gologin-web-access 0.3.0

package/dist/lib/readSource.js

@@ -0,0 +1,297 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeReadSourceMode = normalizeReadSourceMode;
+exports.readMarkdownContent = readMarkdownContent;
+exports.readHtmlContent = readHtmlContent;
+exports.readTextContent = readTextContent;
+exports.readRenderedHtmlContent = readRenderedHtmlContent;
+exports.assessReadableContent = assessReadableContent;
+exports.extractReadableSegmentFromHtml = extractReadableSegmentFromHtml;
+const cheerio_1 = require("cheerio");
+const config_1 = require("../config");
+const browserRead_1 = require("./browserRead");
+const unlocker_1 = require("./unlocker");
+function normalizeReadSourceMode(value, defaultMode = "auto") {
+    if (!value) {
+        return defaultMode;
+    }
+    if (value === "auto" || value === "unlocker" || value === "browser") {
+        return value;
+    }
+    throw new Error(`Unsupported source mode: ${value}`);
+}
+async function readMarkdownContent(url, config, apiKey, options = {}) {
+    return readReadableContent(url, config, apiKey, {
+        ...options,
+        format: "markdown",
+    });
+}
+async function readHtmlContent(url, config, apiKey, options = {}) {
+    return readReadableContent(url, config, apiKey, {
+        ...options,
+        format: "html",
+    });
+}
+async function readTextContent(url, config, apiKey, options = {}) {
+    return readReadableContent(url, config, apiKey, {
+        ...options,
+        format: "text",
+    });
+}
+async function readRenderedHtmlContent(url, config, apiKey, options = {}) {
+    const source = options.source ?? "auto";
+    if (source === "browser") {
+        (0, config_1.requireCloudToken)(config);
+        const browser = await (0, browserRead_1.scrapeRenderedHtmlViaBrowser)(url, config, {
+            profile: options.profile,
+        });
+        return {
+            html: browser.html,
+            renderSource: "browser",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+        };
+    }
+    const unlocker = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, options.request);
+    if (source === "unlocker") {
+        return {
+            html: unlocker.content,
+            renderSource: "unlocker",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+            request: unlocker.request,
+        };
+    }
+    const unlockerText = (0, unlocker_1.htmlToText)(unlocker.content);
+    const assessment = assessReadableContent(unlocker.content, unlockerText);
+    if (!assessment.shouldFallback) {
+        return {
+            html: unlocker.content,
+            renderSource: "unlocker",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+            request: unlocker.request,
+        };
+    }
+    if (!config.cloudToken) {
+        return {
+            html: unlocker.content,
+            renderSource: "unlocker",
+            fallbackAttempted: true,
+            fallbackUsed: false,
+            fallbackReason: `${assessment.reason}; GOLOGIN_CLOUD_TOKEN is not configured`,
+            request: unlocker.request,
+        };
+    }
+    const browser = await (0, browserRead_1.scrapeRenderedHtmlViaBrowser)(url, config, {
+        profile: options.profile,
+    });
+    return {
+        html: browser.html,
+        renderSource: "browser",
+        fallbackAttempted: true,
+        fallbackUsed: true,
+        fallbackReason: assessment.reason,
+        request: unlocker.request,
+    };
+}
+function assessReadableContent(html, content) {
+    const $ = (0, cheerio_1.load)(html);
+    const normalizedContentLength = meaningfulTextLength(content);
+    const mainLength = meaningfulTextLength($("main").first().text());
+    const articleLength = meaningfulTextLength($("article").first().text());
+    const paragraphCount = $("p").length;
+    const headingCount = $("h1, h2, h3").length;
+    const linkCount = $("a[href]").length;
+    const scriptCount = $("script").length;
+    const shellMarkers = /__next_data__|__nuxt__|window\.__|astro-|mintlify|vitepress|docusaurus|hydration|application\/json/i.test(html);
+    const docsUiChromeMarkers = /open in chatgpt|copyask ai|copy ask ai|on this page/i.test(content);
+    if (mainLength < 200 && articleLength < 200 && linkCount > 40 && normalizedContentLength < 5000) {
+        return {
+            shouldFallback: true,
+            reason: "Unlocker returned navigation-heavy shell with weak main/article content",
+        };
+    }
+    if (normalizedContentLength < 600 && (shellMarkers || linkCount > 30 || scriptCount > 10)) {
+        return {
+            shouldFallback: true,
+            reason: "Unlocker returned very little readable text from a likely JS-rendered page",
+        };
+    }
+    if (shellMarkers && mainLength < 300 && paragraphCount < 3) {
+        return {
+            shouldFallback: true,
+            reason: "Unlocker output looks like a JS docs shell without rendered article content",
+        };
+    }
+    if (paragraphCount < 3 && headingCount <= 1 && linkCount > 60 && scriptCount > 15) {
+        return {
+            shouldFallback: true,
+            reason: "Unlocker output is link-heavy and content-light",
+        };
+    }
+    if (docsUiChromeMarkers) {
+        return {
+            shouldFallback: true,
+            reason: "Unlocker output still contains docs UI chrome and action controls",
+        };
+    }
+    return { shouldFallback: false };
+}
+async function readReadableContent(url, config, apiKey, options) {
+    const source = options.source ?? "auto";
+    if (source === "browser") {
+        (0, config_1.requireCloudToken)(config);
+        const browser = await (0, browserRead_1.scrapeReadableContentViaBrowser)(url, config, {
+            profile: options.profile,
+        });
+        return {
+            content: formatReadableContent(options.format, browser.html, browser.text),
+            renderSource: "browser",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+        };
+    }
+    const unlocker = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, options.request);
+    const readable = extractReadableSegmentFromHtml(unlocker.content);
+    const unlockerContent = formatReadableContent(options.format, readable.html, readable.text);
+    if (source === "unlocker") {
+        return {
+            content: unlockerContent,
+            renderSource: "unlocker",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+            request: unlocker.request,
+        };
+    }
+    const assessment = assessReadableContent(unlocker.content, unlockerContent);
+    if (!assessment.shouldFallback) {
+        return {
+            content: unlockerContent,
+            renderSource: "unlocker",
+            fallbackAttempted: false,
+            fallbackUsed: false,
+            request: unlocker.request,
+        };
+    }
+    if (!config.cloudToken) {
+        return {
+            content: unlockerContent,
+            renderSource: "unlocker",
+            fallbackAttempted: true,
+            fallbackUsed: false,
+            fallbackReason: `${assessment.reason}; GOLOGIN_CLOUD_TOKEN is not configured`,
+            request: unlocker.request,
+        };
+    }
+    const browser = await (0, browserRead_1.scrapeReadableContentViaBrowser)(url, config, {
+        profile: options.profile,
+    });
+    const browserContent = formatReadableContent(options.format, browser.html, browser.text);
+    if (meaningfulTextLength(browserContent) < Math.max(300, meaningfulTextLength(unlockerContent))) {
+        return {
+            content: unlockerContent,
+            renderSource: "unlocker",
+            fallbackAttempted: true,
+            fallbackUsed: false,
+            fallbackReason: "Browser fallback did not improve readable output",
+            request: unlocker.request,
+        };
+    }
+    return {
+        content: browserContent,
+        renderSource: "browser",
+        fallbackAttempted: true,
+        fallbackUsed: true,
+        fallbackReason: assessment.reason,
+        request: unlocker.request,
+    };
+}
+function formatReadableContent(format, html, text) {
+    if (format === "html") {
+        return html;
+    }
+    return format === "markdown" ? (0, unlocker_1.htmlToMarkdown)(html) : text.trim();
+}
+function meaningfulTextLength(value) {
+    return value.replace(/\s+/g, " ").trim().length;
+}
+function extractReadableSegmentFromHtml(html) {
+    const $ = (0, cheerio_1.load)(html);
+    const candidates = [
+        { selector: "#content-area", element: $("#content-area").first() },
+        { selector: "main article", element: $("main article").first() },
+        { selector: "article", element: $("article").first() },
+        { selector: "main .prose", element: $("main .prose").first() },
+        { selector: "main", element: $("main").first() },
+        { selector: "[role='main']", element: $("[role='main']").first() },
+        { selector: ".mintlify-content", element: $(".mintlify-content").first() },
+        { selector: ".docs-content", element: $(".docs-content").first() },
+        { selector: ".content", element: $(".content").first() },
+        { selector: ".prose", element: $(".prose").first() },
+        { selector: "body", element: $("body").first() },
+    ];
+    for (const candidate of candidates.filter((item) => item.selector !== "body")) {
+        if (candidate.element.length === 0) {
+            continue;
+        }
+        const text = candidate.element.text();
+        const normalizedLength = meaningfulTextLength(text);
+        const headings = candidate.element.find("h1, h2, h3").length;
+        const paragraphs = candidate.element.find("p, li").length;
+        if (normalizedLength >= 600 && (headings >= 1 || paragraphs >= 3)) {
+            const cleaned = sanitizeReadableFragment($.html(candidate.element));
+            return {
+                selector: candidate.selector,
+                html: cleaned.html,
+                text: cleaned.text,
+            };
+        }
+    }
+    let best = {
+        selector: "body",
+        html: $("body").html() ?? html,
+        text: $("body").text(),
+        score: Number.NEGATIVE_INFINITY,
+    };
+    for (const candidate of candidates) {
+        if (candidate.element.length === 0) {
+            continue;
+        }
+        const text = candidate.element.text();
+        const normalizedLength = meaningfulTextLength(text);
+        if (normalizedLength === 0) {
+            continue;
+        }
+        const headings = candidate.element.find("h1, h2, h3").length;
+        const paragraphs = candidate.element.find("p, li").length;
+        const codeBlocks = candidate.element.find("pre, code").length;
+        const links = candidate.element.find("a[href]").length;
+        let score = Math.min(normalizedLength, 12_000) + headings * 180 + paragraphs * 120 + codeBlocks * 80 - links * 8;
+        if (/^(#content-area|article|main|\[role='main'\])/.test(candidate.selector)) {
+            score += 400;
+        }
+        if (score > best.score) {
+            const cleaned = sanitizeReadableFragment($.html(candidate.element));
+            best = {
+                selector: candidate.selector,
+                html: cleaned.html,
+                text: cleaned.text,
+                score,
+            };
+        }
+    }
+    return {
+        selector: best.selector,
+        html: best.html,
+        text: best.text.replace(/\s+/g, " ").trim(),
+    };
+}
+function sanitizeReadableFragment(fragmentHtml) {
+    const $ = (0, cheerio_1.load)(fragmentHtml);
+    $("script, style, nav, aside, form, button, svg, dialog, [role='button'], [aria-label='More actions'], .sr-only").remove();
+    return {
+        html: $.root().html() ?? fragmentHtml,
+        text: $.root().text().replace(/\s+/g, " ").trim(),
+    };
+}

package/dist/lib/runbooks.js

@@ -0,0 +1,193 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadRunbookDefinition = loadRunbookDefinition;
+exports.loadBatchDefinition = loadBatchDefinition;
+exports.loadVariablesFile = loadVariablesFile;
+exports.executeRunbook = executeRunbook;
+exports.executeBatch = executeBatch;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const selfCli_1 = require("./selfCli");
+const SESSION_SCOPED_COMMANDS = new Set([
+    "open",
+    "search-browser",
+    "tabs",
+    "tabopen",
+    "tabfocus",
+    "tabclose",
+    "snapshot",
+    "click",
+    "dblclick",
+    "focus",
+    "type",
+    "fill",
+    "hover",
+    "select",
+    "check",
+    "uncheck",
+    "press",
+    "scroll",
+    "scrollintoview",
+    "wait",
+    "get",
+    "back",
+    "forward",
+    "reload",
+    "find",
+    "cookies",
+    "cookies-import",
+    "cookies-clear",
+    "storage-export",
+    "storage-import",
+    "storage-clear",
+    "eval",
+    "upload",
+    "pdf",
+    "screenshot",
+    "close"
+]);
+const PROFILE_SCOPED_COMMANDS = new Set(["open", "search-browser", "scrape-screenshot"]);
+const DISALLOWED_COMMANDS = new Set(["run", "batch", "__job-runner"]);
+function loadRunbookDefinition(baseDir, filePath) {
+    const absolutePath = ensureAbsolutePath(baseDir, filePath);
+    return JSON.parse((0, fs_1.readFileSync)(absolutePath, "utf8"));
+}
+function loadBatchDefinition(baseDir, filePath) {
+    const absolutePath = ensureAbsolutePath(baseDir, filePath);
+    const parsed = JSON.parse((0, fs_1.readFileSync)(absolutePath, "utf8"));
+    if (Array.isArray(parsed)) {
+        return { targets: parsed };
+    }
+    return parsed;
+}
+function loadVariablesFile(baseDir, filePath) {
+    const absolutePath = ensureAbsolutePath(baseDir, filePath);
+    return JSON.parse((0, fs_1.readFileSync)(absolutePath, "utf8"));
+}
+async function executeRunbook(runbook, options) {
+    const mergedVariables = {
+        ...(runbook.variables ?? {}),
+        ...(options.variables ?? {})
+    };
+    const continueOnError = options.continueOnError === true || runbook.continueOnError === true;
+    const results = [];
+    for (const step of runbook.steps) {
+        if (DISALLOWED_COMMANDS.has(step.command)) {
+            throw new Error(`Runbook command ${step.command} is not allowed`);
+        }
+        const invocation = buildStepInvocation(step, mergedVariables, {
+            sessionId: options.sessionId,
+            profileId: options.profileId
+        });
+        const startedAt = Date.now();
+        const result = await (0, selfCli_1.runSelfCommandCapture)(invocation, { cwd: options.cwd });
+        const durationMs = Date.now() - startedAt;
+        const stepResult = {
+            command: step.command,
+            label: step.label,
+            status: result.exitCode === 0 ? "ok" : "failed",
+            durationMs,
+            stdout: result.stdout,
+            stderr: result.stderr,
+            error: result.exitCode === 0 ? undefined : result.stderr.trim() || result.stdout.trim() || `exit ${result.exitCode}`
+        };
+        results.push(stepResult);
+        if (stepResult.status === "failed" && !(continueOnError || step.continueOnError)) {
+            throw Object.assign(new Error(stepResult.error ?? `Runbook step failed: ${step.command}`), { steps: results });
+        }
+    }
+    return { steps: results };
+}
+async function executeBatch(runbook, batch, options) {
+    const concurrency = Math.max(1, options.concurrency ?? batch.concurrency ?? 2);
+    const sharedVariables = {
+        ...(batch.variables ?? {}),
+        ...(options.variables ?? {})
+    };
+    const queue = [...batch.targets];
+    const results = [];
+    await Promise.all(Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+        while (queue.length > 0) {
+            const target = queue.shift();
+            if (!target) {
+                return;
+            }
+            const name = target.name ?? target.sessionId ?? target.profileId ?? `target-${results.length + 1}`;
+            const startedAt = Date.now();
+            try {
+                const execution = await executeRunbook(runbook, {
+                    cwd: options.cwd,
+                    sessionId: target.sessionId,
+                    profileId: target.profileId,
+                    variables: {
+                        ...sharedVariables,
+                        ...(target.variables ?? {})
+                    },
+                    continueOnError: options.continueOnError === true || target.continueOnError === true
+                });
+                results.push({
+                    name,
+                    status: execution.steps.some((step) => step.status === "failed") ? "failed" : "ok",
+                    durationMs: Date.now() - startedAt,
+                    sessionId: target.sessionId,
+                    profileId: target.profileId,
+                    steps: execution.steps
+                });
+            }
+            catch (error) {
+                const steps = error && typeof error === "object" && "steps" in error && Array.isArray(error.steps)
+                    ? (error.steps ?? [])
+                    : [];
+                results.push({
+                    name,
+                    status: "failed",
+                    durationMs: Date.now() - startedAt,
+                    sessionId: target.sessionId,
+                    profileId: target.profileId,
+                    steps
+                });
+            }
+        }
+    }));
+    return results.sort((left, right) => left.name.localeCompare(right.name));
+}
+function buildStepInvocation(step, variables, options) {
+    const args = [resolveTemplate(step.command, variables)];
+    const positionals = (step.args ?? []).map((value) => String(resolvePrimitive(value, variables)));
+    const flags = Object.entries(step.flags ?? {}).flatMap(([name, value]) => {
+        const resolved = resolvePrimitive(value, variables);
+        if (typeof resolved === "boolean") {
+            return resolved ? [`--${name}`] : [];
+        }
+        return [`--${name}`, String(resolved)];
+    });
+    args.push(...positionals, ...flags);
+    if (options.sessionId && SESSION_SCOPED_COMMANDS.has(step.command) && !flagPresent(args, "session")) {
+        args.push("--session", options.sessionId);
+    }
+    if (options.profileId && PROFILE_SCOPED_COMMANDS.has(step.command) && !flagPresent(args, "profile")) {
+        args.push("--profile", options.profileId);
+    }
+    return args;
+}
+function resolvePrimitive(value, variables) {
+    return typeof value === "string" ? resolveTemplate(value, variables) : value;
+}
+function resolveTemplate(value, variables) {
+    return value.replace(/\$\{([^}]+)\}/g, (_, rawKey) => {
+        const key = rawKey.trim();
+        if (!(key in variables)) {
+            throw new Error(`Missing runbook variable: ${key}`);
+        }
+        return String(variables[key]);
+    });
+}
+function flagPresent(argv, flagName) {
+    return argv.includes(`--${flagName}`);
+}
+function ensureAbsolutePath(baseDir, targetPath) {
+    return path_1.default.isAbsolute(targetPath) ? targetPath : path_1.default.resolve(baseDir, targetPath);
+}