gologin-web-access 0.3.0

package/dist/lib/changeTracking.js

@@ -0,0 +1,117 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildTrackingKey = buildTrackingKey;
+exports.compareAndPersistSnapshot = compareAndPersistSnapshot;
+exports.scrapeForTracking = scrapeForTracking;
+exports.normalizeTrackingFormat = normalizeTrackingFormat;
+const crypto_1 = require("crypto");
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const diff_1 = require("diff");
+const unlocker_1 = require("./unlocker");
+function buildTrackingKey(url, explicitKey) {
+    return (explicitKey ?? url)
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 120);
+}
+async function compareAndPersistSnapshot(config, snapshot) {
+    await fs_1.promises.mkdir(config.trackingDir, { recursive: true });
+    const next = {
+        ...snapshot,
+        hash: sha256(snapshot.content),
+        updatedAt: new Date().toISOString()
+    };
+    const previous = await readTrackedSnapshot(config, snapshot.key);
+    await fs_1.promises.writeFile(trackingPath(config, snapshot.key), `${JSON.stringify(next, null, 2)}\n`, "utf8");
+    if (!previous) {
+        return {
+            key: next.key,
+            url: next.url,
+            format: next.format,
+            status: "new",
+            currentHash: next.hash,
+            updatedAt: next.updatedAt
+        };
+    }
+    if (previous.hash === next.hash) {
+        return {
+            key: next.key,
+            url: next.url,
+            format: next.format,
+            status: "same",
+            previousHash: previous.hash,
+            currentHash: next.hash,
+            updatedAt: next.updatedAt
+        };
+    }
+    return {
+        key: next.key,
+        url: next.url,
+        format: next.format,
+        status: "changed",
+        previousHash: previous.hash,
+        currentHash: next.hash,
+        updatedAt: next.updatedAt,
+        diff: (0, diff_1.createPatch)(next.key, previous.content, next.content, previous.updatedAt, next.updatedAt)
+    };
+}
+async function scrapeForTracking(url, apiKey, format, options = {}) {
+    switch (format) {
+        case "html": {
+            const result = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, options);
+            return {
+                content: result.content,
+                request: result.request,
+            };
+        }
+        case "text": {
+            const result = await (0, unlocker_1.scrapeText)(url, apiKey, options);
+            return {
+                content: result.text,
+                request: result.request,
+            };
+        }
+        case "json": {
+            const result = await (0, unlocker_1.scrapeJson)(url, apiKey, options);
+            return {
+                title: result.data.title ?? undefined,
+                content: JSON.stringify(result.data, null, 2),
+                request: result.request,
+            };
+        }
+        case "markdown":
+        default: {
+            const result = await (0, unlocker_1.scrapeMarkdown)(url, apiKey, options);
+            return {
+                content: result.markdown,
+                request: result.request,
+            };
+        }
+    }
+}
+function normalizeTrackingFormat(value) {
+    if (value === "html" || value === "markdown" || value === "text" || value === "json") {
+        return value;
+    }
+    throw new Error(`Unsupported change-track format: ${value}`);
+}
+function trackingPath(config, key) {
+    return path_1.default.join(config.trackingDir, `${key}.json`);
+}
+async function readTrackedSnapshot(config, key) {
+    try {
+        const raw = await fs_1.promises.readFile(trackingPath(config, key), "utf8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return undefined;
+    }
+}
+function sha256(value) {
+    return (0, crypto_1.createHash)("sha256").update(value).digest("hex");
+}

package/dist/lib/cloudApi.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateCloudToken = validateCloudToken;
+exports.getProfile = getProfile;
+const errors_1 = require("./errors");
+const GOLOGIN_API_BASE_URL = "https://api.gologin.com";
+async function validateCloudToken(token) {
+    const response = await fetch(`${GOLOGIN_API_BASE_URL}/browser/v2`, {
+        headers: {
+            Authorization: `Bearer ${token}`,
+        },
+    });
+    if (response.ok) {
+        return { ok: true };
+    }
+    const body = await response.text();
+    return {
+        ok: false,
+        status: response.status,
+        detail: body.slice(0, 300) || `status ${response.status}`,
+    };
+}
+async function getProfile(profileId, token) {
+    const response = await fetch(`${GOLOGIN_API_BASE_URL}/browser/${encodeURIComponent(profileId)}`, {
+        headers: {
+            Authorization: `Bearer ${token}`,
+        },
+    });
+    if (response.status === 404) {
+        return null;
+    }
+    if (!response.ok) {
+        const body = await response.text();
+        throw new errors_1.HttpError(`Gologin profile lookup failed with status ${response.status}.`, response.status, body.slice(0, 300));
+    }
+    const data = (await response.json());
+    return {
+        id: data.id ?? profileId,
+        name: data.name,
+    };
+}

package/dist/lib/concurrency.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapWithConcurrency = mapWithConcurrency;
+async function mapWithConcurrency(items, concurrency, mapper) {
+    const results = new Array(items.length);
+    let nextIndex = 0;
+    await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+        while (nextIndex < items.length) {
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            results[currentIndex] = await mapper(items[currentIndex], currentIndex);
+        }
+    }));
+    return results;
+}

package/dist/lib/crawl.js

@@ -0,0 +1,313 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapSite = mapSite;
+exports.crawlSite = crawlSite;
+exports.resolveTraversalStatus = resolveTraversalStatus;
+const readSource_1 = require("./readSource");
+const unlocker_1 = require("./unlocker");
+async function mapSite(rootUrl, apiKey, options) {
+    const pages = await traverseSite(rootUrl, apiKey, options);
+    const status = resolveTraversalStatus(pages.pages.length, pages.pages.filter((page) => !page.ok).length);
+    return {
+        status,
+        rootUrl: pages.rootUrl,
+        visited: pages.pages.length,
+        failed: pages.pages.filter((page) => !page.ok).length,
+        limit: options.limit,
+        maxDepth: options.maxDepth,
+        includeSubdomains: options.includeSubdomains,
+        pages: pages.pages.map((page) => page.ok
+            ? {
+                url: page.url,
+                depth: page.depth,
+                ok: true,
+                title: page.data.title ?? null,
+                description: page.data.description ?? null,
+                canonical: page.data.canonical ?? null,
+                links: page.links,
+            }
+            : {
+                url: page.url,
+                depth: page.depth,
+                ok: false,
+                links: page.links,
+                error: page.error,
+            }),
+    };
+}
+async function crawlSite(rootUrl, apiKey, format, options) {
+    const pages = await traverseSite(rootUrl, apiKey, options);
+    const status = resolveTraversalStatus(pages.pages.length, pages.pages.filter((page) => !page.ok).length);
+    return {
+        status,
+        rootUrl: pages.rootUrl,
+        format,
+        visited: pages.pages.length,
+        failed: pages.pages.filter((page) => !page.ok).length,
+        limit: options.limit,
+        maxDepth: options.maxDepth,
+        includeSubdomains: options.includeSubdomains,
+        pages: pages.pages.map((page) => page.ok
+            ? {
+                url: page.url,
+                depth: page.depth,
+                ok: true,
+                title: page.data.title ?? null,
+                description: page.data.description ?? null,
+                canonical: page.data.canonical ?? null,
+                links: page.links,
+                format,
+                output: page.outputByFormat[format],
+            }
+            : {
+                url: page.url,
+                depth: page.depth,
+                ok: false,
+                links: page.links,
+                error: page.error,
+            }),
+    };
+}
+function resolveTraversalStatus(visited, failed) {
+    if (visited === 0 || failed >= visited) {
+        return "failed";
+    }
+    if (failed > 0) {
+        return "partial";
+    }
+    return "ok";
+}
+async function traverseSite(rootUrl, apiKey, options) {
+    const normalizedRootUrl = normalizeVisitUrl(rootUrl);
+    const scope = createScope(normalizedRootUrl, options.includeSubdomains);
+    const initialFrontier = await buildInitialFrontier(normalizedRootUrl, apiKey, scope, options);
+    const visited = new Set(initialFrontier);
+    const pages = [];
+    let frontier = initialFrontier;
+    for (let depth = 0; depth <= options.maxDepth && frontier.length > 0 && pages.length < options.limit; depth += 1) {
+        const remaining = options.limit - pages.length;
+        const levelUrls = frontier.slice(0, remaining);
+        const levelResults = await mapWithConcurrency(levelUrls, options.concurrency, async (url) => {
+            try {
+                const page = await scrapePage(url, apiKey, scope, options);
+                return {
+                    url,
+                    depth,
+                    ok: true,
+                    data: page.data,
+                    links: page.links,
+                    outputByFormat: page.outputByFormat,
+                };
+            }
+            catch (error) {
+                return {
+                    url,
+                    depth,
+                    ok: false,
+                    error: error instanceof Error ? error.message : "Unknown error",
+                    links: [],
+                };
+            }
+        });
+        pages.push(...levelResults);
+        if (depth === options.maxDepth || pages.length >= options.limit) {
+            break;
+        }
+        const nextFrontier = [];
+        for (const page of levelResults) {
+            if (!page.ok) {
+                continue;
+            }
+            for (const link of page.links) {
+                if (visited.has(link)) {
+                    continue;
+                }
+                if (visited.size >= options.limit) {
+                    break;
+                }
+                visited.add(link);
+                nextFrontier.push(link);
+            }
+        }
+        frontier = nextFrontier;
+    }
+    return {
+        rootUrl: normalizedRootUrl,
+        pages,
+    };
+}
+async function scrapePage(url, apiKey, scope, options) {
+    const scraped = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey);
+    const data = (0, unlocker_1.htmlToStructuredData)(scraped.content);
+    const links = extractScopedLinks(url, data.links, scope, options);
+    const readable = options.onlyMainContent ? (0, readSource_1.extractReadableSegmentFromHtml)(scraped.content) : null;
+    const htmlOutput = readable ? readable.html : scraped.content;
+    return {
+        url,
+        data,
+        links,
+        outputByFormat: {
+            html: htmlOutput,
+            markdown: (0, unlocker_1.htmlToMarkdown)(htmlOutput),
+            text: readable ? readable.text : (0, unlocker_1.htmlToText)(scraped.content),
+            json: data,
+        },
+    };
+}
+async function buildInitialFrontier(rootUrl, apiKey, scope, options) {
+    const rootAllowed = matchesScopeRules(rootUrl, options);
+    const sitemapUrls = options.sitemapMode === "skip" ? [] : await fetchSitemapUrls(rootUrl, apiKey, scope, options).catch(() => []);
+    if (options.sitemapMode === "only") {
+        const seeded = sitemapUrls.filter((url) => matchesScopeRules(url, options));
+        return seeded.length > 0 ? seeded.slice(0, options.limit) : rootAllowed ? [rootUrl] : [];
+    }
+    const urls = [
+        ...(rootAllowed ? [rootUrl] : []),
+        ...sitemapUrls.filter((url) => matchesScopeRules(url, options)),
+    ];
+    return Array.from(new Set(urls)).slice(0, options.limit);
+}
+function createScope(rootUrl, includeSubdomains) {
+    const parsed = new URL(rootUrl);
+    return {
+        rootUrl,
+        rootHost: parsed.hostname.toLowerCase(),
+        rootOrigin: parsed.origin,
+        includeSubdomains,
+    };
+}
+function extractScopedLinks(pageUrl, links, scope, options) {
+    const normalized = new Set();
+    for (const link of links) {
+        const candidate = normalizeDiscoveredUrl(link.href, pageUrl, options.ignoreQueryParameters);
+        if (!candidate) {
+            continue;
+        }
+        if (!isInScope(candidate, scope)) {
+            continue;
+        }
+        if (!matchesScopeRules(candidate, options)) {
+            continue;
+        }
+        normalized.add(candidate);
+    }
+    return Array.from(normalized);
+}
+function normalizeVisitUrl(value, ignoreQueryParameters = false) {
+    const parsed = new URL(value);
+    parsed.hash = "";
+    if (ignoreQueryParameters) {
+        parsed.search = "";
+    }
+    if (parsed.pathname === "") {
+        parsed.pathname = "/";
+    }
+    return parsed.toString();
+}
+function normalizeDiscoveredUrl(href, pageUrl, ignoreQueryParameters) {
+    const trimmed = href.trim();
+    if (!trimmed) {
+        return null;
+    }
+    if (trimmed.startsWith("#") ||
+        trimmed.startsWith("javascript:") ||
+        trimmed.startsWith("mailto:") ||
+        trimmed.startsWith("tel:") ||
+        trimmed.startsWith("data:")) {
+        return null;
+    }
+    try {
+        const resolved = new URL(trimmed, pageUrl);
+        if (resolved.protocol !== "http:" && resolved.protocol !== "https:") {
+            return null;
+        }
+        resolved.hash = "";
+        if (ignoreQueryParameters) {
+            resolved.search = "";
+        }
+        if (resolved.pathname === "") {
+            resolved.pathname = "/";
+        }
+        return resolved.toString();
+    }
+    catch {
+        return null;
+    }
+}
+function isInScope(url, scope) {
+    const parsed = new URL(url);
+    const host = parsed.hostname.toLowerCase();
+    if (scope.includeSubdomains) {
+        return host === scope.rootHost || host.endsWith(`.${scope.rootHost}`);
+    }
+    return parsed.origin === scope.rootOrigin;
+}
+async function fetchSitemapUrls(rootUrl, apiKey, scope, options) {
+    const root = new URL(rootUrl);
+    const sitemapUrl = new URL("/sitemap.xml", root.origin).toString();
+    const scraped = await (0, unlocker_1.scrapeRenderedHtml)(sitemapUrl, apiKey);
+    const directEntries = extractXmlLocValues(scraped.content)
+        .map((url) => normalizeVisitUrl(url, options.ignoreQueryParameters))
+        .filter((url) => isInScope(url, scope) && matchesScopeRules(url, options));
+    if (!scraped.content.includes("<sitemapindex")) {
+        return Array.from(new Set(directEntries));
+    }
+    const nested = directEntries.filter((url) => url.endsWith(".xml"));
+    const pageUrls = directEntries.filter((url) => !url.endsWith(".xml"));
+    const nestedPages = await mapWithConcurrency(nested.slice(0, 10), 2, async (url) => {
+        try {
+            const nestedScrape = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey);
+            return extractXmlLocValues(nestedScrape.content)
+                .map((item) => normalizeVisitUrl(item, options.ignoreQueryParameters))
+                .filter((item) => isInScope(item, scope) && matchesScopeRules(item, options));
+        }
+        catch {
+            return [];
+        }
+    });
+    return Array.from(new Set([...pageUrls, ...nestedPages.flat()]));
+}
+function extractXmlLocValues(xml) {
+    return Array.from(xml.matchAll(/<loc>([\s\S]*?)<\/loc>/gi))
+        .map((match) => decodeXmlEntities(match[1].trim()))
+        .filter(Boolean);
+}
+function decodeXmlEntities(value) {
+    return value
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, "\"")
+        .replace(/&#39;/g, "'");
+}
+function matchesScopeRules(url, options) {
+    if (options.includePatterns.length > 0 && !matchesAnyPattern(url, options.includePatterns)) {
+        return false;
+    }
+    if (options.excludePatterns.length > 0 && matchesAnyPattern(url, options.excludePatterns)) {
+        return false;
+    }
+    return true;
+}
+function matchesAnyPattern(url, patterns) {
+    return patterns.some((pattern) => buildPatternRegex(pattern).test(url));
+}
+function buildPatternRegex(pattern) {
+    const escaped = pattern
+        .trim()
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    return new RegExp(escaped, "i");
+}
+async function mapWithConcurrency(items, concurrency, mapper) {
+    const results = new Array(items.length);
+    let nextIndex = 0;
+    await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), items.length) }, async () => {
+        while (nextIndex < items.length) {
+            const currentIndex = nextIndex;
+            nextIndex += 1;
+            results[currentIndex] = await mapper(items[currentIndex], currentIndex);
+        }
+    }));
+    return results;
+}

package/dist/lib/document.js

@@ -0,0 +1,170 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseDocumentSource = parseDocumentSource;
+const fs_1 = require("fs");
+const path_1 = __importDefault(require("path"));
+const mammoth_1 = __importDefault(require("mammoth"));
+const pdf_parse_1 = require("pdf-parse");
+const XLSX = __importStar(require("xlsx"));
+const unlocker_1 = require("./unlocker");
+async function parseDocumentSource(source) {
+    const loaded = await loadSource(source);
+    const kind = detectDocumentKind(source, loaded.contentType);
+    switch (kind) {
+        case "pdf": {
+            const parser = new pdf_parse_1.PDFParse({ data: loaded.buffer });
+            try {
+                const parsed = await parser.getText();
+                const info = await parser.getInfo().catch(() => undefined);
+                return {
+                    source,
+                    kind,
+                    text: parsed.text.trim(),
+                    metadata: {
+                        pages: parsed.total,
+                        info: info?.info ?? {}
+                    }
+                };
+            }
+            finally {
+                await parser.destroy().catch(() => undefined);
+            }
+        }
+        case "docx": {
+            const parsed = await mammoth_1.default.extractRawText({ buffer: loaded.buffer });
+            return {
+                source,
+                kind,
+                text: parsed.value.trim(),
+                metadata: {
+                    messages: parsed.messages
+                }
+            };
+        }
+        case "xlsx": {
+            const workbook = XLSX.read(loaded.buffer, { type: "buffer" });
+            const parts = workbook.SheetNames.map((sheetName) => {
+                const sheet = workbook.Sheets[sheetName];
+                const rows = XLSX.utils.sheet_to_json(sheet, {
+                    header: 1,
+                    raw: false
+                });
+                const body = rows.map((row) => row.map((cell) => (cell ?? "").toString()).join("\t")).join("\n").trim();
+                return `# ${sheetName}\n${body}`.trim();
+            }).filter(Boolean);
+            return {
+                source,
+                kind,
+                text: parts.join("\n\n").trim(),
+                metadata: {
+                    sheets: workbook.SheetNames
+                }
+            };
+        }
+        case "html": {
+            const raw = loaded.buffer.toString("utf8");
+            return {
+                source,
+                kind,
+                text: (0, unlocker_1.htmlToText)(raw),
+                metadata: {}
+            };
+        }
+        default: {
+            return {
+                source,
+                kind,
+                text: loaded.buffer.toString("utf8").trim(),
+                metadata: {}
+            };
+        }
+    }
+}
+async function loadSource(source) {
+    if (looksLikeUrl(source)) {
+        const response = await fetch(source);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch ${source}: ${response.status}`);
+        }
+        const buffer = Buffer.from(await response.arrayBuffer());
+        return {
+            buffer,
+            contentType: response.headers.get("content-type")
+        };
+    }
+    return {
+        buffer: await fs_1.promises.readFile(path_1.default.resolve(source))
+    };
+}
+function looksLikeUrl(value) {
+    try {
+        const url = new URL(value);
+        return url.protocol === "http:" || url.protocol === "https:";
+    }
+    catch {
+        return false;
+    }
+}
+function detectDocumentKind(source, contentType) {
+    const lowerContentType = contentType?.toLowerCase() ?? "";
+    const lowerSource = source.toLowerCase();
+    if (lowerContentType.includes("pdf") || lowerSource.endsWith(".pdf")) {
+        return "pdf";
+    }
+    if (lowerContentType.includes("wordprocessingml.document") ||
+        lowerContentType.includes("application/vnd.openxmlformats-officedocument.wordprocessingml.document") ||
+        lowerSource.endsWith(".docx")) {
+        return "docx";
+    }
+    if (lowerContentType.includes("spreadsheetml.sheet") ||
+        lowerContentType.includes("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") ||
+        lowerSource.endsWith(".xlsx")) {
+        return "xlsx";
+    }
+    if (lowerContentType.includes("html") || lowerSource.endsWith(".html") || lowerSource.endsWith(".htm")) {
+        return "html";
+    }
+    if (lowerSource.endsWith(".md")) {
+        return "markdown";
+    }
+    if (lowerSource.endsWith(".json")) {
+        return "json";
+    }
+    return "text";
+}