@pagepocket/lighterceptor 0.4.0

package/dist/src/lighterceptor.js

@@ -0,0 +1,697 @@
+import { createJSDOMWithInterceptor } from "./dom.js";
+const DEFAULT_SETTLE_MS = 50;
+export class Lighterceptor {
+    input;
+    options;
+    constructor(input, options = {}) {
+        this.input = input;
+        this.options = options;
+    }
+    async run() {
+        const requests = [];
+        const networkRecords = [];
+        const capturedAt = new Date().toISOString();
+        const settleTimeMs = this.options.settleTimeMs ?? DEFAULT_SETTLE_MS;
+        const recursive = this.options.recursion ?? false;
+        const requestOnly = this.options.requestOnly ?? false;
+        const baseUrl = this.options.baseUrl;
+        const pending = [];
+        const processed = new Set();
+        const resourceCache = new Map();
+        const responseCache = new Map();
+        const pendingNetwork = [];
+        const fetchWithCache = (url) => {
+            const existing = responseCache.get(url);
+            if (existing) {
+                return existing;
+            }
+            const promise = (async () => {
+                if (typeof fetch !== "function") {
+                    return {
+                        ok: false,
+                        error: "fetch-unavailable"
+                    };
+                }
+                try {
+                    const response = await fetch(url);
+                    const cloned = response.clone();
+                    const [buffer, text] = await Promise.all([response.arrayBuffer(), cloned.text()]);
+                    const headers = {};
+                    response.headers.forEach((value, key) => {
+                        headers[key] = value;
+                    });
+                    const bodyEncoding = resolveBodyEncoding(response.headers.get("content-type") ?? undefined);
+                    const body = bodyEncoding === "base64"
+                        ? Buffer.from(buffer).toString("base64")
+                        : decodeText(Buffer.from(buffer), response.headers.get("content-type") ?? undefined, text);
+                    const responseRecord = {
+                        status: response.status,
+                        statusText: response.statusText,
+                        headers,
+                        body,
+                        bodyEncoding
+                    };
+                    return {
+                        ok: response.ok,
+                        response: responseRecord,
+                        contentType: response.headers.get("content-type") ?? undefined,
+                        text: bodyEncoding === "text" ? body : text,
+                        buffer: Buffer.from(buffer)
+                    };
+                }
+                catch (error) {
+                    return {
+                        ok: false,
+                        error: error instanceof Error ? error.message : String(error)
+                    };
+                }
+            })();
+            responseCache.set(url, promise);
+            return promise;
+        };
+        const recordNetwork = (url, source) => {
+            if (requestOnly || isSkippableUrl(url)) {
+                return;
+            }
+            const task = fetchWithCache(url).then((result) => {
+                const record = {
+                    url,
+                    source,
+                    method: "GET",
+                    timestamp: Date.now()
+                };
+                if (result.response) {
+                    record.response = result.response;
+                }
+                if (!result.ok) {
+                    record.error = result.error ?? "request-failed";
+                }
+                networkRecords.push(record);
+            });
+            pendingNetwork.push(task);
+        };
+        const recordUrl = (url, source, baseUrl) => {
+            const resolved = resolveUrl(baseUrl, url);
+            if (!resolved) {
+                return;
+            }
+            requests.push({
+                url: resolved,
+                source,
+                timestamp: Date.now()
+            });
+            recordNetwork(resolved, source);
+        };
+        const enqueue = (url, kind) => {
+            if (!recursive || isSkippableUrl(url)) {
+                return;
+            }
+            if (processed.has(url)) {
+                return;
+            }
+            processed.add(url);
+            pending.push({ url, kind });
+        };
+        const recordCssUrls = (cssText, baseUrl) => {
+            const { imports, urls } = extractCssDependencies(cssText);
+            for (const url of imports) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "css");
+                enqueue(resolved, "css");
+            }
+            for (const url of urls) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "css");
+            }
+        };
+        const analyzeJs = (jsText, baseUrl) => {
+            const { fetches, imports, importScripts, xhrs } = extractJsDependencies(jsText);
+            for (const url of imports) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "resource");
+                enqueue(resolved, inferResourceKindFromUrl(resolved) ?? "js");
+            }
+            for (const url of importScripts) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "resource");
+                enqueue(resolved, "js");
+            }
+            for (const url of fetches) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "fetch");
+                enqueue(resolved, inferResourceKindFromUrl(resolved));
+            }
+            for (const url of xhrs) {
+                const resolved = resolveUrl(baseUrl, url);
+                if (!resolved) {
+                    continue;
+                }
+                recordUrl(resolved, "xhr");
+                enqueue(resolved, inferResourceKindFromUrl(resolved));
+            }
+        };
+        const analyzeHtml = async (htmlText, baseUrl, captureTitle = false) => {
+            const dom = createJSDOMWithInterceptor({
+                html: htmlText,
+                domOptions: {
+                    pretendToBeVisual: true,
+                    runScripts: "dangerously",
+                    url: baseUrl,
+                    beforeParse(window) {
+                        const createStubResponse = (url) => {
+                            const normalizedUrl = url.toLowerCase();
+                            const bodyText = normalizedUrl.endsWith("/figma/manifest.json")
+                                ? JSON.stringify({ figures: [], svgs: [] })
+                                : normalizedUrl.includes("/features/") && normalizedUrl.endsWith(".json")
+                                    ? JSON.stringify({
+                                        isDead: true,
+                                        statistics: {},
+                                        examples_quantiles: []
+                                    })
+                                    : "";
+                            const encoder = typeof TextEncoder === "function" ? new TextEncoder() : undefined;
+                            const buffer = encoder ? encoder.encode(bodyText).buffer : new ArrayBuffer(0);
+                            const headers = typeof window.Headers === "function"
+                                ? new window.Headers()
+                                : {
+                                    append: () => { },
+                                    delete: () => { },
+                                    get: () => null,
+                                    getSetCookie: () => [],
+                                    has: () => false,
+                                    set: () => { },
+                                    forEach: () => { },
+                                    keys: () => [][Symbol.iterator](),
+                                    values: () => [][Symbol.iterator](),
+                                    entries: () => [][Symbol.iterator](),
+                                    [Symbol.iterator]: () => [][Symbol.iterator]()
+                                };
+                            const responseUrl = url;
+                            const response = {
+                                ok: true,
+                                status: 200,
+                                statusText: "OK",
+                                headers,
+                                json: async () => {
+                                    if (!bodyText) {
+                                        return {};
+                                    }
+                                    try {
+                                        return JSON.parse(bodyText);
+                                    }
+                                    catch {
+                                        return {};
+                                    }
+                                },
+                                text: async () => bodyText,
+                                arrayBuffer: async () => buffer,
+                                clone: () => createStubResponse(responseUrl)
+                            };
+                            return response;
+                        };
+                        window.fetch = ((input) => {
+                            let url = "";
+                            if (typeof input === "string") {
+                                url = input;
+                            }
+                            else if (input instanceof URL) {
+                                url = input.toString();
+                            }
+                            else if ("url" in input) {
+                                url = String(input.url);
+                            }
+                            return Promise.resolve(createStubResponse(url));
+                        });
+                        window.XMLHttpRequest.prototype.send = function send() { };
+                    }
+                },
+                interceptor: async (url, options) => {
+                    const resolved = resolveUrl(options.referrer, url);
+                    if (!resolved) {
+                        return Buffer.from("");
+                    }
+                    const source = options.source ?? "unknown";
+                    recordUrl(resolved, source);
+                    const element = options.element;
+                    const tagName = element?.tagName?.toLowerCase();
+                    if (recursive && tagName === "script") {
+                        const result = await fetchWithCache(resolved);
+                        if (result.ok && result.buffer) {
+                            return result.buffer;
+                        }
+                    }
+                    if (recursive) {
+                        if (source === "fetch" || source === "xhr") {
+                            enqueue(resolved, inferResourceKindFromUrl(resolved));
+                        }
+                        else if (source === "resource") {
+                            const kind = inferKindFromElement(options.element);
+                            if (kind) {
+                                enqueue(resolved, kind);
+                            }
+                        }
+                    }
+                    return Buffer.from("");
+                }
+            });
+            const { document } = dom.window;
+            document.querySelectorAll("img").forEach((img) => {
+                if (img instanceof dom.window.HTMLImageElement && img.src) {
+                    recordUrl(img.src, "img", baseUrl);
+                }
+            });
+            document.querySelectorAll("img[srcset]").forEach((img) => {
+                if (!(img instanceof dom.window.HTMLImageElement)) {
+                    return;
+                }
+                const srcset = img.getAttribute("srcset");
+                if (!srcset) {
+                    return;
+                }
+                for (const url of parseSrcsetUrls(srcset)) {
+                    recordUrl(url, "img", baseUrl);
+                }
+            });
+            document.querySelectorAll("source[src]").forEach((source) => {
+                const src = source.getAttribute("src");
+                if (src) {
+                    recordUrl(src, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("source[srcset]").forEach((source) => {
+                const srcset = source.getAttribute("srcset");
+                if (srcset) {
+                    for (const url of parseSrcsetUrls(srcset)) {
+                        recordUrl(url, "resource", baseUrl);
+                    }
+                }
+            });
+            document.querySelectorAll("script[src]").forEach((script) => {
+                if (script instanceof dom.window.HTMLScriptElement && script.src) {
+                    recordUrl(script.src, "resource", baseUrl);
+                    enqueue(script.src, "js");
+                }
+            });
+            document.querySelectorAll("iframe[src]").forEach((iframe) => {
+                if (iframe instanceof dom.window.HTMLIFrameElement && iframe.src) {
+                    recordUrl(iframe.src, "resource", baseUrl);
+                    enqueue(iframe.src, "html");
+                }
+            });
+            document.querySelectorAll("video[src], audio[src]").forEach((media) => {
+                const src = media.getAttribute("src");
+                if (src) {
+                    recordUrl(src, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("video[poster]").forEach((video) => {
+                const poster = video.getAttribute("poster");
+                if (poster) {
+                    recordUrl(poster, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("track[src]").forEach((track) => {
+                const src = track.getAttribute("src");
+                if (src) {
+                    recordUrl(src, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("embed[src]").forEach((embed) => {
+                const src = embed.getAttribute("src");
+                if (src) {
+                    recordUrl(src, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("object[data]").forEach((object) => {
+                const data = object.getAttribute("data");
+                if (data) {
+                    recordUrl(data, "resource", baseUrl);
+                }
+            });
+            document.querySelectorAll("[style]").forEach((element) => {
+                const cssText = element.getAttribute("style");
+                if (cssText) {
+                    recordCssUrls(cssText, baseUrl);
+                }
+            });
+            document.querySelectorAll("style").forEach((style) => {
+                if (style.textContent) {
+                    recordCssUrls(style.textContent, baseUrl);
+                }
+            });
+            document.querySelectorAll("link[rel]").forEach((link) => {
+                if (!(link instanceof dom.window.HTMLLinkElement)) {
+                    return;
+                }
+                const rel = link.getAttribute("rel") ?? "";
+                if (shouldInterceptLinkRel(rel)) {
+                    const href = link.getAttribute("href") ?? link.href;
+                    if (href) {
+                        const resolvedHref = resolveUrl(baseUrl, href) ?? href;
+                        recordUrl(resolvedHref, "resource");
+                        if (rel.toLowerCase().includes("stylesheet")) {
+                            enqueue(resolvedHref, "css");
+                        }
+                        else if (rel.toLowerCase().includes("preload")) {
+                            const kind = inferResourceKindFromUrl(resolvedHref);
+                            if (kind) {
+                                enqueue(resolvedHref, kind);
+                            }
+                        }
+                    }
+                }
+                if (rel.toLowerCase().includes("preload")) {
+                    const imagesrcset = link.getAttribute("imagesrcset");
+                    if (imagesrcset) {
+                        for (const url of parseSrcsetUrls(imagesrcset)) {
+                            recordUrl(url, "resource", baseUrl);
+                        }
+                    }
+                }
+            });
+            await new Promise((resolve) => setTimeout(resolve, settleTimeMs));
+            if (captureTitle) {
+                return dom.window.document.title || undefined;
+            }
+            return undefined;
+        };
+        const loadResource = async (url) => {
+            const existing = resourceCache.get(url);
+            if (existing) {
+                return existing;
+            }
+            const loader = fetchWithCache(url).then((result) => {
+                if (!result.ok || !result.text) {
+                    return null;
+                }
+                return {
+                    text: result.text,
+                    contentType: result.contentType,
+                    buffer: result.buffer
+                };
+            });
+            resourceCache.set(url, loader);
+            return loader;
+        };
+        const processPending = async () => {
+            while (pending.length > 0) {
+                const next = pending.shift();
+                if (!next) {
+                    continue;
+                }
+                const result = await loadResource(next.url);
+                if (!result) {
+                    continue;
+                }
+                const kind = next.kind ?? detectResourceKind(next.url, result.contentType, result.text);
+                if (!kind) {
+                    continue;
+                }
+                if (kind === "html") {
+                    await analyzeHtml(result.text, next.url);
+                    continue;
+                }
+                if (kind === "css") {
+                    recordCssUrls(result.text, next.url);
+                    continue;
+                }
+                analyzeJs(result.text, next.url);
+            }
+        };
+        const initialUrl = parseAbsoluteUrl(this.input);
+        const effectiveBaseUrl = baseUrl ?? initialUrl ?? undefined;
+        let initialInput = this.input;
+        if (initialUrl) {
+            const result = await fetchWithCache(initialUrl);
+            if (!result.ok || !result.text) {
+                return {
+                    title: undefined,
+                    capturedAt,
+                    requests,
+                    networkRecords: requestOnly ? [] : networkRecords
+                };
+            }
+            initialInput = result.text;
+        }
+        const inputKind = detectInputKind(initialInput);
+        let title;
+        if (inputKind === "html") {
+            title = await analyzeHtml(initialInput, effectiveBaseUrl, true);
+        }
+        else if (inputKind === "css") {
+            recordCssUrls(initialInput, effectiveBaseUrl);
+        }
+        else {
+            analyzeJs(initialInput, effectiveBaseUrl);
+        }
+        await processPending();
+        await Promise.allSettled(pendingNetwork);
+        return {
+            title,
+            capturedAt,
+            requests,
+            networkRecords: requestOnly ? [] : networkRecords
+        };
+    }
+}
+function resolveUrl(baseUrl, url) {
+    if (!url) {
+        return undefined;
+    }
+    if (baseUrl) {
+        try {
+            return new URL(url, baseUrl).toString();
+        }
+        catch {
+            return url;
+        }
+    }
+    try {
+        return new URL(url).toString();
+    }
+    catch {
+        return url;
+    }
+}
+function parseAbsoluteUrl(value) {
+    try {
+        const parsed = new URL(value);
+        if (parsed.protocol === "http:" || parsed.protocol === "https:") {
+            return parsed.toString();
+        }
+    }
+    catch {
+        return undefined;
+    }
+    return undefined;
+}
+function isSkippableUrl(url) {
+    const lowered = url.toLowerCase();
+    return (lowered.startsWith("data:") || lowered.startsWith("javascript:") || lowered.startsWith("about:"));
+}
+function inferResourceKindFromUrl(url) {
+    const cleanUrl = url.split("?")[0].split("#")[0];
+    const extension = cleanUrl.split(".").pop()?.toLowerCase();
+    if (!extension) {
+        return undefined;
+    }
+    if (extension === "html" || extension === "htm") {
+        return "html";
+    }
+    if (extension === "css") {
+        return "css";
+    }
+    if (extension === "js" || extension === "mjs" || extension === "cjs") {
+        return "js";
+    }
+    return undefined;
+}
+function detectResourceKind(url, contentType, text) {
+    const normalized = contentType?.toLowerCase() ?? "";
+    if (normalized.includes("text/html")) {
+        return "html";
+    }
+    if (normalized.includes("text/css")) {
+        return "css";
+    }
+    if (normalized.includes("javascript")) {
+        return "js";
+    }
+    const inferred = inferResourceKindFromUrl(url);
+    if (inferred) {
+        return inferred;
+    }
+    const trimmed = text.trimStart();
+    if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html")) {
+        return "html";
+    }
+    if (trimmed.startsWith("<")) {
+        return "html";
+    }
+    if (trimmed.startsWith("@") || trimmed.includes("url(")) {
+        return "css";
+    }
+    if (looksLikeJavaScript(trimmed)) {
+        return "js";
+    }
+    return undefined;
+}
+function detectInputKind(input) {
+    const trimmed = input.trimStart();
+    if (trimmed.startsWith("<")) {
+        return "html";
+    }
+    if (trimmed.startsWith("@") || trimmed.includes("url(")) {
+        return "css";
+    }
+    return "js";
+}
+function looksLikeJavaScript(text) {
+    return (/\b(import|export)\b/.test(text) ||
+        /\b(const|let|var|function)\b/.test(text) ||
+        /\bfetch\s*\(/.test(text) ||
+        /\bXMLHttpRequest\b/.test(text) ||
+        /\bimportScripts\s*\(/.test(text));
+}
+function inferKindFromElement(element) {
+    if (!element || typeof element !== "object") {
+        return undefined;
+    }
+    const tagName = "tagName" in element && typeof element.tagName === "string"
+        ? element.tagName.toLowerCase()
+        : "";
+    if (tagName === "script") {
+        return "js";
+    }
+    if (tagName === "iframe") {
+        return "html";
+    }
+    if (tagName === "link" && "getAttribute" in element) {
+        const rel = String(element.getAttribute("rel") ?? "").toLowerCase();
+        const asValue = String(element.getAttribute("as") ?? "").toLowerCase();
+        if (rel.includes("stylesheet")) {
+            return "css";
+        }
+        if (rel.includes("preload") || rel.includes("prefetch")) {
+            if (asValue === "style") {
+                return "css";
+            }
+            if (asValue === "script") {
+                return "js";
+            }
+        }
+    }
+    return undefined;
+}
+function extractCssDependencies(cssText) {
+    const imports = [];
+    const urls = [];
+    const urlPattern = /url\(\s*(['"]?)(.*?)\1\s*\)/gi;
+    const importPattern = /@import\s+(?:url\(\s*)?(['"]?)([^'")\s]+)\1\s*\)?/gi;
+    let match;
+    while ((match = urlPattern.exec(cssText)) !== null) {
+        const url = match[2].trim();
+        if (url.length > 0) {
+            urls.push(url);
+        }
+    }
+    while ((match = importPattern.exec(cssText)) !== null) {
+        const url = match[2].trim();
+        if (url.length > 0) {
+            imports.push(url);
+        }
+    }
+    return { imports, urls };
+}
+function extractJsDependencies(jsText) {
+    const imports = new Set();
+    const importScripts = new Set();
+    const fetches = new Set();
+    const xhrs = new Set();
+    const importPattern = /\bimport\s+(?:[^'"]+from\s+)?['"]([^'"]+)['"]/g;
+    const dynamicImportPattern = /\bimport\(\s*['"]([^'"]+)['"]\s*\)/g;
+    const importScriptsPattern = /\bimportScripts\(\s*['"]([^'"]+)['"]\s*\)/g;
+    const fetchPattern = /\bfetch\(\s*['"]([^'"]+)['"]/g;
+    const xhrPattern = /\.open\(\s*['"][^'"]+['"]\s*,\s*['"]([^'"]+)['"]/g;
+    let match;
+    while ((match = importPattern.exec(jsText)) !== null) {
+        imports.add(match[1]);
+    }
+    while ((match = dynamicImportPattern.exec(jsText)) !== null) {
+        imports.add(match[1]);
+    }
+    while ((match = importScriptsPattern.exec(jsText)) !== null) {
+        importScripts.add(match[1]);
+    }
+    while ((match = fetchPattern.exec(jsText)) !== null) {
+        fetches.add(match[1]);
+    }
+    while ((match = xhrPattern.exec(jsText)) !== null) {
+        xhrs.add(match[1]);
+    }
+    return {
+        imports: [...imports],
+        importScripts: [...importScripts],
+        fetches: [...fetches],
+        xhrs: [...xhrs]
+    };
+}
+function parseSrcsetUrls(value) {
+    return value
+        .split(",")
+        .map((candidate) => candidate.trim().split(/\s+/)[0])
+        .filter((url) => url.length > 0);
+}
+function shouldInterceptLinkRel(rel) {
+    const normalized = rel.toLowerCase();
+    return (normalized.includes("preload") ||
+        normalized.includes("prefetch") ||
+        normalized.includes("stylesheet") ||
+        normalized.includes("icon"));
+}
+function resolveBodyEncoding(contentType) {
+    if (!contentType) {
+        return "text";
+    }
+    const normalized = contentType.toLowerCase();
+    if (normalized.startsWith("text/") ||
+        normalized.includes("json") ||
+        normalized.includes("xml") ||
+        normalized.includes("javascript") ||
+        normalized.includes("svg")) {
+        return "text";
+    }
+    return "base64";
+}
+function decodeText(buffer, contentType, fallback) {
+    const charset = contentType
+        ?.toLowerCase()
+        .match(/charset=([^;]+)/)?.[1]
+        ?.trim();
+    if (!charset) {
+        return fallback;
+    }
+    try {
+        const decoder = new TextDecoder(charset);
+        return decoder.decode(buffer);
+    }
+    catch {
+        return fallback;
+    }
+}
+//# sourceMappingURL=lighterceptor.js.map