@pagepocket/lighterceptor 0.4.1 → 0.5.0

package/dist/src/lighterceptor.js

@@ -1,4 +1,7 @@
-import { createJSDOMWithInterceptor } from "./dom.js";
+import { analyzeJs as analyzeJsDependencies, recordCssUrls as recordCssUrlsDependencies } from "./dependency-analysis.js";
+import { analyzeHtml } from "./html-analysis.js";
+import { createFetchWithCache } from "./network-utils.js";
+import { detectInputKind, detectResourceKind, isSkippableUrl, parseAbsoluteUrl, resolveUrl } from "./resource-utils.js";
 const DEFAULT_SETTLE_MS = 50;
 export class Lighterceptor {
     input;
@@ -20,55 +23,9 @@ export class Lighterceptor {
         const resourceCache = new Map();
         const responseCache = new Map();
         const pendingNetwork = [];
-        const fetchWithCache = (url) => {
-            const existing = responseCache.get(url);
-            if (existing) {
-                return existing;
-            }
-            const promise = (async () => {
-                if (typeof fetch !== "function") {
-                    return {
-                        ok: false,
-                        error: "fetch-unavailable"
-                    };
-                }
-                try {
-                    const response = await fetch(url);
-                    const cloned = response.clone();
-                    const [buffer, text] = await Promise.all([response.arrayBuffer(), cloned.text()]);
-                    const headers = {};
-                    response.headers.forEach((value, key) => {
-                        headers[key] = value;
-                    });
-                    const bodyEncoding = resolveBodyEncoding(response.headers.get("content-type") ?? undefined);
-                    const body = bodyEncoding === "base64"
-                        ? Buffer.from(buffer).toString("base64")
-                        : decodeText(Buffer.from(buffer), response.headers.get("content-type") ?? undefined, text);
-                    const responseRecord = {
-                        status: response.status,
-                        statusText: response.statusText,
-                        headers,
-                        body,
-                        bodyEncoding
-                    };
-                    return {
-                        ok: response.ok,
-                        response: responseRecord,
-                        contentType: response.headers.get("content-type") ?? undefined,
-                        text: bodyEncoding === "text" ? body : text,
-                        buffer: Buffer.from(buffer)
-                    };
-                }
-                catch (error) {
-                    return {
-                        ok: false,
-                        error: error instanceof Error ? error.message : String(error)
-                    };
-                }
-            })();
-            responseCache.set(url, promise);
-            return promise;
-        };
+        const fetchWithCache = createFetchWithCache(responseCache, {
+            headers: this.options.headers
+        });
         const recordNetwork = (url, source) => {
             if (requestOnly || isSkippableUrl(url)) {
                 return;
@@ -112,284 +69,8 @@ export class Lighterceptor {
             processed.add(url);
             pending.push({ url, kind });
         };
-        const recordCssUrls = (cssText, baseUrl) => {
-            const { imports, urls } = extractCssDependencies(cssText);
-            for (const url of imports) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "css");
-                enqueue(resolved, "css");
-            }
-            for (const url of urls) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "css");
-            }
-        };
-        const analyzeJs = (jsText, baseUrl) => {
-            const { fetches, imports, importScripts, xhrs } = extractJsDependencies(jsText);
-            for (const url of imports) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "resource");
-                enqueue(resolved, inferResourceKindFromUrl(resolved) ?? "js");
-            }
-            for (const url of importScripts) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "resource");
-                enqueue(resolved, "js");
-            }
-            for (const url of fetches) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "fetch");
-                enqueue(resolved, inferResourceKindFromUrl(resolved));
-            }
-            for (const url of xhrs) {
-                const resolved = resolveUrl(baseUrl, url);
-                if (!resolved) {
-                    continue;
-                }
-                recordUrl(resolved, "xhr");
-                enqueue(resolved, inferResourceKindFromUrl(resolved));
-            }
-        };
-        const analyzeHtml = async (htmlText, baseUrl, captureTitle = false) => {
-            const dom = createJSDOMWithInterceptor({
-                html: htmlText,
-                domOptions: {
-                    pretendToBeVisual: true,
-                    runScripts: "dangerously",
-                    url: baseUrl,
-                    beforeParse(window) {
-                        const createStubResponse = (url) => {
-                            const normalizedUrl = url.toLowerCase();
-                            const bodyText = normalizedUrl.endsWith("/figma/manifest.json")
-                                ? JSON.stringify({ figures: [], svgs: [] })
-                                : normalizedUrl.includes("/features/") && normalizedUrl.endsWith(".json")
-                                    ? JSON.stringify({
-                                        isDead: true,
-                                        statistics: {},
-                                        examples_quantiles: []
-                                    })
-                                    : "";
-                            const encoder = typeof TextEncoder === "function" ? new TextEncoder() : undefined;
-                            const buffer = encoder ? encoder.encode(bodyText).buffer : new ArrayBuffer(0);
-                            const headers = typeof window.Headers === "function"
-                                ? new window.Headers()
-                                : {
-                                    append: () => { },
-                                    delete: () => { },
-                                    get: () => null,
-                                    getSetCookie: () => [],
-                                    has: () => false,
-                                    set: () => { },
-                                    forEach: () => { },
-                                    keys: () => [][Symbol.iterator](),
-                                    values: () => [][Symbol.iterator](),
-                                    entries: () => [][Symbol.iterator](),
-                                    [Symbol.iterator]: () => [][Symbol.iterator]()
-                                };
-                            const responseUrl = url;
-                            const response = {
-                                ok: true,
-                                status: 200,
-                                statusText: "OK",
-                                headers,
-                                json: async () => {
-                                    if (!bodyText) {
-                                        return {};
-                                    }
-                                    try {
-                                        return JSON.parse(bodyText);
-                                    }
-                                    catch {
-                                        return {};
-                                    }
-                                },
-                                text: async () => bodyText,
-                                arrayBuffer: async () => buffer,
-                                clone: () => createStubResponse(responseUrl)
-                            };
-                            return response;
-                        };
-                        window.fetch = ((input) => {
-                            let url = "";
-                            if (typeof input === "string") {
-                                url = input;
-                            }
-                            else if (input instanceof URL) {
-                                url = input.toString();
-                            }
-                            else if ("url" in input) {
-                                url = String(input.url);
-                            }
-                            return Promise.resolve(createStubResponse(url));
-                        });
-                        window.XMLHttpRequest.prototype.send = function send() { };
-                    }
-                },
-                interceptor: async (url, options) => {
-                    const resolved = resolveUrl(options.referrer, url);
-                    if (!resolved) {
-                        return Buffer.from("");
-                    }
-                    const source = options.source ?? "unknown";
-                    recordUrl(resolved, source);
-                    const element = options.element;
-                    const tagName = element?.tagName?.toLowerCase();
-                    if (recursive && tagName === "script") {
-                        const result = await fetchWithCache(resolved);
-                        if (result.ok && result.buffer) {
-                            return result.buffer;
-                        }
-                    }
-                    if (recursive) {
-                        if (source === "fetch" || source === "xhr") {
-                            enqueue(resolved, inferResourceKindFromUrl(resolved));
-                        }
-                        else if (source === "resource") {
-                            const kind = inferKindFromElement(options.element);
-                            if (kind) {
-                                enqueue(resolved, kind);
-                            }
-                        }
-                    }
-                    return Buffer.from("");
-                }
-            });
-            const { document } = dom.window;
-            document.querySelectorAll("img").forEach((img) => {
-                if (img instanceof dom.window.HTMLImageElement && img.src) {
-                    recordUrl(img.src, "img", baseUrl);
-                }
-            });
-            document.querySelectorAll("img[srcset]").forEach((img) => {
-                if (!(img instanceof dom.window.HTMLImageElement)) {
-                    return;
-                }
-                const srcset = img.getAttribute("srcset");
-                if (!srcset) {
-                    return;
-                }
-                for (const url of parseSrcsetUrls(srcset)) {
-                    recordUrl(url, "img", baseUrl);
-                }
-            });
-            document.querySelectorAll("source[src]").forEach((source) => {
-                const src = source.getAttribute("src");
-                if (src) {
-                    recordUrl(src, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("source[srcset]").forEach((source) => {
-                const srcset = source.getAttribute("srcset");
-                if (srcset) {
-                    for (const url of parseSrcsetUrls(srcset)) {
-                        recordUrl(url, "resource", baseUrl);
-                    }
-                }
-            });
-            document.querySelectorAll("script[src]").forEach((script) => {
-                if (script instanceof dom.window.HTMLScriptElement && script.src) {
-                    recordUrl(script.src, "resource", baseUrl);
-                    enqueue(script.src, "js");
-                }
-            });
-            document.querySelectorAll("iframe[src]").forEach((iframe) => {
-                if (iframe instanceof dom.window.HTMLIFrameElement && iframe.src) {
-                    recordUrl(iframe.src, "resource", baseUrl);
-                    enqueue(iframe.src, "html");
-                }
-            });
-            document.querySelectorAll("video[src], audio[src]").forEach((media) => {
-                const src = media.getAttribute("src");
-                if (src) {
-                    recordUrl(src, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("video[poster]").forEach((video) => {
-                const poster = video.getAttribute("poster");
-                if (poster) {
-                    recordUrl(poster, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("track[src]").forEach((track) => {
-                const src = track.getAttribute("src");
-                if (src) {
-                    recordUrl(src, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("embed[src]").forEach((embed) => {
-                const src = embed.getAttribute("src");
-                if (src) {
-                    recordUrl(src, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("object[data]").forEach((object) => {
-                const data = object.getAttribute("data");
-                if (data) {
-                    recordUrl(data, "resource", baseUrl);
-                }
-            });
-            document.querySelectorAll("[style]").forEach((element) => {
-                const cssText = element.getAttribute("style");
-                if (cssText) {
-                    recordCssUrls(cssText, baseUrl);
-                }
-            });
-            document.querySelectorAll("style").forEach((style) => {
-                if (style.textContent) {
-                    recordCssUrls(style.textContent, baseUrl);
-                }
-            });
-            document.querySelectorAll("link[rel]").forEach((link) => {
-                if (!(link instanceof dom.window.HTMLLinkElement)) {
-                    return;
-                }
-                const rel = link.getAttribute("rel") ?? "";
-                if (shouldInterceptLinkRel(rel)) {
-                    const href = link.getAttribute("href") ?? link.href;
-                    if (href) {
-                        const resolvedHref = resolveUrl(baseUrl, href) ?? href;
-                        recordUrl(resolvedHref, "resource");
-                        if (rel.toLowerCase().includes("stylesheet")) {
-                            enqueue(resolvedHref, "css");
-                        }
-                        else if (rel.toLowerCase().includes("preload")) {
-                            const kind = inferResourceKindFromUrl(resolvedHref);
-                            if (kind) {
-                                enqueue(resolvedHref, kind);
-                            }
-                        }
-                    }
-                }
-                if (rel.toLowerCase().includes("preload")) {
-                    const imagesrcset = link.getAttribute("imagesrcset");
-                    if (imagesrcset) {
-                        for (const url of parseSrcsetUrls(imagesrcset)) {
-                            recordUrl(url, "resource", baseUrl);
-                        }
-                    }
-                }
-            });
-            await new Promise((resolve) => setTimeout(resolve, settleTimeMs));
-            if (captureTitle) {
-                return dom.window.document.title || undefined;
-            }
-            return undefined;
-        };
+        const recordCssUrls = (cssText, cssBaseUrl) => recordCssUrlsDependencies(cssText, cssBaseUrl, recordUrl, enqueue);
+        const analyzeJs = (jsText, jsBaseUrl) => analyzeJsDependencies(jsText, jsBaseUrl, recordUrl, enqueue);
         const loadResource = async (url) => {
             const existing = resourceCache.get(url);
             if (existing) {
@@ -423,7 +104,16 @@ export class Lighterceptor {
                     continue;
                 }
                 if (kind === "html") {
-                    await analyzeHtml(result.text, next.url);
+                    await analyzeHtml({
+                        htmlText: result.text,
+                        baseUrl: next.url,
+                        settleTimeMs,
+                        recursive,
+                        fetchWithCache,
+                        recordUrl,
+                        enqueue,
+                        recordCssUrls
+                    });
                     continue;
                 }
                 if (kind === "css") {
@@ -439,19 +129,26 @@ export class Lighterceptor {
         if (initialUrl) {
             const result = await fetchWithCache(initialUrl);
             if (!result.ok || !result.text) {
-                return {
-                    title: undefined,
-                    capturedAt,
-                    requests,
-                    networkRecords: requestOnly ? [] : networkRecords
-                };
+                const reason = result.error ?? "request-failed";
+                throw new Error(`Failed to fetch ${initialUrl}: ${reason}`);
             }
+            recordUrl(initialUrl, "resource");
             initialInput = result.text;
         }
         const inputKind = detectInputKind(initialInput);
         let title;
         if (inputKind === "html") {
-            title = await analyzeHtml(initialInput, effectiveBaseUrl, true);
+            title = await analyzeHtml({
+                htmlText: initialInput,
+                baseUrl: effectiveBaseUrl,
+                captureTitle: true,
+                settleTimeMs,
+                recursive,
+                fetchWithCache,
+                recordUrl,
+                enqueue,
+                recordCssUrls
+            });
         }
         else if (inputKind === "css") {
             recordCssUrls(initialInput, effectiveBaseUrl);
@@ -469,229 +166,4 @@ export class Lighterceptor {
         };
     }
 }
-function resolveUrl(baseUrl, url) {
-    if (!url) {
-        return undefined;
-    }
-    if (baseUrl) {
-        try {
-            return new URL(url, baseUrl).toString();
-        }
-        catch {
-            return url;
-        }
-    }
-    try {
-        return new URL(url).toString();
-    }
-    catch {
-        return url;
-    }
-}
-function parseAbsoluteUrl(value) {
-    try {
-        const parsed = new URL(value);
-        if (parsed.protocol === "http:" || parsed.protocol === "https:") {
-            return parsed.toString();
-        }
-    }
-    catch {
-        return undefined;
-    }
-    return undefined;
-}
-function isSkippableUrl(url) {
-    const lowered = url.toLowerCase();
-    return (lowered.startsWith("data:") || lowered.startsWith("javascript:") || lowered.startsWith("about:"));
-}
-function inferResourceKindFromUrl(url) {
-    const cleanUrl = url.split("?")[0].split("#")[0];
-    const extension = cleanUrl.split(".").pop()?.toLowerCase();
-    if (!extension) {
-        return undefined;
-    }
-    if (extension === "html" || extension === "htm") {
-        return "html";
-    }
-    if (extension === "css") {
-        return "css";
-    }
-    if (extension === "js" || extension === "mjs" || extension === "cjs") {
-        return "js";
-    }
-    return undefined;
-}
-function detectResourceKind(url, contentType, text) {
-    const normalized = contentType?.toLowerCase() ?? "";
-    if (normalized.includes("text/html")) {
-        return "html";
-    }
-    if (normalized.includes("text/css")) {
-        return "css";
-    }
-    if (normalized.includes("javascript")) {
-        return "js";
-    }
-    const inferred = inferResourceKindFromUrl(url);
-    if (inferred) {
-        return inferred;
-    }
-    const trimmed = text.trimStart();
-    if (trimmed.startsWith("<!doctype") || trimmed.startsWith("<html")) {
-        return "html";
-    }
-    if (trimmed.startsWith("<")) {
-        return "html";
-    }
-    if (trimmed.startsWith("@") || trimmed.includes("url(")) {
-        return "css";
-    }
-    if (looksLikeJavaScript(trimmed)) {
-        return "js";
-    }
-    return undefined;
-}
-function detectInputKind(input) {
-    const trimmed = input.trimStart();
-    if (trimmed.startsWith("<")) {
-        return "html";
-    }
-    if (trimmed.startsWith("@") || trimmed.includes("url(")) {
-        return "css";
-    }
-    return "js";
-}
-function looksLikeJavaScript(text) {
-    return (/\b(import|export)\b/.test(text) ||
-        /\b(const|let|var|function)\b/.test(text) ||
-        /\bfetch\s*\(/.test(text) ||
-        /\bXMLHttpRequest\b/.test(text) ||
-        /\bimportScripts\s*\(/.test(text));
-}
-function inferKindFromElement(element) {
-    if (!element || typeof element !== "object") {
-        return undefined;
-    }
-    const tagName = "tagName" in element && typeof element.tagName === "string"
-        ? element.tagName.toLowerCase()
-        : "";
-    if (tagName === "script") {
-        return "js";
-    }
-    if (tagName === "iframe") {
-        return "html";
-    }
-    if (tagName === "link" && "getAttribute" in element) {
-        const rel = String(element.getAttribute("rel") ?? "").toLowerCase();
-        const asValue = String(element.getAttribute("as") ?? "").toLowerCase();
-        if (rel.includes("stylesheet")) {
-            return "css";
-        }
-        if (rel.includes("preload") || rel.includes("prefetch")) {
-            if (asValue === "style") {
-                return "css";
-            }
-            if (asValue === "script") {
-                return "js";
-            }
-        }
-    }
-    return undefined;
-}
-function extractCssDependencies(cssText) {
-    const imports = [];
-    const urls = [];
-    const urlPattern = /url\(\s*(['"]?)(.*?)\1\s*\)/gi;
-    const importPattern = /@import\s+(?:url\(\s*)?(['"]?)([^'")\s]+)\1\s*\)?/gi;
-    let match;
-    while ((match = urlPattern.exec(cssText)) !== null) {
-        const url = match[2].trim();
-        if (url.length > 0) {
-            urls.push(url);
-        }
-    }
-    while ((match = importPattern.exec(cssText)) !== null) {
-        const url = match[2].trim();
-        if (url.length > 0) {
-            imports.push(url);
-        }
-    }
-    return { imports, urls };
-}
-function extractJsDependencies(jsText) {
-    const imports = new Set();
-    const importScripts = new Set();
-    const fetches = new Set();
-    const xhrs = new Set();
-    const importPattern = /\bimport\s+(?:[^'"]+from\s+)?['"]([^'"]+)['"]/g;
-    const dynamicImportPattern = /\bimport\(\s*['"]([^'"]+)['"]\s*\)/g;
-    const importScriptsPattern = /\bimportScripts\(\s*['"]([^'"]+)['"]\s*\)/g;
-    const fetchPattern = /\bfetch\(\s*['"]([^'"]+)['"]/g;
-    const xhrPattern = /\.open\(\s*['"][^'"]+['"]\s*,\s*['"]([^'"]+)['"]/g;
-    let match;
-    while ((match = importPattern.exec(jsText)) !== null) {
-        imports.add(match[1]);
-    }
-    while ((match = dynamicImportPattern.exec(jsText)) !== null) {
-        imports.add(match[1]);
-    }
-    while ((match = importScriptsPattern.exec(jsText)) !== null) {
-        importScripts.add(match[1]);
-    }
-    while ((match = fetchPattern.exec(jsText)) !== null) {
-        fetches.add(match[1]);
-    }
-    while ((match = xhrPattern.exec(jsText)) !== null) {
-        xhrs.add(match[1]);
-    }
-    return {
-        imports: [...imports],
-        importScripts: [...importScripts],
-        fetches: [...fetches],
-        xhrs: [...xhrs]
-    };
-}
-function parseSrcsetUrls(value) {
-    return value
-        .split(",")
-        .map((candidate) => candidate.trim().split(/\s+/)[0])
-        .filter((url) => url.length > 0);
-}
-function shouldInterceptLinkRel(rel) {
-    const normalized = rel.toLowerCase();
-    return (normalized.includes("preload") ||
-        normalized.includes("prefetch") ||
-        normalized.includes("stylesheet") ||
-        normalized.includes("icon"));
-}
-function resolveBodyEncoding(contentType) {
-    if (!contentType) {
-        return "text";
-    }
-    const normalized = contentType.toLowerCase();
-    if (normalized.startsWith("text/") ||
-        normalized.includes("json") ||
-        normalized.includes("xml") ||
-        normalized.includes("javascript") ||
-        normalized.includes("svg")) {
-        return "text";
-    }
-    return "base64";
-}
-function decodeText(buffer, contentType, fallback) {
-    const charset = contentType
-        ?.toLowerCase()
-        .match(/charset=([^;]+)/)?.[1]
-        ?.trim();
-    if (!charset) {
-        return fallback;
-    }
-    try {
-        const decoder = new TextDecoder(charset);
-        return decoder.decode(buffer);
-    }
-    catch {
-        return fallback;
-    }
-}
 //# sourceMappingURL=lighterceptor.js.map