@debriefer/sources 1.0.1

package/dist/shared/fetch-page.js

@@ -0,0 +1,212 @@
+/**
+ * Page fetching utility with browser-like headers and archive.org fallback.
+ *
+ * Provides a resilient page fetching pipeline:
+ * 1. Direct fetch with browser-like headers (Chrome UA, Accept text/html)
+ * 2. Block detection (hard HTTP blocks + soft body pattern matching)
+ * 3. Automatic archive.org fallback when blocked or on network error
+ * 4. Non-blocking HTTP errors (404, 500) return immediately without fallback
+ *
+ * Used by WebSearchBase when following links from search results.
+ */
+/** Default browser-like User-Agent string. */
+const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
+/** Default total timeout budget in milliseconds (shared across direct + archive attempts). */
+const DEFAULT_TIMEOUT_MS = 15000;
+/** HTTP status codes that indicate a hard block (should trigger archive fallback). */
+const HARD_BLOCK_STATUSES = new Set([401, 403, 429, 451]);
+/** Maximum body length (in characters) for soft block detection. Pages larger than this are assumed to be real content. */
+const SOFT_BLOCK_MAX_SIZE = 50_000;
+/** Case-insensitive patterns in response body that indicate a soft block. */
+const SOFT_BLOCK_PATTERNS = [
+    "captcha",
+    "please verify you are human",
+    "access denied",
+    "bot detection",
+    "unusual traffic",
+    "automated access",
+    "cloudflare",
+    "ddos protection",
+    "just a moment",
+    "recaptcha",
+    "hcaptcha",
+];
+/**
+ * Build the combined AbortSignal from a caller signal and a timeout.
+ *
+ * Uses `AbortSignal.any()` to combine both so that neither defeats the other.
+ */
+function buildSignal(callerSignal, timeoutMs) {
+    const timeout = timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const timeoutSignal = AbortSignal.timeout(timeout);
+    if (callerSignal) {
+        return AbortSignal.any([callerSignal, timeoutSignal]);
+    }
+    return timeoutSignal;
+}
+/** Build browser-like request headers. */
+function buildHeaders(userAgent) {
+    return {
+        "User-Agent": userAgent,
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Cache-Control": "no-cache",
+    };
+}
+/**
+ * Check whether a response body indicates a soft block (captcha, bot detection, etc.).
+ *
+ * Only checks pages smaller than SOFT_BLOCK_MAX_SIZE to avoid false positives
+ * on large legitimate pages that happen to mention these words.
+ */
+function isSoftBlocked(body) {
+    if (body.length > SOFT_BLOCK_MAX_SIZE) {
+        return false;
+    }
+    const lower = body.toLowerCase();
+    return SOFT_BLOCK_PATTERNS.some((pattern) => lower.includes(pattern));
+}
+/**
+ * Check whether an error is an abort or timeout error.
+ * Node.js 22 AbortSignal.timeout() produces DOMException with name "TimeoutError".
+ */
+function isAbortError(error) {
+    return (error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError"));
+}
+/**
+ * Construct the archive.org Wayback Machine URL for a given URL.
+ * Strips fragments and credentials which aren't part of the archived resource key.
+ */
+function archiveUrl(url) {
+    try {
+        const normalized = new URL(url);
+        normalized.hash = "";
+        normalized.username = "";
+        normalized.password = "";
+        return `https://web.archive.org/web/${normalized.toString()}`;
+    }
+    catch {
+        return `https://web.archive.org/web/${url}`;
+    }
+}
+/**
+ * Attempt to fetch a page from archive.org.
+ *
+ * Returns null if the archive fetch fails for any reason.
+ */
+async function fetchFromArchive(url, headers, signal) {
+    const aUrl = archiveUrl(url);
+    try {
+        const response = await fetch(aUrl, { headers, signal });
+        if (response.ok) {
+            const content = await response.text();
+            return {
+                content,
+                url: response.url || aUrl,
+                fetchMethod: "archive.org",
+            };
+        }
+    }
+    catch {
+        // Archive fetch failed — fall through to return null
+    }
+    return null;
+}
+/**
+ * Fetch a page with browser-like headers and automatic archive.org fallback.
+ *
+ * Pipeline:
+ * 1. Direct fetch with browser-like headers
+ * 2. Block detection (hard HTTP status codes + soft body pattern matching)
+ * 3. If blocked and archiveFallback enabled, try archive.org
+ * 4. Non-blocking HTTP errors (404, 500) return "none" immediately
+ * 5. Network errors on direct fetch trigger archive fallback
+ *
+ * @param options - Fetch options including URL, signal, timeout, etc.
+ * @returns Result with content, final URL, and fetch method
+ */
+export async function fetchPage(options) {
+    const { url, signal: callerSignal, timeoutMs, userAgent = DEFAULT_USER_AGENT, archiveFallback = true, } = options;
+    const signal = buildSignal(callerSignal, timeoutMs);
+    const headers = buildHeaders(userAgent);
+    // --- Direct fetch attempt ---
+    let response;
+    try {
+        response = await fetch(url, { headers, signal });
+    }
+    catch (error) {
+        // Abort/timeout errors should not trigger archive fallback
+        if (isAbortError(error)) {
+            const reason = error instanceof DOMException && error.name === "TimeoutError"
+                ? "Request timed out"
+                : "Request was aborted";
+            return {
+                content: "",
+                url,
+                fetchMethod: "none",
+                error: reason,
+            };
+        }
+        // Network error — try archive fallback
+        if (archiveFallback) {
+            const archiveResult = await fetchFromArchive(url, headers, signal);
+            if (archiveResult) {
+                return archiveResult;
+            }
+        }
+        const message = error instanceof Error ? error.message : String(error);
+        return {
+            content: "",
+            url,
+            fetchMethod: "none",
+            error: `Network error: ${message}`,
+        };
+    }
+    // --- Hard block detection ---
+    if (HARD_BLOCK_STATUSES.has(response.status)) {
+        if (archiveFallback) {
+            const archiveResult = await fetchFromArchive(url, headers, signal);
+            if (archiveResult) {
+                return archiveResult;
+            }
+        }
+        return {
+            content: "",
+            url,
+            fetchMethod: "none",
+            error: `HTTP ${response.status} (blocked)`,
+        };
+    }
+    // --- Non-blocking HTTP errors (404, 500, etc.) — return immediately ---
+    if (!response.ok) {
+        return {
+            content: "",
+            url,
+            fetchMethod: "none",
+            error: `HTTP ${response.status}`,
+        };
+    }
+    // --- Read body and check for soft blocks ---
+    const body = await response.text();
+    if (isSoftBlocked(body)) {
+        if (archiveFallback) {
+            const archiveResult = await fetchFromArchive(url, headers, signal);
+            if (archiveResult) {
+                return archiveResult;
+            }
+        }
+        return {
+            content: "",
+            url,
+            fetchMethod: "none",
+            error: "Soft block detected (captcha/bot detection)",
+        };
+    }
+    // --- Success ---
+    return {
+        content: body,
+        url: response.url || url,
+        fetchMethod: "direct",
+    };
+}
+//# sourceMappingURL=fetch-page.js.map

package/dist/shared/fetch-page.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../../src/shared/fetch-page.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AA4BH,8CAA8C;AAC9C,MAAM,kBAAkB,GACtB,iHAAiH,CAAA;AAEnH,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,KAAK,CAAA;AAEhC,sFAAsF;AACtF,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;AAEzD,2HAA2H;AAC3H,MAAM,mBAAmB,GAAG,MAAM,CAAA;AAElC,6EAA6E;AAC7E,MAAM,mBAAmB,GAAG;IAC1B,SAAS;IACT,6BAA6B;IAC7B,eAAe;IACf,eAAe;IACf,iBAAiB;IACjB,kBAAkB;IAClB,YAAY;IACZ,iBAAiB;IACjB,eAAe;IACf,WAAW;IACX,UAAU;CACX,CAAA;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,YAA0B,EAAE,SAAkB;IACjE,MAAM,OAAO,GAAG,SAAS,IAAI,kBAAkB,CAAA;IAC/C,MAAM,aAAa,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAA;IACvD,CAAC;IACD,OAAO,aAAa,CAAA;AACtB,CAAC;AAED,0CAA0C;AAC1C,SAAS,YAAY,CAAC,SAAiB;IACrC,OAAO;QACL,YAAY,EAAE,SAAS;QACvB,MAAM,EAAE,iEAAiE;QACzE,iBAAiB,EAAE,gBAAgB;QACnC,eAAe,EAAE,UAAU;KAC5B,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;QACtC,OAAO,KAAK,CAAA;IACd,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAChC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;AACvE,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,KAAK,YAAY,YAAY,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC,CAChG,CAAA;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,GAAW;IAC7B,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,UAAU,CAAC,IAAI,GAAG,EAAE,CAAA;QACpB,UAAU,CAAC,QAAQ,GAAG,EAAE,CAAA;QACxB,UAAU,CAAC,QAAQ,GAAG,EAAE,CAAA;QACxB,OAAO,+BAA+B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAA;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,+BAA+B,GAAG,EAAE,CAAA;IAC7C,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,OAA+B,EAC/B,MAAmB;IAEnB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAA;IAC5B,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;QACvD,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;YACrC,OAAO;gBACL,OAAO;gBACP,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,IAAI;gBACzB,WAAW,EAAE,aAAa;aAC3B,CAAA;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,qDAAqD;IACvD,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,MAAM,EACJ,GAAG,EACH,MAAM,EAAE,YAAY,EACpB,SAAS,EACT,SAAS,GAAG,kBAAkB,EAC9B,eAAe,GAAG,IAAI,GACvB,GAAG,OAAO,CAAA;IAEX,MAAM,MAAM,GAAG,WAAW,CAAC,YAAY,EAAE,SAAS,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAA;IAEvC,+BAA+B;IAC/B,IAAI,QAAkB,CAAA;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;IAClD,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,2DAA2D;QAC3D,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,MAAM,GACV,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;gBAC5D,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,qBAAqB,CAAA;YAC3B,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,GAAG;gBACH,WAAW,EAAE,MAAM;gBACnB,KAAK,EAAE,MAAM;aACd,CAAA;QACH,CAAC;QAED,uCAAuC;QACvC,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;YAClE,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO,aAAa,CAAA;YACtB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,OAAO;YACL,OAAO,EAAE,EAAE;YACX,GAAG;YACH,WAAW,EAAE,MAAM;YACnB,KAAK,EAAE,kBAAkB,OAAO,EAAE;SACnC,CAAA;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,mBAAmB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;YAClE,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO,aAAa,CAAA;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,EAAE;YACX,GAAG;YACH,WAAW,EAAE,MAAM;YACnB,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,YAAY;SAC3C,CAAA;IACH,CAAC;IAED,yEAAyE;IACzE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,OAAO;YACL,OAAO,EAAE,EAAE;YACX,GAAG;YACH,WAAW,EAAE,MAAM;YACnB,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;SACjC,CAAA;IACH,CAAC;IAED,8CAA8C;IAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAElC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,aAAa,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;YAClE,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO,aAAa,CAAA;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,OAAO,EAAE,EAAE;YACX,GAAG;YACH,WAAW,EAAE,MAAM;YACnB,KAAK,EAAE,6CAA6C;SACrD,CAAA;IACH,CAAC;IAED,kBAAkB;IAClB,OAAO;QACL,OAAO,EAAE,IAAI;QACb,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG;QACxB,WAAW,EAAE,QAAQ;KACtB,CAAA;AACH,CAAC"}

package/dist/shared/html-utils.d.ts

@@ -0,0 +1,99 @@
+/**
+ * HTML sanitization and text extraction utilities.
+ *
+ * Provides a complete pipeline for converting raw HTML to clean plain text:
+ * script/style tag removal (via state machines for robustness), HTML tag
+ * stripping, entity decoding, whitespace normalization, and optional
+ * code-fragment detection/removal.
+ */
+/**
+ * Decode HTML entities in a string using the `he` library.
+ *
+ * Handles all HTML entities including named (&), decimal (&),
+ * and hexadecimal (&) numeric entities.
+ *
+ * @param text - Text containing HTML entities
+ * @returns Decoded text
+ */
+export declare function decodeHtmlEntities(text: string): string;
+/**
+ * Remove script tags and their content from HTML.
+ *
+ * Uses a state-machine approach that handles edge cases like malformed
+ * tags better than a single regex would.
+ *
+ * @param html - HTML string
+ * @returns HTML with script tags and their content removed
+ */
+export declare function removeScriptTags(html: string): string;
+/**
+ * Remove style tags and their content from HTML.
+ *
+ * Uses the same state-machine approach as removeScriptTags.
+ *
+ * @param html - HTML string
+ * @returns HTML with style tags and their content removed
+ */
+export declare function removeStyleTags(html: string): string;
+/**
+ * Strip all HTML tags from a string, replacing them with spaces.
+ *
+ * @param html - HTML string
+ * @returns Plain text with tags replaced by spaces
+ */
+export declare function stripHtmlTags(html: string): string;
+/**
+ * Convert HTML to clean plain text.
+ *
+ * Applies the full sanitization pipeline:
+ * 1. Remove script tags and content (state machine)
+ * 2. Remove style tags and content (state machine)
+ * 3. Strip remaining HTML tags
+ * 4. Decode HTML entities
+ * 5. Normalize whitespace
+ *
+ * @param html - HTML string to clean
+ * @returns Clean plain text
+ */
+export declare function htmlToText(html: string): string;
+/**
+ * Decode HTML entities and normalize whitespace without removing tags.
+ *
+ * Useful when you want to preserve HTML structure but decode entities.
+ *
+ * @param html - HTML string
+ * @returns HTML with decoded entities and normalized whitespace
+ */
+export declare function cleanHtmlEntities(html: string): string;
+/**
+ * Detect if text looks like programming code using heuristics.
+ *
+ * Designed to catch JavaScript/TypeScript code fragments that might
+ * appear in scraped web pages from client-side rendered sites.
+ *
+ * @param text - Text to analyze
+ * @returns True if text appears to be programming code
+ */
+export declare function looksLikeCode(text: string): boolean;
+/**
+ * Strip code segments from text, keeping natural language content.
+ *
+ * If the entire text looks like code, returns an empty string.
+ * Otherwise splits by sentence/code boundaries and filters out
+ * segments that match code heuristics.
+ *
+ * @param text - Text that may contain code segments
+ * @returns Text with code segments removed
+ */
+export declare function stripCodeFromText(text: string): string;
+/**
+ * Convert HTML to clean plain text, also stripping code-like content.
+ *
+ * Combines the full HTML cleaning pipeline with code detection for
+ * maximum safety when processing scraped web pages.
+ *
+ * @param html - HTML string to clean
+ * @returns Clean plain text with code segments removed
+ */
+export declare function htmlToTextClean(html: string): string;
+//# sourceMappingURL=html-utils.d.ts.map

package/dist/shared/html-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-utils.d.ts","sourceRoot":"","sources":["../../src/shared/html-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuCrD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAiCpD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAiB/C;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAItD;AAsCD;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAcnD;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAoBtD;AAED;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGpD"}

package/dist/shared/html-utils.js

@@ -0,0 +1,246 @@
+/**
+ * HTML sanitization and text extraction utilities.
+ *
+ * Provides a complete pipeline for converting raw HTML to clean plain text:
+ * script/style tag removal (via state machines for robustness), HTML tag
+ * stripping, entity decoding, whitespace normalization, and optional
+ * code-fragment detection/removal.
+ */
+import he from "he";
+/**
+ * Decode HTML entities in a string using the `he` library.
+ *
+ * Handles all HTML entities including named (&), decimal (&),
+ * and hexadecimal (&) numeric entities.
+ *
+ * @param text - Text containing HTML entities
+ * @returns Decoded text
+ */
+export function decodeHtmlEntities(text) {
+    return he.decode(text);
+}
+/**
+ * Remove script tags and their content from HTML.
+ *
+ * Uses a state-machine approach that handles edge cases like malformed
+ * tags better than a single regex would.
+ *
+ * @param html - HTML string
+ * @returns HTML with script tags and their content removed
+ */
+export function removeScriptTags(html) {
+    let result = "";
+    let i = 0;
+    const lowerHtml = html.toLowerCase();
+    while (i < html.length) {
+        const scriptStart = lowerHtml.indexOf("<script", i);
+        if (scriptStart === -1) {
+            result += html.slice(i);
+            break;
+        }
+        // Add content before script tag
+        result += html.slice(i, scriptStart);
+        // Find the end of the opening script tag
+        const tagEnd = html.indexOf(">", scriptStart);
+        if (tagEnd === -1) {
+            // Malformed - no closing bracket, skip rest
+            break;
+        }
+        // Find closing </script> tag
+        const scriptEnd = lowerHtml.indexOf("</script", tagEnd);
+        if (scriptEnd === -1) {
+            // No closing tag, skip rest of document
+            break;
+        }
+        // Find end of closing tag
+        const closeEnd = html.indexOf(">", scriptEnd);
+        if (closeEnd === -1) {
+            break;
+        }
+        i = closeEnd + 1;
+    }
+    return result;
+}
+/**
+ * Remove style tags and their content from HTML.
+ *
+ * Uses the same state-machine approach as removeScriptTags.
+ *
+ * @param html - HTML string
+ * @returns HTML with style tags and their content removed
+ */
+export function removeStyleTags(html) {
+    let result = "";
+    let i = 0;
+    const lowerHtml = html.toLowerCase();
+    while (i < html.length) {
+        const styleStart = lowerHtml.indexOf("<style", i);
+        if (styleStart === -1) {
+            result += html.slice(i);
+            break;
+        }
+        result += html.slice(i, styleStart);
+        const tagEnd = html.indexOf(">", styleStart);
+        if (tagEnd === -1) {
+            break;
+        }
+        const styleEnd = lowerHtml.indexOf("</style", tagEnd);
+        if (styleEnd === -1) {
+            break;
+        }
+        const closeEnd = html.indexOf(">", styleEnd);
+        if (closeEnd === -1) {
+            break;
+        }
+        i = closeEnd + 1;
+    }
+    return result;
+}
+/**
+ * Strip all HTML tags from a string, replacing them with spaces.
+ *
+ * @param html - HTML string
+ * @returns Plain text with tags replaced by spaces
+ */
+export function stripHtmlTags(html) {
+    return html.replace(/<[^>]+>/g, " ");
+}
+/**
+ * Convert HTML to clean plain text.
+ *
+ * Applies the full sanitization pipeline:
+ * 1. Remove script tags and content (state machine)
+ * 2. Remove style tags and content (state machine)
+ * 3. Strip remaining HTML tags
+ * 4. Decode HTML entities
+ * 5. Normalize whitespace
+ *
+ * @param html - HTML string to clean
+ * @returns Clean plain text
+ */
+export function htmlToText(html) {
+    let text = html;
+    // Remove script and style tags first (before stripping all tags)
+    text = removeScriptTags(text);
+    text = removeStyleTags(text);
+    // Remove all other HTML tags
+    text = stripHtmlTags(text);
+    // Decode HTML entities
+    text = decodeHtmlEntities(text);
+    // Normalize whitespace
+    text = text.replace(/\s+/g, " ").trim();
+    return text;
+}
+/**
+ * Decode HTML entities and normalize whitespace without removing tags.
+ *
+ * Useful when you want to preserve HTML structure but decode entities.
+ *
+ * @param html - HTML string
+ * @returns HTML with decoded entities and normalized whitespace
+ */
+export function cleanHtmlEntities(html) {
+    let text = decodeHtmlEntities(html);
+    text = text.replace(/\s+/g, " ").trim();
+    return text;
+}
+// Patterns that strongly indicate JavaScript/TypeScript code.
+// Hoisted to module level to avoid re-allocation on each call.
+const CODE_PATTERNS = [
+    /\bfunction\s*\(/,
+    /\b(?:const|let|var)\s+\w+\s*=/,
+    /\bif\s*\([^)]+\)\s*\{/,
+    /\bdocument\.\w+/,
+    /=>\s*[{(]/,
+    /\bthis\.\w+\s*[=;]/,
+    /\breturn\s+(?:this|null|true|false|undefined)\b/,
+    /\bclass\s+\w+\s*\{/,
+    /\b(?:async|await)\s+\w+/,
+    /\b(?:try|catch|throw)\s*[{(]/,
+    /\bwindow\.\w+/,
+    /\bconsole\.\w+/,
+    /\.(?:push|pop|shift|unshift|slice|splice|map|filter|reduce)\s*\(/,
+    /\b(?:new|delete|typeof|instanceof)\s+\w+/,
+    /\[\s*\d+\s*\]/,
+    /===|!==|&&|\|\|/,
+    /\bfor\s*\([^)]+\)/,
+    /\bwhile\s*\([^)]+\)/,
+    /\bswitch\s*\([^)]+\)/,
+    /\)\s*\{|\{\s*$/,
+    /\.innerHTML\s*=/,
+    /\.innerText\s*=/,
+    /\.textContent\s*=/,
+    /\.value\s*=/,
+    /\.style\.\w+\s*=/,
+    /\.getElementById\s*\(/,
+    /\.querySelector\s*\(/,
+    /\.addEventListener\s*\(/,
+];
+/** Minimum number of pattern matches required to classify text as code. */
+const CODE_PATTERN_THRESHOLD = 2;
+/**
+ * Detect if text looks like programming code using heuristics.
+ *
+ * Designed to catch JavaScript/TypeScript code fragments that might
+ * appear in scraped web pages from client-side rendered sites.
+ *
+ * @param text - Text to analyze
+ * @returns True if text appears to be programming code
+ */
+export function looksLikeCode(text) {
+    if (!text || text.length < 20)
+        return false;
+    let matchCount = 0;
+    for (const pattern of CODE_PATTERNS) {
+        if (pattern.test(text)) {
+            matchCount++;
+            if (matchCount >= CODE_PATTERN_THRESHOLD) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+/**
+ * Strip code segments from text, keeping natural language content.
+ *
+ * If the entire text looks like code, returns an empty string.
+ * Otherwise splits by sentence/code boundaries and filters out
+ * segments that match code heuristics.
+ *
+ * @param text - Text that may contain code segments
+ * @returns Text with code segments removed
+ */
+export function stripCodeFromText(text) {
+    if (!text)
+        return "";
+    // If the whole text looks like code, return empty
+    if (looksLikeCode(text)) {
+        return "";
+    }
+    // Split into segments by sentence endings or code delimiters
+    const segments = text.split(/(?<=[.!?])\s+|(?<=[;{}])\s+/);
+    // Filter out code segments
+    const filtered = segments
+        .map((segment) => segment.trim())
+        .filter((segment) => {
+        if (segment.length < 15)
+            return false;
+        return !looksLikeCode(segment);
+    });
+    return filtered.join(" ").trim();
+}
+/**
+ * Convert HTML to clean plain text, also stripping code-like content.
+ *
+ * Combines the full HTML cleaning pipeline with code detection for
+ * maximum safety when processing scraped web pages.
+ *
+ * @param html - HTML string to clean
+ * @returns Clean plain text with code segments removed
+ */
+export function htmlToTextClean(html) {
+    const text = htmlToText(html);
+    return stripCodeFromText(text);
+}
+//# sourceMappingURL=html-utils.js.map

package/dist/shared/html-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-utils.js","sourceRoot":"","sources":["../../src/shared/html-utils.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAA;AAEnB;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,IAAI,MAAM,GAAG,EAAE,CAAA;IACf,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAEpC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAA;QACnD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YACvB,MAAK;QACP,CAAC;QAED,gCAAgC;QAChC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QAEpC,yCAAyC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;QAC7C,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAClB,4CAA4C;YAC5C,MAAK;QACP,CAAC;QAED,6BAA6B;QAC7B,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QACvD,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,wCAAwC;YACxC,MAAK;QACP,CAAC;QAED,0BAA0B;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;QAC7C,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAK;QACP,CAAC;QAED,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,MAAM,GAAG,EAAE,CAAA;IACf,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAEpC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;QACjD,IAAI,UAAU,KAAK,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YACvB,MAAK;QACP,CAAC;QAED,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;QAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,UAAU,CAAC,CAAA;QAC5C,IAAI,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAClB,MAAK;QACP,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACrD,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAK;QACP,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAC5C,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;YACpB,MAAK;QACP,CAAC;QAED,CAAC,GAAG,QAAQ,GAAG,CAAC,CAAA;IAClB,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;AACtC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,IAAI,GAAG,IAAI,CAAA;IAEf,iEAAiE;IACjE,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC7B,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;IAE5B,6BAA6B;IAC7B,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;IAE1B,uBAAuB;IACvB,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IAE/B,uBAAuB;IACvB,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEvC,OAAO,IAAI,CAAA;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IACnC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACvC,OAAO,IAAI,CAAA;AACb,CAAC;AAED,8DAA8D;AAC9D,+DAA+D;AAC/D,MAAM,aAAa,GAAG;IACpB,iBAAiB;IACjB,+BAA+B;IAC/B,uBAAuB;IACvB,iBAAiB;IACjB,WAAW;IACX,oBAAoB;IACpB,iDAAiD;IACjD,oBAAoB;IACpB,yBAAyB;IACzB,8BAA8B;IAC9B,eAAe;IACf,gBAAgB;IAChB,kEAAkE;IAClE,0CAA0C;IAC1C,eAAe;IACf,iBAAiB;IACjB,mBAAmB;IACnB,qBAAqB;IACrB,sBAAsB;IACtB,gBAAgB;IAChB,iBAAiB;IACjB,iBAAiB;IACjB,mBAAmB;IACnB,aAAa;IACb,kBAAkB;IAClB,uBAAuB;IACvB,sBAAsB;IACtB,yBAAyB;CAC1B,CAAA;AAED,2EAA2E;AAC3E,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEhC;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;QAAE,OAAO,KAAK,CAAA;IAE3C,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,UAAU,EAAE,CAAA;YACZ,IAAI,UAAU,IAAI,sBAAsB,EAAE,CAAC;gBACzC,OAAO,IAAI,CAAA;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAA;IAEpB,kDAAkD;IAClD,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,6DAA6D;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAE1D,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,QAAQ;SACtB,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;SAChC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;QAClB,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,KAAK,CAAA;QACrC,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEJ,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AAClC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;IAC7B,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC"}

package/dist/shared/readability-extract.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Article extraction using Mozilla Readability.
+ *
+ * Uses the same algorithm behind Firefox Reader View to extract article
+ * content from raw HTML, stripping navigation, ads, sidebars, and other
+ * non-content elements. Far more reliable than regex-based extraction.
+ *
+ * Dependencies: @mozilla/readability, jsdom
+ */
+/** Result of extracting article content from HTML. */
+export interface ArticleExtractionResult {
+    text: string;
+    title: string | null;
+    author: string | null;
+    excerpt: string | null;
+    siteName: string | null;
+}
+/**
+ * Extract article content from raw HTML using Mozilla Readability.
+ *
+ * Parses the HTML into a DOM, runs Mozilla's Readability algorithm to
+ * identify the main article body, and returns the plain text content
+ * along with metadata (title, author, excerpt, site name).
+ *
+ * Returns null if Readability cannot identify article content or if
+ * the extracted text is shorter than 100 characters.
+ *
+ * @param html - Raw HTML string to extract from
+ * @param url - Optional URL for resolving relative links in the HTML
+ * @returns Extracted article content and metadata, or null
+ */
+export declare function extractArticleContent(html: string, url?: string): ArticleExtractionResult | null;
+//# sourceMappingURL=readability-extract.d.ts.map

package/dist/shared/readability-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability-extract.d.ts","sourceRoot":"","sources":["../../src/shared/readability-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,sDAAsD;AACtD,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,IAAI,CAqBhG"}

package/dist/shared/readability-extract.js

@@ -0,0 +1,45 @@
+/**
+ * Article extraction using Mozilla Readability.
+ *
+ * Uses the same algorithm behind Firefox Reader View to extract article
+ * content from raw HTML, stripping navigation, ads, sidebars, and other
+ * non-content elements. Far more reliable than regex-based extraction.
+ *
+ * Dependencies: @mozilla/readability, jsdom
+ */
+import { Readability } from "@mozilla/readability";
+import { JSDOM, VirtualConsole } from "jsdom";
+/**
+ * Extract article content from raw HTML using Mozilla Readability.
+ *
+ * Parses the HTML into a DOM, runs Mozilla's Readability algorithm to
+ * identify the main article body, and returns the plain text content
+ * along with metadata (title, author, excerpt, site name).
+ *
+ * Returns null if Readability cannot identify article content or if
+ * the extracted text is shorter than 100 characters.
+ *
+ * @param html - Raw HTML string to extract from
+ * @param url - Optional URL for resolving relative links in the HTML
+ * @returns Extracted article content and metadata, or null
+ */
+export function extractArticleContent(html, url) {
+    // Suppress all JSDOM console output (CSS parsing warnings, resource loading errors, etc.)
+    // These are expected and harmless when parsing arbitrary web pages for article extraction.
+    const virtualConsole = new VirtualConsole();
+    // No event listeners attached — all errors are silently suppressed
+    const dom = new JSDOM(html, { url: url || undefined, virtualConsole });
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+    if (!article || !article.textContent || article.textContent.length < 100) {
+        return null;
+    }
+    return {
+        text: article.textContent,
+        title: article.title || null,
+        author: article.byline || null,
+        excerpt: article.excerpt || null,
+        siteName: article.siteName || null,
+    };
+}
+//# sourceMappingURL=readability-extract.js.map

package/dist/shared/readability-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability-extract.js","sourceRoot":"","sources":["../../src/shared/readability-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAA;AAW7C;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,GAAY;IAC9D,0FAA0F;IAC1F,2FAA2F;IAC3F,MAAM,cAAc,GAAG,IAAI,cAAc,EAAE,CAAA;IAC3C,mEAAmE;IAEnE,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,SAAS,EAAE,cAAc,EAAE,CAAC,CAAA;IACtE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAA;IAE9B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACzE,OAAO,IAAI,CAAA;IACb,CAAC;IAED,OAAO;QACL,IAAI,EAAE,OAAO,CAAC,WAAW;QACzB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;QAChC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;KACnC,CAAA;AACH,CAAC"}