@bluefields/extractor 0.2.0

package/dist/cache-key.js

@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Cache-key helpers (Review #6).
+ *
+ * The cache key for an extraction is `(content_hash, schema_hash,
+ * extractor_version)`. Two customers watching the same URL with the
+ * same JSON schema and the same extractor version share the LLM call
+ * exactly once — chunk 11 wires this into the orchestrator.
+ *
+ * Schema hashing must be deterministic across key-order permutations
+ * (`{a: 1, b: 2}` and `{b: 2, a: 1}` must produce the same hash) — we
+ * canonicalize via recursive key-sort before hashing.
+ */
+import { createHash } from 'node:crypto';
+/**
+ * SHA-256 hex of the canonical JSON form of `schema`. Returns null
+ * for null/undefined/empty-object schemas (markdown-only extractions).
+ */
+export function schemaHash(schema) {
+    if (schema === null || schema === undefined)
+        return null;
+    if (typeof schema === 'object' && Object.keys(schema).length === 0)
+        return null;
+    const canonical = canonicalizeJson(schema);
+    return createHash('sha256').update(canonical).digest('hex');
+}
+function canonicalizeJson(v) {
+    if (v === null || typeof v !== 'object')
+        return JSON.stringify(v);
+    if (Array.isArray(v)) {
+        return `[${v.map(canonicalizeJson).join(',')}]`;
+    }
+    const obj = v;
+    const keys = Object.keys(obj).sort();
+    return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalizeJson(obj[k])}`).join(',')}}`;
+}
+//# sourceMappingURL=cache-key.js.map

package/dist/cache-key.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache-key.js","sourceRoot":"","sources":["../src/cache-key.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,MAAe;IACxC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,MAAgB,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1F,MAAM,SAAS,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC3C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,gBAAgB,CAAC,CAAU;IAClC,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAClE,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAClD,CAAC;IACD,MAAM,GAAG,GAAG,CAA4B,CAAC;IACzC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACrC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9F,CAAC"}

package/dist/guards.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Input pre-flight guards shared by the extraction pipeline (perf-extract).
+ *
+ * Everything here runs on the RAW HTML STRING, before any DOM is built.
+ * The point is to keep pathological inputs (50 MB bodies, 5000-deep
+ * nesting, binary blobs mislabeled as HTML) from ever reaching jsdom,
+ * whose memory footprint is a large multiple of the input size and whose
+ * recursive walks overflow the stack on deep markup. Guards must PREVENT
+ * the allocation — an OOM cannot be caught after the fact.
+ *
+ * The fetcher caps response bodies at 5 MiB (packages/fetcher,
+ * DEFAULT_MAX_BODY_BYTES), so real traffic never trips the size ceiling;
+ * these are defense-in-depth for other ingress paths and future callers.
+ */
+/** Why the extractor degraded instead of running the full pipeline. */
+export type DegradeReason = 'input_too_large' | 'nesting_too_deep' | 'binary_input' | 'parse_failed' | 'output_truncated' | 'table_recovery_capped';
+/** Structured degrade signal attached to `ExtractedMarkdown.degraded`. */
+export interface ExtractDegradeInfo {
+    reasons: DegradeReason[];
+    /** Short human-readable specifics (sizes, counts). */
+    detail?: string;
+}
+/**
+ * Hard ceiling on the HTML string length we will hand to jsdom.
+ *
+ * Measured (profile run 2026-07-01, node --max-old-space-size=1024,
+ * mirroring the 1 GB Fly VM): peak RSS is linear at ~270 MB per MB of
+ * article HTML and ~2x that for table-dense pages; a 4.25 MB single-table
+ * page OOM-aborts outright and a 25 MB article GC-thrashes forever. The
+ * HTTP/Bright-Data fetch tiers cap bodies at 5 MiB but the browser tier
+ * (page.content()) is UNCAPPED, so oversized pages do reach the extractor
+ * in production. 1.5M chars keeps a worst-case single extraction around
+ * ~0.5-0.9 GB peak and comfortably clears the largest known-legit corpus
+ * fixture (wiki-zh-python, 1.05M chars). Inputs beyond it degrade to a
+ * bounded tag-strip instead of a DOM parse.
+ */
+export declare const MAX_EXTRACT_HTML_CHARS = 1500000;
+/** Ceiling on degraded-path text output (markdown/text fields). */
+export declare const MAX_DEGRADED_TEXT_CHARS = 1000000;
+/** Tables with more <tr> than this are never snapshotted (checked BEFORE serializing). */
+export declare const MAX_CAPTURE_TABLE_ROWS = 2000;
+/** A single captured table's outerHTML may not exceed this. */
+export declare const MAX_SINGLE_TABLE_CHARS = 1000000;
+/** Total chars across all captured tables. */
+export declare const MAX_TOTAL_CAPTURED_CHARS = 2000000;
+/** Max number of tables snapshotted per page. */
+export declare const MAX_CAPTURED_TABLES = 100;
+/** Skip the recovery re-parse (second jsdom) when the article content exceeds this. */
+export declare const MAX_RECOVERY_CONTENT_CHARS = 1500000;
+/**
+ * Deepest markup we'll build a DOM for. Comfortably above the 2000-deep
+ * contract fixture (which still extracts normally) and below the ~4000-deep
+ * zone where jsdom parse time explodes and serialization overflows the stack.
+ */
+export declare const MAX_SAFE_NESTING_DEPTH = 2500;
+/**
+ * Estimate the maximum element-nesting depth of an HTML string WITHOUT parsing
+ * it — a running open/close balance over element tags. Conservative (it may
+ * over-count on malformed input, which only makes the guard fire sooner) and
+ * returns as soon as the cap is exceeded, so it stays cheap on huge inputs.
+ * Comments / doctype / PIs are ignored (the tag name must start with a letter).
+ *
+ * Hand-rolled char scan, NOT a regex: the previous
+ * `/<(\/?)([a-zA-Z][\w-]*)([^>]*)>/g` backtracked quadratically on
+ * '>'-free adversarial input (ReDoS on the public sandbox route). This
+ * scan is O(n) by construction and preserves the regex's semantics.
+ */
+export declare function estimateMaxNestingDepth(html: string): number;
+/**
+ * Cheap sniff for binary-pretending-to-be-HTML. True when the input starts
+ * with a known binary magic, or has a high density of NUL/C0 control
+ * characters (legit HTML text has essentially none besides whitespace —
+ * even mojibake from a wrong charset decodes to printable or C1 ranges,
+ * which we deliberately do NOT count to avoid false positives). A single
+ * stray NUL is deliberately NOT enough: a one-byte decoding artifact must
+ * not zero out an otherwise-normal page — NULs count toward the density
+ * rule instead (UTF-16-mislabeled text is ~50% NUL, real binary ~11% C0;
+ * both clear the 5% bar by a wide margin).
+ */
+export declare function looksLikeBinary(html: string): boolean;
+/**
+ * Linear-time removal of <script>/<style> blocks via indexOf scanning.
+ * Replaces the old lazy-regex approach, which backtracked quadratically on
+ * adversarial input (thousands of unclosed `<script` openers). An opener
+ * with no matching closer drops the rest of the string — correct for the
+ * degraded path (script bodies are never content).
+ */
+export declare function stripScriptStyleBlocks(html: string): string;
+/**
+ * Linear-time tag stripper for the degraded path, mimicking the semantics
+ * of `/<[^>]+>/g → ' '` without its backtracking (which is quadratic on
+ * '>'-free adversarial input): a complete `<...>` span with at least one
+ * char inside is replaced by a space; a `<` with no closing `>` in the
+ * remainder stays as literal text, exactly like the regex.
+ */
+export declare function stripTagsLinear(html: string): string;
+/** Widest table (columns) the markdown converter will emit. */
+export declare const MAX_TABLE_COLS = 500;
+/** Max cells (cols × rows) the markdown converter will emit per table. */
+export declare const MAX_TABLE_CELLS = 250000;

package/dist/guards.js

@@ -0,0 +1,276 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Input pre-flight guards shared by the extraction pipeline (perf-extract).
+ *
+ * Everything here runs on the RAW HTML STRING, before any DOM is built.
+ * The point is to keep pathological inputs (50 MB bodies, 5000-deep
+ * nesting, binary blobs mislabeled as HTML) from ever reaching jsdom,
+ * whose memory footprint is a large multiple of the input size and whose
+ * recursive walks overflow the stack on deep markup. Guards must PREVENT
+ * the allocation — an OOM cannot be caught after the fact.
+ *
+ * The fetcher caps response bodies at 5 MiB (packages/fetcher,
+ * DEFAULT_MAX_BODY_BYTES), so real traffic never trips the size ceiling;
+ * these are defense-in-depth for other ingress paths and future callers.
+ */
+/**
+ * Hard ceiling on the HTML string length we will hand to jsdom.
+ *
+ * Measured (profile run 2026-07-01, node --max-old-space-size=1024,
+ * mirroring the 1 GB Fly VM): peak RSS is linear at ~270 MB per MB of
+ * article HTML and ~2x that for table-dense pages; a 4.25 MB single-table
+ * page OOM-aborts outright and a 25 MB article GC-thrashes forever. The
+ * HTTP/Bright-Data fetch tiers cap bodies at 5 MiB but the browser tier
+ * (page.content()) is UNCAPPED, so oversized pages do reach the extractor
+ * in production. 1.5M chars keeps a worst-case single extraction around
+ * ~0.5-0.9 GB peak and comfortably clears the largest known-legit corpus
+ * fixture (wiki-zh-python, 1.05M chars). Inputs beyond it degrade to a
+ * bounded tag-strip instead of a DOM parse.
+ */
+export const MAX_EXTRACT_HTML_CHARS = 1_500_000;
+/** Ceiling on degraded-path text output (markdown/text fields). */
+export const MAX_DEGRADED_TEXT_CHARS = 1_000_000;
+// ── Table-recovery ceilings ──────────────────────────────────────────────
+// Table capture snapshots outerHTML pre-Readability and recovery builds a
+// SECOND jsdom of the article content — profiling shows this double-DOM is
+// what makes table-dense pages cost ~2x articles per byte. Caps sit far
+// above every corpus fixture (max seen: 257 rows, 13 tables, ~0.8M-char
+// table page) so legit pages never trip them.
+/** Tables with more <tr> than this are never snapshotted (checked BEFORE serializing). */
+export const MAX_CAPTURE_TABLE_ROWS = 2_000;
+/** A single captured table's outerHTML may not exceed this. */
+export const MAX_SINGLE_TABLE_CHARS = 1_000_000;
+/** Total chars across all captured tables. */
+export const MAX_TOTAL_CAPTURED_CHARS = 2_000_000;
+/** Max number of tables snapshotted per page. */
+export const MAX_CAPTURED_TABLES = 100;
+/** Skip the recovery re-parse (second jsdom) when the article content exceeds this. */
+export const MAX_RECOVERY_CONTENT_CHARS = 1_500_000;
+/**
+ * Deepest markup we'll build a DOM for. Comfortably above the 2000-deep
+ * contract fixture (which still extracts normally) and below the ~4000-deep
+ * zone where jsdom parse time explodes and serialization overflows the stack.
+ */
+export const MAX_SAFE_NESTING_DEPTH = 2500;
+/** HTML void elements — they never open a nesting level. */
+const VOID_ELEMENTS = new Set([
+    'area',
+    'base',
+    'br',
+    'col',
+    'embed',
+    'hr',
+    'img',
+    'input',
+    'link',
+    'meta',
+    'param',
+    'source',
+    'track',
+    'wbr',
+]);
+/** Is `c` an ASCII letter? */
+function isAsciiLetter(c) {
+    return (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a);
+}
+/** Is `c` a tag-name char per the old regex's `[\w-]`? */
+function isNameChar(c) {
+    return isAsciiLetter(c) || (c >= 0x30 && c <= 0x39) || c === 0x5f /* _ */ || c === 0x2d /* - */;
+}
+/**
+ * Estimate the maximum element-nesting depth of an HTML string WITHOUT parsing
+ * it — a running open/close balance over element tags. Conservative (it may
+ * over-count on malformed input, which only makes the guard fire sooner) and
+ * returns as soon as the cap is exceeded, so it stays cheap on huge inputs.
+ * Comments / doctype / PIs are ignored (the tag name must start with a letter).
+ *
+ * Hand-rolled char scan, NOT a regex: the previous
+ * `/<(\/?)([a-zA-Z][\w-]*)([^>]*)>/g` backtracked quadratically on
+ * '>'-free adversarial input (ReDoS on the public sandbox route). This
+ * scan is O(n) by construction and preserves the regex's semantics.
+ */
+export function estimateMaxNestingDepth(html) {
+    const n = html.length;
+    let depth = 0;
+    let max = 0;
+    let i = 0;
+    while (i < n) {
+        const lt = html.indexOf('<', i);
+        if (lt === -1)
+            break;
+        let p = lt + 1;
+        const isClosing = p < n && html.charCodeAt(p) === 0x2f; /* '/' */
+        if (isClosing)
+            p += 1;
+        if (p >= n || !isAsciiLetter(html.charCodeAt(p))) {
+            // Not an element tag (comment/doctype/PI/stray '<') — skip the '<'.
+            i = lt + 1;
+            continue;
+        }
+        const nameStart = p;
+        while (p < n && isNameChar(html.charCodeAt(p)))
+            p += 1;
+        const gt = html.indexOf('>', p);
+        if (gt === -1)
+            break; // no '>' remains — no further tag can complete
+        if (isClosing) {
+            depth = Math.max(0, depth - 1);
+        }
+        else {
+            // Self-closing: last non-whitespace char before '>' is '/'.
+            let q = gt - 1;
+            while (q >= p) {
+                const c = html.charCodeAt(q);
+                if (c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d || c === 0x0c)
+                    q -= 1;
+                else
+                    break;
+            }
+            const selfClosed = q >= p && html.charCodeAt(q) === 0x2f;
+            const name = html.slice(nameStart, p).toLowerCase();
+            if (!VOID_ELEMENTS.has(name) && !selfClosed) {
+                depth += 1;
+                if (depth > max) {
+                    max = depth;
+                    if (max > MAX_SAFE_NESTING_DEPTH)
+                        return max; // cap hit — stop scanning
+                }
+            }
+        }
+        i = gt + 1;
+    }
+    return max;
+}
+/** Window of the input the binary sniff examines. */
+const BINARY_SNIFF_CHARS = 65_536;
+/**
+ * Magic-byte prefixes of common binary formats served with a text/html
+ * content-type (misconfigured servers, hotlinked assets). Checked after
+ * skipping an optional BOM + leading whitespace. Written as \\uXXXX escapes
+ * and kept byte-precise — a bare 'PK'/'PNG'/'ELF' prefix would
+ * false-positive on legitimate text that happens to start with those letters.
+ */
+const BINARY_MAGIC_PREFIXES = [
+    '%PDF-', // PDF (real PDFs are routed to extractPdfText upstream)
+    '\u0089PNG\r\n\u001a\n', // PNG
+    'GIF87a',
+    'GIF89a',
+    '\u00ff\u00d8\u00ff', // JPEG
+    'PK\u0003\u0004', // zip / docx / epub
+    'PK\u0005\u0006', // empty zip
+    'PK\u0007\u0008', // spanned zip
+    '\u007fELF', // ELF
+    '\u001f\u008b', // gzip
+    'OggS', // Ogg
+    'wOFF', // WOFF font
+    'wOF2', // WOFF2 font
+];
+/**
+ * Cheap sniff for binary-pretending-to-be-HTML. True when the input starts
+ * with a known binary magic, or has a high density of NUL/C0 control
+ * characters (legit HTML text has essentially none besides whitespace —
+ * even mojibake from a wrong charset decodes to printable or C1 ranges,
+ * which we deliberately do NOT count to avoid false positives). A single
+ * stray NUL is deliberately NOT enough: a one-byte decoding artifact must
+ * not zero out an otherwise-normal page — NULs count toward the density
+ * rule instead (UTF-16-mislabeled text is ~50% NUL, real binary ~11% C0;
+ * both clear the 5% bar by a wide margin).
+ */
+export function looksLikeBinary(html) {
+    if (html.length === 0)
+        return false;
+    const window = html.slice(0, BINARY_SNIFF_CHARS);
+    // BOM / whitespace-tolerant magic check.
+    const trimmed = window.replace(/^[\ufeff\s]+/, '');
+    for (const magic of BINARY_MAGIC_PREFIXES) {
+        if (trimmed.startsWith(magic))
+            return true;
+    }
+    // Density of NUL + C0 controls excluding \t \n \r \f.
+    let controls = 0;
+    for (let i = 0; i < window.length; i++) {
+        const c = window.charCodeAt(i);
+        if (c < 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d && c !== 0x0c)
+            controls += 1;
+    }
+    return controls / window.length > 0.05;
+}
+/**
+ * Linear-time removal of <script>/<style> blocks via indexOf scanning.
+ * Replaces the old lazy-regex approach, which backtracked quadratically on
+ * adversarial input (thousands of unclosed `<script` openers). An opener
+ * with no matching closer drops the rest of the string — correct for the
+ * degraded path (script bodies are never content).
+ */
+export function stripScriptStyleBlocks(html) {
+    const lower = html.toLowerCase();
+    const parts = [];
+    let i = 0;
+    // Needle cursors advance monotonically and stick at -1 once absent —
+    // re-searching an absent needle from every position was itself quadratic.
+    let s = lower.indexOf('<script');
+    let t = lower.indexOf('<style');
+    while (i < html.length) {
+        if (s !== -1 && s < i)
+            s = lower.indexOf('<script', i);
+        if (t !== -1 && t < i)
+            t = lower.indexOf('<style', i);
+        const start = s === -1 ? t : t === -1 ? s : Math.min(s, t);
+        if (start === -1) {
+            parts.push(html.slice(i));
+            break;
+        }
+        parts.push(html.slice(i, start));
+        const closer = start === s ? '</script' : '</style';
+        const end = lower.indexOf(closer, start);
+        if (end === -1)
+            break; // unclosed — drop the remainder
+        const gt = html.indexOf('>', end);
+        i = gt === -1 ? html.length : gt + 1;
+    }
+    return parts.join(' ');
+}
+/**
+ * Linear-time tag stripper for the degraded path, mimicking the semantics
+ * of `/<[^>]+>/g → ' '` without its backtracking (which is quadratic on
+ * '>'-free adversarial input): a complete `<...>` span with at least one
+ * char inside is replaced by a space; a `<` with no closing `>` in the
+ * remainder stays as literal text, exactly like the regex.
+ */
+export function stripTagsLinear(html) {
+    const parts = [];
+    let i = 0;
+    while (i < html.length) {
+        const lt = html.indexOf('<', i);
+        if (lt === -1) {
+            parts.push(html.slice(i));
+            break;
+        }
+        const gt = html.indexOf('>', lt + 1);
+        if (gt === -1) {
+            // No complete tag remains — the tail (including the '<') is text.
+            parts.push(html.slice(i));
+            break;
+        }
+        if (gt === lt + 1) {
+            // '<>' — the regex required at least one char inside; keep it.
+            parts.push(html.slice(i, gt + 1));
+            i = gt + 1;
+            continue;
+        }
+        parts.push(html.slice(i, lt));
+        parts.push(' ');
+        i = gt + 1;
+    }
+    return parts.join('');
+}
+// ── Markdown table geometry ceilings (gfmTable rule) ─────────────────────
+// GFM conversion pads EVERY row to the widest row's column count, so one
+// ragged 100k-cell row multiplies across all rows: a page under all the
+// input caps could still emit hundreds of MB of markdown. Corpus max seen:
+// 257 rows / small column counts — these sit far above every legit page.
+/** Widest table (columns) the markdown converter will emit. */
+export const MAX_TABLE_COLS = 500;
+/** Max cells (cols × rows) the markdown converter will emit per table. */
+export const MAX_TABLE_CELLS = 250_000;
+//# sourceMappingURL=guards.js.map

package/dist/guards.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"guards.js","sourceRoot":"","sources":["../src/guards.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;GAaG;AAkBH;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,SAAS,CAAC;AAEhD,mEAAmE;AACnE,MAAM,CAAC,MAAM,uBAAuB,GAAG,SAAS,CAAC;AAEjD,4EAA4E;AAC5E,0EAA0E;AAC1E,2EAA2E;AAC3E,wEAAwE;AACxE,wEAAwE;AACxE,8CAA8C;AAE9C,0FAA0F;AAC1F,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAC5C,+DAA+D;AAC/D,MAAM,CAAC,MAAM,sBAAsB,GAAG,SAAS,CAAC;AAChD,8CAA8C;AAC9C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAClD,iDAAiD;AACjD,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AACvC,uFAAuF;AACvF,MAAM,CAAC,MAAM,0BAA0B,GAAG,SAAS,CAAC;AAEpD;;;;GAIG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,4DAA4D;AAC5D,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,MAAM;IACN,MAAM;IACN,IAAI;IACJ,KAAK;IACL,OAAO;IACP,IAAI;IACJ,KAAK;IACL,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,QAAQ;IACR,OAAO;IACP,KAAK;CACN,CAAC,CAAC;AAEH,8BAA8B;AAC9B,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,0DAA0D;AAC1D,SAAS,UAAU,CAAC,CAAS;IAC3B,OAAO,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC;AAClG,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;IACtB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,MAAM;QACrB,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACf,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS;QACjE,IAAI,SAAS;YAAE,CAAC,IAAI,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,oEAAoE;YACpE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,CAAC,CAAC;QACpB,OAAO,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,EAAE,KAAK,CAAC,CAAC;YAAE,MAAM,CAAC,+CAA+C;QACrE,IAAI,SAAS,EAAE,CAAC;YACd,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;aAAM,CAAC;YACN,4DAA4D;YAC5D,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACd,MAAM,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAC7B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;oBAAE,CAAC,IAAI,CAAC,CAAC;;oBAC1E,MAAM;YACb,CAAC;YACD,MAAM,UAAU,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5C,KAAK,IAAI,CAAC,CAAC;gBACX,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;oBAChB,GAAG,GAAG,KAAK,CAAC;oBACZ,IAAI,GAAG,GAAG,sBAAsB;wBAAE,OAAO,GAAG,CAAC,CAAC,0BAA0B;gBAC1E,CAAC;YACH,CAAC;QACH,CAAC;QACD,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACb,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,qDAAqD;AACrD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC;;;;;;GAMG;AACH,MAAM,qBAAqB,GAAG;IAC5B,OAAO,EAAE,wDAAwD;IACjE,uBAAuB,EAAE,MAAM;IAC/B,QAAQ;IACR,QAAQ;IACR,oBAAoB,EAAE,OAAO;IAC7B,gBAAgB,EAAE,oBAAoB;IACtC,gBAAgB,EAAE,YAAY;IAC9B,gBAAgB,EAAE,cAAc;IAChC,WAAW,EAAE,MAAM;IACnB,cAAc,EAAE,OAAO;IACvB,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,aAAa;CACtB,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAEjD,yCAAyC;IACzC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACnD,KAAK,MAAM,KAAK,IAAI,qBAAqB,EAAE,CAAC;QAC1C,IAAI,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IAC7C,CAAC;IAED,sDAAsD;IACtD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;YAAE,QAAQ,IAAI,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,QAAQ,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;AACzC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,qEAAqE;IACrE,0EAA0E;IAC1E,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjC,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAChC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3D,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM;QACR,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACzC,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAM,CAAC,gCAAgC;QACvD,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM;QACR,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QACrC,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;YACd,kEAAkE;YAClE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM;QACR,CAAC;QACD,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;YAClB,+DAA+D;YAC/D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YAClC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,4EAA4E;AAC5E,yEAAyE;AACzE,wEAAwE;AACxE,2EAA2E;AAC3E,yEAAyE;AAEzE,+DAA+D;AAC/D,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,0EAA0E;AAC1E,MAAM,CAAC,MAAM,eAAe,GAAG,OAAO,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Extractor orchestrator (chunks 10 + 11).
+ *
+ * Pipeline:
+ *   1. Look up existing extraction by (content_hash, schema_hash, extractor_version)
+ *      — Review #6 cross-customer cache. Cache hit → return + skip everything below.
+ *   2. JS-shell pre-flight — if the HTML is just a loading shell, return
+ *      extraction_status='js_shell_detected' without burning a Sonnet call.
+ *   3. Mozilla Readability → markdown.
+ *   4. If schema is null/empty → markdown-only extraction (no Sonnet).
+ *   5. Truncate markdown if too long (>50k chars → 40k, flag status_detail).
+ *   6. Sonnet structured extraction via tool_use, threading customer intent.
+ *   7. Write the extractions row + cost_records row.
+ *
+ * Per the four-layer rule: the extractor returns the
+ * extraction row and nothing more. It does NOT poll, fetch, diff, or
+ * deliver — those are other layers.
+ */
+import type Anthropic from '@anthropic-ai/sdk';
+import { type DbClient, type Extraction } from '@bluefields/db';
+export interface ExtractInput {
+    /** Snapshot row id (extractions.snapshot_id). */
+    snapshotId: string;
+    /** Subscription this extraction belongs to (null for one-shot scrapes). */
+    subscriptionId: string | null;
+    customerId: string;
+    /** The HTML to extract from. */
+    html: string;
+    /** Source URL (canonical). Used by Readability + Turndown for resolving relative links. */
+    url: string;
+    /** SHA-256 hex of the raw response body — keys the cross-customer cache. */
+    contentHash: string;
+    /** Customer's JSON schema. Null/empty → markdown-only. */
+    schema: Record<string, unknown> | null;
+    /** Customer-provided one-sentence intent. Null if not set. */
+    intent: string | null;
+}
+export interface ExtractResult {
+    extraction: Extraction;
+    /** True when the result came from the (content_hash, schema_hash, extractor_version) cache. */
+    cacheHit: boolean;
+    /**
+     * Plain-text body from THIS run's Readability pass. Extraction rows don't
+     * persist text, so this is null on cache hits and failed runs — a caller
+     * that needs text on a cache hit recomputes via `extractMarkdown` (it
+     * still holds the html).
+     */
+    text: string | null;
+}
+export interface ExtractOptions {
+    anthropicClient?: Anthropic;
+    anthropicApiKey?: string;
+    /** Test seam: override Date.now()-derived timestamps. */
+    nowFn?: () => Date;
+}
+/**
+ * Run the extraction pipeline for one (snapshot, schema) pair.
+ *
+ * On cache hit: a single SELECT and an early return. No LLM call, no
+ * cost_records row, but a new extractions row is still inserted so the
+ * subscription's extraction history is complete (the row references the
+ * cached structured_data + markdown but registers under this snapshot).
+ *
+ * Actually — Review #6 cache means we reuse the EXISTING extractions
+ * row across subscriptions for the same (content_hash, schema_hash,
+ * extractor_version). For a per-subscription history view, callers
+ * resolve via the join: events.previous_extraction_id → extractions.
+ * Two subscriptions watching the same URL with the same schema literally
+ * share an extractions row.
+ */
+export declare function extract(db: DbClient, input: ExtractInput, opts?: ExtractOptions): Promise<ExtractResult>;
+export { type DegradeReason, type ExtractDegradeInfo, type ExtractedMarkdown, READABILITY_VERSION, extractMarkdown, } from './readability.js';
+export { MAX_DEGRADED_TEXT_CHARS, MAX_EXTRACT_HTML_CHARS, MAX_SAFE_NESTING_DEPTH, } from './guards.js';
+export { type SonnetExtractResult, SONNET_MODEL, extractStructured } from './sonnet.js';
+export { HAIKU_MODEL, SONNET_MODEL_ID, type PickedModel, type ModelPickReason, pickModel, pickModelWithReason, } from './model-picker.js';
+export { isJsShell } from './js-shell-detect.js';
+export { schemaHash } from './cache-key.js';
+export { type ExtractLinksOptions, extractLinks } from './links.js';
+export { type PdfExtractionResult, extractPdfText, isPdfContentType, looksLikePdfByUrl, } from './pdf.js';
+export { type DiscoverUrlsOptions, type DiscoverUrlsResult, type ParsedSitemap, discoverUrls, parseRobotsTxtForSitemaps, parseSitemapXml, } from './sitemap.js';