@bluefields/extractor 0.2.0

package/dist/index.js

@@ -0,0 +1,281 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Extractor orchestrator (chunks 10 + 11).
+ *
+ * Pipeline:
+ *   1. Look up existing extraction by (content_hash, schema_hash, extractor_version)
+ *      — Review #6 cross-customer cache. Cache hit → return + skip everything below.
+ *   2. JS-shell pre-flight — if the HTML is just a loading shell, return
+ *      extraction_status='js_shell_detected' without burning a Sonnet call.
+ *   3. Mozilla Readability → markdown.
+ *   4. If schema is null/empty → markdown-only extraction (no Sonnet).
+ *   5. Truncate markdown if too long (>50k chars → 40k, flag status_detail).
+ *   6. Sonnet structured extraction via tool_use, threading customer intent.
+ *   7. Write the extractions row + cost_records row.
+ *
+ * Per the four-layer rule: the extractor returns the
+ * extraction row and nothing more. It does NOT poll, fetch, diff, or
+ * deliver — those are other layers.
+ */
+import { and, eq } from 'drizzle-orm';
+import { costRecords, extractions, } from '@bluefields/db';
+import { schemaHash as computeSchemaHash } from './cache-key.js';
+import { isJsShell } from './js-shell-detect.js';
+import { costPerMtokInput, costPerMtokOutput, pickModel } from './model-picker.js';
+import { READABILITY_VERSION, extractMarkdown } from './readability.js';
+import { PROMPT_VERSION, extractStructured, extractorVersionFor } from './sonnet.js';
+const SCHEMA_CONVERTER_VERSION = 'schema-v1';
+const MAX_MARKDOWN_CHARS = 50_000;
+const TRUNCATE_TARGET_CHARS = 40_000;
+/**
+ * Run the extraction pipeline for one (snapshot, schema) pair.
+ *
+ * On cache hit: a single SELECT and an early return. No LLM call, no
+ * cost_records row, but a new extractions row is still inserted so the
+ * subscription's extraction history is complete (the row references the
+ * cached structured_data + markdown but registers under this snapshot).
+ *
+ * Actually — Review #6 cache means we reuse the EXISTING extractions
+ * row across subscriptions for the same (content_hash, schema_hash,
+ * extractor_version). For a per-subscription history view, callers
+ * resolve via the join: events.previous_extraction_id → extractions.
+ * Two subscriptions watching the same URL with the same schema literally
+ * share an extractions row.
+ */
+export async function extract(db, input, opts = {}) {
+    const now = opts.nowFn ? opts.nowFn() : new Date();
+    const schemaHash = computeSchemaHash(input.schema);
+    // Haiku for flat schemas (~3x cheaper); Sonnet for nested/deep. Model
+    // is part of extractor_version, so this also keys the cache — a schema
+    // routed to Haiku won't reuse a prior Sonnet extraction (which is fine;
+    // re-running on Haiku is cheap).
+    const llmModel = pickModel(input.schema);
+    const extractorVersion = extractorVersionFor({
+        readabilityVersion: READABILITY_VERSION,
+        model: llmModel,
+        promptVersion: PROMPT_VERSION,
+        schemaVersion: SCHEMA_CONVERTER_VERSION,
+    });
+    // ── 1. Cache lookup ──────────────────────────────────────────────────
+    const cached = await findCachedExtraction(db, input.contentHash, schemaHash, extractorVersion);
+    if (cached) {
+        return { extraction: cached, cacheHit: true, text: null };
+    }
+    // ── 2. JS-shell detection ────────────────────────────────────────────
+    if (isJsShell(input.html)) {
+        return {
+            extraction: await persistExtraction(db, {
+                snapshotId: input.snapshotId,
+                subscriptionId: input.subscriptionId,
+                customerId: input.customerId,
+                contentHash: input.contentHash,
+                schemaHash,
+                extractorVersion,
+                markdown: null,
+                structuredData: null,
+                extractionStatus: 'js_shell_detected',
+                statusDetail: null,
+                llmModel: null,
+                llmInputTokens: null,
+                llmOutputTokens: null,
+                costMicroUsd: 0,
+                durationMs: Math.max(1, Date.now() - now.getTime()),
+            }),
+            cacheHit: false,
+            text: null,
+        };
+    }
+    const startMs = Date.now();
+    // ── 3. Markdown extraction ───────────────────────────────────────────
+    let markdown;
+    let text;
+    // Set when the extractor degraded (oversized/binary/deep input, capped
+    // table recovery) — surfaced via status_detail so operators can see WHY a
+    // page came back as stripped text. status_detail is free text that nothing
+    // parses (per the contract audit); extraction_status stays 'ok'.
+    let degradeDetail = null;
+    try {
+        const extracted = extractMarkdown(input.html, input.url);
+        markdown = extracted.markdown;
+        text = extracted.text;
+        if (extracted.degraded) {
+            degradeDetail = `degraded:${extracted.degraded.reasons.join('+')}`;
+        }
+    }
+    catch (_err) {
+        return {
+            extraction: await persistExtraction(db, {
+                snapshotId: input.snapshotId,
+                subscriptionId: input.subscriptionId,
+                customerId: input.customerId,
+                contentHash: input.contentHash,
+                schemaHash,
+                extractorVersion,
+                markdown: null,
+                structuredData: null,
+                extractionStatus: 'markdown_failed',
+                statusDetail: 'readability + turndown threw',
+                llmModel: null,
+                llmInputTokens: null,
+                llmOutputTokens: null,
+                costMicroUsd: 0,
+                durationMs: Math.max(1, Date.now() - startMs),
+            }),
+            cacheHit: false,
+            text: null,
+        };
+    }
+    // ── 4. Markdown-only path (no schema) ────────────────────────────────
+    if (!input.schema || schemaHash === null) {
+        return {
+            extraction: await persistExtraction(db, {
+                snapshotId: input.snapshotId,
+                subscriptionId: input.subscriptionId,
+                customerId: input.customerId,
+                contentHash: input.contentHash,
+                schemaHash,
+                extractorVersion,
+                markdown,
+                structuredData: null,
+                extractionStatus: 'ok',
+                statusDetail: degradeDetail,
+                llmModel: null,
+                llmInputTokens: null,
+                llmOutputTokens: null,
+                costMicroUsd: 0,
+                durationMs: Math.max(1, Date.now() - startMs),
+            }),
+            cacheHit: false,
+            text,
+        };
+    }
+    // ── 5. Truncation ────────────────────────────────────────────────────
+    let truncatedMarkdown = markdown;
+    const detailParts = degradeDetail ? [degradeDetail] : [];
+    if (markdown.length > MAX_MARKDOWN_CHARS) {
+        truncatedMarkdown = markdown.slice(0, TRUNCATE_TARGET_CHARS);
+        detailParts.push(`truncated_at_${TRUNCATE_TARGET_CHARS}`);
+    }
+    const statusDetail = detailParts.length > 0 ? detailParts.join(';') : null;
+    // ── 6. LLM structured extraction (Haiku or Sonnet) ───────────────────
+    let llmResult;
+    try {
+        llmResult = await extractStructured(truncatedMarkdown, input.schema, input.intent, {
+            client: opts.anthropicClient,
+            apiKey: opts.anthropicApiKey,
+            model: llmModel,
+        });
+    }
+    catch (err) {
+        const errMsg = err instanceof Error ? err.message : 'llm call failed';
+        return {
+            extraction: await persistExtraction(db, {
+                snapshotId: input.snapshotId,
+                subscriptionId: input.subscriptionId,
+                customerId: input.customerId,
+                contentHash: input.contentHash,
+                schemaHash,
+                extractorVersion,
+                markdown: truncatedMarkdown,
+                structuredData: null,
+                extractionStatus: 'llm_failed',
+                statusDetail: errMsg.slice(0, 500),
+                llmModel,
+                llmInputTokens: null,
+                llmOutputTokens: null,
+                costMicroUsd: 0,
+                durationMs: Math.max(1, Date.now() - startMs),
+            }),
+            cacheHit: false,
+            text,
+        };
+    }
+    const costMicroUsd = computeLlmCostMicros(llmResult.tokens_in, llmResult.tokens_out, llmResult.model);
+    const extraction = await persistExtraction(db, {
+        snapshotId: input.snapshotId,
+        subscriptionId: input.subscriptionId,
+        customerId: input.customerId,
+        contentHash: input.contentHash,
+        schemaHash,
+        extractorVersion,
+        markdown: truncatedMarkdown,
+        structuredData: llmResult.data,
+        extractionStatus: 'ok',
+        statusDetail,
+        llmModel: llmResult.model,
+        llmInputTokens: llmResult.tokens_in,
+        llmOutputTokens: llmResult.tokens_out,
+        costMicroUsd,
+        durationMs: Math.max(1, Date.now() - startMs),
+    });
+    // ── 7. Cost record ───────────────────────────────────────────────────
+    if (costMicroUsd > 0) {
+        await db.insert(costRecords).values({
+            customerId: input.customerId,
+            subscriptionId: input.subscriptionId,
+            urlCanonical: input.url,
+            costType: llmResult.model.includes('haiku') ? 'llm_extract_haiku' : 'llm_extract_sonnet',
+            costSubtype: llmResult.model,
+            costMicroUsd,
+            // units is numeric in Postgres; Drizzle types it as string to preserve precision.
+            units: String(llmResult.tokens_in + llmResult.tokens_out),
+        });
+    }
+    return { extraction, cacheHit: false, text };
+}
+async function persistExtraction(db, args) {
+    const row = {
+        snapshotId: args.snapshotId,
+        subscriptionId: args.subscriptionId,
+        customerId: args.customerId,
+        contentHash: args.contentHash,
+        schemaHash: args.schemaHash,
+        extractorVersion: args.extractorVersion,
+        markdown: args.markdown,
+        structuredData: args.structuredData,
+        extractionStatus: args.extractionStatus,
+        statusDetail: args.statusDetail,
+        llmModel: args.llmModel,
+        llmInputTokens: args.llmInputTokens,
+        llmOutputTokens: args.llmOutputTokens,
+        costMicroUsd: args.costMicroUsd,
+        durationMs: args.durationMs,
+    };
+    const inserted = await db.insert(extractions).values(row).returning();
+    const result = inserted[0];
+    if (!result)
+        throw new Error('extractions insert returned no rows');
+    return result;
+}
+async function findCachedExtraction(db, contentHash, schemaHash, extractorVersion) {
+    // The unique index idx_ext_cache is on (content_hash, schema_hash,
+    // extractor_version). NULL schema_hash is allowed and treated as a
+    // distinct cache bucket.
+    const rows = await db
+        .select()
+        .from(extractions)
+        .where(schemaHash === null
+        ? and(eq(extractions.contentHash, contentHash), eq(extractions.extractorVersion, extractorVersion))
+        : and(eq(extractions.contentHash, contentHash), eq(extractions.schemaHash, schemaHash), eq(extractions.extractorVersion, extractorVersion)))
+        .limit(1);
+    return rows[0] ?? null;
+}
+function computeLlmCostMicros(tokensIn, tokensOut, model) {
+    // dollars_per_MTok is dollars per million tokens. One dollar = 1e6
+    // micros, so micros = (tokens * dollars_per_million_tokens / 1e6) * 1e6
+    // = tokens * dollars_per_million_tokens. Floor to integer because
+    // cost_records.cost_micro_usd is INT4.
+    const inputMicros = tokensIn * costPerMtokInput(model);
+    const outputMicros = tokensOut * costPerMtokOutput(model);
+    return Math.floor(inputMicros + outputMicros);
+}
+export { READABILITY_VERSION, extractMarkdown, } from './readability.js';
+export { MAX_DEGRADED_TEXT_CHARS, MAX_EXTRACT_HTML_CHARS, MAX_SAFE_NESTING_DEPTH, } from './guards.js';
+export { SONNET_MODEL, extractStructured } from './sonnet.js';
+export { HAIKU_MODEL, SONNET_MODEL_ID, pickModel, pickModelWithReason, } from './model-picker.js';
+export { isJsShell } from './js-shell-detect.js';
+export { schemaHash } from './cache-key.js';
+export { extractLinks } from './links.js';
+export { extractPdfText, isPdfContentType, looksLikePdfByUrl, } from './pdf.js';
+export { discoverUrls, parseRobotsTxtForSitemaps, parseSitemapXml, } from './sitemap.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAEtC,OAAO,EAIL,WAAW,EACX,WAAW,GACZ,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAgB,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAEnG,MAAM,wBAAwB,GAAG,WAAoB,CAAC;AACtD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAwCrC;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,EAAY,EACZ,KAAmB,EACnB,OAAuB,EAAE;IAEzB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IACnD,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACnD,sEAAsE;IACtE,uEAAuE;IACvE,wEAAwE;IACxE,iCAAiC;IACjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC;QAC3C,kBAAkB,EAAE,mBAAmB;QACvC,KAAK,EAAE,QAAQ;QACf,aAAa,EAAE,cAAc;QAC7B,aAAa,EAAE,wBAAwB;KACxC,CAAC,CAAC;IAEH,wEAAwE;IACxE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,UAAU,EAAE,gBAAgB,CAAC,CAAC;IAC/F,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5D,CAAC;IAED,wEAAwE;IACxE,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,UAAU,EAAE,MAAM,iBAAiB,CAAC,EAAE,EAAE;gBACtC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,UAAU;gBACV,gBAAgB;gBAChB,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,IAAI;gBACpB,gBAAgB,EAAE,mBAAmB;gBACrC,YAAY,EAAE,IAAI;gBAClB,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC;aACpD,CAAC;YACF,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,IAAI;SACX,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,wEAAwE;IACxE,IAAI,QAAgB,CAAC;IACrB,IAAI,IAAmB,CAAC;IACxB,uEAAuE;IACvE,0EAA0E;IAC1E,2EAA2E;IAC3E,iEAAiE;IACjE,IAAI,aAAa,GAAkB,IAAI,CAAC;IACxC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACzD,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC9B,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC;QACtB,IAAI,SAAS,CAAC,QAAQ,EAAE,CAAC;YACvB,aAAa,GAAG,YAAY,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACrE,CAAC;IACH,CAAC;IAAC,OAAO,IAAI,EAAE,CAAC;QACd,OAAO;YACL,UAAU,EAAE,MAAM,iBAAiB,CAAC,EAAE,EAAE;gBACtC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,UAAU;gBACV,gBAAgB;gBAChB,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,IAAI;gBACpB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,8BAA8B;gBAC5C,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;aAC9C,CAAC;YACF,QAAQ,EAAE,KAAK;YACf,IAAI,EAAE,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACzC,OAAO;YACL,UAAU,EAAE,MAAM,iBAAiB,CAAC,EAAE,EAAE;gBACtC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,UAAU;gBACV,gBAAgB;gBAChB,QAAQ;gBACR,cAAc,EAAE,IAAI;gBACpB,gBAAgB,EAAE,IAAI;gBACtB,YAAY,EAAE,aAAa;gBAC3B,QAAQ,EAAE,IAAI;gBACd,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;aAC9C,CAAC;YACF,QAAQ,EAAE,KAAK;YACf,IAAI;SACL,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,IAAI,iBAAiB,GAAG,QAAQ,CAAC;IACjC,MAAM,WAAW,GAAa,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QACzC,iBAAiB,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QAC7D,WAAW,CAAC,IAAI,CAAC,gBAAgB,qBAAqB,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,YAAY,GAAkB,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE1F,wEAAwE;IACxE,IAAI,SAAwD,CAAC;IAC7D,IAAI,CAAC;QACH,SAAS,GAAG,MAAM,iBAAiB,CAAC,iBAAiB,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE;YACjF,MAAM,EAAE,IAAI,CAAC,eAAe;YAC5B,MAAM,EAAE,IAAI,CAAC,eAAe;YAC5B,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC;QACtE,OAAO;YACL,UAAU,EAAE,MAAM,iBAAiB,CAAC,EAAE,EAAE;gBACtC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;gBACpC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,UAAU;gBACV,gBAAgB;gBAChB,QAAQ,EAAE,iBAAiB;gBAC3B,cAAc,EAAE,IAAI;gBACpB,gBAAgB,EAAE,YAAY;gBAC9B,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAClC,QAAQ;gBACR,cAAc,EAAE,IAAI;gBACpB,eAAe,EAAE,IAAI;gBACrB,YAAY,EAAE,CAAC;gBACf,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;aAC9C,CAAC;YACF,QAAQ,EAAE,KAAK;YACf,IAAI;SACL,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,oBAAoB,CACvC,SAAS,CAAC,SAAS,EACnB,SAAS,CAAC,UAAU,EACpB,SAAS,CAAC,KAAK,CAChB,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,EAAE,EAAE;QAC7C,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU;QACV,gBAAgB;QAChB,QAAQ,EAAE,iBAAiB;QAC3B,cAAc,EAAE,SAAS,CAAC,IAAsC;QAChE,gBAAgB,EAAE,IAAI;QACtB,YAAY;QACZ,QAAQ,EAAE,SAAS,CAAC,KAAK;QACzB,cAAc,EAAE,SAAS,CAAC,SAAS;QACnC,eAAe,EAAE,SAAS,CAAC,UAAU;QACrC,YAAY;QACZ,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;KAC9C,CAAC,CAAC;IAEH,wEAAwE;IACxE,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;YAClC,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,cAAc,EAAE,KAAK,CAAC,cAAc;YACpC,YAAY,EAAE,KAAK,CAAC,GAAG;YACvB,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,oBAAoB;YACxF,WAAW,EAAE,SAAS,CAAC,KAAK;YAC5B,YAAY;YACZ,kFAAkF;YAClF,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,SAAS,GAAG,SAAS,CAAC,UAAU,CAAC;SAC1D,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AAC/C,CAAC;AAsBD,KAAK,UAAU,iBAAiB,CAAC,EAAY,EAAE,IAAiB;IAC9D,MAAM,GAAG,GAAkB;QACzB,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;QAC3B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;QACvC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;KAC5B,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC;IACtE,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACpE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,EAAY,EACZ,WAAmB,EACnB,UAAyB,EACzB,gBAAwB;IAExB,mEAAmE;IACnE,mEAAmE;IACnE,yBAAyB;IACzB,MAAM,IAAI,GAAG,MAAM,EAAE;SAClB,MAAM,EAAE;SACR,IAAI,CAAC,WAAW,CAAC;SACjB,KAAK,CACJ,UAAU,KAAK,IAAI;QACjB,CAAC,CAAC,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,EACxC,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CACnD;QACH,CAAC,CAAC,GAAG,CACD,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,WAAW,CAAC,EACxC,EAAE,CAAC,WAAW,CAAC,UAAU,EAAE,UAAU,CAAC,EACtC,EAAE,CAAC,WAAW,CAAC,gBAAgB,EAAE,gBAAgB,CAAC,CACnD,CACN;SACA,KAAK,CAAC,CAAC,CAAC,CAAC;IACZ,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;AACzB,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAgB,EAAE,SAAiB,EAAE,KAAa;IAC9E,mEAAmE;IACnE,wEAAwE;IACxE,kEAAkE;IAClE,uCAAuC;IACvC,MAAM,WAAW,GAAG,QAAQ,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,YAAY,CAAC,CAAC;AAChD,CAAC;AAED,OAAO,EAIL,mBAAmB,EACnB,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,uBAAuB,EACvB,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,aAAa,CAAC;AACrB,OAAO,EAA4B,YAAY,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACxF,OAAO,EACL,WAAW,EACX,eAAe,EAGf,SAAS,EACT,mBAAmB,GACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAA4B,YAAY,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAEL,cAAc,EACd,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,UAAU,CAAC;AAClB,OAAO,EAIL,YAAY,EACZ,yBAAyB,EACzB,eAAe,GAChB,MAAM,cAAc,CAAC"}

package/dist/js-shell-detect.d.ts

@@ -0,0 +1,21 @@
+/**
+ * JS-shell detection (Review #6).
+ *
+ * Many modern sites (React/Vue/Angular SPAs) serve a near-empty HTML
+ * shell that's filled in by JS at runtime. Our Tier 1 fetcher
+ * (Patchright Chromium) executes JS so usually returns the rendered
+ * DOM, but Tier 0 (curl-impersonate) does not. When we end up with a
+ * shell-only response, running the LLM extractor on it is wasted spend
+ * — the extractor will hallucinate or return empty.
+ *
+ * isJsShell() returns true when the HTML looks like an unrendered
+ * shell. Used as a pre-flight by the extractor to skip the Sonnet call
+ * and return extraction_status='js_shell_detected'.
+ *
+ * Heuristics (any one tripped → shell):
+ *   - <body> text content < 200 chars after stripping <script> + <style>
+ *   - Contains a known "loading shell" marker (e.g. "Loading...",
+ *     "JavaScript is required", `<div id="root"></div>` with empty body)
+ *   - <noscript> tag says "you need javascript" but no other content
+ */
+export declare function isJsShell(html: string): boolean;

package/dist/js-shell-detect.js

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * JS-shell detection (Review #6).
+ *
+ * Many modern sites (React/Vue/Angular SPAs) serve a near-empty HTML
+ * shell that's filled in by JS at runtime. Our Tier 1 fetcher
+ * (Patchright Chromium) executes JS so usually returns the rendered
+ * DOM, but Tier 0 (curl-impersonate) does not. When we end up with a
+ * shell-only response, running the LLM extractor on it is wasted spend
+ * — the extractor will hallucinate or return empty.
+ *
+ * isJsShell() returns true when the HTML looks like an unrendered
+ * shell. Used as a pre-flight by the extractor to skip the Sonnet call
+ * and return extraction_status='js_shell_detected'.
+ *
+ * Heuristics (any one tripped → shell):
+ *   - <body> text content < 200 chars after stripping <script> + <style>
+ *   - Contains a known "loading shell" marker (e.g. "Loading...",
+ *     "JavaScript is required", `<div id="root"></div>` with empty body)
+ *   - <noscript> tag says "you need javascript" but no other content
+ */
+import { JSDOM } from 'jsdom';
+import { MAX_EXTRACT_HTML_CHARS, MAX_SAFE_NESTING_DEPTH, estimateMaxNestingDepth, looksLikeBinary, stripScriptStyleBlocks, stripTagsLinear, } from './guards.js';
+const SHELL_TEXT_MIN_LENGTH = 200;
+const SHELL_MARKERS = [
+    /\bplease enable javascript\b/i,
+    /\byou need(?:\s+to enable)?\s+javascript\b/i,
+    /\bjavascript is required\b/i,
+    /\bthis app requires javascript\b/i,
+];
+const ROOT_DIV_IDS = ['root', '__next', 'app', '__nuxt', 'svelte', 'vue-app'];
+export function isJsShell(html) {
+    if (typeof html !== 'string' || html.length === 0)
+        return true;
+    // Pre-flight guards (perf-extract). isJsShell runs BEFORE extractMarkdown
+    // in the extract() pipeline, so without its own guards it was the actual
+    // process-killer on pathological input: profiling showed a full jsdom
+    // parse + body cloneNode OOM-aborting alone on a 50 MB page and throwing
+    // an uncaught RangeError at ~6000-deep nesting.
+    if (html.length > MAX_EXTRACT_HTML_CHARS) {
+        // Too big for a DOM, but oversized SHELLS are real (SPAs with a
+        // megabyte-scale inlined JS bundle and an empty root div). Estimate the
+        // visible text with the linear string helpers instead: strip script/
+        // style bodies, strip tags, collapse whitespace. Mirrors the DOM
+        // heuristics below minus the root-div check.
+        const visible = stripTagsLinear(stripScriptStyleBlocks(html)).replace(/\s+/g, ' ').trim();
+        for (const marker of SHELL_MARKERS) {
+            if (marker.test(visible) && visible.length < 500)
+                return true;
+        }
+        return visible.length < 30;
+    }
+    // Binary/deep pages are not EMPTY shells — return false without building
+    // a DOM and let extractMarkdown degrade them under its own guards.
+    if (looksLikeBinary(html))
+        return false; // extractMarkdown classifies it as binary_input
+    if (estimateMaxNestingDepth(html) > MAX_SAFE_NESTING_DEPTH)
+        return false;
+    let dom;
+    try {
+        dom = new JSDOM(html);
+    }
+    catch {
+        // Malformed HTML — treat as shell so we don't burn Sonnet tokens on it.
+        return true;
+    }
+    try {
+        const doc = dom.window.document;
+        const body = doc.body;
+        if (!body)
+            return true;
+        // Strip <script> + <style> from a clone so their content isn't counted
+        // as "real" body text.
+        const clone = body.cloneNode(true);
+        for (const sel of ['script', 'style', 'noscript']) {
+            for (const el of clone.querySelectorAll(sel))
+                el.remove();
+        }
+        const bodyText = (clone.textContent ?? '').trim();
+        // Check for known "loading" or "enable JS" markers in the full body
+        // text (incl. noscript content), then check the stripped body length.
+        const fullText = (body.textContent ?? '').trim();
+        for (const marker of SHELL_MARKERS) {
+            if (marker.test(fullText) && bodyText.length < 500)
+                return true;
+        }
+        // Single root-div pattern: <body><div id="root"></div></body>
+        // — only flag when both the body AND an SPA-style root container are
+        // empty. This catches React/Vue/Angular shells without false-positiving
+        // on legitimately short pages (e.g. example.com, status pages).
+        if (bodyText.length < SHELL_TEXT_MIN_LENGTH) {
+            for (const id of ROOT_DIV_IDS) {
+                const rootEl = doc.getElementById(id);
+                if (rootEl && (rootEl.textContent ?? '').trim().length < SHELL_TEXT_MIN_LENGTH) {
+                    return true;
+                }
+            }
+        }
+        // Empty body with no real text content AND nothing in <head> → shell.
+        // Use a stricter threshold so a single-sentence page like example.com
+        // isn't flagged. Real shells have ZERO meaningful text in <body>.
+        if (bodyText.length < 30)
+            return true;
+        return false;
+    }
+    catch {
+        // DOM traversal blew up on a hostile document (e.g. stack overflow in
+        // cloneNode) — NOT evidence of a shell. Say "has content" and let the
+        // guarded extractor deal with it; isJsShell must never throw.
+        return false;
+    }
+}
+//# sourceMappingURL=js-shell-detect.js.map

package/dist/js-shell-detect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"js-shell-detect.js","sourceRoot":"","sources":["../src/js-shell-detect.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,eAAe,EACf,sBAAsB,EACtB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,aAAa,GAAG;IACpB,+BAA+B;IAC/B,6CAA6C;IAC7C,6BAA6B;IAC7B,mCAAmC;CACpC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AAE9E,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE/D,0EAA0E;IAC1E,yEAAyE;IACzE,sEAAsE;IACtE,yEAAyE;IACzE,gDAAgD;IAChD,IAAI,IAAI,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QACzC,gEAAgE;QAChE,wEAAwE;QACxE,qEAAqE;QACrE,iEAAiE;QACjE,6CAA6C;QAC7C,MAAM,OAAO,GAAG,eAAe,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1F,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;gBAAE,OAAO,IAAI,CAAC;QAChE,CAAC;QACD,OAAO,OAAO,CAAC,MAAM,GAAG,EAAE,CAAC;IAC7B,CAAC;IACD,yEAAyE;IACzE,mEAAmE;IACnE,IAAI,eAAe,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,CAAC,gDAAgD;IACzF,IAAI,uBAAuB,CAAC,IAAI,CAAC,GAAG,sBAAsB;QAAE,OAAO,KAAK,CAAC;IAEzE,IAAI,GAAU,CAAC;IACf,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAChC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,uEAAuE;QACvE,uBAAuB;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAgB,CAAC;QAClD,KAAK,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC;YAClD,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC;gBAAE,EAAE,CAAC,MAAM,EAAE,CAAC;QAC5D,CAAC;QACD,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAElD,oEAAoE;QACpE,sEAAsE;QACtE,MAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;gBAAE,OAAO,IAAI,CAAC;QAClE,CAAC;QAED,8DAA8D;QAC9D,qEAAqE;QACrE,wEAAwE;QACxE,gEAAgE;QAChE,IAAI,QAAQ,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YAC5C,KAAK,MAAM,EAAE,IAAI,YAAY,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;gBACtC,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;oBAC/E,OAAO,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,sEAAsE;QACtE,kEAAkE;QAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QAEtC,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,sEAAsE;QACtE,sEAAsE;QACtE,8DAA8D;QAC9D,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}

package/dist/links.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Link extraction (Phase A.2).
+ *
+ * Returns the deduplicated set of absolute URLs reachable via `<a href>`
+ * from a page's HTML. Relative URLs are resolved against `baseUrl`.
+ *
+ * Used by `/scrape` when `formats` includes `'links'` and by `/map` (Phase B).
+ */
+export interface ExtractLinksOptions {
+    /** Skip same-host links. Useful for off-site link discovery. */
+    excludeSameHost?: boolean;
+    /** Include `<link rel="..." href="...">` from <head> alongside <a>. */
+    includeMetaLinks?: boolean;
+    /** Maximum number of links to return. Defaults to no cap. */
+    limit?: number;
+    /**
+     * Strip common tracking query parameters (utm_*, gclid, fbclid, etc.)
+     * before deduping. Default: true. Catches the common case where the
+     * same page appears N times with different campaign tags.
+     */
+    stripTrackingParams?: boolean;
+    /**
+     * Drop links to known analytics / ad / share-button domains
+     * (doubleclick.net, fb.com share endpoints, twitter intents, etc.).
+     * Default: true. Saves noise in `/scrape` links output + speeds
+     * `/crawl` BFS by skipping these dead-end domains.
+     */
+    dropTrackingDomains?: boolean;
+}
+/**
+ * Returns deduplicated, absolute URLs. `mailto:`, `tel:`, `javascript:`,
+ * and `data:` URIs are dropped; only `http(s):` survives.
+ */
+export declare function extractLinks(html: string, baseUrl: string, options?: ExtractLinksOptions): string[];

package/dist/links.js

@@ -0,0 +1,154 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Link extraction (Phase A.2).
+ *
+ * Returns the deduplicated set of absolute URLs reachable via `<a href>`
+ * from a page's HTML. Relative URLs are resolved against `baseUrl`.
+ *
+ * Used by `/scrape` when `formats` includes `'links'` and by `/map` (Phase B).
+ */
+import { JSDOM } from 'jsdom';
+/** Tracking params stripped before dedupe. Catches Google Analytics, Facebook, Hubspot, generic. */
+const TRACKING_PARAM_PREFIXES = ['utm_', 'mc_', '_hs', 'hsa_', 'mtm_', 'pk_'];
+const TRACKING_PARAM_EXACT = new Set([
+    'gclid',
+    'fbclid',
+    'msclkid',
+    'dclid',
+    'yclid',
+    'igshid',
+    'twclid',
+    'ttclid',
+    'wbraid',
+    'gbraid',
+    'oly_anon_id',
+    'oly_enc_id',
+    '_ga',
+    '__cf_chl_tk',
+    'ref',
+    'source',
+]);
+/** Known noise domains — share buttons, analytics beacons, ad networks. */
+const NOISE_DOMAINS = new Set([
+    'doubleclick.net',
+    'googleadservices.com',
+    'google-analytics.com',
+    'googletagmanager.com',
+    'facebook.com/sharer',
+    'twitter.com/intent',
+    'x.com/intent',
+    't.co',
+    'linkedin.com/sharing',
+    'pinterest.com/pin/create',
+    'reddit.com/submit',
+    'plus.google.com/share',
+]);
+function isNoiseDomain(u) {
+    // Strip a leading `www.` so doubleclick.net catches www.doubleclick.net etc.
+    const host = u.host.toLowerCase().replace(/^www\./, '');
+    if (NOISE_DOMAINS.has(host))
+        return true;
+    // host/path match for "x.com/intent" etc.
+    const hostAndPath = `${host}${u.pathname}`;
+    for (const noise of NOISE_DOMAINS) {
+        if (noise.includes('/') && hostAndPath.startsWith(noise))
+            return true;
+    }
+    return false;
+}
+function stripTrackingParams(u) {
+    const toRemove = [];
+    for (const key of u.searchParams.keys()) {
+        if (TRACKING_PARAM_EXACT.has(key)) {
+            toRemove.push(key);
+            continue;
+        }
+        for (const prefix of TRACKING_PARAM_PREFIXES) {
+            if (key.startsWith(prefix)) {
+                toRemove.push(key);
+                break;
+            }
+        }
+    }
+    for (const key of toRemove)
+        u.searchParams.delete(key);
+}
+/**
+ * Returns deduplicated, absolute URLs. `mailto:`, `tel:`, `javascript:`,
+ * and `data:` URIs are dropped; only `http(s):` survives.
+ */
+export function extractLinks(html, baseUrl, options = {}) {
+    // JSDOM throws on invalid base URLs. Callers upstream already
+    // canonicalize the URL, but be defensive — return [] rather than
+    // propagate the error.
+    let dom;
+    try {
+        dom = new JSDOM(html, { url: baseUrl });
+    }
+    catch {
+        try {
+            dom = new JSDOM(html);
+        }
+        catch {
+            return [];
+        }
+    }
+    const doc = dom.window.document;
+    const baseHost = (() => {
+        try {
+            return new URL(baseUrl).host.toLowerCase();
+        }
+        catch {
+            return '';
+        }
+    })();
+    const seen = new Set();
+    const out = [];
+    const candidates = [];
+    for (const a of Array.from(doc.querySelectorAll('a[href]'))) {
+        const href = a.href;
+        if (href)
+            candidates.push(href);
+    }
+    if (options.includeMetaLinks) {
+        for (const l of Array.from(doc.querySelectorAll('link[href]'))) {
+            const href = l.href;
+            if (href)
+                candidates.push(href);
+        }
+    }
+    // Defaults: noise + tracking filters ON. Customers can opt out
+    // explicitly if they want raw output.
+    const wantStripParams = options.stripTrackingParams !== false;
+    const wantDropNoise = options.dropTrackingDomains !== false;
+    for (const raw of candidates) {
+        let abs;
+        try {
+            abs = new URL(raw);
+        }
+        catch {
+            continue;
+        }
+        if (abs.protocol !== 'http:' && abs.protocol !== 'https:')
+            continue;
+        if (options.excludeSameHost && abs.host.toLowerCase() === baseHost)
+            continue;
+        if (wantDropNoise && isNoiseDomain(abs))
+            continue;
+        // Drop the fragment when deduping — same page, different anchor = same link
+        abs.hash = '';
+        // Strip campaign/tracking params before dedup so utm_source variants
+        // collapse to one entry.
+        if (wantStripParams)
+            stripTrackingParams(abs);
+        const final = abs.toString();
+        if (seen.has(final))
+            continue;
+        seen.add(final);
+        out.push(final);
+        if (options.limit && out.length >= options.limit)
+            break;
+    }
+    return out;
+}
+//# sourceMappingURL=links.js.map

package/dist/links.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.js","sourceRoot":"","sources":["../src/links.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;GAOG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAwB9B,oGAAoG;AACpG,MAAM,uBAAuB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAU,CAAC;AACvF,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,OAAO;IACP,QAAQ;IACR,SAAS;IACT,OAAO;IACP,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,YAAY;IACZ,KAAK;IACL,aAAa;IACb,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,2EAA2E;AAC3E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC5B,iBAAiB;IACjB,sBAAsB;IACtB,sBAAsB;IACtB,sBAAsB;IACtB,qBAAqB;IACrB,oBAAoB;IACpB,cAAc;IACd,MAAM;IACN,sBAAsB;IACtB,0BAA0B;IAC1B,mBAAmB;IACnB,uBAAuB;CACxB,CAAC,CAAC;AAEH,SAAS,aAAa,CAAC,CAAM;IAC3B,6EAA6E;IAC7E,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACxD,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACzC,0CAA0C;IAC1C,MAAM,WAAW,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IACxE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAM;IACjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,uBAAuB,EAAE,CAAC;YAC7C,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,QAAQ;QAAE,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,OAAe,EACf,UAA+B,EAAE;IAEjC,8DAA8D;IAC9D,iEAAiE;IACjE,uBAAuB;IACvB,IAAI,GAAU,CAAC;IACf,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAEhC,MAAM,QAAQ,GAAG,CAAC,GAAG,EAAE;QACrB,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,GAAI,CAAuB,CAAC,IAAI,CAAC;QAC3C,IAAI,IAAI;YAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,GAAI,CAAqB,CAAC,IAAI,CAAC;YACzC,IAAI,IAAI;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,sCAAsC;IACtC,MAAM,eAAe,GAAG,OAAO,CAAC,mBAAmB,KAAK,KAAK,CAAC;IAC9D,MAAM,aAAa,GAAG,OAAO,CAAC,mBAAmB,KAAK,KAAK,CAAC;IAE5D,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;YAAE,SAAS;QACpE,IAAI,OAAO,CAAC,eAAe,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,QAAQ;YAAE,SAAS;QAC7E,IAAI,aAAa,IAAI,aAAa,CAAC,GAAG,CAAC;YAAE,SAAS;QAClD,4EAA4E;QAC5E,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACd,qEAAqE;QACrE,yBAAyB;QACzB,IAAI,eAAe;YAAE,mBAAmB,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAC9B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChB,IAAI,OAAO,CAAC,KAAK,IAAI,GAAG,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK;YAAE,MAAM;IAC1D,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}