@bluefields/extractor 0.2.0

package/dist/readability.js

@@ -0,0 +1,458 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+/**
+ * Mozilla Readability + Turndown pipeline (Review #6).
+ *
+ * Pulls the "main content" from an HTML page (sidebar/nav/ads stripped),
+ * then converts to GitHub-Flavored Markdown for downstream consumption.
+ *
+ * Readability is the same library Firefox Reader View uses — battle-tested
+ * across millions of articles. It's not perfect (especially on
+ * non-article pages like pricing tables or app dashboards), but it's the
+ * established baseline. Site-specific overrides land in host_extractors
+ * (chunk M14-17).
+ *
+ * Robustness contract (perf-extract): `extractMarkdown` NEVER throws and
+ * NEVER builds a DOM for input that could OOM or stack-overflow the
+ * process. Pathological inputs (oversized bodies, binary blobs, 2500+-deep
+ * nesting) degrade to a bounded tag-strip and report WHY via the
+ * `degraded` field — see guards.ts for the measured ceilings.
+ */
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import TurndownService from 'turndown';
+import { MAX_CAPTURED_TABLES, MAX_CAPTURE_TABLE_ROWS, MAX_DEGRADED_TEXT_CHARS, MAX_EXTRACT_HTML_CHARS, MAX_RECOVERY_CONTENT_CHARS, MAX_SAFE_NESTING_DEPTH, MAX_SINGLE_TABLE_CHARS, MAX_TABLE_CELLS, MAX_TABLE_COLS, MAX_TOTAL_CAPTURED_CHARS, estimateMaxNestingDepth, looksLikeBinary, stripScriptStyleBlocks, stripTagsLinear, } from './guards.js';
+// v3 — adds input ceilings + typed degrade signaling + table-recovery caps
+// (perf-extract). Output changes ONLY for pathological pages (oversized,
+// binary, table-recovery beyond caps); bumping invalidates the extractions
+// cache so any such page cached under v2 gets re-extracted under the new
+// guards instead of serving a mixed-format row.
+export const READABILITY_VERSION = 'readability-v3';
+/**
+ * Extract main-content markdown from an HTML string.
+ *
+ * `url` is the source URL — used by Readability to resolve relative
+ * links + by Turndown for absolute-URL preservation. Pass the
+ * canonicalized URL (so the resolved hrefs in the markdown are stable
+ * across re-fetches).
+ *
+ * Never throws: binary input, oversized bodies (> MAX_EXTRACT_HTML_CHARS),
+ * pathologically deep markup, and any parse failure degrade to a bounded
+ * iterative plain-text strip instead of propagating — with the cause
+ * reported in `degraded`. The contract is "garbage in, best-effort text
+ * out" — callers can treat the result as always-present.
+ */
+export function extractMarkdown(html, url, opts = {}) {
+    // Table recovery is a win on data/reference pages but a liability on news
+    // articles (it re-adds non-content tables as false positives). Page-type
+    // routing toggles this; default on.
+    const recoverTables = opts.recoverTables ?? true;
+    // Binary sniff first — cheapest guard, and tag-stripping a PNG would
+    // emit multi-MB garbage "text" downstream. Typed empty result instead.
+    if (looksLikeBinary(html)) {
+        return emptyResult({
+            reasons: ['binary_input'],
+            detail: 'input looks like a binary format, not HTML/text',
+        });
+    }
+    // Size ceiling BEFORE any DOM work. Measured: jsdom peaks at ~270 MB RSS
+    // per input MB (2x for tables) — see guards.ts. Oversized bodies get a
+    // bounded tag-strip of the prefix we would otherwise have parsed.
+    if (html.length > MAX_EXTRACT_HTML_CHARS) {
+        return degradedTextResult(html.slice(0, MAX_EXTRACT_HTML_CHARS), {
+            reasons: ['input_too_large'],
+            detail: `input ${html.length} chars > ${MAX_EXTRACT_HTML_CHARS}`,
+        });
+    }
+    // Pre-flight depth guard (F-2). jsdom's serializer and Readability's DOM
+    // walks recurse with the markup depth, so ~5000-deep nesting overflows the
+    // stack and a ~4000-deep page already costs ~20s just to parse. Detect that
+    // on the raw string and skip the DOM entirely — degrade to a tag strip.
+    if (estimateMaxNestingDepth(html) > MAX_SAFE_NESTING_DEPTH) {
+        return degradedTextResult(html, {
+            reasons: ['nesting_too_deep'],
+            detail: `estimated nesting depth > ${MAX_SAFE_NESTING_DEPTH}`,
+        });
+    }
+    try {
+        return extractViaReadability(html, url, recoverTables);
+    }
+    catch {
+        // Belt-and-suspenders: any unforeseen deep-recursion / parser failure
+        // degrades to text rather than throwing.
+        return degradedTextResult(html, { reasons: ['parse_failed'] });
+    }
+}
+function extractViaReadability(html, url, recoverTables) {
+    const dom = new JSDOM(html, { url });
+    const doc = dom.window.document;
+    // Strip UI noise (e.g. MediaWiki "[edit]" section links) before Readability:
+    // they are link-dense enough that Readability drops the whole heading wrapper,
+    // collapsing distinct sections together and losing their headings.
+    stripPreReadabilityNoise(doc);
+    // Capture significant data tables BEFORE Readability runs: it mutates the
+    // document and frequently strips large/link-dense tables it scores as
+    // clutter, so on reference/data pages the main table would otherwise be lost
+    // entirely (Turndown's table rule never receives it).
+    const capture = recoverTables
+        ? collectSignificantTables(doc)
+        : { tables: [], capped: false, cappedDetail: undefined };
+    // The page's own <h1> is usually the clean title; Readability's article.title
+    // is often derived from <title> and carries a " — Site" suffix. Capture it now
+    // (before Readability mutates the DOM) so we can strip that suffix.
+    const pageH1 = doc.querySelector('h1')?.textContent ?? null;
+    // keepClasses preserves `language-*` on <code> so Turndown's preCode rule can
+    // emit a fenced-code language hint (F-1); Readability strips all classes
+    // otherwise. Classes never reach the markdown output, so this is invisible
+    // everywhere except that one rule.
+    const article = new Readability(doc, { keepClasses: true }).parse();
+    if (!article) {
+        // Readability gave up — fall back to body text.
+        const bodyText = dom.window.document.body?.textContent?.trim() ?? '';
+        return {
+            title: null,
+            markdown: bodyText,
+            text: bodyText,
+            excerpt: null,
+            byline: null,
+            siteName: null,
+            length: bodyText.length,
+        };
+    }
+    const turndown = makeTurndown();
+    // Re-attach significant tables Readability dropped so the table rule can
+    // convert them — otherwise the data (often the page's whole point) is gone.
+    const recovery = recoverTables
+        ? recoverDroppedTables(article.content, capture.tables, url)
+        : { html: '', capped: false };
+    const contentHtml = recovery.html ? `${article.content}\n${recovery.html}` : article.content;
+    const body = turndown.turndown(contentHtml);
+    // Prefer the page <h1> over a suffixed article.title (see cleanTitle).
+    const title = cleanTitle(article.title ?? '', pageH1);
+    // Readability extracts the title separately and strips the source <h1> from
+    // article.content, so the converted body has no title heading. Restore it
+    // (this otherwise drops the lead heading and depresses content recall).
+    const markdown = prependTitleHeading(body, title);
+    // A tripped capture/recovery cap means table recovery was not fully
+    // attempted — signal it, the rest of the extraction is still first-class.
+    const capped = capture.capped || recovery.capped;
+    return {
+        title: title || null,
+        markdown,
+        text: article.textContent ?? null,
+        excerpt: article.excerpt ?? null,
+        byline: article.byline ?? null,
+        siteName: article.siteName ?? null,
+        length: markdown.length,
+        ...(capped
+            ? {
+                degraded: {
+                    reasons: ['table_recovery_capped'],
+                    detail: capture.cappedDetail ?? 'article content too large for recovery re-parse',
+                },
+            }
+            : {}),
+    };
+}
+/** Empty typed result for input we refuse to process at all (binary). */
+function emptyResult(degraded) {
+    return {
+        title: null,
+        markdown: '',
+        text: '',
+        excerpt: null,
+        byline: null,
+        siteName: null,
+        length: 0,
+        degraded,
+    };
+}
+/**
+ * Iterative HTML→text fallback for input we won't risk in the DOM. Drops
+ * script/style bodies (indexOf scan — the old lazy regex backtracked
+ * quadratically on unclosed-`<script` bombs), strips tags, and decodes the
+ * few entities common in visible text. Never recurses, so it can't overflow
+ * on pathological depth. Output is capped at MAX_DEGRADED_TEXT_CHARS.
+ */
+function degradedTextResult(html, degraded) {
+    let text = stripTagsLinear(stripScriptStyleBlocks(html))
+        .replace(/&nbsp;/gi, ' ')
+        .replace(/&amp;/gi, '&')
+        .replace(/&lt;/gi, '<')
+        .replace(/&gt;/gi, '>')
+        .replace(/&#39;|&apos;/gi, "'")
+        .replace(/&quot;/gi, '"')
+        .replace(/\s+/g, ' ')
+        .trim();
+    const reasons = [...degraded.reasons];
+    if (text.length > MAX_DEGRADED_TEXT_CHARS) {
+        text = text.slice(0, MAX_DEGRADED_TEXT_CHARS);
+        reasons.push('output_truncated');
+    }
+    return {
+        title: null,
+        markdown: text,
+        text,
+        excerpt: null,
+        byline: null,
+        siteName: null,
+        length: text.length,
+        degraded: { reasons, detail: degraded.detail },
+    };
+}
+/**
+ * Prefer the page's own `<h1>` text over Readability's `article.title` when the
+ * h1 is a (shorter) case-insensitive prefix of it — Readability often derives
+ * the title from the `<title>` tag, which carries a " — SiteName" suffix the
+ * page `<h1>` does not. The prefix check guards against a wrong/banner `<h1>`.
+ */
+function cleanTitle(rawTitle, pageH1) {
+    const title = rawTitle.trim();
+    const h1 = (pageH1 ?? '').replace(/\s+/g, ' ').trim();
+    if (h1 && h1.length < title.length && title.toLowerCase().startsWith(h1.toLowerCase())) {
+        return h1;
+    }
+    return title;
+}
+/**
+ * Prepend the page title as a top-level (`#`) heading. Readability returns the
+ * title separately and drops the source `<h1>` from the content body, which
+ * leaves the markdown with no title and lowers content recall. Skips
+ * prepending when the body already opens with that title (some pages keep
+ * their `<h1>` in content), to avoid a duplicate heading.
+ */
+function prependTitleHeading(body, rawTitle) {
+    const title = rawTitle.trim();
+    if (!title)
+        return body;
+    const firstLine = body.replace(/^\s+/, '').split('\n', 1)[0] ?? '';
+    const normalize = (s) => s
+        .replace(/^#+\s*/, '')
+        .trim()
+        .toLowerCase();
+    if (normalize(firstLine) === title.toLowerCase())
+        return body;
+    return `# ${title}\n\n${body}`;
+}
+/**
+ * Remove UI noise that confuses Readability's content cleaning, run before it.
+ * MediaWiki `[edit]` section links (`.mw-editsection`) are link-dense enough
+ * that Readability drops the entire heading wrapper, collapsing sections and
+ * losing their headings; removing them recovers the section headings AND strips
+ * the `[edit]` clutter from the output. No-op on non-MediaWiki pages.
+ */
+function stripPreReadabilityNoise(doc) {
+    for (const el of Array.from(doc.querySelectorAll('.mw-editsection, .mw-editsection-bracket'))) {
+        el.remove();
+    }
+}
+// Recover a DROPPED table only when it is a SUBSTANTIAL data table. Small
+// dropped tables on article pages are almost always chrome ("related",
+// "most read") — recovering them adds noise and hurts article extraction;
+// large dropped tables on data/reference pages ARE the content. In effect this
+// is page-type routing: article pages drop only small tables (nothing recovered
+// → lean Readability path), data/reference pages drop big ones (recovered).
+const RECOVER_MIN_ROWS = 12;
+const RECOVER_MIN_TEXT = 1200;
+/** Heading-row text — a stable identity for a table, used to dedupe recoveries. */
+function tableSignature(table) {
+    const firstRow = table.querySelector('tr');
+    return (firstRow?.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, 120);
+}
+/**
+ * A table worth recovering = a real data table, not a layout/nav/infobox box.
+ * Requires ≥ 3 rows and ≥ 2 columns, and excludes known chrome containers.
+ */
+function isSignificantDataTable(table) {
+    const cls = (table.getAttribute('class') ?? '').toLowerCase();
+    if (/\b(navbox|vertical-navbox|sidebar|infobox|metadata|nomobile|toccolours|noprint)\b/.test(cls)) {
+        return false;
+    }
+    if (table.closest('.navbox, .vertical-navbox, .sidebar, .infobox, nav, header, footer')) {
+        return false;
+    }
+    const rows = Array.from(table.querySelectorAll('tr'));
+    if (rows.length < 3)
+        return false;
+    const maxCols = rows.reduce((m, r) => Math.max(m, r.querySelectorAll('th, td').length), 0);
+    return maxCols >= 2;
+}
+/**
+ * Snapshot significant data tables (outerHTML + signature) from a document,
+ * under the guards.ts ceilings: giant tables are rejected by row count
+ * BEFORE their outerHTML is serialized (serialization itself is the
+ * allocation we're guarding), and the total snapshot budget is bounded so a
+ * many-tables page can't duplicate the whole document into strings.
+ */
+function collectSignificantTables(doc) {
+    const tables = [];
+    let totalChars = 0;
+    let capped = false;
+    let cappedDetail;
+    for (const t of Array.from(doc.querySelectorAll('table'))) {
+        if (!isSignificantDataTable(t))
+            continue;
+        if (tables.length >= MAX_CAPTURED_TABLES) {
+            capped = true;
+            cappedDetail = `more than ${MAX_CAPTURED_TABLES} significant tables`;
+            break;
+        }
+        const rows = t.querySelectorAll('tr').length;
+        if (rows > MAX_CAPTURE_TABLE_ROWS) {
+            capped = true;
+            cappedDetail = `table with ${rows} rows > ${MAX_CAPTURE_TABLE_ROWS}`;
+            continue;
+        }
+        const html = t.outerHTML;
+        if (html.length > MAX_SINGLE_TABLE_CHARS ||
+            totalChars + html.length > MAX_TOTAL_CAPTURED_CHARS) {
+            capped = true;
+            cappedDetail = 'table snapshot budget exceeded';
+            continue;
+        }
+        totalChars += html.length;
+        tables.push({
+            html,
+            signature: tableSignature(t),
+            rows,
+            textLen: (t.textContent ?? '').replace(/\s+/g, ' ').trim().length,
+        });
+    }
+    return { tables, capped, cappedDetail };
+}
+/**
+ * HTML of captured tables NOT already present in the Readability content
+ * (matched by heading-row signature, so a table Readability kept is never
+ * duplicated). Empty string when there is nothing to recover. Skips the
+ * whole re-parse (a SECOND jsdom of the article content — the expensive
+ * half of recovery) when the content is too large to risk.
+ */
+function recoverDroppedTables(contentHtml, captured, url) {
+    if (captured.length === 0)
+        return { html: '', capped: false };
+    if (contentHtml.length > MAX_RECOVERY_CONTENT_CHARS) {
+        return { html: '', capped: true };
+    }
+    const presentDoc = new JSDOM(contentHtml, { url }).window.document;
+    const present = new Set(collectSignificantTables(presentDoc).tables.map((t) => t.signature));
+    const html = captured
+        .filter((t) => !present.has(t.signature))
+        .filter((t) => t.rows >= RECOVER_MIN_ROWS || t.textLen >= RECOVER_MIN_TEXT)
+        .map((t) => t.html)
+        .join('\n');
+    return { html, capped: false };
+}
+/**
+ * Canonicalize a raw highlighter token into a Markdown fence language. Drops
+ * non-languages ("default"/"text"/"plaintext"/"none") and normalizes a couple
+ * of aliases so a scraping customer gets a stable, conventional tag.
+ */
+function normalizeFenceLang(raw) {
+    const l = (raw ?? '').trim().toLowerCase();
+    if (!l || l === 'default' || l === 'text' || l === 'plaintext' || l === 'none')
+        return '';
+    if (l === 'python3' || l === 'python2')
+        return 'python';
+    return l;
+}
+/**
+ * Derive a code-fence language from a <pre> node by checking, in order:
+ *   1. `language-xxx` / `lang-xxx` on the <code> (or <pre>) — highlight.js etc.
+ *   2. `highlight-source-xxx` on an ancestor — GitHub's blob wrapper.
+ *   3. `highlight-xxx` on an ancestor — Sphinx / Python-docs wrapper (the <pre>
+ *      has no language class of its own; the lang lives on a grandparent div).
+ * Returns '' for a genuinely bare fence (no convention matched).
+ */
+function detectFenceLang(pre) {
+    const code = pre.querySelector('code');
+    for (const el of [code, pre]) {
+        const m = el?.className?.match(/(?:^|\s)(?:language|lang)-([a-zA-Z0-9+#.-]+)/);
+        if (m)
+            return normalizeFenceLang(m[1]);
+    }
+    // Walk a few ancestors for the wrapper conventions. GitHub before Sphinx,
+    // since `highlight-source-go` also matches the looser `highlight-xxx` pattern.
+    let node = pre;
+    for (let depth = 0; node && depth < 4; depth++) {
+        const cls = node.className ?? '';
+        const gh = cls.match(/(?:^|\s)highlight-source-([a-zA-Z0-9+#.-]+)/);
+        if (gh)
+            return normalizeFenceLang(gh[1]);
+        const sphinx = cls.match(/(?:^|\s)highlight-([a-zA-Z0-9+#.-]+)/);
+        if (sphinx)
+            return normalizeFenceLang(sphinx[1]);
+        node = node.parentElement;
+    }
+    return '';
+}
+function makeTurndown() {
+    const td = new TurndownService({
+        headingStyle: 'atx',
+        codeBlockStyle: 'fenced',
+        bulletListMarker: '-',
+        emDelimiter: '_',
+        strongDelimiter: '**',
+        linkStyle: 'inlined',
+    });
+    // Preserve <pre> code blocks with their language hint. The hint lives under
+    // several conventions across the web; detectFenceLang covers the common ones
+    // so a bare fence doesn't silently drop information a scraping customer asked
+    // for (syntax highlighting, language routing).
+    td.addRule('preCode', {
+        filter: ['pre'],
+        replacement: (_content, node) => {
+            const pre = node;
+            const codeEl = pre.querySelector('code');
+            const lang = detectFenceLang(pre);
+            const code = codeEl?.textContent ?? pre.textContent ?? '';
+            return `\n\n\`\`\`${lang}\n${code}\n\`\`\`\n\n`;
+        },
+    });
+    // Convert <table> to GitHub-flavored Markdown. Turndown has no table
+    // support by default, so a data table would otherwise collapse into a run
+    // of concatenated cell text and lose all structure.
+    td.addRule('gfmTable', {
+        filter: 'table',
+        replacement: (_content, node) => {
+            const rows = Array.from(node.querySelectorAll('tr'));
+            if (rows.length === 0)
+                return '';
+            const cellsOf = (tr) => Array.from(tr.querySelectorAll('th, td')).map((c) => (c.textContent ?? '')
+                .trim()
+                .replace(/\s*\n\s*/g, ' ')
+                .replace(/\|/g, '\\|'));
+            let grid = rows.map(cellsOf).filter((r) => r.length > 0);
+            if (grid.length === 0)
+                return '';
+            // reduce, not Math.max(...spread): the spread throws RangeError past
+            // ~65k rows, which would degrade the whole page instead of the table.
+            const rawCols = grid.reduce((m, r) => Math.max(m, r.length), 0);
+            // Geometry ceilings (perf-extract): padding multiplies the widest row
+            // across ALL rows, so a single ragged 100k-cell row would otherwise
+            // blow the output up to hundreds of MB. Far above any legit corpus
+            // table — output is byte-identical for sane pages.
+            const cols = Math.min(rawCols, MAX_TABLE_COLS);
+            let clipped = rawCols > MAX_TABLE_COLS;
+            const maxRows = Math.max(2, Math.floor(MAX_TABLE_CELLS / Math.max(1, cols)));
+            if (grid.length > maxRows) {
+                grid = grid.slice(0, maxRows);
+                clipped = true;
+            }
+            const pad = (r) => {
+                const out = r.slice(0, cols);
+                while (out.length < cols)
+                    out.push('');
+                return out;
+            };
+            const line = (r) => `| ${pad(r).join(' | ')} |`;
+            const header = line(grid[0]);
+            const sep = `| ${Array.from({ length: cols }, () => '---').join(' | ')} |`;
+            const body = grid.slice(1).map(line);
+            const note = clipped ? '\n\n_(table truncated: exceeds size limits)_' : '';
+            return `\n\n${[header, sep, ...body].join('\n')}${note}\n\n`;
+        },
+    });
+    // Drop noisy elements that survive Readability sometimes.
+    td.remove(['script', 'style', 'noscript', 'iframe']);
+    return td;
+}
+//# sourceMappingURL=readability.js.map

package/dist/readability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability.js","sourceRoot":"","sources":["../src/readability.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,OAAO,EAGL,mBAAmB,EACnB,sBAAsB,EACtB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,EAC1B,sBAAsB,EACtB,sBAAsB,EACtB,eAAe,EACf,cAAc,EACd,wBAAwB,EACxB,uBAAuB,EACvB,eAAe,EACf,sBAAsB,EACtB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB,2EAA2E;AAC3E,yEAAyE;AACzE,2EAA2E;AAC3E,yEAAyE;AACzE,gDAAgD;AAChD,MAAM,CAAC,MAAM,mBAAmB,GAAG,gBAAyB,CAAC;AA4B7D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,GAAW,EACX,OAAoC,EAAE;IAEtC,0EAA0E;IAC1E,yEAAyE;IACzE,oCAAoC;IACpC,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC;IAEjD,qEAAqE;IACrE,uEAAuE;IACvE,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,WAAW,CAAC;YACjB,OAAO,EAAE,CAAC,cAAc,CAAC;YACzB,MAAM,EAAE,iDAAiD;SAC1D,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,uEAAuE;IACvE,kEAAkE;IAClE,IAAI,IAAI,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QACzC,OAAO,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,sBAAsB,CAAC,EAAE;YAC/D,OAAO,EAAE,CAAC,iBAAiB,CAAC;YAC5B,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,YAAY,sBAAsB,EAAE;SACjE,CAAC,CAAC;IACL,CAAC;IAED,yEAAyE;IACzE,2EAA2E;IAC3E,4EAA4E;IAC5E,wEAAwE;IACxE,IAAI,uBAAuB,CAAC,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAC3D,OAAO,kBAAkB,CAAC,IAAI,EAAE;YAC9B,OAAO,EAAE,CAAC,kBAAkB,CAAC;YAC7B,MAAM,EAAE,6BAA6B,sBAAsB,EAAE;SAC9D,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC;QACH,OAAO,qBAAqB,CAAC,IAAI,EAAE,GAAG,EAAE,aAAa,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,sEAAsE;QACtE,yCAAyC;QACzC,OAAO,kBAAkB,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED,SAAS,qBAAqB,CAC5B,IAAY,EACZ,GAAW,EACX,aAAsB;IAEtB,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAChC,6EAA6E;IAC7E,+EAA+E;IAC/E,mEAAmE;IACnE,wBAAwB,CAAC,GAAG,CAAC,CAAC;IAC9B,0EAA0E;IAC1E,sEAAsE;IACtE,6EAA6E;IAC7E,sDAAsD;IACtD,MAAM,OAAO,GAAG,aAAa;QAC3B,CAAC,CAAC,wBAAwB,CAAC,GAAG,CAAC;QAC/B,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC;IAC3D,8EAA8E;IAC9E,+EAA+E;IAC/E,oEAAoE;IACpE,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC;IAC5D,8EAA8E;IAC9E,yEAAyE;IACzE,2EAA2E;IAC3E,mCAAmC;IACnC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACpE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,gDAAgD;QAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACrE,OAAO;YACL,KAAK,EAAE,IAAI;YACX,QAAQ,EAAE,QAAQ;YAClB,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,QAAQ,CAAC,MAAM;SACxB,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,yEAAyE;IACzE,4EAA4E;IAC5E,MAAM,QAAQ,GAAG,aAAa;QAC5B,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;QAC5D,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAChC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC;IAC7F,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC5C,uEAAuE;IACvE,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;IACtD,4EAA4E;IAC5E,0EAA0E;IAC1E,wEAAwE;IACxE,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAElD,oEAAoE;IACpE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC;IACjD,OAAO;QACL,KAAK,EAAE,KAAK,IAAI,IAAI;QACpB,QAAQ;QACR,IAAI,EAAE,OAAO,CAAC,WAAW,IAAI,IAAI;QACjC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI;QAChC,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;QAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,IAAI;QAClC,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,GAAG,CAAC,MAAM;YACR,CAAC,CAAC;gBACE,QAAQ,EAAE;oBACR,OAAO,EAAE,CAAC,uBAAwC,CAAC;oBACnD,MAAM,EAAE,OAAO,CAAC,YAAY,IAAI,iDAAiD;iBAClF;aACF;YACH,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,yEAAyE;AACzE,SAAS,WAAW,CAAC,QAA4B;IAC/C,OAAO;QACL,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,IAAI;QACd,MAAM,EAAE,CAAC;QACT,QAAQ;KACT,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,IAAY,EAAE,QAA4B;IACpE,IAAI,IAAI,GAAG,eAAe,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;SACrD,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;SAC9B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IACV,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,IAAI,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC1C,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACnC,CAAC;IACD,OAAO;QACL,KAAK,EAAE,IAAI;QACX,QAAQ,EAAE,IAAI;QACd,IAAI;QACJ,OAAO,EAAE,IAAI;QACb,MAAM,EAAE,IAAI;QACZ,QAAQ,EAAE,IAAI;QACd,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,QAAQ,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE;KAC/C,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,SAAS,UAAU,CAAC,QAAgB,EAAE,MAAqB;IACzD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,IAAI,EAAE,IAAI,EAAE,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;QACvF,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAS,mBAAmB,CAAC,IAAY,EAAE,QAAgB;IACzD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,MAAM,SAAS,GAAG,CAAC,CAAS,EAAU,EAAE,CACtC,CAAC;SACE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,IAAI,EAAE;SACN,WAAW,EAAE,CAAC;IACnB,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,KAAK,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAC;IAC9D,OAAO,KAAK,KAAK,OAAO,IAAI,EAAE,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,wBAAwB,CAAC,GAAa;IAC7C,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,0CAA0C,CAAC,CAAC,EAAE,CAAC;QAC9F,EAAE,CAAC,MAAM,EAAE,CAAC;IACd,CAAC;AACH,CAAC;AAgBD,0EAA0E;AAC1E,uEAAuE;AACvE,0EAA0E;AAC1E,+EAA+E;AAC/E,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAC5B,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,mFAAmF;AACnF,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,QAAQ,GAAG,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACjF,CAAC;AAED;;;GAGG;AACH,SAAS,sBAAsB,CAAC,KAAc;IAC5C,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9D,IACE,mFAAmF,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7F,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,oEAAoE,CAAC,EAAE,CAAC;QACxF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3F,OAAO,OAAO,IAAI,CAAC,CAAC;AACtB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,wBAAwB,CAAC,GAAa;IAC7C,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,YAAgC,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC;YAAE,SAAS;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,mBAAmB,EAAE,CAAC;YACzC,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,GAAG,aAAa,mBAAmB,qBAAqB,CAAC;YACrE,MAAM;QACR,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC7C,IAAI,IAAI,GAAG,sBAAsB,EAAE,CAAC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,GAAG,cAAc,IAAI,WAAW,sBAAsB,EAAE,CAAC;YACrE,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,CAAC,SAAS,CAAC;QACzB,IACE,IAAI,CAAC,MAAM,GAAG,sBAAsB;YACpC,UAAU,GAAG,IAAI,CAAC,MAAM,GAAG,wBAAwB,EACnD,CAAC;YACD,MAAM,GAAG,IAAI,CAAC;YACd,YAAY,GAAG,gCAAgC,CAAC;YAChD,SAAS;QACX,CAAC;QACD,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,SAAS,EAAE,cAAc,CAAC,CAAC,CAAC;YAC5B,IAAI;YACJ,OAAO,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM;SAClE,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AAC1C,CAAC;AAED;;;;;;GAMG;AACH,SAAS,oBAAoB,CAC3B,WAAmB,EACnB,QAAyB,EACzB,GAAW;IAEX,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC9D,IAAI,WAAW,CAAC,MAAM,GAAG,0BAA0B,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC;IACnE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,wBAAwB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7F,MAAM,IAAI,GAAG,QAAQ;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;SACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,gBAAgB,IAAI,CAAC,CAAC,OAAO,IAAI,gBAAgB,CAAC;SAC1E,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;SAClB,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAuB;IACjD,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,MAAM;QAAE,OAAO,EAAE,CAAC;IAC1F,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACxD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,eAAe,CAAC,GAAgB;IACvC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACvC,KAAK,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC/E,IAAI,CAAC;YAAE,OAAO,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,CAAC;IACD,0EAA0E;IAC1E,+EAA+E;IAC/E,IAAI,IAAI,GAAuB,GAAG,CAAC;IACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;QAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QACjC,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACpE,IAAI,EAAE;YAAE,OAAO,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACjE,IAAI,MAAM;YAAE,OAAO,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,GAAG,IAAI,eAAe,CAAC;QAC7B,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;QACrB,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IAEH,4EAA4E;IAC5E,6EAA6E;IAC7E,8EAA8E;IAC9E,+CAA+C;IAC/C,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE;QACpB,MAAM,EAAE,CAAC,KAAK,CAAC;QACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAG,IAAmB,CAAC;YAChC,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,EAAE,WAAW,IAAI,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC;YAC1D,OAAO,aAAa,IAAI,KAAK,IAAI,cAAc,CAAC;QAClD,CAAC;KACF,CAAC,CAAC;IAEH,qEAAqE;IACrE,0EAA0E;IAC1E,oDAAoD;IACpD,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE;QACrB,MAAM,EAAE,OAAO;QACf,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAE,IAAoB,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YACtE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAY,EAAE,CACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClD,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;iBAClB,IAAI,EAAE;iBACN,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;iBACzB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CACzB,CAAC;YACJ,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACzD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACjC,qEAAqE;YACrE,sEAAsE;YACtE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAChE,sEAAsE;YACtE,oEAAoE;YACpE,mEAAmE;YACnE,mDAAmD;YACnD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;YAC/C,IAAI,OAAO,GAAG,OAAO,GAAG,cAAc,CAAC;YACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;YAC7E,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,CAAC;gBAC1B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBAC9B,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,MAAM,GAAG,GAAG,CAAC,CAAW,EAAY,EAAE;gBACpC,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC7B,OAAO,GAAG,CAAC,MAAM,GAAG,IAAI;oBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACvC,OAAO,GAAG,CAAC;YACb,CAAC,CAAC;YACF,MAAM,IAAI,GAAG,CAAC,CAAW,EAAU,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAClE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAC3E,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,8CAA8C,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC;QAC/D,CAAC;KACF,CAAC,CAAC;IAEH,0DAA0D;IAC1D,EAAE,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;IAErD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/sitemap.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Sitemap discovery + parsing (Phase B).
+ *
+ * Sources for /map URL discovery:
+ *   1. `robots.txt` Sitemap: directive(s)
+ *   2. `/sitemap.xml` at the host root
+ *   3. Recursive: sitemapindex → sub-sitemaps → urls
+ *
+ * Sitemap XML is constrained enough that a regex parse beats a full
+ * XML parser dep. Both `<urlset>` and `<sitemapindex>` use the same
+ * `<loc>` element; we discriminate by the wrapping element to decide
+ * whether each `<loc>` is a URL or a nested sitemap.
+ */
+export interface ParsedSitemap {
+    /** Direct page URLs from <urlset>/<url>/<loc>. */
+    urls: string[];
+    /** Sub-sitemap URLs from <sitemapindex>/<sitemap>/<loc>. */
+    subsitemaps: string[];
+}
+/**
+ * Parse a sitemap XML blob. Returns separately the page URLs and the
+ * nested sub-sitemap URLs. A document can technically contain both
+ * (rare but allowed); we split based on whether the document has
+ * <sitemapindex> as a top-level element.
+ */
+export declare function parseSitemapXml(xml: string): ParsedSitemap;
+/**
+ * Pull `Sitemap:` directives out of a robots.txt body.
+ *
+ * Per https://datatracker.ietf.org/doc/html/rfc9309, Sitemap is a
+ * separate field with absolute-URL value. Case-insensitive.
+ */
+export declare function parseRobotsTxtForSitemaps(robotsTxt: string): string[];
+/**
+ * Discover URLs for a host via the standard cascade.
+ *
+ * - First fetches `/robots.txt` and extracts any Sitemap: directives.
+ * - If none, falls back to `/sitemap.xml` at the host root.
+ * - Recursively follows sitemapindex → sub-sitemaps (bounded by
+ *   `maxSitemapsToFollow` to avoid infinite loops / sitemap bombs).
+ *
+ * Pure — the caller (`/map` route) passes in a fetchImpl so SSRF + the
+ * shared dispatcher live at the boundary.
+ */
+export interface DiscoverUrlsOptions {
+    fetchImpl: typeof fetch;
+    /** Max distinct sitemap URLs to fetch (prevents bombs). Default 20. */
+    maxSitemapsToFollow?: number;
+    /** Max URLs to collect (stops early once hit). Default 10_000. */
+    limit?: number;
+    /** Include subdomains? Default false (same-host only). */
+    includeSubdomains?: boolean;
+    /** Substring filter on URLs. */
+    search?: string;
+    /** Per-fetch timeout ms. Default 10_000. */
+    timeoutMs?: number;
+    /** Optional override for the user agent. */
+    userAgent?: string;
+    /**
+     * SSRF pre-flight for each discovered sitemap URL. The /map route passes
+     * validateUrlForExternalRequest; child sitemaps come from a target's
+     * robots.txt `Sitemap:` directives / sitemapindex `<loc>` and are otherwise
+     * attacker-controlled. A rejected URL is skipped. (PENTEST_REPORT.md P5.)
+     */
+    validateUrl?: (url: string) => Promise<void>;
+}
+export interface DiscoverUrlsResult {
+    urls: string[];
+    /** Distinct sitemap URLs we actually fetched. */
+    sitemapsConsidered: string[];
+    /** True if we stopped because we hit the URL limit. */
+    truncated: boolean;
+}
+export declare function discoverUrls(startUrl: string, options: DiscoverUrlsOptions): Promise<DiscoverUrlsResult>;