@intentic/webq 1.317.0

package/dist/lib/markdown.js

@@ -0,0 +1,292 @@
+import { attr, isElement, isText, rawTextOf } from "./dom.js";
+const NON_CONTENT = new Set(["script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed", "link", "meta", "head"]);
+const INLINE = {
+    strong: { open: "**", close: "**" },
+    b: { open: "**", close: "**" },
+    em: { open: "*", close: "*" },
+    i: { open: "*", close: "*" },
+    del: { open: "~~", close: "~~" },
+    s: { open: "~~", close: "~~" },
+    strike: { open: "~~", close: "~~" },
+    kbd: { open: "`", close: "`" },
+};
+const HEADINGS = { h1: 1, h2: 2, h3: 3, h4: 4, h5: 5, h6: 6 };
+export const renderMarkdown = (root, options = {}) => {
+    const blocks = renderFlow(root.childNodes, options);
+    return `${blocks
+        .join("\n\n")
+        .replaceAll(/[ \t]+$/gm, "")
+        .replaceAll(/\n{3,}/g, "\n\n")
+        .trim()}\n`;
+};
+const renderFlow = (nodes, options) => {
+    const blocks = [];
+    let run = "";
+    const flush = () => {
+        const paragraph = collapse(run);
+        if (paragraph !== "") {
+            blocks.push(paragraph);
+        }
+        run = "";
+    };
+    for (const node of nodes) {
+        if (isText(node)) {
+            run += node.value;
+            continue;
+        }
+        if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+            continue;
+        }
+        const block = renderBlock(node, options);
+        if (block === undefined) {
+            run += renderInline(node, options);
+        }
+        else {
+            flush();
+            if (block !== "") {
+                blocks.push(block);
+            }
+        }
+    }
+    flush();
+    return blocks;
+};
+const renderBlock = (el, options) => {
+    const heading = HEADINGS[el.tagName];
+    if (heading !== undefined) {
+        const text = collapse(renderInlineChildren(el, options));
+        return text === "" ? "" : `${"#".repeat(heading)} ${text}`;
+    }
+    switch (el.tagName) {
+        case "p":
+            return collapse(renderInlineChildren(el, options));
+        case "ul":
+        case "ol":
+            return renderList(el, options, 0);
+        case "pre":
+            return renderPre(el);
+        case "blockquote":
+            return renderFlow(el.childNodes, options)
+                .join("\n\n")
+                .split("\n")
+                .map((line) => `> ${line}`.trimEnd())
+                .join("\n");
+        case "table":
+            return renderTable(el, options);
+        case "hr":
+            return "---";
+        case "figure": {
+            const caption = el.childNodes.filter(isElement).find((child) => child.tagName === "figcaption");
+            const body = renderFlow(el.childNodes.filter((child) => !(isElement(child) && child.tagName === "figcaption")), options);
+            const captionText = caption === undefined ? "" : collapse(renderInlineChildren(caption, options));
+            return [body.join("\n\n"), captionText === "" ? "" : `*${captionText}*`].filter(Boolean).join("\n\n");
+        }
+        case "dl":
+            return renderDefinitionList(el, options);
+        case "details": {
+            const summary = el.childNodes.filter(isElement).find((child) => child.tagName === "summary");
+            const rest = renderFlow(el.childNodes.filter((child) => !(isElement(child) && child.tagName === "summary")), options);
+            const label = summary === undefined ? "" : `**${collapse(renderInlineChildren(summary, options))}**`;
+            return [label, rest.join("\n\n")].filter(Boolean).join("\n\n");
+        }
+        case "div":
+        case "section":
+        case "article":
+        case "main":
+        case "body":
+        case "aside":
+        case "header":
+        case "footer":
+        case "nav":
+        case "form":
+        case "fieldset":
+        case "address":
+        case "center":
+            return renderFlow(el.childNodes, options).join("\n\n");
+        default:
+            return undefined;
+    }
+};
+const renderInlineChildren = (el, options) => el.childNodes.map((child) => (isText(child) ? child.value : isElement(child) ? renderInline(child, options) : "")).join("");
+const renderInline = (el, options) => {
+    if (NON_CONTENT.has(el.tagName)) {
+        return "";
+    }
+    if (el.tagName === "br") {
+        return BR;
+    }
+    if (el.tagName === "img") {
+        return renderImage(el, options);
+    }
+    if (el.tagName === "a") {
+        return renderLink(el, options);
+    }
+    if (el.tagName === "code") {
+        const code = rawTextOf(el).trim();
+        return code.includes("`") ? `\`\` ${code} \`\`` : `\`${code}\``;
+    }
+    const wrap = INLINE[el.tagName];
+    const inner = renderInlineChildren(el, options);
+    if (wrap === undefined) {
+        return inner;
+    }
+    const trimmed = collapse(inner);
+    return trimmed === "" ? "" : `${wrap.open}${trimmed}${wrap.close}`;
+};
+const renderLink = (el, options) => {
+    const text = collapse(renderInlineChildren(el, options));
+    const href = resolveUrl(attr(el, "href"), options.baseUrl);
+    if (href === undefined || href.startsWith("javascript:")) {
+        return text;
+    }
+    if (href.startsWith("mailto:") || href.startsWith("tel:")) {
+        return text === "" ? href.slice(href.indexOf(":") + 1) : text;
+    }
+    if (text === "") {
+        return "";
+    }
+    return `[${text}](${href})`;
+};
+const renderImage = (el, options) => {
+    const alt = attr(el, "alt")?.trim() ?? "";
+    const src = resolveUrl(attr(el, "src"), options.baseUrl);
+    if (src === undefined || src.startsWith("data:")) {
+        return alt;
+    }
+    return `![${alt}](${src})`;
+};
+const renderList = (el, options, depth) => {
+    const ordered = el.tagName === "ol";
+    const start = Number(attr(el, "start") ?? "1");
+    const items = el.childNodes.filter(isElement).filter((child) => child.tagName === "li");
+    const lines = [];
+    items.forEach((item, index) => {
+        const marker = ordered ? `${start + index}. ` : "- ";
+        const indent = "    ".repeat(depth);
+        const nested = [];
+        const inlineParts = [];
+        for (const child of item.childNodes) {
+            if (isElement(child) && (child.tagName === "ul" || child.tagName === "ol")) {
+                nested.push(renderList(child, options, depth + 1));
+            }
+            else if (isElement(child) && renderBlock(child, options) !== undefined) {
+                const block = renderBlock(child, options) ?? "";
+                if (block !== "") {
+                    nested.push(block
+                        .split("\n")
+                        .map((line) => `${indent}    ${line}`.trimEnd())
+                        .join("\n"));
+                }
+            }
+            else if (isText(child)) {
+                inlineParts.push(child.value);
+            }
+            else if (isElement(child)) {
+                inlineParts.push(renderInline(child, options));
+            }
+        }
+        lines.push(`${indent}${marker}${collapse(inlineParts.join(""))}`);
+        lines.push(...nested);
+    });
+    return lines.join("\n");
+};
+const renderPre = (el) => {
+    const codeChild = el.childNodes.filter(isElement).find((child) => child.tagName === "code");
+    const classes = `${attr(el, "class") ?? ""} ${codeChild === undefined ? "" : (attr(codeChild, "class") ?? "")}`;
+    const lang = /(?:language|lang)-([\w+-]+)/.exec(classes)?.[1] ?? "";
+    const code = preTextOf(el).replace(/^\n/, "").trimEnd();
+    const longestRun = Math.max(2, ...[...code.matchAll(/`+/g)].map((m) => m[0].length));
+    const fence = "`".repeat(Math.max(3, longestRun + 1));
+    return `${fence}${lang}\n${code}\n${fence}`;
+};
+const PRE_LINE = new Set(["div", "p", "li", "tr", "section", "article", "table", "tbody", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6"]);
+const preTextOf = (node) => {
+    if (isText(node)) {
+        return node.value;
+    }
+    if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+        return "";
+    }
+    if (node.tagName === "br") {
+        return "\n";
+    }
+    let text = "";
+    let afterLine = false;
+    for (const child of node.childNodes) {
+        if (isElement(child) && PRE_LINE.has(child.tagName)) {
+            if (text !== "" && !text.endsWith("\n")) {
+                text += "\n";
+            }
+            text += `${preTextOf(child).replace(/\n$/, "")}\n`;
+            afterLine = true;
+            continue;
+        }
+        if (afterLine && isText(child) && /^\r?\n$/.test(child.value)) {
+            continue;
+        }
+        text += preTextOf(child);
+        afterLine = false;
+    }
+    return text;
+};
+const renderTable = (el, options) => {
+    const rows = [];
+    for (const tr of collectRows(el)) {
+        const cells = tr.childNodes.filter(isElement).filter((cell) => cell.tagName === "td" || cell.tagName === "th");
+        rows.push(cells.map((cell) => collapse(renderInlineChildren(cell, options)).replaceAll("|", "\\|").replaceAll("\n", " ")));
+    }
+    if (rows.length === 0) {
+        return "";
+    }
+    const width = Math.max(...rows.map((row) => row.length));
+    const pad = (row) => [...row, ...Array.from({ length: width - row.length }, () => "")];
+    const [header, ...body] = rows.map(pad);
+    return [tableRow(header), tableRow(header.map(() => "---")), ...body.map(tableRow)].join("\n");
+};
+const tableRow = (cells) => `| ${cells.join(" | ")} |`;
+const collectRows = (el) => {
+    const rows = [];
+    for (const child of el.childNodes.filter(isElement)) {
+        if (child.tagName === "tr") {
+            rows.push(child);
+        }
+        else if (child.tagName === "thead" || child.tagName === "tbody" || child.tagName === "tfoot") {
+            rows.push(...child.childNodes.filter(isElement).filter((row) => row.tagName === "tr"));
+        }
+    }
+    return rows;
+};
+const renderDefinitionList = (el, options) => {
+    const lines = [];
+    for (const child of el.childNodes.filter(isElement)) {
+        if (child.tagName === "dt") {
+            lines.push(`**${collapse(renderInlineChildren(child, options))}**`);
+        }
+        else if (child.tagName === "dd") {
+            lines.push(`: ${collapse(renderInlineChildren(child, options))}`);
+        }
+    }
+    return lines.join("\n");
+};
+const resolveUrl = (raw, base) => {
+    if (raw === undefined || raw === "") {
+        return undefined;
+    }
+    const trimmed = raw.trim();
+    if (/^(?:javascript|mailto|tel|data):/i.test(trimmed)) {
+        return trimmed;
+    }
+    try {
+        return base === undefined ? trimmed : new URL(trimmed, base).href;
+    }
+    catch {
+        return undefined;
+    }
+};
+const BR = "\u0001";
+const collapse = (text) => text
+    .split(BR)
+    .map((part) => part.replaceAll(/\s+/g, " ").trim())
+    .join("\n")
+    .trim();
+//# sourceMappingURL=markdown.js.map

package/dist/lib/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/lib/markdown.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,IAAI,EAAgB,SAAS,EAAE,MAAM,EAAa,SAAS,EAAE,MAAM,UAAU,CAAC;AAEvF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE/I,MAAM,MAAM,GAAoD;IAC5D,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;IACnC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;IAC9B,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;IAC7B,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;IAC5B,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;IAChC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;IAC9B,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE;IACnC,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;CACjC,CAAC;AAEF,MAAM,QAAQ,GAA2B,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;AAOtF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAa,EAAE,OAAO,GAAoB,EAAE,EAAU,EAAE;IACnF,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,GAAG,MAAM;SACX,IAAI,CAAC,MAAM,CAAC;SACZ,UAAU,CAAC,WAAW,EAAE,EAAE,CAAC;SAC3B,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC;SAC7B,IAAI,EAAE,IAAI,CAAC;AACpB,CAAC,CAAC;AAGF,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,OAAwB,EAAY,EAAE;IACrE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,MAAM,KAAK,GAAG,GAAS,EAAE;QACrB,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC3B,CAAC;QACD,GAAG,GAAG,EAAE,CAAC;IACb,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACf,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC;YAClB,SAAS;QACb,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACpD,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACzC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACJ,KAAK,EAAE,CAAC;YACR,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAGF,MAAM,WAAW,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAsB,EAAE;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IACrC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,QAAQ,CAAC,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;IAC/D,CAAC;IACD,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACjB,KAAK,GAAG;YACJ,OAAO,QAAQ,CAAC,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;QACvD,KAAK,IAAI,CAAC;QACV,KAAK,IAAI;YACL,OAAO,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QACtC,KAAK,KAAK;YACN,OAAO,SAAS,CAAC,EAAE,CAAC,CAAC;QACzB,KAAK,YAAY;YACb,OAAO,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;iBACpC,IAAI,CAAC,MAAM,CAAC;iBACZ,KAAK,CAAC,IAAI,CAAC;iBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;iBACpC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,KAAK,OAAO;YACR,OAAO,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACpC,KAAK,IAAI;YACL,OAAO,KAAK,CAAC;QACjB,KAAK,QAAQ,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,YAAY,CAAC,CAAC;YAChG,MAAM,IAAI,GAAG,UAAU,CACnB,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,YAAY,CAAC,CAAC,EACtF,OAAO,CACV,CAAC;YACF,MAAM,WAAW,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;YAClG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1G,CAAC;QACD,KAAK,IAAI;YACL,OAAO,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC7C,KAAK,SAAS,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;YAC7F,MAAM,IAAI,GAAG,UAAU,CACnB,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,EACnF,OAAO,CACV,CAAC;YACF,MAAM,KAAK,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,oBAAoB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACrG,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnE,CAAC;QACD,KAAK,KAAK,CAAC;QACX,KAAK,SAAS,CAAC;QACf,KAAK,SAAS,CAAC;QACf,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM,CAAC;QACZ,KAAK,OAAO,CAAC;QACb,KAAK,QAAQ,CAAC;QACd,KAAK,QAAQ,CAAC;QACd,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,KAAK,UAAU,CAAC;QAChB,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACT,OAAO,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3D;YACI,OAAO,SAAS,CAAC;IACzB,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE,CAC3E,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAEhI,MAAM,YAAY,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE;IACnE,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,EAAE,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAEtB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,EAAE,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QACvB,OAAO,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,EAAE,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;QACrB,OAAO,UAAU,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,EAAE,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;IACpE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QAErB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAChC,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;AACvE,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE;IACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,oBAAoB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxD,OAAO,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClE,CAAC;IACD,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,IAAI,IAAI,KAAK,IAAI,GAAG,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC1C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACzD,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/C,OAAO,GAAG,CAAC;IACf,CAAC;IACD,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAE,KAAa,EAAU,EAAE;IAChF,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,KAAK,IAAI,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;IACxF,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAEpC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAClC,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,EAAE,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;YACvD,CAAC;iBAAM,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,SAAS,EAAE,CAAC;gBACvE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;oBACf,MAAM,CAAC,IAAI,CACP,KAAK;yBACA,KAAK,CAAC,IAAI,CAAC;yBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,MAAM,OAAO,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;yBAC/C,IAAI,CAAC,IAAI,CAAC,CAClB,CAAC;gBACN,CAAC;YACL,CAAC;iBAAM,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACvB,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC;iBAAM,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC1B,WAAW,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,EAAW,EAAU,EAAE;IACtC,MAAM,SAAS,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;IAChH,MAAM,IAAI,GAAG,6BAA6B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACpE,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IAExD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC;IACtD,OAAO,GAAG,KAAK,GAAG,IAAI,KAAK,IAAI,KAAK,KAAK,EAAE,CAAC;AAChD,CAAC,CAAC;AAIF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAG3I,MAAM,SAAS,GAAG,CAAC,IAAU,EAAU,EAAE;IACrC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACf,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,EAAE,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,SAAS,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAClD,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,IAAI,IAAI,IAAI,CAAC;YACjB,CAAC;YAED,IAAI,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC;YACnD,SAAS,GAAG,IAAI,CAAC;YACjB,SAAS;QACb,CAAC;QAED,IAAI,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,SAAS;QACb,CAAC;QACD,IAAI,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QACzB,SAAS,GAAG,KAAK,CAAC;IACtB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE;IAClE,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,KAAK,MAAM,EAAE,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QAC/G,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/H,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACzD,MAAM,GAAG,GAAG,CAAC,GAAa,EAAY,EAAE,CAAC,CAAC,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3G,MAAM,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAA8B,CAAC;IACrE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnG,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,KAAe,EAAU,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;AAEzE,MAAM,WAAW,GAAG,CAAC,EAAW,EAAa,EAAE;IAC3C,MAAM,IAAI,GAAc,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC7F,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC;QAC3F,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,EAAW,EAAE,OAAwB,EAAU,EAAE;IAC3E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,oBAAoB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC;QACxE,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,oBAAoB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;QACtE,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,GAAuB,EAAE,IAAwB,EAAsB,EAAE;IACzF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QAClC,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,mCAAmC,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;IACtE,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,EAAE,GAAG,QAAQ,CAAC;AAIpB,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE,CACtC,IAAI;KACC,KAAK,CAAC,EAAE,CAAC;KACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;KAClD,IAAI,CAAC,IAAI,CAAC;KACV,IAAI,EAAE,CAAC"}

package/dist/lib/output.d.ts

@@ -0,0 +1,20 @@
+import type { PageResult } from "./page.js";
+export interface SavedPage {
+    readonly path: string;
+    readonly tokens: number;
+}
+export declare const slugFor: (url: string) => string;
+export declare const savePage: (outDir: string, page: PageResult, fetchedAt: Date) => Promise<SavedPage>;
+export interface IndexEntry {
+    readonly url: string;
+    readonly title: string;
+    readonly path: string;
+    readonly tokens: number;
+    readonly depth: number;
+}
+export interface CrawlIndex {
+    readonly markdownPath: string;
+    readonly jsonPath: string;
+}
+export declare const saveIndex: (outDir: string, startUrl: string, entries: IndexEntry[], skipped: Record<string, number>) => Promise<CrawlIndex>;
+//# sourceMappingURL=output.d.ts.map

package/dist/lib/output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.d.ts","sourceRoot":"","sources":["../../src/lib/output.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE5C,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAerC,CAAC;AAEF,eAAO,MAAM,QAAQ,WAAkB,MAAM,QAAQ,UAAU,aAAa,IAAI,KAAG,OAAO,CAAC,SAAS,CAiBnG,CAAC;AAEF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC7B;AAGD,eAAO,MAAM,SAAS,WAAkB,MAAM,YAAY,MAAM,WAAW,UAAU,EAAE,WAAW,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAG,OAAO,CAAC,UAAU,CAuB5I,CAAC"}

package/dist/lib/output.js

@@ -0,0 +1,64 @@
+import { createHash } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { estimateTokens } from "@intentic/base/format";
+export const slugFor = (url) => {
+    const hash = createHash("sha256").update(url).digest("hex").slice(0, 8);
+    let readable;
+    try {
+        const parsed = new URL(url);
+        readable = `${parsed.host}${parsed.pathname}`;
+    }
+    catch {
+        readable = url;
+    }
+    const slug = readable
+        .toLowerCase()
+        .replaceAll(/[^a-z0-9]+/g, "-")
+        .replaceAll(/^-+|-+$/g, "")
+        .slice(0, 80);
+    return `${slug === "" ? "page" : slug}-${hash}.md`;
+};
+export const savePage = async (outDir, page, fetchedAt) => {
+    await mkdir(outDir, { recursive: true });
+    const path = join(outDir, slugFor(page.url));
+    const frontMatter = [
+        "---",
+        `url: ${page.url}`,
+        ...(page.finalUrl === page.url ? [] : [`final_url: ${page.finalUrl}`]),
+        `title: ${JSON.stringify(page.meta.title)}`,
+        `fetched_at: ${fetchedAt.toISOString()}`,
+        `source: ${page.source}`,
+        ...page.notes.map((note) => `note: ${JSON.stringify(note)}`),
+        "---",
+        "",
+    ].join("\n");
+    const content = `${frontMatter}${page.markdown}`;
+    await writeFile(path, content);
+    return { path, tokens: estimateTokens(page.markdown) };
+};
+export const saveIndex = async (outDir, startUrl, entries, skipped) => {
+    await mkdir(outDir, { recursive: true });
+    const sorted = entries.toSorted((a, b) => a.url.localeCompare(b.url));
+    const markdownPath = join(outDir, "index.md");
+    const jsonPath = join(outDir, "index.json");
+    const skippedLine = Object.entries(skipped)
+        .filter(([, count]) => count > 0)
+        .map(([reason, count]) => `${reason} ${count}`)
+        .join(", ");
+    const lines = [
+        `# Crawl of ${startUrl}`,
+        "",
+        `${entries.length} pages, ${entries.reduce((sum, entry) => sum + entry.tokens, 0)} tokens total.`,
+        ...(skippedLine === "" ? [] : [`Skipped: ${skippedLine}.`]),
+        "",
+        "| title | url | file | tokens |",
+        "| --- | --- | --- | --- |",
+        ...sorted.map((entry) => `| ${entry.title.replaceAll("|", "\\|")} | ${entry.url} | ${entry.path} | ${entry.tokens} |`),
+        "",
+    ];
+    await writeFile(markdownPath, lines.join("\n"));
+    await writeFile(jsonPath, JSON.stringify({ startUrl, pages: sorted, skipped }, null, 2));
+    return { markdownPath, jsonPath };
+};
+//# sourceMappingURL=output.js.map

package/dist/lib/output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"output.js","sourceRoot":"","sources":["../../src/lib/output.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAQvD,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE;IAC3C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxE,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,QAAQ,GAAG,GAAG,MAAM,CAAC,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACL,QAAQ,GAAG,GAAG,CAAC;IACnB,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ;SAChB,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClB,OAAO,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,CAAC;AACvD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAc,EAAE,IAAgB,EAAE,SAAe,EAAsB,EAAE;IACpG,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG;QAChB,KAAK;QACL,QAAQ,IAAI,CAAC,GAAG,EAAE;QAClB,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QACtE,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC3C,eAAe,SAAS,CAAC,WAAW,EAAE,EAAE;QACxC,WAAW,IAAI,CAAC,MAAM,EAAE;QACxB,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5D,KAAK;QACL,EAAE;KACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,GAAG,WAAW,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IACjD,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC3D,CAAC,CAAC;AAgBF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,MAAc,EAAE,QAAgB,EAAE,OAAqB,EAAE,OAA+B,EAAuB,EAAE;IAC7I,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACtE,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SACtC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;SAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,MAAM,KAAK,GAAG;QACV,cAAc,QAAQ,EAAE;QACxB,EAAE;QACF,GAAG,OAAO,CAAC,MAAM,WAAW,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,gBAAgB;QACjG,GAAG,CAAC,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,WAAW,GAAG,CAAC,CAAC;QAC3D,EAAE;QACF,iCAAiC;QACjC,2BAA2B;QAC3B,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,GAAG,MAAM,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,MAAM,IAAI,CAAC;QACtH,EAAE;KACL,CAAC;IACF,MAAM,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzF,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC;AACtC,CAAC,CAAC"}

package/dist/lib/page.d.ts

@@ -0,0 +1,26 @@
+import { type PageMeta } from "./dom.js";
+import { type Link } from "./links.js";
+export type BrowserMode = "auto" | "never" | "force";
+export interface FetchPageOptions {
+    readonly raw?: boolean;
+    readonly query?: string | undefined;
+    readonly browser?: BrowserMode;
+    readonly maxAgeS?: number;
+    readonly timeoutMs?: number;
+    readonly maxBytes?: number;
+    readonly threshold?: number | undefined;
+}
+export interface PageResult {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly markdown: string;
+    readonly meta: PageMeta;
+    readonly links: Link[];
+    readonly source: "cache" | "network" | "browser";
+    readonly prunedShare: number | undefined;
+    readonly notes: string[];
+}
+export declare const fetchPage: (url: string, options?: FetchPageOptions) => Promise<PageResult>;
+//# sourceMappingURL=page.d.ts.map

package/dist/lib/page.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../../src/lib/page.ts"],"names":[],"mappings":"AAOA,OAAO,EAA8D,KAAK,QAAQ,EAAmC,MAAM,UAAU,CAAC;AAEtI,OAAO,EAAgB,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAIrD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;AAErD,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC;IAE/B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C;AAED,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,SAAS,GAAG,SAAS,CAAC;IAEjD,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,eAAO,MAAM,SAAS,QAAe,MAAM,YAAW,gBAAgB,KAAQ,OAAO,CAAC,UAAU,CAmD/F,CAAC"}

package/dist/lib/page.js

@@ -0,0 +1,153 @@
+import { neutralizeOutsideText } from "@intentic/base/outside-text";
+import { bm25Rank } from "./bm25.js";
+import { chromiumAvailable, renderPage } from "./browser.js";
+import { cacheRead, cacheWrite, DEFAULT_MAX_AGE_S } from "./cache.js";
+import { bodyOf, elementsByTag, metaOf, parseHtml, remove, textOf, walk } from "./dom.js";
+import { httpFetch, USER_AGENT } from "./http.js";
+import { extractLinks } from "./links.js";
+import { renderMarkdown } from "./markdown.js";
+import { pruneTree } from "./prune.js";
+export const fetchPage = async (url, options = {}) => {
+    const browserMode = options.browser ?? "auto";
+    const maxAgeS = options.maxAgeS ?? DEFAULT_MAX_AGE_S;
+    const timeoutMs = options.timeoutMs ?? 20_000;
+    const notes = [];
+    let acquired = browserMode === "force" ? undefined : await acquireStatic(url, options, maxAgeS, timeoutMs, notes);
+    if (acquired === undefined || (browserMode !== "never" && looksLikeAppShell(acquired.doc, acquired.body))) {
+        const rendered = await acquireBrowser(url, maxAgeS, timeoutMs, notes);
+        if (rendered !== undefined) {
+            acquired = rendered;
+        }
+        else if (acquired === undefined) {
+            throw new Error("browser rendering unavailable (no Chromium in this image) and --browser force was asked for");
+        }
+    }
+    const { doc, body, finalUrl, status, contentType, source } = acquired;
+    if (!isHtml(contentType)) {
+        return {
+            url,
+            finalUrl,
+            status,
+            contentType,
+            markdown: neutralizeOutsideText(nonHtmlMarkdown(acquired.rawBody, contentType, notes)),
+            meta: { title: url, description: undefined, lang: undefined },
+            links: [],
+            source,
+            prunedShare: undefined,
+            notes,
+        };
+    }
+    const rawMeta = metaOf(doc);
+    const meta = {
+        title: neutralizeOutsideText(rawMeta.title),
+        description: rawMeta.description === undefined ? undefined : neutralizeOutsideText(rawMeta.description),
+        lang: rawMeta.lang,
+    };
+    const links = extractLinks(doc, finalUrl);
+    let prunedShare;
+    if (body !== undefined && options.raw !== true) {
+        prunedShare = pruneTree(body, options.threshold === undefined ? {} : { threshold: options.threshold });
+    }
+    if (body !== undefined && options.query !== undefined && options.query !== "") {
+        notes.push(applyQueryFilter(body, options.query));
+    }
+    const markdown = body === undefined ? "" : neutralizeOutsideText(renderMarkdown(body, { baseUrl: finalUrl }));
+    return { url, finalUrl, status, contentType, markdown, meta, links, source, prunedShare, notes };
+};
+const acquireStatic = async (url, options, maxAgeS, timeoutMs, notes) => {
+    const cached = maxAgeS > 0 ? await cacheRead(url, "static", maxAgeS) : undefined;
+    if (cached !== undefined) {
+        return parsed(cached.body, cached.finalUrl, cached.status, cached.contentType, "cache");
+    }
+    const fetched = await httpFetch(url, {
+        timeoutMs,
+        userAgent: USER_AGENT,
+        ...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }),
+    });
+    if (fetched.truncated) {
+        notes.push("byte cap hit mid-body: this page is cut short");
+    }
+    await cacheWrite({
+        url,
+        finalUrl: fetched.finalUrl,
+        status: fetched.status,
+        contentType: fetched.contentType,
+        body: fetched.body,
+        mode: "static",
+        fetchedAt: Date.now(),
+    });
+    return parsed(fetched.body, fetched.finalUrl, fetched.status, fetched.contentType, "network");
+};
+const acquireBrowser = async (url, maxAgeS, timeoutMs, notes) => {
+    const mode = "browser";
+    const cached = maxAgeS > 0 ? await cacheRead(url, mode, maxAgeS) : undefined;
+    if (cached !== undefined) {
+        return parsed(cached.body, cached.finalUrl, cached.status, cached.contentType, "browser");
+    }
+    if (!(await chromiumAvailable())) {
+        notes.push("page looks JS-rendered but this image has no Chromium (browser feature pack): static HTML served best-effort");
+        return undefined;
+    }
+    const rendered = await renderPage(url, timeoutMs);
+    await cacheWrite({ url, finalUrl: rendered.finalUrl, status: 200, contentType: "text/html", body: rendered.html, mode, fetchedAt: Date.now() });
+    return parsed(rendered.html, rendered.finalUrl, 200, "text/html", "browser");
+};
+const parsed = (html, finalUrl, status, contentType, source) => {
+    const doc = isHtml(contentType) ? parseHtml(html) : parseHtml("");
+    return { doc, body: bodyOf(doc), rawBody: html, finalUrl, status, contentType, source };
+};
+const isHtml = (contentType) => contentType === "" || /text\/html|application\/xhtml/i.test(contentType);
+const looksLikeAppShell = (doc, body) => {
+    if (body === undefined) {
+        return false;
+    }
+    const text = textOf(body);
+    if (text.length >= 200) {
+        return false;
+    }
+    return elementsByTag(doc, "script").length > 0;
+};
+const nonHtmlMarkdown = (rawBody, contentType, notes) => {
+    if (/json/i.test(contentType)) {
+        return `\`\`\`json\n${rawBody.trim()}\n\`\`\`\n`;
+    }
+    if (/xml/i.test(contentType)) {
+        return `\`\`\`xml\n${rawBody.trim()}\n\`\`\`\n`;
+    }
+    if (/^text\//i.test(contentType)) {
+        return `${rawBody.trim()}\n`;
+    }
+    notes.push(`unsupported content type ${contentType.split(";")[0] ?? contentType}: no markdown produced`);
+    return "";
+};
+const FILTERABLE = new Set(["p", "li", "pre", "table", "blockquote", "dd"]);
+const applyQueryFilter = (body, query) => {
+    const candidates = [];
+    walk(body, (el) => {
+        if (FILTERABLE.has(el.tagName) && !hasFilterableAncestorWithin(el, body)) {
+            candidates.push(el);
+        }
+    });
+    const ranked = bm25Rank(candidates, (el) => textOf(el), query);
+    const keep = new Set(ranked.map((entry) => entry.block));
+    if (keep.size < Math.min(3, candidates.length)) {
+        return `query matched too little (${keep.size}/${candidates.length} blocks): kept the whole page`;
+    }
+    for (const candidate of candidates) {
+        if (!keep.has(candidate)) {
+            remove(candidate);
+        }
+    }
+    return `query kept ${keep.size}/${candidates.length} blocks`;
+};
+const hasFilterableAncestorWithin = (el, root) => {
+    let parent = el.parentNode;
+    while (parent !== null && parent !== root) {
+        if ("tagName" in parent && FILTERABLE.has(parent.tagName)) {
+            return true;
+        }
+        parent = "parentNode" in parent && parent.parentNode !== undefined ? parent.parentNode : null;
+    }
+    return false;
+};
+//# sourceMappingURL=page.js.map

package/dist/lib/page.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page.js","sourceRoot":"","sources":["../../src/lib/page.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,iBAAiB,EAAmB,MAAM,YAAY,CAAC;AACvF,OAAO,EAAE,MAAM,EAA+B,aAAa,EAAE,MAAM,EAAiB,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AACtI,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAClD,OAAO,EAAE,YAAY,EAAa,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AA6BvC,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAE,OAAO,GAAqB,EAAE,EAAuB,EAAE;IAChG,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,iBAAiB,CAAC;IACrD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,IAAI,QAAQ,GAAG,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IAClH,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,WAAW,KAAK,OAAO,IAAI,iBAAiB,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACxG,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACtE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,QAAQ,GAAG,QAAQ,CAAC;QACxB,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6FAA6F,CAAC,CAAC;QACnH,CAAC;IACL,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,QAAQ,CAAC;IAEtE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QACvB,OAAO;YACH,GAAG;YACH,QAAQ;YACR,MAAM;YACN,WAAW;YACX,QAAQ,EAAE,qBAAqB,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YACtF,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE;YAC7D,KAAK,EAAE,EAAE;YACT,MAAM;YACN,WAAW,EAAE,SAAS;YACtB,KAAK;SACR,CAAC;IACN,CAAC;IAGD,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC5B,MAAM,IAAI,GAAa;QACnB,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;QAC3C,WAAW,EAAE,OAAO,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,WAAW,CAAC;QACvG,IAAI,EAAE,OAAO,CAAC,IAAI;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE1C,IAAI,WAA+B,CAAC;IACpC,IAAI,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;QAC7C,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAC3G,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;QAC5E,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC9G,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;AACrG,CAAC,CAAC;AAYF,MAAM,aAAa,GAAG,KAAK,EAAE,GAAW,EAAE,OAAyB,EAAE,OAAe,EAAE,SAAiB,EAAE,KAAe,EAAqB,EAAE;IAC3I,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;QACjC,SAAS;QACT,SAAS,EAAE,UAAU;QACrB,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;KAC5E,CAAC,CAAC;IACH,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACpB,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,UAAU,CAAC;QACb,GAAG;QACH,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,IAAI,EAAE,QAAQ;QACd,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;KACxB,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,KAAK,EAAE,GAAW,EAAE,OAAe,EAAE,SAAiB,EAAE,KAAe,EAAiC,EAAE;IAC7H,MAAM,IAAI,GAAe,SAAS,CAAC;IACnC,MAAM,MAAM,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7E,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAC9F,CAAC;IACD,IAAI,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,8GAA8G,CAAC,CAAC;QAC3H,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAClD,MAAM,UAAU,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IAChJ,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;AACjF,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,QAAgB,EAAE,MAAc,EAAE,WAAmB,EAAE,MAA0B,EAAY,EAAE;IACzH,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAClE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAC5F,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,WAAmB,EAAW,EAAE,CAAC,WAAW,KAAK,EAAE,IAAI,gCAAgC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAI1H,MAAM,iBAAiB,GAAG,CAAC,GAAa,EAAE,IAAyB,EAAW,EAAE;IAC5E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC1B,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AACnD,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,OAAe,EAAE,WAAmB,EAAE,KAAe,EAAU,EAAE;IACtF,IAAI,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5B,OAAO,eAAe,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC;IACrD,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3B,OAAO,cAAc,OAAO,CAAC,IAAI,EAAE,YAAY,CAAC;IACpD,CAAC;IACD,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC;IACjC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,4BAA4B,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,wBAAwB,CAAC,CAAC;IACzG,OAAO,EAAE,CAAC;AACd,CAAC,CAAC;AAGF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC;AAE5E,MAAM,gBAAgB,GAAG,CAAC,IAAa,EAAE,KAAa,EAAU,EAAE;IAC9D,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;QACd,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;YACvE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAEzD,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,OAAO,6BAA6B,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,+BAA+B,CAAC;IACtG,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACvB,MAAM,CAAC,SAAS,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IACD,OAAO,cAAc,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,MAAM,SAAS,CAAC;AACjE,CAAC,CAAC;AAIF,MAAM,2BAA2B,GAAG,CAAC,EAAW,EAAE,IAAa,EAAW,EAAE;IACxE,IAAI,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC;IAC3B,OAAO,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,SAAS,IAAI,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,YAAY,IAAI,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;IAClG,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC"}

package/dist/lib/prune.d.ts

@@ -0,0 +1,7 @@
+import { type Element } from "./dom.js";
+export interface PruneOptions {
+    readonly threshold?: number;
+    readonly minWords?: number;
+}
+export declare const pruneTree: (body: Element, options?: PruneOptions) => number;
+//# sourceMappingURL=prune.d.ts.map

package/dist/lib/prune.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.d.ts","sourceRoot":"","sources":["../../src/lib/prune.ts"],"names":[],"mappings":"AAIA,OAAO,EAAuB,KAAK,OAAO,EAAqC,MAAM,UAAU,CAAC;AA2BhG,MAAM,WAAW,YAAY;IAEzB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAE5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC9B;AASD,eAAO,MAAM,SAAS,SAAU,OAAO,YAAW,YAAY,KAAQ,MAWrE,CAAC"}