@intentic/webq 1.317.0

package/src/lib/markdown.ts

@@ -0,0 +1,337 @@
+// DOM to GitHub-flavored markdown for an agent reader: no character escaping except table pipes and code backticks,
+// absolute URLs (a relative link is a broken fetch), data: images dropped to alt text. Mixed content flows: inline runs
+// pool into a paragraph buffer flushed by a block; unknown elements are transparent containers.
+import { attr, type Element, isElement, isText, type Node, rawTextOf } from "./dom.js";
+
+const NON_CONTENT = new Set(["script", "style", "noscript", "template", "svg", "canvas", "iframe", "object", "embed", "link", "meta", "head"]);
+
+const INLINE: Record<string, { open: string; close: string }> = {
+    strong: { open: "**", close: "**" },
+    b: { open: "**", close: "**" },
+    em: { open: "*", close: "*" },
+    i: { open: "*", close: "*" },
+    del: { open: "~~", close: "~~" },
+    s: { open: "~~", close: "~~" },
+    strike: { open: "~~", close: "~~" },
+    kbd: { open: "`", close: "`" },
+};
+
+const HEADINGS: Record<string, number> = { h1: 1, h2: 2, h3: 3, h4: 4, h5: 5, h6: 6 };
+
+export interface MarkdownOptions {
+    /** The page's final URL; every href/src resolves against it. Absent, URLs pass through untouched. */
+    readonly baseUrl?: string | undefined;
+}
+
+export const renderMarkdown = (root: Element, options: MarkdownOptions = {}): string => {
+    const blocks = renderFlow(root.childNodes, options);
+    return `${blocks
+        .join("\n\n")
+        .replaceAll(/[ \t]+$/gm, "")
+        .replaceAll(/\n{3,}/g, "\n\n")
+        .trim()}\n`;
+};
+
+/* Children of a container: inline runs pool into paragraphs, block elements flush and stand alone. */
+const renderFlow = (nodes: Node[], options: MarkdownOptions): string[] => {
+    const blocks: string[] = [];
+    let run = "";
+    const flush = (): void => {
+        const paragraph = collapse(run);
+        if (paragraph !== "") {
+            blocks.push(paragraph);
+        }
+        run = "";
+    };
+    for (const node of nodes) {
+        if (isText(node)) {
+            run += node.value;
+            continue;
+        }
+        if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+            continue;
+        }
+        const block = renderBlock(node, options);
+        if (block === undefined) {
+            run += renderInline(node, options);
+        } else {
+            flush();
+            if (block !== "") {
+                blocks.push(block);
+            }
+        }
+    }
+    flush();
+    return blocks;
+};
+
+/** A block element's markdown, or undefined when the element is inline and belongs to the current run. */
+const renderBlock = (el: Element, options: MarkdownOptions): string | undefined => {
+    const heading = HEADINGS[el.tagName];
+    if (heading !== undefined) {
+        const text = collapse(renderInlineChildren(el, options));
+        return text === "" ? "" : `${"#".repeat(heading)} ${text}`;
+    }
+    switch (el.tagName) {
+        case "p":
+            return collapse(renderInlineChildren(el, options));
+        case "ul":
+        case "ol":
+            return renderList(el, options, 0);
+        case "pre":
+            return renderPre(el);
+        case "blockquote":
+            return renderFlow(el.childNodes, options)
+                .join("\n\n")
+                .split("\n")
+                .map((line) => `> ${line}`.trimEnd())
+                .join("\n");
+        case "table":
+            return renderTable(el, options);
+        case "hr":
+            return "---";
+        case "figure": {
+            const caption = el.childNodes.filter(isElement).find((child) => child.tagName === "figcaption");
+            const body = renderFlow(
+                el.childNodes.filter((child) => !(isElement(child) && child.tagName === "figcaption")),
+                options,
+            );
+            const captionText = caption === undefined ? "" : collapse(renderInlineChildren(caption, options));
+            return [body.join("\n\n"), captionText === "" ? "" : `*${captionText}*`].filter(Boolean).join("\n\n");
+        }
+        case "dl":
+            return renderDefinitionList(el, options);
+        case "details": {
+            const summary = el.childNodes.filter(isElement).find((child) => child.tagName === "summary");
+            const rest = renderFlow(
+                el.childNodes.filter((child) => !(isElement(child) && child.tagName === "summary")),
+                options,
+            );
+            const label = summary === undefined ? "" : `**${collapse(renderInlineChildren(summary, options))}**`;
+            return [label, rest.join("\n\n")].filter(Boolean).join("\n\n");
+        }
+        case "div":
+        case "section":
+        case "article":
+        case "main":
+        case "body":
+        case "aside":
+        case "header":
+        case "footer":
+        case "nav":
+        case "form":
+        case "fieldset":
+        case "address":
+        case "center":
+            return renderFlow(el.childNodes, options).join("\n\n");
+        default:
+            return undefined;
+    }
+};
+
+const renderInlineChildren = (el: Element, options: MarkdownOptions): string =>
+    el.childNodes.map((child) => (isText(child) ? child.value : isElement(child) ? renderInline(child, options) : "")).join("");
+
+const renderInline = (el: Element, options: MarkdownOptions): string => {
+    if (NON_CONTENT.has(el.tagName)) {
+        return "";
+    }
+    if (el.tagName === "br") {
+        // Sentinel, not "\n": HTML newlines are whitespace and collapse to spaces; only this survives as a break.
+        return BR;
+    }
+    if (el.tagName === "img") {
+        return renderImage(el, options);
+    }
+    if (el.tagName === "a") {
+        return renderLink(el, options);
+    }
+    if (el.tagName === "code") {
+        const code = rawTextOf(el).trim();
+        return code.includes("`") ? `\`\` ${code} \`\`` : `\`${code}\``;
+    }
+    const wrap = INLINE[el.tagName];
+    const inner = renderInlineChildren(el, options);
+    if (wrap === undefined) {
+        // Unknown/neutral inline (span, small, q) is transparent; blocks never reach here, flow dispatches those first.
+        return inner;
+    }
+    const trimmed = collapse(inner);
+    return trimmed === "" ? "" : `${wrap.open}${trimmed}${wrap.close}`;
+};
+
+const renderLink = (el: Element, options: MarkdownOptions): string => {
+    const text = collapse(renderInlineChildren(el, options));
+    const href = resolveUrl(attr(el, "href"), options.baseUrl);
+    if (href === undefined || href.startsWith("javascript:")) {
+        return text;
+    }
+    if (href.startsWith("mailto:") || href.startsWith("tel:")) {
+        return text === "" ? href.slice(href.indexOf(":") + 1) : text;
+    }
+    if (text === "") {
+        return "";
+    }
+    return `[${text}](${href})`;
+};
+
+const renderImage = (el: Element, options: MarkdownOptions): string => {
+    const alt = attr(el, "alt")?.trim() ?? "";
+    const src = resolveUrl(attr(el, "src"), options.baseUrl);
+    if (src === undefined || src.startsWith("data:")) {
+        return alt;
+    }
+    return `![${alt}](${src})`;
+};
+
+const renderList = (el: Element, options: MarkdownOptions, depth: number): string => {
+    const ordered = el.tagName === "ol";
+    const start = Number(attr(el, "start") ?? "1");
+    const items = el.childNodes.filter(isElement).filter((child) => child.tagName === "li");
+    const lines: string[] = [];
+    items.forEach((item, index) => {
+        const marker = ordered ? `${start + index}. ` : "- ";
+        const indent = "    ".repeat(depth);
+        // An item is flow of its own: loose text becomes the item line, nested lists/blocks follow.
+        const nested: string[] = [];
+        const inlineParts: string[] = [];
+        for (const child of item.childNodes) {
+            if (isElement(child) && (child.tagName === "ul" || child.tagName === "ol")) {
+                nested.push(renderList(child, options, depth + 1));
+            } else if (isElement(child) && renderBlock(child, options) !== undefined) {
+                const block = renderBlock(child, options) ?? "";
+                if (block !== "") {
+                    nested.push(
+                        block
+                            .split("\n")
+                            .map((line) => `${indent}    ${line}`.trimEnd())
+                            .join("\n"),
+                    );
+                }
+            } else if (isText(child)) {
+                inlineParts.push(child.value);
+            } else if (isElement(child)) {
+                inlineParts.push(renderInline(child, options));
+            }
+        }
+        lines.push(`${indent}${marker}${collapse(inlineParts.join(""))}`);
+        lines.push(...nested);
+    });
+    return lines.join("\n");
+};
+
+const renderPre = (el: Element): string => {
+    const codeChild = el.childNodes.filter(isElement).find((child) => child.tagName === "code");
+    const classes = `${attr(el, "class") ?? ""} ${codeChild === undefined ? "" : (attr(codeChild, "class") ?? "")}`;
+    const lang = /(?:language|lang)-([\w+-]+)/.exec(classes)?.[1] ?? "";
+    const code = preTextOf(el).replace(/^\n/, "").trimEnd();
+    // The fence must be longer than any backtick run inside the code it fences.
+    const longestRun = Math.max(2, ...[...code.matchAll(/`+/g)].map((m) => m[0].length));
+    const fence = "`".repeat(Math.max(3, longestRun + 1));
+    return `${fence}${lang}\n${code}\n${fence}`;
+};
+
+// Elements that are a line of their own inside a <pre>: editors like CodeMirror render one <div> per line and no
+// newline text between them, so the line breaks exist only as structure.
+const PRE_LINE = new Set(["div", "p", "li", "tr", "section", "article", "table", "tbody", "ul", "ol", "h1", "h2", "h3", "h4", "h5", "h6"]);
+
+/** A code block's text, with a line break at each <br> and around each block-level child. */
+const preTextOf = (node: Node): string => {
+    if (isText(node)) {
+        return node.value;
+    }
+    if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+        return "";
+    }
+    if (node.tagName === "br") {
+        return "\n";
+    }
+    let text = "";
+    let afterLine = false;
+    for (const child of node.childNodes) {
+        if (isElement(child) && PRE_LINE.has(child.tagName)) {
+            if (text !== "" && !text.endsWith("\n")) {
+                text += "\n";
+            }
+            // An empty line is a line element holding only a <br>: the element is the line, the <br> adds nothing.
+            text += `${preTextOf(child).replace(/\n$/, "")}\n`;
+            afterLine = true;
+            continue;
+        }
+        // Pretty-printed markup puts a newline between line elements that already ended their line.
+        if (afterLine && isText(child) && /^\r?\n$/.test(child.value)) {
+            continue;
+        }
+        text += preTextOf(child);
+        afterLine = false;
+    }
+    return text;
+};
+
+const renderTable = (el: Element, options: MarkdownOptions): string => {
+    const rows: string[][] = [];
+    for (const tr of collectRows(el)) {
+        const cells = tr.childNodes.filter(isElement).filter((cell) => cell.tagName === "td" || cell.tagName === "th");
+        rows.push(cells.map((cell) => collapse(renderInlineChildren(cell, options)).replaceAll("|", "\\|").replaceAll("\n", " ")));
+    }
+    if (rows.length === 0) {
+        return "";
+    }
+    const width = Math.max(...rows.map((row) => row.length));
+    const pad = (row: string[]): string[] => [...row, ...Array.from({ length: width - row.length }, () => "")];
+    const [header, ...body] = rows.map(pad) as [string[], ...string[][]];
+    return [tableRow(header), tableRow(header.map(() => "---")), ...body.map(tableRow)].join("\n");
+};
+
+const tableRow = (cells: string[]): string => `| ${cells.join(" | ")} |`;
+
+const collectRows = (el: Element): Element[] => {
+    const rows: Element[] = [];
+    for (const child of el.childNodes.filter(isElement)) {
+        if (child.tagName === "tr") {
+            rows.push(child);
+        } else if (child.tagName === "thead" || child.tagName === "tbody" || child.tagName === "tfoot") {
+            rows.push(...child.childNodes.filter(isElement).filter((row) => row.tagName === "tr"));
+        }
+    }
+    return rows;
+};
+
+const renderDefinitionList = (el: Element, options: MarkdownOptions): string => {
+    const lines: string[] = [];
+    for (const child of el.childNodes.filter(isElement)) {
+        if (child.tagName === "dt") {
+            lines.push(`**${collapse(renderInlineChildren(child, options))}**`);
+        } else if (child.tagName === "dd") {
+            lines.push(`: ${collapse(renderInlineChildren(child, options))}`);
+        }
+    }
+    return lines.join("\n");
+};
+
+const resolveUrl = (raw: string | undefined, base: string | undefined): string | undefined => {
+    if (raw === undefined || raw === "") {
+        return undefined;
+    }
+    const trimmed = raw.trim();
+    if (/^(?:javascript|mailto|tel|data):/i.test(trimmed)) {
+        return trimmed;
+    }
+    try {
+        return base === undefined ? trimmed : new URL(trimmed, base).href;
+    } catch {
+        return undefined;
+    }
+};
+
+// Control character no HTML text node carries; whitespace collapse skips it, and only it becomes a line break.
+const BR = "\u0001";
+
+// Collapses whitespace without eating <br>'s line breaks: splits on the sentinel first, so only the rejoins are
+// newlines. Split, not a regex over the control character (a lint error elsewhere).
+const collapse = (text: string): string =>
+    text
+        .split(BR)
+        .map((part) => part.replaceAll(/\s+/g, " ").trim())
+        .join("\n")
+        .trim();

package/src/lib/output.test.ts

@@ -0,0 +1,16 @@
+import { slugFor } from "./output.js";
+
+describe("slugFor", () => {
+    it("is readable, stable, and distinguishes URLs that slug identically", () => {
+        const first = slugFor("https://docs.ex.com/guide/setup");
+        expect(first).toMatch(/^docs-ex-com-guide-setup-[0-9a-f]{8}\.md$/);
+        expect(slugFor("https://docs.ex.com/guide/setup")).toBe(first);
+        expect(slugFor("https://docs.ex.com/guide/setup?page=2")).not.toBe(first);
+    });
+
+    it("cannot spell a path that escapes the output directory", () => {
+        const slug = slugFor("https://ex.com/../../etc/passwd");
+        expect(slug).not.toContain("/");
+        expect(slug).not.toContain("..");
+    });
+});

package/src/lib/output.ts

@@ -0,0 +1,86 @@
+/* Where markdown lands and how it is named. */
+import { createHash } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { estimateTokens } from "@intentic/base/format";
+import type { PageResult } from "./page.js";
+
+export interface SavedPage {
+    readonly path: string;
+    readonly tokens: number;
+}
+
+export const slugFor = (url: string): string => {
+    const hash = createHash("sha256").update(url).digest("hex").slice(0, 8);
+    let readable: string;
+    try {
+        const parsed = new URL(url);
+        readable = `${parsed.host}${parsed.pathname}`;
+    } catch {
+        readable = url;
+    }
+    const slug = readable
+        .toLowerCase()
+        .replaceAll(/[^a-z0-9]+/g, "-")
+        .replaceAll(/^-+|-+$/g, "")
+        .slice(0, 80);
+    return `${slug === "" ? "page" : slug}-${hash}.md`;
+};
+
+export const savePage = async (outDir: string, page: PageResult, fetchedAt: Date): Promise<SavedPage> => {
+    await mkdir(outDir, { recursive: true });
+    const path = join(outDir, slugFor(page.url));
+    const frontMatter = [
+        "---",
+        `url: ${page.url}`,
+        ...(page.finalUrl === page.url ? [] : [`final_url: ${page.finalUrl}`]),
+        `title: ${JSON.stringify(page.meta.title)}`,
+        `fetched_at: ${fetchedAt.toISOString()}`,
+        `source: ${page.source}`,
+        ...page.notes.map((note) => `note: ${JSON.stringify(note)}`),
+        "---",
+        "",
+    ].join("\n");
+    const content = `${frontMatter}${page.markdown}`;
+    await writeFile(path, content);
+    return { path, tokens: estimateTokens(page.markdown) };
+};
+
+export interface IndexEntry {
+    readonly url: string;
+    readonly title: string;
+    readonly path: string;
+    readonly tokens: number;
+    readonly depth: number;
+}
+
+export interface CrawlIndex {
+    readonly markdownPath: string;
+    readonly jsonPath: string;
+}
+
+/** The crawl's table of contents, as markdown for a reader and JSON for a program. */
+export const saveIndex = async (outDir: string, startUrl: string, entries: IndexEntry[], skipped: Record<string, number>): Promise<CrawlIndex> => {
+    await mkdir(outDir, { recursive: true });
+    const sorted = entries.toSorted((a, b) => a.url.localeCompare(b.url));
+    const markdownPath = join(outDir, "index.md");
+    const jsonPath = join(outDir, "index.json");
+    const skippedLine = Object.entries(skipped)
+        .filter(([, count]) => count > 0)
+        .map(([reason, count]) => `${reason} ${count}`)
+        .join(", ");
+    const lines = [
+        `# Crawl of ${startUrl}`,
+        "",
+        `${entries.length} pages, ${entries.reduce((sum, entry) => sum + entry.tokens, 0)} tokens total.`,
+        ...(skippedLine === "" ? [] : [`Skipped: ${skippedLine}.`]),
+        "",
+        "| title | url | file | tokens |",
+        "| --- | --- | --- | --- |",
+        ...sorted.map((entry) => `| ${entry.title.replaceAll("|", "\\|")} | ${entry.url} | ${entry.path} | ${entry.tokens} |`),
+        "",
+    ];
+    await writeFile(markdownPath, lines.join("\n"));
+    await writeFile(jsonPath, JSON.stringify({ startUrl, pages: sorted, skipped }, null, 2));
+    return { markdownPath, jsonPath };
+};

package/src/lib/page.ts

@@ -0,0 +1,213 @@
+// One URL to one PageResult: cache first, static HTTP second, Chromium only for an empty-looking app shell (or forced).
+// Result carries its own honesty: source, prune share, query-filter outcome and every degradation as a note; a silent
+// fallback would read as the whole page.
+import { neutralizeOutsideText } from "@intentic/base/outside-text";
+import { bm25Rank } from "./bm25.js";
+import { chromiumAvailable, renderPage } from "./browser.js";
+import { cacheRead, cacheWrite, DEFAULT_MAX_AGE_S, type RenderMode } from "./cache.js";
+import { bodyOf, type Document, type Element, elementsByTag, metaOf, type PageMeta, parseHtml, remove, textOf, walk } from "./dom.js";
+import { httpFetch, USER_AGENT } from "./http.js";
+import { extractLinks, type Link } from "./links.js";
+import { renderMarkdown } from "./markdown.js";
+import { pruneTree } from "./prune.js";
+
+export type BrowserMode = "auto" | "never" | "force";
+
+export interface FetchPageOptions {
+    readonly raw?: boolean;
+    readonly query?: string | undefined;
+    readonly browser?: BrowserMode;
+    /** Cache freshness window in seconds; 0 means bypass reads (a write still lands). */
+    readonly maxAgeS?: number;
+    readonly timeoutMs?: number;
+    readonly maxBytes?: number;
+    readonly threshold?: number | undefined;
+}
+
+export interface PageResult {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly markdown: string;
+    readonly meta: PageMeta;
+    readonly links: Link[];
+    readonly source: "cache" | "network" | "browser";
+    /** Share of text mass fit-pruning removed, when it ran. */
+    readonly prunedShare: number | undefined;
+    readonly notes: string[];
+}
+
+export const fetchPage = async (url: string, options: FetchPageOptions = {}): Promise<PageResult> => {
+    const browserMode = options.browser ?? "auto";
+    const maxAgeS = options.maxAgeS ?? DEFAULT_MAX_AGE_S;
+    const timeoutMs = options.timeoutMs ?? 20_000;
+    const notes: string[] = [];
+
+    let acquired = browserMode === "force" ? undefined : await acquireStatic(url, options, maxAgeS, timeoutMs, notes);
+    if (acquired === undefined || (browserMode !== "never" && looksLikeAppShell(acquired.doc, acquired.body))) {
+        const rendered = await acquireBrowser(url, maxAgeS, timeoutMs, notes);
+        if (rendered !== undefined) {
+            acquired = rendered;
+        } else if (acquired === undefined) {
+            throw new Error("browser rendering unavailable (no Chromium in this image) and --browser force was asked for");
+        }
+    }
+    const { doc, body, finalUrl, status, contentType, source } = acquired;
+
+    if (!isHtml(contentType)) {
+        return {
+            url,
+            finalUrl,
+            status,
+            contentType,
+            markdown: neutralizeOutsideText(nonHtmlMarkdown(acquired.rawBody, contentType, notes)),
+            meta: { title: url, description: undefined, lang: undefined },
+            links: [],
+            source,
+            prunedShare: undefined,
+            notes,
+        };
+    }
+
+    // Read before pruning mutates the tree: navigation is chrome to the reader, structure to the crawler.
+    const rawMeta = metaOf(doc);
+    const meta: PageMeta = {
+        title: neutralizeOutsideText(rawMeta.title),
+        description: rawMeta.description === undefined ? undefined : neutralizeOutsideText(rawMeta.description),
+        lang: rawMeta.lang,
+    };
+    const links = extractLinks(doc, finalUrl);
+
+    let prunedShare: number | undefined;
+    if (body !== undefined && options.raw !== true) {
+        prunedShare = pruneTree(body, options.threshold === undefined ? {} : { threshold: options.threshold });
+    }
+    if (body !== undefined && options.query !== undefined && options.query !== "") {
+        notes.push(applyQueryFilter(body, options.query));
+    }
+    // Neutralized here: stdout rides the daemon's untrusted wrapper, but a saved file is read back unwrapped.
+    const markdown = body === undefined ? "" : neutralizeOutsideText(renderMarkdown(body, { baseUrl: finalUrl }));
+    return { url, finalUrl, status, contentType, markdown, meta, links, source, prunedShare, notes };
+};
+
+interface Acquired {
+    readonly doc: Document;
+    readonly body: Element | undefined;
+    readonly rawBody: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly source: "cache" | "network" | "browser";
+}
+
+const acquireStatic = async (url: string, options: FetchPageOptions, maxAgeS: number, timeoutMs: number, notes: string[]): Promise<Acquired> => {
+    const cached = maxAgeS > 0 ? await cacheRead(url, "static", maxAgeS) : undefined;
+    if (cached !== undefined) {
+        return parsed(cached.body, cached.finalUrl, cached.status, cached.contentType, "cache");
+    }
+    const fetched = await httpFetch(url, {
+        timeoutMs,
+        userAgent: USER_AGENT,
+        ...(options.maxBytes === undefined ? {} : { maxBytes: options.maxBytes }),
+    });
+    if (fetched.truncated) {
+        notes.push("byte cap hit mid-body: this page is cut short");
+    }
+    await cacheWrite({
+        url,
+        finalUrl: fetched.finalUrl,
+        status: fetched.status,
+        contentType: fetched.contentType,
+        body: fetched.body,
+        mode: "static",
+        fetchedAt: Date.now(),
+    });
+    return parsed(fetched.body, fetched.finalUrl, fetched.status, fetched.contentType, "network");
+};
+
+const acquireBrowser = async (url: string, maxAgeS: number, timeoutMs: number, notes: string[]): Promise<Acquired | undefined> => {
+    const mode: RenderMode = "browser";
+    const cached = maxAgeS > 0 ? await cacheRead(url, mode, maxAgeS) : undefined;
+    if (cached !== undefined) {
+        return parsed(cached.body, cached.finalUrl, cached.status, cached.contentType, "browser");
+    }
+    if (!(await chromiumAvailable())) {
+        notes.push("page looks JS-rendered but this image has no Chromium (browser feature pack): static HTML served best-effort");
+        return undefined;
+    }
+    const rendered = await renderPage(url, timeoutMs);
+    await cacheWrite({ url, finalUrl: rendered.finalUrl, status: 200, contentType: "text/html", body: rendered.html, mode, fetchedAt: Date.now() });
+    return parsed(rendered.html, rendered.finalUrl, 200, "text/html", "browser");
+};
+
+const parsed = (html: string, finalUrl: string, status: number, contentType: string, source: Acquired["source"]): Acquired => {
+    const doc = isHtml(contentType) ? parseHtml(html) : parseHtml("");
+    return { doc, body: bodyOf(doc), rawBody: html, finalUrl, status, contentType, source };
+};
+
+const isHtml = (contentType: string): boolean => contentType === "" || /text\/html|application\/xhtml/i.test(contentType);
+
+// JS-shell tell: almost no text next to scripts meant to produce some. Thresholded on text, not framework fingerprints,
+// which age.
+const looksLikeAppShell = (doc: Document, body: Element | undefined): boolean => {
+    if (body === undefined) {
+        return false;
+    }
+    const text = textOf(body);
+    if (text.length >= 200) {
+        return false;
+    }
+    return elementsByTag(doc, "script").length > 0;
+};
+
+const nonHtmlMarkdown = (rawBody: string, contentType: string, notes: string[]): string => {
+    if (/json/i.test(contentType)) {
+        return `\`\`\`json\n${rawBody.trim()}\n\`\`\`\n`;
+    }
+    if (/xml/i.test(contentType)) {
+        return `\`\`\`xml\n${rawBody.trim()}\n\`\`\`\n`;
+    }
+    if (/^text\//i.test(contentType)) {
+        return `${rawBody.trim()}\n`;
+    }
+    notes.push(`unsupported content type ${contentType.split(";")[0] ?? contentType}: no markdown produced`);
+    return "";
+};
+
+// Blocks a query filter may drop; headings stay put so the survivors keep their skeleton.
+const FILTERABLE = new Set(["p", "li", "pre", "table", "blockquote", "dd"]);
+
+const applyQueryFilter = (body: Element, query: string): string => {
+    const candidates: Element[] = [];
+    walk(body, (el) => {
+        if (FILTERABLE.has(el.tagName) && !hasFilterableAncestorWithin(el, body)) {
+            candidates.push(el);
+        }
+    });
+    const ranked = bm25Rank(candidates, (el) => textOf(el), query);
+    const keep = new Set(ranked.map((entry) => entry.block));
+    // Almost no match would erase the page; keep everything rather than three coincidentally matching sentences.
+    if (keep.size < Math.min(3, candidates.length)) {
+        return `query matched too little (${keep.size}/${candidates.length} blocks): kept the whole page`;
+    }
+    for (const candidate of candidates) {
+        if (!keep.has(candidate)) {
+            remove(candidate);
+        }
+    }
+    return `query kept ${keep.size}/${candidates.length} blocks`;
+};
+
+// Only top-most filterable blocks compete; a <li> inside a kept <table> cell must not be scored (and possibly removed)
+// twice.
+const hasFilterableAncestorWithin = (el: Element, root: Element): boolean => {
+    let parent = el.parentNode;
+    while (parent !== null && parent !== root) {
+        if ("tagName" in parent && FILTERABLE.has(parent.tagName)) {
+            return true;
+        }
+        parent = "parentNode" in parent && parent.parentNode !== undefined ? parent.parentNode : null;
+    }
+    return false;
+};