@intentic/webq 1.317.0

package/src/commands/cache.command.ts

@@ -0,0 +1,54 @@
+/* `webq cache`: what the shared cache holds and the one switch that empties it. */
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { toolHome } from "@intentic/agent-cli/env";
+import { buildCommand, type CommandContext } from "@stricli/core";
+import { cacheClear } from "../lib/cache.js";
+import { cacheDir } from "../lib/env.js";
+
+export const cacheCommand = buildCommand({
+    docs: { brief: "Show the shared fetch cache; --clear empties it" },
+    parameters: {
+        flags: {
+            clear: { kind: "boolean", default: false, brief: "Delete every cached fetch" },
+        },
+        positional: { kind: "tuple", parameters: [] },
+    },
+    async func(this: CommandContext, flags: { readonly clear: boolean }) {
+        if (flags.clear) {
+            await cacheClear();
+            this.process.stdout.write("cache cleared\n");
+            return;
+        }
+        const { entries, bytes } = await measure(cacheDir());
+        this.process.stdout.write(`webq home: ${toolHome("webq")}\ncache: ${entries} entries, ${(bytes / 1024 / 1024).toFixed(1)} MiB\n`);
+    },
+});
+
+const measure = async (dir: string): Promise<{ entries: number; bytes: number }> => {
+    let entries = 0;
+    let bytes = 0;
+    const walkDir = async (current: string): Promise<void> => {
+        let names: string[];
+        try {
+            names = await readdir(current);
+        } catch {
+            return;
+        }
+        for (const name of names) {
+            const path = join(current, name);
+            const info = await stat(path).catch(() => undefined);
+            if (info === undefined) {
+                continue;
+            }
+            if (info.isDirectory()) {
+                await walkDir(path);
+            } else {
+                entries += 1;
+                bytes += info.size;
+            }
+        }
+    };
+    await walkDir(dir);
+    return { entries, bytes };
+};

package/src/commands/crawl.command.ts

@@ -0,0 +1,94 @@
+/* `webq crawl <url>`: a bounded same-site crawl into a directory of markdown files plus an index. */
+import { join } from "node:path";
+import { toolOutDir } from "@intentic/agent-cli/env";
+import { countParser } from "@intentic/agent-cli/flags";
+import { buildCommand, type CommandContext } from "@stricli/core";
+import { DEFAULT_MAX_AGE_S } from "../lib/cache.js";
+import { crawl } from "../lib/crawl.js";
+import { sharedFlagParameters, type SharedFlags, urlParser } from "../lib/flags.js";
+import { slugFor } from "../lib/output.js";
+
+// Failed pages named one per line before the rest are counted; the JSON report carries all of them.
+const FAILURES_SHOWN = 10;
+
+type CrawlFlags = SharedFlags & {
+    readonly maxPages: number;
+    readonly depth: number;
+    readonly concurrency: number;
+    readonly sitemap: boolean;
+    readonly external: boolean;
+    readonly ignoreRobots: boolean;
+    readonly include?: string[];
+    readonly exclude?: string[];
+    readonly delay: number;
+};
+
+export const crawlCommand = buildCommand({
+    docs: { brief: "Bounded same-site crawl: one markdown file per page, plus an index" },
+    parameters: {
+        flags: {
+            ...sharedFlagParameters,
+            maxPages: { kind: "parsed", parse: countParser, default: "20", brief: "Hard page cap" },
+            depth: { kind: "parsed", parse: countParser, default: "2", brief: "Link hops from the start URL" },
+            concurrency: { kind: "parsed", parse: countParser, default: "4", brief: "Pages in flight at once" },
+            sitemap: { kind: "boolean", default: false, brief: "Seed the frontier from the site's sitemap" },
+            external: { kind: "boolean", default: false, brief: "Follow links off the start origin too" },
+            ignoreRobots: { kind: "boolean", default: false, brief: "Crawl paths robots.txt disallows (your responsibility)" },
+            include: { kind: "parsed", parse: String, variadic: true, optional: true, brief: "Only URLs containing this (or *-glob); repeatable" },
+            exclude: { kind: "parsed", parse: String, variadic: true, optional: true, brief: "Skip URLs containing this (or *-glob); repeatable" },
+            delay: { kind: "parsed", parse: countParser, default: "0", brief: "Milliseconds between requests (robots crawl-delay still respected)" },
+        },
+        positional: { kind: "tuple", parameters: [{ parse: urlParser, brief: "Where the crawl starts", placeholder: "url" }] },
+    },
+    async func(this: CommandContext, flags: CrawlFlags, url: string) {
+        const outDir = flags.out ?? join(toolOutDir("webq"), slugFor(url).replace(/\.md$/, ""));
+        const report = await crawl(url, {
+            outDir,
+            maxPages: flags.maxPages,
+            maxDepth: flags.depth,
+            concurrency: flags.concurrency,
+            robots: !flags.ignoreRobots,
+            sitemap: flags.sitemap,
+            external: flags.external,
+            ...(flags.include === undefined ? {} : { include: flags.include }),
+            ...(flags.exclude === undefined ? {} : { exclude: flags.exclude }),
+            delayMs: flags.delay,
+            raw: flags.raw,
+            query: flags.query,
+            browser: flags.browser,
+            maxAgeS: flags.fresh ? 0 : (flags.maxAge ?? DEFAULT_MAX_AGE_S),
+            timeoutMs: flags.timeout * 1000,
+            threshold: flags.threshold,
+        });
+        if (flags.json) {
+            this.process.stdout.write(`${JSON.stringify(report)}\n`);
+            process.exitCode = report.pages.length > 0 ? 0 : 1;
+            return;
+        }
+        const totalTokens = report.pages.reduce((sum, page) => sum + page.tokens, 0);
+        const skipLine = Object.entries(report.skipped)
+            .filter(([, count]) => count > 0)
+            .map(([reason, count]) => `${reason} ${count}`)
+            .join(", ");
+        this.process.stdout.write(
+            `webq crawl: ${url} · ${report.pages.length} pages · ${totalTokens} tokens${skipLine === "" ? "" : ` · skipped: ${skipLine}`}\n`,
+        );
+        for (const { url: failed, reason } of report.failures.slice(0, FAILURES_SHOWN)) {
+            this.process.stdout.write(`failed: ${failed}: ${reason}\n`);
+        }
+        if (report.failures.length > FAILURES_SHOWN) {
+            this.process.stdout.write(`…and ${report.failures.length - FAILURES_SHOWN} more failed pages (--json lists every one)\n`);
+        }
+        if (report.sitemapCapped) {
+            this.process.stdout.write("note: sitemap larger than the seed cap, frontier seeded from a prefix of it\n");
+        }
+        this.process.stdout.write(`index: ${report.indexMarkdown}\n`);
+        for (const page of report.pages) {
+            this.process.stdout.write(`  ${page.tokens}t d${page.depth} ${page.url} → ${page.path}\n`);
+            for (const note of page.notes) {
+                this.process.stdout.write(`    note: ${note}\n`);
+            }
+        }
+        process.exitCode = report.pages.length > 0 ? 0 : 1;
+    },
+});

package/src/commands/fetch.command.ts

@@ -0,0 +1,61 @@
+/* `webq fetch <url>`: one page as markdown — a capsule line saying what happened, the content up to a token budget. */
+import { toolOutDir } from "@intentic/agent-cli/env";
+import { countParser } from "@intentic/agent-cli/flags";
+import { capsule, clip } from "@intentic/agent-cli/output";
+import { buildCommand, type CommandContext } from "@stricli/core";
+import { closeBrowser } from "../lib/browser.js";
+import { DEFAULT_MAX_AGE_S } from "../lib/cache.js";
+import { sharedFlagParameters, type SharedFlags, urlParser } from "../lib/flags.js";
+import { savePage } from "../lib/output.js";
+import { fetchPage, type PageResult } from "../lib/page.js";
+
+type FetchFlags = SharedFlags & { readonly budget: number };
+
+export const fetchCommand = buildCommand({
+    docs: { brief: "One URL as clean markdown: budgeted on stdout, whole in a saved file" },
+    parameters: {
+        flags: {
+            ...sharedFlagParameters,
+            budget: { kind: "parsed", parse: countParser, default: "4000", brief: "Max stdout tokens; 0 prints only the capsule" },
+        },
+        positional: { kind: "tuple", parameters: [{ parse: urlParser, brief: "The page to fetch", placeholder: "url" }] },
+    },
+    async func(this: CommandContext, flags: FetchFlags, url: string) {
+        try {
+            const page = await fetchPage(url, {
+                raw: flags.raw,
+                query: flags.query,
+                browser: flags.browser,
+                maxAgeS: flags.fresh ? 0 : (flags.maxAge ?? DEFAULT_MAX_AGE_S),
+                timeoutMs: flags.timeout * 1000,
+                threshold: flags.threshold,
+            });
+            const saved = await savePage(flags.out ?? toolOutDir("webq"), page, new Date());
+            if (flags.json) {
+                this.process.stdout.write(
+                    `${JSON.stringify({ url, finalUrl: page.finalUrl, status: page.status, title: page.meta.title, tokens: saved.tokens, path: saved.path, source: page.source, notes: page.notes })}\n`,
+                );
+            } else {
+                this.process.stdout.write(capsule("webq", capsuleFields(page, saved.tokens, flags.raw), page.notes));
+                this.process.stdout.write(`saved: ${saved.path}\n`);
+                if (flags.budget > 0 && page.markdown !== "") {
+                    this.process.stdout.write("---\n");
+                    this.process.stdout.write(clip(page.markdown, flags.budget, saved.path, "page"));
+                }
+            }
+            process.exitCode = page.status >= 400 ? 1 : 0;
+        } finally {
+            await closeBrowser();
+        }
+    },
+});
+
+// What a fetch has to say before its content: the page's name, where it really came from (redirects included),
+// what it will cost to read, how much chrome was pruned, and whether the bytes were cached, fetched or rendered.
+const capsuleFields = (page: PageResult, tokens: number, raw: boolean): string[] => [
+    page.meta.title === "" ? page.finalUrl : page.meta.title,
+    page.finalUrl,
+    `${tokens} tokens`,
+    raw ? "raw" : page.prunedShare === undefined ? "fit" : `fit (pruned ${Math.round(page.prunedShare * 100)}% of text mass)`,
+    page.source,
+];

package/src/lib/bm25.test.ts

@@ -0,0 +1,33 @@
+import { bm25Rank, tokenize } from "./bm25.js";
+
+describe("tokenize", () => {
+    it("lowercases, splits on non-word runs and drops one-char tokens", () => {
+        expect(tokenize("The API’s rate-limit: 5/s")).toEqual(["the", "api", "rate", "limit"]);
+    });
+
+    it("folds English plurals so a query's form matches the page's", () => {
+        expect(tokenize("webhooks retries pages boxes patches class press")).toEqual(["webhook", "retry", "page", "box", "patch", "class", "press"]);
+    });
+});
+
+describe("bm25Rank", () => {
+    const blocks = [
+        "installation guide for the command line tool",
+        "webhook delivery retries use exponential backoff with jitter",
+        "pricing tiers and billing frequently asked questions",
+        "configuring webhook endpoints and secrets",
+    ];
+
+    it("ranks the block about the query first and drops zero-signal blocks", () => {
+        const ranked = bm25Rank(blocks, (block) => block, "webhook retry backoff");
+        expect(ranked[0]?.block).toBe(blocks[1]);
+        const kept = ranked.map((entry) => entry.block);
+        expect(kept).not.toContain(blocks[0]);
+        expect(kept).not.toContain(blocks[2]);
+    });
+
+    it("returns nothing for an empty or unmatched query", () => {
+        expect(bm25Rank(blocks, (block) => block, "")).toEqual([]);
+        expect(bm25Rank(blocks, (block) => block, "zebra")).toEqual([]);
+    });
+});

package/src/lib/bm25.ts

@@ -0,0 +1,70 @@
+// Scores blocks against the query with BM25, keeping only ones with signal, in document order; headings are always
+// kept. Follows crawl4ai's BM25ContentFilter (Apache-2.0, https://github.com/unclecode/crawl4ai). Plain BM25, k1=1.2
+// b=0.75, no stemming.
+
+export interface ScoredBlock<T> {
+    readonly block: T;
+    readonly score: number;
+}
+
+export const tokenize = (text: string): string[] =>
+    text
+        .toLowerCase()
+        .split(/[^\p{L}\p{N}]+/u)
+        .filter((token) => token.length >= 2)
+        .map(foldPlural);
+
+// Folds English plurals (webhooks to webhook, retries to retry) instead of a real stemmer, which would add a dependency
+// and a language commitment.
+const foldPlural = (token: string): string => {
+    if (token.length <= 3) {
+        return token;
+    }
+    if (token.endsWith("ies")) {
+        return `${token.slice(0, -3)}y`;
+    }
+    // -es only marks the plural after a sibilant or o (boxes, patches, heroes); "pages" is page + s.
+    if (/(?:[xz]|ch|sh|o)es$/.test(token)) {
+        return token.slice(0, -2);
+    }
+    return token.endsWith("s") && !token.endsWith("ss") ? token.slice(0, -1) : token;
+};
+
+const K1 = 1.2;
+const B = 0.75;
+
+/**
+ * Ranks `blocks` against `query`; returns every block with a positive score, highest first. Caller decides how many to
+ * keep.
+ */
+export const bm25Rank = <T>(blocks: readonly T[], textOf: (block: T) => string, query: string): ScoredBlock<T>[] => {
+    const queryTerms = [...new Set(tokenize(query))];
+    if (queryTerms.length === 0 || blocks.length === 0) {
+        return [];
+    }
+    const docs = blocks.map((block) => tokenize(textOf(block)));
+    const avgLen = docs.reduce((sum, doc) => sum + doc.length, 0) / docs.length || 1;
+    const docFrequency = new Map<string, number>();
+    for (const term of queryTerms) {
+        docFrequency.set(term, docs.filter((doc) => doc.includes(term)).length);
+    }
+    const scored = blocks.map((block, index) => {
+        const doc = docs[index] ?? [];
+        const counts = new Map<string, number>();
+        for (const token of doc) {
+            counts.set(token, (counts.get(token) ?? 0) + 1);
+        }
+        let score = 0;
+        for (const term of queryTerms) {
+            const df = docFrequency.get(term) ?? 0;
+            if (df === 0) {
+                continue;
+            }
+            const idf = Math.log((docs.length - df + 0.5) / (df + 0.5) + 1);
+            const tf = counts.get(term) ?? 0;
+            score += idf * ((tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (doc.length / avgLen))));
+        }
+        return { block, score };
+    });
+    return scored.filter((entry) => entry.score > 0).toSorted((a, b) => b.score - a.score);
+};

package/src/lib/browser.ts

@@ -0,0 +1,42 @@
+// JS-rendering fallback via the image's own headless Chromium, a feature pack that may not be present; existsSync is
+// what confirms it, not executablePath() alone. One browser per process; images, media and fonts are aborted since only
+// markdown is read from the page.
+import { existsSync } from "node:fs";
+import type { Browser } from "playwright";
+
+export interface RenderedPage {
+    readonly finalUrl: string;
+    readonly html: string;
+}
+
+let browser: Browser | undefined;
+
+export const chromiumAvailable = async (): Promise<boolean> => {
+    const { chromium } = await import("playwright");
+    return existsSync(chromium.executablePath());
+};
+
+export const renderPage = async (url: string, timeoutMs: number): Promise<RenderedPage> => {
+    const { chromium } = await import("playwright");
+    // executablePath is explicit: a bare launch reaches for the headless-shell build, absent from this image.
+    browser ??= await chromium.launch({ executablePath: chromium.executablePath(), headless: true, args: ["--no-sandbox"] });
+    const context = await browser.newContext({ javaScriptEnabled: true, viewport: { width: 1280, height: 900 } });
+    try {
+        await context.route("**/*", (route) => {
+            const kind = route.request().resourceType();
+            return kind === "image" || kind === "media" || kind === "font" ? route.abort() : route.continue();
+        });
+        const page = await context.newPage();
+        await page.goto(url, { waitUntil: "domcontentloaded", timeout: timeoutMs });
+        // Network-idle is a bonus with its own short deadline; a long-polling page must not hold the command hostage.
+        await page.waitForLoadState("networkidle", { timeout: Math.min(5_000, timeoutMs) }).catch(() => undefined);
+        return { finalUrl: page.url(), html: await page.content() };
+    } finally {
+        await context.close();
+    }
+};
+
+export const closeBrowser = async (): Promise<void> => {
+    await browser?.close();
+    browser = undefined;
+};

package/src/lib/cache.ts

@@ -0,0 +1,44 @@
+/* Cache raw HTML by URL and render mode; transforms never share derived markdown. */
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { cacheDir } from "./env.js";
+
+export type RenderMode = "static" | "browser";
+
+export interface CacheEntry {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly body: string;
+    readonly mode: RenderMode;
+    readonly fetchedAt: number;
+}
+
+export const DEFAULT_MAX_AGE_S = 900;
+
+const entryPath = (url: string, mode: RenderMode): string => {
+    const hash = createHash("sha256").update(`${mode}\n${url}`).digest("hex");
+    return join(cacheDir(), hash.slice(0, 2), `${hash}.json`);
+};
+
+export const cacheRead = async (url: string, mode: RenderMode, maxAgeS: number): Promise<CacheEntry | undefined> => {
+    try {
+        const raw = await readFile(entryPath(url, mode), "utf8");
+        const entry = JSON.parse(raw) as CacheEntry;
+        return Date.now() - entry.fetchedAt <= maxAgeS * 1000 ? entry : undefined;
+    } catch {
+        return undefined;
+    }
+};
+
+export const cacheWrite = async (entry: CacheEntry): Promise<void> => {
+    const path = entryPath(entry.url, entry.mode);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, JSON.stringify(entry));
+};
+
+export const cacheClear = async (): Promise<void> => {
+    await rm(cacheDir(), { recursive: true, force: true });
+};

package/src/lib/crawl.ts

@@ -0,0 +1,247 @@
+// Bounded crawl: BFS by default, best-first once a query scores anchor/URL word overlap (best-first and pruning follow
+// crawl4ai, Apache-2.0). Caps (max-pages, depth, robots.txt) are a contract: everything skipped returns as a per-reason
+// count. Shares `webq fetch`'s page pipeline and cache.
+import { setTimeout as sleep } from "node:timers/promises";
+import { errorMessage } from "@intentic/base/errors";
+import { tokenize } from "./bm25.js";
+import { closeBrowser } from "./browser.js";
+import { httpFetch } from "./http.js";
+import { normalizeUrl, sameOrigin } from "./links.js";
+import { saveIndex, savePage, type IndexEntry } from "./output.js";
+import { fetchPage, type FetchPageOptions } from "./page.js";
+import { EVERYTHING_ALLOWED, isAllowed, parseRobots, type RobotsRules } from "./robots.js";
+import { fetchSitemapUrls } from "./sitemap.js";
+
+export interface CrawlOptions extends FetchPageOptions {
+    readonly maxPages?: number;
+    readonly maxDepth?: number;
+    readonly concurrency?: number;
+    /** Off only when the caller explicitly takes responsibility (--ignore-robots). */
+    readonly robots?: boolean;
+    readonly sitemap?: boolean;
+    readonly external?: boolean;
+    /** Substring (or *-glob) filters over the URL; include must match when given, exclude must not. */
+    readonly include?: string[];
+    readonly exclude?: string[];
+    readonly delayMs?: number;
+    readonly outDir: string;
+}
+
+export interface CrawlPageReport extends IndexEntry {
+    readonly source: string;
+    readonly notes: string[];
+}
+
+export interface CrawlReport {
+    readonly pages: CrawlPageReport[];
+    readonly skipped: Record<string, number>;
+    // Each page counted under `skipped.errors`, with what failed: a count alone cannot tell a flaky site from a full disk.
+    readonly failures: ReadonlyArray<{ readonly url: string; readonly reason: string }>;
+    readonly indexMarkdown: string;
+    readonly indexJson: string;
+    readonly sitemapCapped: boolean;
+}
+
+interface Candidate {
+    readonly url: string;
+    depth: number;
+    score: number;
+    readonly order: number;
+}
+
+const MAX_CRAWL_DELAY_MS = 10_000;
+
+export const crawl = async (startUrl: string, options: CrawlOptions): Promise<CrawlReport> => {
+    const maxPages = options.maxPages ?? 20;
+    const maxDepth = options.maxDepth ?? 2;
+    const concurrency = Math.max(1, options.concurrency ?? 4);
+    const queryTerms = options.query === undefined ? [] : [...new Set(tokenize(options.query))];
+    const skipped = { robots: 0, offsite: 0, filtered: 0, "beyond-depth": 0, "beyond-cap": 0, "http-errors": 0, errors: 0 };
+
+    const robots = options.robots === false ? EVERYTHING_ALLOWED : await loadRobots(startUrl, options.timeoutMs);
+    const delayMs = Math.max(options.delayMs ?? 0, Math.min((robots.crawlDelayS ?? 0) * 1000, MAX_CRAWL_DELAY_MS));
+
+    const taken = new Set<string>();
+    const pending = new Map<string, Candidate>();
+    let order = 0;
+    const admit = (raw: string | undefined, depth: number, anchorText: string): void => {
+        const url = normalizeUrl(raw);
+        if (url === undefined || taken.has(url)) {
+            return;
+        }
+        const queued = pending.get(url);
+        if (queued !== undefined) {
+            // A URL reached through a better link (real anchor text) can promote a candidate the sitemap seeded blind.
+            const score = scoreCandidate(url, anchorText, depth, queryTerms);
+            if (score > queued.score) {
+                queued.score = score;
+                queued.depth = Math.min(queued.depth, depth);
+            }
+            return;
+        }
+        if (options.external !== true && !sameOrigin(url, startUrl)) {
+            skipped.offsite += 1;
+            return;
+        }
+        if (!passesFilters(url, options.include, options.exclude)) {
+            skipped.filtered += 1;
+            return;
+        }
+        if (options.robots !== false && !isAllowed(robots, pathOf(url))) {
+            skipped.robots += 1;
+            return;
+        }
+        if (depth > maxDepth) {
+            skipped["beyond-depth"] += 1;
+            return;
+        }
+        pending.set(url, { url, depth, score: scoreCandidate(url, anchorText, depth, queryTerms), order: order++ });
+    };
+
+    admit(startUrl, 0, "");
+    let sitemapCapped = false;
+    if (options.sitemap === true) {
+        const seed = await fetchSitemapUrls(startUrl, robots.sitemaps, sitemapFetcher(options.timeoutMs));
+        sitemapCapped = seed.capped;
+        for (const url of seed.urls) {
+            admit(url, 1, "");
+        }
+    }
+
+    const pages: CrawlPageReport[] = [];
+    const failures: { url: string; reason: string }[] = [];
+    const fetchedAt = new Date();
+    let inFlight = 0;
+    let landed = 0;
+
+    // Works the best candidate until the frontier is idle or the cap is hit; the rest then count as beyond-cap.
+    await new Promise<void>((resolve) => {
+        const pump = (): void => {
+            if (landed >= maxPages || (pending.size === 0 && inFlight === 0)) {
+                if (inFlight === 0) {
+                    skipped["beyond-cap"] += landed >= maxPages ? pending.size : 0;
+                    resolve();
+                }
+                return;
+            }
+            // Start query crawls with one page so anchor scores exist before fan-out.
+            const width = queryTerms.length > 0 && landed === 0 ? 1 : concurrency;
+            while (inFlight < width && pending.size > 0 && landed + inFlight < maxPages) {
+                const next = takeBest(pending);
+                taken.add(next.url);
+                inFlight += 1;
+                void workOne(next).finally(() => {
+                    inFlight -= 1;
+                    pump();
+                });
+            }
+        };
+        const workOne = async (candidate: Candidate): Promise<void> => {
+            try {
+                if (delayMs > 0) {
+                    await sleep(delayMs);
+                }
+                const page = await fetchPage(candidate.url, options);
+                // A 4xx/5xx is a broken link, not a page: counted but never written to the index.
+                if (page.status >= 400) {
+                    skipped["http-errors"] += 1;
+                    return;
+                }
+                const saved = await savePage(options.outDir, page, fetchedAt);
+                landed += 1;
+                pages.push({
+                    url: candidate.url,
+                    title: page.meta.title,
+                    path: saved.path,
+                    tokens: saved.tokens,
+                    depth: candidate.depth,
+                    source: page.source,
+                    notes: page.notes,
+                });
+                for (const link of page.links) {
+                    admit(link.url, candidate.depth + 1, link.text);
+                }
+            } catch (error) {
+                skipped.errors += 1;
+                failures.push({ url: candidate.url, reason: errorMessage(error).split("\n")[0] ?? "" });
+            }
+        };
+        pump();
+    });
+
+    await closeBrowser();
+    const index = await saveIndex(options.outDir, startUrl, pages, skipped);
+    return { pages, skipped, failures, indexMarkdown: index.markdownPath, indexJson: index.jsonPath, sitemapCapped };
+};
+
+const takeBest = (pending: Map<string, Candidate>): Candidate => {
+    let best: Candidate | undefined;
+    for (const candidate of pending.values()) {
+        if (best === undefined || candidate.score > best.score || (candidate.score === best.score && candidate.order < best.order)) {
+            best = candidate;
+        }
+    }
+    pending.delete((best as Candidate).url);
+    return best as Candidate;
+};
+
+// Without a query this is plain BFS (depth only). With one, a link whose anchor text or URL echoes the query's words is
+// visited before its siblings.
+const scoreCandidate = (url: string, anchorText: string, depth: number, queryTerms: string[]): number => {
+    if (queryTerms.length === 0) {
+        return -depth;
+    }
+    const haystack = new Set(tokenize(`${anchorText} ${pathOf(url).replaceAll(/[/\-_.]/g, " ")}`));
+    const overlap = queryTerms.filter((term) => haystack.has(term)).length / queryTerms.length;
+    return overlap - depth * 0.05;
+};
+
+const passesFilters = (url: string, include: string[] | undefined, exclude: string[] | undefined): boolean => {
+    const test = (pattern: string): boolean =>
+        pattern.includes("*")
+            ? new RegExp(pattern.replaceAll(/[.+?^${}()|[\]\\]/g, "\\$&").replaceAll("\\*", ".*")).test(url)
+            : url.includes(pattern);
+    if (include !== undefined && include.length > 0 && !include.some(test)) {
+        return false;
+    }
+    return !(exclude ?? []).some(test);
+};
+
+const pathOf = (url: string): string => {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.pathname}${parsed.search}`;
+    } catch {
+        return url;
+    }
+};
+
+const loadRobots = async (startUrl: string, timeoutMs: number | undefined): Promise<RobotsRules> => {
+    try {
+        const origin = new URL(startUrl).origin;
+        const response = await httpFetch(`${origin}/robots.txt`, { timeoutMs: Math.min(timeoutMs ?? 10_000, 10_000) });
+        // A missing robots.txt allows everything; an unreachable site fails loudly on the first page fetch instead.
+        return response.status >= 200 && response.status < 300 ? parseRobots(response.body) : EVERYTHING_ALLOWED;
+    } catch {
+        return EVERYTHING_ALLOWED;
+    }
+};
+
+const sitemapFetcher =
+    (timeoutMs: number | undefined) =>
+    async (url: string): Promise<{ body: string; bytes?: Uint8Array } | undefined> => {
+        try {
+            if (url.endsWith(".gz")) {
+                const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs ?? 10_000) });
+                if (!response.ok) {
+                    return undefined;
+                }
+                const bytes = new Uint8Array(await response.arrayBuffer());
+                return { body: "", bytes };
+            }
+            const response = await httpFetch(url, { timeoutMs: timeoutMs ?? 10_000 });
+            return response.status >= 200 && response.status < 300 ? { body: response.body } : undefined;
+        } catch {
+            return undefined;
+        }
+    };