@mxalbert/context-mode 2.0.0

package/build/exit-classify.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Classify non-zero exit codes for ctx_execute / ctx_execute_file.
+ *
+ * Shell commands like `grep` exit 1 for "no matches" — not a real error.
+ * We treat exit code 1 as a soft failure when:
+ *   - language is "shell"
+ *   - exit code is exactly 1
+ *   - stdout has non-whitespace content
+ */
+export interface ExitClassification {
+    isError: boolean;
+    output: string;
+}
+export declare function classifyNonZeroExit(params: {
+    language: string;
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+}): ExitClassification;

package/build/exit-classify.js

@@ -0,0 +1,12 @@
+export function classifyNonZeroExit(params) {
+    const { language, exitCode, stdout, stderr } = params;
+    const isSoftFail = language === "shell" &&
+        exitCode === 1 &&
+        stdout.trim().length > 0;
+    return {
+        isError: !isSoftFail,
+        output: isSoftFail
+            ? stdout
+            : `Exit code: ${exitCode}\n\nstdout:\n${stdout}\n\nstderr:\n${stderr}`,
+    };
+}

package/build/fetch/blocks.d.ts

@@ -0,0 +1,145 @@
+/**
+ * blocks — block-level template/content extraction for fetched documents.
+ *
+ * WHY THIS EXISTS
+ * ───────────────
+ * A format converter answers "what format". It can never answer "which part of
+ * the page". Turndown (and Cloudflare's `toMarkdown`, and every other
+ * converter) transliterates the whole document, chrome included: nav lists,
+ * sidebars, footers, cookie banners. Bolting a link-density or byte threshold
+ * onto the converter is a symptom detector — it guesses from the shape of a
+ * single page, and the right threshold differs per host. Measured 2026-08-12 on
+ * the shipped converter: link-only lines are 28.3% of the Stripe API reference
+ * and 0.3% of the Resend send-email page. No single number separates those.
+ *
+ * The signal that does work needs no threshold at all:
+ *
+ *   CHROME IS WHAT REPEATS ACROSS PAGES OF THE SAME HOST.
+ *   CONTENT IS WHAT DOES NOT.
+ *
+ * Every page of docs.stripe.com carries the same nav; only the article differs.
+ * So a block is classified `template` only when that exact block was already
+ * seen on a DIFFERENT page of the same host. The rule never guesses from the
+ * shape of one page, which makes it lossless by construction.
+ *
+ * NO DATA LOSS MEANS LABEL, NEVER DROP
+ * ────────────────────────────────────
+ * This module only ever LABELS. It does not delete, trim, summarise or
+ * truncate. `splitBlocks` is exactly reversible — see `reassemble`, and the
+ * invariant test that asserts `reassemble(splitBlocks(x)) === x` byte for byte.
+ * The caller stores every block; only `content` blocks reach the search index.
+ *
+ * No regular expressions anywhere (repo-wide ban): all scanning is explicit
+ * character and prefix work.
+ */
+export interface Block {
+    /** 0-based position in the document. */
+    ordinal: number;
+    /**
+     * The block's source bytes VERBATIM, including the newlines and blank-line
+     * separators that followed it. Concatenating `raw` over all blocks in
+     * ordinal order reproduces the input document exactly.
+     */
+    raw: string;
+    /** The block's visible text: `raw` with leading/trailing whitespace removed. */
+    text: string;
+    /** sha256 (full digest, never shortened) of the normalised text. */
+    hash: string;
+}
+export type BlockKind = "content" | "template";
+export interface ClassifiedBlock extends Block {
+    kind: BlockKind;
+}
+export interface ClassifyResult {
+    blocks: ClassifiedBlock[];
+    /**
+     * True when this host had no other recorded page at classification time, so
+     * every block was admitted as `content` on no evidence. The caller must
+     * re-run classification for this page once a second page of the host lands.
+     */
+    provisional: boolean;
+    /**
+     * True when every block of this page was already seen on other pages of the
+     * host — the fetch returned the site shell, not this page. The caller must
+     * refuse rather than index it.
+     */
+    allTemplate: boolean;
+    contentBytes: number;
+    templateBytes: number;
+}
+/**
+ * Split converted markdown into blocks.
+ *
+ * A block boundary is a blank line, or a heading line, outside a fenced code
+ * block. Blank lines stay attached to the block they follow, so every input
+ * byte belongs to exactly one block and `reassemble` is exact.
+ *
+ * Fenced code is never split: a nav list and a JSON example must each be a
+ * single unit, or a code sample that happens to share one line with another
+ * page would be classified independently of its surroundings.
+ */
+export declare function splitBlocks(markdown: string): Block[];
+/**
+ * Exact inverse of `splitBlocks`. `reassemble(splitBlocks(x)) === x` for every
+ * input — this is the formal statement of "nothing was lost".
+ */
+export declare function reassemble(blocks: Array<{
+    raw: string;
+}>): string;
+/**
+ * Normalise a block for cross-page comparison: lower-cased, with every run of
+ * whitespace collapsed to a single space. Two renderings of the same nav that
+ * differ only in indentation or line wrapping must hash identically, or the
+ * repeat is missed and chrome leaks into the index.
+ *
+ * Deliberately conservative: no punctuation stripping, no token dropping. A
+ * looser normaliser risks collapsing two DIFFERENT blocks to one hash, which
+ * would demote real content. Under-matching leaks chrome (recoverable, and
+ * visible); over-matching hides content (a silent loss, which is forbidden).
+ */
+export declare function normalizeBlockText(text: string): string;
+/** Full sha256 hex digest of the normalised block. Never shortened. */
+export declare function hashBlockText(text: string): string;
+export interface ClassifyOptions {
+    blocks: Block[];
+    /**
+     * Number of DISTINCT other pages of this host on which the given block hash
+     * has been seen. Must exclude the page being classified, or a page compared
+     * against itself marks its own article as template.
+     */
+    otherPageCount: (hash: string) => number;
+    /** Distinct pages already recorded for this host, excluding the current one. */
+    hostPageCount: number;
+    /**
+     * Set when the document came straight from the site's own machine-readable
+     * endpoint. Such a document was authored for machines and carries no chrome
+     * to classify, so every block is content and the result is never provisional.
+     */
+    authored?: boolean;
+}
+/**
+ * Classify every block of one page.
+ *
+ * COLD START — the honest hard case. The first page of a host has no
+ * comparison set, so there is no evidence on which to call anything template.
+ * The decision here is to admit EVERY block as `content`, mark the page
+ * `provisional`, and re-run the moment a second page of that host arrives.
+ *
+ * The alternative — guessing per-block from link density, position or
+ * container on page one — is rejected on the measurements above: the correct
+ * threshold is 28.3% on Stripe and 0.3% on Resend, so any fixed value demotes
+ * real content on one of them. Over-indexing on page one is fully recoverable
+ * (the re-run removes the chrome, and the page's own bytes were never
+ * discarded); under-indexing is a silent loss with nothing left to detect it.
+ * Never demote on evidence we do not have.
+ */
+export declare function classifyBlocks(opts: ClassifyOptions): ClassifyResult;
+/**
+ * The text that goes to the search index: the `content` blocks, in document
+ * order, joined by their own separators. Template blocks are NOT deleted —
+ * the caller has already stored every block verbatim; they are simply left
+ * out of the index.
+ */
+export declare function contentText(blocks: ClassifiedBlock[]): string;
+/** The template stream, in document order. Stored and retrievable, never indexed. */
+export declare function templateText(blocks: ClassifiedBlock[]): string;

package/build/fetch/blocks.js

@@ -0,0 +1,244 @@
+/**
+ * blocks — block-level template/content extraction for fetched documents.
+ *
+ * WHY THIS EXISTS
+ * ───────────────
+ * A format converter answers "what format". It can never answer "which part of
+ * the page". Turndown (and Cloudflare's `toMarkdown`, and every other
+ * converter) transliterates the whole document, chrome included: nav lists,
+ * sidebars, footers, cookie banners. Bolting a link-density or byte threshold
+ * onto the converter is a symptom detector — it guesses from the shape of a
+ * single page, and the right threshold differs per host. Measured 2026-08-12 on
+ * the shipped converter: link-only lines are 28.3% of the Stripe API reference
+ * and 0.3% of the Resend send-email page. No single number separates those.
+ *
+ * The signal that does work needs no threshold at all:
+ *
+ *   CHROME IS WHAT REPEATS ACROSS PAGES OF THE SAME HOST.
+ *   CONTENT IS WHAT DOES NOT.
+ *
+ * Every page of docs.stripe.com carries the same nav; only the article differs.
+ * So a block is classified `template` only when that exact block was already
+ * seen on a DIFFERENT page of the same host. The rule never guesses from the
+ * shape of one page, which makes it lossless by construction.
+ *
+ * NO DATA LOSS MEANS LABEL, NEVER DROP
+ * ────────────────────────────────────
+ * This module only ever LABELS. It does not delete, trim, summarise or
+ * truncate. `splitBlocks` is exactly reversible — see `reassemble`, and the
+ * invariant test that asserts `reassemble(splitBlocks(x)) === x` byte for byte.
+ * The caller stores every block; only `content` blocks reach the search index.
+ *
+ * No regular expressions anywhere (repo-wide ban): all scanning is explicit
+ * character and prefix work.
+ */
+import { createHash } from "node:crypto";
+// ─────────────────────────────────────────────────────────
+// Splitting
+// ─────────────────────────────────────────────────────────
+/** True when `s` starts with `prefix`. Explicit so no regex is involved. */
+function startsWith(s, prefix) {
+    return s.lastIndexOf(prefix, 0) === 0;
+}
+function isSpace(ch) {
+    return ch === " " || ch === "\t" || ch === "\n" || ch === "\r" || ch === "\f" || ch === "\v";
+}
+/** Trim ASCII whitespace from both ends without a regex. */
+function trimEdges(s) {
+    let start = 0;
+    let end = s.length;
+    while (start < end && isSpace(s.charAt(start)))
+        start++;
+    while (end > start && isSpace(s.charAt(end - 1)))
+        end--;
+    return start === 0 && end === s.length ? s : s.substring(start, end);
+}
+/**
+ * Split a document into lines, KEEPING each line's terminator on the line.
+ * Concatenating the result reproduces the input exactly — the property the
+ * whole lossless story rests on.
+ */
+function splitLinesKeepEnds(text) {
+    const lines = [];
+    let lineStart = 0;
+    for (let i = 0; i < text.length; i++) {
+        if (text.charAt(i) === "\n") {
+            lines.push(text.substring(lineStart, i + 1));
+            lineStart = i + 1;
+        }
+    }
+    if (lineStart < text.length)
+        lines.push(text.substring(lineStart));
+    return lines;
+}
+/**
+ * Split converted markdown into blocks.
+ *
+ * A block boundary is a blank line, or a heading line, outside a fenced code
+ * block. Blank lines stay attached to the block they follow, so every input
+ * byte belongs to exactly one block and `reassemble` is exact.
+ *
+ * Fenced code is never split: a nav list and a JSON example must each be a
+ * single unit, or a code sample that happens to share one line with another
+ * page would be classified independently of its surroundings.
+ */
+export function splitBlocks(markdown) {
+    const blocks = [];
+    if (markdown.length === 0)
+        return blocks;
+    const lines = splitLinesKeepEnds(markdown);
+    let rawParts = [];
+    let hasText = false;
+    let inFence = false;
+    let pendingBreak = false;
+    const flush = () => {
+        if (rawParts.length === 0)
+            return;
+        const raw = rawParts.join("");
+        const text = trimEdges(raw);
+        blocks.push({ ordinal: blocks.length, raw, text, hash: hashBlockText(text) });
+        rawParts = [];
+        hasText = false;
+    };
+    for (const line of lines) {
+        const trimmed = trimEdges(line);
+        const isFenceMarker = startsWith(trimmed, "```") || startsWith(trimmed, "~~~");
+        const blank = !inFence && trimmed.length === 0;
+        const heading = !inFence && !isFenceMarker && startsWith(trimmed, "#");
+        // A new block starts at the first non-blank line after a blank run, or at
+        // a heading — but only if the block being built already carries text, so
+        // leading separators never produce an empty block.
+        if (!blank && (pendingBreak || heading) && hasText) {
+            flush();
+            pendingBreak = false;
+        }
+        if (isFenceMarker)
+            inFence = !inFence;
+        rawParts.push(line);
+        if (!blank)
+            hasText = true;
+        if (blank)
+            pendingBreak = true;
+    }
+    flush();
+    return blocks;
+}
+/**
+ * Exact inverse of `splitBlocks`. `reassemble(splitBlocks(x)) === x` for every
+ * input — this is the formal statement of "nothing was lost".
+ */
+export function reassemble(blocks) {
+    let out = "";
+    for (const b of blocks)
+        out += b.raw;
+    return out;
+}
+// ─────────────────────────────────────────────────────────
+// Hashing
+// ─────────────────────────────────────────────────────────
+/**
+ * Normalise a block for cross-page comparison: lower-cased, with every run of
+ * whitespace collapsed to a single space. Two renderings of the same nav that
+ * differ only in indentation or line wrapping must hash identically, or the
+ * repeat is missed and chrome leaks into the index.
+ *
+ * Deliberately conservative: no punctuation stripping, no token dropping. A
+ * looser normaliser risks collapsing two DIFFERENT blocks to one hash, which
+ * would demote real content. Under-matching leaks chrome (recoverable, and
+ * visible); over-matching hides content (a silent loss, which is forbidden).
+ */
+export function normalizeBlockText(text) {
+    let out = "";
+    let inRun = false;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text.charAt(i);
+        if (isSpace(ch)) {
+            inRun = true;
+            continue;
+        }
+        if (inRun && out.length > 0)
+            out += " ";
+        inRun = false;
+        out += ch;
+    }
+    return out.toLowerCase();
+}
+/** Full sha256 hex digest of the normalised block. Never shortened. */
+export function hashBlockText(text) {
+    return createHash("sha256").update(normalizeBlockText(text), "utf-8").digest("hex");
+}
+/**
+ * Classify every block of one page.
+ *
+ * COLD START — the honest hard case. The first page of a host has no
+ * comparison set, so there is no evidence on which to call anything template.
+ * The decision here is to admit EVERY block as `content`, mark the page
+ * `provisional`, and re-run the moment a second page of that host arrives.
+ *
+ * The alternative — guessing per-block from link density, position or
+ * container on page one — is rejected on the measurements above: the correct
+ * threshold is 28.3% on Stripe and 0.3% on Resend, so any fixed value demotes
+ * real content on one of them. Over-indexing on page one is fully recoverable
+ * (the re-run removes the chrome, and the page's own bytes were never
+ * discarded); under-indexing is a silent loss with nothing left to detect it.
+ * Never demote on evidence we do not have.
+ */
+export function classifyBlocks(opts) {
+    const { blocks, otherPageCount, hostPageCount, authored } = opts;
+    const coldStart = !authored && hostPageCount < 1;
+    const classified = [];
+    let contentBytes = 0;
+    let templateBytes = 0;
+    let sawContent = false;
+    let sawTextBlock = false;
+    for (const b of blocks) {
+        // A whitespace-only separator carries no information either way; keep it
+        // as content so reassembly of the content stream stays readable.
+        const textual = b.text.length > 0;
+        if (textual)
+            sawTextBlock = true;
+        const repeated = textual && !authored && !coldStart && otherPageCount(b.hash) >= 1;
+        const kind = repeated ? "template" : "content";
+        if (kind === "content") {
+            if (textual)
+                sawContent = true;
+            contentBytes += Buffer.byteLength(b.text, "utf-8");
+        }
+        else {
+            templateBytes += Buffer.byteLength(b.text, "utf-8");
+        }
+        classified.push({ ...b, kind });
+    }
+    return {
+        blocks: classified,
+        provisional: coldStart,
+        // Every textual block of this page already exists on other pages of this
+        // host: the response carried no page-specific content at all.
+        allTemplate: sawTextBlock && !sawContent,
+        contentBytes,
+        templateBytes,
+    };
+}
+/**
+ * The text that goes to the search index: the `content` blocks, in document
+ * order, joined by their own separators. Template blocks are NOT deleted —
+ * the caller has already stored every block verbatim; they are simply left
+ * out of the index.
+ */
+export function contentText(blocks) {
+    let out = "";
+    for (const b of blocks) {
+        if (b.kind === "content")
+            out += b.raw;
+    }
+    return trimEdges(out);
+}
+/** The template stream, in document order. Stored and retrievable, never indexed. */
+export function templateText(blocks) {
+    let out = "";
+    for (const b of blocks) {
+        if (b.kind === "template")
+            out += b.raw;
+    }
+    return trimEdges(out);
+}

package/build/fetch/extract.d.ts

@@ -0,0 +1,68 @@
+/**
+ * extract — orchestrates the lossless template/content pass for one fetch.
+ *
+ * Order of operations, cheapest correct answer first:
+ *
+ *   1. If the site publishes a machine-readable version of the page, take it.
+ *      The fetch subprocess now sends `Accept: text/markdown` on the SAME
+ *      request it was already making, so this costs zero extra round trips.
+ *      Measured 2026-08-12: Stripe 1,846,885 B of HTML → 11,744 B of pure
+ *      article; GitBook, Mintlify, Resend and Polygon likewise. Such a
+ *      document was authored for machines and carries no chrome, so it skips
+ *      extraction entirely (`authored`).
+ *   2. Otherwise convert the HTML, then classify each block against the other
+ *      pages already seen from that host.
+ *   3. Index only `content`. Store everything.
+ *   4. If the page turns out to carry no page-specific content at all, refuse
+ *      and say what to try instead.
+ *
+ * No regular expressions (repo-wide ban). Nothing here truncates.
+ */
+import { PageStore } from "./page-store.js";
+/** How the document reached us. `markdown` = the site served it; `html` = we converted it. */
+export type FetchRoute = "markdown" | "html" | "json" | "text";
+export interface ExtractInput {
+    url: string;
+    /** The storage label the caller will index under. */
+    sourceLabel: string;
+    /** The COMPLETE converted document. Stored verbatim; never trimmed here. */
+    document: string;
+    route: FetchRoute;
+    store: PageStore;
+}
+export interface Relabelled {
+    sourceLabel: string;
+    url: string;
+    /** Text to re-index for that page under the same label (replaces the old rows). */
+    indexText: string;
+    templateBytes: number;
+}
+export type ExtractOutcome = {
+    kind: "index";
+    /** The text that should reach the FTS index — `content` blocks only. */
+    indexText: string;
+    /** Complete document as stored. Retained for the caller's byte accounting. */
+    storedBytes: number;
+    contentBytes: number;
+    templateBytes: number;
+    templateBlocks: number;
+    totalBlocks: number;
+    provisional: boolean;
+    route: FetchRoute;
+    /** Cold-start pages of this host that a second page has now resolved. */
+    relabelled: Relabelled[];
+} | {
+    kind: "refuse";
+    reason: string;
+    storedBytes: number;
+    route: FetchRoute;
+};
+/**
+ * JSON and plain-text responses are not web pages and have no chrome; they are
+ * passed through untouched so the existing JSON/text indexing strategies keep
+ * their exact behaviour.
+ */
+export declare function routeSkipsExtraction(route: FetchRoute): boolean;
+export declare function extractAndStore(input: ExtractInput): ExtractOutcome;
+/** The chrome that was labelled out of a stored page, for retrieval. */
+export declare function storedTemplateText(store: PageStore, url: string): string;

package/build/fetch/extract.js

@@ -0,0 +1,123 @@
+/**
+ * extract — orchestrates the lossless template/content pass for one fetch.
+ *
+ * Order of operations, cheapest correct answer first:
+ *
+ *   1. If the site publishes a machine-readable version of the page, take it.
+ *      The fetch subprocess now sends `Accept: text/markdown` on the SAME
+ *      request it was already making, so this costs zero extra round trips.
+ *      Measured 2026-08-12: Stripe 1,846,885 B of HTML → 11,744 B of pure
+ *      article; GitBook, Mintlify, Resend and Polygon likewise. Such a
+ *      document was authored for machines and carries no chrome, so it skips
+ *      extraction entirely (`authored`).
+ *   2. Otherwise convert the HTML, then classify each block against the other
+ *      pages already seen from that host.
+ *   3. Index only `content`. Store everything.
+ *   4. If the page turns out to carry no page-specific content at all, refuse
+ *      and say what to try instead.
+ *
+ * No regular expressions (repo-wide ban). Nothing here truncates.
+ */
+import { splitBlocks, classifyBlocks, contentText, templateText, } from "./blocks.js";
+import { pageKeyFor, hostFor } from "./page-store.js";
+/**
+ * JSON and plain-text responses are not web pages and have no chrome; they are
+ * passed through untouched so the existing JSON/text indexing strategies keep
+ * their exact behaviour.
+ */
+export function routeSkipsExtraction(route) {
+    return route === "json" || route === "text";
+}
+export function extractAndStore(input) {
+    const { url, sourceLabel, document, route, store } = input;
+    const pageKey = pageKeyFor(url);
+    const host = hostFor(url);
+    const storedBytes = Buffer.byteLength(document, "utf-8");
+    const blocks = splitBlocks(document);
+    const authored = route === "markdown";
+    const hostPageCount = store.hostPageCount(host, pageKey);
+    const counts = authored
+        ? new Map()
+        : store.otherPageCounts(host, pageKey, blocks.map((b) => b.hash));
+    const result = classifyBlocks({
+        blocks,
+        otherPageCount: (h) => counts.get(h) ?? 0,
+        hostPageCount,
+        authored,
+    });
+    // Store the page WHOLE regardless of the verdict — including when we are
+    // about to refuse. Refusing to index is not a licence to discard bytes.
+    store.recordPage({
+        pageKey, host, url, sourceLabel, route,
+        provisional: result.provisional,
+        fullText: document,
+    }, result.blocks);
+    if (result.allTemplate) {
+        return {
+            kind: "refuse",
+            storedBytes,
+            route,
+            reason: `every block of this page is byte-identical to blocks already seen on other pages of ${host}, ` +
+                `so the response carried the site shell rather than this page — its content is rendered ` +
+                `client-side by JavaScript and an HTTP fetch cannot see it. Nothing was indexed (the response ` +
+                `is stored whole and unaltered). Retrying this URL returns the same shell; look for this site's ` +
+                `llms.txt, a raw .md source, an OpenAPI spec, or a repository README instead.`,
+        };
+    }
+    // A second page of this host resolves every cold-start page that came
+    // before it. Re-run those now — a first page that is wrongly labelled and
+    // never revisited is exactly the silent loss this design forbids.
+    const relabelled = [];
+    if (!result.provisional) {
+        for (const prev of store.provisionalPages(host, pageKey)) {
+            const prevBlocks = store.blocksOf(prev.pageKey);
+            const prevCounts = store.otherPageCounts(host, prev.pageKey, prevBlocks.map((b) => b.hash));
+            const rerun = classifyBlocks({
+                blocks: prevBlocks.map((b) => ({
+                    ordinal: b.ordinal, raw: b.raw, text: b.text, hash: b.hash,
+                })),
+                otherPageCount: (h) => prevCounts.get(h) ?? 0,
+                hostPageCount: store.hostPageCount(prev.host, prev.pageKey),
+                authored: prev.route === "markdown",
+            });
+            // An all-template re-run means the earlier page was a shell too. Leave
+            // its stored bytes alone and leave its index rows alone rather than
+            // emptying a source the user may already be searching.
+            if (rerun.allTemplate)
+                continue;
+            store.relabelPage(prev.pageKey, rerun.blocks, rerun.provisional);
+            if (rerun.templateBytes > 0) {
+                relabelled.push({
+                    sourceLabel: prev.sourceLabel,
+                    url: prev.url,
+                    indexText: contentText(rerun.blocks),
+                    templateBytes: rerun.templateBytes,
+                });
+            }
+        }
+    }
+    return {
+        kind: "index",
+        indexText: contentText(result.blocks),
+        storedBytes,
+        contentBytes: result.contentBytes,
+        templateBytes: result.templateBytes,
+        templateBlocks: countTemplate(result.blocks),
+        totalBlocks: result.blocks.length,
+        provisional: result.provisional,
+        route,
+        relabelled,
+    };
+}
+function countTemplate(blocks) {
+    let n = 0;
+    for (const b of blocks)
+        if (b.kind === "template")
+            n++;
+    return n;
+}
+/** The chrome that was labelled out of a stored page, for retrieval. */
+export function storedTemplateText(store, url) {
+    const blocks = store.blocksOf(pageKeyFor(url));
+    return templateText(blocks.map((b) => ({ ordinal: b.ordinal, raw: b.raw, text: b.text, hash: b.hash, kind: b.kind })));
+}

package/build/fetch/page-store.d.ts

@@ -0,0 +1,66 @@
+/**
+ * page-store — the lossless side of fetch extraction.
+ *
+ * Holds, per project, the COMPLETE converted document for every page ever
+ * fetched, plus every block of it with its `content` / `template` label. The
+ * FTS index receives only `content` blocks; this store is what makes that
+ * safe, because nothing has been thrown away — the whole document and every
+ * template block remain here, verbatim and retrievable.
+ *
+ * It is also the comparison set. Classification asks one question — "was this
+ * exact block already seen on a DIFFERENT page of this host?" — and that
+ * question is answered by `page_blocks` joined to `pages`.
+ *
+ * Nothing in this file truncates. Full documents are stored whole; there is no
+ * size cap, no prefix, no summary. (The fetch path upstream already refuses
+ * responses above 50 MB before conversion.)
+ *
+ * No regular expressions (repo-wide ban).
+ */
+import type { BlockKind, ClassifiedBlock } from "./blocks.js";
+export interface StoredPage {
+    pageKey: string;
+    host: string;
+    url: string;
+    sourceLabel: string;
+    route: string;
+    provisional: boolean;
+    fullText: string;
+}
+export interface StoredBlock {
+    ordinal: number;
+    hash: string;
+    kind: BlockKind;
+    raw: string;
+    text: string;
+}
+/**
+ * Canonical identity of a fetched page. The URL minus its fragment: two
+ * fetches of the same page must be the same row, or a page compared against
+ * an older copy of itself marks its own article as template.
+ */
+export declare function pageKeyFor(url: string): string;
+/** Host of a URL, lower-cased. Empty string when the URL will not parse. */
+export declare function hostFor(url: string): string;
+export declare class PageStore {
+    #private;
+    constructor(dbPath: string);
+    /** Distinct pages already recorded for a host, excluding `exceptPageKey`. */
+    hostPageCount(host: string, exceptPageKey: string): number;
+    /**
+     * How many DISTINCT other pages of this host carry this exact block. The
+     * `page_key <> ?` clause is what keeps a page from classifying itself.
+     */
+    otherPageCounts(host: string, exceptPageKey: string, hashes: string[]): Map<string, number>;
+    /** Store one page whole: the complete document plus every labelled block. */
+    recordPage(page: StoredPage, blocks: ClassifiedBlock[]): void;
+    /** Update the stored labels after a re-run, and clear the provisional flag. */
+    relabelPage(pageKey: string, blocks: ClassifiedBlock[], provisional: boolean): void;
+    /** Every page of a host still carrying a cold-start (provisional) labelling. */
+    provisionalPages(host: string, exceptPageKey: string): StoredPage[];
+    /** Every stored block of a page, in document order — content and template alike. */
+    blocksOf(pageKey: string): StoredBlock[];
+    /** The complete converted document as it was stored, or null. */
+    fullTextOf(pageKey: string): string | null;
+    close(): void;
+}