markindex 0.0.0-stage → 0.1.0

package/dist/tools/impl.js

@@ -0,0 +1,441 @@
+import { RESPONSE_CHAR_BUDGET, SIMILAR_NAMES_LIMIT, STRUCTURE_FIRST_TOKEN_THRESHOLD } from '../config.js';
+import { searchNodes } from '../store/search.js';
+import { walk } from '../tree/build.js';
+import { TOOL_CONTRACT } from './contract.js';
+import { failure, jsonSize, splitByBudget, success } from './envelope.js';
+import { similarNames } from './similar.js';
+const TOOL_NAMES = Object.keys(TOOL_CONTRACT);
+/** Placeholder for part numbers when measuring envelope overhead (no real document has a million parts). */
+const MAX_PART = 999_999;
+/** Minimum room for one content segment, even if a huge title path eats the budget (best effort). */
+const MIN_SEGMENT = 256;
+/** A label for an untrusted tool name that cannot throw (no String() on arbitrary objects). */
+const nameLabel = (name) => (typeof name === 'string' ? name : typeof name);
+/** `String(e)` / `e.message` can run user code (getters, toString); never let that escape. */
+function describeError(e) {
+    try {
+        if (e instanceof Error && typeof e.message === 'string')
+            return e.message;
+        return typeof e === 'string' ? e : typeof e;
+    }
+    catch {
+        return 'unknown error';
+    }
+}
+const internalError = (name, message) => failure('INTERNAL_ERROR', `Tool failed: ${message}`, {
+    summary: 'The library could not be read.',
+    options: [
+        'Try the request again',
+        'If it keeps failing, the index may be closed, missing or corrupt: ask the user to re-run `markindex index`',
+    ],
+}, { tool: nameLabel(name) });
+const TOO_LARGE = failure('INTERNAL_ERROR', 'Response too large for the response budget', {
+    summary: 'The result does not fit in one response.',
+    options: ['Narrow the request: use a smaller `limit`, a deeper `folder`, or fewer `node_ids`'],
+});
+/**
+ * Runs one tool against `store` and returns its JSON envelope. Never throws: invalid input, unknown names,
+ * internal failures and unserializable values all come back as error envelopes. The result is at most
+ * RESPONSE_CHAR_BUDGET characters.
+ */
+export function runTool(store, name, args) {
+    let json;
+    try {
+        json = JSON.stringify(dispatch(store, name, args));
+    }
+    catch (e) {
+        // every value in an internal-error envelope is a plain string, so this stringify cannot throw
+        json = JSON.stringify(internalError(name, describeError(e).slice(0, 2_000)));
+    }
+    return json.length > RESPONSE_CHAR_BUDGET ? JSON.stringify(TOO_LARGE) : json;
+}
+function dispatch(store, name, args) {
+    if (typeof name !== 'string' || !TOOL_NAMES.includes(name)) {
+        return failure('INVALID_INPUT', `Unknown tool: ${nameLabel(name)}`, { summary: 'No tool with this name exists.', options: [`Use one of: ${TOOL_NAMES.join(', ')}`] }, { tool: nameLabel(name) });
+    }
+    const tool = name;
+    const parsed = TOOL_CONTRACT[tool].input.safeParse(args === undefined ? {} : args);
+    if (!parsed.success)
+        return invalidArgs(tool, parsed.error);
+    const input = parsed.data;
+    switch (tool) {
+        case 'browse_documents':
+            return browse(store, input);
+        case 'get_document':
+            return getDocument(store, input);
+        case 'get_document_structure':
+            return getStructure(store, input);
+        case 'get_node_content':
+            return getContent(store, input);
+        case 'search_documents':
+            return search(store, input);
+    }
+}
+function invalidArgs(tool, error) {
+    const issues = error.issues.map((i) => ({ path: i.path.map(String).join('.') || '(arguments)', message: i.message }));
+    return failure('INVALID_INPUT', `Invalid arguments for ${tool}: ${issues.map((i) => `${i.path}: ${i.message}`).join('; ')}`, {
+        summary: 'The arguments do not match the tool schema.',
+        options: [
+            `Call ${tool}() with a JSON object matching its input schema`,
+            'Copy doc_path, folder and node_id values verbatim from earlier tool results',
+        ],
+    }, { issues });
+}
+// ── lookup helpers ──
+const normFolder = (f) => f.trim().replace(/^\/+|\/+$/g, '');
+function allFolders(paths) {
+    const out = new Set();
+    for (const p of paths) {
+        const segs = p.split('/');
+        for (let i = 1; i < segs.length; i++)
+            out.add(segs.slice(0, i).join('/'));
+    }
+    return [...out].sort();
+}
+function resolveDoc(store, path) {
+    const doc = store.getDoc(path);
+    if (doc)
+        return { doc };
+    const similar = similarNames(path, store.allDocPaths(), SIMILAR_NAMES_LIMIT);
+    return {
+        error: failure('NOT_FOUND', similar.length
+            ? `Document not found. Did you mean: ${similar.map((s) => `"${s}"`).join(', ')}?`
+            : 'Document not found', {
+            summary: 'The requested document does not exist in the library index.',
+            options: [
+                ...similar.map((s) => `Retry with doc_path: "${s}"`),
+                'Copy doc_path verbatim from a browse_documents() or search_documents() result (case-sensitive, include the extension)',
+                'Use browse_documents() to list the library',
+                'Use search_documents() to find the document by keyword',
+            ],
+        }, { doc_path: path, similar_paths: similar }),
+    };
+}
+function folderNotFound(store, folder) {
+    const similar = similarNames(folder, allFolders(store.allDocPaths()), SIMILAR_NAMES_LIMIT);
+    return failure('NOT_FOUND', similar.length
+        ? `Folder not found. Did you mean: ${similar.map((s) => `"${s}"`).join(', ')}?`
+        : 'Folder not found', {
+        summary: 'No indexed documents live under this folder.',
+        options: [
+            ...similar.map((s) => `Retry with folder: "${s}"`),
+            'Copy folder verbatim from the `folders` of a browse_documents() response',
+            'Call browse_documents() without `folder` to start at the library root',
+        ],
+    }, { folder, similar_folders: similar });
+}
+const folderExists = (store, folder) => folder === '' || store.allDocPaths().some((p) => p.startsWith(`${folder}/`));
+function findNode(tree, nodeId) {
+    for (const e of walk(tree))
+        if (e.node.nodeId === nodeId)
+            return e;
+    return undefined;
+}
+function staleNodes(docPath, unknown) {
+    return failure('NOT_FOUND', `Unknown node_id for this document: ${unknown.join(', ')}`, {
+        summary: 'These node ids do not exist in the current index of the document; it may have been re-indexed since you read its structure.',
+        options: [
+            `Call get_document_structure({ doc_path: "${docPath}" }) again and copy the current node ids verbatim`,
+            'Use search_documents() with doc_path to locate the section by keyword',
+        ],
+    }, { doc_path: docPath, unknown_node_ids: unknown });
+}
+function partOutOfRange(tool, part, total) {
+    return failure('INVALID_INPUT', `part ${part} is out of range: this response has ${total} part${total === 1 ? '' : 's'}`, {
+        summary: 'The requested part does not exist.',
+        options: [`Request a part between 1 and ${total} (last part: part: ${total})`, `Call ${tool}() with part: 1 to start over`],
+    }, { part, total_parts: total });
+}
+const pagination = (part, total) => ({ part, total_parts: total, has_more: part < total });
+/**
+ * Budget for the paginated payload: RESPONSE_CHAR_BUDGET minus the envelope around it, measured with an empty
+ * payload (`[]`) and placeholder part numbers in every next_steps variant.
+ */
+function payloadBudget(make) {
+    const overhead = Math.max(jsonSize(make([], MAX_PART - 1, MAX_PART)), jsonSize(make([], MAX_PART, MAX_PART))) - 2;
+    return RESPONSE_CHAR_BUDGET - overhead;
+}
+/** Drops trailing items until the envelope fits the budget (keeps at least one). */
+function fitList(items, make) {
+    let n = items.length;
+    let env = make(items);
+    while (n > 1 && jsonSize(env) > RESPONSE_CHAR_BUDGET) {
+        n--;
+        env = make(items.slice(0, n));
+    }
+    return { env, kept: n };
+}
+// ── browse_documents ──
+function browse(store, input) {
+    const folder = normFolder(input.folder);
+    if (!folderExists(store, folder))
+        return folderNotFound(store, folder);
+    const { offset, limit } = input;
+    const listing = store.listDocs(folder, offset, limit);
+    if (offset > 0 && offset >= listing.total) {
+        return failure('INVALID_INPUT', `offset ${offset} is past the end of this folder (${listing.total} documents)`, {
+            summary: 'There are no documents at this offset.',
+            options: ['Call browse_documents() with offset: 0', 'Use `next_offset` from the previous response, not a guessed value'],
+        }, { folder, offset, total: listing.total });
+    }
+    const docs = listing.docs.map((d) => ({
+        path: d.path,
+        title: d.title,
+        ...(d.description === undefined ? {} : { description: d.description }),
+        tokens: d.tokens,
+    }));
+    const make = (page) => {
+        const next = offset + page.length < listing.total ? offset + page.length : null;
+        const options = [
+            'Use get_document() / get_document_structure() with a doc_path copied from `docs`',
+            'Pass a `folders` entry as `folder` to drill into a sub-folder',
+        ];
+        if (next !== null)
+            options.unshift(`More documents: call browse_documents() with offset: ${next}`);
+        return success({ folder, folders: listing.folders, docs: page, total: listing.total, next_offset: next, has_more: next !== null }, {
+            summary: `Showing documents ${listing.total ? offset + 1 : 0}-${offset + page.length} of ${listing.total} in ${folder === '' ? 'the library root' : `"${folder}"`}, plus ${listing.folders.length} sub-folder(s).`,
+            options,
+        });
+    };
+    return fitList(docs, make).env;
+}
+// ── get_document ──
+function getDocument(store, input) {
+    const r = resolveDoc(store, input.doc_path);
+    if ('error' in r)
+        return r.error;
+    const d = r.doc;
+    const readStructureFirst = d.tokens > STRUCTURE_FIRST_TOKEN_THRESHOLD;
+    return success({
+        doc_path: d.path,
+        title: d.title,
+        ...(d.description === undefined ? {} : { description: d.description }),
+        line_count: d.lineCount,
+        tokens: d.tokens,
+        metadata: d.metadata,
+        indexed_at: d.indexedAt,
+        read_structure_first: readStructureFirst,
+    }, readStructureFirst
+        ? {
+            summary: `Large document (${d.tokens} tokens > ${STRUCTURE_FIRST_TOKEN_THRESHOLD}): read its structure first.`,
+            options: [
+                `Call get_document_structure({ doc_path: "${d.path}" }) to locate the relevant sections`,
+                'Then read only those nodes with get_node_content()',
+            ],
+        }
+        : {
+            summary: `Small document (${d.tokens} tokens): it can be read directly, no outline needed.`,
+            options: [
+                `Read its top-level nodes with get_node_content() (ids from get_document_structure({ doc_path: "${d.path}", depth: 1 }))`,
+            ],
+        });
+}
+function formatStructure(node, depth) {
+    const out = {
+        node_id: node.nodeId,
+        title: node.title,
+        kind: node.kind,
+        start_line: node.startLine,
+        end_line: node.endLine,
+        tokens: node.tokens,
+    };
+    const summary = node.summary ?? node.prefixSummary;
+    if (summary !== undefined)
+        out.summary = summary;
+    if (node.children.length) {
+        if (depth > 1)
+            out.nodes = node.children.map((c) => formatStructure(c, depth - 1));
+        else
+            out.child_count = node.children.length;
+    }
+    return out;
+}
+function getStructure(store, input) {
+    const r = resolveDoc(store, input.doc_path);
+    if ('error' in r)
+        return r.error;
+    const docPath = r.doc.path;
+    const tree = store.getTree(docPath) ?? [];
+    let roots = tree;
+    let titlePath;
+    if (input.node_id !== undefined) {
+        const found = findNode(tree, input.node_id);
+        if (!found)
+            return staleNodes(docPath, [input.node_id]);
+        roots = [found.node];
+        titlePath = found.titlePath;
+    }
+    const structure = roots.map((n) => formatStructure(n, input.depth));
+    const make = (chunk, part, total) => success({
+        doc_path: docPath,
+        ...(titlePath === undefined ? {} : { node_id: input.node_id, title_path: titlePath }),
+        structure: chunk,
+        pagination: pagination(part, total),
+    }, part < total
+        ? {
+            summary: `Showing part ${part} of ${total} of the outline.`,
+            options: [
+                `Request the next part with part: ${part + 1}`,
+                'Read relevant nodes with get_node_content() using node_id values copied verbatim',
+            ],
+        }
+        : {
+            summary: total > 1 ? `Showing the last part (${part} of ${total}) of the outline.` : 'Document outline retrieved.',
+            options: [
+                'Read relevant nodes with get_node_content() using node_id values copied verbatim',
+                'Pass the node_id of a node with child_count as `node_id` to expand it',
+            ],
+        });
+    const chunks = splitByBudget(structure, payloadBudget(make));
+    if (input.part > chunks.length)
+        return partOutOfRange('get_document_structure', input.part, chunks.length);
+    return make(chunks[input.part - 1], input.part, chunks.length);
+}
+/** JSON-escaped length of a string's content (without the quotes). */
+const escLen = (s) => JSON.stringify(s).length - 2;
+/** `n│ line`, split into several segments with the same prefix when one would exceed `max` escaped chars. */
+function segments(n, line, max) {
+    const prefix = `${n}│ `;
+    const whole = prefix + line;
+    if (escLen(whole) <= max)
+        return [whole];
+    const out = [];
+    let cur = '';
+    let curLen = escLen(prefix);
+    for (const ch of line) {
+        const l = escLen(ch);
+        if (cur !== '' && curLen + l > max) {
+            out.push(prefix + cur);
+            cur = '';
+            curLen = escLen(prefix);
+        }
+        cur += ch;
+        curLen += l;
+    }
+    out.push(prefix + cur);
+    return out;
+}
+/** Packs the numbered lines of `targets` into parts whose JSON array fits `budget`, splitting nodes as needed. */
+function packContent(store, docPath, targets, budget) {
+    const parts = [];
+    let part = [];
+    let used = 2; // "[]" plus closed entries and their separating commas
+    const flushPart = () => {
+        if (part.length)
+            parts.push(part);
+        part = [];
+        used = 2;
+    };
+    for (const { node, titlePath } of targets) {
+        const meta = { node_id: node.nodeId, title_path: titlePath, start_line: node.startLine, end_line: node.endLine };
+        const entryBase = jsonSize({ ...meta, complete: false, text: '' });
+        const maxSegment = Math.max(MIN_SEGMENT, budget - 2 - entryBase);
+        let lines = [];
+        let textLen = 0;
+        let split = false;
+        const closeEntry = (complete) => {
+            const entry = { ...meta, complete, text: lines.join('\n') };
+            used += jsonSize(entry) + (part.length ? 1 : 0);
+            part.push(entry);
+            lines = [];
+            textLen = 0;
+        };
+        const projected = (add) => used + (part.length ? 1 : 0) + entryBase + textLen + add;
+        const raw = store.getLines(docPath, node.startLine, node.endLine);
+        raw.forEach((line, i) => {
+            for (const seg of segments(node.startLine + i, line, maxSegment)) {
+                const add = escLen(seg) + (lines.length ? 2 : 0); // "\n" separator is 2 escaped chars
+                if (projected(add) > budget && (lines.length || part.length)) {
+                    if (lines.length) {
+                        split = true;
+                        closeEntry(false);
+                    }
+                    flushPart();
+                    lines = [];
+                }
+                textLen += escLen(seg) + (lines.length ? 2 : 0);
+                lines.push(seg);
+            }
+        });
+        closeEntry(!split);
+    }
+    flushPart();
+    return parts.length ? parts : [[]];
+}
+function getContent(store, input) {
+    const r = resolveDoc(store, input.doc_path);
+    if ('error' in r)
+        return r.error;
+    const docPath = r.doc.path;
+    const tree = store.getTree(docPath) ?? [];
+    const byId = new Map();
+    for (const e of walk(tree))
+        byId.set(e.node.nodeId, e);
+    const ids = [...new Set(input.node_ids)];
+    const unknown = ids.filter((id) => !byId.has(id));
+    if (unknown.length)
+        return staleNodes(docPath, unknown);
+    const make = (content, part, total) => success({ doc_path: docPath, content, pagination: pagination(part, total) }, part < total
+        ? {
+            summary: `Showing part ${part} of ${total} of the requested content.`,
+            options: [
+                `Request the next part with part: ${part + 1}`,
+                'Or read a more specific (child) node: get_document_structure() with node_id lists them',
+            ],
+        }
+        : {
+            summary: total > 1 ? `Showing the last part (${part} of ${total}) of the requested content.` : 'Content retrieved.',
+            options: [
+                `Cite as [${docPath}, L<start>-<end>] using the line numbers shown`,
+                'Use get_document_structure() to find related sections',
+            ],
+        });
+    const parts = packContent(store, docPath, ids.map((id) => byId.get(id)), payloadBudget(make));
+    if (input.part > parts.length)
+        return partOutOfRange('get_node_content', input.part, parts.length);
+    return make(parts[input.part - 1], input.part, parts.length);
+}
+// ── search_documents ──
+function search(store, input) {
+    let docPath;
+    let folder;
+    if (input.doc_path !== undefined) {
+        const r = resolveDoc(store, input.doc_path);
+        if ('error' in r)
+            return r.error;
+        docPath = r.doc.path;
+    }
+    else if (input.folder !== undefined) {
+        folder = normFolder(input.folder);
+        if (!folderExists(store, folder))
+            return folderNotFound(store, folder);
+    }
+    const hits = searchNodes(store, input.query, { docPath, folder, limit: input.limit }).map((h) => ({
+        doc_path: h.docPath,
+        node_id: h.nodeId,
+        title_path: h.titlePath,
+        start_line: h.startLine,
+        end_line: h.endLine,
+        snippet: h.snippet,
+        score: h.score,
+        via: h.via,
+    }));
+    const scope = docPath !== undefined ? { doc_path: docPath } : folder ? { folder } : {};
+    const make = (results) => success({ query: input.query, ...scope, results }, results.length
+        ? {
+            summary: `Found ${results.length} matching node(s).`,
+            options: [
+                'Read the most relevant hits with get_node_content() using doc_path and node_id copied verbatim',
+                'Use get_document_structure() with node_id to see the surrounding sections',
+            ],
+        }
+        : {
+            summary: 'No matching nodes.',
+            options: [
+                'Rephrase with synonyms, alternative terms or the other language (Chinese/English)',
+                'Widen the scope: drop `doc_path` / `folder`',
+                'Browse with browse_documents() and read document structures instead',
+            ],
+        });
+    return fitList(hits, make).env;
+}

package/dist/tools/similar.d.ts

@@ -0,0 +1,8 @@
+/** Optimal string alignment distance (Levenshtein + adjacent transposition), by code point. */
+export declare function editDistance(a: string, b: string): number;
+/**
+ * Up to `limit` candidates most similar to `query` (difflib `get_close_matches`-like, cutoff 0.5). A candidate is
+ * compared both as a whole and by its last path segment against the query's last segment, so `annual.md` finds
+ * `reports/annual.md`. Ties keep the candidates' order.
+ */
+export declare function similarNames(query: string, candidates: readonly string[], limit: number): string[];

package/dist/tools/similar.js

@@ -0,0 +1,40 @@
+/** Optimal string alignment distance (Levenshtein + adjacent transposition), by code point. */
+export function editDistance(a, b) {
+    const s = [...a];
+    const t = [...b];
+    const rows = Array.from({ length: s.length + 1 }, (_, i) => Array.from({ length: t.length + 1 }, (_, j) => (i === 0 ? j : j === 0 ? i : 0)));
+    for (let i = 1; i <= s.length; i++) {
+        for (let j = 1; j <= t.length; j++) {
+            const cost = s[i - 1] === t[j - 1] ? 0 : 1;
+            let d = Math.min(rows[i - 1][j] + 1, rows[i][j - 1] + 1, rows[i - 1][j - 1] + cost);
+            if (i > 1 && j > 1 && s[i - 1] === t[j - 2] && s[i - 2] === t[j - 1])
+                d = Math.min(d, rows[i - 2][j - 2] + 1);
+            rows[i][j] = d;
+        }
+    }
+    return rows[s.length][t.length];
+}
+const basename = (p) => p.slice(p.lastIndexOf('/') + 1);
+/** 1 = identical, 0 = nothing in common; like difflib's ratio, based on edit distance. */
+function similarity(a, b) {
+    const len = Math.max([...a].length, [...b].length);
+    return len === 0 ? 1 : 1 - editDistance(a, b) / len;
+}
+/**
+ * Up to `limit` candidates most similar to `query` (difflib `get_close_matches`-like, cutoff 0.5). A candidate is
+ * compared both as a whole and by its last path segment against the query's last segment, so `annual.md` finds
+ * `reports/annual.md`. Ties keep the candidates' order.
+ */
+export function similarNames(query, candidates, limit) {
+    const q = query.trim().toLowerCase();
+    const qBase = basename(q);
+    return candidates
+        .map((c, i) => {
+        const lc = c.toLowerCase();
+        return { c, i, score: Math.max(similarity(q, lc), similarity(qBase, basename(lc))) };
+    })
+        .filter((x) => x.score >= 0.5)
+        .sort((x, y) => y.score - x.score || x.i - y.i)
+        .slice(0, limit)
+        .map((x) => x.c);
+}

package/dist/tree/build.d.ts

@@ -0,0 +1,27 @@
+import type { ParsedDoc } from '../parse/markdown.js';
+import { type TokenCounter } from '../text/tokens.js';
+import type { TreeNode } from '../types.js';
+export interface BuildTreeOptions {
+    /** Title for a doc without headings (root node). */
+    fileTitle: string;
+    countTokens?: TokenCounter;
+}
+/**
+ * Build the section tree (stack algorithm, like PageIndex `build_tree_from_nodes`).
+ *
+ * Conventions:
+ * - `level` is 0 for the preface and for the title-less root of a headless doc.
+ * - Blank-only lines before the first heading are absorbed into the first heading's node
+ *   (no spurious preface); non-blank text produces a `preface` node.
+ * - A doc without headings yields one root `section` titled `fileTitle`, covering all non-frontmatter lines.
+ * - A doc with no lines outside frontmatter yields `[]`.
+ * - Every line outside frontmatter belongs to exactly one node's own range [startLine, ownEndLine].
+ */
+export declare function buildTree(doc: ParsedDoc, opts: BuildTreeOptions): TreeNode[];
+/** Depth-first pre-order traversal with the title path of each node. */
+export declare function walk(roots: TreeNode[]): Generator<{
+    node: TreeNode;
+    titlePath: string[];
+}>;
+/** Throws when spec §5.2 invariants are violated. Test / debug helper. */
+export declare function assertCoverage(doc: ParsedDoc, roots: TreeNode[]): void;

package/dist/tree/build.js

@@ -0,0 +1,139 @@
+import { createHash } from 'node:crypto';
+import { approxTokens } from '../text/tokens.js';
+const SEP = '\u001f';
+const PREFACE_TITLE = 'Preface';
+/**
+ * Build the section tree (stack algorithm, like PageIndex `build_tree_from_nodes`).
+ *
+ * Conventions:
+ * - `level` is 0 for the preface and for the title-less root of a headless doc.
+ * - Blank-only lines before the first heading are absorbed into the first heading's node
+ *   (no spurious preface); non-blank text produces a `preface` node.
+ * - A doc without headings yields one root `section` titled `fileTitle`, covering all non-frontmatter lines.
+ * - A doc with no lines outside frontmatter yields `[]`.
+ * - Every line outside frontmatter belongs to exactly one node's own range [startLine, ownEndLine].
+ */
+export function buildTree(doc, opts) {
+    const count = opts.countTokens ?? approxTokens;
+    const total = doc.lines.length;
+    const first = doc.firstContentLine;
+    if (first > total)
+        return [];
+    const text = (from, to) => doc.lines.slice(from - 1, to).join('\n');
+    const make = (title, kind, level, startLine) => ({
+        nodeId: '',
+        title,
+        kind,
+        level,
+        startLine,
+        endLine: total,
+        ownEndLine: total,
+        tokens: 0,
+        ownTokens: 0,
+        children: [],
+    });
+    const roots = [];
+    const headings = doc.headings;
+    if (headings.length === 0) {
+        roots.push(make(opts.fileTitle, 'section', 0, first));
+    }
+    else {
+        const firstHeading = headings[0].line;
+        let hasPreface = false;
+        for (let i = first; i < firstHeading; i++) {
+            if (doc.lines[i - 1].trim() !== '') {
+                hasPreface = true;
+                break;
+            }
+        }
+        if (hasPreface) {
+            const p = make(PREFACE_TITLE, 'preface', 0, first);
+            p.endLine = p.ownEndLine = firstHeading - 1;
+            roots.push(p);
+        }
+        const stack = [];
+        headings.forEach((h, idx) => {
+            const node = make(h.title, 'section', h.level, !hasPreface && idx === 0 ? first : h.line);
+            while (stack.length > 0 && stack[stack.length - 1].level >= h.level) {
+                stack.pop().endLine = h.line - 1;
+            }
+            const parent = stack[stack.length - 1];
+            if (parent)
+                parent.children.push(node);
+            else
+                roots.push(node);
+            stack.push(node);
+        });
+    }
+    finalize(roots, [], [], new Set(), text, count);
+    return roots;
+}
+/**
+ * Fills ownEndLine, tokens and nodeId, in document (pre-)order.
+ *
+ * Primary id = hash(title path + the node's ordinal among same-title siblings). Only when that id is already taken
+ * in the document (two same-title parents with same-title children) does the node fall back to a disambiguated key:
+ * the chain of (title, same-title ordinal) pairs of every ancestor, JSON-encoded (unambiguous for any title), plus
+ * a counter in the (practically impossible) case that this collides too. Unique ids therefore never change.
+ */
+function finalize(siblings, parentPath, parentChain, used, text, count) {
+    const seen = new Map();
+    for (const node of siblings) {
+        const ordinal = seen.get(node.title) ?? 0;
+        seen.set(node.title, ordinal + 1);
+        const path = [...parentPath, node.title];
+        const chain = [...parentChain, [node.title, ordinal]];
+        let id = makeId(path, ordinal);
+        for (let n = 0; used.has(id); n++)
+            id = hashHex(JSON.stringify([chain, n]));
+        used.add(id);
+        node.nodeId = id;
+        const firstChild = node.children[0];
+        node.ownEndLine = firstChild ? firstChild.startLine - 1 : node.endLine;
+        node.ownTokens = count(text(node.startLine, node.ownEndLine));
+        node.tokens = firstChild ? count(text(node.startLine, node.endLine)) : node.ownTokens;
+        finalize(node.children, path, chain, used, text, count);
+    }
+}
+const hashHex = (s) => createHash('sha1').update(s).digest('hex').slice(0, 10);
+function makeId(titlePath, ordinal) {
+    return hashHex(titlePath.join(SEP) + SEP + ordinal);
+}
+/** Depth-first pre-order traversal with the title path of each node. */
+export function* walk(roots) {
+    function* rec(nodes, parent) {
+        for (const node of nodes) {
+            const titlePath = [...parent, node.title];
+            yield { node, titlePath };
+            yield* rec(node.children, titlePath);
+        }
+    }
+    yield* rec(roots, []);
+}
+/** Throws when spec §5.2 invariants are violated. Test / debug helper. */
+export function assertCoverage(doc, roots) {
+    const total = doc.lines.length;
+    const first = doc.firstContentLine;
+    let expected = first;
+    for (const { node, titlePath } of walk(roots)) {
+        const label = titlePath.join(' > ');
+        if (node.startLine !== expected) {
+            throw new Error(`coverage: "${label}" starts at ${node.startLine}, expected ${expected}`);
+        }
+        if (node.ownEndLine < node.startLine || node.ownEndLine > node.endLine) {
+            throw new Error(`coverage: "${label}" has invalid own range ${node.startLine}-${node.ownEndLine} (end ${node.endLine})`);
+        }
+        const firstChild = node.children[0];
+        if (firstChild && firstChild.startLine !== node.ownEndLine + 1) {
+            throw new Error(`coverage: "${label}" ownEndLine ${node.ownEndLine} does not precede first child at ${firstChild.startLine}`);
+        }
+        const lastChild = node.children[node.children.length - 1];
+        if (lastChild && lastChild.endLine !== node.endLine) {
+            throw new Error(`coverage: "${label}" endLine ${node.endLine} != last child endLine ${lastChild.endLine}`);
+        }
+        expected = node.ownEndLine + 1;
+    }
+    if (expected !== Math.max(first, total + 1)) {
+        throw new Error(`coverage: lines ${expected}..${total} are not covered`);
+    }
+}