markindex 0.0.0-stage → 0.1.0

package/dist/store/db.js

@@ -0,0 +1,399 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { linkSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import Database from 'better-sqlite3';
+import { segmentForIndex } from '../text/segment.js';
+export class StoreLockedError extends Error {
+    lockFile;
+    pid;
+    constructor(lockFile, pid) {
+        super(`index is locked by another writer (pid ${pid ?? 'unknown'}): ${lockFile}. ` +
+            'Wait for it to finish, or open the store read-only.');
+        this.lockFile = lockFile;
+        this.pid = pid;
+        this.name = 'StoreLockedError';
+    }
+}
+const SCHEMA_VERSION = 2;
+const LINE_CHUNK = 200;
+const MIGRATIONS = {
+    1: `
+    CREATE TABLE docs (
+      path TEXT PRIMARY KEY,
+      title TEXT NOT NULL,
+      description TEXT,
+      metadata TEXT NOT NULL,
+      content_hash TEXT NOT NULL,
+      line_count INTEGER NOT NULL,
+      tokens INTEGER NOT NULL,
+      indexed_at TEXT NOT NULL
+    );
+    CREATE TABLE nodes (
+      doc_path TEXT NOT NULL,
+      node_id TEXT NOT NULL,
+      parent_id TEXT,
+      ord INTEGER NOT NULL,
+      json TEXT NOT NULL,
+      fts_rowid INTEGER NOT NULL,
+      PRIMARY KEY (doc_path, node_id)
+    );
+    CREATE TABLE lines (
+      doc_path TEXT NOT NULL,
+      start_line INTEGER NOT NULL,
+      text TEXT NOT NULL,
+      PRIMARY KEY (doc_path, start_line)
+    );
+    CREATE TABLE summaries (
+      content_hash TEXT PRIMARY KEY,
+      summary TEXT NOT NULL
+    );
+    CREATE VIRTUAL TABLE fts_words USING fts5(
+      title, key_items, summary, body, doc_path UNINDEXED, node_id UNINDEXED,
+      tokenize = "unicode61 tokenchars '+#.-_'"
+    );
+    CREATE VIRTUAL TABLE fts_tri USING fts5(
+      title, body, doc_path UNINDEXED, node_id UNINDEXED,
+      tokenize = 'trigram'
+    );
+  `,
+    2: `
+    CREATE TABLE meta (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `,
+};
+/** Locks held by this process, released on exit. */
+const heldLocks = new Set();
+let exitHookInstalled = false;
+function releaseLock(lockFile) {
+    if (!heldLocks.delete(lockFile))
+        return;
+    try {
+        unlinkSync(lockFile);
+    }
+    catch {
+        // already gone
+    }
+}
+function pidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        return e.code === 'EPERM';
+    }
+}
+/** An empty / unparsable lock younger than this is assumed to be held (its writer may be mid-takeover). */
+const UNREADABLE_LOCK_GRACE_MS = 10_000;
+const MAX_LOCK_ATTEMPTS = 5;
+const uniqueSuffix = () => `${process.pid}.${randomBytes(6).toString('hex')}`;
+function parsePid(content) {
+    const t = content.trim();
+    if (!/^\d+$/.test(t))
+        return undefined;
+    const n = Number(t);
+    return Number.isSafeInteger(n) && n > 0 ? n : undefined;
+}
+/** Atomically creates `lockFile` with our pid: the lock never exists without its content. */
+function tryLink(lockFile) {
+    const tmp = `${lockFile}.${uniqueSuffix()}.tmp`;
+    writeFileSync(tmp, `${process.pid}\n`);
+    try {
+        linkSync(tmp, lockFile);
+        return true;
+    }
+    catch (e) {
+        if (e.code === 'EEXIST')
+            return false;
+        throw e;
+    }
+    finally {
+        rmSync(tmp, { force: true });
+    }
+}
+function holdLock(lockFile) {
+    heldLocks.add(lockFile);
+    if (!exitHookInstalled) {
+        exitHookInstalled = true;
+        process.on('exit', () => {
+            for (const f of [...heldLocks])
+                releaseLock(f);
+        });
+    }
+}
+function acquireLock(lockFile) {
+    for (let attempt = 0; attempt < MAX_LOCK_ATTEMPTS; attempt++) {
+        if (tryLink(lockFile)) {
+            holdLock(lockFile);
+            return;
+        }
+        let content;
+        let mtimeMs;
+        try {
+            content = readFileSync(lockFile, 'utf8');
+            mtimeMs = statSync(lockFile).mtimeMs;
+        }
+        catch (e) {
+            if (e.code === 'ENOENT')
+                continue; // released meanwhile: retry
+            throw e;
+        }
+        const pid = parsePid(content);
+        if (pid === undefined) {
+            if (Date.now() - mtimeMs < UNREADABLE_LOCK_GRACE_MS)
+                throw new StoreLockedError(lockFile, undefined);
+        }
+        else if (pidAlive(pid)) {
+            throw new StoreLockedError(lockFile, pid);
+        }
+        // Stale: move it aside atomically, then confirm we moved the lock we judged (not a fresh one).
+        const aside = `${lockFile}.stale.${uniqueSuffix()}`;
+        try {
+            renameSync(lockFile, aside);
+        }
+        catch (e) {
+            if (e.code === 'ENOENT')
+                continue; // someone else took it over
+            throw e;
+        }
+        let movedContent;
+        try {
+            movedContent = readFileSync(aside, 'utf8');
+        }
+        catch {
+            movedContent = undefined;
+        }
+        if (movedContent !== content) {
+            // We displaced a lock created after our read: put it back (unless a newer one exists) and back off.
+            try {
+                linkSync(aside, lockFile);
+            }
+            catch {
+                // a newer lock is already in place
+            }
+            rmSync(aside, { force: true });
+            throw new StoreLockedError(lockFile, movedContent === undefined ? undefined : parsePid(movedContent));
+        }
+        rmSync(aside, { force: true });
+    }
+    throw new StoreLockedError(lockFile, undefined);
+}
+function migrate(db) {
+    const current = db.pragma('user_version', { simple: true });
+    for (let v = current + 1; v <= SCHEMA_VERSION; v++) {
+        db.transaction(() => {
+            db.exec(MIGRATIONS[v]);
+            db.pragma(`user_version = ${v}`);
+        })();
+    }
+}
+/**
+ * Opens (creating if needed) a store file. Write mode takes an exclusive `${file}.lock` holding the pid, created
+ * atomically (temp file + link). A lock whose pid is dead, or an empty / unparsable lock older than 10 s, is taken over. Read-only opens take no lock and never block on a writer (WAL).
+ */
+export function openStore(file, opts = {}) {
+    if (opts.readonly) {
+        const db = new Database(file, { readonly: true, fileMustExist: true });
+        return createStore(db, () => { });
+    }
+    mkdirSync(dirname(file), { recursive: true });
+    const lockFile = `${file}.lock`;
+    acquireLock(lockFile);
+    let db;
+    try {
+        db = new Database(file);
+        db.pragma('journal_mode = WAL');
+        db.pragma('synchronous = NORMAL');
+        migrate(db);
+    }
+    catch (e) {
+        try {
+            db?.close();
+        }
+        finally {
+            releaseLock(lockFile);
+        }
+        throw e;
+    }
+    return createStore(db, () => releaseLock(lockFile));
+}
+/** Stable short id derived from the library-relative path. */
+function docIdFor(path) {
+    return createHash('sha1').update(path).digest('hex').slice(0, 12);
+}
+function toRecord(r) {
+    const rec = {
+        docId: docIdFor(r.path),
+        path: r.path,
+        title: r.title,
+        metadata: JSON.parse(r.metadata),
+        contentHash: r.content_hash,
+        lineCount: r.line_count,
+        tokens: r.tokens,
+        indexedAt: r.indexed_at,
+    };
+    if (r.description !== null)
+        rec.description = r.description;
+    return rec;
+}
+function createStore(db, unlock) {
+    let closed = false;
+    // Prepared on first use, so a read-only store never prepares write statements.
+    const cache = new Map();
+    const stmt = (sql) => {
+        let s = cache.get(sql);
+        if (!s) {
+            s = db.prepare(sql);
+            cache.set(sql, s);
+        }
+        return s;
+    };
+    const deleteRows = (path) => {
+        stmt('DELETE FROM fts_words WHERE rowid IN (SELECT fts_rowid FROM nodes WHERE doc_path = ?)').run(path);
+        stmt('DELETE FROM fts_tri WHERE rowid IN (SELECT fts_rowid FROM nodes WHERE doc_path = ?)').run(path);
+        stmt('DELETE FROM nodes WHERE doc_path = ?').run(path);
+        stmt('DELETE FROM lines WHERE doc_path = ?').run(path);
+        stmt('DELETE FROM docs WHERE path = ?').run(path);
+    };
+    const replaceTx = db.transaction((doc, tree, lines) => {
+        const path = doc.path;
+        deleteRows(path);
+        stmt(`INSERT INTO docs (path, title, description, metadata, content_hash, line_count, tokens, indexed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run(path, doc.title, doc.description ?? null, JSON.stringify(doc.metadata), doc.contentHash, doc.lineCount, doc.tokens, doc.indexedAt);
+        for (let i = 0; i < lines.length; i += LINE_CHUNK) {
+            stmt('INSERT INTO lines (doc_path, start_line, text) VALUES (?, ?, ?)').run(path, i + 1, lines.slice(i, i + LINE_CHUNK).join('\n'));
+        }
+        const insertNodes = (nodes, parentId) => {
+            nodes.forEach((node, ord) => {
+                const { children, ...own } = node;
+                const json = JSON.stringify(own);
+                const body = lines.slice(node.startLine - 1, node.ownEndLine).join('\n');
+                const keyItems = (node.keyItems ?? []).join('\n');
+                const info = stmt('INSERT INTO fts_words (title, key_items, summary, body, doc_path, node_id) VALUES (?, ?, ?, ?, ?, ?)').run(segmentForIndex(node.title), segmentForIndex(keyItems), '', segmentForIndex(body), path, node.nodeId);
+                stmt('INSERT INTO fts_tri (rowid, title, body, doc_path, node_id) VALUES (?, ?, ?, ?, ?)').run(info.lastInsertRowid, node.title, body, path, node.nodeId);
+                stmt('INSERT INTO nodes (doc_path, node_id, parent_id, ord, json, fts_rowid) VALUES (?, ?, ?, ?, ?, ?)').run(path, node.nodeId, parentId, ord, json, info.lastInsertRowid);
+                insertNodes(children, node.nodeId);
+            });
+        };
+        insertNodes(tree, null);
+    });
+    const updateSummariesTx = db.transaction((docPath, updates) => {
+        for (const u of updates) {
+            const row = stmt('SELECT json, fts_rowid FROM nodes WHERE doc_path = ? AND node_id = ?').get(docPath, u.nodeId);
+            if (!row)
+                throw new Error(`node ${u.nodeId} not found in ${docPath}`);
+            const node = JSON.parse(row.json);
+            if (u.summary !== undefined)
+                node.summary = u.summary;
+            if (u.prefixSummary !== undefined)
+                node.prefixSummary = u.prefixSummary;
+            stmt('UPDATE nodes SET json = ? WHERE doc_path = ? AND node_id = ?').run(JSON.stringify(node), docPath, u.nodeId);
+            stmt('UPDATE fts_words SET summary = ? WHERE rowid = ?').run(segmentForIndex(node.summary ?? node.prefixSummary ?? ''), row.fts_rowid);
+        }
+    });
+    const deleteTx = db.transaction((path) => deleteRows(path));
+    const getDoc = (path) => {
+        const row = stmt('SELECT * FROM docs WHERE path = ?').get(path);
+        return row && toRecord(row);
+    };
+    return {
+        db,
+        getDoc,
+        listDocs(folder, offset, limit) {
+            const prefix = folder.replace(/^\/+|\/+$/g, '');
+            const p = prefix === '' ? '' : `${prefix}/`;
+            const paths = stmt('SELECT path FROM docs WHERE substr(path, 1, length(?)) = ? ORDER BY path').all(p, p).map((r) => r.path);
+            const folders = new Set();
+            const direct = [];
+            for (const path of paths) {
+                const rest = path.slice(p.length);
+                const slash = rest.indexOf('/');
+                if (slash === -1)
+                    direct.push(path);
+                else
+                    folders.add(p + rest.slice(0, slash));
+            }
+            const page = direct.slice(Math.max(0, offset), Math.max(0, offset) + Math.max(0, limit));
+            const docs = page.map((path) => getDoc(path));
+            return { folders: [...folders].sort(), docs, total: direct.length };
+        },
+        allDocPaths() {
+            return stmt('SELECT path FROM docs ORDER BY path').all().map((r) => r.path);
+        },
+        getTree(path) {
+            if (!stmt('SELECT 1 FROM docs WHERE path = ?').get(path))
+                return undefined;
+            const rows = stmt('SELECT node_id, parent_id, ord, json FROM nodes WHERE doc_path = ? ORDER BY ord').all(path);
+            const byId = new Map();
+            for (const r of rows)
+                byId.set(r.node_id, { ...JSON.parse(r.json), children: [] });
+            const roots = [];
+            for (const r of rows) {
+                const node = byId.get(r.node_id);
+                const parent = r.parent_id === null ? undefined : byId.get(r.parent_id);
+                if (parent)
+                    parent.children.push(node);
+                else
+                    roots.push(node);
+            }
+            return roots;
+        },
+        getLines(path, start, end) {
+            const from = Math.max(1, Math.floor(start));
+            const to = Math.floor(end);
+            if (to < from)
+                return [];
+            const firstChunk = Math.floor((from - 1) / LINE_CHUNK) * LINE_CHUNK + 1;
+            const rows = stmt('SELECT start_line, text FROM lines WHERE doc_path = ? AND start_line >= ? AND start_line <= ? ORDER BY start_line').all(path, firstChunk, to);
+            const out = [];
+            for (const r of rows) {
+                r.text.split('\n').forEach((line, i) => {
+                    const n = r.start_line + i;
+                    if (n >= from && n <= to)
+                        out.push(line);
+                });
+            }
+            return out;
+        },
+        replaceDoc(doc, tree, lines) {
+            replaceTx(doc, tree, lines);
+        },
+        deleteDoc(path) {
+            deleteTx(path);
+        },
+        updateNodeSummaries(docPath, updates) {
+            updateSummariesTx(docPath, updates);
+        },
+        setDocDescription(docPath, description) {
+            stmt('UPDATE docs SET description = ? WHERE path = ?').run(description, docPath);
+        },
+        getCachedSummary(contentHash) {
+            return stmt('SELECT summary FROM summaries WHERE content_hash = ?').get(contentHash)?.summary;
+        },
+        putCachedSummary(contentHash, summary) {
+            stmt('INSERT OR REPLACE INTO summaries (content_hash, summary) VALUES (?, ?)').run(contentHash, summary);
+        },
+        getMeta(key) {
+            // a read-only store opened on an index written before schema v2 has no meta table
+            if (!stmt("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").get())
+                return undefined;
+            return stmt('SELECT value FROM meta WHERE key = ?').get(key)?.value;
+        },
+        setMeta(key, value) {
+            stmt('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)').run(key, value);
+        },
+        close() {
+            if (closed)
+                return;
+            closed = true;
+            try {
+                db.close();
+            }
+            finally {
+                unlock();
+            }
+        },
+    };
+}

package/dist/store/search.d.ts

@@ -0,0 +1,27 @@
+import type { Store } from './db.js';
+export interface SearchHit {
+    docPath: string;
+    nodeId: string;
+    /** Ancestor titles from the top-level section down to the hit node. */
+    titlePath: string[];
+    startLine: number;
+    endLine: number;
+    /** <= 300 chars of the node's own original text around the first match. */
+    snippet: string;
+    /** Higher is better (negated FTS5 bm25). Only comparable within the same `via`. */
+    score: number;
+    via: 'bm25' | 'trigram';
+}
+export interface SearchOptions {
+    /** Restrict to documents under this folder ('' or omitted = whole library). */
+    folder?: string;
+    /** Restrict to exactly this document; takes precedence over `folder`. */
+    docPath?: string;
+    limit?: number;
+}
+/**
+ * BM25 node search with trigram fallback (spec §7). Ranks `fts_words` by bm25(title 10, key_items 5, summary 2,
+ * body 1); if that yields fewer than `limit` nodes and the whitespace-stripped query has >= 3 chars, the
+ * `fts_tri` substring index tops the list up (deduplicated by doc+node). Never throws on malformed input.
+ */
+export declare function searchNodes(store: Store, query: string, opts?: SearchOptions): SearchHit[];

package/dist/store/search.js

@@ -0,0 +1,116 @@
+import { segmentQuery } from '../text/segment.js';
+const SNIPPET_MAX = 300;
+const DEFAULT_LIMIT = 10;
+/** Own text beyond this many chars is never scanned for the snippet window. */
+const SNIPPET_SCAN_MAX = 20_000;
+function scopeClause(opts) {
+    if (opts.docPath !== undefined && opts.docPath !== '')
+        return { sql: 'AND f.doc_path = ?', params: [opts.docPath] };
+    const folder = (opts.folder ?? '').replace(/^\/+|\/+$/g, '');
+    if (folder === '')
+        return { sql: '', params: [] };
+    const prefix = `${folder}/`;
+    return { sql: 'AND substr(f.doc_path, 1, length(?)) = ?', params: [prefix, prefix] };
+}
+/** Quoted tokens of a segmentQuery() expression, un-escaped. */
+function queryTokens(expr) {
+    return [...expr.matchAll(/"((?:[^"]|"")*)"/g)].map((m) => m[1].replaceAll('""', '"'));
+}
+/**
+ * Snippet from the ORIGINAL own text of the node: a window of <= 300 chars starting a little before the first
+ * occurrence of any needle (case-insensitive); falls back to the start of the text, then to the title.
+ */
+function makeSnippet(text, needles, title) {
+    const body = text.slice(0, SNIPPET_SCAN_MAX);
+    if (body.trim() === '')
+        return title.slice(0, SNIPPET_MAX);
+    const lower = body.toLowerCase();
+    let at = -1;
+    for (const n of needles) {
+        const i = n === '' ? -1 : lower.indexOf(n.toLowerCase());
+        if (i !== -1 && (at === -1 || i < at))
+            at = i;
+    }
+    const from = at === -1 ? 0 : Math.max(0, at - 60);
+    return body.slice(from, from + SNIPPET_MAX).trim();
+}
+/**
+ * BM25 node search with trigram fallback (spec §7). Ranks `fts_words` by bm25(title 10, key_items 5, summary 2,
+ * body 1); if that yields fewer than `limit` nodes and the whitespace-stripped query has >= 3 chars, the
+ * `fts_tri` substring index tops the list up (deduplicated by doc+node). Never throws on malformed input.
+ */
+export function searchNodes(store, query, opts = {}) {
+    const limit = Math.max(0, Math.floor(opts.limit ?? DEFAULT_LIMIT));
+    if (limit === 0)
+        return [];
+    const scope = scopeClause(opts);
+    const { db } = store;
+    const hits = [];
+    const seen = new Set();
+    const titleCache = new Map();
+    const nodeStmt = db.prepare("SELECT json_extract(json, '$.title') AS t, parent_id AS p FROM nodes WHERE doc_path = ? AND node_id = ?");
+    const titlePath = (docPath, nodeId) => {
+        const out = [];
+        let cur = nodeId;
+        while (cur !== null) {
+            const key = `${docPath}\0${cur}`;
+            let e = titleCache.get(key);
+            if (!e) {
+                const r = nodeStmt.get(docPath, cur);
+                if (!r)
+                    break;
+                e = { title: r.t, parent: r.p };
+                titleCache.set(key, e);
+            }
+            out.push(e.title);
+            cur = e.parent;
+        }
+        return out.reverse();
+    };
+    const push = (r, needles, via) => {
+        const key = `${r.doc_path}\0${r.node_id}`;
+        if (seen.has(key) || hits.length >= limit)
+            return;
+        seen.add(key);
+        const node = JSON.parse(r.json);
+        const own = store.getLines(r.doc_path, node.startLine, node.ownEndLine).join('\n');
+        hits.push({
+            docPath: r.doc_path,
+            nodeId: r.node_id,
+            titlePath: titlePath(r.doc_path, r.node_id),
+            startLine: node.startLine,
+            endLine: node.endLine,
+            snippet: makeSnippet(own, needles, node.title),
+            score: -r.rank,
+            via,
+        });
+    };
+    const run = (table, match, rankExpr) => {
+        try {
+            return db
+                .prepare(`SELECT f.doc_path AS doc_path, f.node_id AS node_id, n.json AS json, ${rankExpr} AS rank
+           FROM ${table} f JOIN nodes n ON n.doc_path = f.doc_path AND n.node_id = f.node_id
+           WHERE ${table} MATCH ? ${scope.sql}
+           ORDER BY rank LIMIT ?`)
+                .all(match, ...scope.params, limit);
+        }
+        catch {
+            return []; // a MATCH expression FTS5 refuses is treated as "no results"
+        }
+    };
+    const words = segmentQuery(query);
+    if (words !== '') {
+        const needles = queryTokens(words);
+        for (const r of run('fts_words', words, 'bm25(fts_words, 10.0, 5.0, 2.0, 1.0, 0.0, 0.0)'))
+            push(r, needles, 'bm25');
+    }
+    if (hits.length < limit) {
+        const compact = query.replace(/\s+/g, '');
+        if ([...compact].length >= 3) {
+            const phrase = `"${compact.replaceAll('"', '""')}"`;
+            for (const r of run('fts_tri', phrase, 'bm25(fts_tri, 10.0, 1.0, 0.0, 0.0)'))
+                push(r, [compact], 'trigram');
+        }
+    }
+    return hits;
+}

package/dist/text/segment.d.ts

@@ -0,0 +1,4 @@
+/** Space-separated lowercase tokens, suitable for an FTS5 `unicode61` column. */
+export declare function segmentForIndex(text: string): string;
+/** FTS5 MATCH expression: each token double-quoted, joined with OR (ranked by bm25). Empty if no tokens. */
+export declare function segmentQuery(q: string): string;

package/dist/text/segment.js

@@ -0,0 +1,37 @@
+const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' });
+/**
+ * Terms like v2.1, C++, ISO-27001 that the word segmenter would split or truncate. The lookbehind anchors each
+ * attempt at the start of an alphanumeric run: without it a long run with no separator backtracks from every
+ * position (quadratic). Matches are unchanged, since a match never starts inside a run.
+ */
+const COMPOUND = /(?<![A-Za-z0-9])[A-Za-z0-9]+(?:[.\-_][A-Za-z0-9]+|[+#]+)+/g;
+/** Appends the lowercased word-like segments of `text` to `out` (no spread: spans may hold 10^5+ words). */
+function words(text, out) {
+    for (const s of segmenter.segment(text)) {
+        if (s.isWordLike)
+            out.push(s.segment.toLowerCase());
+    }
+}
+/** NFKC first, so full-width forms (ＡＢＣ, ２７００１) index and query like their ASCII forms. */
+function tokenize(raw) {
+    const text = raw.normalize('NFKC');
+    const out = [];
+    let last = 0;
+    for (const m of text.matchAll(COMPOUND)) {
+        words(text.slice(last, m.index), out);
+        out.push(m[0].toLowerCase());
+        last = m.index + m[0].length;
+    }
+    words(text.slice(last), out);
+    return out;
+}
+/** Space-separated lowercase tokens, suitable for an FTS5 `unicode61` column. */
+export function segmentForIndex(text) {
+    return tokenize(text).join(' ');
+}
+/** FTS5 MATCH expression: each token double-quoted, joined with OR (ranked by bm25). Empty if no tokens. */
+export function segmentQuery(q) {
+    return tokenize(q)
+        .map((t) => `"${t.replaceAll('"', '""')}"`)
+        .join(' OR ');
+}

package/dist/text/tokens.d.ts

@@ -0,0 +1,3 @@
+export type TokenCounter = (text: string) => number;
+/** Approximate tokens: each CJK character is 1 token, everything else is chars / 4 rounded up. */
+export declare const approxTokens: TokenCounter;

package/dist/text/tokens.js

@@ -0,0 +1,13 @@
+const CJK = /\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul}/u;
+/** Approximate tokens: each CJK character is 1 token, everything else is chars / 4 rounded up. */
+export const approxTokens = (text) => {
+    let cjk = 0;
+    let other = 0;
+    for (const ch of text) {
+        if (CJK.test(ch))
+            cjk++;
+        else
+            other++;
+    }
+    return cjk + Math.ceil(other / 4);
+};

package/dist/tools/contract.d.ts

@@ -0,0 +1,51 @@
+import { z } from 'zod';
+export type ToolName = 'browse_documents' | 'get_document' | 'get_document_structure' | 'get_node_content' | 'search_documents';
+/**
+ * Tool names, descriptions and input schemas. The descriptions are the agent's usage guide and part of the
+ * contract (spec §8): a snapshot test guards them.
+ */
+export declare const TOOL_CONTRACT: {
+    readonly browse_documents: {
+        readonly description: string;
+        readonly input: z.ZodObject<{
+            folder: z.ZodDefault<z.ZodString>;
+            offset: z.ZodDefault<z.ZodInt>;
+            limit: z.ZodDefault<z.ZodInt>;
+        }, z.core.$strict>;
+    };
+    readonly get_document: {
+        readonly description: string;
+        readonly input: z.ZodObject<{
+            doc_path: z.ZodString;
+        }, z.core.$strict>;
+    };
+    readonly get_document_structure: {
+        readonly description: string;
+        readonly input: z.ZodObject<{
+            doc_path: z.ZodString;
+            node_id: z.ZodOptional<z.ZodString>;
+            depth: z.ZodDefault<z.ZodInt>;
+            part: z.ZodDefault<z.ZodInt>;
+        }, z.core.$strict>;
+    };
+    readonly get_node_content: {
+        readonly description: string;
+        readonly input: z.ZodObject<{
+            doc_path: z.ZodString;
+            node_ids: z.ZodArray<z.ZodString>;
+            part: z.ZodDefault<z.ZodInt>;
+        }, z.core.$strict>;
+    };
+    readonly search_documents: {
+        readonly description: string;
+        readonly input: z.ZodObject<{
+            query: z.ZodString;
+            folder: z.ZodOptional<z.ZodString>;
+            doc_path: z.ZodOptional<z.ZodString>;
+            limit: z.ZodDefault<z.ZodInt>;
+        }, z.core.$strict>;
+    };
+};
+export type ToolInput<N extends ToolName> = z.output<(typeof TOOL_CONTRACT)[N]['input']>;
+/** System-prompt guidance for an agent using the tools in `TOOL_CONTRACT`. */
+export declare const AGENT_INSTRUCTIONS: string;