markindex 0.0.0-stage → 0.1.1

package/dist/library/indexer.js

@@ -0,0 +1,199 @@
+import { createHash } from 'node:crypto';
+import { INDEX_FORMAT_VERSION, MAX_NODE_TOKENS, ROUTING_COST_TOKENS } from '../config.js';
+import { readFileSync } from 'node:fs';
+import { basename, extname } from 'node:path';
+import { parseMarkdown } from '../parse/markdown.js';
+import { approxTokens } from '../text/tokens.js';
+import { buildTree } from '../tree/build.js';
+import { optimizeTree } from '../tree/optimize.js';
+import { summarizeDocument } from '../llm/summarize.js';
+import { assertLibraryRoot, scanLibrary } from './scan.js';
+const FORMAT_KEY = 'index_format';
+/** Fingerprint of everything besides file content that shapes the stored trees (INDEX_FORMAT_VERSION + build parameters). */
+export function indexFormat(opts) {
+    const custom = opts.countTokens !== undefined && opts.countTokens !== approxTokens;
+    return JSON.stringify({
+        version: INDEX_FORMAT_VERSION,
+        maxNodeTokens: opts.maxNodeTokens ?? MAX_NODE_TOKENS,
+        routingCost: opts.routingCost ?? ROUTING_COST_TOKENS,
+        tokens: custom ? `custom${opts.tokenCounterId === undefined ? '' : `:${opts.tokenCounterId}`}` : 'approx',
+        // only recorded when off, so indexes built before the option existed (setext always on) are not rebuilt
+        ...(opts.setext === false ? { setext: false } : {}),
+    });
+}
+/** A setext h2 whose heading text is longer than this many code points (or spans several lines) looks like a page. */
+export const SUSPECT_SETEXT_CHARS = 120;
+/**
+ * Setext h2 headings that are probably a converter's page break (`---` right under a paragraph) rather than a real
+ * heading: heading text over SUSPECT_SETEXT_CHARS code points or spread over more than one source line.
+ */
+function suspectSetextHeadings(doc) {
+    const out = [];
+    for (const h of doc.headings) {
+        if (h.level !== 2 || h.endLine === h.line)
+            continue; // ATX headings are one line; setext ones span text + underline
+        const textLines = h.endLine - h.line;
+        const raw = doc.lines.slice(h.line - 1, h.endLine - 1).join('\n').trim();
+        if (textLines > 1 || Array.from(raw).length > SUSPECT_SETEXT_CHARS)
+            out.push(h.line);
+    }
+    return out;
+}
+/** The setext setting recorded in a stored fingerprint (true when absent or unreadable). */
+function storedSetext(stored) {
+    if (stored === undefined)
+        return true;
+    try {
+        return JSON.parse(stored).setext !== false;
+    }
+    catch {
+        return true;
+    }
+}
+const utf8 = new TextDecoder('utf-8', { fatal: true });
+/**
+ * Incrementally indexes the Markdown files under `root` (spec §6): unchanged content hashes are skipped,
+ * new / changed files are rebuilt one transaction per document, and documents missing on disk are deleted.
+ * A file that cannot be read, decoded or indexed is reported in `skipped`; any previous version of it is
+ * dropped so the index never serves content that no longer matches the disk. An unreadable subdirectory is
+ * reported in `skipped` (path with trailing `/`) and the documents previously indexed under it are kept.
+ * When the stored index format (see `indexFormat`) differs from this run's, every document is re-indexed
+ * (summaries are reused through the content-hash cache) and the report carries `rebuilt`.
+ */
+export async function indexLibrary(opts) {
+    assertLibraryRoot(opts.root);
+    const { store } = opts;
+    const countTokens = opts.countTokens ?? approxTokens;
+    const report = { added: [], updated: [], removed: [], unchanged: [], skipped: [], summaryErrors: [], warnings: [] };
+    const storedFormat = store.getMeta(FORMAT_KEY);
+    const setext = opts.setext ?? storedSetext(storedFormat);
+    const format = indexFormat({ ...opts, setext });
+    const rebuild = storedFormat !== format && store.allDocPaths().length > 0;
+    if (rebuild) {
+        report.rebuilt = {
+            reason: `index format changed (stored ${storedFormat ?? 'none'}, now ${format}): re-indexing every document`,
+        };
+    }
+    const unreadableDirs = [];
+    const files = scanLibrary(opts.root, {
+        ignore: opts.ignore,
+        onError: (dir, error) => {
+            unreadableDirs.push(dir);
+            report.skipped.push({ path: dir, reason: `directory not readable: ${error.message}` });
+        },
+    });
+    const total = files.length;
+    let done = 0;
+    const emit = (path, status, reason) => opts.onProgress?.(reason === undefined ? { path, status, done, total } : { path, status, reason, done, total });
+    const summarize = async (path) => {
+        if (!opts.summary)
+            return;
+        try {
+            const res = await summarizeDocument(store, path, opts.summary);
+            for (const f of res.failures)
+                report.summaryErrors.push({ path, ...f });
+        }
+        catch (e) {
+            report.summaryErrors.push({ path, nodeId: null, message: errorMessage(e) });
+        }
+    };
+    for (const file of files) {
+        done++;
+        const existing = store.getDoc(file.path);
+        try {
+            const bytes = readFileSync(file.abs);
+            const hash = createHash('sha256').update(bytes).digest('hex');
+            if (!rebuild && existing?.contentHash === hash) {
+                report.unchanged.push(file.path);
+                emit(file.path, 'unchanged');
+                await summarize(file.path); // enables summaries on an existing index and retries earlier failures (cache-assisted)
+                continue;
+            }
+            let src;
+            try {
+                src = utf8.decode(bytes);
+            }
+            catch {
+                throw new Error('not valid utf-8 text');
+            }
+            const doc = parseMarkdown(src, { setext });
+            const suspects = setext ? suspectSetextHeadings(doc) : [];
+            const title = docTitle(doc, file.path);
+            const tree = optimizeTree(buildTree(doc, { fileTitle: title, countTokens }), doc, {
+                countTokens,
+                maxNodeTokens: opts.maxNodeTokens,
+                routingCost: opts.routingCost,
+            });
+            const record = {
+                docId: '',
+                path: file.path,
+                title,
+                metadata: doc.frontmatter?.data ?? {},
+                contentHash: hash,
+                lineCount: doc.lines.length,
+                tokens: tree.reduce((sum, n) => sum + n.tokens, 0),
+                indexedAt: new Date().toISOString(),
+            };
+            store.replaceDoc(record, tree, doc.lines);
+            if (suspects.length > 0) {
+                report.warnings.push({
+                    path: file.path,
+                    message: `${suspects.length} setext h2 heading(s) look like page breaks (a \`---\` line under a paragraph that spans ` +
+                        `several lines or exceeds ${SUSPECT_SETEXT_CHARS} characters), first at line ${suspects[0]}; if \`---\` ` +
+                        'marks page breaks in this file, index with setext headings off',
+                });
+            }
+            const status = existing ? 'updated' : 'added';
+            report[status].push(file.path);
+            emit(file.path, status);
+            await summarize(file.path);
+        }
+        catch (e) {
+            let reason = errorMessage(e);
+            if (existing) {
+                try {
+                    store.deleteDoc(file.path);
+                }
+                catch (e2) {
+                    reason += `; previous version could not be removed: ${errorMessage(e2)}`;
+                }
+            }
+            report.skipped.push({ path: file.path, reason });
+            emit(file.path, 'skipped', reason);
+        }
+    }
+    const onDisk = new Set(files.map((f) => f.path));
+    for (const path of store.allDocPaths()) {
+        if (onDisk.has(path))
+            continue;
+        // Contents of an unreadable directory are unknown: keep their previous versions.
+        if (unreadableDirs.some((dir) => path.startsWith(dir)))
+            continue;
+        try {
+            store.deleteDoc(path);
+        }
+        catch (e) {
+            const reason = `could not remove deleted document: ${errorMessage(e)}`;
+            report.skipped.push({ path, reason });
+            emit(path, 'skipped', reason);
+            continue;
+        }
+        report.removed.push(path);
+        emit(path, 'removed');
+    }
+    // Written last: an interrupted rebuild keeps the old fingerprint and is redone in full next time.
+    if (storedFormat !== format)
+        store.setMeta(FORMAT_KEY, format);
+    return report;
+}
+const errorMessage = (e) => (e instanceof Error ? e.message : String(e));
+/** frontmatter `title` > first h1 > file name without extension. */
+function docTitle(doc, path) {
+    const fm = doc.frontmatter?.data.title;
+    if (typeof fm === 'string' && fm.trim() !== '')
+        return fm.trim();
+    const h1 = doc.headings.find((h) => h.level === 1 && h.title !== '');
+    if (h1)
+        return h1.title;
+    return basename(path, extname(path));
+}

package/dist/library/scan.d.ts

@@ -0,0 +1,29 @@
+export interface ScannedFile {
+    /** Library-relative path with posix separators. */
+    path: string;
+    abs: string;
+}
+/** Optional ignore file at the library root (gitignore syntax), for rules that should not live in .gitignore. */
+export declare const IGNORE_FILE = ".markindexignore";
+export interface ScanOptions {
+    /** Called for a subdirectory that cannot be read (e.g. EACCES); it is skipped and the scan continues. */
+    onError?: (path: string, error: Error) => void;
+    /** Extra gitignore-syntax patterns, relative to `root`, applied on top of `.gitignore` and `.markindexignore`. */
+    ignore?: string[];
+}
+/**
+ * Makes `ignore` option patterns usable as root-level gitignore patterns: a leading `./` (also after `!`) is
+ * stripped, since gitignore would never match it. A leading `/` stays a gitignore anchor (`/drafts/` = only the
+ * top-level folder). Throws for filesystem paths — a pattern starting with the library root's absolute path, a
+ * drive letter or `~` — and for `../` patterns, which can never match inside the library.
+ */
+export declare function normalizeIgnorePatterns(root: string, patterns: string[]): string[];
+/**
+ * Lists `.md` / `.markdown` files under `root` (spec §6), sorted by path.
+ * Skips `.mdx`, symbolic links (files and directories), dot-directories (`.git`, `.markindex`, ...)
+ * and anything matched by `.gitignore` files at any level, a `.markindexignore` file at the root, or `opts.ignore`. An unreadable root throws; an unreadable
+ * subdirectory is skipped and reported through `opts.onError` with its posix path plus a trailing `/`.
+ */
+export declare function scanLibrary(root: string, opts?: ScanOptions): ScannedFile[];
+/** Throws a clear error unless `root` is an existing directory. Never creates anything. */
+export declare function assertLibraryRoot(root: string): void;

package/dist/library/scan.js

@@ -0,0 +1,106 @@
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join, resolve, sep } from 'node:path';
+import ignore from 'ignore';
+const EXTENSIONS = ['.md', '.markdown'];
+/** Holds the index database. */
+const STORE_DIR = '.markindex';
+/** Optional ignore file at the library root (gitignore syntax), for rules that should not live in .gitignore. */
+export const IGNORE_FILE = '.markindexignore';
+/**
+ * Makes `ignore` option patterns usable as root-level gitignore patterns: a leading `./` (also after `!`) is
+ * stripped, since gitignore would never match it. A leading `/` stays a gitignore anchor (`/drafts/` = only the
+ * top-level folder). Throws for filesystem paths — a pattern starting with the library root's absolute path, a
+ * drive letter or `~` — and for `../` patterns, which can never match inside the library.
+ */
+export function normalizeIgnorePatterns(root, patterns) {
+    const absRoot = resolve(root).split(sep).join('/');
+    return patterns.map((raw) => {
+        const neg = raw.startsWith('!') ? '!' : '';
+        let p = raw.slice(neg.length); // backslashes are gitignore escapes, so they are left alone
+        if (p.startsWith(`${absRoot}/`) || p === absRoot || /^[A-Za-z]:[\\/]/.test(p) || p.startsWith('~')) {
+            const rel = p.startsWith(`${absRoot}/`) ? p.slice(absRoot.length + 1) : '<path relative to the library root>';
+            throw new Error(`ignore pattern "${raw}" is an absolute path; patterns are gitignore syntax relative to the library root, e.g. "${neg}${rel}"`);
+        }
+        while (p.startsWith('./'))
+            p = p.slice(2);
+        if (p === '..' || p.startsWith('../'))
+            throw new Error(`ignore pattern "${raw}" points outside the library root`);
+        return neg + p;
+    });
+}
+/**
+ * Lists `.md` / `.markdown` files under `root` (spec §6), sorted by path.
+ * Skips `.mdx`, symbolic links (files and directories), dot-directories (`.git`, `.markindex`, ...)
+ * and anything matched by `.gitignore` files at any level, a `.markindexignore` file at the root, or `opts.ignore`. An unreadable root throws; an unreadable
+ * subdirectory is skipped and reported through `opts.onError` with its posix path plus a trailing `/`.
+ */
+export function scanLibrary(root, opts = {}) {
+    const out = [];
+    const ignored = (rel, isDir, rules) => rules.some(({ base, ig }) => {
+        const sub = base === '' ? rel : rel.slice(base.length + 1);
+        return ig.ignores(isDir ? `${sub}/` : sub);
+    });
+    const visit = (dirRel, inherited) => {
+        const abs = dirRel === '' ? root : join(root, dirRel);
+        let entries;
+        try {
+            entries = readdirSync(abs, { withFileTypes: true });
+        }
+        catch (e) {
+            if (dirRel === '')
+                throw e;
+            opts.onError?.(`${dirRel}/`, e);
+            return;
+        }
+        const rules = [...inherited];
+        try {
+            rules.push({ base: dirRel, ig: ignore().add(readFileSync(join(abs, '.gitignore'), 'utf8')) });
+        }
+        catch {
+            // no .gitignore here
+        }
+        for (const entry of entries) {
+            if (entry.isSymbolicLink())
+                continue;
+            const rel = dirRel === '' ? entry.name : `${dirRel}/${entry.name}`;
+            if (entry.isDirectory()) {
+                if (entry.name.startsWith('.') || entry.name === STORE_DIR)
+                    continue;
+                if (ignored(rel, true, rules))
+                    continue;
+                visit(rel, rules);
+            }
+            else if (entry.isFile()) {
+                const lower = entry.name.toLowerCase();
+                if (!EXTENSIONS.some((ext) => lower.endsWith(ext)))
+                    continue;
+                if (ignored(rel, false, rules))
+                    continue;
+                out.push({ path: rel, abs: join(root, rel) });
+            }
+        }
+    };
+    const rootRules = [];
+    try {
+        rootRules.push({ base: '', ig: ignore().add(readFileSync(join(root, IGNORE_FILE), 'utf8')) });
+    }
+    catch {
+        // no .markindexignore
+    }
+    if (opts.ignore && opts.ignore.length > 0)
+        rootRules.push({ base: '', ig: ignore().add(normalizeIgnorePatterns(root, opts.ignore)) });
+    visit('', rootRules);
+    return out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+}
+/** Throws a clear error unless `root` is an existing directory. Never creates anything. */
+export function assertLibraryRoot(root) {
+    let st;
+    try {
+        st = statSync(root);
+    }
+    catch {
+        throw new Error(`Library root does not exist: ${root}`);
+    }
+    if (!st.isDirectory())
+        throw new Error(`Library root is not a directory: ${root}`);
+}

package/dist/library.d.ts

@@ -0,0 +1,30 @@
+import { type IndexOptions, type IndexReport } from './library/indexer.js';
+import { type Store } from './store/db.js';
+import type { ToolName } from './tools/contract.js';
+export interface OpenLibraryOptions {
+    /** Library root directory (the folder of Markdown files). */
+    root: string;
+    /** Index database; defaults to `${root}/.markindex/index.db`. */
+    dbFile?: string;
+    /** Defaults to true: agents only read, and a read-only store never waits on (or blocks) an indexer. */
+    readonly?: boolean;
+}
+export interface Library {
+    readonly store: Store;
+    /** Absolute library root. */
+    readonly root: string;
+    /** Runs an agent tool and returns its JSON envelope. Never rejects, whatever the name or arguments. */
+    callTool(name: ToolName, args: unknown): Promise<string>;
+    /**
+     * Incrementally (re)indexes the library into this library's database. A read-only library opens a separate
+     * writable store for the duration of the call (throws StoreLockedError if another writer holds the index).
+     */
+    index(opts?: Omit<IndexOptions, 'root' | 'store'>): Promise<IndexReport>;
+    close(): void;
+}
+export declare const defaultDbFile: (root: string) => string;
+/**
+ * Opens an existing library index for agent use. Does not index: run `markindex index` (or `indexLibrary`)
+ * first. Throws if a read-only open finds no index.
+ */
+export declare function openLibrary(opts: OpenLibraryOptions): Library;

package/dist/library.js

@@ -0,0 +1,42 @@
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { assertLibraryRoot } from './library/scan.js';
+import { indexLibrary } from './library/indexer.js';
+import { openStore } from './store/db.js';
+import { runTool } from './tools/impl.js';
+export const defaultDbFile = (root) => join(root, '.markindex', 'index.db');
+/**
+ * Opens an existing library index for agent use. Does not index: run `markindex index` (or `indexLibrary`)
+ * first. Throws if a read-only open finds no index.
+ */
+export function openLibrary(opts) {
+    const root = resolve(opts.root);
+    assertLibraryRoot(root);
+    const dbFile = resolve(opts.dbFile ?? defaultDbFile(root));
+    const readonly = opts.readonly ?? true;
+    if (readonly && !existsSync(dbFile)) {
+        throw new Error(`No MarkIndex index at ${dbFile}. Build it first with \`markindex index ${root}\`.`);
+    }
+    const store = openStore(dbFile, { readonly });
+    return {
+        store,
+        root,
+        async callTool(name, args) {
+            return runTool(store, name, args);
+        },
+        async index(indexOpts = {}) {
+            if (!readonly)
+                return indexLibrary({ ...indexOpts, root, store });
+            const writer = openStore(dbFile);
+            try {
+                return await indexLibrary({ ...indexOpts, root, store: writer });
+            }
+            finally {
+                writer.close();
+            }
+        },
+        close() {
+            store.close();
+        },
+    };
+}

package/dist/llm/provider.d.ts

@@ -0,0 +1,10 @@
+import type { LanguageModel } from 'ai';
+/** Thrown when a `provider:model` spec cannot be resolved; the message is user-facing. */
+export declare class ProviderError extends Error {
+}
+/**
+ * Resolves `<provider>:<model>` (e.g. `anthropic:claude-sonnet-5-5`) to a language model by dynamically importing
+ * `@ai-sdk/<provider>` and calling its `<provider>(model)` export. The provider packages are not dependencies of
+ * MarkIndex; a missing one yields a `ProviderError` with an install hint.
+ */
+export declare function resolveSummaryModel(spec: string, importer?: (specifier: string) => Promise<Record<string, unknown>>): Promise<LanguageModel>;

package/dist/llm/provider.js

@@ -0,0 +1,32 @@
+/** Thrown when a `provider:model` spec cannot be resolved; the message is user-facing. */
+export class ProviderError extends Error {
+}
+/**
+ * Resolves `<provider>:<model>` (e.g. `anthropic:claude-sonnet-5-5`) to a language model by dynamically importing
+ * `@ai-sdk/<provider>` and calling its `<provider>(model)` export. The provider packages are not dependencies of
+ * MarkIndex; a missing one yields a `ProviderError` with an install hint.
+ */
+export async function resolveSummaryModel(spec, importer = (s) => import(s)) {
+    const i = spec.indexOf(':');
+    const provider = i > 0 ? spec.slice(0, i) : '';
+    const modelId = i > 0 ? spec.slice(i + 1) : '';
+    if (!/^[a-z][a-z0-9-]*$/.test(provider) || modelId === '') {
+        throw new ProviderError(`Invalid --summary-model "${spec}". Expected <provider>:<model>, e.g. anthropic:claude-sonnet-5-5.`);
+    }
+    const pkg = `@ai-sdk/${provider}`;
+    let mod;
+    try {
+        mod = await importer(pkg);
+    }
+    catch (e) {
+        if (e.code === 'ERR_MODULE_NOT_FOUND' || e.code === 'MODULE_NOT_FOUND') {
+            throw new ProviderError(`Provider package ${pkg} is not installed. Install it with: npm install ${pkg}`);
+        }
+        throw e;
+    }
+    const factory = mod[provider];
+    if (typeof factory !== 'function') {
+        throw new ProviderError(`${pkg} does not export a "${provider}" model factory; cannot resolve "${spec}".`);
+    }
+    return factory(modelId);
+}

package/dist/llm/rate-limit.d.ts

@@ -0,0 +1,33 @@
+export interface RateLimitOptions {
+    /** Max LLM requests started in any 60 s window. */
+    requestsPerMinute?: number;
+    /** Max estimated input tokens sent in any 60 s window (estimated with `approxTokens`). */
+    inputTokensPerMinute?: number;
+}
+export interface RateLimiter {
+    /**
+     * Resolves once a request of `inputTokens` estimated tokens may start, and reserves its slot. Callers are served
+     * strictly in arrival order. A single request larger than `inputTokensPerMinute` is let through when no other
+     * tokens are in the window (it then occupies the whole window), so it can never deadlock.
+     */
+    acquire(inputTokens: number): Promise<void>;
+}
+/** Injectable for tests. */
+export interface RateLimiterClock {
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+/** Sliding-window limiter (requests and estimated input tokens over the last 60 s) with a FIFO queue. No dependencies. */
+export declare function createRateLimiter(opts: RateLimitOptions, clock?: RateLimiterClock): RateLimiter;
+/**
+ * One limiter per options object, so every call that receives the same `rateLimit` object (all documents of an index
+ * run, all questions of an eval run) draws from the same budget. Undefined when no limit is set.
+ */
+export declare function rateLimiterFor(opts: RateLimitOptions | undefined): RateLimiter | undefined;
+/** Estimated input tokens of anything that will be sent to the model (strings as-is, other values as JSON). */
+export declare const estimateInputTokens: (value: unknown) => number;
+/** CLI helper: `{ rpm, tpm }` strings from the option flags -> `RateLimitOptions` (undefined when neither is set). */
+export declare function parseRateLimit(o: {
+    rpm?: string;
+    tpm?: string;
+}, rpmFlag: string, tpmFlag: string): RateLimitOptions | undefined;

package/dist/llm/rate-limit.js

@@ -0,0 +1,95 @@
+import { approxTokens } from '../text/tokens.js';
+const WINDOW_MS = 60_000;
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function checkLimit(name, value) {
+    if (value !== undefined && (!Number.isInteger(value) || value < 1))
+        throw new Error(`${name} must be an integer >= 1, got ${value}`);
+}
+/** Sliding-window limiter (requests and estimated input tokens over the last 60 s) with a FIFO queue. No dependencies. */
+export function createRateLimiter(opts, clock = {}) {
+    checkLimit('requestsPerMinute', opts.requestsPerMinute);
+    checkLimit('inputTokensPerMinute', opts.inputTokensPerMinute);
+    const rpm = opts.requestsPerMinute;
+    const tpm = opts.inputTokensPerMinute;
+    const now = clock.now ?? (() => performance.now());
+    const sleep = clock.sleep ?? defaultSleep;
+    let log = [];
+    let tail = Promise.resolve();
+    const fits = (at, tokens) => {
+        let requests = 0;
+        let used = 0;
+        for (const r of log) {
+            if (r.t + WINDOW_MS > at) {
+                requests++;
+                used += r.tokens;
+            }
+        }
+        if (rpm !== undefined && requests + 1 > rpm)
+            return false;
+        if (tpm !== undefined && used > 0 && used + tokens > tpm)
+            return false;
+        return true;
+    };
+    /** Earliest time at which the request fits: now, or the moment an entry leaves the window. */
+    const waitMs = (tokens) => {
+        const t = now();
+        if (fits(t, tokens))
+            return 0;
+        for (const r of log) {
+            const at = r.t + WINDOW_MS;
+            if (at > t && fits(at, tokens))
+                return at - t;
+        }
+        throw new Error('rate limiter: no admissible time found (internal error)'); // unreachable: the window empties after 60 s
+    };
+    const take = async (tokens) => {
+        for (;;) {
+            const wait = waitMs(tokens);
+            if (wait <= 0)
+                break;
+            await sleep(wait);
+        }
+        const t = now();
+        log = log.filter((r) => r.t + WINDOW_MS > t);
+        log.push({ t, tokens });
+    };
+    return {
+        acquire(inputTokens) {
+            if (rpm === undefined && tpm === undefined)
+                return Promise.resolve();
+            const run = tail.then(() => take(Math.max(0, inputTokens)));
+            tail = run.catch(() => undefined);
+            return run;
+        },
+    };
+}
+const shared = new WeakMap();
+/**
+ * One limiter per options object, so every call that receives the same `rateLimit` object (all documents of an index
+ * run, all questions of an eval run) draws from the same budget. Undefined when no limit is set.
+ */
+export function rateLimiterFor(opts) {
+    if (!opts)
+        return undefined;
+    if (!shared.has(opts)) {
+        shared.set(opts, opts.requestsPerMinute === undefined && opts.inputTokensPerMinute === undefined ? undefined : createRateLimiter(opts));
+    }
+    return shared.get(opts);
+}
+/** Estimated input tokens of anything that will be sent to the model (strings as-is, other values as JSON). */
+export const estimateInputTokens = (value) => approxTokens(typeof value === 'string' ? value : JSON.stringify(value) ?? '');
+function parsePositiveInt(v, flag) {
+    const n = Number(v);
+    if (!Number.isInteger(n) || n < 1)
+        throw new Error(`${flag} must be an integer >= 1, got "${v}"`);
+    return n;
+}
+/** CLI helper: `{ rpm, tpm }` strings from the option flags -> `RateLimitOptions` (undefined when neither is set). */
+export function parseRateLimit(o, rpmFlag, tpmFlag) {
+    if (o.rpm === undefined && o.tpm === undefined)
+        return undefined;
+    return {
+        ...(o.rpm !== undefined ? { requestsPerMinute: parsePositiveInt(o.rpm, rpmFlag) } : {}),
+        ...(o.tpm !== undefined ? { inputTokensPerMinute: parsePositiveInt(o.tpm, tpmFlag) } : {}),
+    };
+}

package/dist/llm/summarize.d.ts

@@ -0,0 +1,67 @@
+import { type LanguageModel } from 'ai';
+import type { Store } from '../store/db.js';
+import type { TreeNode } from '../types.js';
+import { type RateLimiter, type RateLimitOptions } from './rate-limit.js';
+export interface SummaryOptions {
+    model: LanguageModel;
+    /** Nodes with `ownTokens` below this use their own text as the summary (default SUMMARY_TOKEN_THRESHOLD). */
+    threshold?: number;
+    /** Max parallel LLM calls (default 8). */
+    concurrency?: number;
+    /** Retries per call, handled by the AI SDK (default 3). */
+    maxRetries?: number;
+    /** Also generate a one-sentence document description (default true). */
+    describe?: boolean;
+    /**
+     * Client-side rate limit for the summary calls (sliding 60 s window; input tokens are estimated from the prompt).
+     * The same options object shares one budget across every document it is passed with.
+     */
+    rateLimit?: RateLimitOptions;
+    /** Advanced: use this limiter instead of building one from `rateLimit` (e.g. to share a budget or inject a clock). */
+    rateLimiter?: RateLimiter;
+}
+export interface SummaryFailure {
+    /** Node id, or `null` for the document description. */
+    nodeId: string | null;
+    message: string;
+}
+export interface SummarizeResult {
+    /** LLM calls made (successful or failed), including the description call. */
+    llmCalls: number;
+    /** Nodes whose summary came from the `summaries` cache. */
+    cached: number;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+    };
+    /** Calls that still failed after retries. Successful summaries are persisted regardless. */
+    failures: SummaryFailure[];
+}
+/** Outline given to the description prompt is cut to this many chars so it cannot overflow the context. */
+export declare const OUTLINE_MAX_CHARS = 12000;
+/** Port of PageIndex `generate_node_summary`, plus language and length constraints. */
+export declare function nodeSummaryPrompt(text: string): string;
+/** Port of PageIndex `generate_doc_description`, plus a language constraint. */
+export declare function docDescriptionPrompt(outline: string): string;
+/** Titles + truncated summaries, indented by depth, cut to OUTLINE_MAX_CHARS. */
+export declare function buildOutline(tree: TreeNode[], maxChars?: number): string;
+/**
+ * Fills node summaries (leaf -> `summary`, non-leaf -> `prefixSummary` of the node's own text only) and, unless
+ * `describe` is false and the doc has none yet, the document description. Nodes under the threshold use their own
+ * text; others are looked up in the `summaries` cache by sha256 of their own text, then sent to the LLM. A failed
+ * call never throws and never touches the store: it is returned in `failures`, everything that succeeded is saved.
+ */
+export declare function summarizeDocument(store: Store, docPath: string, opts: SummaryOptions): Promise<SummarizeResult>;
+/**
+ * Dry-run over every indexed document: how many nodes are over the threshold, how many LLM calls a full
+ * `summarizeDocument` pass would make (unique uncached node texts + one description per undescribed document unless
+ * `describe` is false) and the node-text input tokens (description outlines are not counted).
+ */
+export declare function estimateSummaryCost(store: Store, opts?: {
+    threshold?: number;
+    describe?: boolean;
+}): {
+    nodes: number;
+    llmCalls: number;
+    inputTokens: number;
+};