markindex 0.0.0-stage → 0.1.1

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+export declare function buildProgram(): Command;

package/dist/cli.js

@@ -0,0 +1,297 @@
+#!/usr/bin/env node
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { Command } from 'commander';
+import { createMcpServer, packageVersion } from './adapters/mcp.js';
+import { parseQuestions, QuestionFileError } from './eval/questions.js';
+import { checkGold, runEval } from './eval/run.js';
+import { indexLibrary } from './library/indexer.js';
+import { assertLibraryRoot } from './library/scan.js';
+import { defaultDbFile, openLibrary } from './library.js';
+import { estimateSummaryCost } from './llm/summarize.js';
+import { ProviderError, resolveSummaryModel } from './llm/provider.js';
+import { parseRateLimit } from './llm/rate-limit.js';
+import { RETRIEVE_MAX_INPUT_TOKENS, RETRIEVE_MAX_STEPS } from './retrieve.js';
+import { searchNodes } from './store/search.js';
+import { openStore, StoreLockedError } from './store/db.js';
+const out = (s) => process.stdout.write(`${s}\n`);
+const err = (s) => process.stderr.write(`${s}\n`);
+const json = (v) => out(JSON.stringify(v, null, 2));
+function printReport(r) {
+    if (r.rebuilt)
+        out(`full rebuild: ${r.rebuilt.reason}`);
+    out(`added ${r.added.length}, updated ${r.updated.length}, removed ${r.removed.length}, unchanged ${r.unchanged.length}, skipped ${r.skipped.length}`);
+    for (const s of r.skipped)
+        out(`  skipped ${s.path}: ${s.reason}`);
+    for (const s of r.summaryErrors)
+        out(`  summary failed ${s.path}${s.nodeId ? ` [${s.nodeId}]` : ''}: ${s.message}`);
+    for (const w of r.warnings)
+        out(`  warning ${w.path}: ${w.message}`);
+    if (r.warnings.some((w) => w.message.includes('setext'))) {
+        out('  hint: if your converter writes `---` page breaks, re-run with --no-setext (rebuilds the whole index)');
+    }
+}
+async function indexCommand(root, o) {
+    root = resolve(root);
+    assertLibraryRoot(root);
+    const rateLimit = parseRateLimit({ rpm: o.summaryRpm, tpm: o.summaryTpm }, '--summary-rpm', '--summary-tpm');
+    if (rateLimit && (!o.summaryModel || o.estimate)) {
+        err(`markindex: ${[o.summaryRpm && '--summary-rpm', o.summaryTpm && '--summary-tpm'].filter(Boolean).join(', ')} ignored (needs --summary-model and no --estimate)`);
+    }
+    const model = o.summaryModel && !o.estimate ? await resolveSummaryModel(o.summaryModel) : undefined;
+    const store = openStore(resolve(o.db ?? defaultDbFile(root)));
+    try {
+        const report = await indexLibrary({
+            root,
+            store,
+            setext: o.setext,
+            ignore: o.ignore,
+            summary: model ? { model, ...(rateLimit ? { rateLimit } : {}) } : undefined,
+            onProgress: o.json ? undefined : (e) => err(`[${e.done}/${e.total}] ${e.status} ${e.path}`),
+        });
+        if (o.estimate) {
+            const est = estimateSummaryCost(store);
+            if (o.json)
+                json({ report, estimate: est });
+            else {
+                printReport(report);
+                out(`Summary estimate: ${est.nodes} nodes to summarize, ${est.llmCalls} LLM calls, ~${est.inputTokens} input tokens (no LLM calls were made).`);
+            }
+            return;
+        }
+        if (o.json)
+            json(report);
+        else
+            printReport(report);
+    }
+    finally {
+        store.close();
+    }
+}
+function renderTree(nodes, depth, level = 1, lines = []) {
+    for (const n of nodes) {
+        const more = n.children.length > 0 && level >= depth ? ` (+${n.children.length} children)` : '';
+        lines.push(`${'  '.repeat(level - 1)}${n.title}  [${n.nodeId}] L${n.startLine}-${n.endLine} ${n.tokens}t${more}`);
+        if (level < depth)
+            renderTree(n.children, depth, level + 1, lines);
+    }
+    return lines;
+}
+function pruneDepth(nodes, depth) {
+    return nodes.map((n) => ({ ...n, children: depth > 1 ? pruneDepth(n.children, depth - 1) : [] }));
+}
+function treeCommand(root, doc, o) {
+    const lib = openLibrary({ root, dbFile: o.db });
+    try {
+        const tree = lib.store.getTree(doc);
+        if (!tree)
+            throw new Error(`Document not found in index: ${doc}. Use \`markindex search\` or check the path (library-relative, case-sensitive).`);
+        const depth = Math.max(1, Number.parseInt(o.depth, 10) || 1);
+        if (o.json)
+            json(pruneDepth(tree, depth));
+        else
+            out(renderTree(tree, depth).join('\n'));
+    }
+    finally {
+        lib.close();
+    }
+}
+function searchCommand(root, query, o) {
+    const lib = openLibrary({ root, dbFile: o.db });
+    try {
+        const hits = searchNodes(lib.store, query, { folder: o.folder, docPath: o.doc, limit: Number.parseInt(o.limit, 10) || 10 });
+        if (o.json) {
+            json(hits);
+            return;
+        }
+        if (hits.length === 0) {
+            out('No matches.');
+            return;
+        }
+        for (const h of hits) {
+            out(`${h.docPath} > ${h.titlePath.join(' > ')}  [${h.nodeId}] L${h.startLine}-${h.endLine}`);
+            out(`  ${h.snippet.replace(/\s+/g, ' ')}`);
+        }
+    }
+    finally {
+        lib.close();
+    }
+}
+async function mcpCommand(root, o) {
+    root = resolve(root);
+    assertLibraryRoot(root);
+    const dbFile = resolve(o.db ?? defaultDbFile(root));
+    if (o.reindexOnStart) {
+        const store = openStore(dbFile);
+        try {
+            const report = await indexLibrary({ root, store, setext: o.setext, ignore: o.ignore });
+            for (const w of report.warnings)
+                err(`markindex: warning ${w.path}: ${w.message} (--no-setext)`);
+        }
+        finally {
+            store.close();
+        }
+    }
+    else {
+        const unused = [...(o.ignore.length > 0 ? ['--ignore'] : []), ...(o.setext === undefined ? [] : [o.setext ? '--setext' : '--no-setext'])];
+        if (unused.length > 0)
+            err(`markindex: ${unused.join(', ')} only apply when indexing; ignored without --reindex-on-start`);
+    }
+    if (!o.reindexOnStart && !existsSync(dbFile)) {
+        throw new Error(`No MarkIndex index at ${dbFile}. Run \`markindex index ${root}\` first (or pass --reindex-on-start).`);
+    }
+    const lib = openLibrary({ root, dbFile });
+    const server = createMcpServer(lib);
+    process.on('exit', () => lib.close());
+    await server.connect(new StdioServerTransport());
+    err(`markindex MCP server ready (stdio), library: ${root}`);
+}
+const collect = (value, previous) => [...previous, value];
+const EVAL_MODES = { bm25: 'bm25', agent: 'agent', tools: 'agent', retrieve: 'agent' };
+const int = (v, name, min = 1) => {
+    const n = Number(v);
+    if (!Number.isInteger(n) || n < min)
+        throw new Error(`${name} must be an integer >= ${min}, got "${v}"`);
+    return n;
+};
+function printSummary(report) {
+    const pct = (x) => `${(x * 100).toFixed(1)}%`;
+    const row = (label, s) => s &&
+        out(`${label.padEnd(10)}${String(s.questions).padStart(5)}${String(s.hits).padStart(6)}${pct(s.hitRate).padStart(9)}` +
+            `${pct(s.hitAt1).padStart(9)}${s.mrr.toFixed(3).padStart(8)}${String(Math.round(s.avgTokens)).padStart(11)}` +
+            `${s.avgSteps.toFixed(1).padStart(7)}${String(s.errors).padStart(7)}` +
+            `${s.meanEvidenceLines.toFixed(1).padStart(10)}${String(s.medianEvidenceLines).padStart(7)}` +
+            `${(s.meanPrecision === null ? '-' : pct(s.meanPrecision)).padStart(8)}`);
+    const table = (title, a) => {
+        out(title);
+        out(`${'set'.padEnd(10)}${'n'.padStart(5)}${'hits'.padStart(6)}${'hit'.padStart(9)}${'hit@1'.padStart(9)}${'mrr'.padStart(8)}${'avg tok'.padStart(11)}${'steps'.padStart(7)}${'errors'.padStart(7)}${'ev lines'.padStart(10)}${'med'.padStart(7)}${'prec'.padStart(8)}`);
+        row('overall', a.overall);
+        for (const lang of ['zh', 'en', 'mixed'])
+            row(lang, a.byLang[lang]);
+    };
+    const maxLines = report.options.maxEvidenceLines ? `, hits need evidence <= ${report.options.maxEvidenceLines} lines` : '';
+    table(`mode ${report.mode}${report.model ? ` (${report.model})` : ''}${report.options.topK ? `, top-${report.options.topK}` : ''}${maxLines}`, report.summary);
+    if (report.baseline) {
+        out('');
+        table(`baseline bm25 top-${report.baseline.topK}`, report.baseline.summary);
+    }
+}
+async function evalCommand(root, o) {
+    const rateLimit = parseRateLimit(o, '--rpm', '--tpm');
+    const mode = EVAL_MODES[o.mode];
+    if (!mode)
+        throw new Error(`--mode must be bm25 or agent (aliases: tools, retrieve), got "${o.mode}"`);
+    if (mode === 'agent' && !o.model)
+        throw new Error('agent mode needs --model <provider:model>, e.g. --model anthropic:claude-sonnet-5-5');
+    if (rateLimit && mode !== 'agent')
+        err('markindex: --rpm, --tpm only apply to agent mode; ignored');
+    const maxEvidenceLines = o.maxEvidenceLines === undefined ? undefined : int(o.maxEvidenceLines, '--max-evidence-lines');
+    const questions = parseQuestions(readFileSync(o.questions, 'utf8'), o.questions);
+    const lib = openLibrary({ root, dbFile: o.db });
+    try {
+        const goldIssues = checkGold(lib, questions);
+        if (goldIssues.length > 0)
+            throw new QuestionFileError(goldIssues, o.questions);
+        const model = mode === 'agent'
+            ? await resolveSummaryModel(o.model).catch((e) => {
+                throw e instanceof ProviderError ? new ProviderError(e.message.replace('--summary-model', '--model')) : e;
+            })
+            : undefined;
+        const report = await runEval(lib, questions, {
+            mode,
+            topK: int(o.topK, '--top-k'),
+            maxSteps: int(o.maxSteps, '--max-steps'),
+            maxInputTokens: int(o.maxInputTokens, '--max-input-tokens'),
+            concurrency: int(o.concurrency, '--concurrency'),
+            baseline: o.baseline,
+            ...(maxEvidenceLines !== undefined ? { maxEvidenceLines } : {}),
+            ...(model ? { model, modelId: o.model } : {}),
+            ...(model && rateLimit ? { rateLimit } : {}),
+            onProgress: o.json
+                ? undefined
+                : (done, total, r) => err(`[${done}/${total}] ${r.id} ${r.hit ? 'HIT' : 'miss'}${r.error ? ` (error: ${r.error})` : ''}`),
+        });
+        if (o.out)
+            writeFileSync(o.out, `${JSON.stringify(report, null, 2)}\n`);
+        if (o.json)
+            json(report);
+        else {
+            printSummary(report);
+            if (o.out)
+                out(`\nFull report: ${resolve(o.out)}`);
+        }
+    }
+    finally {
+        lib.close();
+    }
+}
+export function buildProgram() {
+    const program = new Command('markindex').description('Section-tree index of Markdown libraries for agent retrieval').version(packageVersion());
+    program
+        .command('index <root>')
+        .description('Build or update the index of a Markdown folder')
+        .option('--db <file>', 'index database file (default <root>/.markindex/index.db)')
+        .option('--summary-model <provider:model>', 'summarize with an LLM, e.g. anthropic:claude-sonnet-5-5 (needs @ai-sdk/<provider> installed)')
+        .option('--summary-rpm <n>', 'limit summary LLM requests per minute (sliding 60 s window)')
+        .option('--summary-tpm <n>', 'limit estimated summary input tokens per minute (sliding 60 s window)')
+        .option('--estimate', 'index, then print the summarization cost estimate and exit without LLM calls')
+        .option('--ignore <pattern>', 'gitignore-syntax pattern relative to <root> to leave out (repeatable; adds to .gitignore / .markindexignore)', collect, [])
+        .option('--setext', 'treat `Text` + `---` / `===` as headings (CommonMark; the default for a new index); rebuilds when it was off')
+        .option('--no-setext', 'do not treat `---` / `===` under a paragraph as a heading (converter page breaks); stored in the index, rebuilds when changed')
+        .option('--json', 'machine-readable output')
+        .action(indexCommand);
+    program
+        .command('tree <root> <doc>')
+        .description('Print the section tree of one document (library-relative path)')
+        .option('--db <file>', 'index database file')
+        .option('--depth <n>', 'outline levels to show', '3')
+        .option('--json', 'machine-readable output')
+        .action(treeCommand);
+    program
+        .command('search <root> <query>')
+        .description('Keyword search over the index')
+        .option('--db <file>', 'index database file')
+        .option('--folder <folder>', 'restrict to a library-relative folder')
+        .option('--doc <path>', 'restrict to one document')
+        .option('--limit <n>', 'maximum hits', '10')
+        .option('--json', 'machine-readable output')
+        .action(searchCommand);
+    program
+        .command('mcp <root>')
+        .description('Serve the library to agents as an MCP server over stdio (read-only)')
+        .option('--db <file>', 'index database file')
+        .option('--reindex-on-start', 'update the index before serving')
+        .option('--ignore <pattern>', 'with --reindex-on-start: pattern to leave out (repeatable, see `index --ignore`)', collect, [])
+        .option('--setext', 'with --reindex-on-start: turn setext headings back on (see `index --setext`)')
+        .option('--no-setext', 'with --reindex-on-start: index without setext headings (see `index --no-setext`)')
+        .action(mcpCommand);
+    program
+        .command('eval <root>')
+        .description('Measure retrieval quality on a JSONL question file (see eval/README.md for the format)')
+        .requiredOption('--questions <file>', 'JSONL question file: {"id","question","lang":"zh|en|mixed","gold":[{"doc_path","start_line"?,"end_line"?,"title_path"?}]}')
+        .option('--mode <mode>', 'bm25 (keyword baseline, no LLM) or agent (built-in retrieve() loop over the agent tools; "tools" and "retrieve" are aliases)', 'bm25')
+        .option('--model <provider:model>', 'agent mode: model, e.g. anthropic:claude-sonnet-5-5 (needs @ai-sdk/<provider> installed)')
+        .option('--top-k <n>', 'bm25: hits per question', '5')
+        .option('--max-steps <n>', 'agent: model steps per question', String(RETRIEVE_MAX_STEPS))
+        .option('--max-input-tokens <n>', 'agent: input-token budget per question', String(RETRIEVE_MAX_INPUT_TOKENS))
+        .option('--concurrency <n>', 'agent: questions run in parallel', '1')
+        .option('--rpm <n>', 'agent: limit model requests per minute, shared by all questions (sliding 60 s window)')
+        .option('--tpm <n>', 'agent: limit estimated input tokens per minute, shared by all questions (sliding 60 s window)')
+        .option('--no-baseline', 'agent: skip the bm25 baseline')
+        .option('--max-evidence-lines <n>', 'count a hit only if the matching evidence item spans at most n lines')
+        .option('--db <file>', 'index database file')
+        .option('--out <file>', 'write the full JSON report (per-question results) to this file')
+        .option('--json', 'print the full JSON report instead of the summary table')
+        .action(evalCommand);
+    return program;
+}
+try {
+    await buildProgram().parseAsync(process.argv);
+}
+catch (e) {
+    err(e instanceof StoreLockedError || e instanceof ProviderError || e instanceof Error
+        ? `markindex: ${e.message}`
+        : `markindex: ${String(e)}`);
+    process.exitCode = 1;
+}

package/dist/config.d.ts

@@ -0,0 +1,18 @@
+/** Per-response character budget (PageIndex 100_000 x 0.95). */
+export declare const RESPONSE_CHAR_BUDGET = 95000;
+export declare const DEFAULT_STRUCTURE_DEPTH = 2;
+/** PageIndex `--summary-token-threshold` default. */
+export declare const SUMMARY_TOKEN_THRESHOLD = 200;
+export declare const MAX_NODE_TOKENS = 8000;
+export declare const ROUTING_COST_TOKENS = 150;
+/** PageIndex 20 pages x ~750 tokens; to be calibrated in Task 12. */
+export declare const STRUCTURE_FIRST_TOKEN_THRESHOLD = 15000;
+/** PageIndex `_SIMILAR_NAMES_LIMIT`. */
+export declare const SIMILAR_NAMES_LIMIT = 3;
+/** Node titles longer than this many code points are cut and end with an ellipsis; the body keeps the full heading. */
+export declare const MAX_TITLE_CHARS = 300;
+/**
+ * Bump whenever parsing, tree building or segmentation changes what gets stored, so existing indexes are rebuilt.
+ * 2: CR / CRLF / LF line endings, NFKC segmentation, capped heading titles.
+ */
+export declare const INDEX_FORMAT_VERSION = 2;

package/dist/config.js

@@ -0,0 +1,18 @@
+/** Per-response character budget (PageIndex 100_000 x 0.95). */
+export const RESPONSE_CHAR_BUDGET = 95_000;
+export const DEFAULT_STRUCTURE_DEPTH = 2;
+/** PageIndex `--summary-token-threshold` default. */
+export const SUMMARY_TOKEN_THRESHOLD = 200;
+export const MAX_NODE_TOKENS = 8_000;
+export const ROUTING_COST_TOKENS = 150;
+/** PageIndex 20 pages x ~750 tokens; to be calibrated in Task 12. */
+export const STRUCTURE_FIRST_TOKEN_THRESHOLD = 15_000;
+/** PageIndex `_SIMILAR_NAMES_LIMIT`. */
+export const SIMILAR_NAMES_LIMIT = 3;
+/** Node titles longer than this many code points are cut and end with an ellipsis; the body keeps the full heading. */
+export const MAX_TITLE_CHARS = 300;
+/**
+ * Bump whenever parsing, tree building or segmentation changes what gets stored, so existing indexes are rebuilt.
+ * 2: CR / CRLF / LF line endings, NFKC segmentation, capped heading titles.
+ */
+export const INDEX_FORMAT_VERSION = 2;

package/dist/eval/questions.d.ts

@@ -0,0 +1,34 @@
+/** One acceptable answer location. Lines are 1-based, inclusive, in the original file. */
+export interface Gold {
+    /** Library-relative document path (same form as `markindex search` prints). */
+    doc_path: string;
+    /** Titles from the top-level section down to the answer node (used when no line range is given). */
+    title_path?: string[];
+    start_line?: number;
+    end_line?: number;
+}
+export interface EvalQuestion {
+    id: string;
+    question: string;
+    lang: 'zh' | 'en' | 'mixed';
+    /** Any one gold location counts as a hit. */
+    gold: Gold[];
+    note?: string;
+    /** 1-based line of the question file this question came from. */
+    line: number;
+}
+export interface QuestionIssue {
+    line: number;
+    message: string;
+}
+export declare class QuestionFileError extends Error {
+    readonly issues: QuestionIssue[];
+    constructor(issues: QuestionIssue[], file?: string);
+}
+/**
+ * Parses a JSONL question file (one JSON object per line; blank lines ignored; BOM / CRLF tolerated).
+ * Collects every problem and throws one `QuestionFileError` listing them with line numbers.
+ *
+ * Line format: {"id", "question", "lang": "zh"|"en"|"mixed", "gold": [{"doc_path", "title_path"?, "start_line"?, "end_line"?}], "note"?}
+ */
+export declare function parseQuestions(text: string, file?: string): EvalQuestion[];

package/dist/eval/questions.js

@@ -0,0 +1,80 @@
+import { z } from 'zod';
+export class QuestionFileError extends Error {
+    issues;
+    constructor(issues, file = 'questions') {
+        super(`${issues.length} problem(s) in the question file:\n` +
+            issues.map((i) => `${file}:${i.line}: ${i.message}`).join('\n'));
+        this.issues = issues;
+    }
+}
+const lineNo = z.int().min(1);
+const goldSchema = z
+    .strictObject({
+    doc_path: z.string().min(1),
+    title_path: z.array(z.string().min(1)).min(1).optional(),
+    start_line: lineNo.optional(),
+    end_line: lineNo.optional(),
+})
+    .superRefine((g, ctx) => {
+    if ((g.start_line === undefined) !== (g.end_line === undefined)) {
+        ctx.addIssue({ code: 'custom', message: 'start_line and end_line must be given together' });
+    }
+    else if (g.start_line !== undefined && g.end_line !== undefined && g.end_line < g.start_line) {
+        ctx.addIssue({ code: 'custom', path: ['end_line'], message: 'end_line must be >= start_line' });
+    }
+});
+const questionSchema = z.strictObject({
+    id: z.string().trim().min(1),
+    question: z.string().trim().min(1),
+    lang: z.enum(['zh', 'en', 'mixed']),
+    gold: z.array(goldSchema).min(1, 'gold needs at least one location'),
+    note: z.string().optional(),
+});
+const describeIssue = (i) => {
+    const path = i.path.length ? `${i.path.join('.')}: ` : '';
+    if (i.code === 'unrecognized_keys')
+        return `${path}unknown field(s) ${i.keys.map((k) => `"${k}"`).join(', ')}`;
+    return `${path}${i.message}`;
+};
+/**
+ * Parses a JSONL question file (one JSON object per line; blank lines ignored; BOM / CRLF tolerated).
+ * Collects every problem and throws one `QuestionFileError` listing them with line numbers.
+ *
+ * Line format: {"id", "question", "lang": "zh"|"en"|"mixed", "gold": [{"doc_path", "title_path"?, "start_line"?, "end_line"?}], "note"?}
+ */
+export function parseQuestions(text, file) {
+    const issues = [];
+    const out = [];
+    const seen = new Map();
+    const lines = text.replace(/^/, '').split(/\r?\n/);
+    lines.forEach((raw, i) => {
+        const line = i + 1;
+        if (raw.trim() === '')
+            return;
+        let value;
+        try {
+            value = JSON.parse(raw);
+        }
+        catch (e) {
+            issues.push({ line, message: `invalid JSON (${e.message})` });
+            return;
+        }
+        const parsed = questionSchema.safeParse(value);
+        if (!parsed.success) {
+            issues.push({ line, message: parsed.error.issues.map(describeIssue).join('; ') });
+            return;
+        }
+        const prev = seen.get(parsed.data.id);
+        if (prev !== undefined) {
+            issues.push({ line, message: `duplicate id "${parsed.data.id}" (first used on line ${prev})` });
+            return;
+        }
+        seen.set(parsed.data.id, line);
+        out.push({ ...parsed.data, line });
+    });
+    if (issues.length === 0 && out.length === 0)
+        issues.push({ line: 1, message: 'no questions found (expected one JSON object per line)' });
+    if (issues.length > 0)
+        throw new QuestionFileError(issues, file);
+    return out;
+}

package/dist/eval/run.d.ts

@@ -0,0 +1,62 @@
+import type { LanguageModel } from 'ai';
+import type { Library } from '../library.js';
+import { type RateLimiter, type RateLimitOptions } from '../llm/rate-limit.js';
+import type { EvalQuestion, QuestionIssue } from './questions.js';
+import { type Aggregate, type QuestionResult } from './score.js';
+/**
+ * - `bm25`: baseline — `searchNodes(question)` top-k, no LLM. Evidence = each hit node's own text range.
+ * - `agent`: the built-in `retrieve()` loop (generateText + the five library tools + AGENT_INSTRUCTIONS +
+ *   select_evidence). This is also what an external agent does in "tools" mode, so the two are one mode here.
+ */
+export type EvalMode = 'bm25' | 'agent';
+export interface RunEvalOptions {
+    mode: EvalMode;
+    /** bm25: hits per question (default 5). */
+    topK?: number;
+    /** agent: the model driving retrieve(). */
+    model?: LanguageModel;
+    /** agent: label recorded in the report (e.g. "anthropic:claude-sonnet-5-5"). */
+    modelId?: string;
+    /** agent: retrieve() maxSteps (default RETRIEVE_MAX_STEPS). */
+    maxSteps?: number;
+    /** agent: retrieve() maxInputTokens per question (default RETRIEVE_MAX_INPUT_TOKENS). */
+    maxInputTokens?: number;
+    /** agent: questions run in parallel (default 1). */
+    concurrency?: number;
+    /** agent: client-side rate limit shared by every model step of every question (see `RetrieveOptions.rateLimit`). */
+    rateLimit?: RateLimitOptions;
+    /** agent: advanced, use this limiter instead of building one from `rateLimit`. */
+    rateLimiter?: RateLimiter;
+    /** agent: also run the bm25 baseline on the same questions (default true). */
+    baseline?: boolean;
+    /** When set, a matching evidence item only counts as a hit if it spans at most this many lines. */
+    maxEvidenceLines?: number;
+    onProgress?: (done: number, total: number, result: QuestionResult) => void;
+}
+export interface EvalReport {
+    mode: EvalMode;
+    model?: string;
+    createdAt: string;
+    options: {
+        topK?: number;
+        maxSteps?: number;
+        maxInputTokens?: number;
+        maxEvidenceLines?: number;
+        rateLimit?: RateLimitOptions;
+    };
+    summary: Aggregate;
+    /** agent mode: the bm25 top-k baseline on the same questions. */
+    baseline?: {
+        mode: 'bm25';
+        topK: number;
+        summary: Aggregate;
+    };
+    results: QuestionResult[];
+}
+/**
+ * Checks every gold location against the index: the document exists, the line range is inside the file and a
+ * title_path names an existing node. Returns issues keyed by question-file line (empty = all good).
+ */
+export declare function checkGold(lib: Library, questions: EvalQuestion[]): QuestionIssue[];
+/** Runs every question in the given mode and aggregates the results (overall + by language). */
+export declare function runEval(lib: Library, questions: EvalQuestion[], opts: RunEvalOptions): Promise<EvalReport>;