markindex 0.0.0-stage → 0.1.0

package/dist/eval/run.js

@@ -0,0 +1,163 @@
+import { rateLimiterFor } from '../llm/rate-limit.js';
+import { retrieve, RETRIEVE_MAX_INPUT_TOKENS, RETRIEVE_MAX_STEPS } from '../retrieve.js';
+import { searchNodes } from '../store/search.js';
+import { approxTokens } from '../text/tokens.js';
+import { walk } from '../tree/build.js';
+import { aggregate, evidenceLineRanges, evidencePrecision, firstHitRank, } from './score.js';
+const DEFAULT_TOP_K = 5;
+/**
+ * Checks every gold location against the index: the document exists, the line range is inside the file and a
+ * title_path names an existing node. Returns issues keyed by question-file line (empty = all good).
+ */
+export function checkGold(lib, questions) {
+    const issues = [];
+    const titlePaths = new Map();
+    for (const q of questions) {
+        for (const g of q.gold) {
+            const doc = lib.store.getDoc(g.doc_path);
+            if (!doc) {
+                issues.push({ line: q.line, message: `question "${q.id}": gold doc_path "${g.doc_path}" is not in the index (use a library-relative path as printed by \`markindex search\`)` });
+                continue;
+            }
+            if (g.end_line !== undefined && g.end_line > doc.lineCount) {
+                issues.push({ line: q.line, message: `question "${q.id}": gold lines ${g.start_line}-${g.end_line} exceed "${g.doc_path}" (${doc.lineCount} lines)` });
+            }
+            if (g.title_path && g.start_line === undefined) {
+                let paths = titlePaths.get(g.doc_path);
+                if (!paths) {
+                    paths = new Set([...walk(lib.store.getTree(g.doc_path) ?? [])].map((e) => e.titlePath.join('\0')));
+                    titlePaths.set(g.doc_path, paths);
+                }
+                if (!paths.has(g.title_path.join('\0'))) {
+                    issues.push({ line: q.line, message: `question "${q.id}": gold title_path ${JSON.stringify(g.title_path)} matches no node of "${g.doc_path}" (see \`markindex tree\`)` });
+                }
+            }
+        }
+    }
+    return issues;
+}
+/** Hit / rank (honouring maxEvidenceLines) plus evidence size and precision for one question. */
+function judge(lib, q, evidence, maxEvidenceLines) {
+    const ranges = evidenceLineRanges(evidence);
+    const rank = firstHitRank(evidence, q.gold, maxEvidenceLines);
+    return {
+        hit: rank !== null,
+        rank,
+        evidenceLines: ranges.reduce((n, r) => n + r.endLine - r.startLine + 1, 0),
+        evidenceTokens: ranges.reduce((n, r) => n + approxTokens(lib.store.getLines(r.docPath, r.startLine, r.endLine).join('\n')), 0),
+        precision: evidencePrecision(evidence, q.gold),
+    };
+}
+function bm25Runner(lib, topK, maxEvidenceLines) {
+    const nodes = new Map();
+    const node = (docPath, nodeId) => {
+        let m = nodes.get(docPath);
+        if (!m) {
+            m = new Map([...walk(lib.store.getTree(docPath) ?? [])].map((e) => [e.node.nodeId, e.node]));
+            nodes.set(docPath, m);
+        }
+        return m.get(nodeId);
+    };
+    return (q) => {
+        const hits = searchNodes(lib.store, q.question, { limit: topK });
+        const evidence = hits.map((h) => ({
+            docPath: h.docPath,
+            titlePath: h.titlePath,
+            startLine: h.startLine,
+            // what BM25 scored is the node's own text, not its whole subtree
+            endLine: node(h.docPath, h.nodeId)?.ownEndLine ?? h.endLine,
+        }));
+        return {
+            id: q.id,
+            lang: q.lang,
+            ...judge(lib, q, evidence, maxEvidenceLines),
+            tokens: approxTokens(JSON.stringify(hits)),
+            steps: 1,
+            evidence,
+        };
+    };
+}
+async function agentResult(lib, q, opts, rateLimiter) {
+    try {
+        const r = await retrieve(lib, q.question, {
+            model: opts.model,
+            ...(rateLimiter ? { rateLimiter } : {}),
+            ...(opts.maxSteps !== undefined ? { maxSteps: opts.maxSteps } : {}),
+            ...(opts.maxInputTokens !== undefined ? { maxInputTokens: opts.maxInputTokens } : {}),
+        });
+        const evidence = r.evidence.map(({ docPath, titlePath, startLine, endLine }) => ({ docPath, titlePath, startLine, endLine }));
+        return {
+            id: q.id,
+            lang: q.lang,
+            ...judge(lib, q, evidence, opts.maxEvidenceLines),
+            tokens: r.usage.inputTokens + r.usage.outputTokens,
+            steps: r.steps,
+            evidence,
+            stopReason: r.stopReason,
+        };
+    }
+    catch (e) {
+        return {
+            id: q.id,
+            lang: q.lang,
+            hit: false,
+            rank: null,
+            tokens: 0,
+            steps: 0,
+            evidence: [],
+            evidenceLines: 0,
+            evidenceTokens: 0,
+            precision: null,
+            error: e instanceof Error ? e.message : String(e),
+        };
+    }
+}
+/** Runs every question in the given mode and aggregates the results (overall + by language). */
+export async function runEval(lib, questions, opts) {
+    const topK = opts.topK ?? DEFAULT_TOP_K;
+    const total = questions.length;
+    const createdAt = new Date().toISOString();
+    const maxLinesOption = opts.maxEvidenceLines === undefined ? {} : { maxEvidenceLines: opts.maxEvidenceLines };
+    if (opts.mode === 'bm25') {
+        const run = bm25Runner(lib, topK, opts.maxEvidenceLines);
+        const results = questions.map((q, i) => {
+            const r = run(q);
+            opts.onProgress?.(i + 1, total, r);
+            return r;
+        });
+        return { mode: 'bm25', createdAt, options: { topK, ...maxLinesOption }, summary: aggregate(results), results };
+    }
+    if (!opts.model)
+        throw new Error('agent mode needs a model (e.g. --model anthropic:claude-sonnet-5-5)');
+    const rateLimiter = opts.rateLimiter ?? rateLimiterFor(opts.rateLimit);
+    const results = new Array(total);
+    let next = 0;
+    let done = 0;
+    const worker = async () => {
+        while (next < total) {
+            const i = next++;
+            results[i] = await agentResult(lib, questions[i], opts, rateLimiter);
+            opts.onProgress?.(++done, total, results[i]);
+        }
+    };
+    const concurrency = Math.max(1, Math.min(total, Math.floor(opts.concurrency ?? 1)));
+    await Promise.all(Array.from({ length: concurrency }, worker));
+    const report = {
+        mode: 'agent',
+        ...(opts.modelId ? { model: opts.modelId } : {}),
+        createdAt,
+        options: {
+            maxSteps: opts.maxSteps ?? RETRIEVE_MAX_STEPS,
+            maxInputTokens: opts.maxInputTokens ?? RETRIEVE_MAX_INPUT_TOKENS,
+            ...maxLinesOption,
+            ...(opts.rateLimit ? { rateLimit: opts.rateLimit } : {}),
+        },
+        summary: aggregate(results),
+        results,
+    };
+    if (opts.baseline ?? true) {
+        const run = bm25Runner(lib, topK, opts.maxEvidenceLines);
+        report.baseline = { mode: 'bm25', topK, summary: aggregate(questions.map(run)) };
+    }
+    return report;
+}

package/dist/eval/score.d.ts

@@ -0,0 +1,74 @@
+import type { EvalQuestion, Gold } from './questions.js';
+/** Where a returned piece of evidence points (lines 1-based, inclusive). */
+export interface EvidenceRef {
+    docPath: string;
+    titlePath: string[];
+    startLine: number;
+    endLine: number;
+}
+export interface QuestionResult {
+    id: string;
+    lang: EvalQuestion['lang'];
+    hit: boolean;
+    /** 1-based position of the first evidence item matching a gold location; null when none does. */
+    rank: number | null;
+    /** bm25: approx tokens of the search response; agent: model input + output tokens. */
+    tokens: number;
+    /** bm25: 1 (one search call); agent: model steps. */
+    steps: number;
+    evidence: EvidenceRef[];
+    /** Distinct document lines covered by the evidence (overlapping items counted once). */
+    evidenceLines: number;
+    /** Approx tokens of those lines' text. */
+    evidenceTokens: number;
+    /** Share of evidenceLines inside any gold line range; null when no gold has a line range or there is no evidence. */
+    precision: number | null;
+    stopReason?: string;
+    /** Set when the question could not be run (e.g. a provider error); counts as a miss. */
+    error?: string;
+}
+export interface Summary {
+    questions: number;
+    hits: number;
+    /** hits / questions */
+    hitRate: number;
+    /** Share of questions whose FIRST evidence item is a hit (meaningful for ranked bm25 output). */
+    hitAt1: number;
+    /** Mean reciprocal rank of the first hit (0 for misses). */
+    mrr: number;
+    avgTokens: number;
+    avgSteps: number;
+    errors: number;
+    meanEvidenceLines: number;
+    medianEvidenceLines: number;
+    meanEvidenceTokens: number;
+    /** Mean over the questions whose precision is not null; null when none has one. */
+    meanPrecision: number | null;
+}
+export interface Aggregate {
+    overall: Summary;
+    byLang: Partial<Record<EvalQuestion['lang'], Summary>>;
+}
+/**
+ * Hit judgement: same document, and
+ * - gold has a line range → the evidence range overlaps it (inclusive);
+ * - else gold has title_path → one title path is a prefix of the other (evidence inside, or containing, the gold section);
+ * - else (doc-level gold) → any evidence from the document.
+ */
+export declare function matchesGold(e: EvidenceRef, g: Gold): boolean;
+/**
+ * 1-based rank of the first evidence item matching any gold. With `maxEvidenceLines`, only items spanning at most
+ * that many lines count (a whole-chapter "hit" does not).
+ */
+export declare function firstHitRank(evidence: EvidenceRef[], gold: Gold[], maxEvidenceLines?: number): number | null;
+export interface LineRange {
+    docPath: string;
+    startLine: number;
+    endLine: number;
+}
+/** Evidence as disjoint line ranges: sorted by document then line, overlapping or adjacent items merged. */
+export declare function evidenceLineRanges(evidence: LineRange[]): LineRange[];
+/** Share of distinct evidence lines that fall inside any gold line range (gold without lines is ignored). */
+export declare function evidencePrecision(evidence: EvidenceRef[], gold: Gold[]): number | null;
+/** Overall and per-language summaries; averages run over all questions (errors count as misses with their partial cost). */
+export declare function aggregate(results: QuestionResult[]): Aggregate;

package/dist/eval/score.js

@@ -0,0 +1,104 @@
+const prefixOf = (a, b) => a.length <= b.length && a.every((t, i) => t === b[i]);
+/**
+ * Hit judgement: same document, and
+ * - gold has a line range → the evidence range overlaps it (inclusive);
+ * - else gold has title_path → one title path is a prefix of the other (evidence inside, or containing, the gold section);
+ * - else (doc-level gold) → any evidence from the document.
+ */
+export function matchesGold(e, g) {
+    if (e.docPath !== g.doc_path)
+        return false;
+    if (g.start_line !== undefined && g.end_line !== undefined)
+        return e.startLine <= g.end_line && g.start_line <= e.endLine;
+    if (g.title_path)
+        return prefixOf(g.title_path, e.titlePath) || prefixOf(e.titlePath, g.title_path);
+    return true;
+}
+/**
+ * 1-based rank of the first evidence item matching any gold. With `maxEvidenceLines`, only items spanning at most
+ * that many lines count (a whole-chapter "hit" does not).
+ */
+export function firstHitRank(evidence, gold, maxEvidenceLines) {
+    const i = evidence.findIndex((e) => (maxEvidenceLines === undefined || e.endLine - e.startLine + 1 <= maxEvidenceLines) &&
+        gold.some((g) => matchesGold(e, g)));
+    return i === -1 ? null : i + 1;
+}
+/** Evidence as disjoint line ranges: sorted by document then line, overlapping or adjacent items merged. */
+export function evidenceLineRanges(evidence) {
+    const sorted = evidence
+        .map(({ docPath, startLine, endLine }) => ({ docPath, startLine, endLine }))
+        .filter((r) => r.endLine >= r.startLine)
+        .sort((a, b) => (a.docPath < b.docPath ? -1 : a.docPath > b.docPath ? 1 : a.startLine - b.startLine));
+    const out = [];
+    for (const r of sorted) {
+        const last = out[out.length - 1];
+        if (last && last.docPath === r.docPath && r.startLine <= last.endLine + 1)
+            last.endLine = Math.max(last.endLine, r.endLine);
+        else
+            out.push(r);
+    }
+    return out;
+}
+const lineCount = (ranges) => ranges.reduce((n, r) => n + r.endLine - r.startLine + 1, 0);
+/** Share of distinct evidence lines that fall inside any gold line range (gold without lines is ignored). */
+export function evidencePrecision(evidence, gold) {
+    const goldRanges = evidenceLineRanges(gold.flatMap((g) => g.start_line !== undefined && g.end_line !== undefined
+        ? [{ docPath: g.doc_path, startLine: g.start_line, endLine: g.end_line }]
+        : []));
+    const ev = evidenceLineRanges(evidence);
+    const total = lineCount(ev);
+    if (goldRanges.length === 0 || total === 0)
+        return null;
+    let inside = 0;
+    for (const e of ev) {
+        for (const g of goldRanges) {
+            if (g.docPath !== e.docPath)
+                continue;
+            const lo = Math.max(e.startLine, g.startLine);
+            const hi = Math.min(e.endLine, g.endLine);
+            if (hi >= lo)
+                inside += hi - lo + 1;
+        }
+    }
+    return round(inside / total);
+}
+function round(x) {
+    return Math.round(x * 10_000) / 10_000;
+}
+function median(xs) {
+    if (xs.length === 0)
+        return 0;
+    const s = [...xs].sort((a, b) => a - b);
+    const mid = s.length >> 1;
+    return s.length % 2 === 1 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+function summarize(rs) {
+    const n = rs.length;
+    const avg = (f) => (n === 0 ? 0 : round(rs.reduce((s, r) => s + f(r), 0) / n));
+    const hits = rs.filter((r) => r.hit).length;
+    const precisions = rs.flatMap((r) => (r.precision === null ? [] : [r.precision]));
+    return {
+        questions: n,
+        hits,
+        hitRate: n === 0 ? 0 : round(hits / n),
+        hitAt1: avg((r) => (r.rank === 1 ? 1 : 0)),
+        mrr: avg((r) => (r.rank === null ? 0 : 1 / r.rank)),
+        avgTokens: avg((r) => r.tokens),
+        avgSteps: avg((r) => r.steps),
+        errors: rs.filter((r) => r.error !== undefined).length,
+        meanEvidenceLines: avg((r) => r.evidenceLines),
+        medianEvidenceLines: median(rs.map((r) => r.evidenceLines)),
+        meanEvidenceTokens: avg((r) => r.evidenceTokens),
+        meanPrecision: precisions.length === 0 ? null : round(precisions.reduce((a, b) => a + b, 0) / precisions.length),
+    };
+}
+/** Overall and per-language summaries; averages run over all questions (errors count as misses with their partial cost). */
+export function aggregate(results) {
+    const byLang = {};
+    for (const lang of ['zh', 'en', 'mixed']) {
+        const rs = results.filter((r) => r.lang === lang);
+        if (rs.length > 0)
+            byLang[lang] = summarize(rs);
+    }
+    return { overall: summarize(results), byLang };
+}

package/dist/index.d.ts

@@ -0,0 +1,23 @@
+export * from './config.js';
+export * from './types.js';
+export { approxTokens, type TokenCounter } from './text/tokens.js';
+export { segmentForIndex, segmentQuery } from './text/segment.js';
+export * from './parse/markdown.js';
+export * from './tree/build.js';
+export * from './tree/optimize.js';
+export { openStore, StoreLockedError, type Store } from './store/db.js';
+export { scanLibrary, normalizeIgnorePatterns, IGNORE_FILE, type ScanOptions, type ScannedFile } from './library/scan.js';
+export { indexLibrary, indexFormat, SUSPECT_SETEXT_CHARS, type IndexFormatOptions, type IndexEvent, type IndexOptions, type IndexReport, type IndexStatus, } from './library/indexer.js';
+export { searchNodes, type SearchHit, type SearchOptions } from './store/search.js';
+export { summarizeDocument, estimateSummaryCost, type SummaryOptions, type SummarizeResult, type SummaryFailure, } from './llm/summarize.js';
+export { createRateLimiter, rateLimiterFor, type RateLimiter, type RateLimitOptions, type RateLimiterClock } from './llm/rate-limit.js';
+export { TOOL_CONTRACT, AGENT_INSTRUCTIONS, type ToolName, type ToolInput, } from './tools/contract.js';
+export { splitByBudget, type Envelope, type ErrorCode, type NextSteps } from './tools/envelope.js';
+export { openLibrary, defaultDbFile, type Library, type OpenLibraryOptions } from './library.js';
+export { createAiSdkTools } from './adapters/ai-sdk.js';
+export { retrieve, RETRIEVE_MAX_STEPS, RETRIEVE_MAX_INPUT_TOKENS, type Evidence, type RetrieveOptions, type RetrieveResult } from './retrieve.js';
+export { createMcpServer } from './adapters/mcp.js';
+export { resolveSummaryModel, ProviderError } from './llm/provider.js';
+export { parseQuestions, QuestionFileError, type EvalQuestion, type Gold, type QuestionIssue, } from './eval/questions.js';
+export { matchesGold, firstHitRank, aggregate, evidenceLineRanges, evidencePrecision, type LineRange, type EvidenceRef, type QuestionResult, type Summary, type Aggregate } from './eval/score.js';
+export { runEval, checkGold, type EvalMode, type RunEvalOptions, type EvalReport } from './eval/run.js';

package/dist/index.js

@@ -0,0 +1,23 @@
+export * from './config.js';
+export * from './types.js';
+export { approxTokens } from './text/tokens.js';
+export { segmentForIndex, segmentQuery } from './text/segment.js';
+export * from './parse/markdown.js';
+export * from './tree/build.js';
+export * from './tree/optimize.js';
+export { openStore, StoreLockedError } from './store/db.js';
+export { scanLibrary, normalizeIgnorePatterns, IGNORE_FILE } from './library/scan.js';
+export { indexLibrary, indexFormat, SUSPECT_SETEXT_CHARS, } from './library/indexer.js';
+export { searchNodes } from './store/search.js';
+export { summarizeDocument, estimateSummaryCost, } from './llm/summarize.js';
+export { createRateLimiter, rateLimiterFor } from './llm/rate-limit.js';
+export { TOOL_CONTRACT, AGENT_INSTRUCTIONS, } from './tools/contract.js';
+export { splitByBudget } from './tools/envelope.js';
+export { openLibrary, defaultDbFile } from './library.js';
+export { createAiSdkTools } from './adapters/ai-sdk.js';
+export { retrieve, RETRIEVE_MAX_STEPS, RETRIEVE_MAX_INPUT_TOKENS } from './retrieve.js';
+export { createMcpServer } from './adapters/mcp.js';
+export { resolveSummaryModel, ProviderError } from './llm/provider.js';
+export { parseQuestions, QuestionFileError, } from './eval/questions.js';
+export { matchesGold, firstHitRank, aggregate, evidenceLineRanges, evidencePrecision } from './eval/score.js';
+export { runEval, checkGold } from './eval/run.js';

package/dist/library/indexer.d.ts

@@ -0,0 +1,89 @@
+import type { Store } from '../store/db.js';
+import { type TokenCounter } from '../text/tokens.js';
+import { type SummaryOptions } from '../llm/summarize.js';
+export type IndexStatus = 'added' | 'updated' | 'unchanged' | 'removed' | 'skipped';
+export interface IndexEvent {
+    path: string;
+    status: IndexStatus;
+    reason?: string;
+    /** Files processed so far (1-based); removals do not count. */
+    done: number;
+    /** Files found by the scan. */
+    total: number;
+}
+export interface IndexOptions {
+    root: string;
+    store: Store;
+    /** Used for both buildTree and optimizeTree. Defaults to approxTokens. */
+    countTokens?: TokenCounter;
+    /**
+     * Names a custom `countTokens` in the index format fingerprint, so swapping one custom counter for another
+     * also triggers a rebuild. Without it every custom counter is recorded as `custom`.
+     */
+    tokenCounterId?: string;
+    /** optimizeTree `maxNodeTokens` (default MAX_NODE_TOKENS). */
+    maxNodeTokens?: number;
+    /** optimizeTree `routingCost` (default ROUTING_COST_TOKENS). */
+    routingCost?: number;
+    /**
+     * parseMarkdown `setext`. Set false for converter output whose bare `---` lines are page breaks.
+     * Stored in the index format fingerprint: when omitted, the value the index was built with is kept (true for a new
+     * index); an explicit value that differs from the stored one re-indexes every document.
+     */
+    setext?: boolean;
+    /**
+     * Extra gitignore-syntax patterns relative to `root` (on top of `.gitignore` files and a root `.markindexignore`).
+     * Documents that become ignored are removed from the index like deleted files.
+     */
+    ignore?: string[];
+    onProgress?: (e: IndexEvent) => void;
+    /** When set, every document (including unchanged ones, so earlier failures are retried) is summarized; omitted = no LLM calls at all. */
+    summary?: SummaryOptions;
+}
+export interface IndexReport {
+    added: string[];
+    updated: string[];
+    removed: string[];
+    unchanged: string[];
+    skipped: {
+        path: string;
+        reason: string;
+    }[];
+    /**
+     * Only with `summary`: documents whose summarization failed (in whole or in part). The document itself stays
+     * indexed and searchable; whatever succeeded was saved, the rest is retried (cache-assisted) on the next
+     * `summarizeDocument` call.
+     */
+    summaryErrors: {
+        path: string;
+        nodeId: string | null;
+        message: string;
+    }[];
+    /** Present when the stored index format differed from this run's, so every document was re-indexed. */
+    rebuilt?: {
+        reason: string;
+    };
+    /**
+     * Non-fatal findings about documents parsed in this run (unchanged documents are not re-checked). Currently:
+     * setext h2 headings that look like page breaks (see `SUSPECT_SETEXT_CHARS`), reported only while `setext` is on.
+     */
+    warnings: {
+        path: string;
+        message: string;
+    }[];
+}
+export type IndexFormatOptions = Pick<IndexOptions, 'countTokens' | 'tokenCounterId' | 'maxNodeTokens' | 'routingCost' | 'setext'>;
+/** Fingerprint of everything besides file content that shapes the stored trees (INDEX_FORMAT_VERSION + build parameters). */
+export declare function indexFormat(opts: IndexFormatOptions): string;
+/** A setext h2 whose heading text is longer than this many code points (or spans several lines) looks like a page. */
+export declare const SUSPECT_SETEXT_CHARS = 120;
+/**
+ * Incrementally indexes the Markdown files under `root` (spec §6): unchanged content hashes are skipped,
+ * new / changed files are rebuilt one transaction per document, and documents missing on disk are deleted.
+ * A file that cannot be read, decoded or indexed is reported in `skipped`; any previous version of it is
+ * dropped so the index never serves content that no longer matches the disk. An unreadable subdirectory is
+ * reported in `skipped` (path with trailing `/`) and the documents previously indexed under it are kept.
+ * When the stored index format (see `indexFormat`) differs from this run's, every document is re-indexed
+ * (summaries are reused through the content-hash cache) and the report carries `rebuilt`.
+ */
+export declare function indexLibrary(opts: IndexOptions): Promise<IndexReport>;