ollama-intern-mcp 1.0.0

package/dist/corpus/fusion.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fusion.d.ts","sourceRoot":"","sources":["../../src/corpus/fusion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,MAAM,WAAW,MAAM;IACrB,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,GAAE,MAAW,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAQhF;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,EAAE,CAErE;AAED,MAAM,WAAW,gBAAgB;IAC/B,8FAA8F;IAC9F,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,sEAAsE;IACtE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,yEAAyE;IACzE,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,mEAAmE;IACnE,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AASD,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,cAAc,EAAE,EACxB,GAAG,EAAE,gBAAgB,EACrB,IAAI,GAAE,gBAAqB,GAC1B,cAAc,EAAE,CAelB"}

package/dist/corpus/fusion.js

@@ -0,0 +1,76 @@
+/**
+ * Score fusion primitives — pure, deterministic, no I/O.
+ *
+ * RRF (Reciprocal Rank Fusion) combines multiple ranked lists into one.
+ * It is the standard fusion strategy in modern hybrid retrieval because
+ * it needs no score calibration — only ranks — so BM25 and cosine can
+ * be safely combined without learning a joint normalization.
+ *
+ * `applyFactBoost` is the "fact mode" reranker: a dominant multiplier
+ * for exact-substring matches plus a secondary multiplier for short
+ * chunks. Boost, never filter — chunks without a match keep their
+ * fused score and stay in the result so a near-miss query never
+ * collapses to empty.
+ */
+/**
+ * Reciprocal Rank Fusion. Score for doc d =
+ *   sum over input lists i: weight_i / (K + rank_i(d))
+ * where K is a smoothing constant (default 60 — the canonical value from
+ * Cormack et al. 2009). A doc that appears in only one list is still
+ * scored; a doc absent from every list never appears in the output.
+ */
+export function rrfFuse(lists, k = 60) {
+    const scores = new Map();
+    for (const { ranked, weight = 1 } of lists) {
+        for (const r of ranked) {
+            scores.set(r.chunkId, (scores.get(r.chunkId) ?? 0) + weight / (k + r.rank));
+        }
+    }
+    return scores;
+}
+/**
+ * Convert a score-sorted list into a rank list. Ties are broken by the
+ * order the scores arrive — callers are responsible for passing in a
+ * deterministically sorted list. Ranks are 1-indexed as RRF expects.
+ */
+export function toRanked(scored) {
+    return scored.map((s, i) => ({ chunkId: s.chunkId, rank: i + 1 }));
+}
+const DEFAULT_FACT_BOOST = {
+    exactSubstringMultiplier: 2.5,
+    shortChunkMaxMultiplier: 1.15,
+    shortChunkFloorChars: 200,
+    shortChunkCeilingChars: 1600,
+};
+/**
+ * Apply the fact-mode reranker. Returns a new list; input is not mutated.
+ * Dominant boost = exact substring match (multiplier ≈ 2.5x by default).
+ * Secondary boost = short-chunk preference (up to ≈ 1.15x, decays to 1.0).
+ * Non-matching chunks keep their fused score unchanged and stay in the list.
+ */
+export function applyFactBoost(scored, ctx, opts = {}) {
+    const cfg = { ...DEFAULT_FACT_BOOST, ...opts };
+    const q = ctx.query.toLowerCase();
+    const hasQuery = q.length > 0;
+    const boosted = [];
+    for (const s of scored) {
+        const text = ctx.chunkText.get(s.chunkId) ?? "";
+        let mult = 1;
+        if (hasQuery && text.toLowerCase().includes(q)) {
+            mult *= cfg.exactSubstringMultiplier;
+        }
+        mult *= shortChunkMultiplier(text.length, cfg);
+        boosted.push({ chunkId: s.chunkId, score: s.score * mult });
+    }
+    return boosted;
+}
+function shortChunkMultiplier(len, cfg) {
+    if (len <= cfg.shortChunkFloorChars)
+        return cfg.shortChunkMaxMultiplier;
+    if (len >= cfg.shortChunkCeilingChars)
+        return 1.0;
+    const range = cfg.shortChunkCeilingChars - cfg.shortChunkFloorChars;
+    const delta = cfg.shortChunkMaxMultiplier - 1.0;
+    return cfg.shortChunkMaxMultiplier - (delta * (len - cfg.shortChunkFloorChars) / range);
+}
+//# sourceMappingURL=fusion.js.map

package/dist/corpus/fusion.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fusion.js","sourceRoot":"","sources":["../../src/corpus/fusion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAeH;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CAAC,KAAmB,EAAE,IAAY,EAAE;IACzD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAkC;IACzD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AACrE,CAAC;AAaD,MAAM,kBAAkB,GAA+B;IACrD,wBAAwB,EAAE,GAAG;IAC7B,uBAAuB,EAAE,IAAI;IAC7B,oBAAoB,EAAE,GAAG;IACzB,sBAAsB,EAAE,IAAI;CAC7B,CAAC;AAYF;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAwB,EACxB,GAAqB,EACrB,OAAyB,EAAE;IAE3B,MAAM,GAAG,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,IAAI,EAAE,CAAC;IAC/C,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9B,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,QAAQ,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,IAAI,IAAI,GAAG,CAAC,wBAAwB,CAAC;QACvC,CAAC;QACD,IAAI,IAAI,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,GAAG,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,oBAAoB,CAC3B,GAAW,EACX,GAA+B;IAE/B,IAAI,GAAG,IAAI,GAAG,CAAC,oBAAoB;QAAE,OAAO,GAAG,CAAC,uBAAuB,CAAC;IACxE,IAAI,GAAG,IAAI,GAAG,CAAC,sBAAsB;QAAE,OAAO,GAAG,CAAC;IAClD,MAAM,KAAK,GAAG,GAAG,CAAC,sBAAsB,GAAG,GAAG,CAAC,oBAAoB,CAAC;IACpE,MAAM,KAAK,GAAG,GAAG,CAAC,uBAAuB,GAAG,GAAG,CAAC;IAChD,OAAO,GAAG,CAAC,uBAAuB,GAAG,CAAC,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,oBAAoB,CAAC,GAAG,KAAK,CAAC,CAAC;AAC1F,CAAC"}

package/dist/corpus/indexer.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Corpus indexer — read files, chunk, embed in batches, persist.
+ *
+ * Idempotent by file hash: if a path's sha256 matches what's already in
+ * the corpus under the same model_version, chunks for that path are
+ * reused verbatim (no re-embedding). Files no longer in the input set
+ * are dropped. New or changed files are embedded fresh.
+ *
+ * This means `index` can be called repeatedly in daily use without
+ * burning the embed tier on unchanged content.
+ */
+import type { OllamaClient } from "../ollama.js";
+export interface IndexParams {
+    name: string;
+    paths: string[];
+    model: string;
+    chunk_chars?: number;
+    chunk_overlap?: number;
+    client: OllamaClient;
+}
+export interface IndexReport {
+    name: string;
+    model_version: string;
+    documents: number;
+    chunks: number;
+    total_chars: number;
+    reused_chunks: number;
+    newly_embedded_chunks: number;
+    dropped_files: string[];
+    elapsed_ms: number;
+}
+export declare function indexCorpus(params: IndexParams): Promise<IndexReport>;
+//# sourceMappingURL=indexer.d.ts.map

package/dist/corpus/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/corpus/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAQjD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,YAAY,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;CACpB;AASD,wBAAsB,WAAW,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAoM3E"}

package/dist/corpus/indexer.js

@@ -0,0 +1,201 @@
+/**
+ * Corpus indexer — read files, chunk, embed in batches, persist.
+ *
+ * Idempotent by file hash: if a path's sha256 matches what's already in
+ * the corpus under the same model_version, chunks for that path are
+ * reused verbatim (no re-embedding). Files no longer in the input set
+ * are dropped. New or changed files are embedded fresh.
+ *
+ * This means `index` can be called repeatedly in daily use without
+ * burning the embed tier on unchanged content.
+ */
+import { readFile, stat } from "node:fs/promises";
+import { resolve } from "node:path";
+import { createHash } from "node:crypto";
+import { chunkDocument, DEFAULT_CHUNK } from "./chunker.js";
+import { CORPUS_SCHEMA_VERSION, loadCorpus, saveCorpus } from "./storage.js";
+import { MANIFEST_SCHEMA_VERSION, loadManifest, saveManifest } from "./manifest.js";
+import { InternError } from "../errors.js";
+const EMBED_BATCH = 64;
+async function sha256File(path) {
+    const content = await readFile(path, "utf8");
+    const hash = "sha256:" + createHash("sha256").update(content).digest("hex");
+    const st = await stat(path);
+    return { hash, mtime: st.mtime.toISOString(), content };
+}
+export async function indexCorpus(params) {
+    const t0 = Date.now();
+    const opts = {
+        chunk_chars: params.chunk_chars ?? DEFAULT_CHUNK.chunk_chars,
+        chunk_overlap: params.chunk_overlap ?? DEFAULT_CHUNK.chunk_overlap,
+    };
+    // Load existing corpus (if any) to reuse unchanged chunks.
+    // An out-of-date schema version throws SCHEMA_INVALID from loadCorpus —
+    // treat that as "no existing corpus" so this very call can rewrite the
+    // file fresh under the current schema (which is the whole point of
+    // re-indexing after a bump).
+    let existing = null;
+    try {
+        existing = await loadCorpus(params.name);
+    }
+    catch (err) {
+        if (err instanceof InternError && err.code === "SCHEMA_INVALID") {
+            existing = null;
+        }
+        else {
+            throw err;
+        }
+    }
+    const reusable = new Map();
+    if (existing && existing.model_version === params.model) {
+        for (const c of existing.chunks) {
+            const key = `${c.path}::${c.file_hash}`;
+            const arr = reusable.get(key) ?? [];
+            arr.push(c);
+            reusable.set(key, arr);
+        }
+    }
+    const allChunks = [];
+    const titles = {};
+    let reusedCount = 0;
+    let newlyEmbeddedCount = 0;
+    let totalChars = 0;
+    const seenPaths = new Set();
+    // Preserve titles from reusable files (they were captured at previous index).
+    if (existing) {
+        for (const [p, t] of Object.entries(existing.titles ?? {})) {
+            titles[p] = t;
+        }
+    }
+    // Pass 1: read + hash every input, reuse where possible, collect chunks to embed.
+    const toEmbedTexts = [];
+    const toEmbedMeta = [];
+    for (const rawPath of params.paths) {
+        const absPath = resolve(rawPath);
+        seenPaths.add(absPath);
+        let fileInfo;
+        try {
+            fileInfo = await sha256File(absPath);
+        }
+        catch (err) {
+            throw new InternError("SOURCE_PATH_NOT_FOUND", `Cannot read input file: ${rawPath} — ${err.message}`, "Check the path exists and is readable.", false);
+        }
+        totalChars += fileInfo.content.length;
+        const reuseKey = `${absPath}::${fileInfo.hash}`;
+        const reused = reusable.get(reuseKey);
+        if (reused && reused.length > 0) {
+            allChunks.push(...reused);
+            reusedCount += reused.length;
+            continue;
+        }
+        // Fresh chunking for this file — heading-aware.
+        const { title, chunks } = chunkDocument(fileInfo.content, opts);
+        titles[absPath] = title;
+        for (const ck of chunks) {
+            toEmbedMeta.push({
+                path: absPath,
+                file_hash: fileInfo.hash,
+                file_mtime: fileInfo.mtime,
+                chunk_index: ck.index,
+                char_start: ck.char_start,
+                char_end: ck.char_end,
+                text: ck.text,
+                heading_path: ck.heading_path,
+                chunk_type: ck.chunk_type,
+            });
+            toEmbedTexts.push(ck.text);
+        }
+    }
+    // Pass 2: embed everything that needs embedding, in batches.
+    if (toEmbedTexts.length > 0) {
+        for (let i = 0; i < toEmbedTexts.length; i += EMBED_BATCH) {
+            const batch = toEmbedTexts.slice(i, i + EMBED_BATCH);
+            const resp = await params.client.embed({ model: params.model, input: batch });
+            if (resp.embeddings.length !== batch.length) {
+                throw new Error(`Embed returned ${resp.embeddings.length} vectors for ${batch.length} inputs`);
+            }
+            for (let j = 0; j < batch.length; j++) {
+                const meta = toEmbedMeta[i + j];
+                allChunks.push({
+                    id: `${params.name}-${allChunks.length.toString(16).padStart(6, "0")}`,
+                    path: meta.path,
+                    file_hash: meta.file_hash,
+                    file_mtime: meta.file_mtime,
+                    chunk_index: meta.chunk_index,
+                    char_start: meta.char_start,
+                    char_end: meta.char_end,
+                    text: meta.text,
+                    vector: resp.embeddings[j],
+                    heading_path: meta.heading_path,
+                    chunk_type: meta.chunk_type,
+                });
+                newlyEmbeddedCount += 1;
+            }
+        }
+    }
+    // Drop files that were in the old corpus but not in the input.
+    const droppedFiles = [];
+    if (existing) {
+        const previousPaths = new Set(existing.chunks.map((c) => c.path));
+        for (const p of previousPaths) {
+            if (!seenPaths.has(p))
+                droppedFiles.push(p);
+        }
+    }
+    for (const p of droppedFiles)
+        delete titles[p];
+    // Scope titles to paths actually present in this index.
+    const livingTitles = {};
+    for (const c of allChunks) {
+        if (c.path in titles)
+            livingTitles[c.path] = titles[c.path];
+        else if (!(c.path in livingTitles))
+            livingTitles[c.path] = null;
+    }
+    const corpus = {
+        schema_version: CORPUS_SCHEMA_VERSION,
+        name: params.name,
+        model_version: params.model,
+        model_digest: null,
+        indexed_at: new Date().toISOString(),
+        chunk_chars: opts.chunk_chars,
+        chunk_overlap: opts.chunk_overlap,
+        stats: {
+            documents: new Set(allChunks.map((c) => c.path)).size,
+            chunks: allChunks.length,
+            total_chars: totalChars,
+        },
+        titles: livingTitles,
+        chunks: allChunks,
+    };
+    await saveCorpus(corpus);
+    // Write the manifest alongside the corpus. The corpus is "reality";
+    // the manifest is "intent" — what the caller declared should be here.
+    // Refresh later reconciles the two.
+    const manifestPaths = [...seenPaths].sort();
+    const prevManifest = await loadManifest(params.name).catch(() => null);
+    const now = new Date().toISOString();
+    const manifest = {
+        schema_version: MANIFEST_SCHEMA_VERSION,
+        name: params.name,
+        paths: manifestPaths,
+        embed_model: params.model,
+        chunk_chars: opts.chunk_chars,
+        chunk_overlap: opts.chunk_overlap,
+        created_at: prevManifest?.created_at ?? now,
+        updated_at: now,
+    };
+    await saveManifest(manifest);
+    return {
+        name: corpus.name,
+        model_version: corpus.model_version,
+        documents: corpus.stats.documents,
+        chunks: corpus.stats.chunks,
+        total_chars: corpus.stats.total_chars,
+        reused_chunks: reusedCount,
+        newly_embedded_chunks: newlyEmbeddedCount,
+        dropped_files: droppedFiles,
+        elapsed_ms: Date.now() - t0,
+    };
+}
+//# sourceMappingURL=indexer.js.map

package/dist/corpus/indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/corpus/indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAqC,MAAM,cAAc,CAAC;AAC/F,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,UAAU,EAAqC,MAAM,cAAc,CAAC;AAChH,OAAO,EAAE,uBAAuB,EAAE,YAAY,EAAE,YAAY,EAAuB,MAAM,eAAe,CAAC;AACzG,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,MAAM,WAAW,GAAG,EAAE,CAAC;AAuBvB,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAmB;IACnD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACtB,MAAM,IAAI,GAAiB;QACzB,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW;QAC5D,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa;KACnE,CAAC;IAEF,2DAA2D;IAC3D,wEAAwE;IACxE,uEAAuE;IACvE,mEAAmE;IACnE,6BAA6B;IAC7B,IAAI,QAAQ,GAAsB,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAChE,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;IAClD,IAAI,QAAQ,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YACpC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACZ,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAkB,EAAE,CAAC;IACpC,MAAM,MAAM,GAAkC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IAEpC,8EAA8E;IAC9E,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;YAC3D,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IAED,kFAAkF;IAClF,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,MAAM,WAAW,GAUZ,EAAE,CAAC;IAER,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;QACjC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,QAA0D,CAAC;QAC/D,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,WAAW,CACnB,uBAAuB,EACvB,2BAA2B,OAAO,MAAO,GAAa,CAAC,OAAO,EAAE,EAChE,wCAAwC,EACxC,KAAK,CACN,CAAC;QACJ,CAAC;QACD,UAAU,IAAI,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;QACtC,MAAM,QAAQ,GAAG,GAAG,OAAO,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtC,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;YAC7B,SAAS;QACX,CAAC;QACD,gDAAgD;QAChD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAChE,MAAM,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;QACxB,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACxB,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,OAAO;gBACb,SAAS,EAAE,QAAQ,CAAC,IAAI;gBACxB,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,EAAE,CAAC,KAAK;gBACrB,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ;gBACrB,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,YAAY,EAAE,EAAE,CAAC,YAAY;gBAC7B,UAAU,EAAE,EAAE,CAAC,UAAU;aAC1B,CAAC,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC9E,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5C,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,UAAU,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAC9E,CAAC;YACJ,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBAChC,SAAS,CAAC,IAAI,CAAC;oBACb,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;oBACtE,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,SAAS,EAAE,IAAI,CAAC,SAAS;oBACzB,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,WAAW,EAAE,IAAI,CAAC,WAAW;oBAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;oBAC3B,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;oBAC1B,YAAY,EAAE,IAAI,CAAC,YAAY;oBAC/B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC5B,CAAC,CAAC;gBACH,kBAAkB,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAE/C,wDAAwD;IACxD,MAAM,YAAY,GAAkC,EAAE,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,CAAC,IAAI,IAAI,MAAM;YAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;aACvD,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC;YAAE,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAClE,CAAC;IAED,MAAM,MAAM,GAAe;QACzB,cAAc,EAAE,qBAAqB;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,aAAa,EAAE,MAAM,CAAC,KAAK;QAC3B,YAAY,EAAE,IAAI;QAClB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACpC,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,KAAK,EAAE;YACL,SAAS,EAAE,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACrD,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,UAAU;SACxB;QACD,MAAM,EAAE,YAAY;QACpB,MAAM,EAAE,SAAS;KAClB,CAAC;IAEF,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAEzB,oEAAoE;IACpE,sEAAsE;IACtE,oCAAoC;IACpC,MAAM,aAAa,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,MAAM,YAAY,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACvE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAmB;QAC/B,cAAc,EAAE,uBAAuB;QACvC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,aAAa;QACpB,WAAW,EAAE,MAAM,CAAC,KAAK;QACzB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,UAAU,EAAE,YAAY,EAAE,UAAU,IAAI,GAAG;QAC3C,UAAU,EAAE,GAAG;KAChB,CAAC;IACF,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAE7B,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,aAAa,EAAE,MAAM,CAAC,aAAa;QACnC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS;QACjC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;QAC3B,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,WAAW;QACrC,aAAa,EAAE,WAAW;QAC1B,qBAAqB,EAAE,kBAAkB;QACzC,aAAa,EAAE,YAAY;QAC3B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE;KAC5B,CAAC;AACJ,CAAC"}

package/dist/corpus/lexical.d.ts

@@ -0,0 +1,95 @@
+/**
+ * Lexical index + BM25 scorer — the "keyword truth" rail.
+ *
+ * This file is deliberately narrow: it produces one trustworthy primitive,
+ * `scoreLexical(query, index)`, that returns deterministic field-aware
+ * BM25 scores over v2 chunks. No fusion, no modes, no synthesis — slice 3
+ * layers those on top.
+ *
+ * Four fields are scored independently and then combined into a single
+ * `score`, but the per-field breakdown is preserved on every result so
+ * slice 3 can recombine them (e.g. title_path mode) without reworking
+ * this layer.
+ *
+ * Field identity matters:
+ *   - title         : the file's first H1 (if any)
+ *   - heading_path  : the breadcrumb stack for the chunk's section
+ *   - path          : file path tokens (dirs, base name, extension)
+ *   - body          : the chunk text itself
+ *
+ * Determinism guarantees:
+ *   - Sort by score desc, then (path asc, chunk_index asc) as tie-break.
+ *   - Zero-score chunks are omitted.
+ *   - Tokenizer is pure: same input → same tokens, independent of locale.
+ *   - Stopword set is a fixed list in this file (no env lookup).
+ */
+import type { CorpusChunk } from "./storage.js";
+export declare const DEFAULT_FIELD_WEIGHTS: {
+    readonly title: 3;
+    readonly heading: 2;
+    readonly path: 1.5;
+    readonly body: 1;
+};
+export declare function isStopword(term: string): boolean;
+/**
+ * Generic tokenizer for body/title/heading text.
+ * Lowercases, splits on non-alphanumeric, drops stopwords and empties.
+ * No stemming, no fuzzy matching — those are explicit slice-3+ decisions.
+ */
+export declare function tokenize(text: string): string[];
+/**
+ * Path tokenizer — same splitter as body, but runs the full path through
+ * so dirs, base name stem, and extension each become searchable tokens.
+ * Absolute-path prefixes (drive letters, leading slashes) contribute
+ * nothing once split, which is the right behavior.
+ */
+export declare function tokenizePath(path: string): string[];
+/**
+ * Heading tokenizer — flattens the heading_path breadcrumb into tokens.
+ * Deeper headings contribute the same as shallower ones; weighting by
+ * depth is a slice-3+ concern.
+ */
+export declare function tokenizeHeadings(headingPath: string[]): string[];
+export type FieldName = "body" | "heading" | "title" | "path";
+interface LexicalDoc {
+    chunkId: string;
+    chunkIndex: number;
+    path: string;
+    tokens: Record<FieldName, Map<string, number>>;
+    len: Record<FieldName, number>;
+}
+export interface LexicalIndex {
+    chunkCount: number;
+    avg: Record<FieldName, number>;
+    /** Document frequency per field: term -> number of chunks whose field contains it. */
+    df: Record<FieldName, Map<string, number>>;
+    docs: LexicalDoc[];
+}
+export interface LexicalScore {
+    chunkId: string;
+    chunkIndex: number;
+    path: string;
+    score: number;
+    fieldScores: Record<FieldName, number>;
+    matchedTerms: string[];
+}
+/**
+ * Build the in-memory lexical index from the chunks of a loaded corpus.
+ * Idempotent and deterministic: same input corpus → identical index.
+ */
+export declare function buildLexicalIndex(chunks: CorpusChunk[], titles: Record<string, string | null>): LexicalIndex;
+export interface ScoreLexicalOptions {
+    weights?: Partial<Record<FieldName, number>>;
+    /** Override the query tokenizer. Default = tokenize(). */
+    queryTokens?: string[];
+}
+/**
+ * Score every chunk in the index against `query`. Returns only chunks
+ * with score > 0, sorted score desc, (path asc, chunk_index asc) tie-break.
+ *
+ * Field scores are preserved on every result so slice 3 can recombine
+ * without rescoring.
+ */
+export declare function scoreLexical(query: string, index: LexicalIndex, opts?: ScoreLexicalOptions): LexicalScore[];
+export {};
+//# sourceMappingURL=lexical.d.ts.map

package/dist/corpus/lexical.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexical.d.ts","sourceRoot":"","sources":["../../src/corpus/lexical.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAShD,eAAO,MAAM,qBAAqB;;;;;CAKxB,CAAC;AAcX,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEhD;AAED;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAW/C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAEnD;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAGhE;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAE9D,UAAU,UAAU;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/C,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/B,sFAAsF;IACtF,EAAE,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3C,IAAI,EAAE,UAAU,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,WAAW,EAAE,EACrB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,GACpC,YAAY,CAqDd;AAkCD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7C,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,YAAY,EACnB,IAAI,GAAE,mBAAwB,GAC7B,YAAY,EAAE,CA0ChB"}

package/dist/corpus/lexical.js

@@ -0,0 +1,227 @@
+/**
+ * Lexical index + BM25 scorer — the "keyword truth" rail.
+ *
+ * This file is deliberately narrow: it produces one trustworthy primitive,
+ * `scoreLexical(query, index)`, that returns deterministic field-aware
+ * BM25 scores over v2 chunks. No fusion, no modes, no synthesis — slice 3
+ * layers those on top.
+ *
+ * Four fields are scored independently and then combined into a single
+ * `score`, but the per-field breakdown is preserved on every result so
+ * slice 3 can recombine them (e.g. title_path mode) without reworking
+ * this layer.
+ *
+ * Field identity matters:
+ *   - title         : the file's first H1 (if any)
+ *   - heading_path  : the breadcrumb stack for the chunk's section
+ *   - path          : file path tokens (dirs, base name, extension)
+ *   - body          : the chunk text itself
+ *
+ * Determinism guarantees:
+ *   - Sort by score desc, then (path asc, chunk_index asc) as tie-break.
+ *   - Zero-score chunks are omitted.
+ *   - Tokenizer is pure: same input → same tokens, independent of locale.
+ *   - Stopword set is a fixed list in this file (no env lookup).
+ */
+// BM25 constants — classic Robertson/Walker defaults.
+const BM25_K1 = 1.2;
+const BM25_B = 0.75;
+// Default per-field weights. Metadata matches outweigh body because a
+// term in the title almost always means "this document is about that
+// thing" while a term in the body can be incidental.
+export const DEFAULT_FIELD_WEIGHTS = {
+    title: 3.0,
+    heading: 2.0,
+    path: 1.5,
+    body: 1.0,
+};
+// Small, conservative English stopword list. Applied at BOTH index and
+// query time so IDF statistics stay consistent. Kept short on purpose —
+// overly aggressive stopword removal eats legitimate technical terms.
+const STOPWORDS = new Set([
+    "a", "an", "and", "are", "as", "at", "be", "been", "being", "but", "by",
+    "do", "does", "did", "for", "from", "had", "has", "have", "he", "her",
+    "him", "his", "i", "if", "in", "into", "is", "it", "its", "me", "my",
+    "of", "on", "or", "our", "she", "so", "than", "that", "the", "their",
+    "them", "these", "they", "this", "those", "to", "us", "was", "we", "were",
+    "will", "with", "you", "your",
+]);
+export function isStopword(term) {
+    return STOPWORDS.has(term);
+}
+/**
+ * Generic tokenizer for body/title/heading text.
+ * Lowercases, splits on non-alphanumeric, drops stopwords and empties.
+ * No stemming, no fuzzy matching — those are explicit slice-3+ decisions.
+ */
+export function tokenize(text) {
+    if (!text)
+        return [];
+    const lower = text.toLowerCase();
+    const raw = lower.split(/[^a-z0-9]+/);
+    const out = [];
+    for (const t of raw) {
+        if (t.length === 0)
+            continue;
+        if (STOPWORDS.has(t))
+            continue;
+        out.push(t);
+    }
+    return out;
+}
+/**
+ * Path tokenizer — same splitter as body, but runs the full path through
+ * so dirs, base name stem, and extension each become searchable tokens.
+ * Absolute-path prefixes (drive letters, leading slashes) contribute
+ * nothing once split, which is the right behavior.
+ */
+export function tokenizePath(path) {
+    return tokenize(path);
+}
+/**
+ * Heading tokenizer — flattens the heading_path breadcrumb into tokens.
+ * Deeper headings contribute the same as shallower ones; weighting by
+ * depth is a slice-3+ concern.
+ */
+export function tokenizeHeadings(headingPath) {
+    const joined = headingPath.join(" ");
+    return tokenize(joined);
+}
+function countTokens(tokens) {
+    const m = new Map();
+    for (const t of tokens)
+        m.set(t, (m.get(t) ?? 0) + 1);
+    return m;
+}
+/**
+ * Build the in-memory lexical index from the chunks of a loaded corpus.
+ * Idempotent and deterministic: same input corpus → identical index.
+ */
+export function buildLexicalIndex(chunks, titles) {
+    const docs = [];
+    const df = {
+        body: new Map(),
+        heading: new Map(),
+        title: new Map(),
+        path: new Map(),
+    };
+    const totals = { body: 0, heading: 0, title: 0, path: 0 };
+    for (const c of chunks) {
+        const bodyTokens = tokenize(c.text);
+        const headingTokens = tokenizeHeadings(c.heading_path);
+        const title = titles[c.path] ?? "";
+        const titleTokens = tokenize(title);
+        const pathTokens = tokenizePath(c.path);
+        const doc = {
+            chunkId: c.id,
+            chunkIndex: c.chunk_index,
+            path: c.path,
+            tokens: {
+                body: countTokens(bodyTokens),
+                heading: countTokens(headingTokens),
+                title: countTokens(titleTokens),
+                path: countTokens(pathTokens),
+            },
+            len: {
+                body: bodyTokens.length,
+                heading: headingTokens.length,
+                title: titleTokens.length,
+                path: pathTokens.length,
+            },
+        };
+        docs.push(doc);
+        for (const field of ["body", "heading", "title", "path"]) {
+            totals[field] += doc.len[field];
+            for (const term of doc.tokens[field].keys()) {
+                df[field].set(term, (df[field].get(term) ?? 0) + 1);
+            }
+        }
+    }
+    const n = docs.length;
+    const avg = {
+        body: n === 0 ? 0 : totals.body / n,
+        heading: n === 0 ? 0 : totals.heading / n,
+        title: n === 0 ? 0 : totals.title / n,
+        path: n === 0 ? 0 : totals.path / n,
+    };
+    return { chunkCount: n, avg, df, docs };
+}
+/**
+ * BM25+ IDF variant: ln((N - df + 0.5) / (df + 0.5) + 1).
+ * The +1 inside the log keeps IDF non-negative for very common terms.
+ */
+function bm25Idf(n, df) {
+    return Math.log((n - df + 0.5) / (df + 0.5) + 1);
+}
+function bm25FieldScore(queryTerms, doc, field, index) {
+    const avgLen = index.avg[field];
+    const docLen = doc.len[field];
+    let score = 0;
+    const matched = [];
+    if (docLen === 0)
+        return { score: 0, matched };
+    for (const term of queryTerms) {
+        const tf = doc.tokens[field].get(term) ?? 0;
+        if (tf === 0)
+            continue;
+        const df = index.df[field].get(term) ?? 0;
+        const idf = bm25Idf(index.chunkCount, df);
+        const normLen = avgLen === 0 ? 1 : docLen / avgLen;
+        const denom = tf + BM25_K1 * (1 - BM25_B + BM25_B * normLen);
+        score += idf * ((tf * (BM25_K1 + 1)) / denom);
+        matched.push(term);
+    }
+    return { score, matched };
+}
+/**
+ * Score every chunk in the index against `query`. Returns only chunks
+ * with score > 0, sorted score desc, (path asc, chunk_index asc) tie-break.
+ *
+ * Field scores are preserved on every result so slice 3 can recombine
+ * without rescoring.
+ */
+export function scoreLexical(query, index, opts = {}) {
+    const queryTerms = opts.queryTokens ?? tokenize(query);
+    if (queryTerms.length === 0 || index.chunkCount === 0)
+        return [];
+    const weights = {
+        title: opts.weights?.title ?? DEFAULT_FIELD_WEIGHTS.title,
+        heading: opts.weights?.heading ?? DEFAULT_FIELD_WEIGHTS.heading,
+        path: opts.weights?.path ?? DEFAULT_FIELD_WEIGHTS.path,
+        body: opts.weights?.body ?? DEFAULT_FIELD_WEIGHTS.body,
+    };
+    const results = [];
+    for (const doc of index.docs) {
+        const fieldScores = { body: 0, heading: 0, title: 0, path: 0 };
+        const matchedSet = new Set();
+        for (const field of ["body", "heading", "title", "path"]) {
+            const { score, matched } = bm25FieldScore(queryTerms, doc, field, index);
+            fieldScores[field] = score;
+            for (const m of matched)
+                matchedSet.add(m);
+        }
+        const combined = weights.title * fieldScores.title +
+            weights.heading * fieldScores.heading +
+            weights.path * fieldScores.path +
+            weights.body * fieldScores.body;
+        if (combined <= 0)
+            continue;
+        results.push({
+            chunkId: doc.chunkId,
+            chunkIndex: doc.chunkIndex,
+            path: doc.path,
+            score: combined,
+            fieldScores,
+            matchedTerms: [...matchedSet].sort(),
+        });
+    }
+    results.sort((a, b) => {
+        if (b.score !== a.score)
+            return b.score - a.score;
+        if (a.path !== b.path)
+            return a.path < b.path ? -1 : 1;
+        return a.chunkIndex - b.chunkIndex;
+    });
+    return results;
+}
+//# sourceMappingURL=lexical.js.map