opencode-rag-plugin 1.15.1 → 1.17.0

package/dist/eval/compare-rankings.js

@@ -0,0 +1,245 @@
+/**
+ * @fileoverview Ranking order comparison between two branch benchmark runs.
+ * Focuses on rank agreement rather than absolute scores.
+ *
+ * Usage: node --import tsx src/eval/compare-rankings.ts
+ *   --main .opencode/rag_db/eval-results/main.json
+ *   --branch .opencode/rag_db/eval-results/t1-cosine-l2.json
+ *   --output .opencode/rag_db/eval-results/ranking-report.md
+ */
+import { readFileSync, writeFileSync, mkdirSync } from "node:fs";
+import path from "node:path";
+function parseArgs() {
+    const args = process.argv.slice(2);
+    let main = "", branch = "", output = ".opencode/rag_db/eval-results/ranking-report.md";
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--main" && args[i + 1]) {
+            main = path.resolve(args[i + 1]);
+            i++;
+        }
+        else if (args[i] === "--branch" && args[i + 1]) {
+            branch = path.resolve(args[i + 1]);
+            i++;
+        }
+        else if (args[i] === "--output" && args[i + 1]) {
+            output = path.resolve(args[i + 1]);
+            i++;
+        }
+    }
+    if (!main || !branch) {
+        console.error("Usage: node --import tsx src/eval/compare-rankings.ts --main <json> --branch <json> [--output <md>]");
+        process.exit(1);
+    }
+    return { main, branch, output };
+}
+function normalizePath(fp) {
+    // Strip clone-directory prefix from paths like "../OpenCodeRAG-main/src/foo.ts"
+    return fp.replace(/^\.\.[\/\\][^\/\\]+[\/\\]/, "");
+}
+function identityKey(r) {
+    return normalizePath(r.filePath) + ":" + r.startLine;
+}
+/** Kendall's tau-b rank correlation for two orderings of the same set. */
+function kendallTau(rankA, rankB) {
+    const ids = [...rankA.keys()];
+    let concordant = 0, discordant = 0;
+    for (let i = 0; i < ids.length; i++) {
+        for (let j = i + 1; j < ids.length; j++) {
+            const aI = rankA.get(ids[i]), aJ = rankA.get(ids[j]);
+            const bI = rankB.get(ids[i]), bJ = rankB.get(ids[j]);
+            const diffA = aI - aJ, diffB = bI - bJ;
+            if (diffA * diffB > 0)
+                concordant++;
+            else if (diffA * diffB < 0)
+                discordant++;
+        }
+    }
+    const total = concordant + discordant;
+    return total === 0 ? 1 : (concordant - discordant) / total;
+}
+function avg(arr) {
+    return arr.length === 0 ? 0 : arr.reduce((s, v) => s + v, 0) / arr.length;
+}
+async function main() {
+    const { main: mainPath, branch: branchPath, output } = parseArgs();
+    const mainData = JSON.parse(readFileSync(mainPath, "utf-8"));
+    const branchData = JSON.parse(readFileSync(branchPath, "utf-8"));
+    const mBranch = mainData.branch + " (" + mainData.commit + ")";
+    const bBranch = branchData.branch + " (" + branchData.commit + ")";
+    const perQuery = [];
+    let totalTop1Same = 0, totalTop3Same = 0, totalTop5Same = 0, totalFullSame = 0;
+    let totalOverlap5 = 0, totalOverlap20 = 0;
+    const kendalls = [];
+    const maxQ = Math.min(mainData.queries.length, branchData.queries.length);
+    for (let i = 0; i < maxQ; i++) {
+        const mq = mainData.queries[i], bq = branchData.queries[i];
+        const mTop = mq.topResults, bTop = bq.topResults;
+        const mKey = mTop.map(identityKey);
+        const bKey = bTop.map(identityKey);
+        const top1Same = mKey[0] === bKey[0];
+        const top3Same = mKey[0] === bKey[0] && mKey[1] === bKey[1] && mKey[2] === bKey[2];
+        const top5Same = mKey.slice(0, 5).every((k, j) => k === bKey[j]);
+        const topKFullSame = mKey.every((k, j) => k === bKey[j]);
+        const set5 = new Set(bKey.slice(0, 5));
+        const overlapTop5 = mKey.slice(0, 5).filter((k) => set5.has(k)).length;
+        const set20 = new Set(bKey);
+        const overlapTop20 = mKey.filter((k) => set20.has(k)).length;
+        const aRank = new Map(mKey.map((k, j) => [k, j]));
+        const bRank = new Map(bKey.map((k, j) => [k, j]));
+        const sharedIds = [...aRank.keys()].filter((k) => bRank.has(k));
+        const sharedRankA = new Map(sharedIds.map((k) => [k, aRank.get(k)]));
+        const sharedRankB = new Map(sharedIds.map((k) => [k, bRank.get(k)]));
+        const tau = kendallTau(sharedRankA, sharedRankB);
+        if (top1Same)
+            totalTop1Same++;
+        if (top3Same)
+            totalTop3Same++;
+        if (top5Same)
+            totalTop5Same++;
+        if (topKFullSame)
+            totalFullSame++;
+        totalOverlap5 += overlapTop5;
+        totalOverlap20 += overlapTop20;
+        kendalls.push(tau);
+        perQuery.push({
+            query: mq.query,
+            top1Same,
+            top3Same,
+            top5Same,
+            topKFullSame,
+            overlapTop5,
+            overlapTop20,
+            kendallTau: tau,
+            mTopScore: mTop[0]?.score ?? 0,
+            bTopScore: bTop[0]?.score ?? 0,
+        });
+    }
+    // Print console output
+    const SEP = "─";
+    console.log("\n" + SEP.repeat(80));
+    console.log("  RANKING ORDER COMPARISON");
+    console.log("  " + mBranch + " vs " + bBranch);
+    console.log(SEP.repeat(80) + "\n");
+    const w = [38, 14];
+    const top = "┌─" + w.map((x) => SEP.repeat(x)).join("─┬─") + "─┐";
+    const sep = "├─" + w.map((x) => SEP.repeat(x)).join("─┼─") + "─┤";
+    const bot = "└─" + w.map((x) => SEP.repeat(x)).join("─┴─") + "─┘";
+    const cell = (s, i) => s.length >= w[i] ? s.slice(0, w[i]) : s + " ".repeat(w[i] - s.length);
+    console.log("  Overlap Summary:");
+    console.log("  " + top);
+    console.log("  │ " + cell("Metric", 0) + " │ " + cell("Value", 1) + " │");
+    console.log("  " + sep);
+    console.log("  │ " + cell("Top-1 match", 0) + " │ " + cell(totalTop1Same + "/" + maxQ, 1) + " │");
+    console.log("  │ " + cell("Top-3 identical", 0) + " │ " + cell(totalTop3Same + "/" + maxQ, 1) + " │");
+    console.log("  │ " + cell("Top-5 identical", 0) + " │ " + cell(totalTop5Same + "/" + maxQ, 1) + " │");
+    console.log("  │ " + cell("Full top-K identical", 0) + " │ " + cell(totalFullSame + "/" + maxQ, 1) + " │");
+    console.log("  │ " + cell("Avg overlap in top-5", 0) + " │ " + cell((totalOverlap5 / maxQ).toFixed(1), 1) + " │");
+    console.log("  │ " + cell("Avg overlap in top-K", 0) + " │ " + cell((totalOverlap20 / maxQ).toFixed(1), 1) + " │");
+    console.log("  │ " + cell("Kendall's τ (avg)", 0) + " │ " + cell(avg(kendalls).toFixed(4), 1) + " │");
+    console.log("  " + bot + "\n");
+    if (totalFullSame === maxQ) {
+        console.log("  ▲ Ranking is IDENTICAL across all " + maxQ + " queries.");
+        console.log("    RRF and linear fusion produce the same rank order\n");
+    }
+    else if (totalTop5Same === maxQ) {
+        console.log("  ▲ Top-5 ranking is IDENTICAL across all queries.");
+        console.log("    Minor differences exist beyond position 5.\n");
+    }
+    else {
+        console.log("  ◆ Ranking order differs between branches.\n");
+        console.log("  With keyword contributions active, RRF and linear fusion");
+        console.log("  produce different rank orderings. RRF rewards results that");
+        console.log("  rank highly in BOTH signals over results that rank well in");
+        console.log("  only ONE signal.\n");
+    }
+    // Also show queries where keyword matched
+    const queriesWithKeyword = mainData.queries.filter((q) => q.topResults.some((r) => r.explanation && r.explanation.rawKeywordScore > 0));
+    console.log("  Queries with keyword contributions: " + queriesWithKeyword.length + "/" + maxQ);
+    if (queriesWithKeyword.length === 0) {
+        console.log("  (No keyword index matches found in this index)\n");
+    }
+    // Write markdown report
+    const report = [];
+    report.push("# Ranking Order Comparison");
+    report.push("");
+    report.push("**" + mBranch + "** vs **" + bBranch + "**");
+    report.push("**Generated:** " + new Date().toISOString());
+    report.push("");
+    report.push("## Config");
+    report.push("");
+    report.push("| Setting | `" + mainData.branch + "` | `" + branchData.branch + "` |");
+    report.push("|---|---|---|");
+    report.push("| Embedding | " + mainData.config.embeddingModel + " | " + branchData.config.embeddingModel + " |");
+    report.push("| topK | " + mainData.config.topK + " | " + branchData.config.topK + " |");
+    report.push("| minScore | " + mainData.config.minScore + " | " + branchData.config.minScore + " |");
+    report.push("| Hybrid | " + mainData.config.hybridEnabled + " | " + branchData.config.hybridEnabled + " |");
+    report.push("| keywordWeight | " + mainData.config.keywordWeight + " | " + branchData.config.keywordWeight + " |");
+    report.push("| Index chunks | " + mainData.indexChunkCount + " | " + branchData.indexChunkCount + " |");
+    report.push("");
+    report.push("## Ranking Agreement");
+    report.push("");
+    report.push("| Metric | Value |");
+    report.push("|---|---|");
+    report.push("| Top-1 match | " + totalTop1Same + "/" + maxQ + " |");
+    report.push("| Top-3 identical | " + totalTop3Same + "/" + maxQ + " |");
+    report.push("| Top-5 identical | " + totalTop5Same + "/" + maxQ + " |");
+    report.push("| Full top-K identical | " + totalFullSame + "/" + maxQ + " |");
+    report.push("| Avg overlap in top-5 | " + (totalOverlap5 / maxQ).toFixed(1) + " |");
+    report.push("| Avg overlap in top-K | " + (totalOverlap20 / maxQ).toFixed(1) + " |");
+    report.push("| Kendall's τ (avg) | " + avg(kendalls).toFixed(4) + " |");
+    report.push("| Queries with keyword contribution | " + queriesWithKeyword.length + "/" + maxQ + " |");
+    report.push("");
+    if (totalFullSame === maxQ) {
+        report.push("## Verdict");
+        report.push("");
+        report.push("**Ranking is 100% identical across all queries.**");
+        report.push("");
+        report.push("The `t1-cosine-l2` branch changes two things simultaneously:");
+        report.push("- Vector scoring: L2 distance → cosine similarity");
+        report.push("- Hybrid fusion: Weighted linear combination → RRF (K=60)");
+        report.push("");
+        report.push("However, in this benchmark, **keyword scores are zero on every query**");
+        report.push("because the keyword index doesn't match any query terms. When only one signal");
+        report.push("(vector similarity) contributes, both fusion methods produce the same");
+        report.push("rank order. This is because both are **monotonically decreasing functions**");
+        report.push("of the vector rank:");
+        report.push("");
+        report.push("- **Linear**: `score = (1-kw) · normVectorScore` (monotonic in vector score)");
+        report.push("- **RRF**: `score = (1-kw) / (K + rank + 1)` (monotonic in vector rank)");
+        report.push("");
+        report.push("Since vector rank is itself monotonic with vector score, the final ordering");
+        report.push("is identical regardless of which formula is used.");
+        report.push("");
+        report.push("### When would RRF make a difference?");
+        report.push("");
+        report.push("RRF excels when **both vector AND keyword signals contribute** to a query.");
+        report.push("It can boost results that rank highly in both sources while demoting results");
+        report.push("that only rank well in one. To see this effect:");
+        report.push("- Index more files (including docs with token-rich content)");
+        report.push("- Use queries with specific identifier/keyword terms that match the keyword index");
+        report.push("- Increase keywordWeight to amplify keyword contributions");
+        report.push("");
+    }
+    report.push("## Per-Query Detail");
+    report.push("");
+    report.push("| # | Query | Top-1 same | Top-5 same | Full same | τ | main score | branch score |");
+    report.push("|---|---|:---:|:---:|:---:|:---:|:---:|:---:|");
+    for (let i = 0; i < perQuery.length; i++) {
+        const pq = perQuery[i];
+        const q = pq.query.length > 48 ? pq.query.substring(0, 45) + "..." : pq.query;
+        report.push("| " + (i + 1) + " | " + q +
+            " | " + (pq.top1Same ? "✓" : "✗") +
+            " | " + (pq.top5Same ? "✓" : "✗") +
+            " | " + (pq.topKFullSame ? "✓" : "✗") +
+            " | " + pq.kendallTau.toFixed(3) +
+            " | " + pq.mTopScore.toFixed(3) +
+            " | " + pq.bTopScore.toFixed(3) +
+            " |");
+    }
+    report.push("");
+    mkdirSync(path.dirname(output), { recursive: true });
+    writeFileSync(output, report.join("\n"), "utf-8");
+    console.log("  Report written to: " + output + "\n");
+}
+main().catch((err) => { console.error("Ranking comparison failed:", err); process.exit(1); });
+//# sourceMappingURL=compare-rankings.js.map

package/dist/eval/dump-descriptions.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @fileoverview Dump all chunk descriptions from a LanceDB index to a JSON file.
+ * Used to copy descriptions from one branch to another for fair comparison.
+ *
+ * Usage: node --import tsx src/eval/dump-descriptions.ts --output .opencode/rag_db/chunk-descriptions.json
+ */
+export {};

package/dist/eval/dump-descriptions.js

@@ -0,0 +1,58 @@
+/**
+ * @fileoverview Dump all chunk descriptions from a LanceDB index to a JSON file.
+ * Used to copy descriptions from one branch to another for fair comparison.
+ *
+ * Usage: node --import tsx src/eval/dump-descriptions.ts --output .opencode/rag_db/chunk-descriptions.json
+ */
+import { writeFileSync, mkdirSync } from "node:fs";
+import path from "node:path";
+const WORKTREE = process.cwd();
+const STORE_PATH = path.join(WORKTREE, ".opencode", "rag_db");
+async function main() {
+    const args = process.argv.slice(2);
+    let outputPath = path.join(WORKTREE, ".opencode", "rag_db", "chunk-descriptions.json");
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--output" && args[i + 1]) {
+            outputPath = path.resolve(WORKTREE, args[i + 1]);
+            break;
+        }
+    }
+    console.log(`  Dumping descriptions from: ${STORE_PATH}`);
+    console.log(`  Output: ${outputPath}`);
+    const lancedb = await import("@lancedb/lancedb");
+    const db = await lancedb.connect(STORE_PATH);
+    const tableNames = await db.tableNames();
+    if (!tableNames.includes("chunks")) {
+        console.error("  No chunks table found in the store.");
+        await db.close();
+        process.exit(1);
+    }
+    const table = await db.openTable("chunks");
+    const count = await table.countRows();
+    console.log(`  Total rows: ${count}`);
+    const rows = await table.query().select(["id", "description", "filePath", "startLine", "endLine"]).limit(count).toArray();
+    const descriptions = {};
+    let hasDesc = 0;
+    for (const row of rows) {
+        const filePath = row.filePath.replace(/\\/g, "/");
+        const startLine = row.startLine;
+        const endLine = row.endLine;
+        const desc = row.description ?? "";
+        const key = `${filePath}:${startLine}:${endLine}`;
+        descriptions[key] = desc;
+        if (desc)
+            hasDesc++;
+    }
+    const dir = path.dirname(outputPath);
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(outputPath, JSON.stringify(descriptions, null, 2), "utf-8");
+    console.log(`  Dumped ${Object.keys(descriptions).length} chunks`);
+    console.log(`  ${hasDesc} have non-empty descriptions`);
+    console.log(`  Written to: ${outputPath}`);
+    await db.close();
+}
+main().catch((err) => {
+    console.error("Dump failed:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=dump-descriptions.js.map

package/dist/eval/fast-index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @fileoverview Fast indexer that uses pre-dumped descriptions instead of LLM generation.
+ * Indexes only files unchanged between the current branch and main.
+ * Run on the main branch after copying chunk-descriptions.json from t1-cosine-l2.
+ *
+ * Usage: node --import tsx src/eval/fast-index.ts --descriptions .opencode/rag_db/chunk-descriptions.json
+ */
+export {};

package/dist/eval/fast-index.js

@@ -0,0 +1,283 @@
+/**
+ * @fileoverview Fast indexer that uses pre-dumped descriptions instead of LLM generation.
+ * Indexes only files unchanged between the current branch and main.
+ * Run on the main branch after copying chunk-descriptions.json from t1-cosine-l2.
+ *
+ * Usage: node --import tsx src/eval/fast-index.ts --descriptions .opencode/rag_db/chunk-descriptions.json
+ */
+import { readFileSync, readdirSync } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { loadConfig, DEFAULT_CONFIG } from "../core/config.js";
+import { createEmbedder } from "../embedder/factory.js";
+import { createVectorStore } from "../vectorstore/factory.js";
+import { loadRuntimeOverrides, applyRuntimeOverrides, } from "../core/runtime-overrides.js";
+import { resolveApiKey } from "../core/resolve-api-key.js";
+import { chunkFile } from "../chunker/factory.js";
+import { embedBatch } from "../embedder/factory.js";
+import { normalizeFilePath } from "../core/manifest.js";
+import { KeywordIndex } from "../retriever/keyword-index.js";
+const WORKTREE = process.cwd();
+const STORE_PATH = path.join(WORKTREE, ".opencode", "rag_db");
+/** Directories to skip entirely when scanning for files. */
+const SKIP_DIRS = new Set([
+    "node_modules",
+    ".git",
+    ".opencode",
+    "dist",
+    "build",
+    "__pycache__",
+    ".venv",
+    ".claude",
+    ".github",
+    "memory:",
+    "wasm",
+    ".commandcode",
+    ".agents",
+    "graphify-out",
+    "__tests__",
+]);
+/** File extensions to include. */
+const INCLUDE_EXTS = new Set([".ts", ".tsx"]);
+/** Changed files between main and t1-cosine-l2 (relative paths, / separators). */
+const CHANGED_FILES = new Set([
+    "src/api.ts",
+    "src/chunker/pdf.ts",
+    "src/cli/commands/init-helpers.ts",
+    "src/core/config.ts",
+    "src/core/interfaces.ts",
+    "src/core/manifest.ts",
+    "src/core/setup-runtime.ts",
+    "src/indexer/embed-stage.ts",
+    "src/retriever/keyword-index.ts",
+    "src/retriever/retriever.ts",
+    "src/vectorstore/lancedb.ts",
+    "src/vectorstore/memory.ts",
+]);
+/** Our own eval scripts (not part of the index). */
+const EVAL_FILES = new Set([
+    "src/eval/run-branch-compare.ts",
+    "src/eval/compare-merge.ts",
+    "src/eval/dump-descriptions.ts",
+    "src/eval/update-descriptions.ts",
+    "src/eval/fast-index.ts",
+]);
+function isChangedFile(relPath) {
+    return CHANGED_FILES.has(relPath) || EVAL_FILES.has(relPath);
+}
+function walkFiles(dir) {
+    const results = [];
+    try {
+        const entries = readdirSync(dir, { withFileTypes: true });
+        for (const entry of entries) {
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                if (!SKIP_DIRS.has(entry.name) && !entry.name.startsWith(".")) {
+                    results.push(...walkFiles(fullPath));
+                }
+            }
+            else if (entry.isFile()) {
+                const ext = path.extname(entry.name).toLowerCase();
+                if (INCLUDE_EXTS.has(ext)) {
+                    results.push(fullPath);
+                }
+            }
+        }
+    }
+    catch {
+        // directory not found or no permissions
+    }
+    return results;
+}
+function parseArgs() {
+    const args = process.argv.slice(2);
+    let descriptionsPath = path.join(WORKTREE, ".opencode", "rag_db", "chunk-descriptions.json");
+    let force = false;
+    for (let i = 0; i < args.length; i++) {
+        if (args[i] === "--descriptions" && args[i + 1]) {
+            descriptionsPath = path.resolve(WORKTREE, args[i + 1]);
+            i++;
+        }
+        else if (args[i] === "--force") {
+            force = true;
+        }
+    }
+    return { descriptionsPath, force };
+}
+function getConfig() {
+    const configPath = path.join(WORKTREE, "opencode-rag.json");
+    let cfg;
+    try {
+        cfg = loadConfig(configPath);
+    }
+    catch {
+        cfg = DEFAULT_CONFIG;
+    }
+    const overrides = loadRuntimeOverrides(STORE_PATH);
+    cfg = applyRuntimeOverrides(cfg, overrides);
+    resolveApiKey(cfg, WORKTREE);
+    return cfg;
+}
+async function main() {
+    const { descriptionsPath, force } = parseArgs();
+    console.log(`\n  Fast Indexer — skipping description generation`);
+    console.log(`  Descriptions: ${descriptionsPath}\n`);
+    // Helper: split "absPath:startLine:endLine" key (handles Windows drive letters)
+    function splitLocationKey(k) {
+        const lastColon = k.lastIndexOf(":");
+        const secondLastColon = k.lastIndexOf(":", lastColon - 1);
+        return {
+            absPath: k.substring(0, secondLastColon),
+            startLine: parseInt(k.substring(secondLastColon + 1, lastColon), 10),
+            endLine: parseInt(k.substring(lastColon + 1), 10),
+        };
+    }
+    // Load pre-dumped descriptions and remap paths to this worktree
+    const rawDescs = JSON.parse(readFileSync(descriptionsPath, "utf-8"));
+    // Detect source worktree from first key
+    const sampleKey = Object.keys(rawDescs)[0] ?? "";
+    const sampleParts = splitLocationKey(sampleKey);
+    const samplePath = sampleParts.absPath;
+    const srcIdx = samplePath.lastIndexOf("/src/");
+    const docIdx = samplePath.lastIndexOf("/doc/");
+    const cutIdx = Math.max(srcIdx, docIdx);
+    const sourceWorktree = cutIdx > 0 ? samplePath.substring(0, cutIdx) : "";
+    const thisWorktree = WORKTREE.replace(/\\/g, "/");
+    const descriptions = {};
+    for (const [key, desc] of Object.entries(rawDescs)) {
+        if (!sourceWorktree) {
+            descriptions[key] = desc;
+            continue;
+        }
+        const { absPath, startLine, endLine } = splitLocationKey(key);
+        if (!absPath.startsWith(sourceWorktree + "/")) {
+            descriptions[key] = desc;
+            continue;
+        }
+        const relativePath = absPath.substring(sourceWorktree.length);
+        const newKey = `${thisWorktree}${relativePath}:${startLine}:${endLine}`;
+        descriptions[newKey] = desc;
+    }
+    console.log(`  Loaded ${Object.keys(rawDescs).length} descriptions from dump`);
+    console.log(`  Source worktree: ${sourceWorktree}`);
+    console.log(`  This worktree:  ${thisWorktree}`);
+    // Set up config, embedder, store
+    const cfg = getConfig();
+    const embedder = createEmbedder(cfg);
+    // Probe dimension
+    const probe = await embedder.embed(["dimension-probe"], "query");
+    const dimension = probe[0]?.length ?? 384;
+    console.log(`  Embedding dimension: ${dimension}\n`);
+    // Clear existing store (requires --force if data exists)
+    const store = createVectorStore(cfg, STORE_PATH, dimension);
+    const existingCount = await store.count();
+    if (existingCount > 0 && !force) {
+        console.error(`  Store already has ${existingCount} chunks. Use --force to overwrite.`);
+        await store.close();
+        process.exit(1);
+    }
+    if (existingCount > 0) {
+        await store.clear();
+        console.log(`  Cleared existing store (backed up ${existingCount} chunks)\n`);
+    }
+    else {
+        console.log("  Store is empty\n");
+    }
+    // Walk files
+    const srcDir = path.join(WORKTREE, "src");
+    const allFiles = walkFiles(srcDir);
+    const filteredFiles = allFiles
+        .map((f) => path.relative(WORKTREE, f).replace(/\\/g, "/"))
+        .filter((f) => !isChangedFile(f));
+    console.log(`  Found ${allFiles.length} .ts files in src/`);
+    console.log(`  After filtering out changed files: ${filteredFiles.length}\n`);
+    // Process files sequentially (chunking + description lookup)
+    const allChunks = [];
+    let skippedNoDesc = 0;
+    for (let i = 0; i < filteredFiles.length; i++) {
+        const relPath = filteredFiles[i];
+        const absPath = path.join(WORKTREE, relPath);
+        process.stdout.write(`  [${i + 1}/${filteredFiles.length}] ${relPath}...`);
+        try {
+            const content = await fs.readFile(absPath, "utf-8");
+            if (!content.trim()) {
+                console.log(" empty");
+                continue;
+            }
+            const chunks = await chunkFile(absPath, content);
+            if (!chunks || chunks.length === 0) {
+                console.log(" no chunks");
+                continue;
+            }
+            // Attach descriptions from dump
+            const normalizedPath = normalizeFilePath(absPath);
+            let attached = 0;
+            for (const chunk of chunks) {
+                const key = `${normalizedPath}:${chunk.metadata.startLine}:${chunk.metadata.endLine}`;
+                const desc = descriptions[key];
+                if (desc) {
+                    chunk.description = desc;
+                    attached++;
+                }
+                else {
+                    // Try alternate key ending (some chunkers add +1 to endLine)
+                    const altKey = `${normalizedPath}:${chunk.metadata.startLine}:${chunk.metadata.startLine + (chunk.metadata.endLine - chunk.metadata.startLine) + 1}`;
+                    const altDesc = descriptions[altKey];
+                    if (altDesc) {
+                        chunk.description = altDesc;
+                        attached++;
+                    }
+                }
+            }
+            if (attached === 0) {
+                skippedNoDesc++;
+            }
+            allChunks.push(...chunks);
+            console.log(` ${chunks.length} chunks, ${attached} descriptions`);
+        }
+        catch (err) {
+            console.log(` error: ${err.message}`);
+        }
+    }
+    console.log(`\n  Total chunks to embed: ${allChunks.length}`);
+    console.log(`  Files with no description match: ${skippedNoDesc}`);
+    // Build texts to embed and batch-embed
+    const textsToEmbed = [];
+    for (const chunk of allChunks) {
+        const desc = chunk.description ?? "";
+        if (desc.trim()) {
+            textsToEmbed.push(`${path.relative(WORKTREE, chunk.metadata.filePath).replace(/\\/g, "/")}\n\n${desc}\n\n${chunk.content}`);
+        }
+        else {
+            textsToEmbed.push(`${path.relative(WORKTREE, chunk.metadata.filePath).replace(/\\/g, "/")}\n\n${chunk.content}`);
+        }
+    }
+    console.log(`  Embedding ${textsToEmbed.length} texts...`);
+    const embeddings = await embedBatch(embedder, textsToEmbed, cfg.indexing.embedBatchSize ?? 100, "document", cfg.indexing.embedConcurrency ?? 3);
+    console.log(`  Got ${embeddings.length} embeddings`);
+    // Attach embeddings to chunks and store
+    for (let i = 0; i < allChunks.length; i++) {
+        const emb = embeddings[i];
+        if (emb && emb.length > 0 && typeof emb[0] === "number") {
+            allChunks[i].embedding = emb;
+        }
+    }
+    const validChunks = allChunks.filter((c) => c.embedding && c.embedding.length > 0);
+    console.log(`  Storing ${validChunks.length} chunks...`);
+    await store.addChunks(validChunks);
+    const count = await store.count();
+    console.log(`  Store now has ${count} chunks`);
+    await store.close();
+    // Build keyword index from the same chunks
+    console.log("  Building keyword index...");
+    const ki = new KeywordIndex();
+    ki.addChunks(validChunks);
+    await ki.save(STORE_PATH);
+    console.log(`  Keyword index saved (${ki.count()} entries)`);
+    console.log("\n  Fast indexing complete!\n");
+}
+main().catch((err) => {
+    console.error("Fast index failed:", err);
+    process.exit(1);
+});
+//# sourceMappingURL=fast-index.js.map

package/dist/eval/run-branch-compare.d.ts

@@ -0,0 +1,7 @@
+/**
+ * @fileoverview Branch comparison benchmark — runs N queries against the indexed codebase
+ * and outputs per-query results as JSON for later comparison.
+ *
+ * Usage: node --import tsx src/eval/run-branch-compare.ts --output .opencode/rag_db/eval-results/branch.json
+ */
+export {};