causantic 0.9.3 → 0.10.0

package/dist/eval/experiments/pipeline-dropout/run-experiment.js

@@ -0,0 +1,347 @@
+/**
+ * Pipeline Dropout Analysis
+ *
+ * Traces where target chunks get lost in the retrieval pipeline.
+ * For each query, checks whether the ground-truth chunk survives each stage:
+ *   1. Vector search (raw top-K)
+ *   2. RRF fusion (vector + keyword)
+ *   3. Cluster expansion
+ *   4. Oversized filtering
+ *   5. MMR reranking
+ *   6. Budget assembly
+ *   7. Chain walking (recall path)
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/pipeline-dropout/run-experiment.ts [--sample-size=50]
+ */
+import { getDb } from '../../../storage/db.js';
+import { vectorStore, indexVectorStore } from '../../../storage/vector-store.js';
+import { getChunkById } from '../../../storage/chunk-store.js';
+import { getIndexEntryCount, getIndexedChunkCount, dereferenceToChunkIds, searchIndexEntriesByKeyword, } from '../../../storage/index-entry-store.js';
+import { getAllClusters, getClusterChunkIds } from '../../../storage/cluster-store.js';
+import { Embedder } from '../../../models/embedder.js';
+import { getModel } from '../../../models/model-registry.js';
+import { loadConfig, toRuntimeConfig } from '../../../config/loader.js';
+import { KeywordStore } from '../../../storage/keyword-store.js';
+import { fuseRRF } from '../../../retrieval/rrf.js';
+import { expandViaClusters } from '../../../retrieval/cluster-expander.js';
+import { reorderWithMMR } from '../../../retrieval/mmr.js';
+import { walkChains, selectBestChain } from '../../../retrieval/chain-walker.js';
+import { generateSearchQueries } from '../index-vs-chunk/query-generator.js';
+// ── Helpers ────────────────────────────────────────────────────────────────
+function createRng(seed) {
+    let s = seed;
+    return () => {
+        s = (s * 1664525 + 1013904223) & 0x7fffffff;
+        return s / 0x7fffffff;
+    };
+}
+function sampleChunks(sampleSize, seed) {
+    getDb();
+    const clusters = getAllClusters();
+    if (clusters.length === 0)
+        throw new Error('No clusters found.');
+    const rng = createRng(seed);
+    const result = [];
+    const shuffled = [...clusters].sort(() => rng() - 0.5);
+    for (const cluster of shuffled) {
+        if (result.length >= sampleSize)
+            break;
+        const chunkIds = getClusterChunkIds(cluster.id);
+        if (chunkIds.length < 2)
+            continue;
+        const numPicks = Math.min(2, Math.ceil(sampleSize / clusters.length), chunkIds.length);
+        const shuffledIds = [...chunkIds].sort(() => rng() - 0.5);
+        for (let i = 0; i < numPicks && result.length < sampleSize; i++) {
+            const chunk = getChunkById(shuffledIds[i]);
+            if (!chunk || chunk.content.length < 100)
+                continue;
+            result.push({
+                id: chunk.id,
+                sessionSlug: chunk.sessionSlug,
+                content: chunk.content,
+                clusterId: cluster.id,
+                clusterName: cluster.name,
+            });
+        }
+    }
+    return result;
+}
+function findIn(items, targetId) {
+    const idx = items.findIndex((i) => i.chunkId === targetId);
+    return idx >= 0 ? idx + 1 : 0;
+}
+// ── Main ───────────────────────────────────────────────────────────────────
+async function runAnalysis() {
+    const args = process.argv.slice(2);
+    const sampleSizeArg = args.find((a) => a.startsWith('--sample-size='));
+    const sampleSize = sampleSizeArg ? parseInt(sampleSizeArg.split('=')[1], 10) : 50;
+    const seed = 42;
+    console.log('=== Pipeline Dropout Analysis ===\n');
+    getDb();
+    const externalConfig = loadConfig();
+    const config = toRuntimeConfig(externalConfig);
+    const { hybridSearch, clusterExpansion, mmrReranking, embeddingModel } = config;
+    const maxTokens = config.mcpMaxResponseTokens;
+    const useIndexSearch = config.semanticIndex.useForSearch && getIndexEntryCount() > 0;
+    console.log(`Search path: ${useIndexSearch ? 'INDEX' : 'CHUNK'}`);
+    console.log(`Max tokens: ${maxTokens}`);
+    vectorStore.setModelId(embeddingModel);
+    if (useIndexSearch)
+        indexVectorStore.setModelId(embeddingModel);
+    // 1. Sample and generate queries
+    console.log(`\nSampling ${sampleSize} chunks...`);
+    const sampledChunks = sampleChunks(sampleSize, seed);
+    console.log(`  Sampled ${sampledChunks.length} chunks`);
+    console.log('Generating queries...');
+    const queries = await generateSearchQueries(sampledChunks, config.clusterRefreshModel);
+    console.log(`  Generated ${queries.length} queries\n`);
+    // 2. Prepare embedder
+    const embedder = new Embedder();
+    await embedder.load(getModel(embeddingModel));
+    let keywordStore = null;
+    try {
+        keywordStore = new KeywordStore();
+    }
+    catch { /* unavailable */ }
+    const vectorLimits = [20, 50, 100, 200];
+    console.log(`Vector search limits: ${vectorLimits.join(', ')}`);
+    // Pre-embed all queries
+    console.log('Embedding queries...');
+    const queryEmbeddings = [];
+    for (let i = 0; i < queries.length; i++) {
+        const { embedding } = await embedder.embed(queries[i].query, true);
+        queryEmbeddings.push(embedding);
+    }
+    console.log(`  Embedded ${queryEmbeddings.length} queries\n`);
+    // 3. Sweep vector search limits
+    for (const vectorSearchLimit of vectorLimits) {
+        console.log(`── Vector limit: ${vectorSearchLimit} ──`);
+        const traces = [];
+        for (let qi = 0; qi < queries.length; qi++) {
+            const q = queries[qi];
+            const embedding = queryEmbeddings[qi];
+            const targetId = q.groundTruthChunkId;
+            const stages = {
+                vectorSearch: false,
+                rrfFusion: false,
+                clusterExpansion: false,
+                oversizedFilter: false,
+                mmrRerank: false,
+                budgetAssembly: false,
+                chainSeeds: false,
+                chainOutput: false,
+            };
+            let vectorRank = 0;
+            let rrfRank = 0;
+            let fusedResults;
+            if (useIndexSearch) {
+                const entryCount = getIndexEntryCount();
+                const indexedChunks = getIndexedChunkCount();
+                const entriesPerChunk = indexedChunks > 0 ? entryCount / indexedChunks : 1;
+                const indexSearchLimit = Math.ceil(vectorSearchLimit * entriesPerChunk);
+                const indexSimilar = await indexVectorStore.search(embedding, indexSearchLimit);
+                for (let i = 0; i < indexSimilar.length; i++) {
+                    const chunkIds = dereferenceToChunkIds([indexSimilar[i].id]);
+                    if (chunkIds.includes(targetId)) {
+                        stages.vectorSearch = true;
+                        if (vectorRank === 0)
+                            vectorRank = i + 1;
+                        break;
+                    }
+                }
+                let indexKeywordResults = [];
+                try {
+                    indexKeywordResults = searchIndexEntriesByKeyword(q.query, hybridSearch.keywordSearchLimit);
+                }
+                catch { /* unavailable */ }
+                const indexVectorItems = indexSimilar.map((s) => ({
+                    chunkId: s.id,
+                    score: Math.max(0, 1 - s.distance),
+                    source: 'vector',
+                }));
+                const indexKeywordItems = indexKeywordResults.map((r) => ({
+                    chunkId: r.id,
+                    score: r.score,
+                    source: 'keyword',
+                }));
+                const indexFused = fuseRRF([
+                    { items: indexVectorItems, weight: hybridSearch.vectorWeight },
+                    ...(indexKeywordItems.length > 0
+                        ? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
+                        : []),
+                ], hybridSearch.rrfK);
+                const chunkScoreMap = new Map();
+                for (const item of indexFused) {
+                    const entryChunkIds = dereferenceToChunkIds([item.chunkId]);
+                    for (const cid of entryChunkIds) {
+                        const existing = chunkScoreMap.get(cid);
+                        if (!existing || item.score > existing.score) {
+                            chunkScoreMap.set(cid, { score: item.score, source: item.source });
+                        }
+                    }
+                }
+                const allChunkIds = [...chunkScoreMap.keys()];
+                fusedResults = allChunkIds.map((cid) => {
+                    const entry = chunkScoreMap.get(cid);
+                    return { chunkId: cid, score: entry.score, source: entry.source };
+                });
+                rrfRank = fusedResults.findIndex((r) => r.chunkId === targetId) + 1;
+                if (rrfRank > 0)
+                    stages.rrfFusion = true;
+            }
+            else {
+                const similar = await vectorStore.search(embedding, vectorSearchLimit);
+                vectorRank = similar.findIndex((s) => s.id === targetId) + 1;
+                if (vectorRank > 0)
+                    stages.vectorSearch = true;
+                let keywordResults = [];
+                if (keywordStore) {
+                    try {
+                        keywordResults = keywordStore.search(q.query, hybridSearch.keywordSearchLimit);
+                    }
+                    catch { /* */ }
+                }
+                const vectorItems = similar.map((s) => ({
+                    chunkId: s.id,
+                    score: Math.max(0, 1 - s.distance),
+                    source: 'vector',
+                }));
+                const keywordItems = keywordResults.map((r) => ({
+                    chunkId: r.id,
+                    score: r.score,
+                    source: 'keyword',
+                }));
+                fusedResults = fuseRRF([
+                    { items: vectorItems, weight: hybridSearch.vectorWeight },
+                    ...(keywordItems.length > 0
+                        ? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
+                        : []),
+                ], hybridSearch.rrfK);
+                rrfRank = findIn(fusedResults, targetId);
+                if (rrfRank > 0)
+                    stages.rrfFusion = true;
+            }
+            // Cluster expansion
+            const expanded = expandViaClusters(fusedResults, clusterExpansion, undefined, undefined, config.feedbackWeight);
+            if (findIn(expanded, targetId) > 0)
+                stages.clusterExpansion = true;
+            // Dedupe
+            const seen = new Set();
+            const deduped = expanded.filter((r) => {
+                if (seen.has(r.chunkId))
+                    return false;
+                seen.add(r.chunkId);
+                return true;
+            });
+            const chunkTokenMap = new Map();
+            for (const item of deduped) {
+                const chunk = getChunkById(item.chunkId);
+                if (chunk)
+                    chunkTokenMap.set(item.chunkId, chunk.approxTokens || 500);
+            }
+            // Oversized filter
+            const sizeBounded = deduped.filter((item) => {
+                const tokens = chunkTokenMap.get(item.chunkId);
+                return tokens !== undefined && tokens <= maxTokens;
+            });
+            if (findIn(sizeBounded, targetId) > 0)
+                stages.oversizedFilter = true;
+            // MMR reranking
+            const reordered = await reorderWithMMR(sizeBounded, embedding, mmrReranking, {
+                tokenBudget: maxTokens,
+                chunkTokenCounts: chunkTokenMap,
+            });
+            const mmrRank = findIn(reordered, targetId);
+            if (mmrRank > 0)
+                stages.mmrRerank = true;
+            // Log MMR dropout details
+            if (stages.oversizedFilter && !stages.mmrRerank) {
+                const preRank = findIn(sizeBounded, targetId);
+                const targetTokens = chunkTokenMap.get(targetId) ?? 0;
+                const totalTokensInReordered = reordered.reduce((s, r) => s + (chunkTokenMap.get(r.chunkId) ?? 0), 0);
+                console.log(`    MMR DROP: pre-rank=${preRank}/${sizeBounded.length} target=${targetTokens}tok reordered=${reordered.length} reorderedTokens=${totalTokensInReordered} budget=${maxTokens}`);
+            }
+            // Budget assembly
+            let budgetUsed = 0;
+            for (const item of reordered) {
+                const tokens = chunkTokenMap.get(item.chunkId) ?? 500;
+                if (budgetUsed + tokens > maxTokens)
+                    break;
+                budgetUsed += tokens;
+                if (item.chunkId === targetId) {
+                    stages.budgetAssembly = true;
+                    break;
+                }
+            }
+            // Chain walking
+            const seedIds = expanded.slice(0, 5).map((r) => r.chunkId);
+            stages.chainSeeds = seedIds.includes(targetId);
+            try {
+                const chains = await walkChains(seedIds, {
+                    direction: 'backward',
+                    tokenBudget: maxTokens,
+                    queryEmbedding: embedding,
+                });
+                const bestChain = selectBestChain(chains);
+                if (bestChain && bestChain.chunkIds.includes(targetId)) {
+                    stages.chainOutput = true;
+                }
+            }
+            catch { /* chain walk failed */ }
+            let droppedAt = null;
+            const stageOrder = [
+                ['vectorSearch', 'Vector Search'],
+                ['rrfFusion', 'RRF Fusion'],
+                ['clusterExpansion', 'Cluster Expansion'],
+                ['oversizedFilter', 'Oversized Filter'],
+                ['mmrRerank', 'MMR Rerank'],
+                ['budgetAssembly', 'Budget Assembly'],
+            ];
+            for (const [key, name] of stageOrder) {
+                if (!stages[key]) {
+                    droppedAt = name;
+                    break;
+                }
+            }
+            traces.push({
+                query: q.query,
+                groundTruthChunkId: targetId,
+                stages,
+                droppedAt,
+                vectorRank,
+                rrfRank,
+            });
+        }
+        // Aggregate for this limit
+        const total = traces.length;
+        const stageCounts = {};
+        for (const trace of traces) {
+            for (const [key, val] of Object.entries(trace.stages)) {
+                stageCounts[key] = (stageCounts[key] ?? 0) + (val ? 1 : 0);
+            }
+        }
+        const stageLabels = [
+            ['vectorSearch', 'Vector'],
+            ['rrfFusion', 'RRF'],
+            ['clusterExpansion', 'Cluster'],
+            ['oversizedFilter', 'Size'],
+            ['mmrRerank', 'MMR'],
+            ['budgetAssembly', 'Budget'],
+            ['chainSeeds', 'Seeds'],
+            ['chainOutput', 'Chain'],
+        ];
+        const rates = stageLabels.map(([key, label]) => {
+            const count = stageCounts[key] ?? 0;
+            return `${label}: ${((count / total) * 100).toFixed(0)}%`;
+        });
+        console.log(`  ${rates.join(' → ')}`);
+    }
+    await embedder.dispose();
+    console.log('\nDone.');
+}
+runAnalysis().catch((err) => {
+    console.error('Analysis failed:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=run-experiment.js.map

package/dist/eval/experiments/pipeline-dropout/run-experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-experiment.js","sourceRoot":"","sources":["../../../../src/eval/experiments/pipeline-dropout/run-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,EAAE,OAAO,EAAmB,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,MAAM,wCAAwC,CAAC;AAC3E,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,oCAAoC,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAyB,MAAM,sCAAsC,CAAC;AA2BpG,8EAA8E;AAE9E,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,OAAO,GAAG,EAAE;QACV,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,GAAG,UAAU,CAAC,GAAG,UAAU,CAAC;QAC5C,OAAO,CAAC,GAAG,UAAU,CAAC;IACxB,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,UAAkB,EAAE,IAAY;IACpD,KAAK,EAAE,CAAC;IACR,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAEjE,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAEvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;QACvC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAElC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG;gBAAE,SAAS;YACnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,KAAiC,EAAE,QAAgB;IACjE,MAAM,GAAG,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;IAC3D,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,8EAA8E;AAE9E,KAAK,UAAU,WAAW;IACxB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClF,MAAM,IAAI,GAAG,EAAE,CAAC;IAEhB,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IAEnD,KAAK,EAAE,CAAC;IACR,MAAM,cAAc,GAAG,UAAU,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAC/C,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,GAAG,MAAM,CAAC;IAChF,MAAM,SAAS,GAAG,MAAM,CAAC,oBAAoB,CAAC;IAE9C,MAAM,cAAc,GAAG,MAAM,CAAC,aAAa,CAAC,YAAY,IAAI,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,CAAC,gBAAgB,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,eAAe,SAAS,EAAE,CAAC,CAAC;IAExC,WAAW,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACvC,IAAI,cAAc;QAAE,gBAAgB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IAEhE,iCAAiC;IACjC,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,YAAY,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;IAExD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,aAAa,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACvF,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,YAAY,CAAC,CAAC;IAEvD,sBAAsB;IACtB,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IAChC,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;IAE9C,IAAI,YAAY,GAAwB,IAAI,CAAC;IAC7C,IAAI,CAAC;QAAC,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAEtE,MAAM,YAAY,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,yBAAyB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEhE,wBAAwB;IACxB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,eAAe,GAAe,EAAE,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACnE,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,cAAc,eAAe,CAAC,MAAM,YAAY,CAAC,CAAC;IAE9D,gCAAgC;IAChC,KAAK,MAAM,iBAAiB,IAAI,YAAY,EAAE,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,oBAAoB,iBAAiB,KAAK,CAAC,CAAC;QACxD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAEhC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;YACtB,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,kBAAkB,CAAC;YAEtC,MAAM,MAAM,GAAkB;gBAC5B,YAAY,EAAE,KAAK;gBACnB,SAAS,EAAE,KAAK;gBAChB,gBAAgB,EAAE,KAAK;gBACvB,eAAe,EAAE,KAAK;gBACtB,SAAS,EAAE,KAAK;gBAChB,cAAc,EAAE,KAAK;gBACrB,UAAU,EAAE,KAAK;gBACjB,WAAW,EAAE,KAAK;aACnB,CAAC;YAEF,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,YAA0B,CAAC;YAE/B,IAAI,cAAc,EAAE,CAAC;gBACnB,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;gBACxC,MAAM,aAAa,GAAG,oBAAoB,EAAE,CAAC;gBAC7C,MAAM,eAAe,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3E,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,GAAG,eAAe,CAAC,CAAC;gBAExE,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;gBAEhF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBAC7C,MAAM,QAAQ,GAAG,qBAAqB,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;oBAC7D,IAAI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;wBAChC,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;wBAC3B,IAAI,UAAU,KAAK,CAAC;4BAAE,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC;wBACzC,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,mBAAmB,GAAyC,EAAE,CAAC;gBACnE,IAAI,CAAC;oBACH,mBAAmB,GAAG,2BAA2B,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,kBAAkB,CAAC,CAAC;gBAC9F,CAAC;gBAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,CAAC;gBAE7B,MAAM,gBAAgB,GAAiB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC9D,OAAO,EAAE,CAAC,CAAC,EAAE;oBACb,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;oBAClC,MAAM,EAAE,QAAiB;iBAC1B,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,GAAiB,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtE,OAAO,EAAE,CAAC,CAAC,EAAE;oBACb,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,SAAkB;iBAC3B,CAAC,CAAC,CAAC;gBAEJ,MAAM,UAAU,GAAG,OAAO,CACxB;oBACE,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,CAAC,YAAY,EAAE;oBAC9D,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;wBAC9B,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,YAAY,CAAC,aAAa,EAAE,CAAC;wBACpE,CAAC,CAAC,EAAE,CAAC;iBACR,EACD,YAAY,CAAC,IAAI,CAClB,CAAC;gBAEF,MAAM,aAAa,GAAG,IAAI,GAAG,EAA2D,CAAC;gBACzF,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC9B,MAAM,aAAa,GAAG,qBAAqB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC5D,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;wBAChC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;wBACxC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;4BAC7C,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;wBACrE,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,MAAM,WAAW,GAAG,CAAC,GAAG,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9C,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;oBACrC,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC;oBACtC,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;gBACpE,CAAC,CAAC,CAAC;gBAEH,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACpE,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;gBACvE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,QAAQ,CAAC,GAAG,CAAC,CAAC;gBAC7D,IAAI,UAAU,GAAG,CAAC;oBAAE,MAAM,CAAC,YAAY,GAAG,IAAI,CAAC;gBAE/C,IAAI,cAAc,GAAyC,EAAE,CAAC;gBAC9D,IAAI,YAAY,EAAE,CAAC;oBACjB,IAAI,CAAC;wBAAC,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,kBAAkB,CAAC,CAAC;oBAAC,CAAC;oBACvF,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC;gBACjB,CAAC;gBAED,MAAM,WAAW,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACpD,OAAO,EAAE,CAAC,CAAC,EAAE;oBACb,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;oBAClC,MAAM,EAAE,QAAiB;iBAC1B,CAAC,CAAC,CAAC;gBACJ,MAAM,YAAY,GAAiB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC5D,OAAO,EAAE,CAAC,CAAC,EAAE;oBACb,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,SAAkB;iBAC3B,CAAC,CAAC,CAAC;gBAEJ,YAAY,GAAG,OAAO,CACpB;oBACE,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,YAAY,EAAE;oBACzD,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;wBACzB,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,aAAa,EAAE,CAAC;wBAC/D,CAAC,CAAC,EAAE,CAAC;iBACR,EACD,YAAY,CAAC,IAAI,CAClB,CAAC;gBAEF,OAAO,GAAG,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;gBACzC,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YAC3C,CAAC;YAED,oBAAoB;YACpB,MAAM,QAAQ,GAAG,iBAAiB,CAChC,YAAY,EACZ,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,MAAM,CAAC,cAAc,CACtB,CAAC;YACF,IAAI,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAEnE,SAAS;YACT,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACtC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBACpB,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;YAChD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,KAAK;oBAAE,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC;YACxE,CAAC;YAED,mBAAmB;YACnB,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC1C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC/C,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,IAAI,SAAS,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,IAAI,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC;YAErE,gBAAgB;YAChB,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE;gBAC3E,WAAW,EAAE,SAAS;gBACtB,gBAAgB,EAAE,aAAa;aAChC,CAAC,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC5C,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YAEzC,0BAA0B;YAC1B,IAAI,MAAM,CAAC,eAAe,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAChD,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACtD,MAAM,sBAAsB,GAAG,SAAS,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CACrD,CAAC;gBACF,OAAO,CAAC,GAAG,CAAC,0BAA0B,OAAO,IAAI,WAAW,CAAC,MAAM,WAAW,YAAY,iBAAiB,SAAS,CAAC,MAAM,oBAAoB,sBAAsB,WAAW,SAAS,EAAE,CAAC,CAAC;YAC/L,CAAC;YAED,kBAAkB;YAClB,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;gBACtD,IAAI,UAAU,GAAG,MAAM,GAAG,SAAS;oBAAE,MAAM;gBAC3C,UAAU,IAAI,MAAM,CAAC;gBACrB,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;oBAC9B,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC;oBAC7B,MAAM;gBACR,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3D,MAAM,CAAC,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAE/C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE;oBACvC,SAAS,EAAE,UAAU;oBACrB,WAAW,EAAE,SAAS;oBACtB,cAAc,EAAE,SAAS;iBAC1B,CAAC,CAAC;gBACH,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,SAAS,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACvD,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC;gBAC5B,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;YAEnC,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,UAAU,GAAoC;gBAClD,CAAC,cAAc,EAAE,eAAe,CAAC;gBACjC,CAAC,WAAW,EAAE,YAAY,CAAC;gBAC3B,CAAC,kBAAkB,EAAE,mBAAmB,CAAC;gBACzC,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;gBACvC,CAAC,WAAW,EAAE,YAAY,CAAC;gBAC3B,CAAC,gBAAgB,EAAE,iBAAiB,CAAC;aACtC,CAAC;YAEF,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBACjB,SAAS,GAAG,IAAI,CAAC;oBACjB,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,kBAAkB,EAAE,QAAQ;gBAC5B,MAAM;gBACN,SAAS;gBACT,UAAU;gBACV,OAAO;aACR,CAAC,CAAC;QACL,CAAC;QAED,2BAA2B;QAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,MAAM,WAAW,GAA2B,EAAE,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBACtD,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAuB;YACtC,CAAC,cAAc,EAAE,QAAQ,CAAC;YAC1B,CAAC,WAAW,EAAE,KAAK,CAAC;YACpB,CAAC,kBAAkB,EAAE,SAAS,CAAC;YAC/B,CAAC,iBAAiB,EAAE,MAAM,CAAC;YAC3B,CAAC,WAAW,EAAE,KAAK,CAAC;YACpB,CAAC,gBAAgB,EAAE,QAAQ,CAAC;YAC5B,CAAC,YAAY,EAAE,OAAO,CAAC;YACvB,CAAC,aAAa,EAAE,OAAO,CAAC;SACzB,CAAC;QAEF,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;YAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpC,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;QAC5D,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAEzB,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACzB,CAAC;AAED,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IAC1B,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Miss Population Analysis
+ *
+ * For chunks that are never found by vector search (even at K=2000),
+ * diagnoses why by comparing hit vs miss populations across:
+ *
+ *   1. Embedding distance: query → chunk embedding, query → best index entry embedding
+ *   2. Index entry count per chunk
+ *   3. Chunk content length (tokens)
+ *   4. Cluster size (competition from similar chunks)
+ *   5. Content samples from hits vs misses
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/rescorer-ceiling/analyze-misses.ts [--sample-size=50]
+ */
+export {};
+//# sourceMappingURL=analyze-misses.d.ts.map

package/dist/eval/experiments/rescorer-ceiling/analyze-misses.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze-misses.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/rescorer-ceiling/analyze-misses.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}

package/dist/eval/experiments/rescorer-ceiling/analyze-misses.js

@@ -0,0 +1,247 @@
+/**
+ * Miss Population Analysis
+ *
+ * For chunks that are never found by vector search (even at K=2000),
+ * diagnoses why by comparing hit vs miss populations across:
+ *
+ *   1. Embedding distance: query → chunk embedding, query → best index entry embedding
+ *   2. Index entry count per chunk
+ *   3. Chunk content length (tokens)
+ *   4. Cluster size (competition from similar chunks)
+ *   5. Content samples from hits vs misses
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/rescorer-ceiling/analyze-misses.ts [--sample-size=50]
+ */
+import { getDb } from '../../../storage/db.js';
+import { vectorStore, indexVectorStore } from '../../../storage/vector-store.js';
+import { getChunkById } from '../../../storage/chunk-store.js';
+import { getIndexEntryCount, getIndexedChunkCount, getIndexEntriesForChunk, dereferenceToChunkIds, } from '../../../storage/index-entry-store.js';
+import { getAllClusters, getClusterChunkIds } from '../../../storage/cluster-store.js';
+import { Embedder } from '../../../models/embedder.js';
+import { getModel } from '../../../models/model-registry.js';
+import { loadConfig, toRuntimeConfig } from '../../../config/loader.js';
+import { cosineSimilarity } from '../../../utils/angular-distance.js';
+import { generateSearchQueries } from '../index-vs-chunk/query-generator.js';
+// ── Helpers ────────────────────────────────────────────────────────────────
+function createRng(seed) {
+    let s = seed;
+    return () => {
+        s = (s * 1664525 + 1013904223) & 0x7fffffff;
+        return s / 0x7fffffff;
+    };
+}
+function sampleChunks(sampleSize, seed) {
+    getDb();
+    const clusters = getAllClusters();
+    if (clusters.length === 0)
+        throw new Error('No clusters found.');
+    const rng = createRng(seed);
+    const result = [];
+    const shuffled = [...clusters].sort(() => rng() - 0.5);
+    for (const cluster of shuffled) {
+        if (result.length >= sampleSize)
+            break;
+        const chunkIds = getClusterChunkIds(cluster.id);
+        if (chunkIds.length < 2)
+            continue;
+        const numPicks = Math.min(2, Math.ceil(sampleSize / clusters.length), chunkIds.length);
+        const shuffledIds = [...chunkIds].sort(() => rng() - 0.5);
+        for (let i = 0; i < numPicks && result.length < sampleSize; i++) {
+            const chunk = getChunkById(shuffledIds[i]);
+            if (!chunk || chunk.content.length < 100)
+                continue;
+            result.push({
+                id: chunk.id,
+                sessionSlug: chunk.sessionSlug,
+                content: chunk.content,
+                clusterId: cluster.id,
+                clusterName: cluster.name,
+            });
+        }
+    }
+    return result;
+}
+function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    return sorted[Math.floor(sorted.length / 2)];
+}
+function mean(values) {
+    if (values.length === 0)
+        return 0;
+    return values.reduce((s, v) => s + v, 0) / values.length;
+}
+// ── Main ───────────────────────────────────────────────────────────────────
+async function runAnalysis() {
+    const args = process.argv.slice(2);
+    const sampleSizeArg = args.find((a) => a.startsWith('--sample-size='));
+    const sampleSize = sampleSizeArg ? parseInt(sampleSizeArg.split('=')[1], 10) : 50;
+    const seed = 42;
+    const K = 2000; // high K for determining hit/miss
+    console.log('=== Miss Population Analysis ===\n');
+    getDb();
+    const externalConfig = loadConfig();
+    const config = toRuntimeConfig(externalConfig);
+    const useIndex = config.semanticIndex.useForSearch && getIndexEntryCount() > 0;
+    let entriesPerChunk = 1;
+    if (useIndex) {
+        const indexedChunks = getIndexedChunkCount();
+        entriesPerChunk = indexedChunks > 0 ? getIndexEntryCount() / indexedChunks : 1;
+    }
+    console.log(`Search path: ${useIndex ? 'INDEX' : 'CHUNK'}`);
+    console.log(`K: ${K}`);
+    vectorStore.setModelId(config.embeddingModel);
+    if (useIndex)
+        indexVectorStore.setModelId(config.embeddingModel);
+    // Build cluster size map
+    const clusterSizeMap = new Map();
+    for (const cluster of getAllClusters()) {
+        const chunkIds = getClusterChunkIds(cluster.id);
+        for (const cid of chunkIds) {
+            clusterSizeMap.set(cid, chunkIds.length);
+        }
+    }
+    // 1. Sample and generate queries
+    console.log(`\nSampling ${sampleSize} chunks...`);
+    const sampledChunks = sampleChunks(sampleSize, seed);
+    console.log(`  Sampled ${sampledChunks.length} chunks`);
+    console.log('Generating queries...');
+    const queries = await generateSearchQueries(sampledChunks, config.clusterRefreshModel);
+    console.log(`  Generated ${queries.length} queries`);
+    // 2. Prepare embedder
+    const embedder = new Embedder();
+    await embedder.load(getModel(config.embeddingModel));
+    console.log('Embedding queries...');
+    const queryEmbeddings = [];
+    for (const q of queries) {
+        const { embedding } = await embedder.embed(q.query, true);
+        queryEmbeddings.push(embedding);
+    }
+    console.log(`  Embedded ${queryEmbeddings.length} queries\n`);
+    // 3. Profile each query-chunk pair
+    const profiles = [];
+    for (let qi = 0; qi < queries.length; qi++) {
+        const q = queries[qi];
+        const queryEmb = queryEmbeddings[qi];
+        const targetId = q.groundTruthChunkId;
+        // Determine hit/miss at K=2000
+        let found = false;
+        if (useIndex) {
+            const indexLimit = Math.ceil(K * entriesPerChunk);
+            const results = await indexVectorStore.search(queryEmb, indexLimit);
+            for (const r of results) {
+                const chunkIds = dereferenceToChunkIds([r.id]);
+                if (chunkIds.includes(targetId)) {
+                    found = true;
+                    break;
+                }
+            }
+        }
+        else {
+            const results = await vectorStore.search(queryEmb, K);
+            found = results.some((r) => r.id === targetId);
+        }
+        // Get chunk's own embedding
+        const chunkEmb = await vectorStore.get(targetId);
+        const queryToChunkSim = chunkEmb ? cosineSimilarity(queryEmb, chunkEmb) : -1;
+        // Get index entry embeddings and find best match
+        const entries = getIndexEntriesForChunk(targetId);
+        let queryToBestEntrySim = -1;
+        let bestEntryText = '(no entries)';
+        for (const entry of entries) {
+            const entryEmb = await indexVectorStore.get(entry.id);
+            if (entryEmb) {
+                const sim = cosineSimilarity(queryEmb, entryEmb);
+                if (sim > queryToBestEntrySim) {
+                    queryToBestEntrySim = sim;
+                    bestEntryText = entry.description;
+                }
+            }
+        }
+        const chunk = getChunkById(targetId);
+        profiles.push({
+            chunkId: targetId,
+            query: q.query,
+            found,
+            queryToChunkSim,
+            queryToBestEntrySim,
+            indexEntryCount: entries.length,
+            chunkTokens: chunk?.approxTokens ?? 0,
+            clusterSize: clusterSizeMap.get(targetId) ?? 0,
+            contentPreview: (chunk?.content ?? '').slice(0, 120).replace(/\n/g, ' '),
+            bestEntryText,
+        });
+    }
+    // 4. Split into hits vs misses and compare
+    const hits = profiles.filter((p) => p.found);
+    const misses = profiles.filter((p) => !p.found);
+    console.log(`══ Population Summary ══\n`);
+    console.log(`  Hits:   ${hits.length}/${profiles.length}`);
+    console.log(`  Misses: ${misses.length}/${profiles.length}\n`);
+    console.log(`══ Embedding Distance Comparison ══\n`);
+    console.log('  Query → Chunk embedding (cosine similarity):');
+    console.log(`    Hits:   mean=${mean(hits.map((p) => p.queryToChunkSim)).toFixed(3)}  median=${median(hits.map((p) => p.queryToChunkSim)).toFixed(3)}`);
+    console.log(`    Misses: mean=${mean(misses.map((p) => p.queryToChunkSim)).toFixed(3)}  median=${median(misses.map((p) => p.queryToChunkSim)).toFixed(3)}`);
+    console.log('\n  Query → Best index entry embedding (cosine similarity):');
+    console.log(`    Hits:   mean=${mean(hits.map((p) => p.queryToBestEntrySim)).toFixed(3)}  median=${median(hits.map((p) => p.queryToBestEntrySim)).toFixed(3)}`);
+    console.log(`    Misses: mean=${mean(misses.map((p) => p.queryToBestEntrySim)).toFixed(3)}  median=${median(misses.map((p) => p.queryToBestEntrySim)).toFixed(3)}`);
+    console.log(`\n══ Index Entry Count ══\n`);
+    console.log(`    Hits:   mean=${mean(hits.map((p) => p.indexEntryCount)).toFixed(1)}  median=${median(hits.map((p) => p.indexEntryCount))}`);
+    console.log(`    Misses: mean=${mean(misses.map((p) => p.indexEntryCount)).toFixed(1)}  median=${median(misses.map((p) => p.indexEntryCount))}`);
+    console.log(`\n══ Chunk Size (tokens) ══\n`);
+    console.log(`    Hits:   mean=${mean(hits.map((p) => p.chunkTokens)).toFixed(0)}  median=${median(hits.map((p) => p.chunkTokens))}`);
+    console.log(`    Misses: mean=${mean(misses.map((p) => p.chunkTokens)).toFixed(0)}  median=${median(misses.map((p) => p.chunkTokens))}`);
+    console.log(`\n══ Cluster Size ══\n`);
+    console.log(`    Hits:   mean=${mean(hits.map((p) => p.clusterSize)).toFixed(1)}  median=${median(hits.map((p) => p.clusterSize))}`);
+    console.log(`    Misses: mean=${mean(misses.map((p) => p.clusterSize)).toFixed(1)}  median=${median(misses.map((p) => p.clusterSize))}`);
+    // 5. Similarity distribution buckets
+    console.log(`\n══ Best Entry Similarity Distribution ══\n`);
+    const simBuckets = [
+        { label: '0.9-1.0', min: 0.9, max: 1.0 },
+        { label: '0.8-0.9', min: 0.8, max: 0.9 },
+        { label: '0.7-0.8', min: 0.7, max: 0.8 },
+        { label: '0.6-0.7', min: 0.6, max: 0.7 },
+        { label: '0.5-0.6', min: 0.5, max: 0.6 },
+        { label: '0.4-0.5', min: 0.4, max: 0.5 },
+        { label: '<0.4', min: -1, max: 0.4 },
+    ];
+    console.log('  Bucket      Hits  Misses');
+    console.log('  ─────────────────────────');
+    for (const { label, min, max } of simBuckets) {
+        const hitCount = hits.filter((p) => p.queryToBestEntrySim >= min && p.queryToBestEntrySim < max).length;
+        const missCount = misses.filter((p) => p.queryToBestEntrySim >= min && p.queryToBestEntrySim < max).length;
+        console.log(`  ${label.padStart(7)}    ${String(hitCount).padStart(4)}  ${String(missCount).padStart(6)}`);
+    }
+    // 6. Sample misses for qualitative review
+    console.log(`\n══ Sample Misses (worst by entry similarity) ══\n`);
+    const sortedMisses = [...misses].sort((a, b) => a.queryToBestEntrySim - b.queryToBestEntrySim);
+    const samplesToShow = Math.min(10, sortedMisses.length);
+    for (let i = 0; i < samplesToShow; i++) {
+        const p = sortedMisses[i];
+        console.log(`  [${i + 1}] chunk=${p.chunkId.slice(0, 12)}… sim_chunk=${p.queryToChunkSim.toFixed(3)} sim_entry=${p.queryToBestEntrySim.toFixed(3)} entries=${p.indexEntryCount}`);
+        console.log(`      Query: ${p.query}`);
+        console.log(`      Entry: ${p.bestEntryText}`);
+        console.log(`      Chunk: ${p.contentPreview}…`);
+        console.log();
+    }
+    // 7. Sample hits for comparison
+    console.log(`══ Sample Hits (best by entry similarity) ══\n`);
+    const sortedHits = [...hits].sort((a, b) => b.queryToBestEntrySim - a.queryToBestEntrySim);
+    const hitSamplesToShow = Math.min(5, sortedHits.length);
+    for (let i = 0; i < hitSamplesToShow; i++) {
+        const p = sortedHits[i];
+        console.log(`  [${i + 1}] chunk=${p.chunkId.slice(0, 12)}… sim_chunk=${p.queryToChunkSim.toFixed(3)} sim_entry=${p.queryToBestEntrySim.toFixed(3)} entries=${p.indexEntryCount}`);
+        console.log(`      Query: ${p.query}`);
+        console.log(`      Entry: ${p.bestEntryText}`);
+        console.log();
+    }
+    await embedder.dispose();
+    console.log('Done.');
+}
+runAnalysis().catch((err) => {
+    console.error('Analysis failed:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=analyze-misses.js.map