causantic 0.9.3 → 0.10.0

package/dist/eval/experiments/index-vs-chunk/query-generator.js

@@ -0,0 +1,113 @@
+/**
+ * Generate natural language search queries from chunks using LLM.
+ *
+ * Each query is something a user might search for that should find
+ * the source chunk. The LLM sees a truncated chunk and generates
+ * a plausible search query.
+ */
+import Anthropic from '@anthropic-ai/sdk';
+import { createSecretStore } from '../../../utils/secret-store.js';
+/**
+ * Generate search queries for a batch of chunks.
+ *
+ * Batches up to 10 chunks per API call to keep cost low.
+ */
+export async function generateSearchQueries(chunks, model) {
+    const client = await getClient();
+    if (!client) {
+        throw new Error('No Anthropic API key available');
+    }
+    const results = [];
+    const batchSize = 10;
+    for (let i = 0; i < chunks.length; i += batchSize) {
+        const batch = chunks.slice(i, i + batchSize);
+        const queries = await generateBatch(client, batch, model);
+        results.push(...queries);
+    }
+    return results;
+}
+async function generateBatch(client, chunks, model) {
+    const maxContentChars = 500 * 4; // ~500 tokens per chunk
+    const chunkTexts = chunks
+        .map((c, i) => {
+        const content = c.content.length > maxContentChars
+            ? c.content.slice(0, maxContentChars) + '\n...[truncated]'
+            : c.content;
+        return `--- Chunk ${i} ---\n${content}`;
+    })
+        .join('\n\n');
+    const prompt = `You are generating natural language search queries for a retrieval benchmark. For each conversation chunk below, write a short search query (5-15 words) that a user would type to find this specific chunk.
+
+Requirements:
+- The query should be something a real user would search for
+- It should target the SPECIFIC content of this chunk, not the general topic
+- Use natural language, not keywords
+- Do NOT quote or copy text directly from the chunk
+- Focus on the key decision, action, or outcome in the chunk
+
+${chunkTexts}
+
+Respond with exactly ${chunks.length} queries, one per line, prefixed with the chunk number:
+0: [query]
+1: [query]
+...`;
+    try {
+        const response = await client.messages.create({
+            model,
+            max_tokens: Math.min(2048, chunks.length * 100),
+            messages: [{ role: 'user', content: prompt }],
+        });
+        const text = response.content[0].type === 'text' ? response.content[0].text : '';
+        const queries = parseResponse(text, chunks.length);
+        return chunks
+            .map((chunk, i) => {
+            const query = queries[i];
+            if (!query)
+                return null;
+            return {
+                query,
+                groundTruthChunkId: chunk.id,
+                sessionSlug: chunk.sessionSlug,
+                clusterId: chunk.clusterId,
+                clusterName: chunk.clusterName,
+            };
+        })
+            .filter((q) => q !== null);
+    }
+    catch (error) {
+        console.warn(`  Query generation batch failed: ${error.message}`);
+        return [];
+    }
+}
+function parseResponse(text, expectedCount) {
+    const queries = new Array(expectedCount).fill(null);
+    const lines = text.split('\n');
+    for (const line of lines) {
+        const match = line.match(/^(\d+):\s*(.+)/);
+        if (match) {
+            const index = parseInt(match[1], 10);
+            if (index >= 0 && index < expectedCount) {
+                queries[index] = match[2].trim();
+            }
+        }
+    }
+    return queries;
+}
+async function getClient() {
+    if (!process.env.ANTHROPIC_API_KEY) {
+        try {
+            const store = createSecretStore();
+            const storedKey = await store.get('anthropic-api-key');
+            if (storedKey) {
+                process.env.ANTHROPIC_API_KEY = storedKey;
+            }
+        }
+        catch {
+            // Keychain not available
+        }
+    }
+    if (!process.env.ANTHROPIC_API_KEY)
+        return null;
+    return new Anthropic();
+}
+//# sourceMappingURL=query-generator.js.map

package/dist/eval/experiments/index-vs-chunk/query-generator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-generator.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/query-generator.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAYnE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAA0B,EAC1B,KAAa;IAEb,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC;IACjC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,EAAE,CAAC;IAErB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAiB,EACjB,MAA0B,EAC1B,KAAa;IAEb,MAAM,eAAe,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,wBAAwB;IAEzD,MAAM,UAAU,GAAG,MAAM;SACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACZ,MAAM,OAAO,GACX,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,eAAe;YAChC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,eAAe,CAAC,GAAG,kBAAkB;YAC1D,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAChB,OAAO,aAAa,CAAC,SAAS,OAAO,EAAE,CAAC;IAC1C,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG;;;;;;;;;EASf,UAAU;;uBAEW,MAAM,CAAC,MAAM;;;IAGhC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC;YAC/C,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;SAC9C,CAAC,CAAC;QAEH,MAAM,IAAI,GACR,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAEnD,OAAO,MAAM;aACV,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE;YAChB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,CAAC,KAAK;gBAAE,OAAO,IAAI,CAAC;YACxB,OAAO;gBACL,KAAK;gBACL,kBAAkB,EAAE,KAAK,CAAC,EAAE;gBAC5B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B,CAAC;QACJ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAuB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CACV,oCAAqC,KAAe,CAAC,OAAO,EAAE,CAC/D,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,aAAqB;IAErB,MAAM,OAAO,GAAsB,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAE/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,iBAAiB,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACvD,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,SAAS,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,IAAI,SAAS,EAAE,CAAC;AACzB,CAAC"}

package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Index vs Chunk Retrieval Benchmark
+ *
+ * Compares search quality when using the semantic index layer vs
+ * direct chunk search, using LLM-generated natural language queries.
+ *
+ * Steps:
+ *   1. Sample chunks across diverse clusters
+ *   2. Generate natural language search queries via LLM
+ *   3. Run each query through both search paths (index + chunk)
+ *   4. Compare recall@K, MRR, latency
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/index-vs-chunk/run-experiment.ts [--sample-size N]
+ */
+export {};
+//# sourceMappingURL=run-experiment.d.ts.map

package/dist/eval/experiments/index-vs-chunk/run-experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-experiment.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/run-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG"}

package/dist/eval/experiments/index-vs-chunk/run-experiment.js

@@ -0,0 +1,341 @@
+/**
+ * Index vs Chunk Retrieval Benchmark
+ *
+ * Compares search quality when using the semantic index layer vs
+ * direct chunk search, using LLM-generated natural language queries.
+ *
+ * Steps:
+ *   1. Sample chunks across diverse clusters
+ *   2. Generate natural language search queries via LLM
+ *   3. Run each query through both search paths (index + chunk)
+ *   4. Compare recall@K, MRR, latency
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/index-vs-chunk/run-experiment.ts [--sample-size N]
+ */
+import { writeFileSync } from 'fs';
+import { getDb } from '../../../storage/db.js';
+import { vectorStore, indexVectorStore } from '../../../storage/vector-store.js';
+import { getChunkById } from '../../../storage/chunk-store.js';
+import { getIndexEntryCount, getIndexedChunkCount, dereferenceToChunkIds, searchIndexEntriesByKeyword, } from '../../../storage/index-entry-store.js';
+import { getAllClusters, getClusterChunkIds } from '../../../storage/cluster-store.js';
+import { Embedder } from '../../../models/embedder.js';
+import { getModel } from '../../../models/model-registry.js';
+import { loadConfig, toRuntimeConfig } from '../../../config/loader.js';
+import { KeywordStore } from '../../../storage/keyword-store.js';
+import { fuseRRF } from '../../../retrieval/rrf.js';
+import { generateSearchQueries } from './query-generator.js';
+/** Seeded PRNG for reproducible sampling. */
+function createRng(seed) {
+    let s = seed;
+    return () => {
+        s = (s * 1664525 + 1013904223) & 0x7fffffff;
+        return s / 0x7fffffff;
+    };
+}
+/**
+ * Sample chunks across diverse clusters.
+ * Takes 1-2 chunks per cluster to ensure diversity.
+ */
+function sampleChunks(sampleSize, seed) {
+    getDb(); // ensure init
+    const clusters = getAllClusters();
+    if (clusters.length === 0) {
+        throw new Error('No clusters found. Run clustering first.');
+    }
+    const rng = createRng(seed);
+    const result = [];
+    // Shuffle clusters
+    const shuffled = [...clusters].sort(() => rng() - 0.5);
+    for (const cluster of shuffled) {
+        if (result.length >= sampleSize)
+            break;
+        const chunkIds = getClusterChunkIds(cluster.id);
+        if (chunkIds.length < 2)
+            continue;
+        // Pick 1-2 random chunks from this cluster
+        const numPicks = Math.min(2, Math.ceil(sampleSize / clusters.length), chunkIds.length);
+        const shuffledIds = [...chunkIds].sort(() => rng() - 0.5);
+        for (let i = 0; i < numPicks && result.length < sampleSize; i++) {
+            const chunk = getChunkById(shuffledIds[i]);
+            if (!chunk || chunk.content.length < 100)
+                continue;
+            result.push({
+                id: chunk.id,
+                sessionSlug: chunk.sessionSlug,
+                content: chunk.content,
+                clusterId: cluster.id,
+                clusterName: cluster.name,
+            });
+        }
+    }
+    return result;
+}
+/**
+ * Search using the index-based path (index entry embeddings + FTS).
+ * Returns ranked chunk IDs.
+ */
+async function searchViaIndex(queryEmbedding, queryText, vectorSearchLimit, hybridSearch) {
+    const runtimeConfig = toRuntimeConfig(loadConfig());
+    indexVectorStore.setModelId(runtimeConfig.embeddingModel);
+    // Scale search limit by entries-per-chunk ratio so the index path
+    // covers roughly the same number of unique chunks as the chunk path.
+    const entryCount = getIndexEntryCount();
+    const indexedChunks = getIndexedChunkCount();
+    const entriesPerChunk = indexedChunks > 0 ? entryCount / indexedChunks : 1;
+    const indexSearchLimit = Math.ceil(vectorSearchLimit * entriesPerChunk);
+    const indexSimilar = await indexVectorStore.search(queryEmbedding, indexSearchLimit);
+    let indexKeywordResults = [];
+    try {
+        indexKeywordResults = searchIndexEntriesByKeyword(queryText, hybridSearch.keywordSearchLimit);
+    }
+    catch {
+        // FTS unavailable
+    }
+    if (indexSimilar.length === 0 && indexKeywordResults.length === 0)
+        return [];
+    const indexVectorItems = indexSimilar.map((s) => ({
+        chunkId: s.id,
+        score: Math.max(0, 1 - s.distance),
+        source: 'vector',
+    }));
+    const indexKeywordItems = indexKeywordResults.map((r) => ({
+        chunkId: r.id,
+        score: r.score,
+        source: 'keyword',
+    }));
+    const indexFused = fuseRRF([
+        { items: indexVectorItems, weight: hybridSearch.vectorWeight },
+        ...(indexKeywordItems.length > 0
+            ? [{ items: indexKeywordItems, weight: hybridSearch.keywordWeight }]
+            : []),
+    ], hybridSearch.rrfK);
+    // Dereference to chunk IDs
+    const allChunkIds = [];
+    for (const item of indexFused) {
+        const chunkIds = dereferenceToChunkIds([item.chunkId]);
+        for (const cid of chunkIds) {
+            if (!allChunkIds.includes(cid)) {
+                allChunkIds.push(cid);
+            }
+        }
+    }
+    return allChunkIds;
+}
+/**
+ * Search using the chunk-based path (chunk embeddings + FTS).
+ * Returns ranked chunk IDs.
+ */
+async function searchViaChunks(queryEmbedding, queryText, vectorSearchLimit, hybridSearch) {
+    const runtimeConfig = toRuntimeConfig(loadConfig());
+    vectorStore.setModelId(runtimeConfig.embeddingModel);
+    const similar = await vectorStore.search(queryEmbedding, vectorSearchLimit);
+    let keywordResults = [];
+    try {
+        const keywordStore = new KeywordStore();
+        keywordResults = keywordStore.search(queryText, hybridSearch.keywordSearchLimit);
+    }
+    catch {
+        // FTS unavailable
+    }
+    if (similar.length === 0 && keywordResults.length === 0)
+        return [];
+    const vectorItems = similar.map((s) => ({
+        chunkId: s.id,
+        score: Math.max(0, 1 - s.distance),
+        source: 'vector',
+    }));
+    const keywordItems = keywordResults.map((r) => ({
+        chunkId: r.id,
+        score: r.score,
+        source: 'keyword',
+    }));
+    const fused = fuseRRF([
+        { items: vectorItems, weight: hybridSearch.vectorWeight },
+        ...(keywordItems.length > 0
+            ? [{ items: keywordItems, weight: hybridSearch.keywordWeight }]
+            : []),
+    ], hybridSearch.rrfK);
+    return fused.map((r) => r.chunkId);
+}
+/**
+ * Compute metrics from per-query results.
+ */
+function computeMetrics(results, path) {
+    const ranks = results.map((r) => r[path].rank);
+    const durations = results.map((r) => r[path].durationMs);
+    const recallAtK = (k) => ranks.filter((r) => r > 0 && r <= k).length / ranks.length;
+    const mrr = ranks
+        .filter((r) => r > 0)
+        .reduce((sum, r) => sum + 1 / r, 0) / ranks.length;
+    const hitRate = ranks.filter((r) => r > 0).length / ranks.length;
+    const sortedDurations = [...durations].sort((a, b) => a - b);
+    const medianLatency = sortedDurations.length > 0
+        ? sortedDurations[Math.floor(sortedDurations.length / 2)]
+        : 0;
+    const meanLatency = durations.length > 0
+        ? durations.reduce((a, b) => a + b, 0) / durations.length
+        : 0;
+    return {
+        recallAt5: recallAtK(5),
+        recallAt10: recallAtK(10),
+        recallAt20: recallAtK(20),
+        mrr,
+        hitRate,
+        meanLatencyMs: meanLatency,
+        medianLatencyMs: medianLatency,
+    };
+}
+function fmt(n, d = 3) {
+    return n.toFixed(d);
+}
+/**
+ * Run the full A/B benchmark.
+ */
+async function runBenchmark() {
+    const args = process.argv.slice(2);
+    const sampleSizeArg = args.find((a) => a.startsWith('--sample-size='));
+    const sampleSize = sampleSizeArg
+        ? parseInt(sampleSizeArg.split('=')[1], 10)
+        : 100;
+    const seed = 42;
+    console.log('=== Index vs Chunk Retrieval Benchmark ===\n');
+    // Check prerequisites
+    const _db = getDb();
+    const entryCount = getIndexEntryCount();
+    console.log(`Index entries: ${entryCount}`);
+    if (entryCount === 0) {
+        console.log('No index entries. Run backfill first.');
+        process.exit(1);
+    }
+    const externalConfig = loadConfig();
+    const config = toRuntimeConfig(externalConfig);
+    // 1. Sample chunks
+    console.log(`\nSampling ${sampleSize} chunks across clusters...`);
+    const sampledChunks = sampleChunks(sampleSize, seed);
+    console.log(`  Sampled ${sampledChunks.length} chunks from ${new Set(sampledChunks.map((c) => c.clusterId)).size} clusters`);
+    // 2. Generate search queries
+    console.log('\nGenerating natural language search queries via LLM...');
+    const queries = await generateSearchQueries(sampledChunks, config.clusterRefreshModel);
+    console.log(`  Generated ${queries.length} queries (${sampledChunks.length - queries.length} failed)`);
+    if (queries.length === 0) {
+        console.log('No queries generated. Check API key.');
+        process.exit(1);
+    }
+    // 3. Prepare embedder
+    const embedder = new Embedder();
+    await embedder.load(getModel(config.embeddingModel));
+    const hybridSearch = config.hybridSearch;
+    const vectorSearchLimit = 20;
+    // 4. Run queries through both paths
+    console.log(`\nRunning ${queries.length} queries through both search paths...`);
+    const perQuery = [];
+    for (let i = 0; i < queries.length; i++) {
+        const q = queries[i];
+        if ((i + 1) % 10 === 0 || i === queries.length - 1) {
+            console.log(`  Query ${i + 1}/${queries.length}`);
+        }
+        // Embed query
+        const embedResult = await embedder.embed(q.query, true);
+        const queryEmbedding = embedResult.embedding;
+        // Index path
+        const indexStart = Date.now();
+        const indexResults = await searchViaIndex(queryEmbedding, q.query, vectorSearchLimit, hybridSearch);
+        const indexDuration = Date.now() - indexStart;
+        // Chunk path
+        const chunkStart = Date.now();
+        const chunkResults = await searchViaChunks(queryEmbedding, q.query, vectorSearchLimit, hybridSearch);
+        const chunkDuration = Date.now() - chunkStart;
+        // Find rank of ground truth
+        const indexRank = indexResults.indexOf(q.groundTruthChunkId) + 1; // 0 = not found
+        const chunkRank = chunkResults.indexOf(q.groundTruthChunkId) + 1;
+        perQuery.push({
+            query: q.query,
+            groundTruthChunkId: q.groundTruthChunkId,
+            index: {
+                rank: indexRank,
+                totalReturned: indexResults.length,
+                durationMs: indexDuration,
+            },
+            chunk: {
+                rank: chunkRank,
+                totalReturned: chunkResults.length,
+                durationMs: chunkDuration,
+            },
+        });
+    }
+    await embedder.dispose();
+    // 5. Compute metrics
+    const indexMetrics = computeMetrics(perQuery, 'index');
+    const chunkMetrics = computeMetrics(perQuery, 'chunk');
+    // 6. Display results
+    console.log('\n══ Results ══\n');
+    console.log('  Metric          Index Path    Chunk Path    Delta');
+    console.log('  ' + '─'.repeat(55));
+    console.log(`  Recall@5        ${fmt(indexMetrics.recallAt5 * 100, 1)}%        ${fmt(chunkMetrics.recallAt5 * 100, 1)}%        ${fmt((indexMetrics.recallAt5 - chunkMetrics.recallAt5) * 100, 1)}%`);
+    console.log(`  Recall@10       ${fmt(indexMetrics.recallAt10 * 100, 1)}%        ${fmt(chunkMetrics.recallAt10 * 100, 1)}%        ${fmt((indexMetrics.recallAt10 - chunkMetrics.recallAt10) * 100, 1)}%`);
+    console.log(`  Recall@20       ${fmt(indexMetrics.recallAt20 * 100, 1)}%        ${fmt(chunkMetrics.recallAt20 * 100, 1)}%        ${fmt((indexMetrics.recallAt20 - chunkMetrics.recallAt20) * 100, 1)}%`);
+    console.log(`  MRR             ${fmt(indexMetrics.mrr)}          ${fmt(chunkMetrics.mrr)}          ${fmt(indexMetrics.mrr - chunkMetrics.mrr)}`);
+    console.log(`  Hit Rate        ${fmt(indexMetrics.hitRate * 100, 1)}%        ${fmt(chunkMetrics.hitRate * 100, 1)}%        ${fmt((indexMetrics.hitRate - chunkMetrics.hitRate) * 100, 1)}%`);
+    console.log(`  Mean Latency    ${fmt(indexMetrics.meanLatencyMs, 0)}ms         ${fmt(chunkMetrics.meanLatencyMs, 0)}ms         ${fmt(indexMetrics.meanLatencyMs - chunkMetrics.meanLatencyMs, 0)}ms`);
+    console.log(`  Median Latency  ${fmt(indexMetrics.medianLatencyMs, 0)}ms         ${fmt(chunkMetrics.medianLatencyMs, 0)}ms         ${fmt(indexMetrics.medianLatencyMs - chunkMetrics.medianLatencyMs, 0)}ms`);
+    // Show examples where paths disagree
+    const indexWins = perQuery.filter((q) => q.index.rank > 0 &&
+        (q.chunk.rank === 0 || q.index.rank < q.chunk.rank));
+    const chunkWins = perQuery.filter((q) => q.chunk.rank > 0 &&
+        (q.index.rank === 0 || q.chunk.rank < q.index.rank));
+    const ties = perQuery.filter((q) => q.index.rank > 0 && q.index.rank === q.chunk.rank);
+    console.log(`\n  Path comparison: Index wins ${indexWins.length}, Chunk wins ${chunkWins.length}, Ties ${ties.length}, Both miss ${perQuery.length - indexWins.length - chunkWins.length - ties.length}`);
+    if (indexWins.length > 0) {
+        console.log('\n  Sample queries where INDEX path wins:');
+        for (const q of indexWins.slice(0, 3)) {
+            console.log(`    "${q.query}" → index rank ${q.index.rank}, chunk rank ${q.chunk.rank || 'miss'}`);
+        }
+    }
+    if (chunkWins.length > 0) {
+        console.log('\n  Sample queries where CHUNK path wins:');
+        for (const q of chunkWins.slice(0, 3)) {
+            console.log(`    "${q.query}" → chunk rank ${q.chunk.rank}, index rank ${q.index.rank || 'miss'}`);
+        }
+    }
+    // 7. Summary
+    const summary = [];
+    summary.push(`Benchmark: ${queries.length} natural language queries across ${new Set(queries.map((q) => q.clusterId)).size} clusters`);
+    const recallDelta5 = indexMetrics.recallAt5 - chunkMetrics.recallAt5;
+    const mrrDelta = indexMetrics.mrr - chunkMetrics.mrr;
+    if (recallDelta5 > 0.05) {
+        summary.push(`Index path BETTER: +${fmt(recallDelta5 * 100, 1)}% recall@5, +${fmt(mrrDelta, 3)} MRR`);
+    }
+    else if (recallDelta5 < -0.05) {
+        summary.push(`Chunk path BETTER: ${fmt(recallDelta5 * 100, 1)}% recall@5, ${fmt(mrrDelta, 3)} MRR`);
+    }
+    else {
+        summary.push(`Paths comparable: ${fmt(recallDelta5 * 100, 1)}% recall@5 delta, ${fmt(mrrDelta, 3)} MRR delta`);
+    }
+    summary.push(`Index path wins ${indexWins.length}/${perQuery.length} queries, chunk path wins ${chunkWins.length}/${perQuery.length}`);
+    console.log('\n══ Summary ══\n');
+    for (const line of summary) {
+        console.log(`  • ${line}`);
+    }
+    return {
+        timestamp: new Date().toISOString(),
+        queryCount: queries.length,
+        failedQueryCount: sampledChunks.length - queries.length,
+        indexMetrics,
+        chunkMetrics,
+        perQuery,
+        summary,
+    };
+}
+// ── CLI entrypoint ──────────────────────────────────────────────────────────
+runBenchmark()
+    .then((report) => {
+    const outPath = 'index-vs-chunk-report.json';
+    writeFileSync(outPath, JSON.stringify(report, null, 2));
+    console.log(`\nReport written to ${outPath}`);
+})
+    .catch((err) => {
+    console.error('Benchmark failed:', err);
+    process.exit(1);
+});
+//# sourceMappingURL=run-experiment.js.map

package/dist/eval/experiments/index-vs-chunk/run-experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-experiment.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/run-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,MAAM,wBAAwB,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,kCAAkC,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,uCAAuC,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvF,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,mCAAmC,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACjE,OAAO,EAAE,OAAO,EAAmB,MAAM,2BAA2B,CAAC;AACrE,OAAO,EAAE,qBAAqB,EAAyB,MAAM,sBAAsB,CAAC;AAOpF,6CAA6C;AAC7C,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,OAAO,GAAG,EAAE;QACV,CAAC,GAAG,CAAC,CAAC,GAAG,OAAO,GAAG,UAAU,CAAC,GAAG,UAAU,CAAC;QAC5C,OAAO,CAAC,GAAG,UAAU,CAAC;IACxB,CAAC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CACnB,UAAkB,EAClB,IAAY;IAEZ,KAAK,EAAE,CAAC,CAAC,cAAc;IAEvB,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,MAAM,GAAuB,EAAE,CAAC;IAEtC,mBAAmB;IACnB,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;IAEvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,MAAM,IAAI,UAAU;YAAE,MAAM;QAEvC,MAAM,QAAQ,GAAG,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAElC,2CAA2C;QAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;QACvF,MAAM,WAAW,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC;QAE1D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG;gBAAE,SAAS;YAEnD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,OAAO,CAAC,EAAE;gBACrB,WAAW,EAAE,OAAO,CAAC,IAAI;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,cAAc,CAC3B,cAAwB,EACxB,SAAiB,EACjB,iBAAyB,EACzB,YAAuG;IAEvG,MAAM,aAAa,GAAG,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,gBAAgB,CAAC,UAAU,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAE1D,kEAAkE;IAClE,qEAAqE;IACrE,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;IACxC,MAAM,aAAa,GAAG,oBAAoB,EAAE,CAAC;IAC7C,MAAM,eAAe,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,iBAAiB,GAAG,eAAe,CAAC,CAAC;IAExE,MAAM,YAAY,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAErF,IAAI,mBAAmB,GAAyC,EAAE,CAAC;IACnE,IAAI,CAAC;QACH,mBAAmB,GAAG,2BAA2B,CAC/C,SAAS,EACT,YAAY,CAAC,kBAAkB,CAChC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB;IACpB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE7E,MAAM,gBAAgB,GAAiB,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,EAAE,CAAC,CAAC,EAAE;QACb,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAClC,MAAM,EAAE,QAAiB;KAC1B,CAAC,CAAC,CAAC;IAEJ,MAAM,iBAAiB,GAAiB,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,CAAC,EAAE;QACb,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,SAAkB;KAC3B,CAAC,CAAC,CAAC;IAEJ,MAAM,UAAU,GAAG,OAAO,CACxB;QACE,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,CAAC,YAAY,EAAE;QAC9D,GAAG,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,EAAE,YAAY,CAAC,aAAa,EAAE,CAAC;YACpE,CAAC,CAAC,EAAE,CAAC;KACR,EACD,YAAY,CAAC,IAAI,CAClB,CAAC;IAEF,2BAA2B;IAC3B,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,qBAAqB,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe,CAC5B,cAAwB,EACxB,SAAiB,EACjB,iBAAyB,EACzB,YAAuG;IAEvG,MAAM,aAAa,GAAG,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,WAAW,CAAC,UAAU,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;IAErD,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IAE5E,IAAI,cAAc,GAAyC,EAAE,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;QACxC,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,kBAAkB,CAAC,CAAC;IACnF,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB;IACpB,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnE,MAAM,WAAW,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACpD,OAAO,EAAE,CAAC,CAAC,EAAE;QACb,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAClC,MAAM,EAAE,QAAiB;KAC1B,CAAC,CAAC,CAAC;IAEJ,MAAM,YAAY,GAAiB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5D,OAAO,EAAE,CAAC,CAAC,EAAE;QACb,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,MAAM,EAAE,SAAkB;KAC3B,CAAC,CAAC,CAAC;IAEJ,MAAM,KAAK,GAAG,OAAO,CACnB;QACE,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,CAAC,YAAY,EAAE;QACzD,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YACzB,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,aAAa,EAAE,CAAC;YAC/D,CAAC,CAAC,EAAE,CAAC;KACR,EACD,YAAY,CAAC,IAAI,CAClB,CAAC;IAEF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CACrB,OAAsB,EACtB,IAAuB;IAEvB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC;IAEzD,MAAM,SAAS,GAAG,CAAC,CAAS,EAAE,EAAE,CAC9B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAE7D,MAAM,GAAG,GACP,KAAK;SACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SACpB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAEvD,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAEjE,MAAM,eAAe,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,aAAa,GACjB,eAAe,CAAC,MAAM,GAAG,CAAC;QACxB,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC,CAAC;IACR,MAAM,WAAW,GACf,SAAS,CAAC,MAAM,GAAG,CAAC;QAClB,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;QACvB,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;QACzB,UAAU,EAAE,SAAS,CAAC,EAAE,CAAC;QACzB,GAAG;QACH,OAAO;QACP,aAAa,EAAE,WAAW;QAC1B,eAAe,EAAE,aAAa;KAC/B,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,CAAS,EAAE,CAAC,GAAG,CAAC;IAC3B,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY;IACzB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACvE,MAAM,UAAU,GAAG,aAAa;QAC9B,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC3C,CAAC,CAAC,GAAG,CAAC;IACR,MAAM,IAAI,GAAG,EAAE,CAAC;IAEhB,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAE5D,sBAAsB;IACtB,MAAM,GAAG,GAAG,KAAK,EAAE,CAAC;IACpB,MAAM,UAAU,GAAG,kBAAkB,EAAE,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAC;IAC5C,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QACrD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,cAAc,GAAG,UAAU,EAAE,CAAC;IACpC,MAAM,MAAM,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAE/C,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,4BAA4B,CAAC,CAAC;IAClE,MAAM,aAAa,GAAG,YAAY,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,aAAa,aAAa,CAAC,MAAM,gBAAgB,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC;IAE7H,6BAA6B;IAC7B,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;IACvE,MAAM,OAAO,GAAG,MAAM,qBAAqB,CACzC,aAAa,EACb,MAAM,CAAC,mBAAmB,CAC3B,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,aAAa,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAEvG,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,sBAAsB;IACtB,MAAM,QAAQ,GAAG,IAAI,QAAQ,EAAE,CAAC;IAChC,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;IAErD,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IACzC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,oCAAoC;IACpC,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,MAAM,uCAAuC,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAkB,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,cAAc;QACd,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC;QAE7C,aAAa;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,MAAM,cAAc,CACvC,cAAc,EACd,CAAC,CAAC,KAAK,EACP,iBAAiB,EACjB,YAAY,CACb,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAE9C,aAAa;QACb,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,MAAM,eAAe,CACxC,cAAc,EACd,CAAC,CAAC,KAAK,EACP,iBAAiB,EACjB,YAAY,CACb,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QAE9C,4BAA4B;QAC5B,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB;QAClF,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAEjE,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,kBAAkB,EAAE,CAAC,CAAC,kBAAkB;YACxC,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,YAAY,CAAC,MAAM;gBAClC,UAAU,EAAE,aAAa;aAC1B;YACD,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS;gBACf,aAAa,EAAE,YAAY,CAAC,MAAM;gBAClC,UAAU,EAAE,aAAa;aAC1B;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;IAEzB,qBAAqB;IACrB,MAAM,YAAY,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,YAAY,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEvD,qBAAqB;IACrB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACnE,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CACxL,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAC5L,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,UAAU,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAC5L,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,GAAG,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,EAAE,CACpI,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,YAAY,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAChL,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC,YAAY,CAAC,aAAa,GAAG,YAAY,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CACzL,CAAC;IACF,OAAO,CAAC,GAAG,CACT,qBAAqB,GAAG,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,cAAc,GAAG,CAAC,YAAY,CAAC,eAAe,GAAG,YAAY,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CACjM,CAAC;IAEF,qCAAqC;IACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;QAChB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CACtD,CAAC;IACF,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAC/B,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC;QAChB,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CACtD,CAAC;IACF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CACzD,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,mCAAmC,SAAS,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,eAAe,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IAE1M,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QACzD,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC,CAAC,KAAK,kBAAkB,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE,CACtF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QACzD,KAAK,MAAM,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,CAAC,GAAG,CACT,QAAQ,CAAC,CAAC,KAAK,kBAAkB,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,MAAM,EAAE,CACtF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,aAAa;IACb,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,OAAO,CAAC,IAAI,CACV,cAAc,OAAO,CAAC,MAAM,oCAAoC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,WAAW,CACzH,CAAC;IAEF,MAAM,YAAY,GAAG,YAAY,CAAC,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC;IACrE,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC;IAErD,IAAI,YAAY,GAAG,IAAI,EAAE,CAAC;QACxB,OAAO,CAAC,IAAI,CACV,uBAAuB,GAAG,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC,CAAC,gBAAgB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CACxF,CAAC;IACJ,CAAC;SAAM,IAAI,YAAY,GAAG,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,CAAC,IAAI,CACV,sBAAsB,GAAG,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC,CAAC,eAAe,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CACtF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,IAAI,CACV,qBAAqB,GAAG,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC,CAAC,qBAAqB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,YAAY,CACjG,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,IAAI,CACV,mBAAmB,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,6BAA6B,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,CACzH,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,gBAAgB,EAAE,aAAa,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;QACvD,YAAY;QACZ,YAAY;QACZ,QAAQ;QACR,OAAO;KACR,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,YAAY,EAAE;KACX,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;IACf,MAAM,OAAO,GAAG,4BAA4B,CAAC;IAC7C,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAC;AAChD,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACb,OAAO,CAAC,KAAK,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}

package/dist/eval/experiments/index-vs-chunk/types.d.ts

@@ -0,0 +1,71 @@
+/**
+ * Types for the index-vs-chunk retrieval benchmark.
+ *
+ * Compares search quality when using the semantic index layer
+ * vs direct chunk search, using LLM-generated natural language queries.
+ */
+/** A single benchmark query with ground truth. */
+export interface BenchmarkQuery {
+    /** LLM-generated natural language search query. */
+    query: string;
+    /** Ground truth chunk ID that should be found. */
+    groundTruthChunkId: string;
+    /** Session slug for context. */
+    sessionSlug: string;
+    /** Cluster ID the chunk belongs to. */
+    clusterId: string;
+    /** Cluster name (for display). */
+    clusterName: string | null;
+}
+/** Result for a single query across both search paths. */
+export interface QueryResult {
+    query: string;
+    groundTruthChunkId: string;
+    /** Index-based search path results. */
+    index: {
+        /** Rank of ground truth chunk (1-indexed, 0 = not found). */
+        rank: number;
+        /** Total chunks returned. */
+        totalReturned: number;
+        /** Duration in milliseconds. */
+        durationMs: number;
+    };
+    /** Chunk-based search path results. */
+    chunk: {
+        rank: number;
+        totalReturned: number;
+        durationMs: number;
+    };
+}
+/** Aggregate metrics for one search path. */
+export interface PathMetrics {
+    /** Recall at K = fraction of queries where ground truth was in top K. */
+    recallAt5: number;
+    recallAt10: number;
+    recallAt20: number;
+    /** Mean Reciprocal Rank. */
+    mrr: number;
+    /** Hit rate = fraction of queries where ground truth was found at all. */
+    hitRate: number;
+    /** Mean latency in ms. */
+    meanLatencyMs: number;
+    /** Median latency in ms. */
+    medianLatencyMs: number;
+}
+/** Full benchmark report. */
+export interface IndexVsChunkReport {
+    timestamp: string;
+    /** Number of queries used. */
+    queryCount: number;
+    /** Number of queries that failed to generate. */
+    failedQueryCount: number;
+    /** Index-based search path metrics. */
+    indexMetrics: PathMetrics;
+    /** Chunk-based search path metrics. */
+    chunkMetrics: PathMetrics;
+    /** Per-query results. */
+    perQuery: QueryResult[];
+    /** Human-readable summary. */
+    summary: string[];
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/eval/experiments/index-vs-chunk/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,kDAAkD;AAClD,MAAM,WAAW,cAAc;IAC7B,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gCAAgC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,0DAA0D;AAC1D,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;IAE3B,uCAAuC;IACvC,KAAK,EAAE;QACL,6DAA6D;QAC7D,IAAI,EAAE,MAAM,CAAC;QACb,6BAA6B;QAC7B,aAAa,EAAE,MAAM,CAAC;QACtB,gCAAgC;QAChC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,uCAAuC;IACvC,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAED,6CAA6C;AAC7C,MAAM,WAAW,WAAW;IAC1B,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,4BAA4B;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB,0BAA0B;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,4BAA4B;IAC5B,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,6BAA6B;AAC7B,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,gBAAgB,EAAE,MAAM,CAAC;IAEzB,uCAAuC;IACvC,YAAY,EAAE,WAAW,CAAC;IAC1B,uCAAuC;IACvC,YAAY,EAAE,WAAW,CAAC;IAE1B,yBAAyB;IACzB,QAAQ,EAAE,WAAW,EAAE,CAAC;IAExB,8BAA8B;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB"}

package/dist/eval/experiments/index-vs-chunk/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for the index-vs-chunk retrieval benchmark.
+ *
+ * Compares search quality when using the semantic index layer
+ * vs direct chunk search, using LLM-generated natural language queries.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/eval/experiments/index-vs-chunk/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/eval/experiments/index-vs-chunk/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Pipeline Dropout Analysis
+ *
+ * Traces where target chunks get lost in the retrieval pipeline.
+ * For each query, checks whether the ground-truth chunk survives each stage:
+ *   1. Vector search (raw top-K)
+ *   2. RRF fusion (vector + keyword)
+ *   3. Cluster expansion
+ *   4. Oversized filtering
+ *   5. MMR reranking
+ *   6. Budget assembly
+ *   7. Chain walking (recall path)
+ *
+ * Usage:
+ *   npx tsx src/eval/experiments/pipeline-dropout/run-experiment.ts [--sample-size=50]
+ */
+export {};
+//# sourceMappingURL=run-experiment.d.ts.map

package/dist/eval/experiments/pipeline-dropout/run-experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-experiment.d.ts","sourceRoot":"","sources":["../../../../src/eval/experiments/pipeline-dropout/run-experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG"}