causantic 0.2.0

package/dist/eval/collection-benchmark/retrieval.js

@@ -0,0 +1,203 @@
+/**
+ * Retrieval quality benchmarks.
+ *
+ * Measures whether retrieval returns the right chunks using
+ * self-supervised ground truth from structural relationships.
+ */
+import { getChunkById } from '../../storage/chunk-store.js';
+import { assembleContext } from '../../retrieval/context-assembler.js';
+import { approximateTokens } from '../../utils/token-counter.js';
+/**
+ * Run retrieval quality benchmarks.
+ */
+export async function runRetrievalBenchmarks(sample, topK, onProgress) {
+    const skipped = [];
+    // Adjacent recall
+    let adjacentRecallAt5 = 0;
+    let adjacentRecallAt10 = 0;
+    let mrrSum = 0;
+    let adjacentCount = 0;
+    if (sample.thresholds.canRunAdjacentRecall && sample.adjacentPairs.length > 0) {
+        let processed = 0;
+        for (const pair of sample.adjacentPairs) {
+            const queryChunk = getChunkById(pair.queryChunkId);
+            if (!queryChunk)
+                continue;
+            onProgress?.(`[${++processed}/${sample.adjacentPairs.length}] Adjacent chunk recall...`);
+            const response = await assembleContext({
+                query: queryChunk.content.slice(0, 500),
+                mode: 'recall',
+                projectFilter: queryChunk.sessionSlug,
+                maxTokens: 10000,
+                vectorSearchLimit: topK * 2,
+            });
+            const resultIds = response.chunks.map(c => c.id);
+            const foundInTop5 = resultIds.slice(0, 5).includes(pair.adjacentChunkId);
+            const foundInTop10 = resultIds.slice(0, topK).includes(pair.adjacentChunkId);
+            if (foundInTop5)
+                adjacentRecallAt5++;
+            if (foundInTop10)
+                adjacentRecallAt10++;
+            // MRR
+            const rank = resultIds.indexOf(pair.adjacentChunkId);
+            if (rank >= 0)
+                mrrSum += 1 / (rank + 1);
+            adjacentCount++;
+        }
+    }
+    else {
+        const reason = sample.thresholds.reasons.get('adjacentRecall') ?? 'insufficient data';
+        skipped.push({
+            name: 'Adjacent Chunk Recall',
+            reason: `Skipped: ${reason}`,
+            threshold: '>=2 sessions with >=3 chunks each',
+            current: reason,
+        });
+    }
+    // Cross-session bridging
+    let bridgingRecallAt10 = 0;
+    let bridgingCount = 0;
+    let randomRecallAt10 = 0;
+    let randomCount = 0;
+    if (sample.thresholds.canRunCrossSessionBridging && sample.crossSessionPairs.length > 0) {
+        let processed = 0;
+        for (const pair of sample.crossSessionPairs) {
+            const queryChunk = getChunkById(pair.chunkIdA);
+            if (!queryChunk)
+                continue;
+            onProgress?.(`[${++processed}/${sample.crossSessionPairs.length}] Cross-session bridging...`);
+            const response = await assembleContext({
+                query: queryChunk.content.slice(0, 500),
+                mode: 'recall',
+                projectFilter: queryChunk.sessionSlug,
+                maxTokens: 10000,
+                vectorSearchLimit: topK * 2,
+            });
+            const resultIds = response.chunks.map(c => c.id);
+            if (resultIds.slice(0, topK).includes(pair.chunkIdB)) {
+                bridgingRecallAt10++;
+            }
+            bridgingCount++;
+        }
+        // Random baseline: use cross-project pairs as negative control
+        if (sample.crossProjectPairs.length > 0) {
+            const randomSample = sample.crossProjectPairs.slice(0, Math.min(20, sample.crossProjectPairs.length));
+            for (const pair of randomSample) {
+                const queryChunk = getChunkById(pair.chunkIdA);
+                if (!queryChunk)
+                    continue;
+                const response = await assembleContext({
+                    query: queryChunk.content.slice(0, 500),
+                    mode: 'recall',
+                    projectFilter: queryChunk.sessionSlug,
+                    maxTokens: 10000,
+                    vectorSearchLimit: topK * 2,
+                });
+                const resultIds = response.chunks.map(c => c.id);
+                if (resultIds.slice(0, topK).includes(pair.chunkIdB)) {
+                    randomRecallAt10++;
+                }
+                randomCount++;
+            }
+        }
+    }
+    else {
+        const reason = sample.thresholds.reasons.get('crossSessionBridging') ?? 'insufficient data';
+        skipped.push({
+            name: 'Cross-Session Bridging',
+            reason: `Skipped: ${reason}`,
+            threshold: '>=3 sessions in same project with cross-session edges',
+            current: reason,
+        });
+    }
+    // Precision@K
+    let precisionAt5 = 0;
+    let precisionAt10 = 0;
+    let precisionCount = 0;
+    if (sample.thresholds.canRunPrecisionAtK && sample.crossProjectPairs.length > 0) {
+        // Use chunk IDs from the first project in cross-project pairs
+        const projectASamples = new Map();
+        for (const pair of sample.crossProjectPairs) {
+            const list = projectASamples.get(pair.projectA) ?? [];
+            list.push(pair.chunkIdA);
+            projectASamples.set(pair.projectA, list);
+        }
+        let processed = 0;
+        for (const [project, chunkIds] of projectASamples) {
+            const uniqueIds = [...new Set(chunkIds)].slice(0, 10);
+            for (const chunkId of uniqueIds) {
+                const queryChunk = getChunkById(chunkId);
+                if (!queryChunk)
+                    continue;
+                onProgress?.(`[${++processed}/${Math.min(20, [...projectASamples.values()].flat().length)}] Precision@K...`);
+                const response = await assembleContext({
+                    query: queryChunk.content.slice(0, 500),
+                    mode: 'recall',
+                    projectFilter: project,
+                    maxTokens: 10000,
+                    vectorSearchLimit: topK * 2,
+                });
+                const top5FromProject = response.chunks.slice(0, 5).filter(c => c.sessionSlug === project).length;
+                const top10FromProject = response.chunks.slice(0, topK).filter(c => c.sessionSlug === project).length;
+                precisionAt5 += top5FromProject / Math.min(5, response.chunks.length || 1);
+                precisionAt10 += top10FromProject / Math.min(topK, response.chunks.length || 1);
+                precisionCount++;
+            }
+        }
+    }
+    else {
+        const reason = sample.thresholds.reasons.get('precisionAtK') ?? 'insufficient data';
+        skipped.push({
+            name: 'Precision@K',
+            reason: `Skipped: ${reason}`,
+            threshold: '>=2 projects with >=10 chunks each',
+            current: reason,
+        });
+    }
+    // Token efficiency
+    let tokenEfficiencySum = 0;
+    let usefulTokenSum = 0;
+    let tokenEfficiencyCount = 0;
+    // Reuse adjacent recall results for token efficiency
+    if (adjacentCount > 0) {
+        for (const pair of sample.adjacentPairs.slice(0, 20)) {
+            const queryChunk = getChunkById(pair.queryChunkId);
+            if (!queryChunk)
+                continue;
+            const response = await assembleContext({
+                query: queryChunk.content.slice(0, 500),
+                mode: 'recall',
+                projectFilter: queryChunk.sessionSlug,
+                maxTokens: 10000,
+                vectorSearchLimit: topK * 2,
+            });
+            if (response.chunks.length === 0)
+                continue;
+            // Relevant = same session or same project
+            const relevantChunks = response.chunks.filter(c => c.sessionSlug === queryChunk.sessionSlug);
+            const totalTokens = response.tokenCount || 1;
+            const relevantTokens = relevantChunks.reduce((sum, c) => {
+                const chunk = getChunkById(c.id);
+                return sum + (chunk?.approxTokens ?? approximateTokens(c.preview));
+            }, 0);
+            tokenEfficiencySum += relevantTokens / totalTokens;
+            usefulTokenSum += relevantTokens;
+            tokenEfficiencyCount++;
+        }
+    }
+    const result = {
+        adjacentRecallAt5: adjacentCount > 0 ? adjacentRecallAt5 / adjacentCount : 0,
+        adjacentRecallAt10: adjacentCount > 0 ? adjacentRecallAt10 / adjacentCount : 0,
+        mrr: adjacentCount > 0 ? mrrSum / adjacentCount : 0,
+        bridgingRecallAt10: bridgingCount > 0 ? bridgingRecallAt10 / bridgingCount : 0,
+        bridgingVsRandom: randomCount > 0 && randomRecallAt10 > 0
+            ? (bridgingRecallAt10 / bridgingCount) / (randomRecallAt10 / randomCount)
+            : bridgingCount > 0 ? bridgingRecallAt10 / bridgingCount : 0,
+        precisionAt5: precisionCount > 0 ? precisionAt5 / precisionCount : 0,
+        precisionAt10: precisionCount > 0 ? precisionAt10 / precisionCount : 0,
+        tokenEfficiency: tokenEfficiencyCount > 0 ? tokenEfficiencySum / tokenEfficiencyCount : 0,
+        meanUsefulTokensPerQuery: tokenEfficiencyCount > 0 ? usefulTokenSum / tokenEfficiencyCount : 0,
+    };
+    return { result, skipped };
+}
+//# sourceMappingURL=retrieval.js.map

package/dist/eval/collection-benchmark/retrieval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval.js","sourceRoot":"","sources":["../../../src/eval/collection-benchmark/retrieval.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAyB,MAAM,sCAAsC,CAAC;AAC9F,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAOjE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAuB,EACvB,IAAY,EACZ,UAAkC;IAElC,MAAM,OAAO,GAAuB,EAAE,CAAC;IAEvC,kBAAkB;IAClB,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,IAAI,MAAM,CAAC,UAAU,CAAC,oBAAoB,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9E,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACnD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAE1B,UAAU,EAAE,CAAC,IAAI,EAAE,SAAS,IAAI,MAAM,CAAC,aAAa,CAAC,MAAM,4BAA4B,CAAC,CAAC;YAEzF,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC;gBACrC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACvC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU,CAAC,WAAW;gBACrC,SAAS,EAAE,KAAK;gBAChB,iBAAiB,EAAE,IAAI,GAAG,CAAC;aAC5B,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACzE,MAAM,YAAY,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAE7E,IAAI,WAAW;gBAAE,iBAAiB,EAAE,CAAC;YACrC,IAAI,YAAY;gBAAE,kBAAkB,EAAE,CAAC;YAEvC,MAAM;YACN,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACrD,IAAI,IAAI,IAAI,CAAC;gBAAE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;YAExC,aAAa,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,mBAAmB,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,uBAAuB;YAC7B,MAAM,EAAE,YAAY,MAAM,EAAE;YAC5B,SAAS,EAAE,mCAAmC;YAC9C,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IAED,yBAAyB;IACzB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,IAAI,MAAM,CAAC,UAAU,CAAC,0BAA0B,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxF,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,CAAC,UAAU;gBAAE,SAAS;YAE1B,UAAU,EAAE,CAAC,IAAI,EAAE,SAAS,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,6BAA6B,CAAC,CAAC;YAE9F,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC;gBACrC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACvC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU,CAAC,WAAW;gBACrC,SAAS,EAAE,KAAK;gBAChB,iBAAiB,EAAE,IAAI,GAAG,CAAC;aAC5B,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACrD,kBAAkB,EAAE,CAAC;YACvB,CAAC;YACD,aAAa,EAAE,CAAC;QAClB,CAAC;QAED,+DAA+D;QAC/D,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxC,MAAM,YAAY,GAAG,MAAM,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;YACtG,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;gBAChC,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC/C,IAAI,CAAC,UAAU;oBAAE,SAAS;gBAE1B,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC;oBACrC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBACvC,IAAI,EAAE,QAAQ;oBACd,aAAa,EAAE,UAAU,CAAC,WAAW;oBACrC,SAAS,EAAE,KAAK;oBAChB,iBAAiB,EAAE,IAAI,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjD,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACrD,gBAAgB,EAAE,CAAC;gBACrB,CAAC;gBACD,WAAW,EAAE,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,IAAI,mBAAmB,CAAC;QAC5F,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,wBAAwB;YAC9B,MAAM,EAAE,YAAY,MAAM,EAAE;YAC5B,SAAS,EAAE,uDAAuD;YAClE,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IAED,cAAc;IACd,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,IAAI,MAAM,CAAC,UAAU,CAAC,kBAAkB,IAAI,MAAM,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChF,8DAA8D;QAC9D,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoB,CAAC;QACpD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC5C,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzB,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,eAAe,EAAE,CAAC;YAClD,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACtD,KAAK,MAAM,OAAO,IAAI,SAAS,EAAE,CAAC;gBAChC,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;gBACzC,IAAI,CAAC,UAAU;oBAAE,SAAS;gBAE1B,UAAU,EAAE,CAAC,IAAI,EAAE,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;gBAE7G,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC;oBACrC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;oBACvC,IAAI,EAAE,QAAQ;oBACd,aAAa,EAAE,OAAO;oBACtB,SAAS,EAAE,KAAK;oBAChB,iBAAiB,EAAE,IAAI,GAAG,CAAC;iBAC5B,CAAC,CAAC;gBAEH,MAAM,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;gBAClG,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;gBAEtG,YAAY,IAAI,eAAe,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;gBAC3E,aAAa,IAAI,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;gBAChF,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,mBAAmB,CAAC;QACpF,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,aAAa;YACnB,MAAM,EAAE,YAAY,MAAM,EAAE;YAC5B,SAAS,EAAE,oCAAoC;YAC/C,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,oBAAoB,GAAG,CAAC,CAAC;IAE7B,qDAAqD;IACrD,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACnD,IAAI,CAAC,UAAU;gBAAE,SAAS;YAE1B,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC;gBACrC,KAAK,EAAE,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACvC,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,UAAU,CAAC,WAAW;gBACrC,SAAS,EAAE,KAAK;gBAChB,iBAAiB,EAAE,IAAI,GAAG,CAAC;aAC5B,CAAC,CAAC;YAEH,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAE3C,0CAA0C;YAC1C,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAC3C,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,UAAU,CAAC,WAAW,CAC9C,CAAC;YAEF,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAC7C,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;gBACtD,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjC,OAAO,GAAG,GAAG,CAAC,KAAK,EAAE,YAAY,IAAI,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACrE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEN,kBAAkB,IAAI,cAAc,GAAG,WAAW,CAAC;YACnD,cAAc,IAAI,cAAc,CAAC;YACjC,oBAAoB,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAoB;QAC9B,iBAAiB,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC5E,kBAAkB,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9E,GAAG,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACnD,kBAAkB,EAAE,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9E,gBAAgB,EAAE,WAAW,GAAG,CAAC,IAAI,gBAAgB,GAAG,CAAC;YACvD,CAAC,CAAC,CAAC,kBAAkB,GAAG,aAAa,CAAC,GAAG,CAAC,gBAAgB,GAAG,WAAW,CAAC;YACzE,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC9D,YAAY,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QACpE,aAAa,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;QACtE,eAAe,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;QACzF,wBAAwB,EAAE,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;KAC/F,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAC7B,CAAC"}

package/dist/eval/collection-benchmark/runner.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Benchmark orchestrator.
+ *
+ * Runs all benchmark categories based on profile, generates samples,
+ * computes composite scores, and assembles the final result.
+ */
+import type { CollectionBenchmarkOptions, CollectionBenchmarkResult, HealthResult, RetrievalResult, GraphValueResult, LatencyResult } from './types.js';
+/**
+ * Compute overall score (0-100) from sub-scores.
+ */
+export declare function computeOverallScore(health: HealthResult, retrieval?: RetrievalResult, graph?: GraphValueResult, latency?: LatencyResult): number;
+/**
+ * Generate human-readable highlights from results.
+ */
+export declare function generateHighlights(health: HealthResult, retrieval?: RetrievalResult, graph?: GraphValueResult, latency?: LatencyResult): string[];
+/**
+ * Run the collection benchmark suite.
+ */
+export declare function runCollectionBenchmark(options?: CollectionBenchmarkOptions): Promise<CollectionBenchmarkResult>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/collection-benchmark/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/collection-benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,OAAO,KAAK,EAGV,0BAA0B,EAC1B,yBAAyB,EAEzB,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,aAAa,EAGd,MAAM,YAAY,CAAC;AAkBpB;;GAEG;AACH,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,YAAY,EACpB,SAAS,CAAC,EAAE,eAAe,EAC3B,KAAK,CAAC,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM,CA+CR;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,SAAS,CAAC,EAAE,eAAe,EAC3B,KAAK,CAAC,EAAE,gBAAgB,EACxB,OAAO,CAAC,EAAE,aAAa,GACtB,MAAM,EAAE,CA+CV;AAED;;GAEG;AACH,wBAAsB,sBAAsB,CAC1C,OAAO,GAAE,0BAA+B,GACvC,OAAO,CAAC,yBAAyB,CAAC,CAiGpC"}

package/dist/eval/collection-benchmark/runner.js

@@ -0,0 +1,189 @@
+/**
+ * Benchmark orchestrator.
+ *
+ * Runs all benchmark categories based on profile, generates samples,
+ * computes composite scores, and assembles the final result.
+ */
+import { generateSamples } from './sampler.js';
+import { runHealthBenchmarks } from './health.js';
+import { runRetrievalBenchmarks } from './retrieval.js';
+import { runGraphValueBenchmarks } from './graph-value.js';
+import { runLatencyBenchmarks } from './latency.js';
+import { generateTuningRecommendations } from './tuning.js';
+import { storeBenchmarkRun, getLatestBenchmarkRun, computeTrend } from './history.js';
+import { loadConfig } from '../../config/loader.js';
+import { getDistinctProjects } from '../../storage/chunk-store.js';
+/**
+ * Resolve profile to category list.
+ */
+function resolveCategories(profile, explicit) {
+    if (explicit && explicit.length > 0)
+        return explicit;
+    switch (profile) {
+        case 'quick': return ['health'];
+        case 'standard': return ['health', 'retrieval'];
+        case 'full': return ['health', 'retrieval', 'graph', 'latency'];
+    }
+}
+/**
+ * Compute overall score (0-100) from sub-scores.
+ */
+export function computeOverallScore(health, retrieval, graph, latency) {
+    const scores = [];
+    // Health score (0-100)
+    const healthScore = Math.min(100, (Math.min(1, health.edgeToChunkRatio / 3) * 30 +
+        health.clusterCoverage * 40 +
+        (1 - health.orphanChunkPercentage) * 30));
+    scores.push({ score: healthScore, weight: 20 });
+    // Retrieval score (0-100)
+    if (retrieval) {
+        const retrievalScore = Math.min(100, retrieval.adjacentRecallAt10 * 30 +
+            retrieval.bridgingRecallAt10 * 20 +
+            retrieval.precisionAt10 * 25 +
+            retrieval.tokenEfficiency * 25);
+        scores.push({ score: retrievalScore, weight: 35 });
+    }
+    // Graph value score (0-100)
+    if (graph) {
+        const augScore = Math.min(1, (graph.sourceAttribution.augmentationRatio - 1) / 2);
+        const liftScore = Math.min(1, graph.lift);
+        const graphScore = Math.min(100, augScore * 50 + liftScore * 50);
+        scores.push({ score: graphScore, weight: 30 });
+    }
+    // Latency score (0-100, based on p95 thresholds)
+    if (latency) {
+        const recallLatencyScore = latency.recall.p95 <= 50 ? 100
+            : latency.recall.p95 <= 100 ? 80
+                : latency.recall.p95 <= 200 ? 60
+                    : latency.recall.p95 <= 500 ? 40
+                        : 20;
+        scores.push({ score: recallLatencyScore, weight: 15 });
+    }
+    // Weighted average with renormalization
+    const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
+    if (totalWeight === 0)
+        return 0;
+    return Math.round(scores.reduce((sum, s) => sum + s.score * s.weight, 0) / totalWeight);
+}
+/**
+ * Generate human-readable highlights from results.
+ */
+export function generateHighlights(health, retrieval, graph, latency) {
+    const highlights = [];
+    // Health highlights
+    highlights.push(`${(health.clusterCoverage * 100).toFixed(0)}% of chunks organized into topic clusters`);
+    if (health.orphanChunkPercentage < 0.05) {
+        highlights.push(`Only ${(health.orphanChunkPercentage * 100).toFixed(1)}% orphan chunks — knowledge graph is well-connected`);
+    }
+    if (health.temporalSpan) {
+        const days = Math.round((new Date(health.temporalSpan.latest).getTime() - new Date(health.temporalSpan.earliest).getTime()) /
+            (1000 * 60 * 60 * 24));
+        highlights.push(`Collection spans ${days} days across ${health.projectCount} project(s)`);
+    }
+    // Retrieval highlights
+    if (retrieval) {
+        if (retrieval.adjacentRecallAt10 > 0) {
+            highlights.push(`Adjacent chunk recall@10: ${(retrieval.adjacentRecallAt10 * 100).toFixed(0)}%`);
+        }
+        if (retrieval.bridgingRecallAt10 > 0) {
+            highlights.push(`Cross-session bridging recall: ${(retrieval.bridgingRecallAt10 * 100).toFixed(0)}%`);
+        }
+        if (retrieval.tokenEfficiency > 0) {
+            highlights.push(`Token efficiency: ${(retrieval.tokenEfficiency * 100).toFixed(0)}% of returned context is relevant`);
+        }
+    }
+    // Graph highlights
+    if (graph) {
+        if (graph.sourceAttribution.augmentationRatio > 1.1) {
+            highlights.push(`Graph traversal adds ${graph.sourceAttribution.augmentationRatio.toFixed(1)}x more results vs vector-only`);
+        }
+        if (graph.lift > 0.1) {
+            highlights.push(`Recall lift from graph: +${(graph.lift * 100).toFixed(0)}% over vector-only`);
+        }
+    }
+    // Latency highlights
+    if (latency) {
+        highlights.push(`p95 recall latency: ${latency.recall.p95.toFixed(0)}ms`);
+    }
+    return highlights;
+}
+/**
+ * Run the collection benchmark suite.
+ */
+export async function runCollectionBenchmark(options = {}) {
+    const { profile = 'standard', sampleSize = 50, seed, projectFilter, topK = 10, includeTuning = true, onProgress, } = options;
+    const categories = resolveCategories(profile, options.categories);
+    const skipped = [];
+    let retrieval;
+    let graphValue;
+    let latency;
+    // 1. Always run health
+    onProgress?.('[1/4] Collection health...');
+    const includeClusterQuality = categories.includes('retrieval') || categories.includes('graph');
+    const health = await runHealthBenchmarks(includeClusterQuality);
+    onProgress?.('[1/4] Collection health... done');
+    // 2. Generate samples if needed
+    let sample;
+    if (categories.some(c => c !== 'health')) {
+        onProgress?.(`[2/4] Sampling queries (${sampleSize} samples${seed !== null && seed !== undefined ? `, seed=${seed}` : ''})...`);
+        sample = generateSamples({ sampleSize, seed, projectFilter });
+        onProgress?.('[2/4] Sampling... done');
+        // 3. Warm-up
+        onProgress?.('[3/4] Warm-up queries...');
+        // Warm-up is handled within each benchmark module
+        onProgress?.('[3/4] Warm-up... done');
+    }
+    // 4. Run requested categories
+    if (categories.includes('retrieval') && sample) {
+        onProgress?.('[4/4] Retrieval quality benchmarks');
+        const retrievalResult = await runRetrievalBenchmarks(sample, topK, onProgress);
+        retrieval = retrievalResult.result;
+        skipped.push(...retrievalResult.skipped);
+    }
+    if (categories.includes('graph') && sample) {
+        onProgress?.('Graph value benchmarks');
+        const graphResult = await runGraphValueBenchmarks(sample, topK, onProgress);
+        graphValue = graphResult.result;
+        skipped.push(...graphResult.skipped);
+    }
+    if (categories.includes('latency') && sample) {
+        onProgress?.('Latency benchmarks');
+        const effectiveProject = projectFilter ?? getDistinctProjects()[0]?.slug;
+        latency = await runLatencyBenchmarks(sample, effectiveProject, onProgress);
+    }
+    // Compute composite score
+    const overallScore = computeOverallScore(health, retrieval, graphValue, latency);
+    // Generate highlights
+    const highlights = generateHighlights(health, retrieval, graphValue, latency);
+    // Build result
+    const result = {
+        timestamp: new Date().toISOString(),
+        profile,
+        collectionStats: health,
+        retrieval,
+        graphValue,
+        latency,
+        overallScore,
+        highlights,
+        skipped,
+    };
+    // Tuning recommendations
+    if (includeTuning) {
+        result.tuning = generateTuningRecommendations(result);
+    }
+    // Historical trending
+    const previousRun = getLatestBenchmarkRun();
+    if (previousRun) {
+        result.trend = computeTrend(previousRun, result);
+    }
+    // Store this run
+    try {
+        const configSnapshot = loadConfig();
+        storeBenchmarkRun(result, configSnapshot);
+    }
+    catch {
+        // Non-critical: don't fail benchmark if history storage fails
+    }
+    return result;
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/collection-benchmark/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/eval/collection-benchmark/runner.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AAenE;;GAEG;AACH,SAAS,iBAAiB,CACxB,OAAyB,EACzB,QAA8B;IAE9B,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IAErD,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,KAAK,UAAU,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChD,KAAK,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAoB,EACpB,SAA2B,EAC3B,KAAwB,EACxB,OAAuB;IAEvB,MAAM,MAAM,GAA6C,EAAE,CAAC;IAE5D,uBAAuB;IACvB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAChC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,GAAG,CAAC,CAAC,GAAG,EAAE;QAC7C,MAAM,CAAC,eAAe,GAAG,EAAE;QAC3B,CAAC,CAAC,GAAG,MAAM,CAAC,qBAAqB,CAAC,GAAG,EAAE,CACxC,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IAEhD,0BAA0B;IAC1B,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EACjC,SAAS,CAAC,kBAAkB,GAAG,EAAE;YACjC,SAAS,CAAC,kBAAkB,GAAG,EAAE;YACjC,SAAS,CAAC,aAAa,GAAG,EAAE;YAC5B,SAAS,CAAC,eAAe,GAAG,EAAE,CAC/B,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,iBAAiB,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,GAAG,EAAE,GAAG,SAAS,GAAG,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IACjD,CAAC;IAED,iDAAiD;IACjD,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG;YACvD,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE;gBAChC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE;oBAChC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE;wBAChC,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,wCAAwC;IACxC,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACjE,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,OAAO,IAAI,CAAC,KAAK,CACf,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,WAAW,CACrE,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAoB,EACpB,SAA2B,EAC3B,KAAwB,EACxB,OAAuB;IAEvB,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,oBAAoB;IACpB,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,2CAA2C,CAAC,CAAC;IAEzG,IAAI,MAAM,CAAC,qBAAqB,GAAG,IAAI,EAAE,CAAC;QACxC,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,qBAAqB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,qDAAqD,CAAC,CAAC;IAChI,CAAC;IAED,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;YACnG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CACtB,CAAC;QACF,UAAU,CAAC,IAAI,CAAC,oBAAoB,IAAI,gBAAgB,MAAM,CAAC,YAAY,aAAa,CAAC,CAAC;IAC5F,CAAC;IAED,uBAAuB;IACvB,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,SAAS,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACrC,UAAU,CAAC,IAAI,CAAC,6BAA6B,CAAC,SAAS,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACnG,CAAC;QACD,IAAI,SAAS,CAAC,kBAAkB,GAAG,CAAC,EAAE,CAAC;YACrC,UAAU,CAAC,IAAI,CAAC,kCAAkC,CAAC,SAAS,CAAC,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACxG,CAAC;QACD,IAAI,SAAS,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAClC,UAAU,CAAC,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,eAAe,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC;QACxH,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IAAI,KAAK,EAAE,CAAC;QACV,IAAI,KAAK,CAAC,iBAAiB,CAAC,iBAAiB,GAAG,GAAG,EAAE,CAAC;YACpD,UAAU,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC;QAC/H,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,GAAG,GAAG,EAAE,CAAC;YACrB,UAAU,CAAC,IAAI,CAAC,4BAA4B,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,IAAI,OAAO,EAAE,CAAC;QACZ,UAAU,CAAC,IAAI,CAAC,uBAAuB,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,UAAsC,EAAE;IAExC,MAAM,EACJ,OAAO,GAAG,UAAU,EACpB,UAAU,GAAG,EAAE,EACf,IAAI,EACJ,aAAa,EACb,IAAI,GAAG,EAAE,EACT,aAAa,GAAG,IAAI,EACpB,UAAU,GACX,GAAG,OAAO,CAAC;IAEZ,MAAM,UAAU,GAAG,iBAAiB,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAElE,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,SAAsC,CAAC;IAC3C,IAAI,UAAwC,CAAC;IAC7C,IAAI,OAAkC,CAAC;IAEvC,uBAAuB;IACvB,UAAU,EAAE,CAAC,4BAA4B,CAAC,CAAC;IAC3C,MAAM,qBAAqB,GAAG,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC/F,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,qBAAqB,CAAC,CAAC;IAChE,UAAU,EAAE,CAAC,iCAAiC,CAAC,CAAC;IAEhD,gCAAgC;IAChC,IAAI,MAAM,CAAC;IACX,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;QACzC,UAAU,EAAE,CAAC,2BAA2B,UAAU,WAAW,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QAChI,MAAM,GAAG,eAAe,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;QAC9D,UAAU,EAAE,CAAC,wBAAwB,CAAC,CAAC;QAEvC,aAAa;QACb,UAAU,EAAE,CAAC,0BAA0B,CAAC,CAAC;QACzC,kDAAkD;QAClD,UAAU,EAAE,CAAC,uBAAuB,CAAC,CAAC;IACxC,CAAC;IAED,8BAA8B;IAC9B,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,MAAM,EAAE,CAAC;QAC/C,UAAU,EAAE,CAAC,oCAAoC,CAAC,CAAC;QACnD,MAAM,eAAe,GAAG,MAAM,sBAAsB,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/E,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QAC3C,UAAU,EAAE,CAAC,wBAAwB,CAAC,CAAC;QACvC,MAAM,WAAW,GAAG,MAAM,uBAAuB,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QAC5E,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;QAChC,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,MAAM,EAAE,CAAC;QAC7C,UAAU,EAAE,CAAC,oBAAoB,CAAC,CAAC;QACnC,MAAM,gBAAgB,GAAG,aAAa,IAAI,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC;QACzE,OAAO,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,gBAAgB,EAAE,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED,0BAA0B;IAC1B,MAAM,YAAY,GAAG,mBAAmB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAEjF,sBAAsB;IACtB,MAAM,UAAU,GAAG,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAE9E,eAAe;IACf,MAAM,MAAM,GAA8B;QACxC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO;QACP,eAAe,EAAE,MAAM;QACvB,SAAS;QACT,UAAU;QACV,OAAO;QACP,YAAY;QACZ,UAAU;QACV,OAAO;KACR,CAAC;IAEF,yBAAyB;IACzB,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,CAAC,MAAM,GAAG,6BAA6B,CAAC,MAAM,CAAC,CAAC;IACxD,CAAC;IAED,sBAAsB;IACtB,MAAM,WAAW,GAAG,qBAAqB,EAAE,CAAC;IAC5C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,GAAG,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED,iBAAiB;IACjB,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,UAAU,EAAE,CAAC;QACpC,iBAAiB,CAAC,MAAM,EAAE,cAAoD,CAAC,CAAC;IAClF,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;IAChE,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/eval/collection-benchmark/sampler.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Query sampling with seeded PRNG for reproducible benchmarks.
+ *
+ * Generates benchmark samples from the user's collection, including
+ * adjacent pairs (ground truth for recall), cross-session pairs, and
+ * cross-project negative controls.
+ */
+import type { StoredChunk, StoredEdge } from '../../storage/types.js';
+import type { BenchmarkSample, SamplerOptions, SamplerThresholds } from './types.js';
+/**
+ * Check minimum thresholds for each benchmark type.
+ */
+export declare function checkThresholds(chunks: StoredChunk[], edges: StoredEdge[]): SamplerThresholds;
+/**
+ * Generate benchmark samples from the collection.
+ */
+export declare function generateSamples(options: SamplerOptions): BenchmarkSample;
+//# sourceMappingURL=sampler.d.ts.map

package/dist/eval/collection-benchmark/sampler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sampler.d.ts","sourceRoot":"","sources":["../../../src/eval/collection-benchmark/sampler.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAiB,MAAM,wBAAwB,CAAC;AACrF,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,iBAAiB,EAIlB,MAAM,YAAY,CAAC;AAgBpB;;GAEG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,WAAW,EAAE,EACrB,KAAK,EAAE,UAAU,EAAE,GAClB,iBAAiB,CAuDnB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,cAAc,GAAG,eAAe,CA4HxE"}

package/dist/eval/collection-benchmark/sampler.js

@@ -0,0 +1,186 @@
+/**
+ * Query sampling with seeded PRNG for reproducible benchmarks.
+ *
+ * Generates benchmark samples from the user's collection, including
+ * adjacent pairs (ground truth for recall), cross-session pairs, and
+ * cross-project negative controls.
+ */
+import { getAllChunks } from '../../storage/chunk-store.js';
+import { getAllEdges } from '../../storage/edge-store.js';
+/**
+ * Deterministic seeded shuffle (same algorithm as annotation-schema.ts).
+ */
+function seededShuffle(arr, seed) {
+    const result = [...arr];
+    let s = seed;
+    for (let i = result.length - 1; i > 0; i--) {
+        s = (s * 1664525 + 1013904223) & 0x7fffffff;
+        const j = s % (i + 1);
+        [result[i], result[j]] = [result[j], result[i]];
+    }
+    return result;
+}
+/**
+ * Check minimum thresholds for each benchmark type.
+ */
+export function checkThresholds(chunks, edges) {
+    const reasons = new Map();
+    // Group chunks by session
+    const sessionChunks = new Map();
+    for (const chunk of chunks) {
+        const list = sessionChunks.get(chunk.sessionId) ?? [];
+        list.push(chunk);
+        sessionChunks.set(chunk.sessionId, list);
+    }
+    // Adjacent recall: need >= 2 sessions with >= 3 chunks each
+    const sessionsWithEnoughChunks = [...sessionChunks.values()].filter(c => c.length >= 3);
+    const canRunAdjacentRecall = sessionsWithEnoughChunks.length >= 2;
+    if (!canRunAdjacentRecall) {
+        reasons.set('adjacentRecall', `need >=2 sessions with >=3 chunks each, you have ${sessionsWithEnoughChunks.length}`);
+    }
+    // Cross-session bridging: need >= 3 sessions in same project with cross-session edges
+    const projectSessions = new Map();
+    for (const chunk of chunks) {
+        const sessions = projectSessions.get(chunk.sessionSlug) ?? new Set();
+        sessions.add(chunk.sessionId);
+        projectSessions.set(chunk.sessionSlug, sessions);
+    }
+    const crossSessionEdges = edges.filter(e => e.referenceType === 'file-path' || e.referenceType === 'code-entity');
+    const projectsWithCrossSession = [...projectSessions.entries()].filter(([, sessions]) => sessions.size >= 3);
+    const canRunCrossSessionBridging = projectsWithCrossSession.length > 0 && crossSessionEdges.length > 0;
+    if (!canRunCrossSessionBridging) {
+        const sessCount = Math.max(...[...projectSessions.values()].map(s => s.size), 0);
+        reasons.set('crossSessionBridging', `need >=3 sessions in same project with cross-session edges, best project has ${sessCount} sessions`);
+    }
+    // Precision@K: need >= 2 projects with >= 10 chunks each
+    const projectChunkCounts = new Map();
+    for (const chunk of chunks) {
+        projectChunkCounts.set(chunk.sessionSlug, (projectChunkCounts.get(chunk.sessionSlug) ?? 0) + 1);
+    }
+    const projectsWithEnoughChunks = [...projectChunkCounts.entries()].filter(([, count]) => count >= 10);
+    const canRunPrecisionAtK = projectsWithEnoughChunks.length >= 2;
+    if (!canRunPrecisionAtK) {
+        reasons.set('precisionAtK', `need >=2 projects with >=10 chunks each, you have ${projectsWithEnoughChunks.length}`);
+    }
+    return {
+        canRunAdjacentRecall,
+        canRunCrossSessionBridging,
+        canRunPrecisionAtK,
+        reasons,
+    };
+}
+/**
+ * Generate benchmark samples from the collection.
+ */
+export function generateSamples(options) {
+    const { sampleSize, seed = 42, projectFilter } = options;
+    let chunks = getAllChunks();
+    let edges = getAllEdges();
+    // Apply project filter if specified
+    if (projectFilter) {
+        chunks = chunks.filter(c => c.sessionSlug === projectFilter);
+        const chunkIds = new Set(chunks.map(c => c.id));
+        edges = edges.filter(e => chunkIds.has(e.sourceChunkId) || chunkIds.has(e.targetChunkId));
+    }
+    const thresholds = checkThresholds(chunks, edges);
+    // Group chunks by session, sorted by start_time
+    const sessionChunks = new Map();
+    for (const chunk of chunks) {
+        const list = sessionChunks.get(chunk.sessionId) ?? [];
+        list.push(chunk);
+        sessionChunks.set(chunk.sessionId, list);
+    }
+    for (const list of sessionChunks.values()) {
+        list.sort((a, b) => a.startTime.localeCompare(b.startTime));
+    }
+    // Sample query chunk IDs
+    const shuffledChunks = seededShuffle(chunks, seed);
+    const queryChunkIds = shuffledChunks.slice(0, sampleSize).map(c => c.id);
+    // Generate adjacent pairs
+    const adjacentPairs = [];
+    if (thresholds.canRunAdjacentRecall) {
+        let pairSeed = seed + 100;
+        for (const [sessionId, sessionList] of sessionChunks) {
+            if (sessionList.length < 3)
+                continue;
+            // Exclude first and last chunks, sample from middle
+            const middleIndices = Array.from({ length: sessionList.length - 2 }, (_, i) => i + 1);
+            const shuffled = seededShuffle(middleIndices, pairSeed++);
+            const limit = Math.min(shuffled.length, Math.ceil(sampleSize / sessionChunks.size) + 1);
+            for (let k = 0; k < limit && adjacentPairs.length < sampleSize; k++) {
+                const idx = shuffled[k];
+                const chunk = sessionList[idx];
+                // Pick one adjacent (previous or next)
+                const adjacentIdx = (pairSeed++ % 2 === 0) ? idx - 1 : idx + 1;
+                if (adjacentIdx >= 0 && adjacentIdx < sessionList.length) {
+                    adjacentPairs.push({
+                        queryChunkId: chunk.id,
+                        adjacentChunkId: sessionList[adjacentIdx].id,
+                        sessionId,
+                    });
+                }
+            }
+        }
+    }
+    // Generate cross-session pairs from edges
+    const crossSessionPairs = [];
+    if (thresholds.canRunCrossSessionBridging) {
+        const chunkSessionMap = new Map();
+        for (const chunk of chunks) {
+            chunkSessionMap.set(chunk.id, chunk.sessionId);
+        }
+        const bridgingEdges = edges.filter(e => {
+            const srcSession = chunkSessionMap.get(e.sourceChunkId);
+            const tgtSession = chunkSessionMap.get(e.targetChunkId);
+            return srcSession && tgtSession && srcSession !== tgtSession &&
+                (e.referenceType === 'file-path' || e.referenceType === 'code-entity');
+        });
+        const shuffledEdges = seededShuffle(bridgingEdges, seed + 200);
+        for (const edge of shuffledEdges.slice(0, sampleSize)) {
+            crossSessionPairs.push({
+                chunkIdA: edge.sourceChunkId,
+                chunkIdB: edge.targetChunkId,
+                edgeType: edge.referenceType,
+            });
+        }
+    }
+    // Generate cross-project pairs (negative controls)
+    const crossProjectPairs = [];
+    if (thresholds.canRunPrecisionAtK) {
+        const projectChunks = new Map();
+        for (const chunk of chunks) {
+            const list = projectChunks.get(chunk.sessionSlug) ?? [];
+            list.push(chunk);
+            projectChunks.set(chunk.sessionSlug, list);
+        }
+        const projectKeys = [...projectChunks.keys()].filter(k => (projectChunks.get(k)?.length ?? 0) >= 10);
+        if (projectKeys.length >= 2) {
+            const shuffledKeys = seededShuffle(projectKeys, seed + 300);
+            let count = 0;
+            for (let pi = 0; pi < shuffledKeys.length - 1 && count < sampleSize; pi++) {
+                const chunksA = seededShuffle(projectChunks.get(shuffledKeys[pi]), seed + 301);
+                for (let pj = pi + 1; pj < shuffledKeys.length && count < sampleSize; pj++) {
+                    const chunksB = seededShuffle(projectChunks.get(shuffledKeys[pj]), seed + 302);
+                    const limit = Math.min(chunksA.length, chunksB.length, sampleSize - count);
+                    for (let i = 0; i < limit; i++) {
+                        crossProjectPairs.push({
+                            chunkIdA: chunksA[i].id,
+                            projectA: shuffledKeys[pi],
+                            chunkIdB: chunksB[i].id,
+                            projectB: shuffledKeys[pj],
+                        });
+                        count++;
+                    }
+                }
+            }
+        }
+    }
+    return {
+        queryChunkIds,
+        adjacentPairs,
+        crossSessionPairs,
+        crossProjectPairs,
+        thresholds,
+    };
+}
+//# sourceMappingURL=sampler.js.map