@devflow-tools/benchmark 0.16.12 → 0.16.13

package/dist/index.d.ts

@@ -9,4 +9,6 @@ export { runBenchmarkSuite } from "./suite-runner.js";
 export type { BenchmarkRetrievalRunner, SuiteRunOptions } from "./suite-runner.js";
 export { renderBenchmarkReport } from "./report-renderer.js";
 export type { BenchmarkReportFormat } from "./report-renderer.js";
+export { runRetrievalQualityBenchmark } from './retrieval-quality.js';
+export type { RetrievalQualityCase, RetrievalQualityFixture, RetrievalQualityReport, RetrievalQualityThresholds, } from './retrieval-quality.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,YAAY,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,wBAAwB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC;AACtE,YAAY,EACV,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,0BAA0B,GAC3B,MAAM,wBAAwB,CAAC"}

package/dist/index.js

@@ -4,4 +4,5 @@ export { ACCURACY_TESTS } from "./accuracy-queries.js";
 export { LexicalBenchmarkRunner } from "./lexical-runner.js";
 export { runBenchmarkSuite } from "./suite-runner.js";
 export { renderBenchmarkReport } from "./report-renderer.js";
+export { runRetrievalQualityBenchmark } from './retrieval-quality.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,4BAA4B,EAAE,MAAM,wBAAwB,CAAC"}

package/dist/retrieval-quality.d.ts

@@ -0,0 +1,76 @@
+import type { ContextFileRole } from '@devflow-tools/sdk';
+export interface RetrievalQualityFixture {
+    schemaVersion: 'devflow.retrieval-quality.v1';
+    thresholds: RetrievalQualityThresholds;
+    queries: RetrievalQualityCase[];
+    forbiddenCodeEdges?: Array<{
+        sourceFile: string;
+        targetFile: string;
+        kind?: string;
+    }>;
+}
+export interface RetrievalQualityThresholds {
+    codeFileRecallAt5: number;
+    symbolNdcgAt5: number;
+    falseCrossFileEdgeRate: number;
+    memoryPrecisionAt5: number;
+    expectedEmptyMemoryAccuracy: number;
+    knowledgeVectorCoverage: number;
+    knowledgeChunkP95: number;
+    knowledgeVersionMatch: number;
+}
+export interface RetrievalQualityCase {
+    id: string;
+    query: string;
+    intent: string;
+    relevantCode?: Array<{
+        path: string;
+        role: ContextFileRole;
+    }>;
+    relevantSymbols?: string[];
+    relevantMemoryIds?: string[];
+    relevantMemoryTerms?: string[];
+    expectedEmptyMemory?: boolean;
+    relevantKnowledgeIds?: string[];
+    relevantKnowledgeSources?: string[];
+    expectedVersions?: Record<string, string>;
+}
+export interface RetrievalQualityReport {
+    schemaVersion: 'devflow.retrieval-quality-report.v1';
+    projectRoot: string;
+    fixturePath: string;
+    createdAt: number;
+    status: 'passed' | 'failed';
+    metrics: {
+        codeFileRecallAt5: number;
+        codeFilePrecisionAt5: number;
+        symbolNdcgAt5: number;
+        falseCrossFileEdgeRate: number;
+        memoryPrecisionAt5: number;
+        expectedEmptyMemoryAccuracy: number;
+        knowledgeNdcgAt5: number;
+        knowledgeVectorCoverage: number;
+        knowledgeChunkP95: number;
+        knowledgeVersionMatch: number;
+    };
+    thresholds: RetrievalQualityThresholds;
+    gates: Record<string, boolean>;
+    cases: Array<{
+        id: string;
+        requestId: string;
+        selectedCode: string[];
+        selectedMemory: string[];
+        selectedKnowledge: string[];
+        codeRecallAt5?: number;
+        memoryPrecisionAt5?: number;
+        expectedEmptyMemoryCorrect?: boolean;
+        knowledgeNdcgAt5?: number;
+    }>;
+    diagnostics: string[];
+}
+export declare function runRetrievalQualityBenchmark(options: {
+    projectRoot: string;
+    fixturePath?: string;
+    persist?: boolean;
+}): Promise<RetrievalQualityReport>;
+//# sourceMappingURL=retrieval-quality.d.ts.map

package/dist/retrieval-quality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-quality.d.ts","sourceRoot":"","sources":["../src/retrieval-quality.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAuB,MAAM,oBAAoB,CAAC;AAE/E,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,8BAA8B,CAAC;IAC9C,UAAU,EAAE,0BAA0B,CAAC;IACvC,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAChC,kBAAkB,CAAC,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvF;AAED,MAAM,WAAW,0BAA0B;IACzC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2BAA2B,EAAE,MAAM,CAAC;IACpC,uBAAuB,EAAE,MAAM,CAAC;IAChC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,eAAe,CAAA;KAAE,CAAC,CAAC;IAC9D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC/B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,EAAE,CAAC;IACpC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,sBAAsB;IACrC,aAAa,EAAE,qCAAqC,CAAC;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,EAAE;QACP,iBAAiB,EAAE,MAAM,CAAC;QAC1B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,aAAa,EAAE,MAAM,CAAC;QACtB,sBAAsB,EAAE,MAAM,CAAC;QAC/B,kBAAkB,EAAE,MAAM,CAAC;QAC3B,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,CAAC;QACzB,uBAAuB,EAAE,MAAM,CAAC;QAChC,iBAAiB,EAAE,MAAM,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;KAC/B,CAAC;IACF,UAAU,EAAE,0BAA0B,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,KAAK,CAAC;QACX,EAAE,EAAE,MAAM,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,iBAAiB,EAAE,MAAM,EAAE,CAAC;QAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,0BAA0B,CAAC,EAAE,OAAO,CAAC;QACrC,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;IACH,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,wBAAsB,4BAA4B,CAAC,OAAO,EAAE;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAkHlC"}

package/dist/retrieval-quality.js

@@ -0,0 +1,236 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { createEngines } from '@devflow-tools/context-engine';
+import { openGlobalDevFlowDatabase } from '@devflow-tools/database';
+export async function runRetrievalQualityBenchmark(options) {
+    const projectRoot = resolve(options.projectRoot);
+    const fixturePath = resolve(options.fixturePath ?? defaultFixturePath());
+    const fixture = loadFixture(fixturePath);
+    const engines = await createEngines(projectRoot, { warmSymbolVectors: false });
+    const envelopes = [];
+    const diagnostics = [];
+    try {
+        for (const testCase of fixture.queries) {
+            const result = await engines.context.buildTaskContext(testCase.query, {
+                mode: 'deep',
+                maxTokens: 8_000,
+                sessionId: `quality:${testCase.id}`,
+                executionId: `quality:${testCase.id}`,
+                requestId: `quality:${testCase.id}:${Date.now()}`,
+                host: 'benchmark',
+                command: 'retrieval-quality',
+            });
+            if (!result.taskEnvelope)
+                throw new Error(`Query ${testCase.id} did not produce a TaskContextEnvelope`);
+            envelopes.push({ fixture: testCase, envelope: result.taskEnvelope });
+        }
+        const graph = engines.graph.export();
+        const rejected = engines.graph.getRejectedRelations();
+        const nodeFiles = new Map(graph.nodes.map(node => [node.id, normalizePath(node.file)]));
+        const acceptedCrossFile = graph.edges.filter(edge => {
+            const source = nodeFiles.get(edge.from);
+            const target = nodeFiles.get(edge.to);
+            return source && target && source !== target;
+        });
+        const forbidden = fixture.forbiddenCodeEdges ?? [];
+        const falseAccepted = acceptedCrossFile.filter(edge => {
+            const source = nodeFiles.get(edge.from) ?? '';
+            const target = nodeFiles.get(edge.to) ?? '';
+            return forbidden.some(candidate => edgeMatches(candidate, source, target, edge.type));
+        });
+        for (const candidate of forbidden) {
+            const rejectedMatch = rejected.some(edge => edgeMatches(candidate, normalizePath(edge.sourceFile ?? ''), normalizePath(edge.targetFile ?? ''), edge.kind));
+            const acceptedMatch = falseAccepted.some(edge => edgeMatches(candidate, nodeFiles.get(edge.from) ?? '', nodeFiles.get(edge.to) ?? '', edge.type));
+            if (!rejectedMatch && !acceptedMatch)
+                diagnostics.push(`forbidden edge was not observable: ${candidate.sourceFile} -> ${candidate.targetFile}`);
+        }
+        const fileCases = envelopes.filter(item => (item.fixture.relevantCode?.length ?? 0) > 0);
+        const symbolCases = envelopes.filter(item => (item.fixture.relevantSymbols?.length ?? 0) > 0);
+        const memoryCases = envelopes.filter(item => (item.fixture.relevantMemoryIds?.length ?? 0) > 0 || (item.fixture.relevantMemoryTerms?.length ?? 0) > 0);
+        const emptyMemoryCases = envelopes.filter(item => item.fixture.expectedEmptyMemory === true);
+        const knowledgeCases = envelopes.filter(item => (item.fixture.relevantKnowledgeIds?.length ?? 0) > 0 || (item.fixture.relevantKnowledgeSources?.length ?? 0) > 0);
+        const versionCases = envelopes.filter(item => Object.keys(item.fixture.expectedVersions ?? {}).length > 0);
+        const knowledgeHealth = engines.knowledge.getIndexHealth();
+        const metrics = {
+            codeFileRecallAt5: average(fileCases.map(item => fileRecall(item.envelope, item.fixture, 5))),
+            codeFilePrecisionAt5: average(fileCases.map(item => filePrecision(item.envelope, item.fixture, 5))),
+            symbolNdcgAt5: average(symbolCases.map(item => symbolNdcg(item.envelope, item.fixture, 5))),
+            falseCrossFileEdgeRate: acceptedCrossFile.length > 0 ? falseAccepted.length / acceptedCrossFile.length : 0,
+            memoryPrecisionAt5: average(memoryCases.map(item => memoryPrecision(item.envelope, item.fixture, 5))),
+            expectedEmptyMemoryAccuracy: average(emptyMemoryCases.map(item => item.envelope.memory.entries.length === 0 ? 1 : 0)),
+            knowledgeNdcgAt5: average(knowledgeCases.map(item => knowledgeNdcg(item.envelope, item.fixture, 5))),
+            knowledgeVectorCoverage: knowledgeHealth.vectorCoverage,
+            knowledgeChunkP95: knowledgeHealth.chunkTokens.p95,
+            knowledgeVersionMatch: average(versionCases.map(item => knowledgeVersionMatch(item.envelope, item.fixture))),
+        };
+        const thresholds = fixture.thresholds;
+        const gates = {
+            codeFileRecallAt5: metrics.codeFileRecallAt5 >= thresholds.codeFileRecallAt5,
+            symbolNdcgAt5: metrics.symbolNdcgAt5 >= thresholds.symbolNdcgAt5,
+            falseCrossFileEdgeRate: metrics.falseCrossFileEdgeRate < thresholds.falseCrossFileEdgeRate,
+            memoryPrecisionAt5: metrics.memoryPrecisionAt5 >= thresholds.memoryPrecisionAt5,
+            expectedEmptyMemoryAccuracy: metrics.expectedEmptyMemoryAccuracy >= thresholds.expectedEmptyMemoryAccuracy,
+            knowledgeVectorCoverage: metrics.knowledgeVectorCoverage >= thresholds.knowledgeVectorCoverage,
+            knowledgeChunkP95: metrics.knowledgeChunkP95 <= thresholds.knowledgeChunkP95,
+            knowledgeVersionMatch: metrics.knowledgeVersionMatch >= thresholds.knowledgeVersionMatch,
+        };
+        const report = {
+            schemaVersion: 'devflow.retrieval-quality-report.v1',
+            projectRoot,
+            fixturePath,
+            createdAt: Date.now(),
+            status: Object.values(gates).every(Boolean) ? 'passed' : 'failed',
+            metrics,
+            thresholds,
+            gates,
+            cases: envelopes.map(item => ({
+                id: item.fixture.id,
+                requestId: item.envelope.request.requestId,
+                selectedCode: item.envelope.code.files.map(file => `${file.role}:${file.path}`),
+                selectedMemory: item.envelope.memory.entries.map(entry => entry.id),
+                selectedKnowledge: item.envelope.knowledge.entries.map(entry => `${entry.source}:${entry.id}`),
+                ...((item.fixture.relevantCode?.length ?? 0) > 0 ? { codeRecallAt5: fileRecall(item.envelope, item.fixture, 5) } : {}),
+                ...((item.fixture.relevantMemoryIds?.length ?? 0) + (item.fixture.relevantMemoryTerms?.length ?? 0) > 0
+                    ? { memoryPrecisionAt5: memoryPrecision(item.envelope, item.fixture, 5) } : {}),
+                ...(item.fixture.expectedEmptyMemory ? { expectedEmptyMemoryCorrect: item.envelope.memory.entries.length === 0 } : {}),
+                ...((item.fixture.relevantKnowledgeIds?.length ?? 0) + (item.fixture.relevantKnowledgeSources?.length ?? 0) > 0
+                    ? { knowledgeNdcgAt5: knowledgeNdcg(item.envelope, item.fixture, 5) } : {}),
+            })),
+            diagnostics,
+        };
+        if (options.persist !== false)
+            persistReport(report);
+        return report;
+    }
+    finally {
+        engines.context.close();
+        engines.memory.close();
+        engines.knowledge.close();
+        engines.telemetry.close();
+        engines.codegraph.close();
+        engines.data.close?.();
+    }
+}
+function persistReport(report) {
+    const database = openGlobalDevFlowDatabase();
+    try {
+        database.insertBenchmarkReport({
+            runId: `retrieval-quality:${report.createdAt}`,
+            suiteId: 'retrieval-quality',
+            suiteVersion: '1',
+            projectRoot: report.projectRoot,
+            commit: null,
+            createdAt: report.createdAt,
+            status: 'completed',
+            tasks: report.cases,
+            qualityStatus: report.status,
+            metrics: report.metrics,
+            thresholds: report.thresholds,
+            gates: report.gates,
+            diagnostics: report.diagnostics,
+            fixturePath: report.fixturePath,
+        });
+    }
+    finally {
+        database.close();
+    }
+}
+function loadFixture(path) {
+    if (!existsSync(path))
+        throw new Error(`Retrieval quality fixture not found: ${path}`);
+    const value = JSON.parse(readFileSync(path, 'utf8'));
+    if (value.schemaVersion !== 'devflow.retrieval-quality.v1' || !Array.isArray(value.queries)) {
+        throw new Error(`Invalid retrieval quality fixture: ${path}`);
+    }
+    return value;
+}
+function defaultFixturePath() {
+    return resolve(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures', 'retrieval-quality.json');
+}
+function fileRecall(envelope, fixture, k) {
+    const expected = new Set((fixture.relevantCode ?? []).map(file => normalizePath(file.path)));
+    const selected = new Set(envelope.code.files.slice(0, k).map(file => normalizePath(file.path)));
+    return expected.size > 0 ? [...expected].filter(path => selected.has(path)).length / expected.size : 1;
+}
+function filePrecision(envelope, fixture, k) {
+    const expected = new Set((fixture.relevantCode ?? []).map(file => normalizePath(file.path)));
+    const selected = envelope.code.files.slice(0, k).map(file => normalizePath(file.path));
+    return selected.length > 0 ? selected.filter(path => expected.has(path)).length / selected.length : 0;
+}
+function symbolNdcg(envelope, fixture, k) {
+    const expected = new Set((fixture.relevantSymbols ?? []).map(normalizeText));
+    const selected = envelope.code.files.flatMap(file => file.symbols).slice(0, k).map(normalizeText);
+    return ndcg(selected.map(value => expected.has(value) ? 1 : 0), Math.min(k, expected.size));
+}
+function memoryPrecision(envelope, fixture, k) {
+    const ids = new Set(fixture.relevantMemoryIds ?? []);
+    const terms = (fixture.relevantMemoryTerms ?? []).map(normalizeText);
+    const selected = envelope.memory.entries.slice(0, k);
+    if (selected.length === 0)
+        return 0;
+    const relevant = selected.filter(entry => ids.has(entry.id)
+        || terms.some(term => normalizeText(`${entry.title} ${entry.applicability.join(' ')}`).includes(term)));
+    return relevant.length / selected.length;
+}
+function knowledgeNdcg(envelope, fixture, k) {
+    const ids = new Set(fixture.relevantKnowledgeIds ?? []);
+    const sources = new Set((fixture.relevantKnowledgeSources ?? []).map(normalizeText));
+    const relevance = envelope.knowledge.entries.slice(0, k)
+        .map(entry => ids.has(entry.id) || sources.has(normalizeText(entry.source)) ? 1 : 0);
+    // A relevant source can contribute multiple useful chunks. When source-level
+    // relevance is specified, every position up to k can therefore be relevant.
+    const idealRelevant = sources.size > 0 ? k : Math.min(k, ids.size);
+    return ndcg(relevance, idealRelevant);
+}
+function knowledgeVersionMatch(envelope, fixture) {
+    const expected = fixture.expectedVersions ?? {};
+    const applicable = envelope.knowledge.entries.filter(entry => expected[entry.source] !== undefined);
+    if (applicable.length === 0)
+        return 0;
+    return applicable.filter(entry => major(entry.version) === major(expected[entry.source])).length / applicable.length;
+}
+function edgeMatches(expected, source, target, kind) {
+    const forward = normalizePath(expected.sourceFile) === source && normalizePath(expected.targetFile) === target;
+    const reverse = normalizePath(expected.sourceFile) === target && normalizePath(expected.targetFile) === source;
+    return (forward || reverse) && (!expected.kind || normalizeText(expected.kind) === normalizeText(kind));
+}
+function ndcg(relevance, idealRelevant) {
+    const dcg = relevance.reduce((sum, value, index) => sum + value / Math.log2(index + 2), 0);
+    const idcg = Array.from({ length: Math.min(relevance.length, idealRelevant) }, (_, index) => 1 / Math.log2(index + 2))
+        .reduce((sum, value) => sum + value, 0);
+    return idcg > 0 ? Math.min(1, dcg / idcg) : 1;
+}
+function average(values) {
+    return values.length > 0 ? values.reduce((sum, value) => sum + value, 0) / values.length : 1;
+}
+function normalizePath(value) {
+    return value.replaceAll('\\', '/').replace(/^\.\//u, '').toLocaleLowerCase('en-US');
+}
+function normalizeText(value) {
+    return value.normalize('NFKC').toLocaleLowerCase('en-US').trim();
+}
+function major(value) {
+    return value?.match(/\d+/u)?.[0];
+}
+async function main() {
+    const args = process.argv.slice(2);
+    const projectIndex = args.indexOf('--project');
+    const fixtureIndex = args.indexOf('--fixture');
+    if (projectIndex < 0 || !args[projectIndex + 1]) {
+        throw new Error('Usage: retrieval-quality --project <path> [--fixture <path>] [--json]');
+    }
+    const report = await runRetrievalQualityBenchmark({
+        projectRoot: args[projectIndex + 1],
+        fixturePath: fixtureIndex >= 0 ? args[fixtureIndex + 1] : undefined,
+    });
+    console.log(JSON.stringify(report, null, args.includes('--json') ? 0 : 2));
+    process.exitCode = report.status === 'passed' ? 0 : 1;
+}
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+    void main().catch(error => {
+        console.error(error instanceof Error ? error.message : String(error));
+        process.exitCode = 1;
+    });
+}
+//# sourceMappingURL=retrieval-quality.js.map

package/dist/retrieval-quality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"retrieval-quality.js","sourceRoot":"","sources":["../src/retrieval-quality.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAqEpE,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAAC,OAIlD;IACC,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,IAAI,kBAAkB,EAAE,CAAC,CAAC;IACzE,MAAM,OAAO,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,WAAW,EAAE,EAAE,iBAAiB,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/E,MAAM,SAAS,GAA4E,EAAE,CAAC;IAC9F,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,IAAI,CAAC;QACH,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACpE,IAAI,EAAE,MAAM;gBACZ,SAAS,EAAE,KAAK;gBAChB,SAAS,EAAE,WAAW,QAAQ,CAAC,EAAE,EAAE;gBACnC,WAAW,EAAE,WAAW,QAAQ,CAAC,EAAE,EAAE;gBACrC,SAAS,EAAE,WAAW,QAAQ,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;gBACjD,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,mBAAmB;aAC7B,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,YAAY;gBAAE,MAAM,IAAI,KAAK,CAAC,SAAS,QAAQ,CAAC,EAAE,wCAAwC,CAAC,CAAC;YACxG,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACxF,MAAM,iBAAiB,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YAClD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACtC,OAAO,MAAM,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,CAAC;QAC/C,CAAC,CAAC,CAAC;QACH,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACnD,MAAM,aAAa,GAAG,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACpD,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAC5C,OAAO,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACxF,CAAC,CAAC,CAAC;QACH,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,aAAa,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YAC3J,MAAM,aAAa,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,WAAW,CAC1D,SAAS,EACT,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAC9B,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,EAC5B,IAAI,CAAC,IAAI,CACV,CAAC,CAAC;YACH,IAAI,CAAC,aAAa,IAAI,CAAC,aAAa;gBAAE,WAAW,CAAC,IAAI,CAAC,sCAAsC,SAAS,CAAC,UAAU,OAAO,SAAS,CAAC,UAAU,EAAE,CAAC,CAAC;QAClJ,CAAC;QAED,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACzF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9F,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAC1C,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5G,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,KAAK,IAAI,CAAC,CAAC;QAC7F,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAC7C,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3G,MAAM,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;QAE3D,MAAM,OAAO,GAAG;YACd,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YAC7F,oBAAoB,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YACnG,aAAa,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YAC3F,sBAAsB,EAAE,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAC1G,kBAAkB,EAAE,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YACrG,2BAA2B,EAAE,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrH,gBAAgB,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;YACpG,uBAAuB,EAAE,eAAe,CAAC,cAAc;YACvD,iBAAiB,EAAE,eAAe,CAAC,WAAW,CAAC,GAAG;YAClD,qBAAqB,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,qBAAqB,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;SAC7G,CAAC;QACF,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACtC,MAAM,KAAK,GAAG;YACZ,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,UAAU,CAAC,iBAAiB;YAC5E,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,UAAU,CAAC,aAAa;YAChE,sBAAsB,EAAE,OAAO,CAAC,sBAAsB,GAAG,UAAU,CAAC,sBAAsB;YAC1F,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,IAAI,UAAU,CAAC,kBAAkB;YAC/E,2BAA2B,EAAE,OAAO,CAAC,2BAA2B,IAAI,UAAU,CAAC,2BAA2B;YAC1G,uBAAuB,EAAE,OAAO,CAAC,uBAAuB,IAAI,UAAU,CAAC,uBAAuB;YAC9F,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,UAAU,CAAC,iBAAiB;YAC5E,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,IAAI,UAAU,CAAC,qBAAqB;SACzF,CAAC;QACF,MAAM,MAAM,GAA2B;YACrC,aAAa,EAAE,qCAAqC;YACpD,WAAW;YACX,WAAW;YACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;YACjE,OAAO;YACP,UAAU;YACV,KAAK;YACL,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC5B,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE;gBACnB,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS;gBAC1C,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC/E,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnE,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,EAAE,EAAE,CAAC;gBAC9F,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtH,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;oBACrG,CAAC,CAAC,EAAE,kBAAkB,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjF,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,0BAA0B,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtH,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,oBAAoB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC;oBAC7G,CAAC,CAAC,EAAE,gBAAgB,EAAE,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9E,CAAC,CAAC;YACH,WAAW;SACZ,CAAC;QACF,IAAI,OAAO,CAAC,OAAO,KAAK,KAAK;YAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAA+B,CAAC,KAAK,EAAE,EAAE,CAAC;IACrD,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,MAA8B;IACnD,MAAM,QAAQ,GAAG,yBAAyB,EAAE,CAAC;IAC7C,IAAI,CAAC;QACH,QAAQ,CAAC,qBAAqB,CAAC;YAC7B,KAAK,EAAE,qBAAqB,MAAM,CAAC,SAAS,EAAE;YAC9C,OAAO,EAAE,mBAAmB;YAC5B,YAAY,EAAE,GAAG;YACjB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,MAAM,EAAE,WAAW;YACnB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,aAAa,EAAE,MAAM,CAAC,MAAM;YAC5B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,WAAW,EAAE,MAAM,CAAC,WAAW;SAChC,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,IAAI,EAAE,CAAC,CAAC;IACvF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAC;IAChF,IAAI,KAAK,CAAC,aAAa,KAAK,8BAA8B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CAAC,sCAAsC,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,kBAAkB;IACzB,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,wBAAwB,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,UAAU,CAAC,QAA6B,EAAE,OAA6B,EAAE,CAAS;IACzF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChG,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AACzG,CAAC;AAED,SAAS,aAAa,CAAC,QAA6B,EAAE,OAA6B,EAAE,CAAS;IAC5F,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACxG,CAAC;AAED,SAAS,UAAU,CAAC,QAA6B,EAAE,OAA6B,EAAE,CAAS;IACzF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAClG,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AAC9F,CAAC;AAED,SAAS,eAAe,CAAC,QAA6B,EAAE,OAA6B,EAAE,CAAS;IAC9F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;WACtD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1G,OAAO,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,aAAa,CAAC,QAA6B,EAAE,OAA6B,EAAE,CAAS;IAC5F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACrD,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,6EAA6E;IAC7E,4EAA4E;IAC5E,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,qBAAqB,CAAC,QAA6B,EAAE,OAA6B;IACzF,MAAM,QAAQ,GAAG,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC;IACpG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtC,OAAO,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;AACvH,CAAC;AAED,SAAS,WAAW,CAClB,QAAmE,EACnE,MAAc,EACd,MAAc,EACd,IAAY;IAEZ,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,MAAM,IAAI,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC;IAC/G,MAAM,OAAO,GAAG,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,MAAM,IAAI,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,MAAM,CAAC;IAC/G,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED,SAAS,IAAI,CAAC,SAAmB,EAAE,aAAqB;IACtD,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3F,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;SACnH,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC1C,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,OAAO,CAAC,MAAgB;IAC/B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/F,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACtF,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IAClC,OAAO,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,KAAK,CAAC,KAAyB;IACtC,OAAO,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACnC,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,YAAY,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,4BAA4B,CAAC;QAChD,WAAW,EAAE,IAAI,CAAC,YAAY,GAAG,CAAC,CAAE;QACpC,WAAW,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;KACpE,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IACnF,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QACxB,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/fixtures/retrieval-quality.json

@@ -0,0 +1,61 @@
+{
+  "schemaVersion": "devflow.retrieval-quality.v1",
+  "thresholds": {
+    "codeFileRecallAt5": 0.85,
+    "symbolNdcgAt5": 0.85,
+    "falseCrossFileEdgeRate": 0.01,
+    "memoryPrecisionAt5": 0.8,
+    "expectedEmptyMemoryAccuracy": 1,
+    "knowledgeVectorCoverage": 0.95,
+    "knowledgeChunkP95": 3000,
+    "knowledgeVersionMatch": 1
+  },
+  "queries": [
+    {
+      "id": "video-player-performance",
+      "query": "分析 src/components/video-player/VideoPlayer.tsx 组件的 React 性能问题",
+      "intent": "performance",
+      "relevantCode": [
+        { "path": "src/components/video-player/VideoPlayer.tsx", "role": "target" },
+        { "path": "src/pages/video-player/VideoPlayerPage.tsx", "role": "caller" },
+        { "path": "src/components/video-player/VideoPlayer.css", "role": "resource" },
+        { "path": "src/routes/video-player.tsx", "role": "route" }
+      ],
+      "relevantSymbols": ["VideoPlayer", "VideoPlayerInner"],
+      "expectedEmptyMemory": true,
+      "relevantKnowledgeSources": ["react"],
+      "expectedVersions": { "react": "19" }
+    },
+    {
+      "id": "animation-convention",
+      "query": "这个项目统一采用什么动画方案",
+      "intent": "understand",
+      "relevantMemoryTerms": ["motion"]
+    },
+    {
+      "id": "package-manager-convention",
+      "query": "这个项目使用什么包管理器",
+      "intent": "understand",
+      "relevantMemoryTerms": ["pnpm"]
+    },
+    {
+      "id": "styling-convention",
+      "query": "这个项目统一采用什么样式和组件方案",
+      "intent": "understand",
+      "relevantMemoryTerms": ["tailwind", "shadcn"]
+    },
+    {
+      "id": "react-version-convention",
+      "query": "这个项目采用什么 React 版本和语法约定",
+      "intent": "understand",
+      "relevantMemoryTerms": ["react"]
+    }
+  ],
+  "forbiddenCodeEdges": [
+    {
+      "sourceFile": "src/components/video-player/VideoPlayer.tsx",
+      "targetFile": "src/services/webrtc/peer-connection.ts",
+      "kind": "calls"
+    }
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devflow-tools/benchmark",
-  "version": "0.16.12",
+  "version": "0.16.13",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -18,15 +18,17 @@
     "accuracy": "npx tsx src/run-accuracy-tests.ts"
   },
   "dependencies": {
-    "@devflow-tools/context-engine": "0.16.12",
-    "@devflow-tools/sdk": "0.16.12"
+    "@devflow-tools/context-engine": "0.16.13",
+    "@devflow-tools/database": "0.16.13",
+    "@devflow-tools/sdk": "0.16.13"
   },
   "devDependencies": {
     "typescript": "^5.5.0",
     "vitest": "^2.0.0"
   },
   "files": [
-    "dist"
+    "dist",
+    "fixtures"
   ],
-  "gitHead": "d4a018f3031a3570c794c253f8edeff405956133"
+  "gitHead": "67c9dfc3709bd0ce36b7c2a9341896eedcc8a725"
 }