@devflow-tools/benchmark 0.17.7 → 0.18.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 shilongfeicool
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,6 +1,6 @@
 # @devflow-tools/benchmark
 
-> 用于比较 baseline 与 DevFlow Agent 结果的评测和检索质量基准库。（v0.17.7）
+> 用于比较 baseline 与 DevFlow Agent 结果的评测和检索质量基准库。（v0.17.8）
 
 [![npm version](https://img.shields.io/npm/v/%40devflow-tools%2Fbenchmark.svg)](https://www.npmjs.com/package/@devflow-tools/benchmark)
 
@@ -30,7 +30,7 @@ console.log(report.summary);
 
 ## 版本与兼容性
 
-- 当前包版本：`0.17.7`
+- 当前包版本：`0.17.8`
 - Node.js：未在 package manifest 声明更高版本时，使用 Node.js 18+；原生 SQLite/Playwright 能力请按对应依赖安装
 - workspace 内部包应使用同一 DevFlow minor/patch 版本，避免类型和数据库 schema 不一致
 

package/dist/accuracy-command-options.d.ts

@@ -0,0 +1,3 @@
+export declare function resolveAccuracyProjectRoot(args: string[], fallback: string): string;
+export declare function resolveAccuracyFixtureDirectory(args: string[]): string | undefined;
+//# sourceMappingURL=accuracy-command-options.d.ts.map

package/dist/accuracy-command-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"accuracy-command-options.d.ts","sourceRoot":"","sources":["../src/accuracy-command-options.ts"],"names":[],"mappings":"AAEA,wBAAgB,0BAA0B,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEnF;AAED,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAElF"}

package/dist/accuracy-command-options.js

@@ -0,0 +1,17 @@
+import { resolve } from 'node:path';
+export function resolveAccuracyProjectRoot(args, fallback) {
+    return resolve(optionalPathOption(args, '--project-root') ?? fallback);
+}
+export function resolveAccuracyFixtureDirectory(args) {
+    return optionalPathOption(args, '--fixture-directory');
+}
+function optionalPathOption(args, name) {
+    const index = args.indexOf(name);
+    if (index < 0)
+        return undefined;
+    const value = args[index + 1];
+    if (!value || value.startsWith('--'))
+        throw new Error(`${name} requires a path`);
+    return resolve(value);
+}
+//# sourceMappingURL=accuracy-command-options.js.map

package/dist/accuracy-command-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"accuracy-command-options.js","sourceRoot":"","sources":["../src/accuracy-command-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,UAAU,0BAA0B,CAAC,IAAc,EAAE,QAAgB;IACzE,OAAO,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE,gBAAgB,CAAC,IAAI,QAAQ,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,IAAc;IAC5D,OAAO,kBAAkB,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAc,EAAE,IAAY;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,kBAAkB,CAAC,CAAC;IACjF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC;AACxB,CAAC"}

package/dist/context-symbol-quality-command.d.ts

@@ -0,0 +1,59 @@
+import { type ProviderExecutionStatus } from '@devflow-tools/sdk';
+import { type EvidenceQualityMetric } from './evidence-quality-metrics.js';
+import { type RetrievalQuerySamplingAdapter } from '@devflow-tools/semantic-engine';
+interface ContextExperimentCaseInput {
+    caseId: string;
+    candidateRefs: string[];
+    selectedRefs: string[];
+    goldRefs: string[];
+    forbiddenRefs: string[];
+    querySampling: ContextQuerySamplingEvidence;
+}
+interface ContextQuerySamplingEvidence {
+    status: ProviderExecutionStatus;
+    receiptHash: string;
+    reasonCodes: string[];
+    proposalCount: number;
+    acceptedCount: number;
+    rejectedCount: number;
+    proposals: Array<{
+        proposalId: string;
+        kind: 'rewrite' | 'expansion';
+        query: string;
+    }>;
+}
+export interface ContextSymbolExperimentReportV1 {
+    schemaVersion: 'devflow.context-symbol-experiment.v1';
+    profile: {
+        profileHash: string;
+        generation: 3;
+        modelId: string;
+    };
+    cases: Array<{
+        caseId: string;
+        candidateRefs: string[];
+        selectedRefs: string[];
+        goldRefs: string[];
+        candidateHit: boolean;
+        recallAt5: EvidenceQualityMetric;
+        forbiddenRefsAt5: string[];
+        querySampling: ContextQuerySamplingEvidence;
+    }>;
+    hits: number;
+    denominator: number;
+    status: 'passed' | 'failed';
+}
+export declare function evaluateContextExperiment(input: {
+    profile: ContextSymbolExperimentReportV1['profile'];
+    cases: ContextExperimentCaseInput[];
+}): ContextSymbolExperimentReportV1;
+export declare function runContextSymbolQualityExperiment(input: {
+    manifestPath: string;
+    sourcesRoot: string;
+    caseIds: string[];
+    samplingAdapter: RetrievalQuerySamplingAdapter;
+    observer?: (event: Record<string, unknown>) => void;
+}): Promise<ContextSymbolExperimentReportV1>;
+export declare function createDeepSeekSamplingAdapter(model: string): RetrievalQuerySamplingAdapter;
+export {};
+//# sourceMappingURL=context-symbol-quality-command.d.ts.map

package/dist/context-symbol-quality-command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-symbol-quality-command.d.ts","sourceRoot":"","sources":["../src/context-symbol-quality-command.ts"],"names":[],"mappings":"AAWA,OAAO,EAQL,KAAK,uBAAuB,EAC7B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAyB,KAAK,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAMlG,OAAO,EAEL,KAAK,6BAA6B,EACnC,MAAM,gCAAgC,CAAC;AAgDxC,UAAU,0BAA0B;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,aAAa,EAAE,4BAA4B,CAAC;CAC7C;AAED,UAAU,4BAA4B;IACpC,MAAM,EAAE,uBAAuB,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,SAAS,GAAG,WAAW,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxF;AAED,MAAM,WAAW,+BAA+B;IAC9C,aAAa,EAAE,sCAAsC,CAAC;IACtD,OAAO,EAAE;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,CAAC,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IACjE,KAAK,EAAE,KAAK,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,YAAY,EAAE,OAAO,CAAC;QACtB,SAAS,EAAE,qBAAqB,CAAC;QACjC,gBAAgB,EAAE,MAAM,EAAE,CAAC;QAC3B,aAAa,EAAE,4BAA4B,CAAC;KAC7C,CAAC,CAAC;IACH,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,QAAQ,GAAG,QAAQ,CAAC;CAC7B;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,OAAO,EAAE,+BAA+B,CAAC,SAAS,CAAC,CAAC;IACpD,KAAK,EAAE,0BAA0B,EAAE,CAAC;CACrC,GAAG,+BAA+B,CAqClC;AAED,wBAAsB,iCAAiC,CAAC,KAAK,EAAE;IAC7D,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,eAAe,EAAE,6BAA6B,CAAC;IAC/C,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;CACrD,GAAG,OAAO,CAAC,+BAA+B,CAAC,CA6B3C;AA2RD,wBAAgB,6BAA6B,CAAC,KAAK,EAAE,MAAM,GAAG,6BAA6B,CAkD1F"}

package/dist/context-symbol-quality-command.js

@@ -0,0 +1,482 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import CGModule from '@colbymchenry/codegraph';
+import { createEngines } from '@devflow-tools/mcp-server/composition';
+import { createSamplingEvidenceReranker } from '@devflow-tools/mcp-server/retrieval';
+import { DevFlowDatabase } from '@devflow-tools/database';
+import { RetrievalCoordinator, RetrievalSessionRepository } from '@devflow-tools/retrieval-engine';
+import { resolveTaskSemanticContext } from '@devflow-tools/semantic-engine';
+import { createRuntimeActivationSnapshotV1, evaluationContextStableRef, projectSnapshotId, stableRuntimeJson, } from '@devflow-tools/sdk';
+import { evidenceQualityMetric } from './evidence-quality-metrics.js';
+import { readProjectSnapshotIdentity } from './evaluation-bundle.js';
+import { loadReleaseQualityMaterializationTargets, } from './release-quality-bundle-materializer.js';
+import { createRetrievalQuerySamplingResolver, } from '@devflow-tools/semantic-engine';
+import { captureProducer, identityFor, noOpTaskRuntime, rankedEvidenceRefs, stageCapture, } from './unified-runtime-runner.js';
+const CodeGraph = CGModule.default ?? CGModule;
+const REQUIRED_CASES = new Set([
+    'llm-wiki/context/explicit-files',
+    'ecc/context/path-safety',
+]);
+const RETRIEVAL_QUERY_TOOL_NAME = 'submit_retrieval_queries';
+const RETRIEVAL_QUERY_TOOL = {
+    name: RETRIEVAL_QUERY_TOOL_NAME,
+    description: 'Submit evidence-bound retrieval query proposals.',
+    input_schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+            proposals: {
+                type: 'array',
+                maxItems: 9,
+                items: {
+                    type: 'object',
+                    additionalProperties: false,
+                    properties: {
+                        channel: { type: 'string', enum: ['code', 'memory', 'knowledge'] },
+                        kind: { type: 'string', enum: ['rewrite', 'expansion'] },
+                        query: { type: 'string', maxLength: 2_000 },
+                        sourceClaimIds: {
+                            type: 'array',
+                            maxItems: 32,
+                            items: { type: 'string' },
+                        },
+                        confidence: { type: 'number', minimum: 0, maximum: 1 },
+                    },
+                    required: ['channel', 'kind', 'query', 'sourceClaimIds', 'confidence'],
+                },
+            },
+        },
+        required: ['proposals'],
+    },
+};
+const EVIDENCE_RERANK_TOOL_NAME = 'rank_evidence';
+export function evaluateContextExperiment(input) {
+    if (input.profile.generation !== 3 || !/^[a-f0-9]{64}$/u.test(input.profile.profileHash)) {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_PROFILE_INVALID');
+    }
+    const caseIds = new Set(input.cases.map(item => item.caseId));
+    if (caseIds.size !== REQUIRED_CASES.size
+        || [...REQUIRED_CASES].some(caseId => !caseIds.has(caseId))) {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CASE_SET_INVALID');
+    }
+    const cases = input.cases.map(item => {
+        const candidates = unique(item.candidateRefs).slice(0, 20);
+        const selected = unique(item.selectedRefs).slice(0, 5);
+        const gold = unique(item.goldRefs);
+        if (gold.length === 0)
+            throw new Error(`CONTEXT_SYMBOL_EXPERIMENT_GOLD_MISSING:${item.caseId}`);
+        const candidateSet = new Set(candidates);
+        const selectedSet = new Set(selected);
+        const forbidden = new Set(item.forbiddenRefs);
+        return {
+            caseId: item.caseId,
+            candidateRefs: candidates,
+            selectedRefs: selected,
+            goldRefs: gold,
+            candidateHit: gold.every(ref => candidateSet.has(ref)),
+            recallAt5: evidenceQualityMetric(gold.filter(ref => selectedSet.has(ref)).length, gold.length),
+            forbiddenRefsAt5: selected.filter(ref => forbidden.has(ref)),
+            querySampling: item.querySampling,
+        };
+    });
+    const hits = cases.filter(item => item.candidateHit && item.forbiddenRefsAt5.length === 0).length;
+    return {
+        schemaVersion: 'devflow.context-symbol-experiment.v1',
+        profile: input.profile,
+        cases,
+        hits,
+        denominator: cases.length,
+        status: hits === cases.length ? 'passed' : 'failed',
+    };
+}
+export async function runContextSymbolQualityExperiment(input) {
+    const requested = new Set(input.caseIds);
+    if (requested.size !== REQUIRED_CASES.size
+        || [...REQUIRED_CASES].some(caseId => !requested.has(caseId))) {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CASE_SET_INVALID');
+    }
+    const targets = loadReleaseQualityMaterializationTargets({
+        manifestPath: input.manifestPath,
+        sourcesRoot: input.sourcesRoot,
+        outputRoot: join(tmpdir(), 'devflow-context-symbol-unused'),
+    }).filter(target => target.input.requiredSlices.some(slice => slice.channel === 'context' && slice.caseIds.some(caseId => requested.has(caseId))));
+    if (targets.length !== 2)
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_TARGETS_INVALID');
+    let profile;
+    const cases = [];
+    for (const target of targets) {
+        assertPinnedCheckout(target);
+        const result = await runTarget(target, requested, input.samplingAdapter, input.observer);
+        profile ??= result.profile;
+        if (result.profile.profileHash !== profile.profileHash
+            || result.profile.generation !== profile.generation
+            || result.profile.modelId !== profile.modelId) {
+            throw new Error('CONTEXT_SYMBOL_EXPERIMENT_PROFILE_MISMATCH');
+        }
+        cases.push(...result.cases);
+    }
+    if (!profile)
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_PROFILE_MISSING');
+    return evaluateContextExperiment({ profile, cases });
+}
+async function runTarget(target, requested, samplingAdapter, observer) {
+    const container = mkdtempSync(join(tmpdir(), 'devflow-context-symbol-'));
+    const runtimeRoot = join(container, 'project');
+    let engines;
+    let operationalDatabase;
+    try {
+        execFileSync('git', ['clone', '--quiet', '--no-hardlinks', '--no-checkout', target.checkoutPath, runtimeRoot]);
+        execFileSync('git', ['-C', runtimeRoot, 'checkout', '--quiet', '--detach', target.input.project.commit]);
+        mkdirSync(join(runtimeRoot, '.devflow'), { recursive: true });
+        writeFileSync(join(runtimeRoot, '.devflow/config.json'), `${stableRuntimeJson({
+            projectId: target.input.project.projectId,
+            memoryNamespace: `memory:${target.input.project.projectId}`,
+        })}\n`);
+        const graph = await CodeGraph.init(runtimeRoot);
+        try {
+            const indexed = await graph.indexAll();
+            if (!indexed.success || graph.getIndexState() !== 'complete') {
+                throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CODEGRAPH_FAILED');
+            }
+        }
+        finally {
+            graph.close();
+        }
+        engines = await createEngines(runtimeRoot, { warmSymbolVectors: false });
+        if (!engines.symbolEmbedder || !engines.symbolStore) {
+            throw new Error('CONTEXT_SYMBOL_VECTOR_RUNTIME_UNAVAILABLE');
+        }
+        const buildStartedAt = performance.now();
+        await engines.symbolEmbedder.buildAll();
+        if (!engines.symbolEmbedder.isBuilt())
+            throw new Error('CONTEXT_SYMBOL_VECTOR_BUILD_INCOMPLETE');
+        const buildDurationMs = Math.round((performance.now() - buildStartedAt) * 1000) / 1000;
+        const embeddingProfile = engines.symbolStore.getProfile();
+        const profile = experimentProfile(embeddingProfile);
+        observer?.({
+            stage: 'vector-build',
+            projectId: target.input.project.projectId,
+            commit: target.input.project.commit,
+            tree: target.input.project.tree,
+            vectorCount: engines.symbolStore.count(),
+            buildDurationMs,
+            profile,
+        });
+        const snapshotId = projectSnapshotId(readProjectSnapshotIdentity(target.checkoutPath));
+        const corpus = readJson(target.corpusPath);
+        operationalDatabase = new DevFlowDatabase(runtimeRoot, {
+            dbPath: join(container, 'operational/devflow.db'),
+        });
+        const cases = [];
+        const retrievalQuerySampling = createRetrievalQuerySamplingResolver(samplingAdapter);
+        for (const item of corpus.cases.filter(item => requested.has(item.caseId))) {
+            const goldRefs = target.input.context.evidence.map(evidence => {
+                const node = engines.codegraph.getNodesInFile(evidence.path)
+                    .find(candidate => candidate.name === evidence.symbol && candidate.startLine === evidence.startLine);
+                if (!node)
+                    throw new Error(`CONTEXT_SYMBOL_EXPERIMENT_GOLD_NODE_MISSING:${item.caseId}`);
+                return evaluationContextStableRef({
+                    projectSnapshotId: snapshotId,
+                    normalizedPath: node.filePath,
+                    symbolId: node.id,
+                    range: [node.startLine, node.endLine],
+                });
+            });
+            const identity = identityFor(item.caseId, 'benchmark:context-symbol-experiment', runtimeRoot, target.input.project.projectId);
+            const semantic = await resolveTaskSemanticContext({
+                normalizedPrompt: item.prompt,
+                identity,
+                revision: 1,
+                activation: createRuntimeActivationSnapshotV1({
+                    semantic: 'authoritative',
+                    retrieval: 'authoritative',
+                    memoryExtraction: 'deferred',
+                    knowledgeCompilation: 'deferred',
+                    embeddingProfiles: { code_retrieval: profile.profileHash },
+                    rolloutPolicyVersion: 'context-symbol-experiment.v1',
+                    createdAt: 1,
+                }),
+                router: engines.semanticRouter,
+                retrievalQuerySampling: { sample: retrievalQuerySampling },
+                policy: {
+                    policyHash: 'context-symbol-experiment.v1',
+                    budgets: {
+                        hookDeadlineMs: 3_000,
+                        contextTokens: 8_000,
+                        responseReserveTokens: 2_000,
+                        perChannelMs: { code: 1_500, memory: 800, knowledge: 1_500 },
+                    },
+                },
+                now: 1,
+            });
+            const context = semantic.context;
+            const queryReceipt = semantic.resolution.providerReceipts
+                .find(receipt => receipt.capability === 'retrieval_query');
+            if (!queryReceipt)
+                throw new Error('CONTEXT_SYMBOL_EXPERIMENT_QUERY_RECEIPT_MISSING');
+            if (context.plan.code.requirement === 'prohibited') {
+                throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CODE_PLAN_PROHIBITED');
+            }
+            const querySampling = {
+                status: queryReceipt.status,
+                receiptHash: queryReceipt.receiptHash,
+                reasonCodes: queryReceipt.reasonCodes,
+                proposalCount: queryReceipt.proposalCount,
+                acceptedCount: queryReceipt.acceptedCount,
+                rejectedCount: queryReceipt.rejectedCount,
+                proposals: context.plan.code.additions.map(proposal => ({
+                    proposalId: proposal.proposalId,
+                    kind: proposal.kind,
+                    query: proposal.query,
+                })),
+            };
+            const stage = stageCapture();
+            const result = await new RetrievalCoordinator({
+                code: captureProducer(engines.codeEvidence, stage),
+                memory: captureProducer(engines.memoryQuery, stage),
+                knowledge: captureProducer(engines.knowledgeQuery, stage),
+                sessions: new RetrievalSessionRepository(operationalDatabase),
+                reranker: createSamplingEvidenceReranker(samplingAdapter),
+                taskRuntime: noOpTaskRuntime(),
+            }).retrieve({ context, taskRevision: 1, tokenBudget: 8_000, now: 1 });
+            const candidates = stage.candidates.filter(candidate => candidate.channel === 'code');
+            assertVectorCandidateProfiles(candidates, embeddingProfile);
+            const producerOutput = stage.outputs.find(output => output.result.receipt.channel === 'code');
+            if (!producerOutput)
+                throw new Error('CONTEXT_SYMBOL_EXPERIMENT_PRODUCER_RECEIPT_MISSING');
+            const refByProducerRef = new Map(candidates.map(candidate => [
+                candidate.stableRef,
+                evaluationRef(snapshotId, candidate),
+            ]));
+            const rankedCandidates = rankedEvidenceRefs(result.receipt, 'context')
+                .map(stableRef => candidates.find(candidate => candidate.stableRef === stableRef))
+                .filter((candidate) => Boolean(candidate));
+            const candidateRefs = unique(rankedCandidates.map(candidate => refByProducerRef.get(candidate.stableRef)));
+            const selectedRefs = unique((result.receipt.selectedEvidence ?? [])
+                .filter(candidate => candidate.channel === 'code')
+                .map(candidate => refByProducerRef.get(candidate.stableRef))
+                .filter((ref) => Boolean(ref)));
+            observer?.({
+                stage: 'retrieval',
+                caseId: item.caseId,
+                goldRanks: goldRefs.map(ref => candidateRefs.indexOf(ref) + 1 || null),
+                vectorGoldRanks: vectorGoldRanks(producerOutput.vectorCandidateRefs ?? [], candidates, snapshotId, goldRefs),
+                producerGoldRanks: goldRefs.map(ref => candidateRefs.indexOf(ref) + 1 || null),
+                selectedGoldRanks: selectedRefs.map(ref => goldRefs.indexOf(ref) + 1 || null),
+                querySampling,
+                topCandidates: rankedCandidates.slice(0, 20).map(candidate => ({
+                    ref: refByProducerRef.get(candidate.stableRef),
+                    path: candidate.payload.path,
+                    symbol: candidate.payload.symbol,
+                    symbolId: candidate.payload.symbolId,
+                    contributingLegs: candidate.contributingLegs,
+                })),
+            });
+            cases.push({
+                caseId: item.caseId,
+                candidateRefs,
+                selectedRefs,
+                goldRefs,
+                forbiddenRefs: [],
+                querySampling,
+            });
+        }
+        return { profile, cases };
+    }
+    finally {
+        operationalDatabase?.close();
+        if (engines)
+            closeEngines(engines);
+        rmSync(container, { recursive: true, force: true });
+    }
+}
+function vectorGoldRanks(vectorRefs, candidates, snapshotId, goldRefs) {
+    const evaluationByProducerRef = new Map(candidates.map(candidate => [
+        candidate.stableRef,
+        evaluationRef(snapshotId, candidate),
+    ]));
+    return goldRefs.map(gold => {
+        const match = vectorRefs.find(item => evaluationByProducerRef.get(item.ref) === gold);
+        return match?.rank ?? null;
+    });
+}
+function experimentProfile(profile) {
+    if (profile.purpose !== 'code_retrieval' || profile.generation !== 3) {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_PROFILE_INVALID');
+    }
+    return { profileHash: profile.profileHash, generation: 3, modelId: profile.modelId };
+}
+function assertVectorCandidateProfiles(candidates, profile) {
+    for (const candidate of candidates.filter(item => item.contributingLegs.some(leg => /vector|embedding/iu.test(leg.leg)))) {
+        if (candidate.retrievalProfile !== profile.profileHash || candidate.generation !== profile.generation) {
+            throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CANDIDATE_PROFILE_MISMATCH');
+        }
+    }
+}
+function evaluationRef(snapshotId, candidate) {
+    const path = candidate.payload.path;
+    const symbolId = candidate.payload.symbolId;
+    const startLine = candidate.payload.startLine;
+    const endLine = candidate.payload.endLine;
+    if (typeof path !== 'string' || typeof symbolId !== 'string'
+        || !Number.isSafeInteger(startLine) || !Number.isSafeInteger(endLine)) {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CANDIDATE_IDENTITY_INVALID');
+    }
+    return evaluationContextStableRef({
+        projectSnapshotId: snapshotId,
+        normalizedPath: path,
+        symbolId,
+        range: [startLine, endLine],
+    });
+}
+function assertPinnedCheckout(target) {
+    if (git(target.checkoutPath, ['rev-parse', 'HEAD']) !== target.input.project.commit
+        || git(target.checkoutPath, ['rev-parse', 'HEAD^{tree}']) !== target.input.project.tree
+        || git(target.checkoutPath, ['status', '--porcelain', '--untracked-files=all']) !== '') {
+        throw new Error('CONTEXT_SYMBOL_EXPERIMENT_CHECKOUT_IDENTITY_MISMATCH');
+    }
+}
+function git(root, args) {
+    return execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim();
+}
+function readJson(path) {
+    return JSON.parse(readFileSync(path, 'utf8'));
+}
+function unique(refs) {
+    return [...new Set(refs)];
+}
+function closeEngines(engines) {
+    engines.context.close();
+    engines.memory.close();
+    engines.knowledge.close();
+    engines.telemetry.close();
+    engines.codegraph.close();
+    engines.data.close?.();
+}
+function requiredOption(argv, name) {
+    const index = argv.indexOf(name);
+    const value = index >= 0 ? argv[index + 1] : undefined;
+    if (!value)
+        throw new Error(`${name} requires a value`);
+    return value;
+}
+async function main(argv) {
+    const output = resolve(requiredOption(argv, '--output'));
+    const report = await runContextSymbolQualityExperiment({
+        manifestPath: resolveInputPath(requiredOption(argv, '--manifest')),
+        sourcesRoot: resolveInputPath(requiredOption(argv, '--sources')),
+        caseIds: requiredOption(argv, '--cases').split(',').filter(Boolean),
+        samplingAdapter: createDeepSeekSamplingAdapter(requiredOption(argv, '--sampling-model')),
+        observer: event => process.stderr.write(`${stableRuntimeJson(event)}\n`),
+    });
+    mkdirSync(dirname(output), { recursive: true });
+    writeFileSync(output, `${stableRuntimeJson(report)}\n`);
+    process.stdout.write(`${stableRuntimeJson(report)}\n`);
+    if (report.status !== 'passed')
+        process.exitCode = 1;
+}
+export function createDeepSeekSamplingAdapter(model) {
+    if (!model.trim())
+        throw new Error('--sampling-model requires a value');
+    const baseUrl = process.env.DEEPSEEK_BASE_URL?.replace(/\/$/u, '');
+    const apiKey = process.env.DEEPSEEK_API_KEY;
+    return {
+        isSupported: () => Boolean(baseUrl && apiKey),
+        createMessage: async (request) => {
+            if (!baseUrl || !apiKey)
+                throw new Error('DEEPSEEK_SAMPLING_ENV_UNAVAILABLE');
+            const reranking = request.systemPrompt.startsWith('You rank retrieved evidence');
+            const tool = reranking ? evidenceRerankTool(request.prompt) : RETRIEVAL_QUERY_TOOL;
+            const response = await fetch(`${baseUrl}/v1/messages`, {
+                method: 'POST',
+                headers: {
+                    'content-type': 'application/json',
+                    'anthropic-version': '2023-06-01',
+                    'x-api-key': apiKey,
+                },
+                body: JSON.stringify({
+                    model,
+                    max_tokens: request.maxTokens,
+                    temperature: 0,
+                    thinking: { type: 'disabled' },
+                    system: request.systemPrompt,
+                    messages: [{ role: 'user', content: request.prompt }],
+                    tools: [tool],
+                    tool_choice: { type: 'tool', name: tool.name },
+                }),
+                signal: AbortSignal.timeout(request.timeoutMs),
+            });
+            const body = await response.json();
+            if (!response.ok) {
+                throw new Error(`DEEPSEEK_SAMPLING_HTTP_${response.status}:${String(body.error?.message ?? '')}`);
+            }
+            const toolOutputs = body.content?.filter(item => item.type === 'tool_use'
+                && item.name === tool.name) ?? [];
+            const input = toolOutputs.length === 1 ? toolOutputs[0]?.input : undefined;
+            const outputKey = reranking ? 'rankedRefs' : 'proposals';
+            if (!isRecord(input) || Object.keys(input).length !== 1 || !Array.isArray(input[outputKey])) {
+                throw new Error('DEEPSEEK_SAMPLING_TOOL_OUTPUT_INVALID');
+            }
+            return {
+                text: reranking ? JSON.stringify(input) : JSON.stringify(input.proposals),
+                model: typeof body.model === 'string' ? body.model : model,
+            };
+        },
+    };
+}
+function evidenceRerankTool(prompt) {
+    let value;
+    try {
+        value = JSON.parse(prompt);
+    }
+    catch {
+        throw new Error('DEEPSEEK_RERANK_REQUEST_INVALID');
+    }
+    if (!isRecord(value) || !Array.isArray(value.candidates)) {
+        throw new Error('DEEPSEEK_RERANK_REQUEST_INVALID');
+    }
+    const rankedRefs = value.candidates.map(candidate => isRecord(candidate) && typeof candidate.stableRef === 'string'
+        ? candidate.stableRef
+        : undefined);
+    if (rankedRefs.length === 0 || rankedRefs.length > 20
+        || rankedRefs.some((ref) => ref === undefined)
+        || new Set(rankedRefs).size !== rankedRefs.length) {
+        throw new Error('DEEPSEEK_RERANK_REQUEST_INVALID');
+    }
+    return {
+        name: EVIDENCE_RERANK_TOOL_NAME,
+        description: 'Rank every supplied evidence ref by direct relevance.',
+        input_schema: {
+            type: 'object',
+            additionalProperties: false,
+            properties: {
+                rankedRefs: {
+                    type: 'array',
+                    minItems: rankedRefs.length,
+                    maxItems: rankedRefs.length,
+                    uniqueItems: true,
+                    items: { type: 'string', enum: rankedRefs },
+                },
+            },
+            required: ['rankedRefs'],
+        },
+    };
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function resolveInputPath(value) {
+    if (value.startsWith('/') || /^[A-Za-z]:[\\/]/u.test(value))
+        return resolve(value);
+    const fromCwd = resolve(value);
+    if (existsSync(fromCwd))
+        return fromCwd;
+    return resolve(import.meta.dirname, '../../..', value);
+}
+if (process.argv[1] && realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url))) {
+    await main(process.argv.slice(2));
+}
+//# sourceMappingURL=context-symbol-quality-command.js.map