@devflow-tools/benchmark 0.17.7 → 0.18.0

package/dist/unified-runtime-runner.js

@@ -0,0 +1,620 @@
+import { createHash } from 'node:crypto';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { createEngines } from '@devflow-tools/mcp-server/composition';
+import { createDomainToolRegistry } from '@devflow-tools/mcp-server/retrieval';
+import { openGlobalDevFlowDatabase } from '@devflow-tools/database';
+import { createSemanticFacts, createRuntimeActivationSnapshotV1, semanticTextHash, stableRuntimeHash, } from '@devflow-tools/sdk';
+import { compileChannelQueryPlan, resolveTaskSemanticContext, planDomainAction, } from '@devflow-tools/semantic-engine';
+import { MemoryCommandService, MemoryStore } from '@devflow-tools/memory-engine';
+import { reciprocalRankFusion, RetrievalCoordinator, RetrievalSessionRepository, } from '@devflow-tools/retrieval-engine';
+import { emitSemanticControlLog } from '@devflow-tools/telemetry';
+const RUNNER_VERSION = 'unified-runtime-runner.v4';
+export async function runUnifiedRuntimeGold(corpus, options = {}) {
+    const projectRoot = options.projectRoot ?? resolve(import.meta.dirname, '../../..');
+    const runtime = options.runtime ?? runtimeIdentity(projectRoot);
+    const runStartedAt = performance.now();
+    emitSemanticControlLog({
+        event: 'benchmark.gold.loaded', identity: { projectRoot, sessionId: 'benchmark:unified-runtime' },
+        level: 'info', timestamp: Date.now(), data: {
+            datasetIds: corpus.datasetIds, labelSourceRevision: corpus.labelProvenance.sourceRevision,
+            semanticCount: corpus.semantic.length, contextCount: corpus.context.length,
+            memoryCount: corpus.memory.length, knowledgeCount: corpus.knowledge.length,
+        },
+    });
+    const activation = createRuntimeActivationSnapshotV1({
+        semantic: 'authoritative', retrieval: 'authoritative', memoryExtraction: 'deferred', knowledgeCompilation: 'deferred',
+        embeddingProfiles: {}, rolloutPolicyVersion: 'benchmark.v2', createdAt: Date.now(),
+    });
+    const engines = options.semanticExecutor ? undefined : await createEngines(projectRoot, { warmSymbolVectors: false });
+    try {
+        let runtimeRetrievalExecutor = options.retrievalExecutor;
+        if (!runtimeRetrievalExecutor && engines) {
+            runtimeRetrievalExecutor = (item, channel) => runRealRetrievalCase(item, channel, engines, projectRoot, activation);
+        }
+        const suiteIdentity = options.suiteIdentity ?? await resolveSuiteIdentity({
+            corpus, runtime, semanticExecutor: options.semanticExecutor,
+            retrievalExecutionSource: retrievalExecutionSource({
+                retrievalExecutor: Boolean(options.retrievalExecutor),
+                enginesAvailable: Boolean(engines),
+            }),
+            router: engines?.semanticRouter,
+        });
+        for (const [suite, identity] of Object.entries(suiteIdentity)) {
+            if ('status' in identity && identity.status === 'unavailable')
+                emitSemanticControlLog({
+                    event: 'benchmark.capability.unavailable',
+                    identity: { projectRoot, sessionId: 'benchmark:unified-runtime' },
+                    level: 'warn', timestamp: Date.now(),
+                    data: { suite, reasonCode: identity.reasonCode ?? 'capability_unavailable' },
+                });
+        }
+        const semantic = [];
+        const semanticStartedAt = performance.now();
+        for (const [index, item] of corpus.semantic.entries()) {
+            semantic.push(await executeSemanticCase(item, activation, options.semanticExecutor, engines?.semanticRouter));
+            if ((index + 1) % 10 === 0 || index + 1 === corpus.semantic.length)
+                options.observer?.({
+                    stage: 'semantic', completed: index + 1, total: corpus.semantic.length,
+                    elapsedMs: Math.round(performance.now() - semanticStartedAt),
+                });
+        }
+        const context = await runRetrievalCases(corpus.context, 'context', runtimeRetrievalExecutor, options.observer, semanticStartedAt);
+        const memory = await runRetrievalCases(corpus.memory, 'memory', runtimeRetrievalExecutor, options.observer, semanticStartedAt);
+        const knowledge = await runRetrievalCases(corpus.knowledge, 'knowledge', runtimeRetrievalExecutor, options.observer, semanticStartedAt);
+        const observation = {
+            schemaVersion: 'devflow.unified-runtime-observation.v4', capturedAt: Date.now(),
+            goldDatasetIds: corpus.datasetIds, runtime: { ...runtime }, suiteIdentity,
+            semantic, context, memory, knowledge,
+        };
+        emitSemanticControlLog({
+            event: 'benchmark.observation.captured', identity: { projectRoot, sessionId: 'benchmark:unified-runtime' },
+            level: 'info', timestamp: Date.now(), data: {
+                runtimeSourceHash: runtime.runtimeSourceHash, pluginRegistryHash: runtime.pluginRegistryHash,
+                semanticCount: semantic.length, contextCount: context.length, memoryCount: memory.length,
+                knowledgeCount: knowledge.length, latencyMs: Math.round(performance.now() - runStartedAt),
+            },
+        });
+        return observation;
+    }
+    finally {
+        closeIfPossible(engines?.memory);
+        closeIfPossible(engines?.knowledge);
+        closeIfPossible(engines?.telemetry);
+    }
+}
+async function executeSemanticCase(item, activation, executor, router) {
+    const startedAt = performance.now();
+    const context = executor ? await executor(item, activation) : await resolveTaskSemanticContext({
+        normalizedPrompt: item.prompt, identity: identityFor(item.id, item.slice.host), revision: 1, activation,
+        ...(router ? { router } : {}), ...(item.expected.targetAnchors ? { targetAnchors: item.expected.targetAnchors } : {}),
+        policy: { policyHash: 'benchmark:policy:v2', activeSkill: item.expected.activeSkill, budgets: { hookDeadlineMs: 3_000, contextTokens: 8_000, responseReserveTokens: 2_000, perChannelMs: { code: 1_500, memory: 800, knowledge: 1_500 } } },
+    }).then(result => result.context);
+    const semantic = context.facts;
+    const plan = item.expected.activeSkill ? planDomainAction({
+        identity: context.identity, taskRevisionHash: context.taskDefinitionHash, semanticContextHash: context.contextHash,
+        facts: semantic,
+        actionAuthority: context.actionAuthority,
+        objectKind: item.expected.domainAction?.tool === 'react_audit_performance'
+            ? 'performance' : item.expected.domainAction ? 'component' : undefined,
+        activeSkill: item.expected.activeSkill,
+        registry: createDomainToolRegistry(item.expected.activeSkill),
+        phase: 'final',
+    }) : undefined;
+    const fieldRefs = [
+        ...(semantic.intents.length > 0 ? semantic.intents.map(intent => `intent:${intent}`) : ['intent:unknown']),
+        ...(semantic.actions.length > 0 ? semantic.actions.map(action => `action:${action}`) : ['action:unknown']),
+        `memory:${semantic.memoryDirective}`,
+        ...(semantic.knowledgeNeeds.length > 0 ? semantic.knowledgeNeeds.map(need => `knowledge:${need.kind}`) : ['knowledge:none']),
+        `permission:${permissionRef(context.actionAuthority.mutation, semantic.conflicts.length > 0)}`,
+    ];
+    return {
+        caseId: item.id, datasetId: item.datasetId, runtimeContextHash: context.contextHash,
+        channelContextHashes: { context: context.contextHash, memory: context.contextHash, knowledge: context.contextHash },
+        fieldRefs, abstain: semantic.conflicts.length > 0,
+        mutationAuthorized: context.actionAuthority.mutation === 'authorized',
+        ...(plan ? { domainAction: {
+                ...(plan.selectedTool ? { tool: plan.selectedTool } : {}),
+                disposition: plan.status === 'selected' ? 'selected'
+                    : plan.status === 'needs_condition_evidence' ? 'conditional_proposal' : 'abstained',
+                reasonCode: plan.reasonCode,
+            } } : {}),
+        terminalFalseSuccess: false,
+        latencyMs: Math.max(0, Math.round((performance.now() - startedAt) * 1000) / 1000),
+    };
+}
+async function runRetrievalCases(cases, channel, executor, observer, runStartedAt = performance.now()) {
+    if (executor)
+        return Promise.all(cases.map(item => executor(item, channel)));
+    const observations = [];
+    const root = mkdtempSync(join(tmpdir(), `devflow-unified-${channel}-`));
+    const database = openGlobalDevFlowDatabase(join(root, 'home'), { busyTimeoutMs: 500 });
+    try {
+        for (const [index, item] of cases.entries()) {
+            observations.push(await runRetrievalCase(item, channel, root, database));
+            if ((index + 1) % 10 === 0 || index + 1 === cases.length)
+                observer?.({
+                    stage: channel, completed: index + 1, total: cases.length,
+                    elapsedMs: Math.round(performance.now() - runStartedAt),
+                });
+        }
+        return observations;
+    }
+    finally {
+        database.close();
+        rmSync(root, { recursive: true, force: true });
+    }
+}
+async function runRetrievalCase(item, channel, root, database) {
+    const startedAt = performance.now();
+    const identity = identityFor(item.id, item.slice.host, root);
+    const context = buildBenchmarkRetrievalContext({
+        caseId: item.id,
+        prompt: item.prompt,
+        identity,
+        channel,
+    });
+    const producer = fixtureProducer(channel, item);
+    const empty = fixtureProducer(channel === 'context' ? 'memory' : 'context', item);
+    const stage = stageCapture();
+    const code = captureProducer(channel === 'context' ? producer : empty, stage);
+    const memory = captureProducer(channel === 'memory' ? producer : empty, stage);
+    const knowledge = captureProducer(channel === 'knowledge' ? producer : empty, stage);
+    const coordinator = new RetrievalCoordinator({
+        code, memory, knowledge, sessions: new RetrievalSessionRepository(database), taskRuntime: noOpTaskRuntime(),
+    });
+    let result = await coordinator.retrieve({ context, taskRevision: 1, tokenBudget: 8_000 });
+    let attempts = 1;
+    while (result.status === 'abstain' && attempts < item.expected.maxCycles) {
+        const next = await coordinator.retrieve({ context, taskRevision: 1, tokenBudget: 8_000 });
+        attempts += 1;
+        if (next.cycles.length <= result.cycles.length)
+            break;
+        result = next;
+    }
+    const selectedEvidence = (result.receipt.selectedEvidence ?? [])
+        .filter(candidate => candidate.channel === evidenceChannel(channel));
+    const selected = refsForRetrievalChannel(selectedEvidence, channel);
+    const sourceVersion = selectedEvidence.find(candidate => typeof candidate.payload.version === 'string')?.payload.version;
+    const episodePersisted = item.expected.explicitSave ? await executeExplicitSave(root, item.prompt, identity) : undefined;
+    const stages = retrievalStages(stage.candidates, result.receipt.rejectedRefs, channel);
+    return {
+        caseId: item.id, datasetId: item.datasetId, channel, runtimeContextHash: context.contextHash,
+        planRequirement: context.plan[evidenceChannel(channel)].requirement,
+        executionStatus: 'executed',
+        ...(item.expected.reasonCode ? { executionReasonCode: item.expected.reasonCode } : {}),
+        ...stages,
+        rankedEvidenceIds: selected,
+        cycles: result.cycles.length, duplicateOrNoGainStopObserved: result.cycles.length < 2 || (result.cycles.at(-1)?.noGainCount ?? 0) > 0,
+        latencyMs: Math.max(0, Math.round((performance.now() - startedAt) * 1000) / 1000),
+        rerankerBaselineIds: stages.fusionBaselineIds,
+        attributionSource: 'none',
+        attributionOutcomes: selected.map(() => 'unknown'),
+        verificationOutcomes: selected.map(() => 'unknown'),
+        ...(sourceVersion ? { sourceVersion } : {}),
+        ...(item.expected.temporalCorrect !== undefined ? { temporalCorrect: item.slice.availability === 'available' && selected.length > 0 } : {}),
+        ...(episodePersisted !== undefined ? { episodePersisted } : {}), terminalFalseSuccess: false,
+    };
+}
+export function buildBenchmarkRetrievalContext(input) {
+    const sourceHash = semanticTextHash(input.prompt);
+    const facts = createSemanticFacts({
+        sourceHash,
+        intents: [input.channel === 'context' ? 'understand' : input.channel],
+        actions: ['analyze'],
+        entities: [], targets: [], constraints: [], prohibitions: [], conditions: [],
+        memoryDirective: input.channel === 'memory' ? 'recall' : 'none',
+        knowledgeNeeds: input.channel === 'knowledge' ? [{ kind: 'overview', required: true }] : [],
+        evidence: [], conflicts: [], confidence: { overall: 1 },
+    });
+    const dispositions = ['code', 'memory', 'knowledge'].map(target => ({
+        channel: target,
+        requirement: target === evidenceChannel(input.channel) ? 'required'
+            : input.exclusive ? 'prohibited' : 'optional',
+        reasonCode: target === evidenceChannel(input.channel) ? 'benchmark_channel_required'
+            : input.exclusive ? 'benchmark_channel_prohibited' : 'benchmark_channel_optional',
+        evidenceIds: [],
+    }));
+    const authorityBody = {
+        mutation: 'not_requested',
+        targetAnchors: [], conditionEvidenceIds: [], authorityEvidenceIds: [],
+        reasonCode: 'benchmark_read_only',
+    };
+    const actionAuthority = { ...authorityBody, authorityHash: stableRuntimeHash(authorityBody) };
+    const plan = compileChannelQueryPlan({
+        identity: input.identity,
+        facts,
+        dispositions,
+        claims: [],
+        query: input.prompt,
+        createdAt: 1,
+    });
+    const contextBody = {
+        schemaVersion: 'semantic-execution-context',
+        identity: input.identity, facts, dispositions, actionAuthority, plan,
+        taskDefinitionHash: `definition:${input.caseId}`,
+        activationSnapshotHash: `activation:${input.caseId}`, embeddingProfiles: {}, scopeBindings: { project: { kind: 'project', id: input.identity.projectId }, session: { kind: 'session', id: input.identity.sessionId } },
+        mode: 'authoritative', createdAt: 1,
+    };
+    return { ...contextBody, contextHash: stableRuntimeHash(contextBody) };
+}
+export function restrictBenchmarkRetrievalChannel(context, channel, query) {
+    const facts = channel === 'memory' && context.facts.memoryDirective !== 'recall'
+        ? createSemanticFacts({ ...context.facts, memoryDirective: 'recall' })
+        : channel === 'knowledge' && context.facts.knowledgeNeeds.length === 0
+            ? createSemanticFacts({ ...context.facts, knowledgeNeeds: [{ kind: 'overview', required: true }] })
+            : context.facts;
+    const dispositions = ['code', 'memory', 'knowledge'].map(target => ({
+        channel: target,
+        requirement: target === evidenceChannel(channel) ? 'required' : 'prohibited',
+        reasonCode: target === evidenceChannel(channel)
+            ? 'benchmark_channel_required' : 'benchmark_channel_prohibited',
+        evidenceIds: [],
+    }));
+    const deterministicPlan = compileChannelQueryPlan({
+        identity: context.identity,
+        facts,
+        dispositions,
+        claims: [],
+        query,
+        createdAt: context.createdAt,
+    });
+    const selectedChannel = evidenceChannel(channel);
+    const preservesSampledPlan = context.plan[selectedChannel].requirement !== 'prohibited';
+    const code = selectedChannel === 'code'
+        ? requireBenchmarkChannel(context.plan.code, deterministicPlan.code)
+        : deterministicPlan.code;
+    const memory = selectedChannel === 'memory'
+        ? requireBenchmarkChannel(context.plan.memory, deterministicPlan.memory)
+        : deterministicPlan.memory;
+    const knowledge = selectedChannel === 'knowledge'
+        ? requireBenchmarkChannel(context.plan.knowledge, deterministicPlan.knowledge)
+        : deterministicPlan.knowledge;
+    const planBody = {
+        identity: context.identity,
+        sourceFactsHash: facts.factsHash,
+        code,
+        memory,
+        knowledge,
+        queryProposalIds: [code, memory, knowledge].flatMap(item => item.requirement === 'prohibited' ? [] : item.additions.flatMap(variant => variant.proposalId ?? [])),
+        generatedBy: preservesSampledPlan
+            ? context.plan.generatedBy
+            : deterministicPlan.generatedBy,
+        createdAt: context.plan.createdAt,
+    };
+    const plan = { ...planBody, planHash: stableRuntimeHash(planBody) };
+    const body = { ...context, facts, dispositions, plan };
+    return { ...body, contextHash: stableRuntimeHash({
+            schemaVersion: body.schemaVersion,
+            identity: body.identity,
+            facts: body.facts,
+            dispositions: body.dispositions,
+            actionAuthority: body.actionAuthority,
+            plan: body.plan,
+            activationSnapshotHash: body.activationSnapshotHash,
+            embeddingProfiles: body.embeddingProfiles,
+            scopeBindings: body.scopeBindings,
+            mode: body.mode,
+            createdAt: body.createdAt,
+        }) };
+}
+function requireBenchmarkChannel(source, deterministic) {
+    if (source.requirement === 'prohibited')
+        return deterministic;
+    return {
+        ...source,
+        requirement: 'required',
+        reasonCode: 'benchmark_channel_required',
+        evidenceIds: [],
+    };
+}
+async function runRealRetrievalCase(item, channel, engines, projectRoot, activation) {
+    const root = mkdtempSync(join(tmpdir(), `devflow-unified-real-${channel}-`));
+    const database = openGlobalDevFlowDatabase(join(root, 'home'), { busyTimeoutMs: 500 });
+    const startedAt = performance.now();
+    try {
+        const identity = identityFor(item.id, item.slice.host, projectRoot);
+        const resolved = await resolveTaskSemanticContext({
+            normalizedPrompt: item.prompt,
+            identity,
+            revision: 1,
+            activation,
+            router: engines.semanticRouter,
+            policy: {
+                policyHash: 'benchmark:policy:v2',
+                budgets: {
+                    hookDeadlineMs: 3_000,
+                    contextTokens: 8_000,
+                    responseReserveTokens: 2_000,
+                    perChannelMs: { code: 1_500, memory: 800, knowledge: 1_500 },
+                },
+            },
+            targetAnchors: [],
+        });
+        const context = resolved.context;
+        const stage = stageCapture();
+        const coordinator = new RetrievalCoordinator({
+            code: captureProducer(engines.codeEvidence, stage),
+            memory: captureProducer(engines.memoryQuery, stage),
+            knowledge: captureProducer(engines.knowledgeQuery, stage),
+            sessions: new RetrievalSessionRepository(database),
+            taskRuntime: noOpTaskRuntime(),
+        });
+        let result = await coordinator.retrieve({ context, taskRevision: 1, tokenBudget: 8_000 });
+        let attempts = 1;
+        while (result.status === 'abstain' && attempts < item.expected.maxCycles) {
+            const next = await coordinator.retrieve({ context, taskRevision: 1, tokenBudget: 8_000 });
+            attempts += 1;
+            if (next.cycles.length <= result.cycles.length)
+                break;
+            result = next;
+        }
+        const selectedEvidence = (result.receipt.selectedEvidence ?? [])
+            .filter(candidate => candidate.channel === evidenceChannel(channel));
+        const selected = refsForRetrievalChannel(selectedEvidence, channel);
+        const sourceVersion = selectedEvidence.find(candidate => typeof candidate.payload.version === 'string')?.payload.version;
+        const episodePersisted = item.expected.explicitSave
+            ? await executeExplicitSave(root, item.prompt, identity)
+            : undefined;
+        const stages = retrievalStages(stage.candidates, result.receipt.rejectedRefs, channel);
+        const execution = retrievalExecution(context, result.receipt, channel);
+        return {
+            caseId: item.id, datasetId: item.datasetId, channel,
+            runtimeContextHash: context.contextHash,
+            ...execution,
+            ...stages,
+            rankedEvidenceIds: selected,
+            cycles: result.cycles.length,
+            duplicateOrNoGainStopObserved: result.cycles.length < 2 || (result.cycles.at(-1)?.noGainCount ?? 0) > 0,
+            latencyMs: Math.max(0, Math.round((performance.now() - startedAt) * 1000) / 1000),
+            rerankerBaselineIds: stages.fusionBaselineIds,
+            attributionSource: 'none',
+            attributionOutcomes: selected.map(() => 'unknown'),
+            verificationOutcomes: selected.map(() => 'unknown'),
+            ...(sourceVersion ? { sourceVersion } : {}),
+            ...(episodePersisted !== undefined ? { episodePersisted } : {}),
+            terminalFalseSuccess: false,
+        };
+    }
+    catch (error) {
+        return {
+            caseId: item.id, datasetId: item.datasetId, channel,
+            runtimeContextHash: semanticTextHash(item.prompt),
+            planRequirement: 'required',
+            executionStatus: 'failed', executionReasonCode: 'runtime_retrieval_exception',
+            candidateEvidenceIds: [],
+            eligibilityRejectedEvidenceIds: [], rankedEvidenceIds: [], rejectedEvidenceIds: [],
+            rejectionReasonCounts: {}, cycles: 1, duplicateOrNoGainStopObserved: true,
+            latencyMs: Math.max(0, Math.round((performance.now() - startedAt) * 1000) / 1000),
+            rerankerBaselineIds: [], attributionSource: 'none', attributionOutcomes: [],
+            verificationOutcomes: [], terminalFalseSuccess: false,
+        };
+    }
+    finally {
+        database.close();
+        rmSync(root, { recursive: true, force: true });
+    }
+}
+function retrievalExecution(context, receipt, channel) {
+    const target = evidenceChannel(channel);
+    if (context.plan[target].requirement === 'prohibited') {
+        return { planRequirement: 'prohibited', executionStatus: 'blocked', executionReasonCode: 'channel_prohibited' };
+    }
+    const originalLegs = (receipt.legs ?? [])
+        .filter(outcome => outcome.channel === target && outcome.queryVariant.kind === 'original');
+    if (originalLegs.length === 0) {
+        return { planRequirement: context.plan[target].requirement, executionStatus: 'failed', executionReasonCode: 'original_leg_missing' };
+    }
+    const status = originalLegs[0].status;
+    return {
+        planRequirement: context.plan[target].requirement,
+        executionStatus: status,
+        executionReasonCode: originalLegs.find(outcome => outcome.reasonCode)?.reasonCode
+            ?? (status === 'executed' ? undefined : `original_leg_${status}`),
+    };
+}
+const ELIGIBILITY_REJECTION_REASONS = new Set([
+    'producer_channel_mismatch', 'scope_mismatch', 'permission_denied', 'permission_unknown',
+    'source_not_ready', 'expired', 'authority_missing', 'authority_below_threshold',
+    'profile_unbound', 'profile_missing', 'profile_mismatch',
+]);
+export function stageCapture() {
+    return { candidates: [], outputs: [] };
+}
+export function captureProducer(producer, stage) {
+    return {
+        ...producer,
+        async search(context, request) {
+            const result = await producer.search(context, request);
+            stage.candidates.push(...result.candidates);
+            const diagnostic = producer.getLastVectorCandidateRefs?.();
+            stage.outputs.push({ result, ...(diagnostic ? { vectorCandidateRefs: diagnostic } : {}) });
+            return result;
+        },
+    };
+}
+export function retrievalStages(candidates, rejected, channel) {
+    const channelCandidates = candidates.filter(candidate => candidate.channel === evidenceChannel(channel));
+    const channelRefs = new Set(channelCandidates.map(candidate => candidate.stableRef));
+    const channelRejected = rejected.filter(item => channelRefs.has(item.ref));
+    const eligibilityRejected = new Set(channelRejected
+        .filter(item => ELIGIBILITY_REJECTION_REASONS.has(item.reason))
+        .map(item => item.ref));
+    const eligible = channelCandidates.filter(candidate => !eligibilityRejected.has(candidate.stableRef));
+    return {
+        candidateEvidenceIds: uniqueRefs(channelCandidates.map(candidate => candidate.stableRef)),
+        eligibilityRejectedEvidenceIds: [...eligibilityRejected].sort(),
+        rejectedEvidenceIds: uniqueRefs(channelRejected.map(item => item.ref)),
+        rejectionReasonCounts: channelRejected.reduce((counts, item) => {
+            counts[item.reason] = (counts[item.reason] ?? 0) + 1;
+            return counts;
+        }, {}),
+        fusionBaselineIds: reciprocalRankFusion(eligible).map(item => item.candidate.stableRef),
+    };
+}
+export function refsForRetrievalChannel(evidence, channel) {
+    const expected = evidenceChannel(channel);
+    return evidence.filter(item => item.channel === expected).map(item => item.stableRef);
+}
+export function rankedEvidenceRefs(receipt, channel) {
+    const expected = evidenceChannel(channel);
+    return [...(receipt.evidenceReceipts ?? [])]
+        .filter(item => item.channel === expected && item.stage === 'ranked' && Number.isSafeInteger(item.rank))
+        .sort((left, right) => left.rank - right.rank || left.stableRef.localeCompare(right.stableRef))
+        .map(item => item.stableRef)
+        .filter((ref, index, refs) => refs.indexOf(ref) === index);
+}
+function evidenceChannel(channel) {
+    return channel === 'context' ? 'code' : channel;
+}
+function uniqueRefs(refs) {
+    return [...new Set(refs)].sort();
+}
+function fixtureProducer(channel, item) {
+    return {
+        async search(context, request) {
+            const available = item.slice.availability === 'available' ? 'available' : 'unavailable';
+            const candidates = available === 'available' ? item.expected.evidenceIds.map((stableRef, index) => candidateFor(channel, stableRef, index, context, item.expected.sourceVersion)) : [];
+            const body = {
+                schemaVersion: 'evidence-receipt.v1',
+                receiptId: `${channel}:${stableRuntimeHash({ context: context.contextHash, request: request.signatureHash }).slice(0, 20)}`,
+                parentReceiptIds: [request.receiptId],
+                taskContextHash: context.contextHash,
+                factsHash: context.facts.factsHash,
+                planHash: context.plan.planHash,
+                activationSnapshotHash: context.activationSnapshotHash,
+                channel: channel === 'context' ? 'code' : channel,
+                availability: available,
+                selectedRefs: [],
+                rejectedRefs: [],
+                gaps: [],
+                createdAt: Date.now(),
+            };
+            return { availability: available, candidates, gaps: [], receipt: { ...body, receiptHash: stableRuntimeHash(body) } };
+        },
+    };
+}
+function candidateFor(channel, stableRef, index, context, sourceVersion) {
+    const actualChannel = channel === 'context' ? 'code' : channel;
+    const payload = channel === 'context' ? { path: stableRef.replace(/^code:/u, '') } : channel === 'knowledge' ? { id: stableRef, source: stableRef, title: stableRef, excerpt: stableRef, ...(sourceVersion ? { version: sourceVersion } : {}), applicability: ['kind:overview'] } : { id: stableRef, title: stableRef, content: stableRef, type: 'project_convention', applicability: ['fixture'] };
+    return {
+        stableRef, channel: actualChannel, strength: actualChannel === 'code' ? 'strong' : 'supporting', rank: index + 1, score: 1 - index * 0.01,
+        evidence: { ref: stableRef, sourceId: stableRef, sourceRevision: 'fixture.v2', producer: 'benchmark-fixture-producer', producerVersion: RUNNER_VERSION, contentHash: stableRuntimeHash(payload), parentRefs: [] },
+        payload, freshness: { current: true, checkedAt: Date.now() }, contributingLegs: [{ leg: `${actualChannel}:fixture`, rank: index + 1, score: 1 - index * 0.01 }],
+        eligibility: { scope: context.scopeBindings.project, permission: 'allowed', sourceReady: true }, estimatedTokens: 16,
+    };
+}
+export function noOpTaskRuntime() {
+    return { sessionOpened() { }, cycleCompleted() { } };
+}
+async function executeExplicitSave(root, prompt, identity) {
+    const projectIdentity = {
+        projectId: identity.projectId,
+        memoryNamespace: `memory:${identity.projectId}`,
+    };
+    const store = new MemoryStore(root, undefined, projectIdentity);
+    try {
+        const result = await new MemoryCommandService(root, store, projectIdentity).commitExplicitEpisode({
+            content: prompt,
+            actorId: 'actor:benchmark-runner',
+            sessionId: identity.sessionId,
+            turnId: identity.turnId,
+            sourceMessageId: identity.requestId,
+        });
+        return result.episode.episodeId.length > 0;
+    }
+    finally {
+        store.close();
+    }
+}
+export function identityFor(caseId, host, projectRoot = '/benchmark', projectId = 'project:benchmark-runtime') {
+    return { projectRoot, projectId, hostId: host, sessionId: `session:${caseId}`, turnId: `turn:${caseId}`, requestId: `request:${caseId}`, executionId: `execution:${caseId}` };
+}
+function permissionRef(value, _conflicted) {
+    return value;
+}
+export function runtimeIdentity(root, sourceRoot = resolve(import.meta.dirname, '../../..')) {
+    // Runtime identity belongs to the DevFlow distribution under test, while
+    // projectRoot may point at an external initialized host project.
+    const packageJson = JSON.parse(readFileSync(resolve(sourceRoot, 'packages/devflow/package.json'), 'utf8'));
+    const pluginRegistry = readFileSync(resolve(sourceRoot, 'packages/devflow/plugin-registry.json'), 'utf8');
+    const files = gitOutput(sourceRoot, ['ls-files', '-co', '--exclude-standard'])
+        .split('\n')
+        .filter(runtimeSourceFile)
+        .sort();
+    const sourceFiles = files.map(path => [
+        path,
+        createHash('sha256').update(readFileSync(resolve(sourceRoot, path))).digest('hex'),
+    ]);
+    return {
+        packageVersion: packageJson.version,
+        pluginRegistryHash: stableRuntimeHash(pluginRegistry),
+        runtimeSourceHash: stableRuntimeHash({ packageVersion: packageJson.version, sourceFiles }),
+        runnerVersion: RUNNER_VERSION,
+    };
+}
+function gitOutput(root, args) {
+    try {
+        return execFileSync('git', ['-C', root, ...args], {
+            encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 10_000,
+            maxBuffer: 16 * 1024 * 1024,
+        }).trim();
+    }
+    catch {
+        return '';
+    }
+}
+function runtimeSourceFile(path) {
+    if (path.includes('/dist/') || path.includes('/fixtures/') || path.includes('/__tests__/'))
+        return false;
+    return path === 'package.json'
+        || path === 'package-lock.json'
+        || path === 'packages/devflow/plugin-registry.json'
+        || /^apps\/[^/]+\/(?:package\.json|src\/)/u.test(path)
+        || /^packages\/[^/]+\/(?:package\.json|src\/)/u.test(path)
+        || path.startsWith('scripts/');
+}
+async function resolveSuiteIdentity(input) {
+    const semanticCapability = input.semanticExecutor
+        ? {
+            status: 'executed', resolver: 'host_sampling',
+            profileHash: stableRuntimeHash({ executor: 'injected', runnerVersion: input.runtime.runnerVersion }),
+        }
+        : input.router && await input.router.warmup()
+            ? {
+                status: 'executed', resolver: 'embedding',
+                profileHash: stableRuntimeHash({
+                    modelId: input.router.modelId,
+                    modelRevision: input.router.modelRevision,
+                    thresholdVersion: input.router.thresholdVersion,
+                }),
+            }
+            : { status: 'unavailable', resolver: 'embedding', reasonCode: 'embedding_unavailable' };
+    return {
+        deterministicSafety: {
+            runtimeSourceHash: input.runtime.runtimeSourceHash,
+            runnerVersion: input.runtime.runnerVersion,
+        },
+        semanticCapability,
+        retrievalExecution: {
+            status: 'executed', source: input.retrievalExecutionSource,
+            datasetIds: [input.corpus.datasetIds.context, input.corpus.datasetIds.memory, input.corpus.datasetIds.knowledge],
+        },
+        candidateGeneration: {
+            status: 'unavailable', source: 'runtime_index',
+            reasonCode: input.retrievalExecutionSource === 'real_project'
+                ? 'evaluation_bundle_runner_required' : 'fixture_seeded_only',
+        },
+        externalReplay: { status: 'unavailable', reasonCode: 'external_replay_not_provided' },
+    };
+}
+export function retrievalExecutionSource(input) {
+    return !input.retrievalExecutor && input.enginesAvailable ? 'real_project' : 'fixture_seeded';
+}
+function closeIfPossible(value) {
+    if (value && typeof value.close === 'function')
+        value.close();
+}
+//# sourceMappingURL=unified-runtime-runner.js.map