@dailephd/my-dev-kit-lab 0.4.8 → 0.5.0

package/dist/src/experiments/plugins/warmIndexReuse/execution.js

@@ -0,0 +1,158 @@
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { groupWarmIndexCases, taskOutputSegment } from "./selection.js";
+import { assertWarmIndexSessionMatchesTarget, prepareWarmIndexSession, } from "./warmIndexSession.js";
+/**
+ * Runs the warm-index lifecycle for already selected cases: one index setup attempt per valid
+ * benchmark-project group, then per task one raw baseline and one retrieval against that group's
+ * session. Never indexes per task and never retries a failed index.
+ */
+export async function executeWarmIndexReuse(options) {
+    const projects = [];
+    for (const group of groupWarmIndexCases(options.cases)) {
+        const indexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment));
+        const commandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        const project = {
+            benchmarkProject: group.benchmarkProject,
+            projectSegment: group.projectSegment,
+            targetRoot: group.targetRoot,
+            sourceRoots: group.sourceRoots,
+            indexDir,
+            tasks: [],
+            status: "skipped",
+            warnings: [],
+            errors: [...group.structuralErrors],
+        };
+        let setupFailure;
+        if (group.structuralErrors.length > 0) {
+            setupFailure = {
+                side: "warm",
+                code: "warm-index-group-inconsistent",
+                message: group.structuralErrors.join(" "),
+            };
+        }
+        else {
+            try {
+                // requireKit: false keeps the measured index command as evidence when setup fails.
+                const prepared = await prepareWarmIndexSession({
+                    target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                    kitCommand: options.kitCommand,
+                    indexDir,
+                    commandsDir: path.join(commandsDir, "index"),
+                    requireKit: false,
+                });
+                project.build = prepared.build;
+                if (prepared.ok) {
+                    project.session = prepared.session;
+                }
+                else {
+                    project.warnings.push(...prepared.warnings);
+                    const message = prepared.warnings.join(" ") || "my-dev-kit index setup failed.";
+                    project.errors.push(message);
+                    setupFailure = { side: "warm", code: "warm-index-setup-failed", message };
+                }
+            }
+            catch (error) {
+                const message = errorMessage(error);
+                project.errors.push(message);
+                setupFailure = { side: "warm", code: "warm-index-setup-failed", message };
+            }
+        }
+        for (const evaluationCase of group.cases) {
+            project.tasks.push(await executeTask({
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                session: project.session,
+                setupFailure,
+                commandsDir: path.join(commandsDir, taskOutputSegment(evaluationCase.id)),
+            }));
+        }
+        project.status = aggregateStatus(project.tasks.map((task) => task.status));
+        projects.push(project);
+    }
+    return projects;
+}
+async function executeTask(args) {
+    const { evaluationCase } = args;
+    const errors = [];
+    const warnings = [];
+    let rawBaseline;
+    let rawStatus;
+    try {
+        rawBaseline = await runRawFullFileBaseline(evaluationCase);
+        rawStatus = "completed";
+    }
+    catch (error) {
+        rawStatus = "failed";
+        errors.push({ side: "raw", code: "raw-baseline-failed", message: errorMessage(error) });
+    }
+    let warmRetrieval;
+    let warmStatus = "failed";
+    if (!args.session) {
+        errors.push(args.setupFailure ?? { side: "warm", code: "warm-index-setup-failed", message: "No warm index session." });
+    }
+    else {
+        try {
+            assertWarmIndexSessionMatchesTarget(args.session, evaluationCase);
+        }
+        catch (error) {
+            errors.push({ side: "warm", code: "warm-session-target-mismatch", message: errorMessage(error) });
+        }
+        if (!errors.some((error) => error.side === "warm")) {
+            try {
+                // requireKit: false retains subordinate command evidence; a skipped result caused by a
+                // failed command is classified as failed below rather than as an honest skip.
+                warmRetrieval = await runMyDevKitRetrievalFromIndex({
+                    evaluationCase,
+                    kitCommand: args.kitCommand,
+                    indexDir: args.session.indexDir,
+                    commandsDir: args.commandsDir,
+                    requireKit: false,
+                });
+                warnings.push(...warmRetrieval.warnings);
+                warmStatus = classifyWarmRetrieval(warmRetrieval);
+                if (warmStatus === "failed") {
+                    errors.push({
+                        side: "warm",
+                        code: "warm-retrieval-failed",
+                        message: warmRetrieval.warnings.join(" ") || "my-dev-kit retrieval command failed.",
+                    });
+                }
+            }
+            catch (error) {
+                errors.push({ side: "warm", code: "warm-retrieval-failed", message: errorMessage(error) });
+            }
+        }
+    }
+    return {
+        caseId: evaluationCase.id,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        status: aggregateStatus([rawStatus, warmStatus]),
+        rawStatus,
+        warmStatus,
+        rawBaseline,
+        warmRetrieval,
+        warnings,
+        errors,
+    };
+}
+function classifyWarmRetrieval(result) {
+    if (!result.skipped)
+        return "completed";
+    return result.commands.some((command) => !command.ok) ? "failed" : "skipped";
+}
+/** Same aggregation rule as the generic per-case summary: uniform statuses win, mixed is partial. */
+export function aggregateStatus(statuses) {
+    if (statuses.length === 0)
+        return "skipped";
+    for (const status of ["completed", "skipped", "failed"]) {
+        if (statuses.every((value) => value === status))
+            return status;
+    }
+    return "partial";
+}
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}

package/dist/src/experiments/plugins/warmIndexReuse/executionArtifact.js

@@ -0,0 +1,77 @@
+export const WARM_INDEX_EXECUTION_ARTIFACT_FILE = "warm-index-execution.json";
+export const WARM_INDEX_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-execution-v1";
+export function summarizeCommand(command) {
+    return {
+        commandId: command.commandId,
+        ok: command.ok,
+        exitCode: command.exitCode,
+        durationMs: command.durationMs,
+        stdoutPath: command.stdoutPath,
+        stderrPath: command.stderrPath,
+        telemetryPath: command.telemetryPath,
+        error: command.error ?? null,
+    };
+}
+function summarizeTask(task) {
+    const raw = task.rawBaseline;
+    const warm = task.warmRetrieval;
+    return {
+        caseId: task.caseId,
+        status: task.status,
+        rawStatus: task.rawStatus,
+        warmStatus: task.warmStatus,
+        rawBaseline: raw
+            ? {
+                targetRoot: raw.targetRoot,
+                filesIncluded: [...raw.filesIncluded],
+                totalFiles: raw.totalFiles,
+                totalChars: raw.totalChars,
+                totalEstimatedTokens: raw.totalEstimatedTokens,
+                tokenCountMethod: raw.tokenCountMethod,
+                durationMs: raw.durationMs,
+            }
+            : null,
+        warmRetrieval: warm
+            ? {
+                skipped: warm.skipped,
+                warnings: [...warm.warnings],
+                totalChars: warm.totalChars,
+                totalEstimatedTokens: warm.totalEstimatedTokens,
+                tokenCountMethod: warm.tokenCountMethod,
+                filesRead: [...warm.filesRead],
+                selectedNodeId: warm.selectedNodeId ?? null,
+                selectedFile: warm.selectedFile ?? null,
+                selectedSymbol: warm.selectedSymbol ?? null,
+                durationMs: warm.durationMs,
+                commands: warm.commands.map(summarizeCommand),
+            }
+            : null,
+        warnings: [...task.warnings],
+        errors: task.errors.map((error) => ({ ...error })),
+    };
+}
+/** Bounded, context-free project summary used for both the persisted artifact and the run record. */
+export function summarizeProjectExecution(project) {
+    return {
+        benchmarkProject: project.benchmarkProject,
+        sessionKey: project.projectSegment,
+        status: project.status,
+        targetRoot: project.targetRoot,
+        sourceRoots: [...project.sourceRoots],
+        indexDir: project.indexDir,
+        sessionPrepared: project.session !== undefined,
+        buildDurationMs: project.build ? project.build.durationMs : null,
+        indexCommand: project.build ? summarizeCommand(project.build.command) : null,
+        tasks: project.tasks.map(summarizeTask),
+        warnings: [...project.warnings],
+        errors: [...project.errors],
+    };
+}
+export function buildWarmIndexExecutionArtifact(args) {
+    return {
+        schemaVersion: WARM_INDEX_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        projects: args.projects.map(summarizeProjectExecution),
+    };
+}

package/dist/src/experiments/plugins/warmIndexReuse/fakeAgentEvaluation.js

@@ -0,0 +1,118 @@
+import path from "node:path";
+import { runAgentPrompt } from "../../../agents/index.js";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
+import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
+import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
+import { generatePromptVariants } from "../../../prompts/index.js";
+import { taskOutputSegment } from "./selection.js";
+// Classified statuses for which the scorer evaluated an actual agent answer.
+const SCOREABLE_STATUSES = new Set(["completed", "invalid-output"]);
+const SIDE_STRATEGIES = {
+    "raw-full-file": "raw-full-file",
+    "warm-index-reuse": "my-dev-kit-guided",
+};
+export async function evaluateWarmIndexFakeAgents(options) {
+    const casesById = new Map(options.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase]));
+    const result = [];
+    for (const project of options.projects) {
+        const tasks = [];
+        for (const task of project.tasks) {
+            const evaluationCase = casesById.get(task.caseId);
+            const run = (variantId) => evaluateSide({
+                evaluationCase,
+                caseId: task.caseId,
+                variantId,
+                projectProfiles: options.projectProfiles,
+                outDir: resolveWithinRoot(options.outputRoot, path.join("agents", project.projectSegment, taskOutputSegment(task.caseId), variantId)),
+                cwd: options.cwd,
+                env: options.env,
+            });
+            tasks.push({
+                caseId: task.caseId,
+                raw: task.rawBaseline ? await run("raw-full-file") : null,
+                warm: task.warmRetrieval ? await run("warm-index-reuse") : null,
+            });
+        }
+        result.push({ benchmarkProject: project.benchmarkProject, tasks });
+    }
+    return result;
+}
+async function evaluateSide(args) {
+    const promptStrategy = SIDE_STRATEGIES[args.variantId];
+    const base = { variantId: args.variantId, agentId: "fake-agent", promptStrategy };
+    try {
+        if (!args.evaluationCase) {
+            throw new Error(`Evaluation case ${args.caseId} is not available for fake-agent evaluation.`);
+        }
+        const [promptVariant] = generatePromptVariants({
+            cases: [args.evaluationCase],
+            projectProfiles: [...args.projectProfiles],
+            strategies: [promptStrategy],
+            complexityLevels: ["short"],
+        });
+        if (!promptVariant) {
+            throw new Error(`Failed to generate a fake-agent prompt for case ${args.caseId}.`);
+        }
+        const agentRunResult = await runAgentPrompt({
+            runId: `${args.caseId}.fake-agent.${args.variantId}`,
+            agentId: "fake-agent",
+            promptVariant,
+            promptText: promptVariant.promptText,
+            cwd: args.cwd,
+            outDir: args.outDir,
+            env: args.env,
+        });
+        const parsedAnswer = parseAgentAnswer({
+            text: agentRunResult.finalAnswerText,
+            answerKey: promptVariant.expectedAnswerKey,
+            tokenUsage: agentRunResult.tokenUsage,
+        });
+        const classification = classifyAgentRunOutcome({ agentRunResult, parsedAnswer });
+        const correctness = scoreCorrectness({
+            caseId: args.caseId,
+            answerKey: promptVariant.expectedAnswerKey,
+            parsedAnswer,
+            status: classification.status,
+        });
+        const scoreable = SCOREABLE_STATUSES.has(classification.status) && Number.isFinite(correctness.correctnessScore);
+        const totalTokens = agentRunResult.tokenUsage.totalTokens;
+        return {
+            ...base,
+            status: classification.status,
+            correctness: {
+                available: scoreable,
+                score: scoreable ? correctness.correctnessScore : null,
+                passed: scoreable ? correctness.passed : null,
+                failureReasons: [...correctness.failureReasons],
+            },
+            tokenUsage: {
+                totalTokens: typeof totalTokens === "number" && Number.isFinite(totalTokens) ? totalTokens : null,
+                source: agentRunResult.tokenUsageSource,
+                reliability: agentRunResult.tokenUsageReliability,
+            },
+            durationMs: agentRunResult.durationMs,
+            warnings: [...classification.warnings],
+            errors: [...classification.errors],
+            artifactPaths: {
+                promptPath: path.join(args.outDir, "prompt.txt"),
+                agentRunResultPath: path.join(args.outDir, "agent-run-result.json"),
+                stdoutPath: agentRunResult.stdoutPath,
+                stderrPath: agentRunResult.stderrPath,
+                telemetryPath: agentRunResult.telemetryPath,
+            },
+        };
+    }
+    catch (error) {
+        return {
+            ...base,
+            status: "failed",
+            correctness: { available: false, score: null, passed: null, failureReasons: [] },
+            tokenUsage: { totalTokens: null, source: "unavailable", reliability: "unavailable" },
+            durationMs: null,
+            warnings: [],
+            errors: [error instanceof Error ? error.message : String(error)],
+            artifactPaths: {},
+        };
+    }
+}

package/dist/src/experiments/plugins/warmIndexReuse/index.js

@@ -0,0 +1,8 @@
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./fakeAgentEvaluation.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./selection.js";
+export * from "./warmIndexSession.js";

package/dist/src/experiments/plugins/warmIndexReuse/metrics.js

@@ -0,0 +1,257 @@
+// ---------------------------------------------------------------------------
+// Warm-index metric owner. Every warm-index formula (amortization, cumulative
+// prefix sums, availability) is calculated here exactly once, as a pure
+// transform over the bounded Prompt 2 execution summaries. Reports render
+// these results; they never recalculate them.
+// ---------------------------------------------------------------------------
+export const WARM_INDEX_METRICS_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-metrics-v1";
+const NO_AGENT_EVIDENCE_REASON = "No fake-agent evaluation evidence was supplied for this task.";
+const AGENT_NOT_RUN_REASON = "The fake agent was not run because this side produced no context evidence.";
+export function availableMetric(value, unit, source, tokenCountMethod = null) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return unavailableMetric(unit, source, `Invalid input: expected a finite number but received ${String(value)}.`, tokenCountMethod);
+    }
+    return { availability: "available", value, unit, source, reason: null, tokenCountMethod };
+}
+export function unavailableMetric(unit, source, reason, tokenCountMethod = null) {
+    return { availability: "unavailable", value: null, unit, source, reason: requireReason(reason), tokenCountMethod };
+}
+export function notApplicableMetric(unit, source, reason) {
+    return { availability: "not-applicable", value: null, unit, source, reason: requireReason(reason), tokenCountMethod: null };
+}
+function requireReason(reason) {
+    if (!reason.trim()) {
+        throw new Error("Unavailable and not-applicable warm-index metrics require a reason.");
+    }
+    return reason;
+}
+/** Strict-prefix running sum: once one ordinal lacks evidence, every later ordinal stays unavailable. */
+class PrefixSum {
+    unit;
+    source;
+    total;
+    brokenReason;
+    method = null;
+    constructor(unit, source, start) {
+        this.unit = unit;
+        this.source = source;
+        this.total = "value" in start ? start.value : 0;
+        this.brokenReason = "unavailableReason" in start ? start.unavailableReason : null;
+    }
+    add(direct, label) {
+        if (this.brokenReason === null) {
+            if (direct.availability !== "available" || direct.value === null) {
+                this.brokenReason = `${label} evidence is missing or invalid; the cumulative prefix is incomplete from this task onward.`;
+            }
+            else if (this.method !== null && direct.tokenCountMethod !== this.method) {
+                this.brokenReason = `${label} uses token-count method ${String(direct.tokenCountMethod)} but earlier tasks used ${this.method}; mixed methods are not summed.`;
+            }
+            else {
+                this.method = direct.tokenCountMethod;
+                this.total += direct.value;
+            }
+        }
+        return this.brokenReason === null
+            ? availableMetric(this.total, this.unit, this.source, this.method)
+            : unavailableMetric(this.unit, this.source, this.brokenReason, this.method);
+    }
+}
+function indexBuildMetric(project) {
+    if (project.buildDurationMs === null) {
+        return unavailableMetric("ms", "measured", project.errors.length > 0
+            ? "No index setup was attempted because the project group was structurally inconsistent."
+            : "No index build evidence was recorded for this project.");
+    }
+    // A failed build is still a measured command duration; it is only excluded from warm costs below.
+    return availableMetric(project.buildDurationMs, "ms", "measured");
+}
+/**
+ * Fake-agent correctness and total tokens come only from bounded fake-agent evidence. Missing
+ * token telemetry stays unavailable; estimated context tokens are never substituted.
+ */
+function agentSideMetrics(evidence) {
+    if (evidence === undefined || evidence === null) {
+        const reason = evidence === undefined ? NO_AGENT_EVIDENCE_REASON : AGENT_NOT_RUN_REASON;
+        return {
+            agentCorrectness: unavailableMetric("score", "agent", reason),
+            agentTotalTokens: unavailableMetric("tokens", "agent", reason),
+        };
+    }
+    const correctnessScore = evidence.correctness.score;
+    const agentCorrectness = evidence.correctness.available && correctnessScore !== null
+        ? availableMetric(correctnessScore, "score", "agent")
+        : unavailableMetric("score", "agent", `Fake-agent output was not scoreable (status ${evidence.status})${evidence.errors[0] ? `: ${evidence.errors[0]}` : "."}`);
+    const agentTotalTokens = evidence.tokenUsage.totalTokens !== null
+        ? availableMetric(evidence.tokenUsage.totalTokens, "tokens", "agent")
+        : unavailableMetric("tokens", "agent", `The fake agent did not report total tokens (token usage source ${evidence.tokenUsage.source}).`);
+    return { agentCorrectness, agentTotalTokens };
+}
+function assertAgentEvidenceAligned(project, evidence) {
+    if (!evidence)
+        return;
+    const aligned = evidence.benchmarkProject === project.benchmarkProject &&
+        evidence.tasks.length === project.tasks.length &&
+        evidence.tasks.every((task, index) => task.caseId === project.tasks[index].caseId);
+    if (!aligned) {
+        throw new Error(`Fake-agent evidence is not aligned with the execution summary for ${project.benchmarkProject}.`);
+    }
+}
+export function calculateWarmIndexProjectMetrics(project, agentEvidence) {
+    assertAgentEvidenceAligned(project, agentEvidence);
+    const indexBuildDurationMs = indexBuildMetric(project);
+    const buildUsable = project.sessionPrepared && indexBuildDurationMs.availability === "available";
+    const noSessionReason = project.sessionPrepared
+        ? "Index build duration is unavailable, so warm index cost cannot be calculated."
+        : "No valid warm index session was prepared for this project, so the index build cost is not charged to warm tasks.";
+    const rawDuration = new PrefixSum("ms", "measured", { value: 0 });
+    const rawTokens = new PrefixSum("estimated-tokens", "estimated-chars-div-4", { value: 0 });
+    // The one-time build is charged exactly once, as the starting value of the warm prefix.
+    const warmDuration = new PrefixSum("ms", "derived", buildUsable ? { value: indexBuildDurationMs.value } : { unavailableReason: noSessionReason });
+    const warmTokens = new PrefixSum("estimated-tokens", "estimated-chars-div-4", { value: 0 });
+    // Fake-agent token totals accumulate separately from estimated context tokens.
+    const rawAgentTokens = new PrefixSum("tokens", "agent", { value: 0 });
+    const warmAgentTokens = new PrefixSum("tokens", "agent", { value: 0 });
+    const tasks = project.tasks.map((task, index) => {
+        const taskOrdinal = index + 1;
+        const label = (side) => `Task ${taskOrdinal} (${task.caseId}) ${side}`;
+        const raw = rawDirect(task);
+        const warm = warmDirect(task);
+        const taskAgentEvidence = agentEvidence?.tasks[index];
+        const rawAgent = agentSideMetrics(agentEvidence ? taskAgentEvidence?.raw ?? null : undefined);
+        const warmAgent = agentSideMetrics(agentEvidence ? taskAgentEvidence?.warm ?? null : undefined);
+        return {
+            caseId: task.caseId,
+            taskOrdinal,
+            raw: {
+                ...raw,
+                cumulativeDurationMs: rawDuration.add(raw.operationDurationMs, label("raw duration")),
+                cumulativeEstimatedContextTokens: rawTokens.add(raw.contextEstimatedTokens, label("raw estimated context tokens")),
+                ...rawAgent,
+                cumulativeAgentTotalTokens: rawAgentTokens.add(rawAgent.agentTotalTokens, label("raw fake-agent total tokens")),
+            },
+            warm: {
+                ...warm,
+                // Amortized build cost after N tasks = build duration / N.
+                amortizedIndexBuildDurationMs: buildUsable
+                    ? availableMetric(indexBuildDurationMs.value / taskOrdinal, "ms", "derived")
+                    : unavailableMetric("ms", "derived", noSessionReason),
+                cumulativeComponentDurationMs: warmDuration.add(warm.retrievalDurationMs, label("warm retrieval duration")),
+                cumulativeEstimatedContextTokens: warmTokens.add(warm.contextEstimatedTokens, label("warm estimated context tokens")),
+                ...warmAgent,
+                cumulativeAgentTotalTokens: warmAgentTokens.add(warmAgent.agentTotalTokens, label("warm fake-agent total tokens")),
+            },
+        };
+    });
+    return {
+        benchmarkProject: project.benchmarkProject,
+        sessionKey: project.sessionKey,
+        sessionPrepared: project.sessionPrepared,
+        indexBuildDurationMs,
+        tasks,
+    };
+}
+function rawDirect(task) {
+    const raw = task.rawBaseline;
+    if (!raw) {
+        const reason = `No raw baseline evidence was produced (raw status ${task.rawStatus}).`;
+        return {
+            contextCharacters: unavailableMetric("characters", "measured", reason),
+            contextEstimatedTokens: unavailableMetric("estimated-tokens", "estimated-chars-div-4", reason),
+            operationDurationMs: unavailableMetric("ms", "measured", reason),
+        };
+    }
+    return {
+        contextCharacters: availableMetric(raw.totalChars, "characters", "measured"),
+        contextEstimatedTokens: availableMetric(raw.totalEstimatedTokens, "estimated-tokens", "estimated-chars-div-4", raw.tokenCountMethod),
+        operationDurationMs: availableMetric(raw.durationMs, "ms", "measured"),
+    };
+}
+function warmDirect(task) {
+    const warm = task.warmRetrieval;
+    if (!warm) {
+        const reason = `No warm retrieval was run for this task (warm status ${task.warmStatus}).`;
+        return {
+            contextCharacters: unavailableMetric("characters", "measured", reason),
+            contextEstimatedTokens: unavailableMetric("estimated-tokens", "estimated-chars-div-4", reason),
+            retrievalDurationMs: unavailableMetric("ms", "measured", reason),
+        };
+    }
+    // A skipped retrieval still carries measured values; its skipped status stays on the outcome.
+    return {
+        contextCharacters: availableMetric(warm.totalChars, "characters", "measured"),
+        contextEstimatedTokens: availableMetric(warm.totalEstimatedTokens, "estimated-tokens", "estimated-chars-div-4", warm.tokenCountMethod),
+        retrievalDurationMs: availableMetric(warm.durationMs, "ms", "measured"),
+    };
+}
+/** Pure, order-preserving calculation; ordinals and cumulative state restart for every project. */
+export function calculateWarmIndexMetrics(projects, agentEvidence) {
+    if (agentEvidence && agentEvidence.length !== projects.length) {
+        throw new Error("Fake-agent evidence is not aligned with the execution summaries.");
+    }
+    return {
+        schemaVersion: WARM_INDEX_METRICS_SCHEMA_VERSION,
+        projects: projects.map((project, index) => calculateWarmIndexProjectMetrics(project, agentEvidence?.[index])),
+    };
+}
+/** Generic outcome metrics: only available numeric values are emitted, never placeholder zeros. */
+export function toRawOutcomeMetrics(task, variantId) {
+    return toExperimentMetrics(task.caseId, variantId, [
+        { id: "context-character-count", name: "Context characters", description: "Characters in the raw full-file context.", metric: task.raw.contextCharacters },
+        { id: "context-estimated-token-count", name: "Estimated context tokens", description: "Character-based estimate of raw context tokens; not provider token usage.", metric: task.raw.contextEstimatedTokens },
+        { id: "operation-duration-ms", name: "Operation duration", description: "Measured raw full-file context construction duration.", metric: task.raw.operationDurationMs },
+        { id: "cumulative-component-duration-ms", name: "Cumulative component duration", description: "Sum of raw context construction durations for tasks 1..N in this project.", metric: task.raw.cumulativeDurationMs },
+        { id: "cumulative-context-estimated-token-count", name: "Cumulative estimated context tokens", description: "Sum of raw estimated context tokens for tasks 1..N in this project; not provider token usage.", metric: task.raw.cumulativeEstimatedContextTokens },
+        ...agentSpecs(task.raw),
+    ]);
+}
+export function toWarmOutcomeMetrics(task, variantId) {
+    return toExperimentMetrics(task.caseId, variantId, [
+        { id: "context-character-count", name: "Context characters", description: "Characters in the retrieved my-dev-kit context.", metric: task.warm.contextCharacters },
+        { id: "context-estimated-token-count", name: "Estimated context tokens", description: "Character-based estimate of retrieved context tokens; not provider token usage.", metric: task.warm.contextEstimatedTokens },
+        { id: "operation-duration-ms", name: "Operation duration", description: "Measured retrieval-only duration (search, lookup, slice, source); excludes index build.", metric: task.warm.retrievalDurationMs },
+        { id: "cumulative-component-duration-ms", name: "Cumulative component duration", description: "One index build plus retrieval durations for tasks 1..N in this project; a component sum, not wall-clock latency.", metric: task.warm.cumulativeComponentDurationMs },
+        { id: "cumulative-context-estimated-token-count", name: "Cumulative estimated context tokens", description: "Sum of retrieved estimated context tokens for tasks 1..N in this project; not provider token usage.", metric: task.warm.cumulativeEstimatedContextTokens },
+        { id: "amortized-index-build-duration-ms", name: "Amortized index build duration", description: "Index build duration divided by this task's ordinal within its project.", metric: task.warm.amortizedIndexBuildDurationMs },
+        ...agentSpecs(task.warm),
+    ]);
+}
+function agentSpecs(side) {
+    return [
+        { id: "agent-correctness-score", name: "Fake-agent correctness score", description: "Deterministic fake-agent correctness from the existing benchmark answer-key scorer; not real-model correctness.", metric: side.agentCorrectness },
+        { id: "agent-total-tokens", name: "Fake-agent total tokens", description: "Simulated fake-agent harness token total; not provider billing telemetry.", metric: side.agentTotalTokens },
+        { id: "cumulative-agent-total-tokens", name: "Cumulative fake-agent total tokens", description: "Sum of simulated fake-agent token totals for tasks 1..N in this project; not provider billing telemetry.", metric: side.cumulativeAgentTotalTokens },
+    ];
+}
+function toExperimentMetrics(caseId, variantId, specs) {
+    return specs
+        .filter((spec) => spec.metric.availability === "available" && spec.metric.value !== null)
+        .map((spec) => ({
+        id: spec.id,
+        name: spec.name,
+        value: spec.metric.value,
+        unit: spec.metric.unit,
+        description: spec.description,
+        variantId,
+        caseId,
+    }));
+}
+/** Run-level metrics are limited to direct counts. */
+export function toRunLevelMetrics(metrics) {
+    return [
+        { id: "warm-index-project-count", name: "Warm-index project count", value: metrics.projects.length, unit: "count", description: "Benchmark-project groups in this run." },
+        {
+            id: "warm-index-task-count",
+            name: "Warm-index task count",
+            value: metrics.projects.reduce((sum, project) => sum + project.tasks.length, 0),
+            unit: "count",
+            description: "Tasks evaluated across all benchmark-project groups.",
+        },
+        {
+            id: "warm-index-session-prepared-project-count",
+            name: "Prepared warm-index session count",
+            value: metrics.projects.filter((project) => project.sessionPrepared).length,
+            unit: "count",
+            description: "Benchmark-project groups for which exactly one warm index session was prepared.",
+        },
+    ];
+}