@dailephd/my-dev-kit-lab 0.4.8 → 0.5.0

package/dist/src/experiments/plugins/warmIndexReuse/plugin.js

@@ -0,0 +1,212 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { summarizeExperimentRun } from "../../results.js";
+import { defaultWarmIndexReuseConfig, validateWarmIndexReuseConfig, warmIndexReuseConfigDefinition, } from "./config.js";
+import { aggregateStatus, executeWarmIndexReuse, } from "./execution.js";
+import { buildWarmIndexExecutionArtifact, WARM_INDEX_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
+import { calculateWarmIndexMetrics, toRawOutcomeMetrics, toRunLevelMetrics, toWarmOutcomeMetrics, } from "./metrics.js";
+import { selectWarmIndexCases } from "./selection.js";
+import { evaluateWarmIndexFakeAgents, } from "./fakeAgentEvaluation.js";
+export const RAW_FULL_FILE_VARIANT_ID = "raw-full-file";
+export const WARM_INDEX_REUSE_VARIANT_ID = "warm-index-reuse";
+export const warmIndexReuseMetadata = {
+    id: "warm-index-reuse",
+    name: "Warm Index Reuse",
+    description: "Measure repeated task execution where one my-dev-kit index is built once per benchmark project and reused across multiple tasks, with a matched raw-full-file baseline per task.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "text", "plot", "artifact"],
+};
+const WARM_INDEX_VARIANTS = [
+    {
+        id: RAW_FULL_FILE_VARIANT_ID,
+        name: "Raw full file",
+        description: "Every file matched by the case's raw include globs, read once per task.",
+    },
+    {
+        id: WARM_INDEX_REUSE_VARIANT_ID,
+        name: "Warm index reuse",
+        description: "my-dev-kit retrieval against one index prepared once per benchmark project.",
+    },
+];
+export const warmIndexReusePlugin = {
+    metadata: warmIndexReuseMetadata,
+    defaultConfig: defaultWarmIndexReuseConfig,
+    configDefinition: warmIndexReuseConfigDefinition,
+    supportedVariants: WARM_INDEX_VARIANTS.map((variant) => variant.id),
+    validateConfig: validateWarmIndexReuseConfig,
+    async run(context) {
+        const startedAt = context.startedAt.toISOString();
+        const cases = selectWarmIndexCases(readCasesInput(context.inputs), context.config);
+        const projectProfiles = readProjectProfilesInput(context.inputs);
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const projects = await executeWarmIndexReuse({
+            cases,
+            kitCommand: context.config.kitCommand,
+            outputRoot: outDir,
+        });
+        // Fake-agent evaluation runs after execution evidence exists and never repeats it.
+        const agentEvidence = await evaluateWarmIndexFakeAgents({
+            projects,
+            cases,
+            projectProfiles,
+            outputRoot: outDir,
+            cwd: context.target.targetRoot,
+            env: readEnvInput(context.inputs),
+        });
+        const artifact = buildWarmIndexExecutionArtifact({
+            runId: context.runId,
+            pluginId: warmIndexReuseMetadata.id,
+            projects,
+        });
+        const artifactPath = resolveWithinRoot(outDir, WARM_INDEX_EXECUTION_ARTIFACT_FILE);
+        await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`, "utf8");
+        const run = mapWarmIndexExecutionToRun({
+            runId: context.runId,
+            startedAt,
+            completedAt: new Date().toISOString(),
+            target: context.target,
+            cases,
+            projects,
+            projectSummaries: artifact.projects,
+            agentEvidence,
+            artifactPath,
+        });
+        return run;
+    },
+    summarize(result) {
+        return result.summary ?? summarizeExperimentRun(result);
+    },
+};
+export function mapWarmIndexExecutionToRun(args) {
+    const titles = new Map(args.cases.map((evaluationCase) => [evaluationCase.id, evaluationCase.title]));
+    const warmIndexMetrics = calculateWarmIndexMetrics(args.projectSummaries, args.agentEvidence);
+    // projectSummaries (and therefore metrics) preserve project and task order of args.projects.
+    const experimentCases = args.projects.flatMap((project, projectIndex) => project.tasks.map((task, taskIndex) => {
+        const taskMetrics = warmIndexMetrics.projects[projectIndex].tasks[taskIndex];
+        const taskAgents = args.agentEvidence[projectIndex].tasks[taskIndex];
+        return {
+            id: task.caseId,
+            name: titles.get(task.caseId) ?? task.caseId,
+            outcomes: [
+                buildRawOutcome(project, task, taskMetrics, taskAgents),
+                buildWarmOutcome(project, task, taskMetrics, taskAgents),
+            ],
+            metadata: { benchmarkProject: task.benchmarkProject, sessionKey: project.projectSegment },
+        };
+    }));
+    const warnings = args.projects.flatMap((project) => project.warnings.map((message) => ({
+        code: "warm-index-setup-warning",
+        message,
+        details: { benchmarkProject: project.benchmarkProject },
+    })));
+    const run = {
+        runId: args.runId,
+        pluginId: warmIndexReuseMetadata.id,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status: aggregateStatus(experimentCases.flatMap((experimentCase) => experimentCase.outcomes.map((outcome) => outcome.status))),
+        target: args.target,
+        variants: WARM_INDEX_VARIANTS.map((variant) => ({ ...variant })),
+        cases: experimentCases,
+        metrics: toRunLevelMetrics(warmIndexMetrics),
+        artifacts: [
+            {
+                id: "warm-index-execution",
+                label: "Warm index execution evidence",
+                path: args.artifactPath,
+                kind: "artifact",
+                mimeType: "application/json",
+                description: "Per-project index setup and per-task raw/warm execution summaries without context text.",
+            },
+        ],
+        warnings,
+        failures: [],
+        metadata: { executionArtifactPath: args.artifactPath },
+        projectExecutions: args.projectSummaries,
+        agentEvidence: args.agentEvidence,
+        warmIndexMetrics,
+    };
+    run.summary = summarizeExperimentRun(run);
+    return run;
+}
+function outcomeMetadata(project, task, agent) {
+    return {
+        benchmarkProject: task.benchmarkProject,
+        sessionKey: project.projectSegment,
+        warmSessionAvailable: project.session !== undefined,
+        taskStatus: task.status,
+        // Agent evaluation status is reported separately; it does not change the execution status.
+        agentStatus: agent ? agent.status : "not-run",
+    };
+}
+function agentWarnings(agent, variantId, caseId) {
+    if (!agent)
+        return [];
+    return agent.errors.map((message) => ({
+        code: "fake-agent-evaluation-error",
+        message,
+        variantId,
+        caseId,
+    }));
+}
+function buildRawOutcome(project, task, taskMetrics, taskAgents) {
+    return {
+        id: `${task.caseId}:${RAW_FULL_FILE_VARIANT_ID}`,
+        caseId: task.caseId,
+        variantId: RAW_FULL_FILE_VARIANT_ID,
+        status: task.rawStatus,
+        metrics: toRawOutcomeMetrics(taskMetrics, RAW_FULL_FILE_VARIANT_ID),
+        artifacts: [],
+        warnings: agentWarnings(taskAgents.raw, RAW_FULL_FILE_VARIANT_ID, task.caseId),
+        failures: toFailures(task, "raw", RAW_FULL_FILE_VARIANT_ID),
+        metadata: outcomeMetadata(project, task, taskAgents.raw),
+    };
+}
+function buildWarmOutcome(project, task, taskMetrics, taskAgents) {
+    return {
+        id: `${task.caseId}:${WARM_INDEX_REUSE_VARIANT_ID}`,
+        caseId: task.caseId,
+        variantId: WARM_INDEX_REUSE_VARIANT_ID,
+        status: task.warmStatus,
+        metrics: toWarmOutcomeMetrics(taskMetrics, WARM_INDEX_REUSE_VARIANT_ID),
+        artifacts: [],
+        warnings: [
+            ...task.warnings.map((message) => ({
+                code: "warm-retrieval-warning",
+                message,
+                variantId: WARM_INDEX_REUSE_VARIANT_ID,
+                caseId: task.caseId,
+            })),
+            ...agentWarnings(taskAgents.warm, WARM_INDEX_REUSE_VARIANT_ID, task.caseId),
+        ],
+        failures: toFailures(task, "warm", WARM_INDEX_REUSE_VARIANT_ID),
+        metadata: outcomeMetadata(project, task, taskAgents.warm),
+    };
+}
+function toFailures(task, side, variantId) {
+    return task.errors
+        .filter((error) => error.side === side)
+        .map((error) => ({ code: error.code, message: error.message, variantId, caseId: task.caseId, recoverable: true }));
+}
+function readProjectProfilesInput(inputs) {
+    const value = inputs?.projectProfiles;
+    if (!Array.isArray(value)) {
+        throw new Error("Warm index reuse requires projectProfiles input for fake-agent evaluation.");
+    }
+    return value;
+}
+function readEnvInput(inputs) {
+    const env = inputs?.env;
+    return env && typeof env === "object" && !Array.isArray(env) ? env : undefined;
+}
+function readCasesInput(inputs) {
+    const value = inputs?.cases;
+    if (!Array.isArray(value)) {
+        throw new Error("Warm index reuse requires cases input.");
+    }
+    return value;
+}

package/dist/src/experiments/plugins/warmIndexReuse/selection.js

@@ -0,0 +1,85 @@
+import path from "node:path";
+import { sanitizePathSegment } from "../../outputPaths.js";
+/**
+ * Keeps source case order, applies case-ID then benchmark-project filters, and fails clearly for
+ * unknown requested IDs or an empty selection.
+ */
+export function selectWarmIndexCases(cases, filters) {
+    const knownCaseIds = new Set(cases.map((evaluationCase) => evaluationCase.id));
+    const missingCases = (filters.caseIds ?? []).filter((caseId) => !knownCaseIds.has(caseId));
+    if (missingCases.length > 0) {
+        throw new Error(`Evaluation case not found: ${missingCases.join(", ")}`);
+    }
+    const knownProjects = new Set(cases.map((evaluationCase) => evaluationCase.benchmarkProject));
+    const missingProjects = (filters.benchmarkProjects ?? []).filter((project) => !knownProjects.has(project));
+    if (missingProjects.length > 0) {
+        throw new Error(`Benchmark project not found: ${missingProjects.join(", ")}`);
+    }
+    const selected = cases
+        .filter((evaluationCase) => !filters.caseIds?.length || filters.caseIds.includes(evaluationCase.id))
+        .filter((evaluationCase) => !filters.benchmarkProjects?.length || filters.benchmarkProjects.includes(evaluationCase.benchmarkProject));
+    if (selected.length === 0) {
+        throw new Error(`No evaluation cases matched the requested filters (cases: ${(filters.caseIds ?? []).join(", ") || "any"}; ` +
+            `benchmark projects: ${(filters.benchmarkProjects ?? []).join(", ") || "any"}).`);
+    }
+    return selected;
+}
+/**
+ * Groups cases by benchmarkProject in first-seen order, keeping task order within a group. Cases
+ * that share a project but disagree on target root or ordered source roots make the group
+ * structurally invalid rather than silently picking one configuration.
+ */
+export function groupWarmIndexCases(cases) {
+    const groups = new Map();
+    for (const evaluationCase of cases) {
+        let group = groups.get(evaluationCase.benchmarkProject);
+        if (!group) {
+            group = {
+                benchmarkProject: evaluationCase.benchmarkProject,
+                projectSegment: safeOutputSegment(evaluationCase.benchmarkProject, "benchmark project"),
+                targetRoot: evaluationCase.absoluteTargetRoot,
+                sourceRoots: Object.freeze([...evaluationCase.sourceRoots]),
+                cases: [],
+                structuralErrors: [],
+            };
+            groups.set(evaluationCase.benchmarkProject, group);
+        }
+        else {
+            if (path.resolve(group.targetRoot) !== path.resolve(evaluationCase.absoluteTargetRoot)) {
+                group.structuralErrors.push(`Case ${evaluationCase.id} target root ${evaluationCase.absoluteTargetRoot} differs from ${group.targetRoot} for benchmark project ${group.benchmarkProject}.`);
+            }
+            if (evaluationCase.sourceRoots.join("\u0000") !== group.sourceRoots.join("\u0000")) {
+                group.structuralErrors.push(`Case ${evaluationCase.id} source roots [${evaluationCase.sourceRoots.join(", ")}] differ from [${group.sourceRoots.join(", ")}] for benchmark project ${group.benchmarkProject}.`);
+            }
+        }
+        group.cases.push(evaluationCase);
+    }
+    const result = [...groups.values()];
+    assertUniqueSegments(result.map((group) => [group.benchmarkProject, group.projectSegment]), "benchmark project");
+    for (const group of result) {
+        assertUniqueSegments(group.cases.map((evaluationCase) => [evaluationCase.id, taskOutputSegment(evaluationCase.id)]), "case");
+    }
+    return result;
+}
+export function taskOutputSegment(caseId) {
+    return safeOutputSegment(caseId, "case");
+}
+// Reuses the shared experiment path-segment sanitizer and additionally rejects dot-only segments,
+// which the sanitizer preserves and which would resolve outside the intended directory.
+function safeOutputSegment(value, label) {
+    const segment = sanitizePathSegment(value);
+    if (/^\.+$/.test(segment)) {
+        throw new Error(`Unsafe ${label} ID for an output path segment: ${JSON.stringify(value)}`);
+    }
+    return segment;
+}
+function assertUniqueSegments(entries, label) {
+    const owners = new Map();
+    for (const [id, segment] of entries) {
+        const owner = owners.get(segment);
+        if (owner !== undefined && owner !== id) {
+            throw new Error(`The ${label} IDs ${owner} and ${id} map to the same output path segment "${segment}".`);
+        }
+        owners.set(segment, id);
+    }
+}

package/dist/src/experiments/plugins/warmIndexReuse/warmIndexSession.js

@@ -0,0 +1,43 @@
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { buildMyDevKitIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+/**
+ * Builds the index once through the shared `buildMyDevKitIndex` owner. A build that reports
+ * success but leaves no index directory is not treated as a prepared session.
+ */
+export async function prepareWarmIndexSession(options) {
+    const build = await buildMyDevKitIndex(options);
+    if (!build.ok) {
+        return { ok: false, warnings: [...build.warnings], build };
+    }
+    if (!existsSync(build.indexDir)) {
+        const message = `my-dev-kit index reported success but index directory is missing: ${build.indexDir}`;
+        if (options.requireKit) {
+            throw new Error(message);
+        }
+        return { ok: false, warnings: [...build.warnings, message], build };
+    }
+    const session = Object.freeze({
+        indexDir: build.indexDir,
+        buildDurationMs: build.durationMs,
+        targetRoot: options.target.absoluteTargetRoot,
+        sourceRoots: Object.freeze([...options.target.sourceRoots]),
+        buildCommand: build.command
+    });
+    return { ok: true, session, build };
+}
+function normalizeRoot(root) {
+    const resolved = path.resolve(root);
+    return process.platform === "win32" ? resolved.toLowerCase() : resolved;
+}
+/** Rejects reuse of a session against a different target root or source-root configuration. */
+export function assertWarmIndexSessionMatchesTarget(session, target) {
+    if (normalizeRoot(session.targetRoot) !== normalizeRoot(target.absoluteTargetRoot)) {
+        throw new Error(`Warm index session target root ${session.targetRoot} does not match ${target.absoluteTargetRoot}.`);
+    }
+    const sameSourceRoots = session.sourceRoots.length === target.sourceRoots.length &&
+        session.sourceRoots.every((sourceRoot, index) => sourceRoot === target.sourceRoots[index]);
+    if (!sameSourceRoots) {
+        throw new Error(`Warm index session source roots [${session.sourceRoots.join(", ")}] do not match [${target.sourceRoots.join(", ")}].`);
+    }
+}

package/dist/src/plots/buildWarmIndexPlotData.js

@@ -0,0 +1,112 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "../report/experiments/warmIndexReuseReportModel.js";
+// ---------------------------------------------------------------------------
+// Warm-index plot data. Maps metrics already calculated by the warm-index
+// metric owner (as carried by the plugin report) to plot points; unavailable
+// metrics become skipped points with their reason. No formula is recalculated.
+// ---------------------------------------------------------------------------
+export const WARM_INDEX_PLOT_IDS = [
+    "warm-index-amortized-index-cost",
+    "warm-index-context-size",
+    "warm-index-correctness",
+    "warm-index-cumulative-token-usage",
+];
+const WARM_INDEX_PLUGIN_ID = "warm-index-reuse";
+const VARIANTS = ["raw-full-file", "warm-index-reuse"];
+/**
+ * Returns the warm-index report section when `<experimentDir>/report.json` belongs to the
+ * warm-index-reuse plugin, or null for any other directory. A warm report that is malformed
+ * fails instead of falling back to legacy controlled-experiment parsing.
+ */
+export async function readWarmIndexPlotSource(experimentDir) {
+    const reportPath = path.join(experimentDir, "report.json");
+    if (!existsSync(reportPath))
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(await readFile(reportPath, "utf8"));
+    }
+    catch {
+        return null;
+    }
+    const report = parsed?.report;
+    if (report?.plugin?.id !== WARM_INDEX_PLUGIN_ID)
+        return null;
+    const section = report.warmIndexReuse;
+    const invalid = (detail) => new Error(`Invalid warm-index-reuse report for plots (${reportPath}): ${detail}`);
+    if (!section || typeof section !== "object")
+        throw invalid("missing warmIndexReuse section.");
+    if (section.schemaVersion !== WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION) {
+        throw invalid(`unsupported schema version ${String(section.schemaVersion)}.`);
+    }
+    if (!Array.isArray(section.projects))
+        throw invalid("projects must be an array.");
+    for (const project of section.projects) {
+        if (!project || typeof project.benchmarkProject !== "string" || !Array.isArray(project.tasks)) {
+            throw invalid("each project needs a benchmarkProject and a tasks array.");
+        }
+        for (const task of project.tasks) {
+            if (!task || typeof task.taskOrdinal !== "number" || !task.raw || !task.warm) {
+                throw invalid("each task needs a taskOrdinal and raw/warm metrics.");
+            }
+        }
+    }
+    return section;
+}
+export function buildWarmIndexPlotData(args) {
+    const skippedPoints = [];
+    const series = (id, title, yLabel) => ({
+        id,
+        title,
+        xLabel: "Task ordinal",
+        yLabel,
+        kind: "scatter",
+        points: [],
+        warnings: [],
+    });
+    const amortized = series("warm-index-amortized-index-cost", "Amortized index build cost", "Amortized index build duration (ms)");
+    const contextSize = series("warm-index-context-size", "Raw vs retrieved context size", "Estimated context tokens");
+    const correctness = series("warm-index-correctness", "Fake-agent correctness by strategy", "Fake-agent correctness score");
+    const cumulativeTokens = series("warm-index-cumulative-token-usage", "Cumulative fake-agent token usage", "Cumulative fake-agent total tokens (simulated)");
+    const add = (plot, metric, group, label, task) => {
+        if (metric.availability === "available" && metric.value !== null && Number.isFinite(metric.value)) {
+            const point = { x: task.taskOrdinal, y: metric.value, group, label, metadata: { caseId: task.caseId } };
+            plot.points.push(point);
+        }
+        else {
+            skippedPoints.push({ plotId: plot.id, label, reason: metric.reason ?? `Metric is ${metric.availability}.` });
+        }
+    };
+    for (const project of args.section.projects) {
+        for (const task of project.tasks) {
+            const taskLabel = `${project.benchmarkProject} task ${task.taskOrdinal} (${task.caseId})`;
+            add(amortized, task.warm.amortizedIndexBuildDurationMs, project.benchmarkProject, taskLabel, task);
+            for (const variantId of VARIANTS) {
+                const side = variantId === "raw-full-file" ? task.raw : task.warm;
+                const group = `${project.benchmarkProject} / ${variantId}`;
+                const label = `${taskLabel} ${variantId}`;
+                add(contextSize, side.contextEstimatedTokens, group, label, task);
+                add(correctness, side.agentCorrectness, group, label, task);
+                // Agent token totals only; estimated context tokens are never used as a fallback.
+                add(cumulativeTokens, side.cumulativeAgentTotalTokens, group, label, task);
+            }
+        }
+    }
+    const plots = [amortized, contextSize, correctness, cumulativeTokens];
+    const warnings = [];
+    for (const plot of plots) {
+        if (plot.points.length === 0) {
+            plot.warnings.push("No comparable data available.");
+            warnings.push(`${plot.title}: no comparable data available.`);
+        }
+    }
+    return {
+        generatedAt: args.generatedAt ?? new Date().toISOString(),
+        sourceExperimentDir: args.experimentDir,
+        plots,
+        skippedPoints,
+        warnings,
+    };
+}

package/dist/src/plots/index.js

@@ -1,4 +1,5 @@
 export * from "./types.js";
 export * from "./buildExperimentPlotData.js";
+export * from "./buildWarmIndexPlotData.js";
 export * from "./renderSvgChart.js";
 export * from "./writePlotArtifacts.js";

package/dist/src/plots/writePlotArtifacts.js

@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { resolveWithinRoot } from "../core/pathSafety.js";
 import { buildExperimentPlotData } from "./buildExperimentPlotData.js";
+import { buildWarmIndexPlotData, readWarmIndexPlotSource } from "./buildWarmIndexPlotData.js";
 import { renderSvgChart } from "./renderSvgChart.js";
 const chartFiles = {
     "token-savings-vs-prompt-length": "token-savings-vs-prompt-length.svg",
@@ -12,7 +13,13 @@ const chartFiles = {
     "run-outcomes-by-agent": "run-outcomes-by-agent.svg"
 };
 export async function writePlotArtifacts(options) {
-    const data = await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
+    // A warm-index-reuse plugin output directory is detected by its plugin report; every other
+    // directory keeps the legacy controlled-experiment plot path unchanged.
+    const experimentDir = path.resolve(options.repoRoot ?? process.cwd(), options.experimentDir);
+    const warmSection = await readWarmIndexPlotSource(experimentDir);
+    const data = warmSection
+        ? buildWarmIndexPlotData({ section: warmSection, experimentDir })
+        : await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
     return writePlotArtifactsFromData({ data, outDir: options.outDir });
 }
 export async function writePlotArtifactsFromData(options) {

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
+import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -9,6 +10,7 @@ export function buildPluginExperimentReport(args) {
     const outputRoot = args.outputRoot ?? readString(args.run.metadata?.outputRoot) ?? null;
     const allOutcomes = args.run.cases.flatMap((experimentCase) => experimentCase.outcomes);
     const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
+    const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -36,8 +38,9 @@ export function buildPluginExperimentReport(args) {
         failures: args.run.failures,
         skippedOutcomes: allOutcomes.filter((outcome) => outcome.status === "skipped"),
         findings: buildFindings(args.run),
+        warmIndexReuse,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse),
         rawRun,
     };
 }
@@ -94,7 +97,20 @@ function buildFindings(run) {
         ...skipped,
     ];
 }
-function buildInterpretation(run, contextStrategyComparisonV043) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse) {
+    if (run.pluginId === "warm-index-reuse" && warmIndexReuse) {
+        const summary = warmIndexReuse.summary;
+        return {
+            summary: `The warm-index run prepared ${summary.preparedSessionProjectCount} of ${summary.projectCount} project indexes and evaluated ${summary.taskCount} tasks. ` +
+                "The report separates one-time index-build duration from per-task retrieval duration and shows how the fixed build cost is amortized across repeated tasks. " +
+                `Deterministic fake-agent correctness is available for ${summary.agentCorrectnessAvailableCount} of ${summary.agentSideCount} task sides and fake-agent token totals for ${summary.agentTotalTokensAvailableCount}; ` +
+                "these are simulated harness evidence, not real-model or provider measurements. " +
+                "Estimated context-token values are context-size estimates, not provider token usage.",
+            recommendedNextStep: run.status === "completed"
+                ? "Review per-task measurements and cumulative component costs for each project; the variants are reported side by side and are not ranked."
+                : "Inspect unavailable metrics, warnings, and failures before drawing conclusions from this run.",
+        };
+    }
     if (run.pluginId === "context-strategy-comparison" && (contextStrategyComparisonV043?.summary.strategyCount ?? 0) > 0) {
         return {
             summary: `Stage-context evidence was recorded for ${contextStrategyComparisonV043.summary.strategyCount} strategy executions. Review each strategy independently; this report does not calculate a composite ranking or winning strategy.`,

package/dist/src/report/experiments/buildWarmIndexReuseReport.js

@@ -0,0 +1,91 @@
+import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "./warmIndexReuseReportModel.js";
+const WARM_INDEX_REUSE_PLUGIN_ID = "warm-index-reuse";
+const COST_MODEL = [
+    "Index construction is a one-time cost per benchmark project; task retrieval is the repeated warm cost.",
+    "Cold start: for the first task in a project, the cumulative warm component duration is the one-time index build plus the first retrieval.",
+    "Warm reuse: later tasks do not rebuild the index; their warm operation duration is retrieval only.",
+    "Amortized index build duration after N tasks is the index build duration divided by N.",
+    "Raw cumulative duration sums one raw-context construction per task; warm cumulative component duration includes one index build plus each warm retrieval once.",
+    "Cumulative durations are sums of measured components, not wall-clock or end-to-end latency, and are not presented as a speedup.",
+];
+const LIMITATIONS = [
+    "Context estimated-token counts use the existing character-based estimate and are not provider billing telemetry.",
+    "Agent evidence comes from the deterministic fake agent only; real agents such as Codex or Claude are not run by warm-index-reuse in v0.5.0.",
+    "Fake-agent correctness is deterministic benchmark-harness evidence from the existing answer-key scorer, not a measure of real-model correctness.",
+    "Fake-agent token totals are simulated harness telemetry, not provider billing telemetry; estimated context tokens are a separate character-based context-size estimate.",
+    "Component-duration sums are not equivalent to provider end-to-end latency.",
+    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
+    "No composite score, winner, ranking, or break-even task is calculated.",
+];
+/**
+ * Presents the warm-index metrics already calculated on the run. It performs no amortization,
+ * cumulative, or availability calculation of its own.
+ */
+export function buildWarmIndexReuseReport(run) {
+    if (run.pluginId !== WARM_INDEX_REUSE_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const executions = candidate.projectExecutions ?? [];
+    const metricProjects = candidate.warmIndexMetrics?.projects ?? [];
+    const agentProjects = candidate.agentEvidence;
+    if (executions.length !== metricProjects.length) {
+        throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
+    }
+    const projects = executions.map((execution, projectIndex) => {
+        const metrics = metricProjects[projectIndex];
+        if (metrics.benchmarkProject !== execution.benchmarkProject ||
+            metrics.tasks.length !== execution.tasks.length ||
+            metrics.tasks.some((task, taskIndex) => task.caseId !== execution.tasks[taskIndex].caseId)) {
+            throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
+        }
+        return {
+            benchmarkProject: execution.benchmarkProject,
+            sessionKey: execution.sessionKey,
+            status: execution.status,
+            sessionPrepared: metrics.sessionPrepared,
+            indexBuildDurationMs: metrics.indexBuildDurationMs,
+            taskCount: metrics.tasks.length,
+            tasks: metrics.tasks.map((task, taskIndex) => {
+                const agents = agentProjects?.[projectIndex]?.tasks[taskIndex];
+                return {
+                    taskOrdinal: task.taskOrdinal,
+                    caseId: task.caseId,
+                    rawStatus: execution.tasks[taskIndex].rawStatus,
+                    warmStatus: execution.tasks[taskIndex].warmStatus,
+                    raw: task.raw,
+                    warm: task.warm,
+                    rawAgent: toReportAgent(agents?.raw ?? null),
+                    warmAgent: toReportAgent(agents?.warm ?? null),
+                };
+            }),
+        };
+    });
+    const sides = projects.flatMap((project) => project.tasks.flatMap((task) => [task.raw, task.warm]));
+    return {
+        schemaVersion: WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION,
+        summary: {
+            projectCount: projects.length,
+            taskCount: projects.reduce((sum, project) => sum + project.taskCount, 0),
+            preparedSessionProjectCount: projects.filter((project) => project.sessionPrepared).length,
+            incompleteProjectCount: projects.filter((project) => project.status !== "completed").length,
+            agentSideCount: sides.length,
+            agentCorrectnessAvailableCount: sides.filter((side) => side.agentCorrectness.availability === "available").length,
+            agentTotalTokensAvailableCount: sides.filter((side) => side.agentTotalTokens.availability === "available").length,
+        },
+        costModel: [...COST_MODEL],
+        limitations: [...LIMITATIONS],
+        projects,
+    };
+}
+function toReportAgent(agent) {
+    if (!agent)
+        return null;
+    return {
+        status: agent.status,
+        passed: agent.correctness.passed,
+        tokenUsageSource: agent.tokenUsage.source,
+        tokenUsageReliability: agent.tokenUsage.reliability,
+        warnings: [...agent.warnings],
+        errors: [...agent.errors],
+    };
+}

package/dist/src/report/experiments/index.js

@@ -6,6 +6,9 @@ export * from "./contextStrategyComparisonV043ReportModel.js";
 export { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
 export { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
 export { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
+export * from "./warmIndexReuseReportModel.js";
+export { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
+export { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 export * from "./contextIntegrityReportModel.js";
 export { buildContextIntegrityReport } from "./buildContextIntegrityReport.js";
 export { renderContextIntegrityJsonReport } from "./renderContextIntegrityJsonReport.js";

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -1,4 +1,5 @@
 import { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
+import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     return `<!DOCTYPE html>
 <html lang="en">
@@ -96,6 +97,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextStrategyComparisonV043Html(report.contextStrategyComparisonV043)}
 
+  ${renderWarmIndexReuseHtml(report.warmIndexReuse)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [