@dailephd/my-dev-kit-lab 0.4.9 → 0.5.1

package/dist/src/plots/buildWarmIndexPlotData.js

@@ -0,0 +1,112 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "../report/experiments/warmIndexReuseReportModel.js";
+// ---------------------------------------------------------------------------
+// Warm-index plot data. Maps metrics already calculated by the warm-index
+// metric owner (as carried by the plugin report) to plot points; unavailable
+// metrics become skipped points with their reason. No formula is recalculated.
+// ---------------------------------------------------------------------------
+export const WARM_INDEX_PLOT_IDS = [
+    "warm-index-amortized-index-cost",
+    "warm-index-context-size",
+    "warm-index-correctness",
+    "warm-index-cumulative-token-usage",
+];
+const WARM_INDEX_PLUGIN_ID = "warm-index-reuse";
+const VARIANTS = ["raw-full-file", "warm-index-reuse"];
+/**
+ * Returns the warm-index report section when `<experimentDir>/report.json` belongs to the
+ * warm-index-reuse plugin, or null for any other directory. A warm report that is malformed
+ * fails instead of falling back to legacy controlled-experiment parsing.
+ */
+export async function readWarmIndexPlotSource(experimentDir) {
+    const reportPath = path.join(experimentDir, "report.json");
+    if (!existsSync(reportPath))
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(await readFile(reportPath, "utf8"));
+    }
+    catch {
+        return null;
+    }
+    const report = parsed?.report;
+    if (report?.plugin?.id !== WARM_INDEX_PLUGIN_ID)
+        return null;
+    const section = report.warmIndexReuse;
+    const invalid = (detail) => new Error(`Invalid warm-index-reuse report for plots (${reportPath}): ${detail}`);
+    if (!section || typeof section !== "object")
+        throw invalid("missing warmIndexReuse section.");
+    if (section.schemaVersion !== WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION) {
+        throw invalid(`unsupported schema version ${String(section.schemaVersion)}.`);
+    }
+    if (!Array.isArray(section.projects))
+        throw invalid("projects must be an array.");
+    for (const project of section.projects) {
+        if (!project || typeof project.benchmarkProject !== "string" || !Array.isArray(project.tasks)) {
+            throw invalid("each project needs a benchmarkProject and a tasks array.");
+        }
+        for (const task of project.tasks) {
+            if (!task || typeof task.taskOrdinal !== "number" || !task.raw || !task.warm) {
+                throw invalid("each task needs a taskOrdinal and raw/warm metrics.");
+            }
+        }
+    }
+    return section;
+}
+export function buildWarmIndexPlotData(args) {
+    const skippedPoints = [];
+    const series = (id, title, yLabel) => ({
+        id,
+        title,
+        xLabel: "Task ordinal",
+        yLabel,
+        kind: "scatter",
+        points: [],
+        warnings: [],
+    });
+    const amortized = series("warm-index-amortized-index-cost", "Amortized index build cost", "Amortized index build duration (ms)");
+    const contextSize = series("warm-index-context-size", "Raw vs retrieved context size", "Estimated context tokens");
+    const correctness = series("warm-index-correctness", "Fake-agent correctness by strategy", "Fake-agent correctness score");
+    const cumulativeTokens = series("warm-index-cumulative-token-usage", "Cumulative fake-agent token usage", "Cumulative fake-agent total tokens (simulated)");
+    const add = (plot, metric, group, label, task) => {
+        if (metric.availability === "available" && metric.value !== null && Number.isFinite(metric.value)) {
+            const point = { x: task.taskOrdinal, y: metric.value, group, label, metadata: { caseId: task.caseId } };
+            plot.points.push(point);
+        }
+        else {
+            skippedPoints.push({ plotId: plot.id, label, reason: metric.reason ?? `Metric is ${metric.availability}.` });
+        }
+    };
+    for (const project of args.section.projects) {
+        for (const task of project.tasks) {
+            const taskLabel = `${project.benchmarkProject} task ${task.taskOrdinal} (${task.caseId})`;
+            add(amortized, task.warm.amortizedIndexBuildDurationMs, project.benchmarkProject, taskLabel, task);
+            for (const variantId of VARIANTS) {
+                const side = variantId === "raw-full-file" ? task.raw : task.warm;
+                const group = `${project.benchmarkProject} / ${variantId}`;
+                const label = `${taskLabel} ${variantId}`;
+                add(contextSize, side.contextEstimatedTokens, group, label, task);
+                add(correctness, side.agentCorrectness, group, label, task);
+                // Agent token totals only; estimated context tokens are never used as a fallback.
+                add(cumulativeTokens, side.cumulativeAgentTotalTokens, group, label, task);
+            }
+        }
+    }
+    const plots = [amortized, contextSize, correctness, cumulativeTokens];
+    const warnings = [];
+    for (const plot of plots) {
+        if (plot.points.length === 0) {
+            plot.warnings.push("No comparable data available.");
+            warnings.push(`${plot.title}: no comparable data available.`);
+        }
+    }
+    return {
+        generatedAt: args.generatedAt ?? new Date().toISOString(),
+        sourceExperimentDir: args.experimentDir,
+        plots,
+        skippedPoints,
+        warnings,
+    };
+}

package/dist/src/plots/index.js

@@ -1,4 +1,5 @@
 export * from "./types.js";
 export * from "./buildExperimentPlotData.js";
+export * from "./buildWarmIndexPlotData.js";
 export * from "./renderSvgChart.js";
 export * from "./writePlotArtifacts.js";

package/dist/src/plots/writePlotArtifacts.js

@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { resolveWithinRoot } from "../core/pathSafety.js";
 import { buildExperimentPlotData } from "./buildExperimentPlotData.js";
+import { buildWarmIndexPlotData, readWarmIndexPlotSource } from "./buildWarmIndexPlotData.js";
 import { renderSvgChart } from "./renderSvgChart.js";
 const chartFiles = {
     "token-savings-vs-prompt-length": "token-savings-vs-prompt-length.svg",
@@ -12,7 +13,13 @@ const chartFiles = {
     "run-outcomes-by-agent": "run-outcomes-by-agent.svg"
 };
 export async function writePlotArtifacts(options) {
-    const data = await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
+    // A warm-index-reuse plugin output directory is detected by its plugin report; every other
+    // directory keeps the legacy controlled-experiment plot path unchanged.
+    const experimentDir = path.resolve(options.repoRoot ?? process.cwd(), options.experimentDir);
+    const warmSection = await readWarmIndexPlotSource(experimentDir);
+    const data = warmSection
+        ? buildWarmIndexPlotData({ section: warmSection, experimentDir })
+        : await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
     return writePlotArtifactsFromData({ data, outDir: options.outDir });
 }
 export async function writePlotArtifactsFromData(options) {

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -1,5 +1,6 @@
 import path from "node:path";
 import { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
+import { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",
@@ -9,6 +10,7 @@ export function buildPluginExperimentReport(args) {
     const outputRoot = args.outputRoot ?? readString(args.run.metadata?.outputRoot) ?? null;
     const allOutcomes = args.run.cases.flatMap((experimentCase) => experimentCase.outcomes);
     const contextStrategyComparisonV043 = buildContextStrategyComparisonV043Report(args.run);
+    const warmIndexReuse = buildWarmIndexReuseReport(args.run);
     const rawRun = { ...args.run, artifacts: relativizeArtifacts(args.run.artifacts, outputRoot) };
     for (const key of V043_BULK_ARRAY_KEYS) {
         delete rawRun[key];
@@ -36,8 +38,9 @@ export function buildPluginExperimentReport(args) {
         failures: args.run.failures,
         skippedOutcomes: allOutcomes.filter((outcome) => outcome.status === "skipped"),
         findings: buildFindings(args.run),
+        warmIndexReuse,
         contextStrategyComparisonV043,
-        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043),
+        interpretation: buildInterpretation(args.run, contextStrategyComparisonV043, warmIndexReuse),
         rawRun,
     };
 }
@@ -94,7 +97,20 @@ function buildFindings(run) {
         ...skipped,
     ];
 }
-function buildInterpretation(run, contextStrategyComparisonV043) {
+function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse) {
+    if (run.pluginId === "warm-index-reuse" && warmIndexReuse) {
+        const summary = warmIndexReuse.summary;
+        return {
+            summary: `The warm-index run prepared ${summary.preparedSessionProjectCount} of ${summary.projectCount} project indexes and evaluated ${summary.taskCount} tasks. ` +
+                "The report separates one-time index-build duration from per-task retrieval duration and shows how the fixed build cost is amortized across repeated tasks. " +
+                `Deterministic fake-agent correctness is available for ${summary.agentCorrectnessAvailableCount} of ${summary.agentSideCount} task sides and fake-agent token totals for ${summary.agentTotalTokensAvailableCount}; ` +
+                "these are simulated harness evidence, not real-model or provider measurements. " +
+                "Estimated context-token values are context-size estimates, not provider token usage.",
+            recommendedNextStep: run.status === "completed"
+                ? "Review per-task measurements and cumulative component costs for each project; the variants are reported side by side and are not ranked."
+                : "Inspect unavailable metrics, warnings, and failures before drawing conclusions from this run.",
+        };
+    }
     if (run.pluginId === "context-strategy-comparison" && (contextStrategyComparisonV043?.summary.strategyCount ?? 0) > 0) {
         return {
             summary: `Stage-context evidence was recorded for ${contextStrategyComparisonV043.summary.strategyCount} strategy executions. Review each strategy independently; this report does not calculate a composite ranking or winning strategy.`,

package/dist/src/report/experiments/buildWarmIndexReuseReport.js

@@ -0,0 +1,91 @@
+import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "./warmIndexReuseReportModel.js";
+const WARM_INDEX_REUSE_PLUGIN_ID = "warm-index-reuse";
+const COST_MODEL = [
+    "Index construction is a one-time cost per benchmark project; task retrieval is the repeated warm cost.",
+    "Cold start: for the first task in a project, the cumulative warm component duration is the one-time index build plus the first retrieval.",
+    "Warm reuse: later tasks do not rebuild the index; their warm operation duration is retrieval only.",
+    "Amortized index build duration after N tasks is the index build duration divided by N.",
+    "Raw cumulative duration sums one raw-context construction per task; warm cumulative component duration includes one index build plus each warm retrieval once.",
+    "Cumulative durations are sums of measured components, not wall-clock or end-to-end latency, and are not presented as a speedup.",
+];
+const LIMITATIONS = [
+    "Context estimated-token counts use the existing character-based estimate and are not provider billing telemetry.",
+    "Agent evidence comes from the deterministic fake agent only; real agents such as Codex or Claude are not run by warm-index-reuse in v0.5.0.",
+    "Fake-agent correctness is deterministic benchmark-harness evidence from the existing answer-key scorer, not a measure of real-model correctness.",
+    "Fake-agent token totals are simulated harness telemetry, not provider billing telemetry; estimated context tokens are a separate character-based context-size estimate.",
+    "Component-duration sums are not equivalent to provider end-to-end latency.",
+    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
+    "No composite score, winner, ranking, or break-even task is calculated.",
+];
+/**
+ * Presents the warm-index metrics already calculated on the run. It performs no amortization,
+ * cumulative, or availability calculation of its own.
+ */
+export function buildWarmIndexReuseReport(run) {
+    if (run.pluginId !== WARM_INDEX_REUSE_PLUGIN_ID)
+        return null;
+    const candidate = run;
+    const executions = candidate.projectExecutions ?? [];
+    const metricProjects = candidate.warmIndexMetrics?.projects ?? [];
+    const agentProjects = candidate.agentEvidence;
+    if (executions.length !== metricProjects.length) {
+        throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
+    }
+    const projects = executions.map((execution, projectIndex) => {
+        const metrics = metricProjects[projectIndex];
+        if (metrics.benchmarkProject !== execution.benchmarkProject ||
+            metrics.tasks.length !== execution.tasks.length ||
+            metrics.tasks.some((task, taskIndex) => task.caseId !== execution.tasks[taskIndex].caseId)) {
+            throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
+        }
+        return {
+            benchmarkProject: execution.benchmarkProject,
+            sessionKey: execution.sessionKey,
+            status: execution.status,
+            sessionPrepared: metrics.sessionPrepared,
+            indexBuildDurationMs: metrics.indexBuildDurationMs,
+            taskCount: metrics.tasks.length,
+            tasks: metrics.tasks.map((task, taskIndex) => {
+                const agents = agentProjects?.[projectIndex]?.tasks[taskIndex];
+                return {
+                    taskOrdinal: task.taskOrdinal,
+                    caseId: task.caseId,
+                    rawStatus: execution.tasks[taskIndex].rawStatus,
+                    warmStatus: execution.tasks[taskIndex].warmStatus,
+                    raw: task.raw,
+                    warm: task.warm,
+                    rawAgent: toReportAgent(agents?.raw ?? null),
+                    warmAgent: toReportAgent(agents?.warm ?? null),
+                };
+            }),
+        };
+    });
+    const sides = projects.flatMap((project) => project.tasks.flatMap((task) => [task.raw, task.warm]));
+    return {
+        schemaVersion: WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION,
+        summary: {
+            projectCount: projects.length,
+            taskCount: projects.reduce((sum, project) => sum + project.taskCount, 0),
+            preparedSessionProjectCount: projects.filter((project) => project.sessionPrepared).length,
+            incompleteProjectCount: projects.filter((project) => project.status !== "completed").length,
+            agentSideCount: sides.length,
+            agentCorrectnessAvailableCount: sides.filter((side) => side.agentCorrectness.availability === "available").length,
+            agentTotalTokensAvailableCount: sides.filter((side) => side.agentTotalTokens.availability === "available").length,
+        },
+        costModel: [...COST_MODEL],
+        limitations: [...LIMITATIONS],
+        projects,
+    };
+}
+function toReportAgent(agent) {
+    if (!agent)
+        return null;
+    return {
+        status: agent.status,
+        passed: agent.correctness.passed,
+        tokenUsageSource: agent.tokenUsage.source,
+        tokenUsageReliability: agent.tokenUsage.reliability,
+        warnings: [...agent.warnings],
+        errors: [...agent.errors],
+    };
+}

package/dist/src/report/experiments/index.js

@@ -6,6 +6,9 @@ export * from "./contextStrategyComparisonV043ReportModel.js";
 export { buildContextStrategyComparisonV043Report } from "./buildContextStrategyComparisonV043Report.js";
 export { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
 export { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
+export * from "./warmIndexReuseReportModel.js";
+export { buildWarmIndexReuseReport } from "./buildWarmIndexReuseReport.js";
+export { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 export * from "./contextIntegrityReportModel.js";
 export { buildContextIntegrityReport } from "./buildContextIntegrityReport.js";
 export { renderContextIntegrityJsonReport } from "./renderContextIntegrityJsonReport.js";

package/dist/src/report/experiments/renderPluginExperimentReportHtml.js

@@ -1,4 +1,5 @@
 import { renderContextStrategyComparisonV043Html } from "./renderContextStrategyComparisonV043Html.js";
+import { renderWarmIndexReuseHtml } from "./renderWarmIndexReuseHtml.js";
 export function renderPluginExperimentReportHtml(report) {
     return `<!DOCTYPE html>
 <html lang="en">
@@ -96,6 +97,8 @@ export function renderPluginExperimentReportHtml(report) {
 
   ${renderContextStrategyComparisonV043Html(report.contextStrategyComparisonV043)}
 
+  ${renderWarmIndexReuseHtml(report.warmIndexReuse)}
+
   <section>
     <h2>Warnings, Skips, And Failures</h2>
     ${report.findings.length ? table(["Severity", "Code", "Message", "Variant", "Case"], report.findings.map((finding) => [

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -256,6 +256,77 @@ function renderV043Section(lines, section) {
         renderStrategy(lines, section.strategies[index], index);
     }
 }
+function formatWarmMetric(metric) {
+    const method = metric.tokenCountMethod ? `, method ${metric.tokenCountMethod}` : "";
+    if (metric.availability === "available") {
+        return `${metric.value} ${metric.unit} (${metric.source}${method})`;
+    }
+    return `${metric.availability} (${metric.reason ?? ""})`;
+}
+function formatWarmAgent(agent) {
+    if (agent === null)
+        return "not run (no context evidence for this side)";
+    const passed = agent.passed === null ? "unavailable" : String(agent.passed);
+    const issues = [...agent.errors, ...agent.warnings].slice(0, 3).join("; ");
+    return `status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
+}
+function renderWarmIndexReuseSection(lines, section) {
+    if (section === null) {
+        lines.push("Not applicable to this plugin.");
+        return;
+    }
+    lines.push(fieldLine("Project Count", section.summary.projectCount));
+    lines.push(fieldLine("Task Count", section.summary.taskCount));
+    lines.push(fieldLine("Prepared Session Project Count", section.summary.preparedSessionProjectCount));
+    lines.push(fieldLine("Incomplete Or Failed Project Count", section.summary.incompleteProjectCount));
+    lines.push(fieldLine("Fake-Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push(fieldLine("Fake-Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push("Cold Start And Warm Reuse:");
+    pushDashList(lines, section.costModel);
+    lines.push("Limitations:");
+    pushDashList(lines, section.limitations);
+    if (section.projects.length === 0) {
+        lines.push("No warm-index project evidence was recorded.");
+        return;
+    }
+    section.projects.forEach((project, index) => {
+        pushSection(lines, `Warm Index Project ${index + 1}: ${sanitizeScalar(project.benchmarkProject)}`);
+        lines.push(fieldLine("Session Key", project.sessionKey));
+        lines.push(fieldLine("Status", project.status));
+        lines.push(fieldLine("Session Prepared", project.sessionPrepared));
+        lines.push(fieldLine("Index Build Duration", formatWarmMetric(project.indexBuildDurationMs)));
+        lines.push(fieldLine("Task Count", project.taskCount));
+        for (const task of project.tasks) {
+            pushSection(lines, `Task ${task.taskOrdinal}: ${sanitizeScalar(task.caseId)}`);
+            const rows = [
+                ["Raw Status", task.rawStatus],
+                ["Warm Status", task.warmStatus],
+                ["Raw Context Characters", formatWarmMetric(task.raw.contextCharacters)],
+                ["Warm Context Characters", formatWarmMetric(task.warm.contextCharacters)],
+                ["Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.contextEstimatedTokens)],
+                ["Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.contextEstimatedTokens)],
+                ["Raw Operation Duration", formatWarmMetric(task.raw.operationDurationMs)],
+                ["Warm Retrieval Duration", formatWarmMetric(task.warm.retrievalDurationMs)],
+                ["Amortized Index Build Duration", formatWarmMetric(task.warm.amortizedIndexBuildDurationMs)],
+                ["Cumulative Raw Duration", formatWarmMetric(task.raw.cumulativeDurationMs)],
+                ["Cumulative Warm Component Duration", formatWarmMetric(task.warm.cumulativeComponentDurationMs)],
+                ["Cumulative Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.cumulativeEstimatedContextTokens)],
+                ["Cumulative Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.cumulativeEstimatedContextTokens)],
+                ["Raw Fake-Agent Evaluation", formatWarmAgent(task.rawAgent)],
+                ["Warm Fake-Agent Evaluation", formatWarmAgent(task.warmAgent)],
+                ["Raw Agent Correctness (fake agent)", formatWarmMetric(task.raw.agentCorrectness)],
+                ["Warm Agent Correctness (fake agent)", formatWarmMetric(task.warm.agentCorrectness)],
+                ["Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.agentTotalTokens)],
+                ["Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.agentTotalTokens)],
+                ["Cumulative Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
+                ["Cumulative Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
+            ];
+            for (const [label, value] of rows) {
+                lines.push(fieldLine(label, value));
+            }
+        }
+    });
+}
 export function renderPluginExperimentReportText(report) {
     const lines = [];
     pushSection(lines, "Plugin Experiment Report");
@@ -284,6 +355,8 @@ export function renderPluginExperimentReportText(report) {
     pushDashList(lines, report.metrics.map((metric) => `${metric.name} (${metric.id}) value=${metric.value ?? "unavailable"} unit=${metric.unit ?? ""}`));
     pushSection(lines, "V0.4.3 Stage-Context Evidence");
     renderV043Section(lines, report.contextStrategyComparisonV043);
+    pushSection(lines, "Warm Index Reuse Evidence");
+    renderWarmIndexReuseSection(lines, report.warmIndexReuse);
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/dist/src/report/experiments/renderWarmIndexReuseHtml.js

@@ -0,0 +1,150 @@
+const UNIT_LABELS = {
+    ms: "ms",
+    characters: "chars",
+    "estimated-tokens": "est. tokens",
+    tokens: "tokens",
+    score: "score",
+};
+export function renderWarmIndexReuseHtml(section) {
+    if (section === null) {
+        return `<section>
+    <h2>Warm Index Reuse Evidence</h2>
+    <p>Not applicable to this plugin.</p>
+  </section>`;
+    }
+    return `<section>
+    <h2>Warm Index Reuse Evidence</h2>
+    ${table(["Field", "Value"], [
+        ["Projects", String(section.summary.projectCount)],
+        ["Tasks", String(section.summary.taskCount)],
+        ["Projects with a prepared index session", String(section.summary.preparedSessionProjectCount)],
+        ["Incomplete or failed projects", String(section.summary.incompleteProjectCount)],
+        ["Fake-agent correctness available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`],
+        ["Fake-agent token totals available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`],
+    ])}
+    <h3>Cold Start And Warm Reuse</h3>
+    ${list(section.costModel)}
+    <h3>Limitations</h3>
+    ${list(section.limitations)}
+    ${section.projects.length === 0 ? "<p>No warm-index project evidence was recorded.</p>" : section.projects.map(renderProject).join("\n")}
+  </section>`;
+}
+function renderProject(project, index) {
+    const unavailableRows = project.tasks.flatMap((task) => [
+        ["Raw context characters", task.raw.contextCharacters],
+        ["Warm context characters", task.warm.contextCharacters],
+        ["Raw estimated context tokens", task.raw.contextEstimatedTokens],
+        ["Warm estimated context tokens", task.warm.contextEstimatedTokens],
+        ["Raw operation duration", task.raw.operationDurationMs],
+        ["Warm retrieval duration", task.warm.retrievalDurationMs],
+        ["Amortized index build duration", task.warm.amortizedIndexBuildDurationMs],
+        ["Cumulative raw duration", task.raw.cumulativeDurationMs],
+        ["Cumulative warm component duration", task.warm.cumulativeComponentDurationMs],
+        ["Cumulative raw estimated context tokens", task.raw.cumulativeEstimatedContextTokens],
+        ["Cumulative warm estimated context tokens", task.warm.cumulativeEstimatedContextTokens],
+        ["Raw fake-agent correctness", task.raw.agentCorrectness],
+        ["Warm fake-agent correctness", task.warm.agentCorrectness],
+        ["Raw fake-agent total tokens", task.raw.agentTotalTokens],
+        ["Warm fake-agent total tokens", task.warm.agentTotalTokens],
+        ["Cumulative raw fake-agent total tokens", task.raw.cumulativeAgentTotalTokens],
+        ["Cumulative warm fake-agent total tokens", task.warm.cumulativeAgentTotalTokens],
+    ]
+        .filter(([, metric]) => metric.availability !== "available")
+        .map(([label, metric]) => [String(task.taskOrdinal), task.caseId, label, metric.availability, metric.reason ?? ""]));
+    return `<h3>Project ${index + 1}: ${escapeHtml(project.benchmarkProject)}</h3>
+    ${table(["Field", "Value"], [
+        ["Session key", project.sessionKey],
+        ["Project status", project.status],
+        ["Session prepared", String(project.sessionPrepared)],
+        ["Index build duration", formatMetric(project.indexBuildDurationMs)],
+        ["Index build duration availability", withReason(project.indexBuildDurationMs)],
+        ["Tasks", String(project.taskCount)],
+    ])}
+    ${table([
+        "Task",
+        "Raw status",
+        "Warm status",
+        "Raw chars",
+        "Warm chars",
+        "Raw est. tokens",
+        "Warm est. tokens",
+        "Raw duration",
+        "Warm retrieval duration",
+        "Amortized index cost",
+        "Cumulative raw duration",
+        "Cumulative warm duration",
+        "Cumulative raw est. tokens",
+        "Cumulative warm est. tokens",
+    ], project.tasks.map((task) => [
+        `${task.taskOrdinal}. ${task.caseId}`,
+        task.rawStatus,
+        task.warmStatus,
+        formatMetric(task.raw.contextCharacters),
+        formatMetric(task.warm.contextCharacters),
+        formatMetric(task.raw.contextEstimatedTokens),
+        formatMetric(task.warm.contextEstimatedTokens),
+        formatMetric(task.raw.operationDurationMs),
+        formatMetric(task.warm.retrievalDurationMs),
+        formatMetric(task.warm.amortizedIndexBuildDurationMs),
+        formatMetric(task.raw.cumulativeDurationMs),
+        formatMetric(task.warm.cumulativeComponentDurationMs),
+        formatMetric(task.raw.cumulativeEstimatedContextTokens),
+        formatMetric(task.warm.cumulativeEstimatedContextTokens),
+    ]))}
+    <p class="muted">Estimated tokens are character-based context-size estimates, not provider token usage.</p>
+    <p class="muted">Agent columns are deterministic fake-agent evidence; token totals are simulated harness telemetry, not provider billing telemetry.</p>
+    ${table([
+        "Task",
+        "Raw fake-agent status",
+        "Warm fake-agent status",
+        "Raw correctness",
+        "Warm correctness",
+        "Raw agent tokens",
+        "Warm agent tokens",
+        "Cumulative raw agent tokens",
+        "Cumulative warm agent tokens",
+        "Token source / reliability",
+    ], project.tasks.map((task) => [
+        `${task.taskOrdinal}. ${task.caseId}`,
+        formatAgent(task.rawAgent),
+        formatAgent(task.warmAgent),
+        formatMetric(task.raw.agentCorrectness),
+        formatMetric(task.warm.agentCorrectness),
+        formatMetric(task.raw.agentTotalTokens),
+        formatMetric(task.warm.agentTotalTokens),
+        formatMetric(task.raw.cumulativeAgentTotalTokens),
+        formatMetric(task.warm.cumulativeAgentTotalTokens),
+        [task.rawAgent, task.warmAgent]
+            .map((agent) => (agent ? `${agent.tokenUsageSource} / ${agent.tokenUsageReliability}` : "not run"))
+            .join("; "),
+    ]))}
+    ${unavailableRows.length === 0 ? "" : table(["Task", "Case", "Metric", "Availability", "Reason"], unavailableRows)}`;
+}
+function formatAgent(agent) {
+    if (agent === null)
+        return "not run";
+    const passed = agent.passed === null ? "" : agent.passed ? ", passed" : ", not passed";
+    return `${agent.status}${passed}${agent.errors[0] ? ` (${agent.errors[0]})` : ""}`;
+}
+function formatMetric(metric) {
+    return metric.availability === "available" ? `${metric.value} ${UNIT_LABELS[metric.unit]}` : metric.availability;
+}
+function withReason(metric) {
+    return metric.availability === "available" ? formatMetric(metric) : `${metric.availability}: ${metric.reason ?? ""}`;
+}
+function list(items) {
+    return `<ul>${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}</ul>`;
+}
+function table(headers, rows) {
+    return `<table><thead><tr>${headers.map((header) => `<th>${escapeHtml(header)}</th>`).join("")}</tr></thead><tbody>${rows
+        .map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`)
+        .join("")}</tbody></table>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/report/experiments/warmIndexReuseReportModel.js

@@ -0,0 +1 @@
+export const WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-report-v1";

package/docs/METRICS.md

@@ -137,6 +137,15 @@ Each entry below uses the same fields: **Meaning**, **Appears in** (the artifact
   Interpretation: human-readable size category.
   Caveat: coarse label; use the score and metrics for detail.
 
+In the released v0.5.1 benchmark suite, the task statistics (`taskCount`, `expectedRelevantFilesAverage`, `expectedRelevantSymbolsAverage`) of the two warm-index benchmark projects are derived from the dedicated corpus `benchmarks/contracts/warm-index-benchmark-cases.json`: the averages are the mean `expectedFiles` and `expectedSymbols` counts of that project's cases, rounded to two decimals. `npm run verify:benchmarks` fails if a profile disagrees with the corpus. The `benchmark-project-complexity-v1` formula, its weights, and its caps are unchanged; the current values are:
+
+| Project | `taskCount` | `expectedRelevantFilesAverage` | `expectedRelevantSymbolsAverage` | `complexityScore` |
+|---|---|---|---|---|
+| `task-workflow-medium-ts` | 6 | 3.83 | 4.33 | 31 |
+| `task-analytics-large-mixed` | 6 | 4.5 | 4.67 | 47 |
+
+Task locality (`localized`, `cross-module`, `broad-change`) is benchmark metadata on each corpus case. It is not a metric, score, or ranking, and it does not enter the complexity formula.
+
 ## Prompt complexity metrics
 
 - `promptChars`
@@ -441,3 +450,69 @@ Implemented in `src/evaluation/stageContextMetrics` and composed once per run by
 - Determinism and fixture self-immutability
   Meaning: reuses `calculateStageContextDeterminism`/`canonicalizeStageContextRun` (`v0.4.3`) to confirm repeated evaluations of the same fixture produce identical canonicalized results, and re-runs fixture hash verification to confirm the frozen bytes were not mutated by evaluation.
   Caveat: this is fixture-level self-immutability, not the `v0.4.3` live-target-mutation check — no live target repository is exercised by this evaluation path.
+
+## Warm-index reuse metrics
+
+Introduced in v0.5.0. The `warm-index-reuse` plugin calculates these metrics once, in `src/experiments/plugins/warmIndexReuse/metrics.ts`, from bounded execution summaries and bounded fake-agent evidence. The report section (`report.warmIndexReuse` in `report.json`, and the text and HTML "Warm Index Reuse Evidence" sections) and the four warm-index plots render these precomputed values; they never recalculate a formula.
+
+The released v0.5.1 expanded benchmark suite adds no warm-index metric and changes no formula: over the dedicated 12-case corpus the same metrics run through task ordinal 6 in each project, and amortization and cumulative sums restart for each project. No token-savings percentage, break-even task, winner, speedup, ranking, or provider billing metric is calculated.
+
+**Availability contract.** Every warm-index metric carries `availability`, `value`, `unit`, `source`, `reason`, and `tokenCountMethod`:
+
+- `available`: `value` is a finite number and `reason` is `null`. Zero is a valid available value.
+- `unavailable`: `value` is `null` and `reason` explains the missing or invalid evidence. Missing evidence is never reported as zero, and a non-finite upstream number becomes `unavailable` with an invalid-input reason.
+- `not-applicable`: `value` is `null` and `reason` explains why the metric does not apply. The contract supports it; the current warm-index metrics do not emit it.
+
+Units are `ms`, `characters`, `estimated-tokens`, `tokens`, and `score`. Sources are `measured`, `derived`, `estimated-chars-div-4`, and `agent`. `tokenCountMethod` is set only for estimated-token metrics (currently `estimated_chars_div_4`).
+
+**Task ordinal.** Tasks keep their filtered source order inside each benchmark project and are numbered `1, 2, 3, …` per project. Ordinals, amortization, and every cumulative sum restart for each benchmark project.
+
+**Direct measurements (per task):**
+
+- Raw context characters / warm retrieved context characters (`characters`, `measured`)
+  Meaning: characters in the raw full-file context and in the retrieved my-dev-kit context.
+  Caveat: unavailable when that side produced no evidence (for example, a raw baseline failure, or no warm retrieval because the project index was not prepared).
+- Raw estimated context tokens / warm estimated context tokens (`estimated-tokens`, `estimated-chars-div-4`)
+  Meaning: the character-based context-size estimate, `ceil(characters / 4)`.
+  Caveat: a context-size estimate, not provider token usage and not fake-agent token usage.
+- Raw context-construction duration (`ms`, `measured`)
+  Meaning: time to build the raw full-file context for the task. It is not agent duration.
+- Warm retrieval duration (`ms`, `measured`)
+  Meaning: search, lookup, slice, and source against the already prepared index. It never includes the index build.
+  Caveat: a skipped retrieval (no usable context) still reports its measured duration and sizes; its outcome remains `skipped`.
+- Index build duration (per project, `ms`, `measured`)
+  Meaning: the measured duration of the project's single my-dev-kit `index` command.
+  Caveat: available whenever a build was attempted and measured, even if the build failed. Unavailable, never zero, when the project group was structurally inconsistent and no index setup was attempted.
+
+**Derived measurements (per task):**
+
+- Amortized index build duration (`ms`, `derived`)
+  Formula: `amortizedIndexBuildDurationMs(N) = indexBuildDurationMs / N` for task ordinal `N`, without rounding.
+  Caveat: available only when the project prepared a valid index session. A failed or never-attempted index is not amortized.
+- Cumulative raw component duration (`ms`, `measured`)
+  Formula: sum of raw context-construction durations for tasks `1..N`.
+- Cumulative warm component duration (`ms`, `derived`)
+  Formula: `indexBuildDurationMs` (counted once) plus the warm retrieval durations for tasks `1..N`. At `N = 1` this is the cold-start warm cost: index build plus the first retrieval.
+  Caveat: requires a prepared index session. It is a sum of measured components, not wall-clock or end-to-end latency, and it is not a speedup claim.
+- Cumulative raw / warm estimated context tokens (`estimated-tokens`, `estimated-chars-div-4`)
+  Formula: sum of that side's estimated context tokens for tasks `1..N`. No index-token cost is added; indexing is a local repository operation, not model context.
+  Caveat: if different token-count methods appear within one prefix, the cumulative value is unavailable rather than summed.
+
+**Strict-prefix rule.** A cumulative metric at ordinal `N` is available only when every task `1..N` has the required input. After the first missing or invalid input, that cumulative metric stays unavailable for every later task in the project, with a reason naming the first incomplete task. The later tasks' direct metrics remain available.
+
+**Fake-agent evidence (per task side):**
+
+The plugin evaluates each task side that has context evidence once with the deterministic fake agent, through the existing prompt generator, answer parser, run classifier, and correctness scorer. A side with no context evidence is not evaluated, and its agent metrics are unavailable with that reason.
+
+- Fake-agent correctness score (`score`, `agent`)
+  Meaning: `correctnessScore` from the existing benchmark answer-key scorer (see "Experiment and run metrics" above).
+  Caveat: available only when the fake-agent answer was scoreable (classified `completed` or `invalid-output`); a failed agent run reports `unavailable` with its status and error. It is deterministic benchmark-harness evidence, not real-model correctness, and retrieval success is never used as correctness.
+- Fake-agent total tokens (`tokens`, `agent`)
+  Meaning: `tokenUsage.totalTokens` exactly as reported by the fake agent (its prompt estimate plus a fixed simulated output count), with the reported token-usage source and reliability kept in the agent evidence.
+  Caveat: simulated harness telemetry, not provider billing telemetry. When the fake agent reports no total (for example `FAKE_AGENT_MODE=missing-token-usage`), the value is unavailable; estimated context tokens are never substituted.
+- Cumulative fake-agent total tokens (`tokens`, `agent`)
+  Formula: sum of fake-agent total tokens for tasks `1..N` on that side, under the strict-prefix rule.
+
+**Generic outcome metrics.** Available values also appear as `ExperimentMetric` entries on each outcome, with `variantId`, `caseId`, `unit`, and `description`. Both the `raw-full-file` and `warm-index-reuse` outcomes can carry `context-character-count`, `context-estimated-token-count`, `operation-duration-ms` (raw context construction or warm retrieval), `cumulative-component-duration-ms`, `cumulative-context-estimated-token-count`, `agent-correctness-score`, `agent-total-tokens`, and `cumulative-agent-total-tokens`. The warm outcome also carries `amortized-index-build-duration-ms`. Unavailable metrics are omitted rather than emitted as zero. Run-level metrics are limited to `warm-index-project-count`, `warm-index-task-count`, and `warm-index-session-prepared-project-count`.
+
+**Interpretation.** The report separates the one-time index cost from per-task retrieval cost and shows the fixed cost falling per task as more tasks reuse the index. It calculates no token-savings percentage, duration-reduction percentage, break-even task, composite score, winner, or ranking. Warm-index plots map the same precomputed values: amortized index build duration, raw versus retrieved estimated context tokens, fake-agent correctness, and cumulative fake-agent total tokens. Unavailable values become skipped plot points with their reason.