@dailephd/my-dev-kit-lab 0.5.1 → 0.6.0

package/dist/src/report/experiments/buildWarmIndexReuseReport.js

@@ -1,5 +1,35 @@
-import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "./warmIndexReuseReportModel.js";
+import { MAX_REPORT_FRESHNESS_CHANGES, MAX_REPORT_FRESHNESS_UNRESOLVED, WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "./warmIndexReuseReportModel.js";
 const WARM_INDEX_REUSE_PLUGIN_ID = "warm-index-reuse";
+const COMMON_LIMITATIONS = [
+    "Context estimated-token counts use the character-based estimate and are not provider billing telemetry.",
+    "Correctness is deterministic answer-key scoring, not semantic LLM judging.",
+    "Component-duration sums are not equivalent to provider end-to-end latency.",
+    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
+    "No composite score, winner, ranking, or break-even task is calculated.",
+];
+const FAKE_AGENT_LIMITATIONS = [
+    "Agent evidence is deterministic simulated fake-agent evidence.",
+    "Fake-agent total tokens are simulated harness telemetry, not provider billing telemetry.",
+];
+const CODEX_CAMPAIGN_LIMITATIONS = [
+    "Agent evidence comes from Codex CLI campaign execution over the exact runner-supplied benchmark contexts.",
+    "Token totals are CLI-exposed adapter telemetry when available.",
+    "Token evidence must be interpreted using tokenUsageSource/tokenUsageReliability.",
+    "Context estimates are never substituted for missing agent token totals.",
+];
+const CLAUDE_CAMPAIGN_LIMITATIONS = [
+    "Agent evidence comes from Claude CLI campaign execution over the exact runner-supplied benchmark contexts.",
+    "Token totals are used when exposed by the CLI output.",
+    "Token totals may be unavailable for some CLI/version/run outcomes.",
+    "Unavailable token totals remain unavailable and are never replaced by zero or a context estimate.",
+];
+const FRESHNESS_LIMITATIONS = [
+    "Freshness compares only files proven by the index snapshot to have been indexed. It does not establish that the entire repository is unchanged.",
+    "A new file that was not represented in the snapshot is outside the current freshness comparison and is not automatically evidence that the index is stale.",
+    "File content SHA-256 is the comparison identity. Modified timestamps are diagnostic metadata only.",
+    "Freshness is observational evidence. It does not trigger reindexing, suppress warm retrieval, alter execution status, or alter provider/agent outcome status.",
+    "Freshness statuses: fresh means all represented indexed files were completely compared and their content identities still match; stale means a complete comparison confirmed at least one represented indexed file is modified or missing; partially-stale means at least one represented indexed file is confirmed changed but comparison evidence is incomplete; unknown means no confirmed change established staleness but evidence is insufficient to prove freshness. A task without an assessment is reported as not assessed, which is not the same as unknown.",
+];
 const COST_MODEL = [
     "Index construction is a one-time cost per benchmark project; task retrieval is the repeated warm cost.",
     "Cold start: for the first task in a project, the cumulative warm component duration is the one-time index build plus the first retrieval.",
@@ -8,15 +38,15 @@ const COST_MODEL = [
     "Raw cumulative duration sums one raw-context construction per task; warm cumulative component duration includes one index build plus each warm retrieval once.",
     "Cumulative durations are sums of measured components, not wall-clock or end-to-end latency, and are not presented as a speedup.",
 ];
-const LIMITATIONS = [
-    "Context estimated-token counts use the existing character-based estimate and are not provider billing telemetry.",
-    "Agent evidence comes from the deterministic fake agent only; real agents such as Codex or Claude are not run by warm-index-reuse in v0.5.0.",
-    "Fake-agent correctness is deterministic benchmark-harness evidence from the existing answer-key scorer, not a measure of real-model correctness.",
-    "Fake-agent token totals are simulated harness telemetry, not provider billing telemetry; estimated context tokens are a separate character-based context-size estimate.",
-    "Component-duration sums are not equivalent to provider end-to-end latency.",
-    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
-    "No composite score, winner, ranking, or break-even task is calculated.",
-];
+const OUTCOME_STATUS_TO_COUNT_KEY = {
+    completed: "completed",
+    failed: "failed",
+    timeout: "timeout",
+    "invalid-output": "invalidOutput",
+    "agent-unavailable": "agentUnavailable",
+    "agent-limit-reached": "agentLimitReached",
+    skipped: "skipped",
+};
 /**
  * Presents the warm-index metrics already calculated on the run. It performs no amortization,
  * cumulative, or availability calculation of its own.
@@ -31,6 +61,14 @@ export function buildWarmIndexReuseReport(run) {
     if (executions.length !== metricProjects.length) {
         throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
     }
+    const campaignPreset = readString(run.metadata?.campaignPreset);
+    const campaignAgentId = readString(run.metadata?.campaignAgentId);
+    const campaignTimeoutMs = readNumber(run.metadata?.campaignTimeoutMs);
+    if (campaignAgentId !== undefined && campaignAgentId !== "codex" && campaignAgentId !== "claude") {
+        throw new Error(`Invalid warm-index report source: unsupported campaign agent id ${campaignAgentId}.`);
+    }
+    const isCampaign = campaignPreset !== undefined;
+    const allSides = [];
     const projects = executions.map((execution, projectIndex) => {
         const metrics = metricProjects[projectIndex];
         if (metrics.benchmarkProject !== execution.benchmarkProject ||
@@ -47,6 +85,9 @@ export function buildWarmIndexReuseReport(run) {
             taskCount: metrics.tasks.length,
             tasks: metrics.tasks.map((task, taskIndex) => {
                 const agents = agentProjects?.[projectIndex]?.tasks[taskIndex];
+                const rawEvidence = agents?.raw ?? null;
+                const warmEvidence = agents?.warm ?? null;
+                allSides.push(rawEvidence, warmEvidence);
                 return {
                     taskOrdinal: task.taskOrdinal,
                     caseId: task.caseId,
@@ -54,12 +95,23 @@ export function buildWarmIndexReuseReport(run) {
                     warmStatus: execution.tasks[taskIndex].warmStatus,
                     raw: task.raw,
                     warm: task.warm,
-                    rawAgent: toReportAgent(agents?.raw ?? null),
-                    warmAgent: toReportAgent(agents?.warm ?? null),
+                    rawAgent: toReportAgent(rawEvidence),
+                    warmAgent: toReportAgent(warmEvidence),
+                    indexFreshness: toReportFreshness(execution.tasks[taskIndex].indexFreshness),
                 };
             }),
         };
     });
+    const agentIdentity = deriveAgentIdentity({ isCampaign, campaignAgentId, sides: allSides });
+    const agentCampaign = isCampaign
+        ? buildCampaignSummary({
+            presetId: campaignPreset,
+            agentId: agentIdentity.id,
+            timeoutMs: campaignTimeoutMs ?? 0,
+            selectedCaseCount: projects.reduce((sum, project) => sum + project.taskCount, 0),
+            sides: allSides,
+        })
+        : null;
     const sides = projects.flatMap((project) => project.tasks.flatMap((task) => [task.raw, task.warm]));
     return {
         schemaVersion: WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION,
@@ -73,14 +125,133 @@ export function buildWarmIndexReuseReport(run) {
             agentTotalTokensAvailableCount: sides.filter((side) => side.agentTotalTokens.availability === "available").length,
         },
         costModel: [...COST_MODEL],
-        limitations: [...LIMITATIONS],
+        limitations: buildLimitations(agentIdentity),
         projects,
+        agent: agentIdentity,
+        agentCampaign,
+        indexFreshnessSummary: summarizeFreshness(projects.flatMap((project) => project.tasks)),
+    };
+}
+function boundedList(source, limit, map) {
+    const items = source.slice(0, limit).map(map);
+    return { totalCount: source.length, displayedCount: items.length, omittedCount: source.length - items.length, items };
+}
+/**
+ * Presents persisted freshness evidence only: it never recomputes a status, count, or hash and
+ * never touches the filesystem. Absent evidence stays null rather than becoming `unknown`.
+ */
+function toReportFreshness(freshness) {
+    if (!freshness)
+        return null;
+    return {
+        status: freshness.status,
+        assessedAt: freshness.assessedAt,
+        baselineSnapshotStatus: freshness.baselineSnapshotStatus,
+        indexedFileCount: freshness.indexedFileCount,
+        comparableFileCount: freshness.comparableFileCount,
+        unchangedFileCount: freshness.unchangedFileCount,
+        changedFileCount: freshness.changedFileCount,
+        missingFileCount: freshness.missingFileCount,
+        unresolvedFileCount: freshness.unresolvedFileCount,
+        changes: boundedList(freshness.changes, MAX_REPORT_FRESHNESS_CHANGES, (change) => ({ path: change.path, changeType: change.changeType })),
+        unresolved: boundedList(freshness.unresolved, MAX_REPORT_FRESHNESS_UNRESOLVED, (entry) => ({
+            path: entry.path,
+            reasonCode: entry.reasonCode,
+            message: entry.message,
+        })),
+        warnings: [...freshness.warnings],
     };
 }
+function summarizeFreshness(tasks) {
+    const assessed = tasks.flatMap((task) => (task.indexFreshness ? [task.indexFreshness] : []));
+    const count = (status) => assessed.filter((freshness) => freshness.status === status).length;
+    return {
+        assessedTaskCount: assessed.length,
+        unassessedTaskCount: tasks.length - assessed.length,
+        freshTaskCount: count("fresh"),
+        staleTaskCount: count("stale"),
+        partiallyStaleTaskCount: count("partially-stale"),
+        unknownTaskCount: count("unknown"),
+    };
+}
+function deriveAgentIdentity(args) {
+    const nonNullAgentIds = new Set(args.sides.filter((side) => side !== null).map((side) => side.agentId));
+    if (!args.isCampaign) {
+        if ([...nonNullAgentIds].some((agentId) => agentId !== "fake-agent")) {
+            throw new Error(`Invalid warm-index report source: a legacy non-campaign run must not contain non-fake agent evidence, found ${[...nonNullAgentIds].join(", ")}.`);
+        }
+        return { id: "fake-agent", mode: "deterministic-fake" };
+    }
+    const campaignAgentId = args.campaignAgentId;
+    if (!campaignAgentId) {
+        throw new Error("Invalid warm-index report source: a campaign run must carry a campaignAgentId.");
+    }
+    for (const agentId of nonNullAgentIds) {
+        if (agentId !== campaignAgentId) {
+            throw new Error(`Invalid warm-index report source: campaign metadata says ${campaignAgentId} but agent evidence contains ${agentId}.`);
+        }
+    }
+    return { id: campaignAgentId, mode: "real-provider" };
+}
+function buildCampaignSummary(args) {
+    const scheduledSideCount = args.selectedCaseCount * 2;
+    const executedSides = args.sides.filter((side) => side !== null);
+    const executedSideCount = executedSides.length;
+    const notRunForMissingContextCount = scheduledSideCount - executedSideCount;
+    const outcomeCounts = {
+        completed: 0,
+        failed: 0,
+        timeout: 0,
+        invalidOutput: 0,
+        agentUnavailable: 0,
+        agentLimitReached: 0,
+        skipped: 0,
+    };
+    for (const side of executedSides) {
+        const key = OUTCOME_STATUS_TO_COUNT_KEY[side.status];
+        if (!key) {
+            throw new Error(`Invalid warm-index report source: unsupported campaign agent status "${side.status}".`);
+        }
+        outcomeCounts[key] += 1;
+    }
+    const tokenAvailableSideCount = executedSides.filter((side) => typeof side.tokenUsage.totalTokens === "number" && Number.isFinite(side.tokenUsage.totalTokens)).length;
+    const agentEvidenceStatus = executedSideCount === 0
+        ? "unavailable"
+        : executedSideCount === scheduledSideCount && outcomeCounts.completed === executedSideCount
+            ? "complete"
+            : "partial";
+    const tokenEvidenceStatus = executedSideCount === 0
+        ? "unavailable"
+        : tokenAvailableSideCount === 0
+            ? "unavailable"
+            : tokenAvailableSideCount === executedSideCount
+                ? "complete"
+                : "partial";
+    return {
+        presetId: args.presetId,
+        agentId: args.agentId,
+        timeoutMs: args.timeoutMs,
+        selectedCaseCount: args.selectedCaseCount,
+        scheduledSideCount,
+        executedSideCount,
+        notRunForMissingContextCount,
+        outcomeCounts,
+        agentEvidenceStatus,
+        tokenEvidenceStatus,
+    };
+}
+function buildLimitations(agent) {
+    if (agent.mode === "deterministic-fake") {
+        return [...COMMON_LIMITATIONS, ...FAKE_AGENT_LIMITATIONS, ...FRESHNESS_LIMITATIONS];
+    }
+    const providerLimitations = agent.id === "codex" ? CODEX_CAMPAIGN_LIMITATIONS : CLAUDE_CAMPAIGN_LIMITATIONS;
+    return [...COMMON_LIMITATIONS, ...providerLimitations, ...FRESHNESS_LIMITATIONS];
+}
 function toReportAgent(agent) {
     if (!agent)
         return null;
     return {
+        agentId: agent.agentId,
         status: agent.status,
         passed: agent.correctness.passed,
         tokenUsageSource: agent.tokenUsage.source,
@@ -89,3 +260,9 @@ function toReportAgent(agent) {
         errors: [...agent.errors],
     };
 }
+function readString(value) {
+    return typeof value === "string" && value ? value : undefined;
+}
+function readNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -265,10 +265,28 @@ function formatWarmMetric(metric) {
 }
 function formatWarmAgent(agent) {
     if (agent === null)
-        return "not run (no context evidence for this side)";
+        return "not run (no usable context evidence for this side)";
     const passed = agent.passed === null ? "unavailable" : String(agent.passed);
     const issues = [...agent.errors, ...agent.warnings].slice(0, 3).join("; ");
-    return `status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
+    return `agent ${agent.agentId}, status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
+}
+function renderCampaignSection(lines, campaign) {
+    pushSection(lines, "Real-Agent Campaign");
+    lines.push(fieldLine("Campaign Preset", campaign.presetId));
+    lines.push(fieldLine("Agent ID", campaign.agentId));
+    lines.push(fieldLine("Campaign Timeout", `${campaign.timeoutMs} ms`));
+    lines.push(fieldLine("Scheduled Agent Sides", campaign.scheduledSideCount));
+    lines.push(fieldLine("Executed Agent Sides", campaign.executedSideCount));
+    lines.push(fieldLine("Not Run For Missing Context", campaign.notRunForMissingContextCount));
+    lines.push(fieldLine("Agent Evidence Status", campaign.agentEvidenceStatus));
+    lines.push(fieldLine("Token Evidence Status", campaign.tokenEvidenceStatus));
+    lines.push(fieldLine("Completed Agent Sides", campaign.outcomeCounts.completed));
+    lines.push(fieldLine("Failed Agent Sides", campaign.outcomeCounts.failed));
+    lines.push(fieldLine("Timeout Agent Sides", campaign.outcomeCounts.timeout));
+    lines.push(fieldLine("Invalid-Output Agent Sides", campaign.outcomeCounts.invalidOutput));
+    lines.push(fieldLine("Agent-Unavailable Sides", campaign.outcomeCounts.agentUnavailable));
+    lines.push(fieldLine("Agent-Limit-Reached Sides", campaign.outcomeCounts.agentLimitReached));
+    lines.push(fieldLine("Skipped Agent Sides", campaign.outcomeCounts.skipped));
 }
 function renderWarmIndexReuseSection(lines, section) {
     if (section === null) {
@@ -279,8 +297,14 @@ function renderWarmIndexReuseSection(lines, section) {
     lines.push(fieldLine("Task Count", section.summary.taskCount));
     lines.push(fieldLine("Prepared Session Project Count", section.summary.preparedSessionProjectCount));
     lines.push(fieldLine("Incomplete Or Failed Project Count", section.summary.incompleteProjectCount));
-    lines.push(fieldLine("Fake-Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
-    lines.push(fieldLine("Fake-Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push(fieldLine("Agent ID", section.agent.id));
+    lines.push(fieldLine("Agent Mode", section.agent.mode));
+    lines.push(fieldLine("Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push(fieldLine("Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
+    renderFreshnessSummary(lines, section);
+    if (section.agentCampaign) {
+        renderCampaignSection(lines, section.agentCampaign);
+    }
     lines.push("Cold Start And Warm Reuse:");
     pushDashList(lines, section.costModel);
     lines.push("Limitations:");
@@ -301,6 +325,7 @@ function renderWarmIndexReuseSection(lines, section) {
             const rows = [
                 ["Raw Status", task.rawStatus],
                 ["Warm Status", task.warmStatus],
+                ["Index Freshness Status", task.indexFreshness ? task.indexFreshness.status : "not assessed"],
                 ["Raw Context Characters", formatWarmMetric(task.raw.contextCharacters)],
                 ["Warm Context Characters", formatWarmMetric(task.warm.contextCharacters)],
                 ["Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.contextEstimatedTokens)],
@@ -312,21 +337,74 @@ function renderWarmIndexReuseSection(lines, section) {
                 ["Cumulative Warm Component Duration", formatWarmMetric(task.warm.cumulativeComponentDurationMs)],
                 ["Cumulative Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.cumulativeEstimatedContextTokens)],
                 ["Cumulative Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.cumulativeEstimatedContextTokens)],
-                ["Raw Fake-Agent Evaluation", formatWarmAgent(task.rawAgent)],
-                ["Warm Fake-Agent Evaluation", formatWarmAgent(task.warmAgent)],
-                ["Raw Agent Correctness (fake agent)", formatWarmMetric(task.raw.agentCorrectness)],
-                ["Warm Agent Correctness (fake agent)", formatWarmMetric(task.warm.agentCorrectness)],
-                ["Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.agentTotalTokens)],
-                ["Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.agentTotalTokens)],
-                ["Cumulative Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
-                ["Cumulative Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
+                ["Raw Agent Evaluation", formatWarmAgent(task.rawAgent)],
+                ["Warm Agent Evaluation", formatWarmAgent(task.warmAgent)],
+                ["Raw Agent Correctness", formatWarmMetric(task.raw.agentCorrectness)],
+                ["Warm Agent Correctness", formatWarmMetric(task.warm.agentCorrectness)],
+                ["Raw Agent Total Tokens", formatWarmMetric(task.raw.agentTotalTokens)],
+                ["Warm Agent Total Tokens", formatWarmMetric(task.warm.agentTotalTokens)],
+                ["Cumulative Raw Agent Total Tokens", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
+                ["Cumulative Warm Agent Total Tokens", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
             ];
             for (const [label, value] of rows) {
                 lines.push(fieldLine(label, value));
             }
+            renderTaskFreshnessDetail(lines, task.indexFreshness);
         }
     });
 }
+function renderFreshnessSummary(lines, section) {
+    // Reports serialized before v0.6.0 Batch 3 lack the summary; render them as having no assessment.
+    const summary = section.indexFreshnessSummary ?? {
+        assessedTaskCount: 0,
+        unassessedTaskCount: section.summary.taskCount,
+        freshTaskCount: 0,
+        staleTaskCount: 0,
+        partiallyStaleTaskCount: 0,
+        unknownTaskCount: 0,
+    };
+    pushSection(lines, "Index Freshness Summary");
+    lines.push(fieldLine("Assessed Tasks", summary.assessedTaskCount));
+    lines.push(fieldLine("Unassessed Tasks", summary.unassessedTaskCount));
+    lines.push(fieldLine("Fresh Tasks", summary.freshTaskCount));
+    lines.push(fieldLine("Stale Tasks", summary.staleTaskCount));
+    lines.push(fieldLine("Partially-Stale Tasks", summary.partiallyStaleTaskCount));
+    lines.push(fieldLine("Unknown Tasks", summary.unknownTaskCount));
+}
+function pushFreshnessBoundedNote(lines, list) {
+    if (list.omittedCount > 0) {
+        lines.push(`Displayed: ${list.displayedCount} of ${list.totalCount}`);
+        lines.push(`Omitted: ${list.omittedCount}`);
+    }
+}
+/** Renders the persisted freshness counts and bounded evidence; hashes are never printed. */
+function renderTaskFreshnessDetail(lines, freshness) {
+    if (!freshness)
+        return;
+    lines.push(fieldLine("Baseline Snapshot Status", freshness.baselineSnapshotStatus));
+    lines.push(fieldLine("Indexed Files", freshness.indexedFileCount));
+    lines.push(fieldLine("Comparable Files", freshness.comparableFileCount));
+    lines.push(fieldLine("Unchanged Files", freshness.unchangedFileCount));
+    lines.push(fieldLine("Modified Files", freshness.changedFileCount));
+    lines.push(fieldLine("Missing Files", freshness.missingFileCount));
+    lines.push(fieldLine("Unresolved Comparisons", freshness.unresolvedFileCount));
+    lines.push("Freshness Changes:");
+    pushFreshnessBoundedNote(lines, freshness.changes);
+    if (freshness.changes.items.length === 0) {
+        lines.push("- none");
+    }
+    for (const change of freshness.changes.items) {
+        lines.push(`- ${sanitizeScalar(change.changeType)} ${sanitizeScalar(change.path)}`);
+    }
+    lines.push("Freshness Unresolved Evidence:");
+    pushFreshnessBoundedNote(lines, freshness.unresolved);
+    if (freshness.unresolved.items.length === 0) {
+        lines.push("- none");
+    }
+    for (const entry of freshness.unresolved.items) {
+        lines.push(`- ${entry.path === null ? "run-level" : sanitizeScalar(entry.path)} [${sanitizeScalar(entry.reasonCode)}] ${sanitizeScalar(entry.message)}`);
+    }
+}
 export function renderPluginExperimentReportText(report) {
     const lines = [];
     pushSection(lines, "Plugin Experiment Report");

package/dist/src/report/experiments/renderWarmIndexReuseHtml.js

@@ -1,3 +1,4 @@
+import { agentLabel } from "../../experiments/plugins/warmIndexReuse/metrics.js";
 const UNIT_LABELS = {
     ms: "ms",
     characters: "chars",
@@ -19,17 +20,108 @@ export function renderWarmIndexReuseHtml(section) {
         ["Tasks", String(section.summary.taskCount)],
         ["Projects with a prepared index session", String(section.summary.preparedSessionProjectCount)],
         ["Incomplete or failed projects", String(section.summary.incompleteProjectCount)],
-        ["Fake-agent correctness available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`],
-        ["Fake-agent token totals available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`],
+        ["Agent ID", section.agent.id],
+        ["Agent mode", section.agent.mode],
+        ["Agent correctness available", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`],
+        ["Agent token totals available", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`],
     ])}
+    ${renderFreshnessSummary(section.indexFreshnessSummary ?? legacyFreshnessSummary(section.summary.taskCount))}
+    ${section.agentCampaign ? renderCampaignSection(section.agentCampaign) : ""}
     <h3>Cold Start And Warm Reuse</h3>
     ${list(section.costModel)}
     <h3>Limitations</h3>
     ${list(section.limitations)}
-    ${section.projects.length === 0 ? "<p>No warm-index project evidence was recorded.</p>" : section.projects.map(renderProject).join("\n")}
+    ${section.projects.length === 0 ? "<p>No warm-index project evidence was recorded.</p>" : section.projects.map((project, index) => renderProject(project, index, section.agent.id)).join("\n")}
   </section>`;
 }
-function renderProject(project, index) {
+// Reports serialized before v0.6.0 Batch 3 lack the summary; they are rendered as having no assessment.
+function legacyFreshnessSummary(taskCount) {
+    return {
+        assessedTaskCount: 0,
+        unassessedTaskCount: taskCount,
+        freshTaskCount: 0,
+        staleTaskCount: 0,
+        partiallyStaleTaskCount: 0,
+        unknownTaskCount: 0,
+    };
+}
+function renderFreshnessSummary(summary) {
+    return `<h3>Index Freshness</h3>
+    ${table(["Field", "Value"], [
+        ["Assessed tasks", String(summary.assessedTaskCount)],
+        ["Unassessed tasks", String(summary.unassessedTaskCount)],
+        ["Fresh tasks", String(summary.freshTaskCount)],
+        ["Stale tasks", String(summary.staleTaskCount)],
+        ["Partially-stale tasks", String(summary.partiallyStaleTaskCount)],
+        ["Unknown tasks", String(summary.unknownTaskCount)],
+    ])}`;
+}
+/** Persisted freshness evidence per task; hashes are deliberately absent from this presentation. */
+function renderProjectFreshness(tasks) {
+    const overview = table(["Task", "Freshness", "Baseline", "Indexed", "Comparable", "Unchanged", "Modified", "Missing", "Unresolved"], tasks.map((task) => {
+        const freshness = task.indexFreshness;
+        const label = `${task.taskOrdinal}. ${task.caseId}`;
+        return freshness
+            ? [
+                label,
+                freshness.status,
+                freshness.baselineSnapshotStatus,
+                String(freshness.indexedFileCount),
+                String(freshness.comparableFileCount),
+                String(freshness.unchangedFileCount),
+                String(freshness.changedFileCount),
+                String(freshness.missingFileCount),
+                String(freshness.unresolvedFileCount),
+            ]
+            : [label, "not assessed", "unavailable", "", "", "", "", "", ""];
+    }));
+    const detailRows = tasks.flatMap((task) => {
+        const freshness = task.indexFreshness;
+        if (!freshness)
+            return [];
+        const label = `${task.taskOrdinal}. ${task.caseId}`;
+        return [
+            ...freshness.changes.items.map((change) => [label, change.changeType, change.path, ""]),
+            ...freshness.unresolved.items.map((entry) => [label, "unresolved", entry.path ?? "run-level", `${entry.reasonCode}: ${entry.message}`]),
+        ];
+    });
+    const omitted = tasks.flatMap((task) => {
+        const freshness = task.indexFreshness;
+        if (!freshness)
+            return [];
+        return [
+            ...(freshness.changes.omittedCount > 0 ? [`Task ${task.taskOrdinal}: ${freshness.changes.omittedCount} of ${freshness.changes.totalCount} changes omitted.`] : []),
+            ...(freshness.unresolved.omittedCount > 0
+                ? [`Task ${task.taskOrdinal}: ${freshness.unresolved.omittedCount} of ${freshness.unresolved.totalCount} unresolved entries omitted.`]
+                : []),
+        ];
+    });
+    return `<h4>Index Freshness</h4>
+    ${overview}
+    ${detailRows.length === 0 ? "" : table(["Task", "Evidence type", "Path", "Detail"], detailRows)}
+    ${omitted.length === 0 ? "" : list(omitted)}`;
+}
+function renderCampaignSection(campaign) {
+    return `<h3>Real-Agent Campaign</h3>
+    ${table(["Field", "Value"], [
+        ["Preset", campaign.presetId],
+        ["Agent", campaign.agentId],
+        ["Timeout (ms)", String(campaign.timeoutMs)],
+        ["Scheduled agent sides", String(campaign.scheduledSideCount)],
+        ["Executed agent sides", String(campaign.executedSideCount)],
+        ["Not run for missing context", String(campaign.notRunForMissingContextCount)],
+        ["Agent evidence status", campaign.agentEvidenceStatus],
+        ["Token evidence status", campaign.tokenEvidenceStatus],
+        ["Completed agent sides", String(campaign.outcomeCounts.completed)],
+        ["Failed agent sides", String(campaign.outcomeCounts.failed)],
+        ["Timeout agent sides", String(campaign.outcomeCounts.timeout)],
+        ["Invalid-output agent sides", String(campaign.outcomeCounts.invalidOutput)],
+        ["Agent-unavailable sides", String(campaign.outcomeCounts.agentUnavailable)],
+        ["Agent-limit-reached sides", String(campaign.outcomeCounts.agentLimitReached)],
+        ["Skipped agent sides", String(campaign.outcomeCounts.skipped)],
+    ])}`;
+}
+function renderProject(project, index, agentId) {
     const unavailableRows = project.tasks.flatMap((task) => [
         ["Raw context characters", task.raw.contextCharacters],
         ["Warm context characters", task.warm.contextCharacters],
@@ -42,12 +134,12 @@ function renderProject(project, index) {
         ["Cumulative warm component duration", task.warm.cumulativeComponentDurationMs],
         ["Cumulative raw estimated context tokens", task.raw.cumulativeEstimatedContextTokens],
         ["Cumulative warm estimated context tokens", task.warm.cumulativeEstimatedContextTokens],
-        ["Raw fake-agent correctness", task.raw.agentCorrectness],
-        ["Warm fake-agent correctness", task.warm.agentCorrectness],
-        ["Raw fake-agent total tokens", task.raw.agentTotalTokens],
-        ["Warm fake-agent total tokens", task.warm.agentTotalTokens],
-        ["Cumulative raw fake-agent total tokens", task.raw.cumulativeAgentTotalTokens],
-        ["Cumulative warm fake-agent total tokens", task.warm.cumulativeAgentTotalTokens],
+        ["Raw agent correctness", task.raw.agentCorrectness],
+        ["Warm agent correctness", task.warm.agentCorrectness],
+        ["Raw agent total tokens", task.raw.agentTotalTokens],
+        ["Warm agent total tokens", task.warm.agentTotalTokens],
+        ["Cumulative raw agent total tokens", task.raw.cumulativeAgentTotalTokens],
+        ["Cumulative warm agent total tokens", task.warm.cumulativeAgentTotalTokens],
     ]
         .filter(([, metric]) => metric.availability !== "available")
         .map(([label, metric]) => [String(task.taskOrdinal), task.caseId, label, metric.availability, metric.reason ?? ""]));
@@ -91,12 +183,13 @@ function renderProject(project, index) {
         formatMetric(task.raw.cumulativeEstimatedContextTokens),
         formatMetric(task.warm.cumulativeEstimatedContextTokens),
     ]))}
+    ${renderProjectFreshness(project.tasks)}
     <p class="muted">Estimated tokens are character-based context-size estimates, not provider token usage.</p>
-    <p class="muted">Agent columns are deterministic fake-agent evidence; token totals are simulated harness telemetry, not provider billing telemetry.</p>
+    <p class="muted">${agentColumnsNote(agentId)}</p>
     ${table([
         "Task",
-        "Raw fake-agent status",
-        "Warm fake-agent status",
+        "Raw agent status",
+        "Warm agent status",
         "Raw correctness",
         "Warm correctness",
         "Raw agent tokens",
@@ -120,11 +213,18 @@ function renderProject(project, index) {
     ]))}
     ${unavailableRows.length === 0 ? "" : table(["Task", "Case", "Metric", "Availability", "Reason"], unavailableRows)}`;
 }
+function agentColumnsNote(agentId) {
+    if (agentId === "fake-agent") {
+        return "Agent columns are deterministic fake-agent evidence; token totals are simulated harness telemetry, not provider billing telemetry.";
+    }
+    const label = agentLabel(agentId);
+    return `Agent columns are ${label} campaign evidence; token totals are provider-reported adapter telemetry when available and are never replaced by a context-size estimate.`;
+}
 function formatAgent(agent) {
     if (agent === null)
         return "not run";
     const passed = agent.passed === null ? "" : agent.passed ? ", passed" : ", not passed";
-    return `${agent.status}${passed}${agent.errors[0] ? ` (${agent.errors[0]})` : ""}`;
+    return `${agentLabel(agent.agentId)}: ${agent.status}${passed}${agent.errors[0] ? ` (${agent.errors[0]})` : ""}`;
 }
 function formatMetric(metric) {
     return metric.availability === "available" ? `${metric.value} ${UNIT_LABELS[metric.unit]}` : metric.availability;

package/dist/src/report/experiments/warmIndexReuseReportModel.js

@@ -1 +1,4 @@
 export const WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION = "my-dev-kit-lab-warm-index-report-v1";
+/** Report presentation limits; the full persisted lists stay in warm-index-execution.json. */
+export const MAX_REPORT_FRESHNESS_CHANGES = 20;
+export const MAX_REPORT_FRESHNESS_UNRESOLVED = 10;

package/docs/METRICS.md

@@ -26,7 +26,7 @@ Each entry below uses the same fields: **Meaning**, **Appears in** (the artifact
 **Token savings notes:**
 - A positive token savings value means the my-dev-kit-guided strategy used fewer tokens than raw-full-file for that run pair.
 - A negative token savings value means my-dev-kit-guided used more tokens. This can happen on small projects where raw-full-file is cheaper.
-- Token savings are only computed when both paired runs expose token totals. Claude does not expose token totals. Codex may expose token totals but can produce timeouts or invalid-output runs.
+- Token savings are only computed when both paired runs expose token totals. Claude's token totals depend on that run's actual CLI output and may be unavailable; Codex may expose token totals but can produce timeouts or invalid-output runs. See "Warm-index real-agent campaign metrics" below for the explicit per-run `tokenEvidenceStatus` signal used by the `warm-index-reuse` campaign path.
 - Token counts in fake-agent runs are estimated using `Math.ceil(characterCount / 4)`. These are context-size estimates, not provider billing totals.
 
 **Correctness scoring notes:**
@@ -515,4 +515,41 @@ The plugin evaluates each task side that has context evidence once with the dete
 
 **Generic outcome metrics.** Available values also appear as `ExperimentMetric` entries on each outcome, with `variantId`, `caseId`, `unit`, and `description`. Both the `raw-full-file` and `warm-index-reuse` outcomes can carry `context-character-count`, `context-estimated-token-count`, `operation-duration-ms` (raw context construction or warm retrieval), `cumulative-component-duration-ms`, `cumulative-context-estimated-token-count`, `agent-correctness-score`, `agent-total-tokens`, and `cumulative-agent-total-tokens`. The warm outcome also carries `amortized-index-build-duration-ms`. Unavailable metrics are omitted rather than emitted as zero. Run-level metrics are limited to `warm-index-project-count`, `warm-index-task-count`, and `warm-index-session-prepared-project-count`.
 
-**Interpretation.** The report separates the one-time index cost from per-task retrieval cost and shows the fixed cost falling per task as more tasks reuse the index. It calculates no token-savings percentage, duration-reduction percentage, break-even task, composite score, winner, or ranking. Warm-index plots map the same precomputed values: amortized index build duration, raw versus retrieved estimated context tokens, fake-agent correctness, and cumulative fake-agent total tokens. Unavailable values become skipped plot points with their reason.
+**Interpretation.** The report separates the one-time index cost from per-task retrieval cost and shows the fixed cost falling per task as more tasks reuse the index. It calculates no token-savings percentage, duration-reduction percentage, break-even task, composite score, winner, or ranking. Warm-index plots map the same precomputed values: amortized index build duration, raw versus retrieved estimated context tokens, fake-agent correctness (or campaign agent correctness for a `--campaign` run), and cumulative fake-agent total tokens. Unavailable values become skipped plot points with their reason.
+
+### Warm-index real-agent campaign metrics (v0.5.2, released)
+
+Available in the installed v0.5.2 CLI. A `--campaign` run replaces the deterministic fake-agent evaluation described above with one real Codex or Claude provider evaluation per task side with context evidence; every direct measurement, derived measurement, and the strict-prefix rule above are unchanged and reused as-is. Only the per-side agent evidence and its rollup are additive.
+
+- `presetId`, `agentId`, `timeoutMs`, `selectedCaseCount`
+  Meaning: which campaign preset ran, which single provider (`codex` or `claude`) it used, its resolved timeout, and how many cases were selected.
+- `scheduledSideCount`, `executedSideCount`, `notRunForMissingContextCount`
+  Meaning: how many task sides were scheduled for real-agent evaluation, how many actually ran, and how many were not run because that side had no context evidence (for example, a warm side with no prepared index).
+- `outcomeCounts` (`completed`, `failed`, `timeout`, `invalidOutput`, `agentUnavailable`, `agentLimitReached`, `skipped`)
+  Meaning: exact counts of each real-agent outcome across executed sides, from `classifyAgentRunOutcome`'s status mapping. None of these are folded into a single pass/fail count.
+- `agentEvidenceStatus` (`complete`, `partial`, `unavailable`)
+  Meaning: `complete` when every scheduled side executed and every executed side reported `completed`; `unavailable` when no side executed; `partial` otherwise.
+  Interpretation: this is infrastructure/provider evidence completeness, not correctness. A campaign can report `agentEvidenceStatus: "partial"` while the underlying index/execution infrastructure completed normally — infrastructure status and provider status are reported separately, never conflated.
+- `tokenEvidenceStatus` (`complete`, `partial`, `unavailable`)
+  Meaning: `complete` when every executed side reports a finite total-token count; `unavailable` when no executed side does; `partial` otherwise.
+  Caveat: Claude and Codex token exposure depends on that provider's actual CLI output for the run; token evidence is reported as `unavailable`/`partial` explicitly rather than assumed. No fixed claim that either provider always or never exposes a token total is made — `tokenEvidenceStatus` is the authoritative, per-run signal.
+
+No token-savings percentage, break-even task, winner, ranking, or provider billing metric is calculated for a campaign run, matching the fake-agent path above.
+
+### Index freshness evidence (v0.6.0)
+
+Status: released in v0.6.0 and present in the current package. The freshness values below are report evidence, not metric formulas: no `ExperimentMetric` ID, plot, or gallery item is added for them.
+
+Before each task's warm retrieval, the warm-index run compares the files the index snapshot proves were indexed against their current state and persists one assessment per task (`indexFreshness` in `warm-index-execution.json`). The report only presents that persisted assessment; it never re-reads the filesystem, re-hashes files, or invokes my-dev-kit.
+
+- **Snapshot-scoped comparison.** Only files listed by the index snapshot are compared. Freshness does not establish that the whole repository is unchanged.
+- **SHA-256 is the authority.** A file is modified when its current SHA-256 differs from the snapshot's, or missing when it no longer exists. A changed modified time with identical content is not a change; timestamps are diagnostic metadata only.
+- **New and non-indexed files.** A file that was not represented in the snapshot is outside the comparison and is not, by itself, evidence that the index is stale. Lab does not guess which files my-dev-kit would index.
+- **Four statuses.**
+  - `fresh`: the snapshot was complete, every represented file was compared, and every content identity still matches.
+  - `stale`: a complete comparison confirmed at least one represented file is modified or missing.
+  - `partially-stale`: at least one represented file is confirmed changed, but comparison evidence is incomplete (a partial snapshot or unresolved comparisons).
+  - `unknown`: no confirmed change established staleness, but evidence is insufficient to prove freshness (an unavailable or partial snapshot, or an unresolved comparison). It is never treated as `fresh`.
+- **`unknown` versus not assessed.** `unknown` means an assessment ran and could not prove freshness. A task with no assessment (for example a report from before v0.6.0) is shown as `not assessed` and counted as unassessed, never as `unknown`.
+- **Report-level counts.** `report.warmIndexReuse.indexFreshnessSummary` holds direct presentation counts of assessed, unassessed, `fresh`, `stale`, `partially-stale`, and `unknown` tasks (assessed equals the sum of the four statuses). They are counts, not `ExperimentMetric` formulas: no percentage, score, or ranking is calculated. Each task's `indexFreshness` lists at most 20 changed files and 10 unresolved comparisons, with the persisted totals and the omitted count; hashes stay in `warm-index-execution.json`.
+- **Observational only.** Freshness does not trigger reindexing, suppress or alter warm retrieval, or change execution status, correctness, provider or agent outcome status, or token evidence. v0.6.0 makes no reindex recommendation.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {