@dailephd/my-dev-kit-lab 0.5.0 → 0.5.2

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -100,15 +100,42 @@ function buildFindings(run) {
 function buildInterpretation(run, contextStrategyComparisonV043, warmIndexReuse) {
     if (run.pluginId === "warm-index-reuse" && warmIndexReuse) {
         const summary = warmIndexReuse.summary;
+        const campaign = warmIndexReuse.agentCampaign;
+        if (!campaign) {
+            return {
+                summary: `The warm-index run prepared ${summary.preparedSessionProjectCount} of ${summary.projectCount} project indexes and evaluated ${summary.taskCount} tasks. ` +
+                    "The report separates one-time index-build duration from per-task retrieval duration and shows how the fixed build cost is amortized across repeated tasks. " +
+                    `Deterministic fake-agent correctness/token evidence is available for ${summary.agentCorrectnessAvailableCount} of ${summary.agentSideCount} task sides (correctness) and ${summary.agentTotalTokensAvailableCount} (total tokens); ` +
+                    "fake-agent totals are simulated harness evidence, not real-model or provider measurements. " +
+                    "Estimated context-token values are context-size estimates, not provider token usage.",
+                recommendedNextStep: run.status === "completed"
+                    ? "Review per-task measurements and cumulative component costs for each project; the variants are reported side by side and are not ranked."
+                    : "Inspect unavailable metrics, warnings, and failures before drawing conclusions from this run.",
+            };
+        }
+        const nonCompletedCounts = [
+            ["failed", campaign.outcomeCounts.failed],
+            ["timeout", campaign.outcomeCounts.timeout],
+            ["invalid-output", campaign.outcomeCounts.invalidOutput],
+            ["agent-unavailable", campaign.outcomeCounts.agentUnavailable],
+            ["agent-limit-reached", campaign.outcomeCounts.agentLimitReached],
+            ["skipped", campaign.outcomeCounts.skipped],
+        ]
+            .filter(([, count]) => count > 0)
+            .map(([label, count]) => `${label}=${count}`)
+            .join(", ");
+        const isInfrastructureComplete = run.status === "completed";
+        const isAgentEvidenceComplete = campaign.agentEvidenceStatus === "complete";
         return {
-            summary: `The warm-index run prepared ${summary.preparedSessionProjectCount} of ${summary.projectCount} project indexes and evaluated ${summary.taskCount} tasks. ` +
-                "The report separates one-time index-build duration from per-task retrieval duration and shows how the fixed build cost is amortized across repeated tasks. " +
-                `Deterministic fake-agent correctness is available for ${summary.agentCorrectnessAvailableCount} of ${summary.agentSideCount} task sides and fake-agent token totals for ${summary.agentTotalTokensAvailableCount}; ` +
-                "these are simulated harness evidence, not real-model or provider measurements. " +
-                "Estimated context-token values are context-size estimates, not provider token usage.",
-            recommendedNextStep: run.status === "completed"
-                ? "Review per-task measurements and cumulative component costs for each project; the variants are reported side by side and are not ranked."
-                : "Inspect unavailable metrics, warnings, and failures before drawing conclusions from this run.",
+            summary: `Campaign preset ${campaign.presetId} selected agent ${campaign.agentId} over ${summary.projectCount} prepared project ` +
+                `index${summary.projectCount === 1 ? "" : "es"} and ${campaign.selectedCaseCount} selected task${campaign.selectedCaseCount === 1 ? "" : "s"}. ` +
+                `Agent evidence status: ${campaign.agentEvidenceStatus} (${campaign.executedSideCount} of ${campaign.scheduledSideCount} scheduled sides executed` +
+                `${nonCompletedCounts ? `; outcome counts: ${nonCompletedCounts}` : ""}). ` +
+                `Token evidence status: ${campaign.tokenEvidenceStatus} (${summary.agentTotalTokensAvailableCount} of ${summary.agentSideCount} task sides have a total-token value). ` +
+                "Context estimated tokens remain a separate character-based context-size estimate and are never substituted for provider telemetry.",
+            recommendedNextStep: isInfrastructureComplete && isAgentEvidenceComplete
+                ? "Review per-task correctness, token provenance, and cumulative component measurements."
+                : "Review infrastructure gaps (index build, raw/warm retrieval) separately from provider/agent limitations before drawing conclusions from this run.",
         };
     }
     if (run.pluginId === "context-strategy-comparison" && (contextStrategyComparisonV043?.summary.strategyCount ?? 0) > 0) {

package/dist/src/report/experiments/buildWarmIndexReuseReport.js

@@ -1,5 +1,28 @@
 import { WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION, } from "./warmIndexReuseReportModel.js";
 const WARM_INDEX_REUSE_PLUGIN_ID = "warm-index-reuse";
+const COMMON_LIMITATIONS = [
+    "Context estimated-token counts use the character-based estimate and are not provider billing telemetry.",
+    "Correctness is deterministic answer-key scoring, not semantic LLM judging.",
+    "Component-duration sums are not equivalent to provider end-to-end latency.",
+    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
+    "No composite score, winner, ranking, or break-even task is calculated.",
+];
+const FAKE_AGENT_LIMITATIONS = [
+    "Agent evidence is deterministic simulated fake-agent evidence.",
+    "Fake-agent total tokens are simulated harness telemetry, not provider billing telemetry.",
+];
+const CODEX_CAMPAIGN_LIMITATIONS = [
+    "Agent evidence comes from Codex CLI campaign execution over the exact runner-supplied benchmark contexts.",
+    "Token totals are CLI-exposed adapter telemetry when available.",
+    "Token evidence must be interpreted using tokenUsageSource/tokenUsageReliability.",
+    "Context estimates are never substituted for missing agent token totals.",
+];
+const CLAUDE_CAMPAIGN_LIMITATIONS = [
+    "Agent evidence comes from Claude CLI campaign execution over the exact runner-supplied benchmark contexts.",
+    "Token totals are used when exposed by the CLI output.",
+    "Token totals may be unavailable for some CLI/version/run outcomes.",
+    "Unavailable token totals remain unavailable and are never replaced by zero or a context estimate.",
+];
 const COST_MODEL = [
     "Index construction is a one-time cost per benchmark project; task retrieval is the repeated warm cost.",
     "Cold start: for the first task in a project, the cumulative warm component duration is the one-time index build plus the first retrieval.",
@@ -8,15 +31,15 @@ const COST_MODEL = [
     "Raw cumulative duration sums one raw-context construction per task; warm cumulative component duration includes one index build plus each warm retrieval once.",
     "Cumulative durations are sums of measured components, not wall-clock or end-to-end latency, and are not presented as a speedup.",
 ];
-const LIMITATIONS = [
-    "Context estimated-token counts use the existing character-based estimate and are not provider billing telemetry.",
-    "Agent evidence comes from the deterministic fake agent only; real agents such as Codex or Claude are not run by warm-index-reuse in v0.5.0.",
-    "Fake-agent correctness is deterministic benchmark-harness evidence from the existing answer-key scorer, not a measure of real-model correctness.",
-    "Fake-agent token totals are simulated harness telemetry, not provider billing telemetry; estimated context tokens are a separate character-based context-size estimate.",
-    "Component-duration sums are not equivalent to provider end-to-end latency.",
-    "Failed or missing prefix measurements make cumulative metrics unavailable; missing values are never treated as zero.",
-    "No composite score, winner, ranking, or break-even task is calculated.",
-];
+const OUTCOME_STATUS_TO_COUNT_KEY = {
+    completed: "completed",
+    failed: "failed",
+    timeout: "timeout",
+    "invalid-output": "invalidOutput",
+    "agent-unavailable": "agentUnavailable",
+    "agent-limit-reached": "agentLimitReached",
+    skipped: "skipped",
+};
 /**
  * Presents the warm-index metrics already calculated on the run. It performs no amortization,
  * cumulative, or availability calculation of its own.
@@ -31,6 +54,14 @@ export function buildWarmIndexReuseReport(run) {
     if (executions.length !== metricProjects.length) {
         throw new Error("Invalid warm-index report source: project executions and metrics are inconsistent.");
     }
+    const campaignPreset = readString(run.metadata?.campaignPreset);
+    const campaignAgentId = readString(run.metadata?.campaignAgentId);
+    const campaignTimeoutMs = readNumber(run.metadata?.campaignTimeoutMs);
+    if (campaignAgentId !== undefined && campaignAgentId !== "codex" && campaignAgentId !== "claude") {
+        throw new Error(`Invalid warm-index report source: unsupported campaign agent id ${campaignAgentId}.`);
+    }
+    const isCampaign = campaignPreset !== undefined;
+    const allSides = [];
     const projects = executions.map((execution, projectIndex) => {
         const metrics = metricProjects[projectIndex];
         if (metrics.benchmarkProject !== execution.benchmarkProject ||
@@ -47,6 +78,9 @@ export function buildWarmIndexReuseReport(run) {
             taskCount: metrics.tasks.length,
             tasks: metrics.tasks.map((task, taskIndex) => {
                 const agents = agentProjects?.[projectIndex]?.tasks[taskIndex];
+                const rawEvidence = agents?.raw ?? null;
+                const warmEvidence = agents?.warm ?? null;
+                allSides.push(rawEvidence, warmEvidence);
                 return {
                     taskOrdinal: task.taskOrdinal,
                     caseId: task.caseId,
@@ -54,12 +88,22 @@ export function buildWarmIndexReuseReport(run) {
                     warmStatus: execution.tasks[taskIndex].warmStatus,
                     raw: task.raw,
                     warm: task.warm,
-                    rawAgent: toReportAgent(agents?.raw ?? null),
-                    warmAgent: toReportAgent(agents?.warm ?? null),
+                    rawAgent: toReportAgent(rawEvidence),
+                    warmAgent: toReportAgent(warmEvidence),
                 };
             }),
         };
     });
+    const agentIdentity = deriveAgentIdentity({ isCampaign, campaignAgentId, sides: allSides });
+    const agentCampaign = isCampaign
+        ? buildCampaignSummary({
+            presetId: campaignPreset,
+            agentId: agentIdentity.id,
+            timeoutMs: campaignTimeoutMs ?? 0,
+            selectedCaseCount: projects.reduce((sum, project) => sum + project.taskCount, 0),
+            sides: allSides,
+        })
+        : null;
     const sides = projects.flatMap((project) => project.tasks.flatMap((task) => [task.raw, task.warm]));
     return {
         schemaVersion: WARM_INDEX_REUSE_REPORT_SCHEMA_VERSION,
@@ -73,14 +117,90 @@ export function buildWarmIndexReuseReport(run) {
             agentTotalTokensAvailableCount: sides.filter((side) => side.agentTotalTokens.availability === "available").length,
         },
         costModel: [...COST_MODEL],
-        limitations: [...LIMITATIONS],
+        limitations: buildLimitations(agentIdentity),
         projects,
+        agent: agentIdentity,
+        agentCampaign,
+    };
+}
+function deriveAgentIdentity(args) {
+    const nonNullAgentIds = new Set(args.sides.filter((side) => side !== null).map((side) => side.agentId));
+    if (!args.isCampaign) {
+        if ([...nonNullAgentIds].some((agentId) => agentId !== "fake-agent")) {
+            throw new Error(`Invalid warm-index report source: a legacy non-campaign run must not contain non-fake agent evidence, found ${[...nonNullAgentIds].join(", ")}.`);
+        }
+        return { id: "fake-agent", mode: "deterministic-fake" };
+    }
+    const campaignAgentId = args.campaignAgentId;
+    if (!campaignAgentId) {
+        throw new Error("Invalid warm-index report source: a campaign run must carry a campaignAgentId.");
+    }
+    for (const agentId of nonNullAgentIds) {
+        if (agentId !== campaignAgentId) {
+            throw new Error(`Invalid warm-index report source: campaign metadata says ${campaignAgentId} but agent evidence contains ${agentId}.`);
+        }
+    }
+    return { id: campaignAgentId, mode: "real-provider" };
+}
+function buildCampaignSummary(args) {
+    const scheduledSideCount = args.selectedCaseCount * 2;
+    const executedSides = args.sides.filter((side) => side !== null);
+    const executedSideCount = executedSides.length;
+    const notRunForMissingContextCount = scheduledSideCount - executedSideCount;
+    const outcomeCounts = {
+        completed: 0,
+        failed: 0,
+        timeout: 0,
+        invalidOutput: 0,
+        agentUnavailable: 0,
+        agentLimitReached: 0,
+        skipped: 0,
+    };
+    for (const side of executedSides) {
+        const key = OUTCOME_STATUS_TO_COUNT_KEY[side.status];
+        if (!key) {
+            throw new Error(`Invalid warm-index report source: unsupported campaign agent status "${side.status}".`);
+        }
+        outcomeCounts[key] += 1;
+    }
+    const tokenAvailableSideCount = executedSides.filter((side) => typeof side.tokenUsage.totalTokens === "number" && Number.isFinite(side.tokenUsage.totalTokens)).length;
+    const agentEvidenceStatus = executedSideCount === 0
+        ? "unavailable"
+        : executedSideCount === scheduledSideCount && outcomeCounts.completed === executedSideCount
+            ? "complete"
+            : "partial";
+    const tokenEvidenceStatus = executedSideCount === 0
+        ? "unavailable"
+        : tokenAvailableSideCount === 0
+            ? "unavailable"
+            : tokenAvailableSideCount === executedSideCount
+                ? "complete"
+                : "partial";
+    return {
+        presetId: args.presetId,
+        agentId: args.agentId,
+        timeoutMs: args.timeoutMs,
+        selectedCaseCount: args.selectedCaseCount,
+        scheduledSideCount,
+        executedSideCount,
+        notRunForMissingContextCount,
+        outcomeCounts,
+        agentEvidenceStatus,
+        tokenEvidenceStatus,
     };
 }
+function buildLimitations(agent) {
+    if (agent.mode === "deterministic-fake") {
+        return [...COMMON_LIMITATIONS, ...FAKE_AGENT_LIMITATIONS];
+    }
+    const providerLimitations = agent.id === "codex" ? CODEX_CAMPAIGN_LIMITATIONS : CLAUDE_CAMPAIGN_LIMITATIONS;
+    return [...COMMON_LIMITATIONS, ...providerLimitations];
+}
 function toReportAgent(agent) {
     if (!agent)
         return null;
     return {
+        agentId: agent.agentId,
         status: agent.status,
         passed: agent.correctness.passed,
         tokenUsageSource: agent.tokenUsage.source,
@@ -89,3 +209,9 @@ function toReportAgent(agent) {
         errors: [...agent.errors],
     };
 }
+function readString(value) {
+    return typeof value === "string" && value ? value : undefined;
+}
+function readNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+}

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -265,10 +265,28 @@ function formatWarmMetric(metric) {
 }
 function formatWarmAgent(agent) {
     if (agent === null)
-        return "not run (no context evidence for this side)";
+        return "not run (no usable context evidence for this side)";
     const passed = agent.passed === null ? "unavailable" : String(agent.passed);
     const issues = [...agent.errors, ...agent.warnings].slice(0, 3).join("; ");
-    return `status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
+    return `agent ${agent.agentId}, status ${agent.status}, passed ${passed}, token source ${agent.tokenUsageSource}, reliability ${agent.tokenUsageReliability}${issues ? `, notes: ${issues}` : ""}`;
+}
+function renderCampaignSection(lines, campaign) {
+    pushSection(lines, "Real-Agent Campaign");
+    lines.push(fieldLine("Campaign Preset", campaign.presetId));
+    lines.push(fieldLine("Agent ID", campaign.agentId));
+    lines.push(fieldLine("Campaign Timeout", `${campaign.timeoutMs} ms`));
+    lines.push(fieldLine("Scheduled Agent Sides", campaign.scheduledSideCount));
+    lines.push(fieldLine("Executed Agent Sides", campaign.executedSideCount));
+    lines.push(fieldLine("Not Run For Missing Context", campaign.notRunForMissingContextCount));
+    lines.push(fieldLine("Agent Evidence Status", campaign.agentEvidenceStatus));
+    lines.push(fieldLine("Token Evidence Status", campaign.tokenEvidenceStatus));
+    lines.push(fieldLine("Completed Agent Sides", campaign.outcomeCounts.completed));
+    lines.push(fieldLine("Failed Agent Sides", campaign.outcomeCounts.failed));
+    lines.push(fieldLine("Timeout Agent Sides", campaign.outcomeCounts.timeout));
+    lines.push(fieldLine("Invalid-Output Agent Sides", campaign.outcomeCounts.invalidOutput));
+    lines.push(fieldLine("Agent-Unavailable Sides", campaign.outcomeCounts.agentUnavailable));
+    lines.push(fieldLine("Agent-Limit-Reached Sides", campaign.outcomeCounts.agentLimitReached));
+    lines.push(fieldLine("Skipped Agent Sides", campaign.outcomeCounts.skipped));
 }
 function renderWarmIndexReuseSection(lines, section) {
     if (section === null) {
@@ -279,8 +297,13 @@ function renderWarmIndexReuseSection(lines, section) {
     lines.push(fieldLine("Task Count", section.summary.taskCount));
     lines.push(fieldLine("Prepared Session Project Count", section.summary.preparedSessionProjectCount));
     lines.push(fieldLine("Incomplete Or Failed Project Count", section.summary.incompleteProjectCount));
-    lines.push(fieldLine("Fake-Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
-    lines.push(fieldLine("Fake-Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push(fieldLine("Agent ID", section.agent.id));
+    lines.push(fieldLine("Agent Mode", section.agent.mode));
+    lines.push(fieldLine("Agent Correctness Available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`));
+    lines.push(fieldLine("Agent Token Totals Available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`));
+    if (section.agentCampaign) {
+        renderCampaignSection(lines, section.agentCampaign);
+    }
     lines.push("Cold Start And Warm Reuse:");
     pushDashList(lines, section.costModel);
     lines.push("Limitations:");
@@ -312,14 +335,14 @@ function renderWarmIndexReuseSection(lines, section) {
                 ["Cumulative Warm Component Duration", formatWarmMetric(task.warm.cumulativeComponentDurationMs)],
                 ["Cumulative Raw Estimated Context Tokens (estimate)", formatWarmMetric(task.raw.cumulativeEstimatedContextTokens)],
                 ["Cumulative Warm Estimated Context Tokens (estimate)", formatWarmMetric(task.warm.cumulativeEstimatedContextTokens)],
-                ["Raw Fake-Agent Evaluation", formatWarmAgent(task.rawAgent)],
-                ["Warm Fake-Agent Evaluation", formatWarmAgent(task.warmAgent)],
-                ["Raw Agent Correctness (fake agent)", formatWarmMetric(task.raw.agentCorrectness)],
-                ["Warm Agent Correctness (fake agent)", formatWarmMetric(task.warm.agentCorrectness)],
-                ["Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.agentTotalTokens)],
-                ["Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.agentTotalTokens)],
-                ["Cumulative Raw Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
-                ["Cumulative Warm Agent Total Tokens (fake-agent simulated)", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
+                ["Raw Agent Evaluation", formatWarmAgent(task.rawAgent)],
+                ["Warm Agent Evaluation", formatWarmAgent(task.warmAgent)],
+                ["Raw Agent Correctness", formatWarmMetric(task.raw.agentCorrectness)],
+                ["Warm Agent Correctness", formatWarmMetric(task.warm.agentCorrectness)],
+                ["Raw Agent Total Tokens", formatWarmMetric(task.raw.agentTotalTokens)],
+                ["Warm Agent Total Tokens", formatWarmMetric(task.warm.agentTotalTokens)],
+                ["Cumulative Raw Agent Total Tokens", formatWarmMetric(task.raw.cumulativeAgentTotalTokens)],
+                ["Cumulative Warm Agent Total Tokens", formatWarmMetric(task.warm.cumulativeAgentTotalTokens)],
             ];
             for (const [label, value] of rows) {
                 lines.push(fieldLine(label, value));

package/dist/src/report/experiments/renderWarmIndexReuseHtml.js

@@ -1,3 +1,4 @@
+import { agentLabel } from "../../experiments/plugins/warmIndexReuse/metrics.js";
 const UNIT_LABELS = {
     ms: "ms",
     characters: "chars",
@@ -19,17 +20,40 @@ export function renderWarmIndexReuseHtml(section) {
         ["Tasks", String(section.summary.taskCount)],
         ["Projects with a prepared index session", String(section.summary.preparedSessionProjectCount)],
         ["Incomplete or failed projects", String(section.summary.incompleteProjectCount)],
-        ["Fake-agent correctness available (task sides)", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`],
-        ["Fake-agent token totals available (task sides)", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`],
+        ["Agent ID", section.agent.id],
+        ["Agent mode", section.agent.mode],
+        ["Agent correctness available", `${section.summary.agentCorrectnessAvailableCount} of ${section.summary.agentSideCount}`],
+        ["Agent token totals available", `${section.summary.agentTotalTokensAvailableCount} of ${section.summary.agentSideCount}`],
     ])}
+    ${section.agentCampaign ? renderCampaignSection(section.agentCampaign) : ""}
     <h3>Cold Start And Warm Reuse</h3>
     ${list(section.costModel)}
     <h3>Limitations</h3>
     ${list(section.limitations)}
-    ${section.projects.length === 0 ? "<p>No warm-index project evidence was recorded.</p>" : section.projects.map(renderProject).join("\n")}
+    ${section.projects.length === 0 ? "<p>No warm-index project evidence was recorded.</p>" : section.projects.map((project, index) => renderProject(project, index, section.agent.id)).join("\n")}
   </section>`;
 }
-function renderProject(project, index) {
+function renderCampaignSection(campaign) {
+    return `<h3>Real-Agent Campaign</h3>
+    ${table(["Field", "Value"], [
+        ["Preset", campaign.presetId],
+        ["Agent", campaign.agentId],
+        ["Timeout (ms)", String(campaign.timeoutMs)],
+        ["Scheduled agent sides", String(campaign.scheduledSideCount)],
+        ["Executed agent sides", String(campaign.executedSideCount)],
+        ["Not run for missing context", String(campaign.notRunForMissingContextCount)],
+        ["Agent evidence status", campaign.agentEvidenceStatus],
+        ["Token evidence status", campaign.tokenEvidenceStatus],
+        ["Completed agent sides", String(campaign.outcomeCounts.completed)],
+        ["Failed agent sides", String(campaign.outcomeCounts.failed)],
+        ["Timeout agent sides", String(campaign.outcomeCounts.timeout)],
+        ["Invalid-output agent sides", String(campaign.outcomeCounts.invalidOutput)],
+        ["Agent-unavailable sides", String(campaign.outcomeCounts.agentUnavailable)],
+        ["Agent-limit-reached sides", String(campaign.outcomeCounts.agentLimitReached)],
+        ["Skipped agent sides", String(campaign.outcomeCounts.skipped)],
+    ])}`;
+}
+function renderProject(project, index, agentId) {
     const unavailableRows = project.tasks.flatMap((task) => [
         ["Raw context characters", task.raw.contextCharacters],
         ["Warm context characters", task.warm.contextCharacters],
@@ -42,12 +66,12 @@ function renderProject(project, index) {
         ["Cumulative warm component duration", task.warm.cumulativeComponentDurationMs],
         ["Cumulative raw estimated context tokens", task.raw.cumulativeEstimatedContextTokens],
         ["Cumulative warm estimated context tokens", task.warm.cumulativeEstimatedContextTokens],
-        ["Raw fake-agent correctness", task.raw.agentCorrectness],
-        ["Warm fake-agent correctness", task.warm.agentCorrectness],
-        ["Raw fake-agent total tokens", task.raw.agentTotalTokens],
-        ["Warm fake-agent total tokens", task.warm.agentTotalTokens],
-        ["Cumulative raw fake-agent total tokens", task.raw.cumulativeAgentTotalTokens],
-        ["Cumulative warm fake-agent total tokens", task.warm.cumulativeAgentTotalTokens],
+        ["Raw agent correctness", task.raw.agentCorrectness],
+        ["Warm agent correctness", task.warm.agentCorrectness],
+        ["Raw agent total tokens", task.raw.agentTotalTokens],
+        ["Warm agent total tokens", task.warm.agentTotalTokens],
+        ["Cumulative raw agent total tokens", task.raw.cumulativeAgentTotalTokens],
+        ["Cumulative warm agent total tokens", task.warm.cumulativeAgentTotalTokens],
     ]
         .filter(([, metric]) => metric.availability !== "available")
         .map(([label, metric]) => [String(task.taskOrdinal), task.caseId, label, metric.availability, metric.reason ?? ""]));
@@ -92,11 +116,11 @@ function renderProject(project, index) {
         formatMetric(task.warm.cumulativeEstimatedContextTokens),
     ]))}
     <p class="muted">Estimated tokens are character-based context-size estimates, not provider token usage.</p>
-    <p class="muted">Agent columns are deterministic fake-agent evidence; token totals are simulated harness telemetry, not provider billing telemetry.</p>
+    <p class="muted">${agentColumnsNote(agentId)}</p>
     ${table([
         "Task",
-        "Raw fake-agent status",
-        "Warm fake-agent status",
+        "Raw agent status",
+        "Warm agent status",
         "Raw correctness",
         "Warm correctness",
         "Raw agent tokens",
@@ -120,11 +144,18 @@ function renderProject(project, index) {
     ]))}
     ${unavailableRows.length === 0 ? "" : table(["Task", "Case", "Metric", "Availability", "Reason"], unavailableRows)}`;
 }
+function agentColumnsNote(agentId) {
+    if (agentId === "fake-agent") {
+        return "Agent columns are deterministic fake-agent evidence; token totals are simulated harness telemetry, not provider billing telemetry.";
+    }
+    const label = agentLabel(agentId);
+    return `Agent columns are ${label} campaign evidence; token totals are provider-reported adapter telemetry when available and are never replaced by a context-size estimate.`;
+}
 function formatAgent(agent) {
     if (agent === null)
         return "not run";
     const passed = agent.passed === null ? "" : agent.passed ? ", passed" : ", not passed";
-    return `${agent.status}${passed}${agent.errors[0] ? ` (${agent.errors[0]})` : ""}`;
+    return `${agentLabel(agent.agentId)}: ${agent.status}${passed}${agent.errors[0] ? ` (${agent.errors[0]})` : ""}`;
 }
 function formatMetric(metric) {
     return metric.availability === "available" ? `${metric.value} ${UNIT_LABELS[metric.unit]}` : metric.availability;

package/docs/METRICS.md

@@ -26,7 +26,7 @@ Each entry below uses the same fields: **Meaning**, **Appears in** (the artifact
 **Token savings notes:**
 - A positive token savings value means the my-dev-kit-guided strategy used fewer tokens than raw-full-file for that run pair.
 - A negative token savings value means my-dev-kit-guided used more tokens. This can happen on small projects where raw-full-file is cheaper.
-- Token savings are only computed when both paired runs expose token totals. Claude does not expose token totals. Codex may expose token totals but can produce timeouts or invalid-output runs.
+- Token savings are only computed when both paired runs expose token totals. Claude's token totals depend on that run's actual CLI output and may be unavailable; Codex may expose token totals but can produce timeouts or invalid-output runs. See "Warm-index real-agent campaign metrics" below for the explicit per-run `tokenEvidenceStatus` signal used by the `warm-index-reuse` campaign path.
 - Token counts in fake-agent runs are estimated using `Math.ceil(characterCount / 4)`. These are context-size estimates, not provider billing totals.
 
 **Correctness scoring notes:**
@@ -137,6 +137,15 @@ Each entry below uses the same fields: **Meaning**, **Appears in** (the artifact
   Interpretation: human-readable size category.
   Caveat: coarse label; use the score and metrics for detail.
 
+In the released v0.5.1 benchmark suite, the task statistics (`taskCount`, `expectedRelevantFilesAverage`, `expectedRelevantSymbolsAverage`) of the two warm-index benchmark projects are derived from the dedicated corpus `benchmarks/contracts/warm-index-benchmark-cases.json`: the averages are the mean `expectedFiles` and `expectedSymbols` counts of that project's cases, rounded to two decimals. `npm run verify:benchmarks` fails if a profile disagrees with the corpus. The `benchmark-project-complexity-v1` formula, its weights, and its caps are unchanged; the current values are:
+
+| Project | `taskCount` | `expectedRelevantFilesAverage` | `expectedRelevantSymbolsAverage` | `complexityScore` |
+|---|---|---|---|---|
+| `task-workflow-medium-ts` | 6 | 3.83 | 4.33 | 31 |
+| `task-analytics-large-mixed` | 6 | 4.5 | 4.67 | 47 |
+
+Task locality (`localized`, `cross-module`, `broad-change`) is benchmark metadata on each corpus case. It is not a metric, score, or ranking, and it does not enter the complexity formula.
+
 ## Prompt complexity metrics
 
 - `promptChars`
@@ -444,7 +453,9 @@ Implemented in `src/evaluation/stageContextMetrics` and composed once per run by
 
 ## Warm-index reuse metrics
 
-Implemented in the current release v0.5.0. The `warm-index-reuse` plugin calculates these metrics once, in `src/experiments/plugins/warmIndexReuse/metrics.ts`, from bounded execution summaries and bounded fake-agent evidence. The report section (`report.warmIndexReuse` in `report.json`, and the text and HTML "Warm Index Reuse Evidence" sections) and the four warm-index plots render these precomputed values; they never recalculate a formula.
+Introduced in v0.5.0. The `warm-index-reuse` plugin calculates these metrics once, in `src/experiments/plugins/warmIndexReuse/metrics.ts`, from bounded execution summaries and bounded fake-agent evidence. The report section (`report.warmIndexReuse` in `report.json`, and the text and HTML "Warm Index Reuse Evidence" sections) and the four warm-index plots render these precomputed values; they never recalculate a formula.
+
+The released v0.5.1 expanded benchmark suite adds no warm-index metric and changes no formula: over the dedicated 12-case corpus the same metrics run through task ordinal 6 in each project, and amortization and cumulative sums restart for each project. No token-savings percentage, break-even task, winner, speedup, ranking, or provider billing metric is calculated.
 
 **Availability contract.** Every warm-index metric carries `availability`, `value`, `unit`, `source`, `reason`, and `tokenCountMethod`:
 
@@ -504,4 +515,23 @@ The plugin evaluates each task side that has context evidence once with the dete
 
 **Generic outcome metrics.** Available values also appear as `ExperimentMetric` entries on each outcome, with `variantId`, `caseId`, `unit`, and `description`. Both the `raw-full-file` and `warm-index-reuse` outcomes can carry `context-character-count`, `context-estimated-token-count`, `operation-duration-ms` (raw context construction or warm retrieval), `cumulative-component-duration-ms`, `cumulative-context-estimated-token-count`, `agent-correctness-score`, `agent-total-tokens`, and `cumulative-agent-total-tokens`. The warm outcome also carries `amortized-index-build-duration-ms`. Unavailable metrics are omitted rather than emitted as zero. Run-level metrics are limited to `warm-index-project-count`, `warm-index-task-count`, and `warm-index-session-prepared-project-count`.
 
-**Interpretation.** The report separates the one-time index cost from per-task retrieval cost and shows the fixed cost falling per task as more tasks reuse the index. It calculates no token-savings percentage, duration-reduction percentage, break-even task, composite score, winner, or ranking. Warm-index plots map the same precomputed values: amortized index build duration, raw versus retrieved estimated context tokens, fake-agent correctness, and cumulative fake-agent total tokens. Unavailable values become skipped plot points with their reason.
+**Interpretation.** The report separates the one-time index cost from per-task retrieval cost and shows the fixed cost falling per task as more tasks reuse the index. It calculates no token-savings percentage, duration-reduction percentage, break-even task, composite score, winner, or ranking. Warm-index plots map the same precomputed values: amortized index build duration, raw versus retrieved estimated context tokens, fake-agent correctness (or campaign agent correctness for a `--campaign` run), and cumulative fake-agent total tokens. Unavailable values become skipped plot points with their reason.
+
+### Warm-index real-agent campaign metrics (v0.5.2, released)
+
+Available in the installed v0.5.2 CLI. A `--campaign` run replaces the deterministic fake-agent evaluation described above with one real Codex or Claude provider evaluation per task side with context evidence; every direct measurement, derived measurement, and the strict-prefix rule above are unchanged and reused as-is. Only the per-side agent evidence and its rollup are additive.
+
+- `presetId`, `agentId`, `timeoutMs`, `selectedCaseCount`
+  Meaning: which campaign preset ran, which single provider (`codex` or `claude`) it used, its resolved timeout, and how many cases were selected.
+- `scheduledSideCount`, `executedSideCount`, `notRunForMissingContextCount`
+  Meaning: how many task sides were scheduled for real-agent evaluation, how many actually ran, and how many were not run because that side had no context evidence (for example, a warm side with no prepared index).
+- `outcomeCounts` (`completed`, `failed`, `timeout`, `invalidOutput`, `agentUnavailable`, `agentLimitReached`, `skipped`)
+  Meaning: exact counts of each real-agent outcome across executed sides, from `classifyAgentRunOutcome`'s status mapping. None of these are folded into a single pass/fail count.
+- `agentEvidenceStatus` (`complete`, `partial`, `unavailable`)
+  Meaning: `complete` when every scheduled side executed and every executed side reported `completed`; `unavailable` when no side executed; `partial` otherwise.
+  Interpretation: this is infrastructure/provider evidence completeness, not correctness. A campaign can report `agentEvidenceStatus: "partial"` while the underlying index/execution infrastructure completed normally — infrastructure status and provider status are reported separately, never conflated.
+- `tokenEvidenceStatus` (`complete`, `partial`, `unavailable`)
+  Meaning: `complete` when every executed side reports a finite total-token count; `unavailable` when no executed side does; `partial` otherwise.
+  Caveat: Claude and Codex token exposure depends on that provider's actual CLI output for the run; token evidence is reported as `unavailable`/`partial` explicitly rather than assumed. No fixed claim that either provider always or never exposes a token total is made — `tokenEvidenceStatus` is the authoritative, per-run signal.
+
+No token-savings percentage, break-even task, winner, ranking, or provider billing metric is calculated for a campaign run, matching the fake-agent path above.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {