@dailephd/my-dev-kit-lab 0.9.0 → 0.9.1

package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js

@@ -1,6 +1,7 @@
 import { mkdtemp, readdir, rm } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
+import { normalizeAgentDurationV091, normalizeAgentTokenUsageV091 } from "../../../agents/index.js";
 import { resolveWithinRoot } from "../../../core/pathSafety.js";
 import { isSameOrInside, resolvePhysicalPath } from "../../../evaluation/benchmarkSandbox/pathPolicy.js";
 import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
@@ -52,18 +53,33 @@ function toProviderStatus(status) {
         case "agent-unavailable":
         case "agent-limit-reached":
         case "invalid-output":
+        case "not-invoked":
             return status;
         default:
             return "failed";
     }
 }
+/** Runtime-only projection. The evaluation pipeline copies named legacy fields, never this envelope. */
+export function projectAgentSuccessProviderTelemetryV091(result, providerStatus, durationMs) {
+    const tokenUsage = normalizeAgentTokenUsageV091(result?.tokenUsage ?? { source: "unavailable" }, result?.tokenUsageReliability ?? "unavailable");
+    const conflict = result !== null && result.tokenUsage.source !== result.tokenUsageSource;
+    if (conflict) {
+        // Retain both identities and reliability; disagreement cannot certify either channel.
+        tokenUsage.evidenceLevel = "unavailable";
+        tokenUsage.classificationReason = "unattributed-source";
+    }
+    return {
+        normalizedTelemetry: { providerStatus, tokenUsage, duration: normalizeAgentDurationV091(durationMs) },
+        tokenUsageSourceConflict: conflict ? { usageSource: result.tokenUsage.source, resultSource: result.tokenUsageSource } : null
+    };
+}
 function tokenEvidence(result) {
-    const finite = (value) => (typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null);
+    const normalized = normalizeAgentTokenUsageV091(result.tokenUsage, result.tokenUsageReliability);
     return {
         // Only provider-, agent-, CLI- or telemetry-reported totals count; a text-length estimate is not provider usage.
-        totalTokens: result.tokenUsageSource === "unavailable" || result.tokenUsageSource === "estimated-from-text" ? null : finite(result.tokenUsage.totalTokens),
-        inputTokens: finite(result.tokenUsage.inputTokens),
-        outputTokens: finite(result.tokenUsage.outputTokens),
+        totalTokens: [result.tokenUsageSource, result.tokenUsage.source].some((source) => source === "unavailable" || source === "estimated-from-text") ? null : normalized.totalTokens,
+        inputTokens: normalized.inputTokens,
+        outputTokens: normalized.outputTokens,
         source: result.tokenUsageSource,
         reliability: result.tokenUsageReliability
     };
@@ -95,6 +111,7 @@ export function createRealAgentProposalSource(args) {
     const { settings } = args;
     const attemptNumber = args.attemptNumber ?? 1;
     return async ({ treatmentId, invoke, notInvokedReason }) => {
+        let runtimeTelemetry = projectAgentSuccessProviderTelemetryV091(null, "not-invoked", null);
         const context = args.contexts[treatmentId];
         const task = args.task;
         const contextArtifactPath = context.text !== null ? agentSuccessContextArtifactPath(task.benchmarkProject, task.caseId, treatmentId) : null;
@@ -128,6 +145,7 @@ export function createRealAgentProposalSource(args) {
             realAgent.providerStatusReason = boundMessage(reason, settings.privateRoots);
             return {
                 kind: "no-proposal",
+                ...runtimeTelemetry,
                 realAgent,
                 agentDurationMs: measured.agentDurationMs ?? null,
                 agentTokenUsage: measured.agentTokenUsage ?? null,
@@ -138,7 +156,7 @@ export function createRealAgentProposalSource(args) {
         if (!invoke) {
             realAgent.context.contextArtifactPath = null;
             realAgent.providerStatusReason = boundMessage(notInvokedReason ?? "not-invoked", settings.privateRoots);
-            return { kind: "no-proposal", realAgent, agentDurationMs: null, agentTokenUsage: null, errors: [], contextFile: null };
+            return { kind: "no-proposal", ...runtimeTelemetry, realAgent, agentDurationMs: null, agentTokenUsage: null, errors: [], contextFile: null };
         }
         if (context.availability === "unavailable" || context.text === null) {
             return noProposal(`context-unavailable: ${context.reason ?? "unknown"}`, "CONTEXT_UNAVAILABLE", `The ${treatmentId} context could not be built (${context.reason ?? "unknown"}); the provider was not invoked.`);
@@ -186,6 +204,7 @@ export function createRealAgentProposalSource(args) {
                     promptVariant: buildTransportVariant(task, treatmentId),
                     promptText,
                     promptTransport: "stdin",
+                    trustedExecutable: settings.trustedExecutable,
                     timeoutMs: settings.timeoutMs,
                     requireAvailable: false,
                     env: settings.env,
@@ -218,7 +237,9 @@ export function createRealAgentProposalSource(args) {
                 realAgent.providerStatus = "failed";
             if (notNeutral || !realAgent.providerInvoked)
                 realAgent.agentArtifactDirectory = null;
-            return noProposal(notNeutral ? "provider-cwd-not-neutral" : "provider-invocation-error", notNeutral ? "AGENT_CWD_NOT_NEUTRAL" : "PROVIDER_FAILED", notNeutral ? "The provider working directory overlapped a protected root; the provider was not invoked." : `The provider invocation raised an error: ${invocationError ?? "unknown"}`, { agentDurationMs: notNeutral ? null : Date.now() - wallStarted });
+            const agentDurationMs = notNeutral ? null : Date.now() - wallStarted;
+            runtimeTelemetry = projectAgentSuccessProviderTelemetryV091(null, realAgent.providerStatus, agentDurationMs);
+            return noProposal(notNeutral ? "provider-cwd-not-neutral" : "provider-invocation-error", notNeutral ? "AGENT_CWD_NOT_NEUTRAL" : "PROVIDER_FAILED", notNeutral ? "The provider working directory overlapped a protected root; the provider was not invoked." : `The provider invocation raised an error: ${invocationError ?? "unknown"}`, { agentDurationMs });
         }
         // Provider health is classified with the answer text withheld: patch content must never be read as a limit/timeout.
         const classification = classifyAgentRunOutcome({ agentRunResult: { ...result, finalAnswerText: "" } });
@@ -234,6 +255,7 @@ export function createRealAgentProposalSource(args) {
         };
         const agentDurationMs = Number.isFinite(result.durationMs) && result.durationMs >= 0 ? result.durationMs : Date.now() - wallStarted;
         const agentTokenUsage = tokenEvidence(result);
+        runtimeTelemetry = projectAgentSuccessProviderTelemetryV091(result, providerStatus, agentDurationMs);
         if (providerStatus !== "completed") {
             return noProposal(classification.statusReason, PROVIDER_STATUS_CODES[providerStatus] ?? "PROVIDER_FAILED", `Provider ${settings.providerId} ended with status ${providerStatus}: ${classification.statusReason}`, {
                 agentDurationMs,
@@ -245,6 +267,6 @@ export function createRealAgentProposalSource(args) {
         }
         realAgent.finalAnswerAvailable = true;
         realAgent.providerStatusReason = null;
-        return { kind: "proposal", text: result.finalAnswerText, realAgent, agentDurationMs, agentTokenUsage, errors: [], contextFile };
+        return { kind: "proposal", ...runtimeTelemetry, text: result.finalAnswerText, realAgent, agentDurationMs, agentTokenUsage, errors: [], contextFile };
     };
 }

package/dist/src/experiments/plugins/agentSuccessRate/timeoutPresets.js

@@ -0,0 +1,9 @@
+export const AGENT_TIMEOUT_PRESETS = Object.freeze({
+    codex: Object.freeze({ standard: 240000, extended: 900000, maximum: 1800000 }),
+    claude: Object.freeze({ standard: 240000, extended: 900000, maximum: 1800000 })
+});
+export function resolveAgentTimeoutPreset(provider, preset) {
+    if (!Object.hasOwn(AGENT_TIMEOUT_PRESETS, provider) || !Object.hasOwn(AGENT_TIMEOUT_PRESETS[provider], preset))
+        throw new Error("--timeout-preset must be standard, extended or maximum for codex or claude.");
+    return AGENT_TIMEOUT_PRESETS[provider][preset];
+}

package/dist/src/report/experiments/agentSuccessCampaignProgressReportModel.js

@@ -0,0 +1 @@
+export { CAMPAIGN_PROGRESS_SCHEMA, validateCampaignProgress } from "../../experiments/plugins/agentSuccessRate/campaignProgress.js";

package/dist/src/report/experiments/buildAgentSuccessCampaignProgressReport.js

@@ -0,0 +1,6 @@
+import { assertValidatedCampaignProgress } from "../../experiments/plugins/agentSuccessRate/campaignProgress.js";
+/** All representations copy the same validated revision; no renderer evaluates tasks. */
+export function buildAgentSuccessCampaignProgressReport(snapshot, boundary) {
+    assertValidatedCampaignProgress(snapshot, boundary);
+    return structuredClone(snapshot.model);
+}

package/dist/src/report/experiments/buildAgentSuccessRateReport.js

@@ -1,3 +1,4 @@
+import { validateCampaignProgress } from "../../experiments/plugins/agentSuccessRate/campaignProgress.js";
 import { attemptEvidenceOf } from "../../experiments/plugins/agentSuccessRate/attemptEvidence.js";
 import { AGENT_SUCCESS_RATE_PLUGIN_ID, AGENT_SUCCESS_RATE_REAL_AGENT_EXECUTION_MODE, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../../experiments/plugins/agentSuccessRate/metadata.js";
 import { AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION } from "./agentSuccessRateReportModel.js";
@@ -211,7 +212,17 @@ export function buildAgentSuccessRateReport(run) {
         })()
         : null;
     const overall = total === 0 || evaluable === 0 ? "unavailable" : unavailable === 0 ? "complete" : "partial";
+    const progress = run.agentSuccessCampaignProgress;
+    let campaignEvidence;
+    if (progress !== undefined) {
+        validateCampaignProgress(progress);
+        if (progress.state !== "complete" || !progress.finalScientificArtifactsAvailable)
+            throw new Error("Campaign final report requires complete validated evidence.");
+        const { schemaVersion, campaignIdentity, checkpointGeneration, checkpointRevision, state, selectedCaseCount, totalTreatmentUnits, unitCounts, caseCounts, comparisons, finalScientificArtifactsAvailable } = progress;
+        campaignEvidence = structuredClone({ schemaVersion, campaignIdentity, checkpointGeneration, checkpointRevision, state, selectedCaseCount, totalTreatmentUnits, unitCounts, caseCounts, comparisons, finalScientificArtifactsAvailable });
+    }
     return {
+        ...(campaignEvidence ? { campaignEvidence } : {}),
         schemaVersion: AGENT_SUCCESS_RATE_REPORT_SCHEMA_VERSION,
         identity: {
             runId: run.runId,

package/dist/src/report/experiments/buildPluginExperimentReport.js

@@ -17,7 +17,7 @@ const RETRIEVAL_QUERY_STRATEGY_COMPARISON_BULK_KEYS = ["caseExecutionEvidence",
 // Same for context-pack-generation: the typed section presents execution evidence and analysis.
 const CONTEXT_PACK_GENERATION_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
 // Same for agent-success-rate: the typed section presents execution evidence and analysis.
-const AGENT_SUCCESS_RATE_BULK_KEYS = ["caseExecutionEvidence", "analysis"];
+const AGENT_SUCCESS_RATE_BULK_KEYS = ["caseExecutionEvidence", "analysis", "agentSuccessCampaignProgress"];
 const V043_BULK_ARRAY_KEYS = [
     "v043StageContextExecutions",
     "v043StageContextEvaluations",

package/dist/src/report/experiments/renderAgentSuccessCampaignProgressHtml.js

@@ -0,0 +1,6 @@
+import { renderAgentSuccessCampaignProgressText } from "./renderAgentSuccessCampaignProgressText.js";
+/** A static accessible projection of the same text; every dynamic character is escaped. */
+export function renderAgentSuccessCampaignProgressHtml(model) {
+    const text = renderAgentSuccessCampaignProgressText(model).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&#39;");
+    return `<!doctype html><html lang="en"><head><meta charset="utf-8"><title>Campaign progress</title></head><body><main><h1>Campaign progress and evidence</h1><pre style="white-space:pre-wrap">${text}</pre></main></body></html>\n`;
+}

package/dist/src/report/experiments/renderAgentSuccessCampaignProgressText.js

@@ -0,0 +1,25 @@
+export function renderAgentSuccessCampaignProgressText(m) {
+    return [
+        "Agent Success Rate Campaign Progress", `Schema: ${m.schemaVersion}`, `Campaign: ${m.campaignIdentity}`, `Execution mode: ${m.executionMode}`,
+        `Checkpoint revision: ${m.checkpointGeneration} (${m.checkpointRevision})`, `State: ${m.state}`,
+        `Completed evaluations: ${m.unitCounts.completed}/${m.totalTreatmentUnits} treatment units`,
+        `Completed cases: ${m.caseCounts.completed}/${m.selectedCaseCount}`,
+        ...Object.entries(m.unitCounts).map(([s, n]) => `Treatment state ${s}: ${n}`),
+        ...Object.entries(m.caseCounts).map(([s, n]) => `Case state ${s}: ${n}`),
+        `Provider limit: ${m.knownProviderLimit}; uncertainty: ${m.knownUncertainty}`,
+        `Diagnostics: ${m.diagnostics.join(", ") || "none"}`,
+        `Scientific results: ${m.scientificCompleteness}; validated final artifacts: ${m.finalScientificArtifactsAvailable}`,
+        `Completed cases: ${m.completedCaseIds.join(", ") || "none"}`, `Unfinished cases: ${m.unfinishedCaseIds.join(", ") || "none"}`,
+        ...m.cases.flatMap(c => c.treatments.flatMap(t => [
+            `${c.caseId} / ${t.treatmentId}: ${t.state}; evaluated task success=${t.evaluatedTaskSuccess ?? "unavailable"}; epochs=${t.epochCount}; historical limits=${t.historicalProviderLimitCount}; diagnostics=${t.diagnostics.join(", ") || "none"}`,
+            ...t.attempts.map(a => `  Attempt ${a.attemptNumber}: provider invoked=${a.providerInvoked}; Lab duration=${a.duration.durationMs ?? "unavailable"} ms (${a.duration.source}); tokens=${a.tokens.totalTokens ?? "unavailable"}; input=${a.tokens.inputTokens ?? "unavailable"}; output=${a.tokens.outputTokens ?? "unavailable"}; source=${a.tokens.source}; reliability=${a.tokens.reliability}; category=${a.tokens.evidenceLevel}; numeric availability=${a.tokens.numericAvailability}; interpretation=${a.tokenInterpretation}`)
+        ])),
+        "Comparisons (context-pack minus raw-full-file; descriptive matched subset means):",
+        ...m.comparisons.flatMap(q => [
+            `${q.quantity}: ${q.quality}; basis=${q.basis}; measurement=${q.measurementBasis}; matched=${q.matchedCaseIds.length}/${q.selectedCaseCount}; excluded=${q.excludedCaseIds.length}`,
+            `  Matched: ${q.matchedCaseIds.join(", ") || "none"}; excluded: ${q.excludedCaseIds.join(", ") || "none"}`,
+            `  Raw mean=${q.rawFullFileMean ?? "unavailable"}; context mean=${q.contextPackMean ?? "unavailable"}; delta=${q.meanDifference ?? "unavailable"}; unit=${q.unit}`,
+            ...q.reasons.map((r, i) => `  ${r}: ${q.reasonExplanations[i]}`)
+        ]), "Scientific limitations:", ...m.limitations.map(l => `- ${l}`), ""
+    ].join("\n");
+}

package/dist/src/report/experiments/renderAgentSuccessRateHtml.js

@@ -1,4 +1,4 @@
-import { AGENT_SUCCESS_BASIS_LABEL, formatAgentSuccessMetric, formatAgentSuccessPairs, formatAgentSuccessTotal } from "./renderAgentSuccessRateText.js";
+import { AGENT_SUCCESS_BASIS_LABEL, formatAgentSuccessMetric, formatAgentSuccessPairs, formatAgentSuccessTotal, renderCampaignEvidenceSummary } from "./renderAgentSuccessRateText.js";
 function escapeHtml(value) {
     return String(value ?? "")
         .replaceAll("&", "&")
@@ -22,6 +22,8 @@ export function renderAgentSuccessRateHtml(report) {
     const realMode = report.identity.executionMode === "real-agent";
     const m = formatAgentSuccessMetric;
     const parts = [];
+    if (report.campaignEvidence)
+        parts.push(`<section><h2>Campaign evidence</h2><pre>${escapeHtml(renderCampaignEvidenceSummary(report))}</pre></section>`);
     parts.push(`<section id="agent-success-rate">
     <h2>Agent Success Rate</h2>
     <h3>Experiment identity</h3>

package/dist/src/report/experiments/renderAgentSuccessRateText.js

@@ -1,3 +1,8 @@
+export function renderCampaignEvidenceSummary(report) {
+    const m = report.campaignEvidence;
+    return m ? [`Campaign evidence: ${m.state}; checkpoint revision ${m.checkpointGeneration} (${m.checkpointRevision}); completed evaluations ${m.unitCounts.completed}/${m.totalTreatmentUnits}; completed cases ${m.caseCounts.completed}/${m.selectedCaseCount}`,
+        ...m.comparisons.map(q => `${q.quantity}: ${q.quality}; ${q.basis}; ${q.measurementBasis}; matched ${q.matchedCaseIds.length}/${q.selectedCaseCount}; excluded ${q.excludedCaseIds.length}; raw=${q.rawFullFileMean ?? "unavailable"}; context=${q.contextPackMean ?? "unavailable"}; delta=${q.meanDifference ?? "unavailable"}; unit=${q.unit}; reasons=${q.reasons.join(", ") || "none"}; ${q.reasonExplanations.join(" ")}`)].join("\n") : "";
+}
 /** One display form for every metric: an absent value is labelled, never rendered as zero. */
 export function formatAgentSuccessMetric(metric) {
     if (!metric)
@@ -135,5 +140,7 @@ export function renderAgentSuccessRateTextLines(report) {
         lines.push(`error ${error.caseId} / ${error.treatmentId} attempt ${error.attemptNumber} ${sanitize(error.code)}: ${sanitize(error.message)}`);
     for (const artifact of report.artifacts)
         lines.push(`artifact ${sanitize(artifact.id)}: ${artifact.path ?? "[reference withheld]"}`);
+    if (report.campaignEvidence)
+        lines.push(renderCampaignEvidenceSummary(report));
     return lines;
 }

package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js

@@ -0,0 +1,135 @@
+import { lstat, mkdir, open, readdir, realpath } from "node:fs/promises";
+import path from "node:path";
+import { campaignAssert, campaignHash, canonicalCampaignJson } from "../../experiments/plugins/agentSuccessRate/campaignIdentity.js";
+import { validateCampaignOutputPath } from "../../experiments/plugins/agentSuccessRate/campaignCheckpointStore.js";
+import { readCampaignProgressScientificFamily } from "../../experiments/plugins/agentSuccessRate/campaignProgress.js";
+import { agentSuccessRatePlugin, mapAgentSuccessRateToRun } from "../../experiments/plugins/agentSuccessRate/plugin.js";
+import { writePluginExperimentReports } from "./writePluginExperimentReports.js";
+import { buildPluginExperimentReport } from "./buildPluginExperimentReport.js";
+import { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
+import { renderPluginExperimentReportHtml } from "./renderPluginExperimentReportHtml.js";
+import { buildAgentSuccessCampaignProgressReport } from "./buildAgentSuccessCampaignProgressReport.js";
+import { renderAgentSuccessCampaignProgressText } from "./renderAgentSuccessCampaignProgressText.js";
+import { renderAgentSuccessCampaignProgressHtml } from "./renderAgentSuccessCampaignProgressHtml.js";
+async function plain(p, kind) {
+    const s = await lstat(p);
+    campaignAssert(!s.isSymbolicLink() && (kind === "file" ? s.isFile() : s.isDirectory()) && path.resolve(await realpath(p)).toLowerCase() === path.resolve(p).toLowerCase(), "PROGRESS_UNSAFE_PATH");
+}
+async function writeExclusive(p, content) {
+    const h = await open(p, "wx");
+    try {
+        await h.writeFile(content, "utf8");
+        await h.sync();
+    }
+    finally {
+        await h.close();
+    }
+}
+async function readExpected(p, expected, code) {
+    await plain(p, "file");
+    const length = Buffer.byteLength(expected);
+    campaignAssert(length <= 64 * 1024 * 1024, "PROGRESS_DATA_BOUND");
+    const h = await open(p, "r");
+    try {
+        const before = await h.stat();
+        campaignAssert(before.isFile() && before.size === length, code);
+        const buffer = Buffer.alloc(length + 1);
+        let size = 0;
+        for (;;) {
+            const n = await h.read(buffer, size, buffer.length - size, size);
+            size += n.bytesRead;
+            if (!n.bytesRead || size === buffer.length)
+                break;
+        }
+        campaignAssert(size === length && (await h.stat()).size === length && buffer.subarray(0, size).toString("utf8") === expected, code);
+    }
+    finally {
+        await h.close();
+    }
+}
+async function owned(parent, name, owner) {
+    const p = path.join(parent, name);
+    let created = false;
+    try {
+        await mkdir(p);
+        created = true;
+    }
+    catch (e) {
+        if (e.code !== "EEXIST")
+            throw e;
+    }
+    await plain(p, "directory");
+    if (created)
+        await writeExclusive(path.join(p, "owner.json"), owner);
+    await readExpected(path.join(p, "owner.json"), owner, "PROGRESS_OWNER_MISMATCH");
+    return p;
+}
+/** Revision-specific files are exclusive. The manifest is published last and verified before returning success. */
+export async function writeAgentSuccessCampaignProgressReport(args) {
+    const model = buildAgentSuccessCampaignProgressReport(args.snapshot, args.boundary);
+    await validateCampaignOutputPath(args.boundary);
+    const root = path.resolve(args.boundary.outputRoot);
+    await plain(root, "directory");
+    const family = await owned(root, "campaign-progress-v1", "agent-success-rate-campaign-progress-v1");
+    const identity = await owned(family, model.campaignIdentity, `agent-success-rate-campaign-progress-v1:${model.campaignIdentity}`);
+    // Ownership blockers can change without a checkpoint transition. Keep both projections immutable at that revision.
+    const modelDigest = campaignHash(canonicalCampaignJson(model));
+    const revision = `revision-${String(model.checkpointGeneration).padStart(10, "0")}-${model.checkpointRevision}-${modelDigest.slice(0, 16)}`;
+    const outDir = path.join(identity, revision);
+    const scientific = model.state === "complete" ? await readCampaignProgressScientificFamily(args.snapshot, args.boundary) : null;
+    const run = scientific ? mapAgentSuccessRateToRun({ runId: scientific.execution.runId, startedAt: scientific.execution.startedAt, completedAt: scientific.execution.completedAt,
+        target: { kind: "self", targetRoot: "[redacted]", toolRoot: "[redacted]", packageName: "@dailephd/my-dev-kit-lab", packageVersion: "0.9.0", hasPackageJson: true, hasLockfile: true, branch: null, commit: null, hasGit: false, isSelf: true },
+        caseEvidence: scientific.execution.cases.map(c => ({ ...c, caseName: c.caseId })), analysis: scientific.analysis.analysis, patchArtifactPaths: [],
+        providerId: scientific.execution.realAgent?.providerId, ...(scientific.execution.realAgent ? { realSettings: { timeoutMs: scientific.execution.realAgent.timeoutMs, repairAttempts: scientific.execution.realAgent.repairAttempts ?? 0 } } : {}) }) : null;
+    if (run)
+        Object.assign(run, { agentSuccessCampaignProgress: model });
+    const files = [
+        { name: "agent-success-rate-campaign-progress.json", content: `${JSON.stringify(model, null, 2)}\n`, phase: "json-written" },
+        { name: "agent-success-rate-campaign-progress.txt", content: renderAgentSuccessCampaignProgressText(model), phase: "text-written" },
+        { name: "agent-success-rate-campaign-progress.html", content: renderAgentSuccessCampaignProgressHtml(model), phase: "html-written" }
+    ];
+    if (run) {
+        const built = buildPluginExperimentReport({ run, plugin: agentSuccessRatePlugin.metadata, outputRoot: outDir, generatedAt: run.completedAt });
+        const report = { ...built, metadata: { ...built.metadata, outputRoot: "[redacted]" } };
+        files.push({ name: "report.json", content: `${JSON.stringify({ report, outputPaths: { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" } }, null, 2)}\n`, phase: "json-written" }, { name: "report.txt", content: renderPluginExperimentReportText(report), phase: "text-written" }, { name: "report.html", content: renderPluginExperimentReportHtml(report), phase: "html-written" });
+    }
+    const marker = canonicalCampaignJson({ schemaVersion: "agent-success-rate-campaign-progress-publication-v1", campaignIdentity: model.campaignIdentity, checkpointGeneration: model.checkpointGeneration, checkpointRevision: model.checkpointRevision, modelDigest, files: files.map(f => ({ name: f.name, sha256: campaignHash(f.content), bytes: Buffer.byteLength(f.content) })) });
+    let created = false;
+    try {
+        await mkdir(outDir);
+        created = true;
+    }
+    catch (e) {
+        if (e.code !== "EEXIST")
+            throw e;
+    }
+    await plain(outDir, "directory");
+    if (created) {
+        for (const file of files.slice(0, 3)) {
+            await writeExclusive(path.join(outDir, file.name), file.content);
+            await args.failureHook?.(file.phase);
+        }
+        // Exclusive directory reservation owns these ordinary report names; the existing writer remains their owner.
+        if (run)
+            await writePluginExperimentReports({ run, plugin: agentSuccessRatePlugin.metadata, outputRoot: outDir, generatedAt: run.completedAt, redactOutputRoot: true, exclusive: true });
+        for (const f of files)
+            await readExpected(path.join(outDir, f.name), f.content, "PROGRESS_SNAPSHOT_INTEGRITY");
+        // The legacy writer owns report rendering. Flush its campaign-only output before committing the group marker.
+        for (const f of files.slice(3)) {
+            const h = await open(path.join(outDir, f.name), "r+");
+            try {
+                await h.sync();
+            }
+            finally {
+                await h.close();
+            }
+        }
+        await args.failureHook?.("before-manifest");
+        await writeExclusive(path.join(outDir, "manifest.json"), marker);
+    }
+    campaignAssert((await readdir(outDir)).sort().join(",") === [...files.map(f => f.name), "manifest.json"].sort().join(","), "PROGRESS_INCOMPLETE_OR_UNKNOWN_SNAPSHOT");
+    await readExpected(path.join(outDir, "manifest.json"), marker, "PROGRESS_SNAPSHOT_COLLISION");
+    for (const f of files)
+        await readExpected(path.join(outDir, f.name), f.content, "PROGRESS_SNAPSHOT_INTEGRITY");
+    return { model, outputPaths: { outDir, jsonPath: path.join(outDir, files[0].name), textPath: path.join(outDir, files[1].name), htmlPath: path.join(outDir, files[2].name), manifestPath: path.join(outDir, "manifest.json") } };
+}

package/dist/src/report/experiments/writePluginExperimentReports.js

@@ -31,9 +31,10 @@ export async function writePluginExperimentReports(args) {
     const serializedPaths = args.redactOutputRoot
         ? { outDir: "[redacted]", jsonPath: "report.json", htmlPath: "report.html", textPath: "report.txt" }
         : outputPaths;
-    await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, "utf8");
-    await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), "utf8");
-    await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), "utf8");
+    const writeOptions = args.exclusive ? { encoding: "utf8", flag: "wx" } : "utf8";
+    await writeFile(outputPaths.jsonPath, `${JSON.stringify({ report, outputPaths: serializedPaths }, null, 2)}\n`, writeOptions);
+    await writeFile(outputPaths.htmlPath, renderPluginExperimentReportHtml(report), writeOptions);
+    await writeFile(outputPaths.textPath, renderPluginExperimentReportText(report), writeOptions);
     return { report, outputPaths };
 }
 function readString(value) {

package/docs/METRICS.md

@@ -938,3 +938,12 @@ Cumulative versus final: durations and tokens accumulate over all provider attem
 - Paired differences report the mean of the `raw-full-file` values, the mean of the `context-pack` values, and the mean of `context-pack` minus `raw-full-file` over matched cases.
 - These are **descriptive**. No weighted composite score, ranking, automatic winner, causal claim, or statistical-significance test is produced (`statisticalEffect` is `not-assessed`). The corpus is finite and controlled, so results do not generalize beyond it, and a single run carries no variance estimate.
 - Provider telemetry can be unavailable, and in deterministic-fixture mode both treatments receive the same reference patch, so those results measure the pipeline only.
+
+
+## Campaign progress evidence (v0.9.1 implemented; unreleased)
+
+Token evidence levels are `provider-reported`, `cli-reported`, `parsed-from-output`, `unavailable`, and `estimated`. The original telemetry source is retained. Numeric availability is recorded per token field, so partial measurements remain partial and normalization never fills missing fields or invents totals; unattributed OpenTelemetry evidence is unavailable. Duration provenance is `lab-wall-clock` for valid observed elapsed time and `unavailable` for missing or invalid values; campaign comparison time is Lab-measured wall-clock duration, not provider-native time.
+
+The plugin-specific progress projection is `my-dev-kit-lab-agent-success-rate-campaign-progress-v1`; the writer publishes immutable revision-specific JSON, HTML, and text snapshots. It accounts for selected cases and treatment units, while execution epochs remain recovery history and are excluded from selected-unit counts. Paused provider-limit and uncertain states remain distinct. Partial progress is not a complete scientific analysis artifact.
+
+Evidence quality is metric-specific: task outcomes, token measurements, and duration can have different strong, partial, or unavailable classifications. Strong describes the completeness and provenance of the available evidence; it is not statistical significance. Partial evidence does not mean task failure. Missing token evidence is unavailable, never zero; estimated tokens are not provider usage. Duration is Lab-measured wall-clock time, not provider-native time. Comparisons use matched completed cases and state their denominator and exclusions; the existing paired direction is preserved. These descriptive comparisons establish no causal effect, winner, p-value, or general provider performance.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {