@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/README.md +19 -3
  3. package/benchmarks/contracts/agent-success-rate-project-profiles.json +33 -0
  4. package/benchmarks/contracts/agent-success-rate-tasks.json +671 -0
  5. package/benchmarks/projects/agent-success-inventory-node/.gitattributes +2 -0
  6. package/benchmarks/projects/agent-success-inventory-node/README.md +74 -0
  7. package/benchmarks/projects/agent-success-inventory-node/package.json +14 -0
  8. package/benchmarks/projects/agent-success-inventory-node/src/fulfillmentReport.js +38 -0
  9. package/benchmarks/projects/agent-success-inventory-node/src/index.js +26 -0
  10. package/benchmarks/projects/agent-success-inventory-node/src/inventoryStore.js +114 -0
  11. package/benchmarks/projects/agent-success-inventory-node/src/quantity.js +19 -0
  12. package/benchmarks/projects/agent-success-inventory-node/src/reservationService.js +45 -0
  13. package/benchmarks/projects/agent-success-inventory-node/tests/quantity-edge.check.mjs +51 -0
  14. package/benchmarks/projects/agent-success-inventory-node/tests/quantity-primary.check.mjs +46 -0
  15. package/benchmarks/projects/agent-success-inventory-node/tests/regression.check.mjs +115 -0
  16. package/benchmarks/projects/agent-success-inventory-node/tests/report-edge.check.mjs +109 -0
  17. package/benchmarks/projects/agent-success-inventory-node/tests/report-primary.check.mjs +47 -0
  18. package/benchmarks/projects/agent-success-inventory-node/tests/reservation-edge.check.mjs +63 -0
  19. package/benchmarks/projects/agent-success-inventory-node/tests/reservation-primary.check.mjs +44 -0
  20. package/benchmarks/projects/agent-success-task-board-node/.gitattributes +2 -0
  21. package/benchmarks/projects/agent-success-task-board-node/README.md +77 -0
  22. package/benchmarks/projects/agent-success-task-board-node/package.json +14 -0
  23. package/benchmarks/projects/agent-success-task-board-node/src/index.js +24 -0
  24. package/benchmarks/projects/agent-success-task-board-node/src/projectSummary.js +15 -0
  25. package/benchmarks/projects/agent-success-task-board-node/src/taskService.js +58 -0
  26. package/benchmarks/projects/agent-success-task-board-node/src/taskStore.js +58 -0
  27. package/benchmarks/projects/agent-success-task-board-node/src/validation.js +31 -0
  28. package/benchmarks/projects/agent-success-task-board-node/tests/import-edge.check.mjs +65 -0
  29. package/benchmarks/projects/agent-success-task-board-node/tests/import-primary.check.mjs +31 -0
  30. package/benchmarks/projects/agent-success-task-board-node/tests/regression.check.mjs +107 -0
  31. package/benchmarks/projects/agent-success-task-board-node/tests/summary-edge.check.mjs +67 -0
  32. package/benchmarks/projects/agent-success-task-board-node/tests/summary-primary.check.mjs +54 -0
  33. package/benchmarks/projects/agent-success-task-board-node/tests/title-edge.check.mjs +39 -0
  34. package/benchmarks/projects/agent-success-task-board-node/tests/title-primary.check.mjs +29 -0
  35. package/dist/src/agents/adapters/claudeAdapter.js +6 -3
  36. package/dist/src/agents/adapters/codexAdapter.js +51 -10
  37. package/dist/src/agents/index.js +1 -0
  38. package/dist/src/agents/normalizeAgentTelemetry.js +50 -0
  39. package/dist/src/agents/parseAgentTokenUsage.js +71 -49
  40. package/dist/src/agents/providerBuildObservation.js +97 -0
  41. package/dist/src/audits/core/projectInventory.js +1 -0
  42. package/dist/src/cli/help.js +44 -1
  43. package/dist/src/commands/runExperimentDescribeCommand.js +83 -1
  44. package/dist/src/commands/runExperimentRunCommand.js +309 -6
  45. package/dist/src/core/processTree.js +55 -0
  46. package/dist/src/core/runMeasuredCommand.js +108 -22
  47. package/dist/src/evaluation/agentSuccess/applyPatchToSandbox.js +37 -0
  48. package/dist/src/evaluation/agentSuccess/assessBaseline.js +23 -0
  49. package/dist/src/evaluation/agentSuccess/corpusTypes.js +74 -0
  50. package/dist/src/evaluation/agentSuccess/extractPatchCandidate.js +46 -0
  51. package/dist/src/evaluation/agentSuccess/index.js +14 -0
  52. package/dist/src/evaluation/agentSuccess/parseUnifiedDiff.js +210 -0
  53. package/dist/src/evaluation/agentSuccess/patchTypes.js +5 -0
  54. package/dist/src/evaluation/agentSuccess/readAgentSuccessCorpus.js +183 -0
  55. package/dist/src/evaluation/agentSuccess/runVerificationCheck.js +100 -0
  56. package/dist/src/evaluation/agentSuccess/taskPaths.js +34 -0
  57. package/dist/src/evaluation/agentSuccess/taskTypes.js +14 -0
  58. package/dist/src/evaluation/agentSuccess/validateAgentSuccessCorpus.js +307 -0
  59. package/dist/src/evaluation/agentSuccess/validateAgentSuccessTask.js +297 -0
  60. package/dist/src/evaluation/agentSuccess/validatePatchPolicy.js +66 -0
  61. package/dist/src/evaluation/agentSuccess/verificationTypes.js +1 -0
  62. package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +127 -0
  63. package/dist/src/evaluation/benchmarkSandbox/errors.js +11 -0
  64. package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +62 -0
  65. package/dist/src/evaluation/benchmarkSandbox/index.js +8 -0
  66. package/dist/src/evaluation/benchmarkSandbox/minimalHostEnv.js +37 -0
  67. package/dist/src/evaluation/benchmarkSandbox/pathPolicy.js +49 -0
  68. package/dist/src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.js +74 -0
  69. package/dist/src/evaluation/benchmarkSandbox/treeSnapshot.js +57 -0
  70. package/dist/src/evaluation/benchmarkSandbox/types.js +3 -0
  71. package/dist/src/evaluation/changeSet/captureChangeSet.js +98 -0
  72. package/dist/src/evaluation/changeSet/index.js +2 -0
  73. package/dist/src/evaluation/changeSet/types.js +9 -0
  74. package/dist/src/experiments/defaultRegistry.js +2 -0
  75. package/dist/src/experiments/plugins/agentSuccessRate/agentTaskProjection.js +12 -0
  76. package/dist/src/experiments/plugins/agentSuccessRate/analysis.js +425 -0
  77. package/dist/src/experiments/plugins/agentSuccessRate/analysisArtifact.js +155 -0
  78. package/dist/src/experiments/plugins/agentSuccessRate/analysisTypes.js +65 -0
  79. package/dist/src/experiments/plugins/agentSuccessRate/attemptEvidence.js +7 -0
  80. package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpoint.js +216 -0
  81. package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +588 -0
  82. package/dist/src/experiments/plugins/agentSuccessRate/campaignEvidenceQuality.js +77 -0
  83. package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +314 -0
  84. package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +111 -0
  85. package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +173 -0
  86. package/dist/src/experiments/plugins/agentSuccessRate/campaignQueue.js +11 -0
  87. package/dist/src/experiments/plugins/agentSuccessRate/campaignRecovery.js +256 -0
  88. package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +207 -0
  89. package/dist/src/experiments/plugins/agentSuccessRate/config.js +123 -0
  90. package/dist/src/experiments/plugins/agentSuccessRate/contextGeneration.js +497 -0
  91. package/dist/src/experiments/plugins/agentSuccessRate/execution.js +441 -0
  92. package/dist/src/experiments/plugins/agentSuccessRate/executionArtifact.js +126 -0
  93. package/dist/src/experiments/plugins/agentSuccessRate/executionTypes.js +2 -0
  94. package/dist/src/experiments/plugins/agentSuccessRate/index.js +26 -0
  95. package/dist/src/experiments/plugins/agentSuccessRate/metadata.js +28 -0
  96. package/dist/src/experiments/plugins/agentSuccessRate/metrics.js +76 -0
  97. package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +517 -0
  98. package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +272 -0
  99. package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +43 -0
  100. package/dist/src/experiments/plugins/agentSuccessRate/repairAnalysis.js +148 -0
  101. package/dist/src/experiments/plugins/agentSuccessRate/repairExecution.js +103 -0
  102. package/dist/src/experiments/plugins/agentSuccessRate/repairFeedback.js +91 -0
  103. package/dist/src/experiments/plugins/agentSuccessRate/repairPolicy.js +88 -0
  104. package/dist/src/experiments/plugins/agentSuccessRate/timeoutPresets.js +9 -0
  105. package/dist/src/experiments/plugins/agentSuccessRate/types.js +1 -0
  106. package/dist/src/experiments/plugins/warmIndexReuse/plugin.js +2 -3
  107. package/dist/src/report/experiments/agentSuccessCampaignProgressReportModel.js +1 -0
  108. package/dist/src/report/experiments/agentSuccessRateReportModel.js +6 -0
  109. package/dist/src/report/experiments/buildAgentSuccessCampaignProgressReport.js +6 -0
  110. package/dist/src/report/experiments/buildAgentSuccessRateReport.js +266 -0
  111. package/dist/src/report/experiments/buildPluginExperimentReport.js +27 -2
  112. package/dist/src/report/experiments/index.js +4 -0
  113. package/dist/src/report/experiments/renderAgentSuccessCampaignProgressHtml.js +6 -0
  114. package/dist/src/report/experiments/renderAgentSuccessCampaignProgressText.js +25 -0
  115. package/dist/src/report/experiments/renderAgentSuccessRateHtml.js +171 -0
  116. package/dist/src/report/experiments/renderAgentSuccessRateText.js +146 -0
  117. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
  118. package/dist/src/report/experiments/renderPluginExperimentReportText.js +5 -0
  119. package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +135 -0
  120. package/dist/src/report/experiments/writePluginExperimentReports.js +4 -3
  121. package/docs/METRICS.md +79 -0
  122. package/package.json +1 -1
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Every complete attempt evidence record of one treatment, in execution order. Real-agent treatments list their attempts;
3
+ * a deterministic-fixture treatment (or a Batch 4 style record) is its own single attempt.
4
+ */
5
+ export function attemptEvidenceOf(treatment) {
6
+ return treatment.attempts && treatment.attempts.length > 0 ? treatment.attempts.map((attempt) => attempt.evidence) : [treatment];
7
+ }
@@ -0,0 +1,216 @@
1
+ import { agentSuccessPatchArtifactPath } from "./execution.js";
2
+ import { agentSuccessContextArtifactPath, agentSuccessAgentArtifactDirectory } from "./realAgentExecution.js";
3
+ import { classifyAgentSuccessAttempt } from "./repairPolicy.js";
4
+ import { analyzeAgentSuccessTreatment } from "./analysis.js";
5
+ import { campaignAssert, campaignExact, campaignHash, campaignIdentifier, canonicalCampaignJson, isCampaignDigest, validateCampaignIdentity } from "./campaignIdentity.js";
6
+ export const CAMPAIGN_CHECKPOINT_SCHEMA = "my-dev-kit-lab-agent-success-rate-campaign-checkpoint-v1";
7
+ export const CAMPAIGN_CHECKPOINT_MAX_BYTES = 16 * 1024 * 1024;
8
+ const nullable = (rule) => (v) => { if (v !== null)
9
+ rule(v); };
10
+ const nil = (v) => campaignAssert(v === null);
11
+ const bool = (v) => campaignAssert(typeof v === "boolean");
12
+ const number = (v) => campaignAssert(typeof v === "number" && Number.isFinite(v) && v >= 0);
13
+ const integer = (v) => { number(v); campaignAssert(Number.isSafeInteger(v)); };
14
+ const identifier = (v) => campaignAssert(campaignIdentifier(v));
15
+ const digest = (v) => campaignAssert(isCampaignDigest(v));
16
+ const one = (...values) => (v) => campaignAssert(values.includes(v));
17
+ const list = (rule, max = 2000) => (v) => { campaignAssert(Array.isArray(v) && v.length <= max, "DATA_BOUND"); v.forEach(rule); };
18
+ const shape = (rules, optional = {}) => (v) => {
19
+ campaignAssert(typeof v === "object" && v !== null && !Array.isArray(v));
20
+ const row = v;
21
+ campaignAssert(Object.keys(row).every((k) => Object.hasOwn(rules, k) || Object.hasOwn(optional, k)));
22
+ for (const [k, r] of Object.entries(rules))
23
+ r(row[k]);
24
+ for (const [k, r] of Object.entries(optional))
25
+ if (Object.hasOwn(row, k))
26
+ r(row[k]);
27
+ };
28
+ const timestamp = (v) => campaignAssert(typeof v === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(v) && Number.isFinite(Date.parse(v)) && new Date(v).toISOString() === v);
29
+ export function validateCampaignRelativePath(v) {
30
+ campaignAssert(typeof v === "string" && v.length > 0 && v.length <= 512 && /^[A-Za-z0-9._/-]+$/.test(v) && !v.startsWith("/") && v.split("/").every((p) => p !== "" && p !== "." && p !== ".." && !p.endsWith(".") && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p)), "UNSAFE_PATH");
31
+ }
32
+ const rel = validateCampaignRelativePath;
33
+ const cleanup = shape({ attempted: bool, removed: bool, reason: nil });
34
+ const check = shape({ checkId: identifier, class: one("task", "regression"), status: one("passed", "failed", "timeout", "error"), exitCode: nullable((v) => campaignAssert(typeof v === "number" && Number.isSafeInteger(v))), durationMs: number, failureReason: nil });
35
+ const verification = shape({ phase: one("baseline", "post-edit"), taskResults: list(check), regressionResults: list(check) });
36
+ const assessment = shape({ evaluable: bool, reasons: list(one("NO_CHECKS", "ALL_TASK_CHECKS_PASSED", "REGRESSION_CHECK_FAILED", "CHECK_NOT_EVALUABLE"), 4) });
37
+ const token = shape({ totalTokens: nullable(integer), inputTokens: nullable(integer), outputTokens: nullable(integer), source: one("provider-reported", "agent-reported", "cli-json", "otel", "estimated-from-text", "unavailable"), reliability: one("high", "medium", "low", "unavailable") });
38
+ const providerStatus = one("not-invoked", "completed", "failed", "skipped", "timeout", "agent-unavailable", "agent-limit-reached", "invalid-output");
39
+ const failure = one("none", "patch-malformed", "patch-policy-rejected", "patch-check-failed", "patch-apply-failed", "task-check-failed", "regression-failed", "required-behavior-unsatisfied", "baseline-invalid", "context-unavailable", "provider-unavailable", "provider-limit-reached", "provider-timeout", "provider-failed", "provider-empty-answer", "provider-not-invoked", "infrastructure-failure", "cleanup-failed", "integrity-unproven", "integrity-mutated", "evidence-incomplete");
40
+ const feedback = shape({ basedOnAttempt: one(1, 2, 3), category: failure, patchProduced: bool, patchApplied: bool, taskChecks: one("all-passed", "some-failed", "not-evaluated"), regressionChecks: one("all-passed", "some-failed", "not-evaluated"), requiredBehavior: one("verified", "not-verified", "not-evaluated"), failedTaskCheckCount: nullable(integer), failedRegressionCheckCount: nullable(integer), previousPatchIncluded: bool, previousPatchTruncated: bool });
41
+ const realAgent = shape({ providerId: one("codex", "claude"), attempt: one(1, 2, 3), promptTransport: one("stdin"), providerInvoked: bool, providerStatus, providerStatusReason: nil, finalAnswerAvailable: bool, promptChars: nullable(integer), context: shape({ contextMode: one("raw-full-file", "context-pack"), availability: one("available", "partial", "unavailable"), reason: nil, selectionPolicyId: identifier, myDevKitVersion: nullable(identifier), includedSourceFiles: list(rel), contextChars: integer, estimatedContextTokens: integer, contextArtifactPath: nullable(rel) }), agentArtifactDirectory: nullable(rel), agentArtifacts: shape({ prompt: nullable(rel), result: nullable(rel), stdout: nullable(rel), stderr: nullable(rel), telemetry: nullable(rel) }), cwdCleanup: cleanup }, { repairFeedback: nullable(feedback) });
42
+ const fileStatus = one("added", "modified", "deleted");
43
+ const patch = shape({ attempted: bool, outcome: nullable(one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success")), code: nullable(identifier), message: nil, appliedFiles: list(shape({ path: rel, status: fileStatus })), rejections: list(shape({ code: identifier, path: nullable(rel) }), 20), attemptedProtectedPaths: list(rel), proposedPatchBytes: integer });
44
+ const change = shape({ baselineCommit: identifier, changedFiles: list(shape({ relativePath: rel, status: fileStatus, beforeSha256: nullable(digest), afterSha256: nullable(digest), additions: nullable(integer), deletions: nullable(integer) })), addedCount: integer, modifiedCount: integer, deletedCount: integer, changedCount: integer, totalAdditions: integer, totalDeletions: integer });
45
+ const baseRules = { treatmentId: one("raw-full-file", "context-pack"), status: one("completed", "partial", "failed", "skipped"), availability: one("complete", "baseline-invalid", "incomplete", "infrastructure-failure"), sandboxId: identifier, sandboxBaseline: nullable(shape({ commit: identifier, fileCount: integer, digest })), baselineTextLineCount: nullable(integer), baselineVerification: nullable(verification), baselineAssessment: nullable(assessment), patch, change: nullable(change), postEditVerification: nullable(verification), protectedIntegrity: shape({ status: one("intact", "mutated", "unproven"), mutatedPaths: list(rel) }), timing: shape({ agentDurationMs: nullable(number), baselineVerificationDurationMs: nullable(number), patchPipelineDurationMs: nullable(number), postEditVerificationDurationMs: nullable(number), evaluationDurationMs: nullable(number) }), agentTokenUsage: nullable(token), cleanup, proposedPatchPath: nullable(rel), appliedPatchPath: nullable(rel), errors: list(shape({ code: identifier, message: one("details-omitted") }), 100) };
46
+ const singleEvidence = shape(baseRules, { realAgent });
47
+ const attempt = shape({ attemptNumber: one(1, 2, 3), providerId: one("codex", "claude"), providerStatus, startedAt: timestamp, completedAt: timestamp, durationMs: nullable(number), tokenUsage: nullable(token), proposedPatchAvailability: one("available", "unavailable"), patchApplicationOutcome: one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success", "not-attempted"), baselineAssessment: nullable(assessment), postEditVerification: nullable(verification), changeEvidenceAvailability: one("available", "unavailable"), taskSuccess: nullable(bool), failureCategory: failure, repairEligible: bool, cleanupResult: cleanup, evidence: singleEvidence });
48
+ const treatment = shape(baseRules, { realAgent, attempts: list(attempt, 3) });
49
+ /** Discard free-form explanations only; unknown fields still fail the closed decoder. Original evidence is never mutated. */
50
+ export function projectCampaignTreatmentEvidence(source) {
51
+ canonicalCampaignJson(source);
52
+ const walk = (v) => Array.isArray(v) ? v.map(walk) : v !== null && typeof v === "object" ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, k === "message" && Object.keys(v).length === 2 && Object.hasOwn(v, "code") ? "details-omitted" : ["message", "reason", "failureReason", "providerStatusReason"].includes(k) ? null : walk(x)])) : v;
53
+ const result = walk(source);
54
+ treatment(result);
55
+ return result;
56
+ }
57
+ export function requiredCampaignArtifacts(e, task, allowUnfinished = false) {
58
+ const refs = new Map();
59
+ const attempts = e.attempts?.map((a) => a.evidence) ?? [e];
60
+ for (let i = 0; i < attempts.length; i++) {
61
+ const a = attempts[i];
62
+ for (const kind of ["proposed", "applied"]) {
63
+ const p = kind === "proposed" ? a.proposedPatchPath : a.appliedPatchPath;
64
+ if (p !== null) {
65
+ campaignAssert(p === agentSuccessPatchArtifactPath(task.benchmarkProject, task.id, a.treatmentId, kind, (i + 1)));
66
+ campaignAssert(!refs.has(p));
67
+ refs.set(p, "patch");
68
+ }
69
+ }
70
+ const real = a.realAgent;
71
+ if (real) {
72
+ const context = real.context.contextArtifactPath;
73
+ if (context !== null || !allowUnfinished) {
74
+ campaignAssert(context === agentSuccessContextArtifactPath(task.benchmarkProject, task.id, a.treatmentId));
75
+ refs.set(context, "context");
76
+ }
77
+ if (real.agentArtifactDirectory !== null || !allowUnfinished)
78
+ campaignAssert(real.agentArtifactDirectory === agentSuccessAgentArtifactDirectory(task.benchmarkProject, task.id, a.treatmentId, (i + 1)));
79
+ else
80
+ campaignAssert(!real.providerInvoked && Object.values(real.agentArtifacts).every(p => p === null), "MISSING_PROVIDER_REFERENCES");
81
+ for (const [key, p] of Object.entries(real.agentArtifacts))
82
+ if (p !== null) {
83
+ campaignAssert(real.agentArtifactDirectory !== null && p.startsWith(`${real.agentArtifactDirectory}/`) && !p.slice(real.agentArtifactDirectory.length + 1).includes("/") && /\.(json|txt|log)$/.test(p), "UNSAFE_PROVIDER_ARTIFACT");
84
+ campaignAssert(key !== "prompt" || p.endsWith(".txt"));
85
+ campaignAssert(!refs.has(p));
86
+ refs.set(p, "provider");
87
+ }
88
+ }
89
+ }
90
+ return [...refs].map(([path, kind]) => ({ path, kind })).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
91
+ }
92
+ export function validateCampaignCompletion(value, identity, task, treatmentId) {
93
+ validateCampaignIdentity(identity);
94
+ campaignAssert(identity.selected.some((s) => s.caseId === task.id && s.projectId === task.benchmarkProject && s.taskDigest === campaignHash(canonicalCampaignJson(task))), "TASK_IDENTITY_MISMATCH");
95
+ campaignExact(value, ["sourceSchema", "evidence", "taskSuccess", "artifacts"]);
96
+ campaignAssert(value.sourceSchema === "my-dev-kit-lab-agent-success-rate-execution-v1");
97
+ bool(value.taskSuccess);
98
+ treatment(value.evidence);
99
+ const e = value.evidence;
100
+ const rows = e.attempts?.map((a) => a.evidence) ?? [e];
101
+ campaignAssert(rows.length > 0 && rows.length <= 1 + identity.config.repairAttempts);
102
+ for (let i = 0; i < rows.length; i++) {
103
+ const a = rows[i];
104
+ campaignAssert(a.treatmentId === treatmentId && a.status === "completed" && a.availability === "complete" && a.cleanup.attempted && a.cleanup.removed && a.protectedIntegrity.status !== "unproven" && a.sandboxBaseline !== null && a.baselineAssessment?.evaluable === true && a.baselineAssessment.reasons.length === 0 && a.patch.attempted && a.patch.outcome !== null && a.proposedPatchPath !== null && a.baselineVerification !== null, "INCOMPLETE_EVIDENCE");
105
+ campaignAssert(a.protectedIntegrity.status !== "intact" || a.protectedIntegrity.mutatedPaths.length === 0);
106
+ campaignAssert(a.protectedIntegrity.status !== "mutated" || a.protectedIntegrity.mutatedPaths.length > 0);
107
+ campaignAssert(a.protectedIntegrity.mutatedPaths.every((p) => task.protectedFiles.includes(p)), "PROTECTED_PROOF_MISMATCH");
108
+ campaignAssert(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(a.sandboxBaseline.commit) && a.sandboxBaseline.fileCount > 0, "BASELINE_PROOF_MISMATCH");
109
+ if (a.change !== null) {
110
+ const changes = a.change;
111
+ campaignAssert(changes.baselineCommit === a.sandboxBaseline.commit && changes.changedCount === changes.changedFiles.length && new Set(changes.changedFiles.map((f) => f.relativePath)).size === changes.changedFiles.length, "CHANGE_SUMMARY_MISMATCH");
112
+ for (const [status, count] of [["added", changes.addedCount], ["modified", changes.modifiedCount], ["deleted", changes.deletedCount]])
113
+ campaignAssert(count === changes.changedFiles.filter((f) => f.status === status).length, "CHANGE_SUMMARY_MISMATCH");
114
+ campaignAssert(changes.totalAdditions === changes.changedFiles.reduce((sum, f) => sum + (f.additions ?? 0), 0) && changes.totalDeletions === changes.changedFiles.reduce((sum, f) => sum + (f.deletions ?? 0), 0), "CHANGE_SUMMARY_MISMATCH");
115
+ campaignAssert(changes.changedFiles.every((f) => f.status === "added" ? f.beforeSha256 === null && f.afterSha256 !== null : f.status === "deleted" ? f.beforeSha256 !== null && f.afterSha256 === null : f.beforeSha256 !== null && f.afterSha256 !== null), "CHANGE_DIGEST_MISMATCH");
116
+ }
117
+ campaignAssert((a.appliedPatchPath !== null) === (a.patch.outcome === "success"));
118
+ campaignAssert(a.patch.outcome === "success" ? a.change !== null && a.postEditVerification !== null : a.change === null && a.postEditVerification === null);
119
+ for (const [phase, v] of [["baseline", a.baselineVerification], ["post-edit", a.postEditVerification]]) {
120
+ if (v === null)
121
+ continue;
122
+ campaignAssert(v.phase === phase);
123
+ for (const [checkClass, definitions, results] of [["task", task.taskChecks, v.taskResults], ["regression", task.regressionChecks, v.regressionResults]]) {
124
+ campaignAssert(results.length === definitions.length && results.every((r, j) => r.checkId === definitions[j].id && r.class === checkClass && ["passed", "failed"].includes(r.status) && (r.status === "passed" ? r.exitCode === 0 : r.exitCode !== null && r.exitCode !== 0)), "VERIFICATION_MISSING");
125
+ }
126
+ }
127
+ campaignAssert(a.baselineVerification.taskResults.some((r) => r.status === "failed") && a.baselineVerification.regressionResults.every((r) => r.status === "passed"));
128
+ if (identity.config.providerId === "deterministic-fixture")
129
+ campaignAssert(a.realAgent === undefined && a.agentTokenUsage === null && a.timing.agentDurationMs === null);
130
+ else {
131
+ const r = a.realAgent;
132
+ campaignAssert(r && r.providerId === identity.config.providerId && r.attempt === i + 1 && r.providerInvoked && r.providerStatus === "completed" && r.finalAnswerAvailable && r.cwdCleanup.attempted && r.cwdCleanup.removed && r.context.contextMode === treatmentId && r.context.availability !== "unavailable" && r.agentArtifacts.prompt !== null && r.agentArtifacts.result !== null);
133
+ }
134
+ const record = e.attempts?.[i];
135
+ if (a.realAgent?.repairFeedback !== undefined && a.realAgent.repairFeedback !== null)
136
+ campaignAssert(i > 0 && a.realAgent.repairFeedback.basedOnAttempt === i && a.realAgent.repairFeedback.category === e.attempts[i - 1].failureCategory, "REPAIR_FEEDBACK_MISMATCH");
137
+ if (record) {
138
+ const classification = classifyAgentSuccessAttempt(a, analyzeAgentSuccessTreatment(task, a).metrics);
139
+ campaignAssert(record.failureCategory === classification.category && record.repairEligible === classification.repairEligible && record.taskSuccess === classification.taskSuccess && record.durationMs === a.timing.agentDurationMs && canonicalCampaignJson(record.tokenUsage) === canonicalCampaignJson(a.agentTokenUsage) && record.proposedPatchAvailability === (a.proposedPatchPath === null ? "unavailable" : "available") && record.patchApplicationOutcome === (a.patch.outcome ?? "not-attempted") && canonicalCampaignJson(record.baselineAssessment) === canonicalCampaignJson(a.baselineAssessment) && canonicalCampaignJson(record.postEditVerification) === canonicalCampaignJson(a.postEditVerification) && record.changeEvidenceAvailability === (a.change === null ? "unavailable" : "available"));
140
+ campaignAssert(record.attemptNumber === i + 1 && record.providerId === identity.config.providerId && record.providerStatus === a.realAgent?.providerStatus && record.taskSuccess === (analyzeAgentSuccessTreatment(task, a).metrics.taskSuccess.value === true) && canonicalCampaignJson(record.cleanupResult) === canonicalCampaignJson(a.cleanup) && record.completedAt >= record.startedAt);
141
+ if (i > 0)
142
+ campaignAssert(e.attempts[i - 1].repairEligible && e.attempts[i - 1].taskSuccess === false);
143
+ }
144
+ }
145
+ campaignAssert(new Set(rows.map((r) => r.sandboxId)).size === rows.length);
146
+ if (e.attempts) {
147
+ const { attempts: _attempts, ...final } = e;
148
+ campaignAssert(canonicalCampaignJson(final) === canonicalCampaignJson(rows[rows.length - 1]));
149
+ }
150
+ campaignAssert(value.taskSuccess === (analyzeAgentSuccessTreatment(task, e).metrics.taskSuccess.value === true));
151
+ const expected = requiredCampaignArtifacts(e, task);
152
+ list(shape({ path: rel, kind: one("patch", "context", "provider"), sha256: digest, bytes: integer }), 32)(value.artifacts);
153
+ const artifacts = value.artifacts;
154
+ campaignAssert(canonicalCampaignJson(artifacts.map(({ path, kind }) => ({ path, kind }))) === canonicalCampaignJson(expected), "ARTIFACT_REFERENCE_MISMATCH");
155
+ }
156
+ export function sealCampaignCheckpoint(body) { return { ...body, integrity: campaignHash(canonicalCampaignJson(body)) }; }
157
+ export function validateCampaignCheckpoint(value, tasks) {
158
+ campaignAssert(Buffer.byteLength(canonicalCampaignJson(value)) <= CAMPAIGN_CHECKPOINT_MAX_BYTES, "DATA_BOUND");
159
+ campaignExact(value, ["schemaVersion", "identity", "generation", "parentIntegrity", "createdAt", "updatedAt", "units", "integrity"]);
160
+ campaignAssert(value.schemaVersion === CAMPAIGN_CHECKPOINT_SCHEMA, "UNSUPPORTED_SCHEMA");
161
+ validateCampaignIdentity(value.identity);
162
+ integer(value.generation);
163
+ campaignAssert(Number(value.generation) >= 1 && Number(value.generation) <= 1_000_000);
164
+ nullable(digest)(value.parentIntegrity);
165
+ timestamp(value.createdAt);
166
+ timestamp(value.updatedAt);
167
+ campaignAssert(String(value.updatedAt) >= String(value.createdAt));
168
+ const identity = value.identity;
169
+ campaignAssert(tasks.length === identity.selected.length && tasks.every((t, i) => t.id === identity.selected[i].caseId && t.benchmarkProject === identity.selected[i].projectId && campaignHash(canonicalCampaignJson(t)) === identity.selected[i].taskDigest), "TASK_IDENTITY_MISMATCH");
170
+ campaignAssert(Array.isArray(value.units) && value.units.length === tasks.length * 2);
171
+ for (let i = 0; i < value.units.length; i++) {
172
+ const u = value.units[i];
173
+ campaignExact(u, ["caseId", "treatmentId", "state", "errorCode", "completion"]);
174
+ const task = tasks[Math.floor(i / 2)];
175
+ campaignAssert(u.caseId === task.id && u.treatmentId === value.identity.treatmentOrder[i % 2]);
176
+ one("pending", "in-progress", "completed", "incomplete", "paused-limit")(u.state);
177
+ nullable(one("INTERRUPTED", "INVALID_EVIDENCE", "PROVIDER_LIMIT"))(u.errorCode);
178
+ if (u.state === "completed") {
179
+ campaignAssert(u.errorCode === null);
180
+ validateCampaignCompletion(u.completion, value.identity, task, String(u.treatmentId));
181
+ }
182
+ else {
183
+ campaignAssert(u.completion === null);
184
+ campaignAssert(u.state === "paused-limit" ? u.errorCode === "PROVIDER_LIMIT" : u.state === "incomplete" ? ["INTERRUPTED", "INVALID_EVIDENCE"].includes(String(u.errorCode)) : u.errorCode === null);
185
+ }
186
+ }
187
+ const { integrity, ...body } = value;
188
+ campaignAssert(isCampaignDigest(integrity) && integrity === campaignHash(canonicalCampaignJson(body)), "CHECKPOINT_INTEGRITY");
189
+ }
190
+ export function createCampaignCheckpoint(identity, tasks, at) {
191
+ const result = sealCampaignCheckpoint({ schemaVersion: CAMPAIGN_CHECKPOINT_SCHEMA, identity: structuredClone(identity), generation: 1, parentIntegrity: null, createdAt: at, updatedAt: at, units: identity.selected.flatMap((s) => identity.treatmentOrder.map((treatmentId) => ({ caseId: s.caseId, treatmentId, state: "pending", errorCode: null, completion: null }))) });
192
+ validateCampaignCheckpoint(result, tasks);
193
+ return result;
194
+ }
195
+ export function transitionCampaignCheckpoint(previous, tasks, index, next, at, completion = null, classifiedProviderOutcome = null) {
196
+ validateCampaignCheckpoint(previous, tasks);
197
+ campaignAssert(Number.isSafeInteger(index) && index >= 0 && index < previous.units.length);
198
+ campaignAssert(next !== "paused-limit" || (classifiedProviderOutcome === "agent-limit-reached" && previous.identity.config.providerId !== "deterministic-fixture"), "LIMIT_CLASSIFICATION_REQUIRED");
199
+ const state = previous.units[index].state;
200
+ campaignAssert(state === "pending" ? next === "in-progress" : state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(next), "INVALID_TRANSITION");
201
+ const { integrity, ...body } = structuredClone(previous);
202
+ body.generation++;
203
+ body.parentIntegrity = integrity;
204
+ body.updatedAt = at;
205
+ body.units[index] = { ...body.units[index], state: next, errorCode: next === "incomplete" ? "INTERRUPTED" : next === "paused-limit" ? "PROVIDER_LIMIT" : null, completion };
206
+ const result = sealCampaignCheckpoint(body);
207
+ validateCampaignCheckpoint(result, tasks);
208
+ return result;
209
+ }
210
+ export function validateCampaignHistory(previous, next) {
211
+ campaignAssert(previous.identity.digest === next.identity.digest && next.generation === previous.generation + 1 && next.parentIntegrity === previous.integrity && next.createdAt === previous.createdAt && next.updatedAt >= previous.updatedAt, "INVALID_HISTORY");
212
+ let changes = 0;
213
+ previous.units.forEach((u, i) => { const v = next.units[i]; if (canonicalCampaignJson(u) === canonicalCampaignJson(v))
214
+ return; changes++; campaignAssert(u.caseId === v.caseId && u.treatmentId === v.treatmentId && (u.state === "pending" ? v.state === "in-progress" : u.state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(v.state)), "INVALID_HISTORY"); });
215
+ campaignAssert(changes === 1, "INVALID_HISTORY");
216
+ }