@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +19 -3
- package/benchmarks/contracts/agent-success-rate-project-profiles.json +33 -0
- package/benchmarks/contracts/agent-success-rate-tasks.json +671 -0
- package/benchmarks/projects/agent-success-inventory-node/.gitattributes +2 -0
- package/benchmarks/projects/agent-success-inventory-node/README.md +74 -0
- package/benchmarks/projects/agent-success-inventory-node/package.json +14 -0
- package/benchmarks/projects/agent-success-inventory-node/src/fulfillmentReport.js +38 -0
- package/benchmarks/projects/agent-success-inventory-node/src/index.js +26 -0
- package/benchmarks/projects/agent-success-inventory-node/src/inventoryStore.js +114 -0
- package/benchmarks/projects/agent-success-inventory-node/src/quantity.js +19 -0
- package/benchmarks/projects/agent-success-inventory-node/src/reservationService.js +45 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/quantity-edge.check.mjs +51 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/quantity-primary.check.mjs +46 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/regression.check.mjs +115 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/report-edge.check.mjs +109 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/report-primary.check.mjs +47 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/reservation-edge.check.mjs +63 -0
- package/benchmarks/projects/agent-success-inventory-node/tests/reservation-primary.check.mjs +44 -0
- package/benchmarks/projects/agent-success-task-board-node/.gitattributes +2 -0
- package/benchmarks/projects/agent-success-task-board-node/README.md +77 -0
- package/benchmarks/projects/agent-success-task-board-node/package.json +14 -0
- package/benchmarks/projects/agent-success-task-board-node/src/index.js +24 -0
- package/benchmarks/projects/agent-success-task-board-node/src/projectSummary.js +15 -0
- package/benchmarks/projects/agent-success-task-board-node/src/taskService.js +58 -0
- package/benchmarks/projects/agent-success-task-board-node/src/taskStore.js +58 -0
- package/benchmarks/projects/agent-success-task-board-node/src/validation.js +31 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/import-edge.check.mjs +65 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/import-primary.check.mjs +31 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/regression.check.mjs +107 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/summary-edge.check.mjs +67 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/summary-primary.check.mjs +54 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/title-edge.check.mjs +39 -0
- package/benchmarks/projects/agent-success-task-board-node/tests/title-primary.check.mjs +29 -0
- package/dist/src/agents/adapters/claudeAdapter.js +6 -3
- package/dist/src/agents/adapters/codexAdapter.js +51 -10
- package/dist/src/agents/index.js +1 -0
- package/dist/src/agents/normalizeAgentTelemetry.js +50 -0
- package/dist/src/agents/parseAgentTokenUsage.js +71 -49
- package/dist/src/agents/providerBuildObservation.js +97 -0
- package/dist/src/audits/core/projectInventory.js +1 -0
- package/dist/src/cli/help.js +44 -1
- package/dist/src/commands/runExperimentDescribeCommand.js +83 -1
- package/dist/src/commands/runExperimentRunCommand.js +309 -6
- package/dist/src/core/processTree.js +55 -0
- package/dist/src/core/runMeasuredCommand.js +108 -22
- package/dist/src/evaluation/agentSuccess/applyPatchToSandbox.js +37 -0
- package/dist/src/evaluation/agentSuccess/assessBaseline.js +23 -0
- package/dist/src/evaluation/agentSuccess/corpusTypes.js +74 -0
- package/dist/src/evaluation/agentSuccess/extractPatchCandidate.js +46 -0
- package/dist/src/evaluation/agentSuccess/index.js +14 -0
- package/dist/src/evaluation/agentSuccess/parseUnifiedDiff.js +210 -0
- package/dist/src/evaluation/agentSuccess/patchTypes.js +5 -0
- package/dist/src/evaluation/agentSuccess/readAgentSuccessCorpus.js +183 -0
- package/dist/src/evaluation/agentSuccess/runVerificationCheck.js +100 -0
- package/dist/src/evaluation/agentSuccess/taskPaths.js +34 -0
- package/dist/src/evaluation/agentSuccess/taskTypes.js +14 -0
- package/dist/src/evaluation/agentSuccess/validateAgentSuccessCorpus.js +307 -0
- package/dist/src/evaluation/agentSuccess/validateAgentSuccessTask.js +297 -0
- package/dist/src/evaluation/agentSuccess/validatePatchPolicy.js +66 -0
- package/dist/src/evaluation/agentSuccess/verificationTypes.js +1 -0
- package/dist/src/evaluation/benchmarkSandbox/createBenchmarkSandbox.js +127 -0
- package/dist/src/evaluation/benchmarkSandbox/errors.js +11 -0
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +62 -0
- package/dist/src/evaluation/benchmarkSandbox/index.js +8 -0
- package/dist/src/evaluation/benchmarkSandbox/minimalHostEnv.js +37 -0
- package/dist/src/evaluation/benchmarkSandbox/pathPolicy.js +49 -0
- package/dist/src/evaluation/benchmarkSandbox/removeBenchmarkSandbox.js +74 -0
- package/dist/src/evaluation/benchmarkSandbox/treeSnapshot.js +57 -0
- package/dist/src/evaluation/benchmarkSandbox/types.js +3 -0
- package/dist/src/evaluation/changeSet/captureChangeSet.js +98 -0
- package/dist/src/evaluation/changeSet/index.js +2 -0
- package/dist/src/evaluation/changeSet/types.js +9 -0
- package/dist/src/experiments/defaultRegistry.js +2 -0
- package/dist/src/experiments/plugins/agentSuccessRate/agentTaskProjection.js +12 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysis.js +425 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysisArtifact.js +155 -0
- package/dist/src/experiments/plugins/agentSuccessRate/analysisTypes.js +65 -0
- package/dist/src/experiments/plugins/agentSuccessRate/attemptEvidence.js +7 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpoint.js +216 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +588 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignEvidenceQuality.js +77 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +314 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +111 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +173 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignQueue.js +11 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignRecovery.js +256 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +207 -0
- package/dist/src/experiments/plugins/agentSuccessRate/config.js +123 -0
- package/dist/src/experiments/plugins/agentSuccessRate/contextGeneration.js +497 -0
- package/dist/src/experiments/plugins/agentSuccessRate/execution.js +441 -0
- package/dist/src/experiments/plugins/agentSuccessRate/executionArtifact.js +126 -0
- package/dist/src/experiments/plugins/agentSuccessRate/executionTypes.js +2 -0
- package/dist/src/experiments/plugins/agentSuccessRate/index.js +26 -0
- package/dist/src/experiments/plugins/agentSuccessRate/metadata.js +28 -0
- package/dist/src/experiments/plugins/agentSuccessRate/metrics.js +76 -0
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +517 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +272 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +43 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairAnalysis.js +148 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairExecution.js +103 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairFeedback.js +91 -0
- package/dist/src/experiments/plugins/agentSuccessRate/repairPolicy.js +88 -0
- package/dist/src/experiments/plugins/agentSuccessRate/timeoutPresets.js +9 -0
- package/dist/src/experiments/plugins/agentSuccessRate/types.js +1 -0
- package/dist/src/experiments/plugins/warmIndexReuse/plugin.js +2 -3
- package/dist/src/report/experiments/agentSuccessCampaignProgressReportModel.js +1 -0
- package/dist/src/report/experiments/agentSuccessRateReportModel.js +6 -0
- package/dist/src/report/experiments/buildAgentSuccessCampaignProgressReport.js +6 -0
- package/dist/src/report/experiments/buildAgentSuccessRateReport.js +266 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +27 -2
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressHtml.js +6 -0
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressText.js +25 -0
- package/dist/src/report/experiments/renderAgentSuccessRateHtml.js +171 -0
- package/dist/src/report/experiments/renderAgentSuccessRateText.js +146 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +3 -0
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +5 -0
- package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +135 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -3
- package/docs/METRICS.md +79 -0
- package/package.json +1 -1
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every complete attempt evidence record of one treatment, in execution order. Real-agent treatments list their attempts;
|
|
3
|
+
* a deterministic-fixture treatment (or a Batch 4 style record) is its own single attempt.
|
|
4
|
+
*/
|
|
5
|
+
export function attemptEvidenceOf(treatment) {
|
|
6
|
+
return treatment.attempts && treatment.attempts.length > 0 ? treatment.attempts.map((attempt) => attempt.evidence) : [treatment];
|
|
7
|
+
}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { agentSuccessPatchArtifactPath } from "./execution.js";
|
|
2
|
+
import { agentSuccessContextArtifactPath, agentSuccessAgentArtifactDirectory } from "./realAgentExecution.js";
|
|
3
|
+
import { classifyAgentSuccessAttempt } from "./repairPolicy.js";
|
|
4
|
+
import { analyzeAgentSuccessTreatment } from "./analysis.js";
|
|
5
|
+
import { campaignAssert, campaignExact, campaignHash, campaignIdentifier, canonicalCampaignJson, isCampaignDigest, validateCampaignIdentity } from "./campaignIdentity.js";
|
|
6
|
+
export const CAMPAIGN_CHECKPOINT_SCHEMA = "my-dev-kit-lab-agent-success-rate-campaign-checkpoint-v1";
|
|
7
|
+
export const CAMPAIGN_CHECKPOINT_MAX_BYTES = 16 * 1024 * 1024;
|
|
8
|
+
const nullable = (rule) => (v) => { if (v !== null)
|
|
9
|
+
rule(v); };
|
|
10
|
+
const nil = (v) => campaignAssert(v === null);
|
|
11
|
+
const bool = (v) => campaignAssert(typeof v === "boolean");
|
|
12
|
+
const number = (v) => campaignAssert(typeof v === "number" && Number.isFinite(v) && v >= 0);
|
|
13
|
+
const integer = (v) => { number(v); campaignAssert(Number.isSafeInteger(v)); };
|
|
14
|
+
const identifier = (v) => campaignAssert(campaignIdentifier(v));
|
|
15
|
+
const digest = (v) => campaignAssert(isCampaignDigest(v));
|
|
16
|
+
const one = (...values) => (v) => campaignAssert(values.includes(v));
|
|
17
|
+
const list = (rule, max = 2000) => (v) => { campaignAssert(Array.isArray(v) && v.length <= max, "DATA_BOUND"); v.forEach(rule); };
|
|
18
|
+
const shape = (rules, optional = {}) => (v) => {
|
|
19
|
+
campaignAssert(typeof v === "object" && v !== null && !Array.isArray(v));
|
|
20
|
+
const row = v;
|
|
21
|
+
campaignAssert(Object.keys(row).every((k) => Object.hasOwn(rules, k) || Object.hasOwn(optional, k)));
|
|
22
|
+
for (const [k, r] of Object.entries(rules))
|
|
23
|
+
r(row[k]);
|
|
24
|
+
for (const [k, r] of Object.entries(optional))
|
|
25
|
+
if (Object.hasOwn(row, k))
|
|
26
|
+
r(row[k]);
|
|
27
|
+
};
|
|
28
|
+
const timestamp = (v) => campaignAssert(typeof v === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(v) && Number.isFinite(Date.parse(v)) && new Date(v).toISOString() === v);
|
|
29
|
+
export function validateCampaignRelativePath(v) {
|
|
30
|
+
campaignAssert(typeof v === "string" && v.length > 0 && v.length <= 512 && /^[A-Za-z0-9._/-]+$/.test(v) && !v.startsWith("/") && v.split("/").every((p) => p !== "" && p !== "." && p !== ".." && !p.endsWith(".") && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p)), "UNSAFE_PATH");
|
|
31
|
+
}
|
|
32
|
+
const rel = validateCampaignRelativePath;
|
|
33
|
+
const cleanup = shape({ attempted: bool, removed: bool, reason: nil });
|
|
34
|
+
const check = shape({ checkId: identifier, class: one("task", "regression"), status: one("passed", "failed", "timeout", "error"), exitCode: nullable((v) => campaignAssert(typeof v === "number" && Number.isSafeInteger(v))), durationMs: number, failureReason: nil });
|
|
35
|
+
const verification = shape({ phase: one("baseline", "post-edit"), taskResults: list(check), regressionResults: list(check) });
|
|
36
|
+
const assessment = shape({ evaluable: bool, reasons: list(one("NO_CHECKS", "ALL_TASK_CHECKS_PASSED", "REGRESSION_CHECK_FAILED", "CHECK_NOT_EVALUABLE"), 4) });
|
|
37
|
+
const token = shape({ totalTokens: nullable(integer), inputTokens: nullable(integer), outputTokens: nullable(integer), source: one("provider-reported", "agent-reported", "cli-json", "otel", "estimated-from-text", "unavailable"), reliability: one("high", "medium", "low", "unavailable") });
|
|
38
|
+
const providerStatus = one("not-invoked", "completed", "failed", "skipped", "timeout", "agent-unavailable", "agent-limit-reached", "invalid-output");
|
|
39
|
+
const failure = one("none", "patch-malformed", "patch-policy-rejected", "patch-check-failed", "patch-apply-failed", "task-check-failed", "regression-failed", "required-behavior-unsatisfied", "baseline-invalid", "context-unavailable", "provider-unavailable", "provider-limit-reached", "provider-timeout", "provider-failed", "provider-empty-answer", "provider-not-invoked", "infrastructure-failure", "cleanup-failed", "integrity-unproven", "integrity-mutated", "evidence-incomplete");
|
|
40
|
+
const feedback = shape({ basedOnAttempt: one(1, 2, 3), category: failure, patchProduced: bool, patchApplied: bool, taskChecks: one("all-passed", "some-failed", "not-evaluated"), regressionChecks: one("all-passed", "some-failed", "not-evaluated"), requiredBehavior: one("verified", "not-verified", "not-evaluated"), failedTaskCheckCount: nullable(integer), failedRegressionCheckCount: nullable(integer), previousPatchIncluded: bool, previousPatchTruncated: bool });
|
|
41
|
+
const realAgent = shape({ providerId: one("codex", "claude"), attempt: one(1, 2, 3), promptTransport: one("stdin"), providerInvoked: bool, providerStatus, providerStatusReason: nil, finalAnswerAvailable: bool, promptChars: nullable(integer), context: shape({ contextMode: one("raw-full-file", "context-pack"), availability: one("available", "partial", "unavailable"), reason: nil, selectionPolicyId: identifier, myDevKitVersion: nullable(identifier), includedSourceFiles: list(rel), contextChars: integer, estimatedContextTokens: integer, contextArtifactPath: nullable(rel) }), agentArtifactDirectory: nullable(rel), agentArtifacts: shape({ prompt: nullable(rel), result: nullable(rel), stdout: nullable(rel), stderr: nullable(rel), telemetry: nullable(rel) }), cwdCleanup: cleanup }, { repairFeedback: nullable(feedback) });
|
|
42
|
+
const fileStatus = one("added", "modified", "deleted");
|
|
43
|
+
const patch = shape({ attempted: bool, outcome: nullable(one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success")), code: nullable(identifier), message: nil, appliedFiles: list(shape({ path: rel, status: fileStatus })), rejections: list(shape({ code: identifier, path: nullable(rel) }), 20), attemptedProtectedPaths: list(rel), proposedPatchBytes: integer });
|
|
44
|
+
const change = shape({ baselineCommit: identifier, changedFiles: list(shape({ relativePath: rel, status: fileStatus, beforeSha256: nullable(digest), afterSha256: nullable(digest), additions: nullable(integer), deletions: nullable(integer) })), addedCount: integer, modifiedCount: integer, deletedCount: integer, changedCount: integer, totalAdditions: integer, totalDeletions: integer });
|
|
45
|
+
const baseRules = { treatmentId: one("raw-full-file", "context-pack"), status: one("completed", "partial", "failed", "skipped"), availability: one("complete", "baseline-invalid", "incomplete", "infrastructure-failure"), sandboxId: identifier, sandboxBaseline: nullable(shape({ commit: identifier, fileCount: integer, digest })), baselineTextLineCount: nullable(integer), baselineVerification: nullable(verification), baselineAssessment: nullable(assessment), patch, change: nullable(change), postEditVerification: nullable(verification), protectedIntegrity: shape({ status: one("intact", "mutated", "unproven"), mutatedPaths: list(rel) }), timing: shape({ agentDurationMs: nullable(number), baselineVerificationDurationMs: nullable(number), patchPipelineDurationMs: nullable(number), postEditVerificationDurationMs: nullable(number), evaluationDurationMs: nullable(number) }), agentTokenUsage: nullable(token), cleanup, proposedPatchPath: nullable(rel), appliedPatchPath: nullable(rel), errors: list(shape({ code: identifier, message: one("details-omitted") }), 100) };
|
|
46
|
+
const singleEvidence = shape(baseRules, { realAgent });
|
|
47
|
+
const attempt = shape({ attemptNumber: one(1, 2, 3), providerId: one("codex", "claude"), providerStatus, startedAt: timestamp, completedAt: timestamp, durationMs: nullable(number), tokenUsage: nullable(token), proposedPatchAvailability: one("available", "unavailable"), patchApplicationOutcome: one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success", "not-attempted"), baselineAssessment: nullable(assessment), postEditVerification: nullable(verification), changeEvidenceAvailability: one("available", "unavailable"), taskSuccess: nullable(bool), failureCategory: failure, repairEligible: bool, cleanupResult: cleanup, evidence: singleEvidence });
|
|
48
|
+
const treatment = shape(baseRules, { realAgent, attempts: list(attempt, 3) });
|
|
49
|
+
/** Discard free-form explanations only; unknown fields still fail the closed decoder. Original evidence is never mutated. */
|
|
50
|
+
export function projectCampaignTreatmentEvidence(source) {
|
|
51
|
+
canonicalCampaignJson(source);
|
|
52
|
+
const walk = (v) => Array.isArray(v) ? v.map(walk) : v !== null && typeof v === "object" ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, k === "message" && Object.keys(v).length === 2 && Object.hasOwn(v, "code") ? "details-omitted" : ["message", "reason", "failureReason", "providerStatusReason"].includes(k) ? null : walk(x)])) : v;
|
|
53
|
+
const result = walk(source);
|
|
54
|
+
treatment(result);
|
|
55
|
+
return result;
|
|
56
|
+
}
|
|
57
|
+
export function requiredCampaignArtifacts(e, task, allowUnfinished = false) {
|
|
58
|
+
const refs = new Map();
|
|
59
|
+
const attempts = e.attempts?.map((a) => a.evidence) ?? [e];
|
|
60
|
+
for (let i = 0; i < attempts.length; i++) {
|
|
61
|
+
const a = attempts[i];
|
|
62
|
+
for (const kind of ["proposed", "applied"]) {
|
|
63
|
+
const p = kind === "proposed" ? a.proposedPatchPath : a.appliedPatchPath;
|
|
64
|
+
if (p !== null) {
|
|
65
|
+
campaignAssert(p === agentSuccessPatchArtifactPath(task.benchmarkProject, task.id, a.treatmentId, kind, (i + 1)));
|
|
66
|
+
campaignAssert(!refs.has(p));
|
|
67
|
+
refs.set(p, "patch");
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const real = a.realAgent;
|
|
71
|
+
if (real) {
|
|
72
|
+
const context = real.context.contextArtifactPath;
|
|
73
|
+
if (context !== null || !allowUnfinished) {
|
|
74
|
+
campaignAssert(context === agentSuccessContextArtifactPath(task.benchmarkProject, task.id, a.treatmentId));
|
|
75
|
+
refs.set(context, "context");
|
|
76
|
+
}
|
|
77
|
+
if (real.agentArtifactDirectory !== null || !allowUnfinished)
|
|
78
|
+
campaignAssert(real.agentArtifactDirectory === agentSuccessAgentArtifactDirectory(task.benchmarkProject, task.id, a.treatmentId, (i + 1)));
|
|
79
|
+
else
|
|
80
|
+
campaignAssert(!real.providerInvoked && Object.values(real.agentArtifacts).every(p => p === null), "MISSING_PROVIDER_REFERENCES");
|
|
81
|
+
for (const [key, p] of Object.entries(real.agentArtifacts))
|
|
82
|
+
if (p !== null) {
|
|
83
|
+
campaignAssert(real.agentArtifactDirectory !== null && p.startsWith(`${real.agentArtifactDirectory}/`) && !p.slice(real.agentArtifactDirectory.length + 1).includes("/") && /\.(json|txt|log)$/.test(p), "UNSAFE_PROVIDER_ARTIFACT");
|
|
84
|
+
campaignAssert(key !== "prompt" || p.endsWith(".txt"));
|
|
85
|
+
campaignAssert(!refs.has(p));
|
|
86
|
+
refs.set(p, "provider");
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return [...refs].map(([path, kind]) => ({ path, kind })).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
91
|
+
}
|
|
92
|
+
export function validateCampaignCompletion(value, identity, task, treatmentId) {
|
|
93
|
+
validateCampaignIdentity(identity);
|
|
94
|
+
campaignAssert(identity.selected.some((s) => s.caseId === task.id && s.projectId === task.benchmarkProject && s.taskDigest === campaignHash(canonicalCampaignJson(task))), "TASK_IDENTITY_MISMATCH");
|
|
95
|
+
campaignExact(value, ["sourceSchema", "evidence", "taskSuccess", "artifacts"]);
|
|
96
|
+
campaignAssert(value.sourceSchema === "my-dev-kit-lab-agent-success-rate-execution-v1");
|
|
97
|
+
bool(value.taskSuccess);
|
|
98
|
+
treatment(value.evidence);
|
|
99
|
+
const e = value.evidence;
|
|
100
|
+
const rows = e.attempts?.map((a) => a.evidence) ?? [e];
|
|
101
|
+
campaignAssert(rows.length > 0 && rows.length <= 1 + identity.config.repairAttempts);
|
|
102
|
+
for (let i = 0; i < rows.length; i++) {
|
|
103
|
+
const a = rows[i];
|
|
104
|
+
campaignAssert(a.treatmentId === treatmentId && a.status === "completed" && a.availability === "complete" && a.cleanup.attempted && a.cleanup.removed && a.protectedIntegrity.status !== "unproven" && a.sandboxBaseline !== null && a.baselineAssessment?.evaluable === true && a.baselineAssessment.reasons.length === 0 && a.patch.attempted && a.patch.outcome !== null && a.proposedPatchPath !== null && a.baselineVerification !== null, "INCOMPLETE_EVIDENCE");
|
|
105
|
+
campaignAssert(a.protectedIntegrity.status !== "intact" || a.protectedIntegrity.mutatedPaths.length === 0);
|
|
106
|
+
campaignAssert(a.protectedIntegrity.status !== "mutated" || a.protectedIntegrity.mutatedPaths.length > 0);
|
|
107
|
+
campaignAssert(a.protectedIntegrity.mutatedPaths.every((p) => task.protectedFiles.includes(p)), "PROTECTED_PROOF_MISMATCH");
|
|
108
|
+
campaignAssert(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(a.sandboxBaseline.commit) && a.sandboxBaseline.fileCount > 0, "BASELINE_PROOF_MISMATCH");
|
|
109
|
+
if (a.change !== null) {
|
|
110
|
+
const changes = a.change;
|
|
111
|
+
campaignAssert(changes.baselineCommit === a.sandboxBaseline.commit && changes.changedCount === changes.changedFiles.length && new Set(changes.changedFiles.map((f) => f.relativePath)).size === changes.changedFiles.length, "CHANGE_SUMMARY_MISMATCH");
|
|
112
|
+
for (const [status, count] of [["added", changes.addedCount], ["modified", changes.modifiedCount], ["deleted", changes.deletedCount]])
|
|
113
|
+
campaignAssert(count === changes.changedFiles.filter((f) => f.status === status).length, "CHANGE_SUMMARY_MISMATCH");
|
|
114
|
+
campaignAssert(changes.totalAdditions === changes.changedFiles.reduce((sum, f) => sum + (f.additions ?? 0), 0) && changes.totalDeletions === changes.changedFiles.reduce((sum, f) => sum + (f.deletions ?? 0), 0), "CHANGE_SUMMARY_MISMATCH");
|
|
115
|
+
campaignAssert(changes.changedFiles.every((f) => f.status === "added" ? f.beforeSha256 === null && f.afterSha256 !== null : f.status === "deleted" ? f.beforeSha256 !== null && f.afterSha256 === null : f.beforeSha256 !== null && f.afterSha256 !== null), "CHANGE_DIGEST_MISMATCH");
|
|
116
|
+
}
|
|
117
|
+
campaignAssert((a.appliedPatchPath !== null) === (a.patch.outcome === "success"));
|
|
118
|
+
campaignAssert(a.patch.outcome === "success" ? a.change !== null && a.postEditVerification !== null : a.change === null && a.postEditVerification === null);
|
|
119
|
+
for (const [phase, v] of [["baseline", a.baselineVerification], ["post-edit", a.postEditVerification]]) {
|
|
120
|
+
if (v === null)
|
|
121
|
+
continue;
|
|
122
|
+
campaignAssert(v.phase === phase);
|
|
123
|
+
for (const [checkClass, definitions, results] of [["task", task.taskChecks, v.taskResults], ["regression", task.regressionChecks, v.regressionResults]]) {
|
|
124
|
+
campaignAssert(results.length === definitions.length && results.every((r, j) => r.checkId === definitions[j].id && r.class === checkClass && ["passed", "failed"].includes(r.status) && (r.status === "passed" ? r.exitCode === 0 : r.exitCode !== null && r.exitCode !== 0)), "VERIFICATION_MISSING");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
campaignAssert(a.baselineVerification.taskResults.some((r) => r.status === "failed") && a.baselineVerification.regressionResults.every((r) => r.status === "passed"));
|
|
128
|
+
if (identity.config.providerId === "deterministic-fixture")
|
|
129
|
+
campaignAssert(a.realAgent === undefined && a.agentTokenUsage === null && a.timing.agentDurationMs === null);
|
|
130
|
+
else {
|
|
131
|
+
const r = a.realAgent;
|
|
132
|
+
campaignAssert(r && r.providerId === identity.config.providerId && r.attempt === i + 1 && r.providerInvoked && r.providerStatus === "completed" && r.finalAnswerAvailable && r.cwdCleanup.attempted && r.cwdCleanup.removed && r.context.contextMode === treatmentId && r.context.availability !== "unavailable" && r.agentArtifacts.prompt !== null && r.agentArtifacts.result !== null);
|
|
133
|
+
}
|
|
134
|
+
const record = e.attempts?.[i];
|
|
135
|
+
if (a.realAgent?.repairFeedback !== undefined && a.realAgent.repairFeedback !== null)
|
|
136
|
+
campaignAssert(i > 0 && a.realAgent.repairFeedback.basedOnAttempt === i && a.realAgent.repairFeedback.category === e.attempts[i - 1].failureCategory, "REPAIR_FEEDBACK_MISMATCH");
|
|
137
|
+
if (record) {
|
|
138
|
+
const classification = classifyAgentSuccessAttempt(a, analyzeAgentSuccessTreatment(task, a).metrics);
|
|
139
|
+
campaignAssert(record.failureCategory === classification.category && record.repairEligible === classification.repairEligible && record.taskSuccess === classification.taskSuccess && record.durationMs === a.timing.agentDurationMs && canonicalCampaignJson(record.tokenUsage) === canonicalCampaignJson(a.agentTokenUsage) && record.proposedPatchAvailability === (a.proposedPatchPath === null ? "unavailable" : "available") && record.patchApplicationOutcome === (a.patch.outcome ?? "not-attempted") && canonicalCampaignJson(record.baselineAssessment) === canonicalCampaignJson(a.baselineAssessment) && canonicalCampaignJson(record.postEditVerification) === canonicalCampaignJson(a.postEditVerification) && record.changeEvidenceAvailability === (a.change === null ? "unavailable" : "available"));
|
|
140
|
+
campaignAssert(record.attemptNumber === i + 1 && record.providerId === identity.config.providerId && record.providerStatus === a.realAgent?.providerStatus && record.taskSuccess === (analyzeAgentSuccessTreatment(task, a).metrics.taskSuccess.value === true) && canonicalCampaignJson(record.cleanupResult) === canonicalCampaignJson(a.cleanup) && record.completedAt >= record.startedAt);
|
|
141
|
+
if (i > 0)
|
|
142
|
+
campaignAssert(e.attempts[i - 1].repairEligible && e.attempts[i - 1].taskSuccess === false);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
campaignAssert(new Set(rows.map((r) => r.sandboxId)).size === rows.length);
|
|
146
|
+
if (e.attempts) {
|
|
147
|
+
const { attempts: _attempts, ...final } = e;
|
|
148
|
+
campaignAssert(canonicalCampaignJson(final) === canonicalCampaignJson(rows[rows.length - 1]));
|
|
149
|
+
}
|
|
150
|
+
campaignAssert(value.taskSuccess === (analyzeAgentSuccessTreatment(task, e).metrics.taskSuccess.value === true));
|
|
151
|
+
const expected = requiredCampaignArtifacts(e, task);
|
|
152
|
+
list(shape({ path: rel, kind: one("patch", "context", "provider"), sha256: digest, bytes: integer }), 32)(value.artifacts);
|
|
153
|
+
const artifacts = value.artifacts;
|
|
154
|
+
campaignAssert(canonicalCampaignJson(artifacts.map(({ path, kind }) => ({ path, kind }))) === canonicalCampaignJson(expected), "ARTIFACT_REFERENCE_MISMATCH");
|
|
155
|
+
}
|
|
156
|
+
export function sealCampaignCheckpoint(body) { return { ...body, integrity: campaignHash(canonicalCampaignJson(body)) }; }
|
|
157
|
+
export function validateCampaignCheckpoint(value, tasks) {
|
|
158
|
+
campaignAssert(Buffer.byteLength(canonicalCampaignJson(value)) <= CAMPAIGN_CHECKPOINT_MAX_BYTES, "DATA_BOUND");
|
|
159
|
+
campaignExact(value, ["schemaVersion", "identity", "generation", "parentIntegrity", "createdAt", "updatedAt", "units", "integrity"]);
|
|
160
|
+
campaignAssert(value.schemaVersion === CAMPAIGN_CHECKPOINT_SCHEMA, "UNSUPPORTED_SCHEMA");
|
|
161
|
+
validateCampaignIdentity(value.identity);
|
|
162
|
+
integer(value.generation);
|
|
163
|
+
campaignAssert(Number(value.generation) >= 1 && Number(value.generation) <= 1_000_000);
|
|
164
|
+
nullable(digest)(value.parentIntegrity);
|
|
165
|
+
timestamp(value.createdAt);
|
|
166
|
+
timestamp(value.updatedAt);
|
|
167
|
+
campaignAssert(String(value.updatedAt) >= String(value.createdAt));
|
|
168
|
+
const identity = value.identity;
|
|
169
|
+
campaignAssert(tasks.length === identity.selected.length && tasks.every((t, i) => t.id === identity.selected[i].caseId && t.benchmarkProject === identity.selected[i].projectId && campaignHash(canonicalCampaignJson(t)) === identity.selected[i].taskDigest), "TASK_IDENTITY_MISMATCH");
|
|
170
|
+
campaignAssert(Array.isArray(value.units) && value.units.length === tasks.length * 2);
|
|
171
|
+
for (let i = 0; i < value.units.length; i++) {
|
|
172
|
+
const u = value.units[i];
|
|
173
|
+
campaignExact(u, ["caseId", "treatmentId", "state", "errorCode", "completion"]);
|
|
174
|
+
const task = tasks[Math.floor(i / 2)];
|
|
175
|
+
campaignAssert(u.caseId === task.id && u.treatmentId === value.identity.treatmentOrder[i % 2]);
|
|
176
|
+
one("pending", "in-progress", "completed", "incomplete", "paused-limit")(u.state);
|
|
177
|
+
nullable(one("INTERRUPTED", "INVALID_EVIDENCE", "PROVIDER_LIMIT"))(u.errorCode);
|
|
178
|
+
if (u.state === "completed") {
|
|
179
|
+
campaignAssert(u.errorCode === null);
|
|
180
|
+
validateCampaignCompletion(u.completion, value.identity, task, String(u.treatmentId));
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
campaignAssert(u.completion === null);
|
|
184
|
+
campaignAssert(u.state === "paused-limit" ? u.errorCode === "PROVIDER_LIMIT" : u.state === "incomplete" ? ["INTERRUPTED", "INVALID_EVIDENCE"].includes(String(u.errorCode)) : u.errorCode === null);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const { integrity, ...body } = value;
|
|
188
|
+
campaignAssert(isCampaignDigest(integrity) && integrity === campaignHash(canonicalCampaignJson(body)), "CHECKPOINT_INTEGRITY");
|
|
189
|
+
}
|
|
190
|
+
export function createCampaignCheckpoint(identity, tasks, at) {
|
|
191
|
+
const result = sealCampaignCheckpoint({ schemaVersion: CAMPAIGN_CHECKPOINT_SCHEMA, identity: structuredClone(identity), generation: 1, parentIntegrity: null, createdAt: at, updatedAt: at, units: identity.selected.flatMap((s) => identity.treatmentOrder.map((treatmentId) => ({ caseId: s.caseId, treatmentId, state: "pending", errorCode: null, completion: null }))) });
|
|
192
|
+
validateCampaignCheckpoint(result, tasks);
|
|
193
|
+
return result;
|
|
194
|
+
}
|
|
195
|
+
export function transitionCampaignCheckpoint(previous, tasks, index, next, at, completion = null, classifiedProviderOutcome = null) {
|
|
196
|
+
validateCampaignCheckpoint(previous, tasks);
|
|
197
|
+
campaignAssert(Number.isSafeInteger(index) && index >= 0 && index < previous.units.length);
|
|
198
|
+
campaignAssert(next !== "paused-limit" || (classifiedProviderOutcome === "agent-limit-reached" && previous.identity.config.providerId !== "deterministic-fixture"), "LIMIT_CLASSIFICATION_REQUIRED");
|
|
199
|
+
const state = previous.units[index].state;
|
|
200
|
+
campaignAssert(state === "pending" ? next === "in-progress" : state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(next), "INVALID_TRANSITION");
|
|
201
|
+
const { integrity, ...body } = structuredClone(previous);
|
|
202
|
+
body.generation++;
|
|
203
|
+
body.parentIntegrity = integrity;
|
|
204
|
+
body.updatedAt = at;
|
|
205
|
+
body.units[index] = { ...body.units[index], state: next, errorCode: next === "incomplete" ? "INTERRUPTED" : next === "paused-limit" ? "PROVIDER_LIMIT" : null, completion };
|
|
206
|
+
const result = sealCampaignCheckpoint(body);
|
|
207
|
+
validateCampaignCheckpoint(result, tasks);
|
|
208
|
+
return result;
|
|
209
|
+
}
|
|
210
|
+
export function validateCampaignHistory(previous, next) {
|
|
211
|
+
campaignAssert(previous.identity.digest === next.identity.digest && next.generation === previous.generation + 1 && next.parentIntegrity === previous.integrity && next.createdAt === previous.createdAt && next.updatedAt >= previous.updatedAt, "INVALID_HISTORY");
|
|
212
|
+
let changes = 0;
|
|
213
|
+
previous.units.forEach((u, i) => { const v = next.units[i]; if (canonicalCampaignJson(u) === canonicalCampaignJson(v))
|
|
214
|
+
return; changes++; campaignAssert(u.caseId === v.caseId && u.treatmentId === v.treatmentId && (u.state === "pending" ? v.state === "in-progress" : u.state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(v.state)), "INVALID_HISTORY"); });
|
|
215
|
+
campaignAssert(changes === 1, "INVALID_HISTORY");
|
|
216
|
+
}
|