@dailephd/my-dev-kit-lab 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +10 -3
- package/dist/src/agents/adapters/claudeAdapter.js +6 -3
- package/dist/src/agents/adapters/codexAdapter.js +51 -10
- package/dist/src/agents/index.js +1 -0
- package/dist/src/agents/normalizeAgentTelemetry.js +50 -0
- package/dist/src/agents/parseAgentTokenUsage.js +71 -49
- package/dist/src/agents/providerBuildObservation.js +97 -0
- package/dist/src/audits/core/projectInventory.js +1 -0
- package/dist/src/cli/help.js +8 -2
- package/dist/src/commands/runExperimentDescribeCommand.js +31 -0
- package/dist/src/commands/runExperimentRunCommand.js +193 -5
- package/dist/src/core/runMeasuredCommand.js +4 -0
- package/dist/src/evaluation/benchmarkSandbox/gitExecutor.js +3 -1
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpoint.js +216 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +588 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignEvidenceQuality.js +77 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +314 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +111 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +175 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignQueue.js +11 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignRecovery.js +256 -0
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +207 -0
- package/dist/src/experiments/plugins/agentSuccessRate/index.js +6 -0
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +32 -22
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +29 -7
- package/dist/src/experiments/plugins/agentSuccessRate/timeoutPresets.js +9 -0
- package/dist/src/report/experiments/agentSuccessCampaignProgressReportModel.js +1 -0
- package/dist/src/report/experiments/buildAgentSuccessCampaignProgressReport.js +6 -0
- package/dist/src/report/experiments/buildAgentSuccessRateReport.js +11 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +1 -1
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressHtml.js +6 -0
- package/dist/src/report/experiments/renderAgentSuccessCampaignProgressText.js +25 -0
- package/dist/src/report/experiments/renderAgentSuccessRateHtml.js +3 -1
- package/dist/src/report/experiments/renderAgentSuccessRateText.js +7 -0
- package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +139 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -3
- package/dist/src/securityValidation/attackScenarios/payloadCorpus.js +0 -0
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +34 -4
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +106 -2
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +31 -16
- package/dist/src/securityValidation/staticScans/semgrep.js +68 -64
- package/dist/src/securityValidation/types.js +0 -4
- package/dist/src/securityValidation/validate/runSecurityValidation.js +1 -1
- package/docs/METRICS.md +9 -0
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
import { createDefaultExperimentPluginRegistry } from "../experiments/index.js";
|
|
2
2
|
import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/index.js";
|
|
3
|
+
import { AGENT_TIMEOUT_PRESETS } from "../experiments/plugins/agentSuccessRate/timeoutPresets.js";
|
|
4
|
+
/** Public command-owned operational contract, shared by human, JSON and run help. */
|
|
5
|
+
export const AGENT_CAMPAIGN_OPTION_HELP = [
|
|
6
|
+
"--agent-campaign <new|resume|continue-limit>: explicit version 2 campaign; requires stable --out <directory>.",
|
|
7
|
+
"--one-case: campaign-only; process at most the first unfinished selected case, with both treatments mandatory.",
|
|
8
|
+
"--campaign-source-root <directory>: campaign-only verified current source checkout; required when installed source proof is unavailable.",
|
|
9
|
+
"--timeout-preset <standard|extended|maximum>: codex/claude real-agent mode only; 240000/900000/1800000 ms per attempt.",
|
|
10
|
+
"--timeout-ms <ms>: numeric override; mutually exclusive with --timeout-preset. Neither supplied preserves 240000 ms default.",
|
|
11
|
+
"--expected-checkpoint-revision <positive-integer>: continue-limit only; exact committed checkpoint generation.",
|
|
12
|
+
"--limit-case <case-id> --limit-treatment <raw-full-file|context-pack> --limit-epoch <epoch-id>: continue-limit only; exact terminal limit unit and epoch.",
|
|
13
|
+
"--authorize-additional-invocation: mandatory boolean with no value for continue-limit; at most two execution epochs per treatment.",
|
|
14
|
+
"Resume revalidates identity, ownership and artifact digests and skips completed work, including completed task failures.",
|
|
15
|
+
"Real campaigns require --agent <codex|claude> and --include-real-agents plus trusted local build observation bound to invocation.",
|
|
16
|
+
"Native executables and attributable restricted node entrypoints are supported; unsupported wrappers are UNVERIFIABLE. No universal installation support or transitive attestation.",
|
|
17
|
+
"Uncertain invocations block resume; no force replay, marker stealing, automatic quota waiting, polling or provider switching.",
|
|
18
|
+
"Partial campaign work is persisted without final scientific reports; complete campaigns write the existing execution/analysis v1 artifacts.",
|
|
19
|
+
"Campaign exits: 0 consistent bounded work (including one-case partial), 1 paused/blocked/failed execution, 2 invalid CLI usage. Legacy behavior is preserved.",
|
|
20
|
+
"--campaign belongs to warm-index-reuse and is rejected by agent-success-rate; other plugins reject these campaign options."
|
|
21
|
+
];
|
|
22
|
+
export const AGENT_CAMPAIGN_EXAMPLES = [
|
|
23
|
+
"my-dev-kit-lab experiment run --experiment agent-success-rate --agent-campaign new --one-case --out <campaign-output-root> --campaign-source-root <source-checkout>",
|
|
24
|
+
"my-dev-kit-lab experiment run --experiment agent-success-rate --agent-campaign resume --one-case --out <same-campaign-output-root> --campaign-source-root <source-checkout>",
|
|
25
|
+
"my-dev-kit-lab experiment run --experiment agent-success-rate --agent codex --include-real-agents --agent-campaign new --timeout-preset extended --out <campaign-output-root> --campaign-source-root <source-checkout>",
|
|
26
|
+
"my-dev-kit-lab experiment run --experiment agent-success-rate --agent codex --include-real-agents --agent-campaign continue-limit --out <campaign-output-root> --campaign-source-root <source-checkout> --expected-checkpoint-revision <revision> --limit-case <case-id> --limit-treatment <treatment> --limit-epoch <epoch-id> --authorize-additional-invocation"
|
|
27
|
+
];
|
|
3
28
|
// ---------------------------------------------------------------------------
|
|
4
29
|
// v0.4.6 Batch 4 -- reusable experiment-describe command owner.
|
|
5
30
|
//
|
|
@@ -72,6 +97,8 @@ function describeAgentSuccessRate() {
|
|
|
72
97
|
realAgentMode: "Opt-in only: --agent <codex|claude> together with --include-real-agents. Neither flag alone enables it; real execution is never the default.",
|
|
73
98
|
providers: [...AGENT_SUCCESS_RATE_REAL_AGENT_IDS],
|
|
74
99
|
timeoutMs: { default: AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, maximum: AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, scope: "each provider attempt" },
|
|
100
|
+
timeoutPresets: AGENT_TIMEOUT_PRESETS,
|
|
101
|
+
campaign: { modes: ["new", "resume", "continue-limit"], options: [...AGENT_CAMPAIGN_OPTION_HELP], examples: [...AGENT_CAMPAIGN_EXAMPLES] },
|
|
75
102
|
repairAttempts: { allowed: [...AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES], default: 0, maximumTotalAttemptsPerTreatment: 3, mode: "real-agent only" },
|
|
76
103
|
agentContext: "Source-only: the provider sees the public task text and the treatment's source context, never tests, trusted checks or reference patches, and runs in a neutral working directory.",
|
|
77
104
|
patchApplication: "Guarded: the provider returns a unified diff only. The Lab validates it against a patch policy and applies it in a fresh disposable sandbox; canonical benchmark projects are never modified.",
|
|
@@ -115,6 +142,8 @@ function printDescription(plugin) {
|
|
|
115
142
|
console.log(` Real-agent mode: ${agentSuccessRate.realAgentMode}`);
|
|
116
143
|
console.log(` Providers: ${agentSuccessRate.providers.join(", ")}`);
|
|
117
144
|
console.log(` Timeout per attempt: default ${agentSuccessRate.timeoutMs.default} ms, maximum ${agentSuccessRate.timeoutMs.maximum} ms`);
|
|
145
|
+
for (const option of agentSuccessRate.campaign.options)
|
|
146
|
+
console.log(` ${option}`);
|
|
118
147
|
console.log(` Repair attempts: ${agentSuccessRate.repairAttempts.allowed.join(", ")} (default ${agentSuccessRate.repairAttempts.default}; at most ${agentSuccessRate.repairAttempts.maximumTotalAttemptsPerTreatment} attempts per treatment; real-agent mode only)`);
|
|
119
148
|
console.log(` Agent context: ${agentSuccessRate.agentContext}`);
|
|
120
149
|
console.log(` Patch application: ${agentSuccessRate.patchApplication}`);
|
|
@@ -212,6 +241,8 @@ function describeExamples(plugin) {
|
|
|
212
241
|
`my-dev-kit-lab experiment describe --experiment ${plugin.metadata.id}`,
|
|
213
242
|
`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id}${agentOptions}`
|
|
214
243
|
];
|
|
244
|
+
if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID)
|
|
245
|
+
examples.push(...AGENT_CAMPAIGN_EXAMPLES);
|
|
215
246
|
// Only advertise an explicit --target example for plugins that support external targets.
|
|
216
247
|
if (plugin.metadata.supportedTargets.includes("external-local") &&
|
|
217
248
|
plugin.metadata.id !== "context-window-scaling" &&
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFile } from "node:fs/promises";
|
|
1
|
+
import { lstat, mkdir, readFile, readdir } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { parseAgentCommandTemplate } from "../agents/index.js";
|
|
4
4
|
import { parseAgentId } from "../agents/agentRegistry.js";
|
|
@@ -17,6 +17,18 @@ import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/inde
|
|
|
17
17
|
import { writePluginExperimentReports } from "../report/index.js";
|
|
18
18
|
import { createLabExecutionContext, resolvePackageResource } from "../runtime/index.js";
|
|
19
19
|
import { runWarmIndexCampaignPresentation } from "./runWarmIndexCampaignPresentation.js";
|
|
20
|
+
import { observeProviderBuild } from "../agents/providerBuildObservation.js";
|
|
21
|
+
import { resolveAgentTimeoutPreset } from "../experiments/plugins/agentSuccessRate/timeoutPresets.js";
|
|
22
|
+
import { validateAgentSuccessRateConfig } from "../experiments/plugins/agentSuccessRate/config.js";
|
|
23
|
+
import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS } from "../experiments/plugins/agentSuccessRate/realAgentExecution.js";
|
|
24
|
+
import { AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/metadata.js";
|
|
25
|
+
import { executeAgentSuccessCampaignV2, prepareAgentSuccessCampaign } from "../experiments/plugins/agentSuccessRate/campaignExecution.js";
|
|
26
|
+
import { readCampaignCheckpointV2, resolveCampaignRoot, validateCampaignOutputPath } from "../experiments/plugins/agentSuccessRate/campaignCheckpointStore.js";
|
|
27
|
+
import { validateCampaignLimitAuthorization } from "../experiments/plugins/agentSuccessRate/campaignRecovery.js";
|
|
28
|
+
import { campaignIdentifier } from "../experiments/plugins/agentSuccessRate/campaignIdentity.js";
|
|
29
|
+
import { readCampaignProgress } from "../experiments/plugins/agentSuccessRate/campaignProgress.js";
|
|
30
|
+
import { writeAgentSuccessCampaignProgressReport } from "../report/experiments/writeAgentSuccessCampaignProgressReport.js";
|
|
31
|
+
import { boundMessage } from "../experiments/plugins/agentSuccessRate/execution.js";
|
|
20
32
|
// ---------------------------------------------------------------------------
|
|
21
33
|
// v0.4.6 Batch 4 -- reusable experiment-run command owner.
|
|
22
34
|
//
|
|
@@ -85,8 +97,11 @@ const AGENT_SUCCESS_RATE_ALLOWED_FLAGS = [
|
|
|
85
97
|
"--agent",
|
|
86
98
|
"--include-real-agents",
|
|
87
99
|
"--timeout-ms",
|
|
88
|
-
"--repair-attempts"
|
|
100
|
+
"--repair-attempts",
|
|
101
|
+
"--agent-campaign", "--one-case", "--campaign-source-root", "--timeout-preset",
|
|
102
|
+
"--expected-checkpoint-revision", "--limit-case", "--limit-treatment", "--limit-epoch", "--authorize-additional-invocation"
|
|
89
103
|
];
|
|
104
|
+
const AGENT_CAMPAIGN_FLAGS = ["--agent-campaign", "--one-case", "--campaign-source-root", "--expected-checkpoint-revision", "--limit-case", "--limit-treatment", "--limit-epoch", "--authorize-additional-invocation"];
|
|
90
105
|
export function assertAgentSuccessRateFlags(argv) {
|
|
91
106
|
const flags = argv.filter((arg) => arg.startsWith("--"));
|
|
92
107
|
const unsupported = [...new Set(flags.filter((flag) => !AGENT_SUCCESS_RATE_ALLOWED_FLAGS.includes(flag)))];
|
|
@@ -95,8 +110,11 @@ export function assertAgentSuccessRateFlags(argv) {
|
|
|
95
110
|
}
|
|
96
111
|
}
|
|
97
112
|
export async function runExperimentRunCommandFromArgs(argv, options = {}) {
|
|
113
|
+
let parsed = false;
|
|
114
|
+
const campaignUsage = argv.includes("--agent-campaign") || argv.some(a => AGENT_CAMPAIGN_FLAGS.includes(a));
|
|
98
115
|
try {
|
|
99
116
|
const args = parseRunExperimentArgs(argv);
|
|
117
|
+
parsed = true;
|
|
100
118
|
const context = options.context ?? createLabExecutionContext();
|
|
101
119
|
const toolRoot = context.packageRoot;
|
|
102
120
|
const registry = createDefaultExperimentPluginRegistry();
|
|
@@ -119,6 +137,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
|
|
|
119
137
|
});
|
|
120
138
|
}
|
|
121
139
|
const inputs = await loadPluginInputs(args, toolRoot, context, outputRoot);
|
|
140
|
+
if (args.agentCampaign)
|
|
141
|
+
return await runAgentCampaignCommand(args, context, outputRoot, inputs, runId, options);
|
|
122
142
|
const result = await runExperiment({
|
|
123
143
|
pluginId: args.experimentId,
|
|
124
144
|
registry,
|
|
@@ -219,16 +239,126 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
|
|
|
219
239
|
return 0;
|
|
220
240
|
}
|
|
221
241
|
catch (error) {
|
|
222
|
-
if (
|
|
242
|
+
if (campaignUsage) {
|
|
243
|
+
const sourceIndex = argv.indexOf("--campaign-source-root");
|
|
244
|
+
const source = sourceIndex >= 0 ? argv[sourceIndex + 1] : undefined;
|
|
245
|
+
const message = error instanceof Error ? error.message : "CAMPAIGN_FAILED";
|
|
246
|
+
console.error(boundMessage(message, [process.cwd(), options.context?.packageRoot ?? "", ...(source ? [path.resolve(options.context?.invocationCwd ?? process.cwd(), source)] : [])]));
|
|
247
|
+
}
|
|
248
|
+
else if (process.env.DEBUG) {
|
|
223
249
|
console.error(error);
|
|
224
250
|
}
|
|
225
251
|
else {
|
|
226
252
|
console.error(error instanceof Error ? error.message : String(error));
|
|
227
253
|
}
|
|
228
|
-
return 1;
|
|
254
|
+
return campaignUsage && !parsed ? 2 : 1;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
async function runAgentCampaignCommand(args, context, outputRoot, inputs, runId, options) {
|
|
258
|
+
const request = args.agentCampaign;
|
|
259
|
+
const sourceRoot = path.resolve(context.invocationCwd, request.sourceRoot ?? context.packageRoot);
|
|
260
|
+
try {
|
|
261
|
+
const packageName = "@dailephd/my-dev-kit-lab";
|
|
262
|
+
const validPackageVersion = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
|
|
263
|
+
const executingMetadata = JSON.parse(await readFile(path.join(context.packageRoot, "package.json"), "utf8"));
|
|
264
|
+
if (executingMetadata.name !== packageName || typeof executingMetadata.version !== "string" || !validPackageVersion.test(executingMetadata.version))
|
|
265
|
+
throw new Error();
|
|
266
|
+
await resolveCampaignRoot(sourceRoot);
|
|
267
|
+
if (!(await lstat(path.join(sourceRoot, "src"))).isDirectory())
|
|
268
|
+
throw new Error();
|
|
269
|
+
const metadata = JSON.parse(await readFile(path.join(sourceRoot, "package.json"), "utf8"));
|
|
270
|
+
if (metadata.name !== packageName || metadata.version !== executingMetadata.version)
|
|
271
|
+
throw new Error();
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
throw new Error("CAMPAIGN_SOURCE_UNVERIFIABLE: supply --campaign-source-root with a current my-dev-kit-lab source checkout; installed source bytes cannot be guessed.");
|
|
275
|
+
}
|
|
276
|
+
const boundary = { toolRoot: context.packageRoot, outputRoot };
|
|
277
|
+
await validateCampaignOutputPath(boundary, sourceRoot);
|
|
278
|
+
let entries = [];
|
|
279
|
+
try {
|
|
280
|
+
entries = await readdir(outputRoot);
|
|
281
|
+
}
|
|
282
|
+
catch (error) {
|
|
283
|
+
if (error.code !== "ENOENT")
|
|
284
|
+
throw error;
|
|
285
|
+
if (request.mode !== "new")
|
|
286
|
+
throw new Error("RESUMABLE_CAMPAIGN_REQUIRED: --out must contain an existing version 2 campaign.");
|
|
287
|
+
}
|
|
288
|
+
if (request.mode === "new" && entries.length > 0)
|
|
289
|
+
throw new Error("CAMPAIGN_OUTPUT_EXISTS: new requires an empty output root; existing artifacts are preserved.");
|
|
290
|
+
if (request.mode !== "new" && !entries.includes("campaign-checkpoints-v2"))
|
|
291
|
+
throw new Error("RESUMABLE_CAMPAIGN_REQUIRED: no version 2 checkpoint in --out.");
|
|
292
|
+
const validation = validateAgentSuccessRateConfig(args.config);
|
|
293
|
+
if (!validation.valid || !validation.config)
|
|
294
|
+
throw new Error("INVALID_CAMPAIGN_CONFIG");
|
|
295
|
+
const config = validation.config;
|
|
296
|
+
const tasks = selectAgentSuccessTasks(inputs.agentSuccessTasks, config);
|
|
297
|
+
const runtime = options.campaignRuntime;
|
|
298
|
+
const provider = config.agentId ? { kind: "real-agent", providerId: config.agentId, includeRealAgents: true, timeoutMs: config.timeoutMs ?? AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, repairAttempts: (config.repairAttempts ?? 0), kitCommand: config.kitCommand, contextDependencies: runtime?.contextDependencies, env: runtime?.env } : { kind: "deterministic-fixture" };
|
|
299
|
+
const preparation = { boundary, sourceRoot, corpusFile: resolvePackageResource(context, AGENT_SUCCESS_TASK_CATALOG_PATH), corpusRoot: context.packageRoot, tasks, treatments: AGENT_SUCCESS_RATE_TREATMENT_IDS, provider, dependencies: runtime?.dependencies };
|
|
300
|
+
// Source, corpus and canonical project byte checks precede even a task-free version probe.
|
|
301
|
+
let prepared = await prepareAgentSuccessCampaign(preparation);
|
|
302
|
+
if (provider.kind === "real-agent") {
|
|
303
|
+
provider.trustedExecutable = await observeProviderBuild(provider.providerId, { env: runtime?.env });
|
|
304
|
+
prepared = await prepareAgentSuccessCampaign(preparation);
|
|
305
|
+
}
|
|
306
|
+
let intent = request.mode === "new" ? { kind: "new-resumable" } : { kind: "resume" };
|
|
307
|
+
let reused = 0;
|
|
308
|
+
const previousEpochs = new Set();
|
|
309
|
+
if (request.mode !== "new") {
|
|
310
|
+
// Identity-scoped stores are not discovery: a different digest cannot create another campaign on resume.
|
|
311
|
+
const identities = await readdir(path.join(outputRoot, "campaign-checkpoints-v2"));
|
|
312
|
+
if (!identities.includes(prepared.identity.digest))
|
|
313
|
+
throw new Error("CAMPAIGN_IDENTITY_MISMATCH: current inputs do not match this output root.");
|
|
314
|
+
const stored = await readCampaignCheckpointV2(boundary, prepared.identity, tasks);
|
|
315
|
+
if (!stored.reusable || !stored.checkpoint || stored.identityVerification !== "MATCH")
|
|
316
|
+
throw new Error("RESUMABLE_CAMPAIGN_REQUIRED_MATCH");
|
|
317
|
+
const cp = stored.checkpoint;
|
|
318
|
+
cp.units.flatMap(u => u.epochs).forEach(e => previousEpochs.add(e.id));
|
|
319
|
+
reused = cp.units.filter(u => u.state === "completed").length;
|
|
320
|
+
if (request.mode === "continue-limit") {
|
|
321
|
+
const unit = cp.units.find(u => u.caseId === request.limitCase && u.treatmentId === request.limitTreatment);
|
|
322
|
+
if (cp.generation !== request.expectedRevision || !unit || unit.epochs.at(-1)?.id !== request.limitEpoch)
|
|
323
|
+
throw new Error("LIMIT_AUTHORIZATION_MISMATCH");
|
|
324
|
+
const authorization = { identityDigest: cp.identity.digest, generation: request.expectedRevision, revision: cp.integrity, caseId: request.limitCase, treatmentId: request.limitTreatment, acknowledgeInvocation: true, acknowledgeExternalLimit: true };
|
|
325
|
+
validateCampaignLimitAuthorization(authorization, cp, unit);
|
|
326
|
+
intent = { kind: "authorize-limit-continuation", authorization };
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
await mkdir(outputRoot, { recursive: true });
|
|
331
|
+
await validateCampaignOutputPath(boundary, sourceRoot);
|
|
332
|
+
if ((await readdir(outputRoot)).length !== 0)
|
|
333
|
+
throw new Error("CAMPAIGN_OUTPUT_EXISTS");
|
|
229
334
|
}
|
|
335
|
+
const result = await executeAgentSuccessCampaignV2(prepared, { intent, runId, limit: request.oneCase ? "one-case" : "all-selected-cases", hooks: runtime?.hooks });
|
|
336
|
+
const progressLines = [];
|
|
337
|
+
let progressFailed = false;
|
|
338
|
+
let progressValidated = false;
|
|
339
|
+
try {
|
|
340
|
+
const snapshot = await readCampaignProgress(boundary, prepared.identity, tasks);
|
|
341
|
+
progressValidated = true;
|
|
342
|
+
const progress = await writeAgentSuccessCampaignProgressReport({ snapshot, boundary });
|
|
343
|
+
progressLines.push(`Progress status: ${progress.model.state}`, `Progress checkpoint revision: ${progress.model.checkpointGeneration} (${progress.model.checkpointRevision})`, `Completed evaluations: ${progress.model.unitCounts.completed}/${progress.model.totalTreatmentUnits}`, `Completed cases: ${progress.model.caseCounts.completed}/${progress.model.selectedCaseCount}`, `Progress JSON: ${progress.outputPaths.jsonPath}`, `Progress Text: ${progress.outputPaths.textPath}`, `Progress HTML: ${progress.outputPaths.htmlPath}`);
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
progressFailed = true;
|
|
347
|
+
progressLines.push(progressValidated ? "Progress persistence: failed (validated checkpoint retained)." : "Progress status: blocked-integrity (checkpoint unavailable or untrusted).");
|
|
348
|
+
progressLines.push("Progress publication: failed; checkpoint/scientific execution outcome is reported separately.");
|
|
349
|
+
}
|
|
350
|
+
const newEpochs = result.checkpoint?.units.flatMap(u => u.epochs).filter(e => !previousEpochs.has(e.id)) ?? [];
|
|
351
|
+
// Provider evidence, never mere creation of a reservation, establishes invocation.
|
|
352
|
+
const newWork = newEpochs.some(e => e.terminal?.evidence.realAgent?.providerInvoked === true) ? "invoked" : newEpochs.some(e => e.phase === "invocation-authorized" || e.phase === "uncertain") ? "unconfirmed" : "none";
|
|
353
|
+
console.log(["Experiment: agent-success-rate", `Campaign mode: ${request.mode}`, `Identity: ${prepared.identity.digest.slice(0, 16)}`, `Status: ${result.status === "provider-limit" ? "paused-provider-limit" : result.status}`, `New provider work: ${newWork}`, `Validated work reused: ${reused > 0 ? "yes" : "no"}`, `Checkpoint revision: ${result.checkpoint?.generation ?? "unavailable"}`, `Final scientific artifacts: ${result.scientificArtifactsWritten ? "written" : "not written"}`, `Output: ${outputRoot}`, ...result.errors.map(e => `Failure: ${e}`)].join("\n"));
|
|
354
|
+
console.log(progressLines.join("\n"));
|
|
355
|
+
return !progressFailed && ["completed", "already-complete", "partial"].includes(result.status) ? 0 : 1;
|
|
230
356
|
}
|
|
231
357
|
export function parseRunExperimentArgs(argv) {
|
|
358
|
+
let agentCampaignMode;
|
|
359
|
+
let oneCase = false, authorize = false;
|
|
360
|
+
let campaignSourceRoot, limitCase, limitTreatment, limitEpoch, timeoutPreset;
|
|
361
|
+
let expectedRevision;
|
|
232
362
|
let experimentId = "";
|
|
233
363
|
let targetPath;
|
|
234
364
|
let outDir;
|
|
@@ -259,9 +389,41 @@ export function parseRunExperimentArgs(argv) {
|
|
|
259
389
|
assertAgentSuccessRateFlags(argv);
|
|
260
390
|
for (let index = 0; index < argv.length; index += 1) {
|
|
261
391
|
const arg = argv[index];
|
|
392
|
+
if (arg !== undefined && arg.startsWith("--") && seenFlags.includes(arg) && (AGENT_CAMPAIGN_FLAGS.includes(arg) || arg === "--timeout-preset" || (argv.includes("agent-success-rate") && arg !== "--case" && arg !== "--benchmark-project")))
|
|
393
|
+
throw new Error(`${arg} may be supplied only once.`);
|
|
262
394
|
if (arg !== undefined && arg.startsWith("--"))
|
|
263
395
|
seenFlags.push(arg);
|
|
264
|
-
if (arg === "--
|
|
396
|
+
if (arg === "--agent-campaign") {
|
|
397
|
+
agentCampaignMode = readRequiredValue(argv, ++index, arg);
|
|
398
|
+
}
|
|
399
|
+
else if (arg === "--one-case") {
|
|
400
|
+
oneCase = true;
|
|
401
|
+
}
|
|
402
|
+
else if (arg === "--authorize-additional-invocation") {
|
|
403
|
+
authorize = true;
|
|
404
|
+
}
|
|
405
|
+
else if (arg === "--campaign-source-root") {
|
|
406
|
+
campaignSourceRoot = readRequiredValue(argv, ++index, arg);
|
|
407
|
+
}
|
|
408
|
+
else if (arg === "--expected-checkpoint-revision") {
|
|
409
|
+
const value = readRequiredValue(argv, ++index, arg);
|
|
410
|
+
if (!/^[1-9]\d*$/.test(value))
|
|
411
|
+
throw new Error("--expected-checkpoint-revision must be a positive integer.");
|
|
412
|
+
expectedRevision = parsePositiveInteger(arg, value);
|
|
413
|
+
}
|
|
414
|
+
else if (arg === "--limit-case") {
|
|
415
|
+
limitCase = readRequiredValue(argv, ++index, arg);
|
|
416
|
+
}
|
|
417
|
+
else if (arg === "--limit-treatment") {
|
|
418
|
+
limitTreatment = readRequiredValue(argv, ++index, arg);
|
|
419
|
+
}
|
|
420
|
+
else if (arg === "--limit-epoch") {
|
|
421
|
+
limitEpoch = readRequiredValue(argv, ++index, arg);
|
|
422
|
+
}
|
|
423
|
+
else if (arg === "--timeout-preset") {
|
|
424
|
+
timeoutPreset = readRequiredValue(argv, ++index, arg);
|
|
425
|
+
}
|
|
426
|
+
else if (arg === "--experiment") {
|
|
265
427
|
experimentId = readRequiredValue(argv, ++index, "--experiment");
|
|
266
428
|
}
|
|
267
429
|
else if (arg === "--target") {
|
|
@@ -353,6 +515,31 @@ export function parseRunExperimentArgs(argv) {
|
|
|
353
515
|
if (!experimentId) {
|
|
354
516
|
throw new Error("Usage: --experiment <id> [--target <path>] [--out <directory>]");
|
|
355
517
|
}
|
|
518
|
+
const campaignFlagsPresent = seenFlags.some(f => AGENT_CAMPAIGN_FLAGS.includes(f));
|
|
519
|
+
if ((campaignFlagsPresent || timeoutPreset !== undefined) && experimentId !== agentSuccessRatePlugin.metadata.id)
|
|
520
|
+
throw new Error("Agent campaign and timeout preset flags are only supported for --experiment agent-success-rate.");
|
|
521
|
+
if (campaignFlagsPresent && agentCampaignMode === undefined)
|
|
522
|
+
throw new Error("Campaign flags require --agent-campaign <new|resume|continue-limit>.");
|
|
523
|
+
if (agentCampaignMode !== undefined && !["new", "resume", "continue-limit"].includes(agentCampaignMode))
|
|
524
|
+
throw new Error("--agent-campaign must be new, resume or continue-limit.");
|
|
525
|
+
if (agentCampaignMode !== undefined && !outDir?.trim())
|
|
526
|
+
throw new Error("--agent-campaign requires a stable --out directory.");
|
|
527
|
+
const limitPresent = expectedRevision !== undefined || limitCase !== undefined || limitTreatment !== undefined || limitEpoch !== undefined || authorize;
|
|
528
|
+
if (agentCampaignMode !== "continue-limit" && limitPresent)
|
|
529
|
+
throw new Error("Limit authorization flags require --agent-campaign continue-limit.");
|
|
530
|
+
if (agentCampaignMode === "continue-limit") {
|
|
531
|
+
if (expectedRevision === undefined || !limitCase || !limitTreatment || !limitEpoch || !authorize)
|
|
532
|
+
throw new Error("continue-limit requires --expected-checkpoint-revision, --limit-case, --limit-treatment, --limit-epoch and --authorize-additional-invocation.");
|
|
533
|
+
if (expectedRevision > 1000000 || !campaignIdentifier(limitCase) || !["raw-full-file", "context-pack"].includes(limitTreatment) || !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(limitEpoch))
|
|
534
|
+
throw new Error("Invalid limit revision, case, treatment or epoch identifier.");
|
|
535
|
+
}
|
|
536
|
+
if (timeoutPreset !== undefined) {
|
|
537
|
+
if (timeoutMs !== undefined)
|
|
538
|
+
throw new Error("--timeout-preset and --timeout-ms cannot be combined.");
|
|
539
|
+
if (agentFlagValues.length !== 1 || includeRealAgents !== true || !["codex", "claude"].includes(agentFlagValues[0]))
|
|
540
|
+
throw new Error("--timeout-preset requires --agent <codex|claude> and --include-real-agents.");
|
|
541
|
+
timeoutMs = resolveAgentTimeoutPreset(agentFlagValues[0], timeoutPreset);
|
|
542
|
+
}
|
|
356
543
|
const KIT_COMMAND_PLUGIN_IDS = [
|
|
357
544
|
warmIndexReusePlugin.metadata.id,
|
|
358
545
|
incrementalChangeStalenessPlugin.metadata.id,
|
|
@@ -540,6 +727,7 @@ export function parseRunExperimentArgs(argv) {
|
|
|
540
727
|
}
|
|
541
728
|
return {
|
|
542
729
|
experimentId,
|
|
730
|
+
...(agentCampaignMode ? { agentCampaign: { mode: agentCampaignMode, oneCase, sourceRoot: campaignSourceRoot, expectedRevision, limitCase, limitTreatment: limitTreatment, limitEpoch, authorize } } : {}),
|
|
543
731
|
targetPath,
|
|
544
732
|
outDir,
|
|
545
733
|
syntheticConfigPath,
|
|
@@ -56,6 +56,7 @@ export async function runMeasuredCommand(options) {
|
|
|
56
56
|
cwd: options.cwd,
|
|
57
57
|
env: childEnv,
|
|
58
58
|
shell: false,
|
|
59
|
+
windowsHide: true,
|
|
59
60
|
stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"],
|
|
60
61
|
// Own process group so a bounded termination can reach descendants. Windows reaches the tree
|
|
61
62
|
// through `taskkill /T`, and `detached` there would open a new console, so it is not set.
|
|
@@ -123,6 +124,9 @@ export async function runMeasuredCommand(options) {
|
|
|
123
124
|
// without asserting a limit of our own.
|
|
124
125
|
if (error.code === "ENOENT" && existsSync(options.cwd)) {
|
|
125
126
|
spawnError += ` (the working directory exists but the process could not be started in it; cwd length ${options.cwd.length} characters)`;
|
|
127
|
+
if (process.platform === "win32") {
|
|
128
|
+
spawnError += "; use a shorter output/sandbox path on this Windows host";
|
|
129
|
+
}
|
|
126
130
|
}
|
|
127
131
|
});
|
|
128
132
|
// A failed spawn can also make the OS report the never-connected stdout/stderr pipes as errors (Windows:
|
|
@@ -31,7 +31,9 @@ export async function runSandboxGit(target, args, options = {}) {
|
|
|
31
31
|
const result = await runMeasuredCommand({
|
|
32
32
|
commandId,
|
|
33
33
|
executable: "git",
|
|
34
|
-
|
|
34
|
+
// Initialization creates .git paths before repository-local configuration exists.
|
|
35
|
+
// Keep long-path support command-local so even that first invocation is covered.
|
|
36
|
+
args: ["-c", "core.quotepath=false", "-c", "core.autocrlf=false", "-c", "core.longpaths=true", "-c", "core.fsmonitor=false", ...args],
|
|
35
37
|
cwd: target.projectRoot,
|
|
36
38
|
outDir: path.join(target.evidenceRoot, "git"),
|
|
37
39
|
env,
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { agentSuccessPatchArtifactPath } from "./execution.js";
|
|
2
|
+
import { agentSuccessContextArtifactPath, agentSuccessAgentArtifactDirectory } from "./realAgentExecution.js";
|
|
3
|
+
import { classifyAgentSuccessAttempt } from "./repairPolicy.js";
|
|
4
|
+
import { analyzeAgentSuccessTreatment } from "./analysis.js";
|
|
5
|
+
import { campaignAssert, campaignExact, campaignHash, campaignIdentifier, canonicalCampaignJson, isCampaignDigest, validateCampaignIdentity } from "./campaignIdentity.js";
|
|
6
|
+
export const CAMPAIGN_CHECKPOINT_SCHEMA = "my-dev-kit-lab-agent-success-rate-campaign-checkpoint-v1";
|
|
7
|
+
export const CAMPAIGN_CHECKPOINT_MAX_BYTES = 16 * 1024 * 1024;
|
|
8
|
+
const nullable = (rule) => (v) => { if (v !== null)
|
|
9
|
+
rule(v); };
|
|
10
|
+
const nil = (v) => campaignAssert(v === null);
|
|
11
|
+
const bool = (v) => campaignAssert(typeof v === "boolean");
|
|
12
|
+
const number = (v) => campaignAssert(typeof v === "number" && Number.isFinite(v) && v >= 0);
|
|
13
|
+
const integer = (v) => { number(v); campaignAssert(Number.isSafeInteger(v)); };
|
|
14
|
+
const identifier = (v) => campaignAssert(campaignIdentifier(v));
|
|
15
|
+
const digest = (v) => campaignAssert(isCampaignDigest(v));
|
|
16
|
+
const one = (...values) => (v) => campaignAssert(values.includes(v));
|
|
17
|
+
const list = (rule, max = 2000) => (v) => { campaignAssert(Array.isArray(v) && v.length <= max, "DATA_BOUND"); v.forEach(rule); };
|
|
18
|
+
const shape = (rules, optional = {}) => (v) => {
|
|
19
|
+
campaignAssert(typeof v === "object" && v !== null && !Array.isArray(v));
|
|
20
|
+
const row = v;
|
|
21
|
+
campaignAssert(Object.keys(row).every((k) => Object.hasOwn(rules, k) || Object.hasOwn(optional, k)));
|
|
22
|
+
for (const [k, r] of Object.entries(rules))
|
|
23
|
+
r(row[k]);
|
|
24
|
+
for (const [k, r] of Object.entries(optional))
|
|
25
|
+
if (Object.hasOwn(row, k))
|
|
26
|
+
r(row[k]);
|
|
27
|
+
};
|
|
28
|
+
const timestamp = (v) => campaignAssert(typeof v === "string" && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(v) && Number.isFinite(Date.parse(v)) && new Date(v).toISOString() === v);
|
|
29
|
+
export function validateCampaignRelativePath(v) {
|
|
30
|
+
campaignAssert(typeof v === "string" && v.length > 0 && v.length <= 512 && /^[A-Za-z0-9._/-]+$/.test(v) && !v.startsWith("/") && v.split("/").every((p) => p !== "" && p !== "." && p !== ".." && !p.endsWith(".") && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p)), "UNSAFE_PATH");
|
|
31
|
+
}
|
|
32
|
+
const rel = validateCampaignRelativePath;
|
|
33
|
+
const cleanup = shape({ attempted: bool, removed: bool, reason: nil });
|
|
34
|
+
const check = shape({ checkId: identifier, class: one("task", "regression"), status: one("passed", "failed", "timeout", "error"), exitCode: nullable((v) => campaignAssert(typeof v === "number" && Number.isSafeInteger(v))), durationMs: number, failureReason: nil });
|
|
35
|
+
const verification = shape({ phase: one("baseline", "post-edit"), taskResults: list(check), regressionResults: list(check) });
|
|
36
|
+
const assessment = shape({ evaluable: bool, reasons: list(one("NO_CHECKS", "ALL_TASK_CHECKS_PASSED", "REGRESSION_CHECK_FAILED", "CHECK_NOT_EVALUABLE"), 4) });
|
|
37
|
+
const token = shape({ totalTokens: nullable(integer), inputTokens: nullable(integer), outputTokens: nullable(integer), source: one("provider-reported", "agent-reported", "cli-json", "otel", "estimated-from-text", "unavailable"), reliability: one("high", "medium", "low", "unavailable") });
|
|
38
|
+
const providerStatus = one("not-invoked", "completed", "failed", "skipped", "timeout", "agent-unavailable", "agent-limit-reached", "invalid-output");
|
|
39
|
+
const failure = one("none", "patch-malformed", "patch-policy-rejected", "patch-check-failed", "patch-apply-failed", "task-check-failed", "regression-failed", "required-behavior-unsatisfied", "baseline-invalid", "context-unavailable", "provider-unavailable", "provider-limit-reached", "provider-timeout", "provider-failed", "provider-empty-answer", "provider-not-invoked", "infrastructure-failure", "cleanup-failed", "integrity-unproven", "integrity-mutated", "evidence-incomplete");
|
|
40
|
+
const feedback = shape({ basedOnAttempt: one(1, 2, 3), category: failure, patchProduced: bool, patchApplied: bool, taskChecks: one("all-passed", "some-failed", "not-evaluated"), regressionChecks: one("all-passed", "some-failed", "not-evaluated"), requiredBehavior: one("verified", "not-verified", "not-evaluated"), failedTaskCheckCount: nullable(integer), failedRegressionCheckCount: nullable(integer), previousPatchIncluded: bool, previousPatchTruncated: bool });
|
|
41
|
+
const realAgent = shape({ providerId: one("codex", "claude"), attempt: one(1, 2, 3), promptTransport: one("stdin"), providerInvoked: bool, providerStatus, providerStatusReason: nil, finalAnswerAvailable: bool, promptChars: nullable(integer), context: shape({ contextMode: one("raw-full-file", "context-pack"), availability: one("available", "partial", "unavailable"), reason: nil, selectionPolicyId: identifier, myDevKitVersion: nullable(identifier), includedSourceFiles: list(rel), contextChars: integer, estimatedContextTokens: integer, contextArtifactPath: nullable(rel) }), agentArtifactDirectory: nullable(rel), agentArtifacts: shape({ prompt: nullable(rel), result: nullable(rel), stdout: nullable(rel), stderr: nullable(rel), telemetry: nullable(rel) }), cwdCleanup: cleanup }, { repairFeedback: nullable(feedback) });
|
|
42
|
+
const fileStatus = one("added", "modified", "deleted");
|
|
43
|
+
const patch = shape({ attempted: bool, outcome: nullable(one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success")), code: nullable(identifier), message: nil, appliedFiles: list(shape({ path: rel, status: fileStatus })), rejections: list(shape({ code: identifier, path: nullable(rel) }), 20), attemptedProtectedPaths: list(rel), proposedPatchBytes: integer });
|
|
44
|
+
const change = shape({ baselineCommit: identifier, changedFiles: list(shape({ relativePath: rel, status: fileStatus, beforeSha256: nullable(digest), afterSha256: nullable(digest), additions: nullable(integer), deletions: nullable(integer) })), addedCount: integer, modifiedCount: integer, deletedCount: integer, changedCount: integer, totalAdditions: integer, totalDeletions: integer });
|
|
45
|
+
const baseRules = { treatmentId: one("raw-full-file", "context-pack"), status: one("completed", "partial", "failed", "skipped"), availability: one("complete", "baseline-invalid", "incomplete", "infrastructure-failure"), sandboxId: identifier, sandboxBaseline: nullable(shape({ commit: identifier, fileCount: integer, digest })), baselineTextLineCount: nullable(integer), baselineVerification: nullable(verification), baselineAssessment: nullable(assessment), patch, change: nullable(change), postEditVerification: nullable(verification), protectedIntegrity: shape({ status: one("intact", "mutated", "unproven"), mutatedPaths: list(rel) }), timing: shape({ agentDurationMs: nullable(number), baselineVerificationDurationMs: nullable(number), patchPipelineDurationMs: nullable(number), postEditVerificationDurationMs: nullable(number), evaluationDurationMs: nullable(number) }), agentTokenUsage: nullable(token), cleanup, proposedPatchPath: nullable(rel), appliedPatchPath: nullable(rel), errors: list(shape({ code: identifier, message: one("details-omitted") }), 100) };
|
|
46
|
+
const singleEvidence = shape(baseRules, { realAgent });
|
|
47
|
+
const attempt = shape({ attemptNumber: one(1, 2, 3), providerId: one("codex", "claude"), providerStatus, startedAt: timestamp, completedAt: timestamp, durationMs: nullable(number), tokenUsage: nullable(token), proposedPatchAvailability: one("available", "unavailable"), patchApplicationOutcome: one("parse-failure", "policy-rejection", "git-check-failure", "git-apply-failure", "success", "not-attempted"), baselineAssessment: nullable(assessment), postEditVerification: nullable(verification), changeEvidenceAvailability: one("available", "unavailable"), taskSuccess: nullable(bool), failureCategory: failure, repairEligible: bool, cleanupResult: cleanup, evidence: singleEvidence });
|
|
48
|
+
const treatment = shape(baseRules, { realAgent, attempts: list(attempt, 3) });
|
|
49
|
+
/** Discard free-form explanations only; unknown fields still fail the closed decoder. Original evidence is never mutated. */
|
|
50
|
+
export function projectCampaignTreatmentEvidence(source) {
|
|
51
|
+
canonicalCampaignJson(source);
|
|
52
|
+
const walk = (v) => Array.isArray(v) ? v.map(walk) : v !== null && typeof v === "object" ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, k === "message" && Object.keys(v).length === 2 && Object.hasOwn(v, "code") ? "details-omitted" : ["message", "reason", "failureReason", "providerStatusReason"].includes(k) ? null : walk(x)])) : v;
|
|
53
|
+
const result = walk(source);
|
|
54
|
+
treatment(result);
|
|
55
|
+
return result;
|
|
56
|
+
}
|
|
57
|
+
export function requiredCampaignArtifacts(e, task, allowUnfinished = false) {
|
|
58
|
+
const refs = new Map();
|
|
59
|
+
const attempts = e.attempts?.map((a) => a.evidence) ?? [e];
|
|
60
|
+
for (let i = 0; i < attempts.length; i++) {
|
|
61
|
+
const a = attempts[i];
|
|
62
|
+
for (const kind of ["proposed", "applied"]) {
|
|
63
|
+
const p = kind === "proposed" ? a.proposedPatchPath : a.appliedPatchPath;
|
|
64
|
+
if (p !== null) {
|
|
65
|
+
campaignAssert(p === agentSuccessPatchArtifactPath(task.benchmarkProject, task.id, a.treatmentId, kind, (i + 1)));
|
|
66
|
+
campaignAssert(!refs.has(p));
|
|
67
|
+
refs.set(p, "patch");
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const real = a.realAgent;
|
|
71
|
+
if (real) {
|
|
72
|
+
const context = real.context.contextArtifactPath;
|
|
73
|
+
if (context !== null || !allowUnfinished) {
|
|
74
|
+
campaignAssert(context === agentSuccessContextArtifactPath(task.benchmarkProject, task.id, a.treatmentId));
|
|
75
|
+
refs.set(context, "context");
|
|
76
|
+
}
|
|
77
|
+
if (real.agentArtifactDirectory !== null || !allowUnfinished)
|
|
78
|
+
campaignAssert(real.agentArtifactDirectory === agentSuccessAgentArtifactDirectory(task.benchmarkProject, task.id, a.treatmentId, (i + 1)));
|
|
79
|
+
else
|
|
80
|
+
campaignAssert(!real.providerInvoked && Object.values(real.agentArtifacts).every(p => p === null), "MISSING_PROVIDER_REFERENCES");
|
|
81
|
+
for (const [key, p] of Object.entries(real.agentArtifacts))
|
|
82
|
+
if (p !== null) {
|
|
83
|
+
campaignAssert(real.agentArtifactDirectory !== null && p.startsWith(`${real.agentArtifactDirectory}/`) && !p.slice(real.agentArtifactDirectory.length + 1).includes("/") && /\.(json|txt|log)$/.test(p), "UNSAFE_PROVIDER_ARTIFACT");
|
|
84
|
+
campaignAssert(key !== "prompt" || p.endsWith(".txt"));
|
|
85
|
+
campaignAssert(!refs.has(p));
|
|
86
|
+
refs.set(p, "provider");
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return [...refs].map(([path, kind]) => ({ path, kind })).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
91
|
+
}
|
|
92
|
+
export function validateCampaignCompletion(value, identity, task, treatmentId) {
|
|
93
|
+
validateCampaignIdentity(identity);
|
|
94
|
+
campaignAssert(identity.selected.some((s) => s.caseId === task.id && s.projectId === task.benchmarkProject && s.taskDigest === campaignHash(canonicalCampaignJson(task))), "TASK_IDENTITY_MISMATCH");
|
|
95
|
+
campaignExact(value, ["sourceSchema", "evidence", "taskSuccess", "artifacts"]);
|
|
96
|
+
campaignAssert(value.sourceSchema === "my-dev-kit-lab-agent-success-rate-execution-v1");
|
|
97
|
+
bool(value.taskSuccess);
|
|
98
|
+
treatment(value.evidence);
|
|
99
|
+
const e = value.evidence;
|
|
100
|
+
const rows = e.attempts?.map((a) => a.evidence) ?? [e];
|
|
101
|
+
campaignAssert(rows.length > 0 && rows.length <= 1 + identity.config.repairAttempts);
|
|
102
|
+
for (let i = 0; i < rows.length; i++) {
|
|
103
|
+
const a = rows[i];
|
|
104
|
+
campaignAssert(a.treatmentId === treatmentId && a.status === "completed" && a.availability === "complete" && a.cleanup.attempted && a.cleanup.removed && a.protectedIntegrity.status !== "unproven" && a.sandboxBaseline !== null && a.baselineAssessment?.evaluable === true && a.baselineAssessment.reasons.length === 0 && a.patch.attempted && a.patch.outcome !== null && a.proposedPatchPath !== null && a.baselineVerification !== null, "INCOMPLETE_EVIDENCE");
|
|
105
|
+
campaignAssert(a.protectedIntegrity.status !== "intact" || a.protectedIntegrity.mutatedPaths.length === 0);
|
|
106
|
+
campaignAssert(a.protectedIntegrity.status !== "mutated" || a.protectedIntegrity.mutatedPaths.length > 0);
|
|
107
|
+
campaignAssert(a.protectedIntegrity.mutatedPaths.every((p) => task.protectedFiles.includes(p)), "PROTECTED_PROOF_MISMATCH");
|
|
108
|
+
campaignAssert(/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(a.sandboxBaseline.commit) && a.sandboxBaseline.fileCount > 0, "BASELINE_PROOF_MISMATCH");
|
|
109
|
+
if (a.change !== null) {
|
|
110
|
+
const changes = a.change;
|
|
111
|
+
campaignAssert(changes.baselineCommit === a.sandboxBaseline.commit && changes.changedCount === changes.changedFiles.length && new Set(changes.changedFiles.map((f) => f.relativePath)).size === changes.changedFiles.length, "CHANGE_SUMMARY_MISMATCH");
|
|
112
|
+
for (const [status, count] of [["added", changes.addedCount], ["modified", changes.modifiedCount], ["deleted", changes.deletedCount]])
|
|
113
|
+
campaignAssert(count === changes.changedFiles.filter((f) => f.status === status).length, "CHANGE_SUMMARY_MISMATCH");
|
|
114
|
+
campaignAssert(changes.totalAdditions === changes.changedFiles.reduce((sum, f) => sum + (f.additions ?? 0), 0) && changes.totalDeletions === changes.changedFiles.reduce((sum, f) => sum + (f.deletions ?? 0), 0), "CHANGE_SUMMARY_MISMATCH");
|
|
115
|
+
campaignAssert(changes.changedFiles.every((f) => f.status === "added" ? f.beforeSha256 === null && f.afterSha256 !== null : f.status === "deleted" ? f.beforeSha256 !== null && f.afterSha256 === null : f.beforeSha256 !== null && f.afterSha256 !== null), "CHANGE_DIGEST_MISMATCH");
|
|
116
|
+
}
|
|
117
|
+
campaignAssert((a.appliedPatchPath !== null) === (a.patch.outcome === "success"));
|
|
118
|
+
campaignAssert(a.patch.outcome === "success" ? a.change !== null && a.postEditVerification !== null : a.change === null && a.postEditVerification === null);
|
|
119
|
+
for (const [phase, v] of [["baseline", a.baselineVerification], ["post-edit", a.postEditVerification]]) {
|
|
120
|
+
if (v === null)
|
|
121
|
+
continue;
|
|
122
|
+
campaignAssert(v.phase === phase);
|
|
123
|
+
for (const [checkClass, definitions, results] of [["task", task.taskChecks, v.taskResults], ["regression", task.regressionChecks, v.regressionResults]]) {
|
|
124
|
+
campaignAssert(results.length === definitions.length && results.every((r, j) => r.checkId === definitions[j].id && r.class === checkClass && ["passed", "failed"].includes(r.status) && (r.status === "passed" ? r.exitCode === 0 : r.exitCode !== null && r.exitCode !== 0)), "VERIFICATION_MISSING");
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
campaignAssert(a.baselineVerification.taskResults.some((r) => r.status === "failed") && a.baselineVerification.regressionResults.every((r) => r.status === "passed"));
|
|
128
|
+
if (identity.config.providerId === "deterministic-fixture")
|
|
129
|
+
campaignAssert(a.realAgent === undefined && a.agentTokenUsage === null && a.timing.agentDurationMs === null);
|
|
130
|
+
else {
|
|
131
|
+
const r = a.realAgent;
|
|
132
|
+
campaignAssert(r && r.providerId === identity.config.providerId && r.attempt === i + 1 && r.providerInvoked && r.providerStatus === "completed" && r.finalAnswerAvailable && r.cwdCleanup.attempted && r.cwdCleanup.removed && r.context.contextMode === treatmentId && r.context.availability !== "unavailable" && r.agentArtifacts.prompt !== null && r.agentArtifacts.result !== null);
|
|
133
|
+
}
|
|
134
|
+
const record = e.attempts?.[i];
|
|
135
|
+
if (a.realAgent?.repairFeedback !== undefined && a.realAgent.repairFeedback !== null)
|
|
136
|
+
campaignAssert(i > 0 && a.realAgent.repairFeedback.basedOnAttempt === i && a.realAgent.repairFeedback.category === e.attempts[i - 1].failureCategory, "REPAIR_FEEDBACK_MISMATCH");
|
|
137
|
+
if (record) {
|
|
138
|
+
const classification = classifyAgentSuccessAttempt(a, analyzeAgentSuccessTreatment(task, a).metrics);
|
|
139
|
+
campaignAssert(record.failureCategory === classification.category && record.repairEligible === classification.repairEligible && record.taskSuccess === classification.taskSuccess && record.durationMs === a.timing.agentDurationMs && canonicalCampaignJson(record.tokenUsage) === canonicalCampaignJson(a.agentTokenUsage) && record.proposedPatchAvailability === (a.proposedPatchPath === null ? "unavailable" : "available") && record.patchApplicationOutcome === (a.patch.outcome ?? "not-attempted") && canonicalCampaignJson(record.baselineAssessment) === canonicalCampaignJson(a.baselineAssessment) && canonicalCampaignJson(record.postEditVerification) === canonicalCampaignJson(a.postEditVerification) && record.changeEvidenceAvailability === (a.change === null ? "unavailable" : "available"));
|
|
140
|
+
campaignAssert(record.attemptNumber === i + 1 && record.providerId === identity.config.providerId && record.providerStatus === a.realAgent?.providerStatus && record.taskSuccess === (analyzeAgentSuccessTreatment(task, a).metrics.taskSuccess.value === true) && canonicalCampaignJson(record.cleanupResult) === canonicalCampaignJson(a.cleanup) && record.completedAt >= record.startedAt);
|
|
141
|
+
if (i > 0)
|
|
142
|
+
campaignAssert(e.attempts[i - 1].repairEligible && e.attempts[i - 1].taskSuccess === false);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
campaignAssert(new Set(rows.map((r) => r.sandboxId)).size === rows.length);
|
|
146
|
+
if (e.attempts) {
|
|
147
|
+
const { attempts: _attempts, ...final } = e;
|
|
148
|
+
campaignAssert(canonicalCampaignJson(final) === canonicalCampaignJson(rows[rows.length - 1]));
|
|
149
|
+
}
|
|
150
|
+
campaignAssert(value.taskSuccess === (analyzeAgentSuccessTreatment(task, e).metrics.taskSuccess.value === true));
|
|
151
|
+
const expected = requiredCampaignArtifacts(e, task);
|
|
152
|
+
list(shape({ path: rel, kind: one("patch", "context", "provider"), sha256: digest, bytes: integer }), 32)(value.artifacts);
|
|
153
|
+
const artifacts = value.artifacts;
|
|
154
|
+
campaignAssert(canonicalCampaignJson(artifacts.map(({ path, kind }) => ({ path, kind }))) === canonicalCampaignJson(expected), "ARTIFACT_REFERENCE_MISMATCH");
|
|
155
|
+
}
|
|
156
|
+
export function sealCampaignCheckpoint(body) { return { ...body, integrity: campaignHash(canonicalCampaignJson(body)) }; }
|
|
157
|
+
export function validateCampaignCheckpoint(value, tasks) {
|
|
158
|
+
campaignAssert(Buffer.byteLength(canonicalCampaignJson(value)) <= CAMPAIGN_CHECKPOINT_MAX_BYTES, "DATA_BOUND");
|
|
159
|
+
campaignExact(value, ["schemaVersion", "identity", "generation", "parentIntegrity", "createdAt", "updatedAt", "units", "integrity"]);
|
|
160
|
+
campaignAssert(value.schemaVersion === CAMPAIGN_CHECKPOINT_SCHEMA, "UNSUPPORTED_SCHEMA");
|
|
161
|
+
validateCampaignIdentity(value.identity);
|
|
162
|
+
integer(value.generation);
|
|
163
|
+
campaignAssert(Number(value.generation) >= 1 && Number(value.generation) <= 1_000_000);
|
|
164
|
+
nullable(digest)(value.parentIntegrity);
|
|
165
|
+
timestamp(value.createdAt);
|
|
166
|
+
timestamp(value.updatedAt);
|
|
167
|
+
campaignAssert(String(value.updatedAt) >= String(value.createdAt));
|
|
168
|
+
const identity = value.identity;
|
|
169
|
+
campaignAssert(tasks.length === identity.selected.length && tasks.every((t, i) => t.id === identity.selected[i].caseId && t.benchmarkProject === identity.selected[i].projectId && campaignHash(canonicalCampaignJson(t)) === identity.selected[i].taskDigest), "TASK_IDENTITY_MISMATCH");
|
|
170
|
+
campaignAssert(Array.isArray(value.units) && value.units.length === tasks.length * 2);
|
|
171
|
+
for (let i = 0; i < value.units.length; i++) {
|
|
172
|
+
const u = value.units[i];
|
|
173
|
+
campaignExact(u, ["caseId", "treatmentId", "state", "errorCode", "completion"]);
|
|
174
|
+
const task = tasks[Math.floor(i / 2)];
|
|
175
|
+
campaignAssert(u.caseId === task.id && u.treatmentId === value.identity.treatmentOrder[i % 2]);
|
|
176
|
+
one("pending", "in-progress", "completed", "incomplete", "paused-limit")(u.state);
|
|
177
|
+
nullable(one("INTERRUPTED", "INVALID_EVIDENCE", "PROVIDER_LIMIT"))(u.errorCode);
|
|
178
|
+
if (u.state === "completed") {
|
|
179
|
+
campaignAssert(u.errorCode === null);
|
|
180
|
+
validateCampaignCompletion(u.completion, value.identity, task, String(u.treatmentId));
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
campaignAssert(u.completion === null);
|
|
184
|
+
campaignAssert(u.state === "paused-limit" ? u.errorCode === "PROVIDER_LIMIT" : u.state === "incomplete" ? ["INTERRUPTED", "INVALID_EVIDENCE"].includes(String(u.errorCode)) : u.errorCode === null);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const { integrity, ...body } = value;
|
|
188
|
+
campaignAssert(isCampaignDigest(integrity) && integrity === campaignHash(canonicalCampaignJson(body)), "CHECKPOINT_INTEGRITY");
|
|
189
|
+
}
|
|
190
|
+
export function createCampaignCheckpoint(identity, tasks, at) {
|
|
191
|
+
const result = sealCampaignCheckpoint({ schemaVersion: CAMPAIGN_CHECKPOINT_SCHEMA, identity: structuredClone(identity), generation: 1, parentIntegrity: null, createdAt: at, updatedAt: at, units: identity.selected.flatMap((s) => identity.treatmentOrder.map((treatmentId) => ({ caseId: s.caseId, treatmentId, state: "pending", errorCode: null, completion: null }))) });
|
|
192
|
+
validateCampaignCheckpoint(result, tasks);
|
|
193
|
+
return result;
|
|
194
|
+
}
|
|
195
|
+
export function transitionCampaignCheckpoint(previous, tasks, index, next, at, completion = null, classifiedProviderOutcome = null) {
|
|
196
|
+
validateCampaignCheckpoint(previous, tasks);
|
|
197
|
+
campaignAssert(Number.isSafeInteger(index) && index >= 0 && index < previous.units.length);
|
|
198
|
+
campaignAssert(next !== "paused-limit" || (classifiedProviderOutcome === "agent-limit-reached" && previous.identity.config.providerId !== "deterministic-fixture"), "LIMIT_CLASSIFICATION_REQUIRED");
|
|
199
|
+
const state = previous.units[index].state;
|
|
200
|
+
campaignAssert(state === "pending" ? next === "in-progress" : state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(next), "INVALID_TRANSITION");
|
|
201
|
+
const { integrity, ...body } = structuredClone(previous);
|
|
202
|
+
body.generation++;
|
|
203
|
+
body.parentIntegrity = integrity;
|
|
204
|
+
body.updatedAt = at;
|
|
205
|
+
body.units[index] = { ...body.units[index], state: next, errorCode: next === "incomplete" ? "INTERRUPTED" : next === "paused-limit" ? "PROVIDER_LIMIT" : null, completion };
|
|
206
|
+
const result = sealCampaignCheckpoint(body);
|
|
207
|
+
validateCampaignCheckpoint(result, tasks);
|
|
208
|
+
return result;
|
|
209
|
+
}
|
|
210
|
+
export function validateCampaignHistory(previous, next) {
|
|
211
|
+
campaignAssert(previous.identity.digest === next.identity.digest && next.generation === previous.generation + 1 && next.parentIntegrity === previous.integrity && next.createdAt === previous.createdAt && next.updatedAt >= previous.updatedAt, "INVALID_HISTORY");
|
|
212
|
+
let changes = 0;
|
|
213
|
+
previous.units.forEach((u, i) => { const v = next.units[i]; if (canonicalCampaignJson(u) === canonicalCampaignJson(v))
|
|
214
|
+
return; changes++; campaignAssert(u.caseId === v.caseId && u.treatmentId === v.treatmentId && (u.state === "pending" ? v.state === "in-progress" : u.state === "in-progress" && ["completed", "incomplete", "paused-limit"].includes(v.state)), "INVALID_HISTORY"); });
|
|
215
|
+
campaignAssert(changes === 1, "INVALID_HISTORY");
|
|
216
|
+
}
|