@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1

package/dist/src/commands/runExperimentRunCommand.js

@@ -1,12 +1,14 @@
-import { readFile } from "node:fs/promises";
+import { lstat, mkdir, readFile, readdir, realpath } from "node:fs/promises";
 import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
+import { AGENT_SUCCESS_PROJECT_PROFILES_PATH, AGENT_SUCCESS_TASK_CATALOG_PATH, assertReadAgentSuccessCorpus } from "../evaluation/agentSuccess/index.js";
 import { summarizeRetrievalGroundTruthIssuesSafely, validateRetrievalPrecisionRecallCorpus } from "../evaluation/retrievalQuality/index.js";
 import { loadLocalRepositorySubject } from "../evaluation/localRepositorySubject/index.js";
 import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, retrievalPrecisionRecallPlugin, retrievalQueryStrategyComparisonPlugin, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
 import { contextPackGenerationPlugin } from "../experiments/plugins/contextPackGeneration/index.js";
+import { AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, agentSuccessRatePlugin, selectAgentSuccessTasks } from "../experiments/plugins/agentSuccessRate/index.js";
 import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
 import { projectExternalLocalTarget, projectRunForExternalLocalPersistence, } from "../experiments/plugins/contextWindowScaling/localSubjectPrivacy.js";
 import { assertWorkRootOutsideTarget } from "../experiments/plugins/contextWindowScaling/localSubjectScratch.js";
@@ -15,6 +17,18 @@ import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/inde
 import { writePluginExperimentReports } from "../report/index.js";
 import { createLabExecutionContext, resolvePackageResource } from "../runtime/index.js";
 import { runWarmIndexCampaignPresentation } from "./runWarmIndexCampaignPresentation.js";
+import { observeProviderBuild } from "../agents/providerBuildObservation.js";
+import { resolveAgentTimeoutPreset } from "../experiments/plugins/agentSuccessRate/timeoutPresets.js";
+import { validateAgentSuccessRateConfig } from "../experiments/plugins/agentSuccessRate/config.js";
+import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS } from "../experiments/plugins/agentSuccessRate/realAgentExecution.js";
+import { AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/metadata.js";
+import { executeAgentSuccessCampaignV2, prepareAgentSuccessCampaign } from "../experiments/plugins/agentSuccessRate/campaignExecution.js";
+import { readCampaignCheckpointV2, validateCampaignOutputPath } from "../experiments/plugins/agentSuccessRate/campaignCheckpointStore.js";
+import { validateCampaignLimitAuthorization } from "../experiments/plugins/agentSuccessRate/campaignRecovery.js";
+import { campaignIdentifier } from "../experiments/plugins/agentSuccessRate/campaignIdentity.js";
+import { readCampaignProgress } from "../experiments/plugins/agentSuccessRate/campaignProgress.js";
+import { writeAgentSuccessCampaignProgressReport } from "../report/experiments/writeAgentSuccessCampaignProgressReport.js";
+import { boundMessage } from "../experiments/plugins/agentSuccessRate/execution.js";
 // ---------------------------------------------------------------------------
 // v0.4.6 Batch 4 -- reusable experiment-run command owner.
 //
@@ -71,9 +85,36 @@ const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_ALLOWED_FLAGS = ["--experimen
 const CONTEXT_PACK_GENERATION_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
 const CONTEXT_PACK_GENERATION_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
 const CONTEXT_PACK_GENERATION_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
+// agent-success-rate (v0.9.0) owns the bundled six-task implementation corpus (task catalog and project profiles), both
+// resolved from the package root, never the cwd. It runs against the Lab itself only and accepts exactly these options;
+// every other flag is rejected, not ignored. Real providers run only with --agent together with --include-real-agents.
+const AGENT_SUCCESS_RATE_ALLOWED_FLAGS = [
+    "--experiment",
+    "--out",
+    "--case",
+    "--benchmark-project",
+    "--kit-command",
+    "--agent",
+    "--include-real-agents",
+    "--timeout-ms",
+    "--repair-attempts",
+    "--agent-campaign", "--one-case", "--campaign-source-root", "--timeout-preset",
+    "--expected-checkpoint-revision", "--limit-case", "--limit-treatment", "--limit-epoch", "--authorize-additional-invocation"
+];
+const AGENT_CAMPAIGN_FLAGS = ["--agent-campaign", "--one-case", "--campaign-source-root", "--expected-checkpoint-revision", "--limit-case", "--limit-treatment", "--limit-epoch", "--authorize-additional-invocation"];
+export function assertAgentSuccessRateFlags(argv) {
+    const flags = argv.filter((arg) => arg.startsWith("--"));
+    const unsupported = [...new Set(flags.filter((flag) => !AGENT_SUCCESS_RATE_ALLOWED_FLAGS.includes(flag)))];
+    if (unsupported.length > 0) {
+        throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${agentSuccessRatePlugin.metadata.id}; supported options: ${AGENT_SUCCESS_RATE_ALLOWED_FLAGS.join(", ")}.`);
+    }
+}
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
+    let parsed = false;
+    const campaignUsage = argv.includes("--agent-campaign") || argv.some(a => AGENT_CAMPAIGN_FLAGS.includes(a));
     try {
         const args = parseRunExperimentArgs(argv);
+        parsed = true;
         const context = options.context ?? createLabExecutionContext();
         const toolRoot = context.packageRoot;
         const registry = createDefaultExperimentPluginRegistry();
@@ -96,6 +137,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
             });
         }
         const inputs = await loadPluginInputs(args, toolRoot, context, outputRoot);
+        if (args.agentCampaign)
+            return await runAgentCampaignCommand(args, context, outputRoot, inputs, runId, options);
         const result = await runExperiment({
             pluginId: args.experimentId,
             registry,
@@ -151,7 +194,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
                 `Target root: ${result.target.targetRoot}`,
                 `Output: ${String(result.metadata?.outputRoot ?? "")}`,
                 `Report JSON: ${reports.outputPaths.jsonPath}`,
-                `Report HTML: ${reports.outputPaths.htmlPath}`
+                `Report HTML: ${reports.outputPaths.htmlPath}`,
+                ...(args.experimentId === agentSuccessRatePlugin.metadata.id ? agentSuccessRateSummaryLines(result, reports.outputPaths.textPath) : [])
             ];
         // Automatic presentation (report already produced above) applies only to warm-index-reuse
         // campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
@@ -195,16 +239,125 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
         return 0;
     }
     catch (error) {
-        if (process.env.DEBUG) {
+        if (campaignUsage) {
+            const sourceIndex = argv.indexOf("--campaign-source-root");
+            const source = sourceIndex >= 0 ? argv[sourceIndex + 1] : undefined;
+            const message = error instanceof Error ? error.message : "CAMPAIGN_FAILED";
+            console.error(boundMessage(message, [process.cwd(), options.context?.packageRoot ?? "", ...(source ? [path.resolve(options.context?.invocationCwd ?? process.cwd(), source)] : [])]));
+        }
+        else if (process.env.DEBUG) {
             console.error(error);
         }
         else {
             console.error(error instanceof Error ? error.message : String(error));
         }
-        return 1;
+        return campaignUsage && !parsed ? 2 : 1;
+    }
+}
+async function runAgentCampaignCommand(args, context, outputRoot, inputs, runId, options) {
+    const request = args.agentCampaign;
+    const sourceRoot = path.resolve(context.invocationCwd, request.sourceRoot ?? context.packageRoot);
+    try {
+        const packageName = "@dailephd/my-dev-kit-lab";
+        const validPackageVersion = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/;
+        const executingMetadata = JSON.parse(await readFile(path.join(context.packageRoot, "package.json"), "utf8"));
+        if (executingMetadata.name !== packageName || typeof executingMetadata.version !== "string" || !validPackageVersion.test(executingMetadata.version))
+            throw new Error();
+        if (!(await lstat(path.join(sourceRoot, "src"))).isDirectory() || path.resolve(await realpath(sourceRoot)).toLowerCase() !== sourceRoot.toLowerCase())
+            throw new Error();
+        const metadata = JSON.parse(await readFile(path.join(sourceRoot, "package.json"), "utf8"));
+        if (metadata.name !== packageName || metadata.version !== executingMetadata.version)
+            throw new Error();
+    }
+    catch {
+        throw new Error("CAMPAIGN_SOURCE_UNVERIFIABLE: supply --campaign-source-root with a current my-dev-kit-lab source checkout; installed source bytes cannot be guessed.");
+    }
+    const boundary = { toolRoot: context.packageRoot, outputRoot };
+    await validateCampaignOutputPath(boundary, sourceRoot);
+    let entries = [];
+    try {
+        entries = await readdir(outputRoot);
+    }
+    catch (error) {
+        if (error.code !== "ENOENT")
+            throw error;
+        if (request.mode !== "new")
+            throw new Error("RESUMABLE_CAMPAIGN_REQUIRED: --out must contain an existing version 2 campaign.");
+    }
+    if (request.mode === "new" && entries.length > 0)
+        throw new Error("CAMPAIGN_OUTPUT_EXISTS: new requires an empty output root; existing artifacts are preserved.");
+    if (request.mode !== "new" && !entries.includes("campaign-checkpoints-v2"))
+        throw new Error("RESUMABLE_CAMPAIGN_REQUIRED: no version 2 checkpoint in --out.");
+    const validation = validateAgentSuccessRateConfig(args.config);
+    if (!validation.valid || !validation.config)
+        throw new Error("INVALID_CAMPAIGN_CONFIG");
+    const config = validation.config;
+    const tasks = selectAgentSuccessTasks(inputs.agentSuccessTasks, config);
+    const runtime = options.campaignRuntime;
+    const provider = config.agentId ? { kind: "real-agent", providerId: config.agentId, includeRealAgents: true, timeoutMs: config.timeoutMs ?? AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, repairAttempts: (config.repairAttempts ?? 0), kitCommand: config.kitCommand, contextDependencies: runtime?.contextDependencies, env: runtime?.env } : { kind: "deterministic-fixture" };
+    const preparation = { boundary, sourceRoot, corpusFile: resolvePackageResource(context, AGENT_SUCCESS_TASK_CATALOG_PATH), corpusRoot: context.packageRoot, tasks, treatments: AGENT_SUCCESS_RATE_TREATMENT_IDS, provider, dependencies: runtime?.dependencies };
+    // Source, corpus and canonical project byte checks precede even a task-free version probe.
+    let prepared = await prepareAgentSuccessCampaign(preparation);
+    if (provider.kind === "real-agent") {
+        provider.trustedExecutable = await observeProviderBuild(provider.providerId, { env: runtime?.env });
+        prepared = await prepareAgentSuccessCampaign(preparation);
+    }
+    let intent = request.mode === "new" ? { kind: "new-resumable" } : { kind: "resume" };
+    let reused = 0;
+    const previousEpochs = new Set();
+    if (request.mode !== "new") {
+        // Identity-scoped stores are not discovery: a different digest cannot create another campaign on resume.
+        const identities = await readdir(path.join(outputRoot, "campaign-checkpoints-v2"));
+        if (!identities.includes(prepared.identity.digest))
+            throw new Error("CAMPAIGN_IDENTITY_MISMATCH: current inputs do not match this output root.");
+        const stored = await readCampaignCheckpointV2(boundary, prepared.identity, tasks);
+        if (!stored.reusable || !stored.checkpoint || stored.identityVerification !== "MATCH")
+            throw new Error("RESUMABLE_CAMPAIGN_REQUIRED_MATCH");
+        const cp = stored.checkpoint;
+        cp.units.flatMap(u => u.epochs).forEach(e => previousEpochs.add(e.id));
+        reused = cp.units.filter(u => u.state === "completed").length;
+        if (request.mode === "continue-limit") {
+            const unit = cp.units.find(u => u.caseId === request.limitCase && u.treatmentId === request.limitTreatment);
+            if (cp.generation !== request.expectedRevision || !unit || unit.epochs.at(-1)?.id !== request.limitEpoch)
+                throw new Error("LIMIT_AUTHORIZATION_MISMATCH");
+            const authorization = { identityDigest: cp.identity.digest, generation: request.expectedRevision, revision: cp.integrity, caseId: request.limitCase, treatmentId: request.limitTreatment, acknowledgeInvocation: true, acknowledgeExternalLimit: true };
+            validateCampaignLimitAuthorization(authorization, cp, unit);
+            intent = { kind: "authorize-limit-continuation", authorization };
+        }
+    }
+    else {
+        await mkdir(outputRoot, { recursive: true });
+        await validateCampaignOutputPath(boundary, sourceRoot);
+        if ((await readdir(outputRoot)).length !== 0)
+            throw new Error("CAMPAIGN_OUTPUT_EXISTS");
+    }
+    const result = await executeAgentSuccessCampaignV2(prepared, { intent, runId, limit: request.oneCase ? "one-case" : "all-selected-cases", hooks: runtime?.hooks });
+    const progressLines = [];
+    let progressFailed = false;
+    let progressValidated = false;
+    try {
+        const snapshot = await readCampaignProgress(boundary, prepared.identity, tasks);
+        progressValidated = true;
+        const progress = await writeAgentSuccessCampaignProgressReport({ snapshot, boundary });
+        progressLines.push(`Progress status: ${progress.model.state}`, `Progress checkpoint revision: ${progress.model.checkpointGeneration} (${progress.model.checkpointRevision})`, `Completed evaluations: ${progress.model.unitCounts.completed}/${progress.model.totalTreatmentUnits}`, `Completed cases: ${progress.model.caseCounts.completed}/${progress.model.selectedCaseCount}`, `Progress JSON: ${progress.outputPaths.jsonPath}`, `Progress Text: ${progress.outputPaths.textPath}`, `Progress HTML: ${progress.outputPaths.htmlPath}`);
+    }
+    catch {
+        progressFailed = true;
+        progressLines.push(progressValidated ? "Progress persistence: failed (validated checkpoint retained)." : "Progress status: blocked-integrity (checkpoint unavailable or untrusted).");
+        progressLines.push("Progress publication: failed; checkpoint/scientific execution outcome is reported separately.");
     }
+    const newEpochs = result.checkpoint?.units.flatMap(u => u.epochs).filter(e => !previousEpochs.has(e.id)) ?? [];
+    // Provider evidence, never mere creation of a reservation, establishes invocation.
+    const newWork = newEpochs.some(e => e.terminal?.evidence.realAgent?.providerInvoked === true) ? "invoked" : newEpochs.some(e => e.phase === "invocation-authorized" || e.phase === "uncertain") ? "unconfirmed" : "none";
+    console.log(["Experiment: agent-success-rate", `Campaign mode: ${request.mode}`, `Identity: ${prepared.identity.digest.slice(0, 16)}`, `Status: ${result.status === "provider-limit" ? "paused-provider-limit" : result.status}`, `New provider work: ${newWork}`, `Validated work reused: ${reused > 0 ? "yes" : "no"}`, `Checkpoint revision: ${result.checkpoint?.generation ?? "unavailable"}`, `Final scientific artifacts: ${result.scientificArtifactsWritten ? "written" : "not written"}`, `Output: ${outputRoot}`, ...result.errors.map(e => `Failure: ${e}`)].join("\n"));
+    console.log(progressLines.join("\n"));
+    return !progressFailed && ["completed", "already-complete", "partial"].includes(result.status) ? 0 : 1;
 }
 export function parseRunExperimentArgs(argv) {
+    let agentCampaignMode;
+    let oneCase = false, authorize = false;
+    let campaignSourceRoot, limitCase, limitTreatment, limitEpoch, timeoutPreset;
+    let expectedRevision;
     let experimentId = "";
     let targetPath;
     let outDir;
@@ -225,13 +378,51 @@ export function parseRunExperimentArgs(argv) {
     let contextBudgets;
     let syntheticConfigPath;
     let localSubjectConfigPath;
+    const agentFlagValues = [];
+    let repairAttempts;
     const seenFlags = [];
     const commandTemplates = {};
+    // The flag whitelist runs before any value is interpreted so an unsupported flag is reported as such.
+    const preScanIndex = argv.indexOf("--experiment");
+    if (preScanIndex >= 0 && argv[preScanIndex + 1] === agentSuccessRatePlugin.metadata.id)
+        assertAgentSuccessRateFlags(argv);
     for (let index = 0; index < argv.length; index += 1) {
         const arg = argv[index];
+        if (arg !== undefined && arg.startsWith("--") && seenFlags.includes(arg) && (AGENT_CAMPAIGN_FLAGS.includes(arg) || arg === "--timeout-preset" || (argv.includes("agent-success-rate") && arg !== "--case" && arg !== "--benchmark-project")))
+            throw new Error(`${arg} may be supplied only once.`);
         if (arg !== undefined && arg.startsWith("--"))
             seenFlags.push(arg);
-        if (arg === "--experiment") {
+        if (arg === "--agent-campaign") {
+            agentCampaignMode = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--one-case") {
+            oneCase = true;
+        }
+        else if (arg === "--authorize-additional-invocation") {
+            authorize = true;
+        }
+        else if (arg === "--campaign-source-root") {
+            campaignSourceRoot = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--expected-checkpoint-revision") {
+            const value = readRequiredValue(argv, ++index, arg);
+            if (!/^[1-9]\d*$/.test(value))
+                throw new Error("--expected-checkpoint-revision must be a positive integer.");
+            expectedRevision = parsePositiveInteger(arg, value);
+        }
+        else if (arg === "--limit-case") {
+            limitCase = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--limit-treatment") {
+            limitTreatment = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--limit-epoch") {
+            limitEpoch = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--timeout-preset") {
+            timeoutPreset = readRequiredValue(argv, ++index, arg);
+        }
+        else if (arg === "--experiment") {
             experimentId = readRequiredValue(argv, ++index, "--experiment");
         }
         else if (arg === "--target") {
@@ -264,6 +455,12 @@ export function parseRunExperimentArgs(argv) {
         else if (arg === "--benchmark-project") {
             benchmarkProjects.push(...splitList(readRequiredValue(argv, ++index, "--benchmark-project")));
         }
+        else if (arg === "--agent") {
+            agentFlagValues.push(readRequiredValue(argv, ++index, "--agent"));
+        }
+        else if (arg === "--repair-attempts") {
+            repairAttempts = parseRepairAttempts(readRequiredValue(argv, ++index, "--repair-attempts"));
+        }
         else if (arg === "--agents") {
             agents = splitList(readRequiredValue(argv, ++index, "--agents")).map((value) => parseAgentId(value));
         }
@@ -317,6 +514,31 @@ export function parseRunExperimentArgs(argv) {
     if (!experimentId) {
         throw new Error("Usage: --experiment <id> [--target <path>] [--out <directory>]");
     }
+    const campaignFlagsPresent = seenFlags.some(f => AGENT_CAMPAIGN_FLAGS.includes(f));
+    if ((campaignFlagsPresent || timeoutPreset !== undefined) && experimentId !== agentSuccessRatePlugin.metadata.id)
+        throw new Error("Agent campaign and timeout preset flags are only supported for --experiment agent-success-rate.");
+    if (campaignFlagsPresent && agentCampaignMode === undefined)
+        throw new Error("Campaign flags require --agent-campaign <new|resume|continue-limit>.");
+    if (agentCampaignMode !== undefined && !["new", "resume", "continue-limit"].includes(agentCampaignMode))
+        throw new Error("--agent-campaign must be new, resume or continue-limit.");
+    if (agentCampaignMode !== undefined && !outDir?.trim())
+        throw new Error("--agent-campaign requires a stable --out directory.");
+    const limitPresent = expectedRevision !== undefined || limitCase !== undefined || limitTreatment !== undefined || limitEpoch !== undefined || authorize;
+    if (agentCampaignMode !== "continue-limit" && limitPresent)
+        throw new Error("Limit authorization flags require --agent-campaign continue-limit.");
+    if (agentCampaignMode === "continue-limit") {
+        if (expectedRevision === undefined || !limitCase || !limitTreatment || !limitEpoch || !authorize)
+            throw new Error("continue-limit requires --expected-checkpoint-revision, --limit-case, --limit-treatment, --limit-epoch and --authorize-additional-invocation.");
+        if (expectedRevision > 1000000 || !campaignIdentifier(limitCase) || !["raw-full-file", "context-pack"].includes(limitTreatment) || !/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/.test(limitEpoch))
+            throw new Error("Invalid limit revision, case, treatment or epoch identifier.");
+    }
+    if (timeoutPreset !== undefined) {
+        if (timeoutMs !== undefined)
+            throw new Error("--timeout-preset and --timeout-ms cannot be combined.");
+        if (agentFlagValues.length !== 1 || includeRealAgents !== true || !["codex", "claude"].includes(agentFlagValues[0]))
+            throw new Error("--timeout-preset requires --agent <codex|claude> and --include-real-agents.");
+        timeoutMs = resolveAgentTimeoutPreset(agentFlagValues[0], timeoutPreset);
+    }
     const KIT_COMMAND_PLUGIN_IDS = [
         warmIndexReusePlugin.metadata.id,
         incrementalChangeStalenessPlugin.metadata.id,
@@ -399,6 +621,49 @@ export function parseRunExperimentArgs(argv) {
             throw new Error("--benchmark-project must list at least one benchmark project id.");
         }
     }
+    if (experimentId === agentSuccessRatePlugin.metadata.id) {
+        // Mode matrix: no flags = deterministic-fixture (default); --agent together with --include-real-agents = real-agent mode.
+        // Everything else is rejected, never coerced: a provider needs both flags, and real-agent options need a provider.
+        if (agentFlagValues.length > 1) {
+            throw new Error("--agent may be supplied only once; agent-success-rate runs exactly one provider.");
+        }
+        const agentId = agentFlagValues[0];
+        if (agentId !== undefined && !AGENT_SUCCESS_RATE_REAL_AGENT_IDS.includes(agentId)) {
+            throw new Error(`--agent must be exactly one of: ${AGENT_SUCCESS_RATE_REAL_AGENT_IDS.join(", ")}.`);
+        }
+        if (agentId !== undefined && includeRealAgents !== true) {
+            throw new Error("--agent requires --include-real-agents; real providers are invoked only with explicit authorization.");
+        }
+        if (agentId === undefined && includeRealAgents === true) {
+            throw new Error("--include-real-agents requires --agent <codex|claude> for agent-success-rate.");
+        }
+        if (agentId === undefined && timeoutMs !== undefined) {
+            throw new Error("--timeout-ms is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && kitCommand !== undefined) {
+            throw new Error("--kit-command is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && repairAttempts !== undefined && repairAttempts > 0) {
+            throw new Error("--repair-attempts greater than 0 requires --agent and --include-real-agents (real-agent mode).");
+        }
+        if (timeoutMs !== undefined && timeoutMs > AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS) {
+            throw new Error(`--timeout-ms must be a positive integer no greater than ${AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
+    else {
+        if (agentFlagValues.length > 0) {
+            throw new Error(`--agent is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+        if (repairAttempts !== undefined) {
+            throw new Error(`--repair-attempts is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+    }
     if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
@@ -441,7 +706,8 @@ export function parseRunExperimentArgs(argv) {
             throw new Error(`External ${experimentId} targets require --local-subject-config.`);
         }
     }
-    if (kitCommand !== undefined && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
+    // agent-success-rate accepts --kit-command in real-agent mode; its own validation above owns that rule and message.
+    if (kitCommand !== undefined && experimentId !== agentSuccessRatePlugin.metadata.id && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
         throw new Error(`--kit-command is only supported for --experiment ${KIT_COMMAND_PLUGIN_IDS.join(" or ")}.`);
     }
     if (campaignPreset !== undefined) {
@@ -460,6 +726,7 @@ export function parseRunExperimentArgs(argv) {
     }
     return {
         experimentId,
+        ...(agentCampaignMode ? { agentCampaign: { mode: agentCampaignMode, oneCase, sourceRoot: campaignSourceRoot, expectedRevision, limitCase, limitTreatment: limitTreatment, limitEpoch, authorize } } : {}),
         targetPath,
         outDir,
         syntheticConfigPath,
@@ -478,6 +745,8 @@ export function parseRunExperimentArgs(argv) {
             requireAgents,
             includeRealAgents,
             commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
+            agentId: agentFlagValues[0],
+            repairAttempts,
             kitCommand,
             campaignPreset,
             contextBudgets
@@ -542,8 +811,27 @@ async function loadPluginInputs(args, toolRoot, context, outputRoot) {
             ? loadRetrievalQueryStrategyComparisonLocalSubjectInputs(args, toolRoot, context, outputRoot)
             : loadRetrievalQueryStrategyComparisonInputs(args, toolRoot, context);
     }
+    if (args.experimentId === agentSuccessRatePlugin.metadata.id) {
+        return loadAgentSuccessRateInputs(args, context);
+    }
     return undefined;
 }
+// agent-success-rate: the bundled task catalog and project profiles are located through the established package-resource
+// resolver (never the cwd) and validated by the corpus reader, which also checks the canonical projects on disk. The
+// validated tasks reach the plugin through its existing programmatic input; the plugin applies --case/--benchmark-project
+// selection in catalog order and owns execution, repair, science and artifacts. No task file is accepted from the CLI.
+async function loadAgentSuccessRateInputs(args, context) {
+    const validation = agentSuccessRatePlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid agent success rate config: ${validation.errors.join("; ")}`);
+    }
+    resolvePackageResource(context, AGENT_SUCCESS_TASK_CATALOG_PATH);
+    resolvePackageResource(context, AGENT_SUCCESS_PROJECT_PROFILES_PATH);
+    const corpus = assertReadAgentSuccessCorpus(context.resourceRoot);
+    // Fail on unknown case or project ids before any output directory exists.
+    selectAgentSuccessTasks(corpus.tasks, validation.config);
+    return { agentSuccessTasks: corpus.tasks };
+}
 // context-pack-generation (bundled/self): the frozen warm-index corpus through the established resource resolver, profile
 // and corpus validation. The plugin applies --case/--benchmark-project selection and owns execution, science and artifacts.
 async function loadContextPackGenerationInputs(args, toolRoot, context) {
@@ -815,6 +1103,12 @@ function readRequiredValue(argv, index, label) {
     }
     return value;
 }
+function parseRepairAttempts(value) {
+    if (!/^[0-2]$/.test(value)) {
+        throw new Error("--repair-attempts must be 0, 1 or 2.");
+    }
+    return Number(value);
+}
 function parsePositiveInteger(label, value) {
     const parsed = Number(value);
     if (!Number.isInteger(parsed) || parsed <= 0) {
@@ -822,6 +1116,15 @@ function parsePositiveInteger(label, value) {
     }
     return parsed;
 }
+function agentSuccessRateSummaryLines(result, textPath) {
+    const mode = readMetadataString(result.metadata?.executionMode) ?? "deterministic-fixture";
+    const provider = readMetadataString(result.metadata?.providerId);
+    return [
+        `Report Text: ${textPath}`,
+        `Execution mode: ${mode}`,
+        ...(provider ? [`Provider: ${provider} (invoked by explicit --include-real-agents authorization)`] : ["Provider: none (deterministic-fixture; no coding agent was invoked)"])
+    ];
+}
 function readMetadataString(value) {
     return typeof value === "string" && value.trim() ? value : undefined;
 }

package/dist/src/core/processTree.js

@@ -20,6 +20,61 @@ export function requestProcessTermination(pid) {
         // The process may have exited between the liveness check and this signal.
     }
 }
+/**
+ * Opt-in, awaitable descendant-tree termination for bounded execution
+ * (see `terminateProcessTree` in `runMeasuredCommand`). The legacy
+ * `forceTerminateProcess` above is intentionally unchanged: it stays
+ * fire-and-forget and, off Windows, signals one pid only.
+ *
+ * Windows: `taskkill /pid <pid> /T /F` (shell:false) is awaited, bounded by
+ * `waitTimeoutMs`, so the caller knows the tree was asked to die before it
+ * proceeds.
+ *
+ * Other platforms: the child must have been spawned `detached` so it leads its
+ * own process group; the whole group is signalled via the negative pid. If no
+ * group exists, the single pid is signalled as a fallback.
+ *
+ * Never throws: the target may already have exited.
+ */
+export async function terminateProcessTree(pid, options = {}) {
+    if (!pid) {
+        return;
+    }
+    const platform = options.platform ?? process.platform;
+    if (platform === "win32") {
+        await new Promise((resolve) => {
+            let timer;
+            const finish = () => {
+                if (timer)
+                    clearTimeout(timer);
+                resolve();
+            };
+            const killer = spawn("taskkill", ["/pid", String(pid), "/T", "/F"], {
+                shell: false,
+                stdio: "ignore",
+                windowsHide: true
+            });
+            timer = setTimeout(finish, options.waitTimeoutMs ?? 5000);
+            killer.on("error", finish);
+            killer.on("close", finish);
+        });
+        return;
+    }
+    const signal = options.posixSignal ?? "SIGKILL";
+    try {
+        process.kill(-pid, signal);
+        return;
+    }
+    catch {
+        // No process group for this pid (or it already exited); fall back to the single pid.
+    }
+    try {
+        process.kill(pid, signal);
+    }
+    catch {
+        // The process may have exited between the liveness check and this signal.
+    }
+}
 /**
  * Forced cleanup for a process that did not stop on request.
  *