@dailephd/my-dev-kit-lab 0.8.2 → 0.9.0

package/benchmarks/projects/agent-success-task-board-node/tests/summary-edge.check.mjs

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_PROJECT, createTaskBoard } from "../src/index.js";
+
+test("a project's completed count never exceeds its total", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "Solo", projectId: "alpha" });
+  for (const title of ["x", "y", "z"]) {
+    const task = service.createTask({ title, projectId: "beta" });
+    service.completeTask(task.id);
+  }
+  const summary = summarize("alpha");
+  assert.equal(summary.total, 1);
+  assert.equal(summary.completed, 0);
+  assert.ok(summary.completed <= summary.total);
+  assert.equal(summary.open, 1);
+});
+
+test("unrelated tasks in the store do not change a project's summary", () => {
+  const { service, summarize } = createTaskBoard();
+  const a = service.createTask({ title: "A", projectId: "core" });
+  service.createTask({ title: "B", projectId: "core" });
+  service.completeTask(a.id);
+  const before = summarize("core");
+  const noise = service.createTask({ title: "Noise", projectId: "core-extras" });
+  service.createTask({ title: "More noise", projectId: "CORE2" });
+  service.completeTask(noise.id);
+  service.importTasks([{ externalId: "n-1", title: "Imported noise", projectId: "c" }]);
+  assert.deepEqual(summarize("core"), before);
+});
+
+test("an unknown project has an empty, consistent summary", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "A", projectId: "known" });
+  assert.deepEqual(summarize("missing"), { projectId: "missing", total: 0, completed: 0, open: 0, taskIds: [] });
+});
+
+test("blank project identities are rejected consistently", () => {
+  const { service, store, summarize } = createTaskBoard();
+  assert.throws(() => service.createTask({ title: "Task", projectId: "   " }), { code: INVALID_PROJECT });
+  assert.throws(() => service.createTask({ title: "Task", projectId: "" }), { code: INVALID_PROJECT });
+  assert.throws(
+    () => service.importTasks([{ externalId: "ext-1", title: "Task", projectId: " " }]),
+    { code: INVALID_PROJECT }
+  );
+  assert.throws(() => summarize("  "), { code: INVALID_PROJECT });
+  assert.equal(store.list().length, 0);
+});
+
+test("summaries are deterministic and list ids in creation order", () => {
+  const build = () => {
+    const { service, summarize } = createTaskBoard();
+    service.createTask({ title: "1", projectId: "p" });
+    service.createTask({ title: "2", projectId: "q" });
+    service.createTask({ title: "3", projectId: "P" });
+    service.importTasks([{ externalId: "e-1", title: "4", projectId: "p " }]);
+    return summarize("p");
+  };
+  assert.deepEqual(build(), build());
+  assert.deepEqual(build().taskIds, ["task-1", "task-3", "task-4"]);
+});
+
+test("the default project is used for tasks created without one", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "No project given" });
+  assert.equal(summarize("inbox").total, 1);
+});

package/benchmarks/projects/agent-success-task-board-node/tests/summary-primary.check.mjs

@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createTaskBoard } from "../src/index.js";
+
+test("tasks created with differently written project names share one project identity", () => {
+  const { service, store, summarize } = createTaskBoard();
+  service.createTask({ title: "One", projectId: "web" });
+  service.createTask({ title: "Two", projectId: "  WEB " });
+  service.createTask({ title: "Three", projectId: "Web" });
+  assert.deepEqual(
+    store.list().map((task) => task.projectId),
+    ["web", "web", "web"]
+  );
+  const summary = summarize("web");
+  assert.equal(summary.projectId, "web");
+  assert.equal(summary.total, 3);
+});
+
+test("a summary only counts tasks of the requested project", () => {
+  const { service, summarize } = createTaskBoard();
+  const web = service.createTask({ title: "Web task", projectId: "web" });
+  service.createTask({ title: "Web app task", projectId: "web-app" });
+  service.createTask({ title: "Webhook task", projectId: "webhooks" });
+  const other = service.createTask({ title: "Docs task", projectId: "docs" });
+  service.completeTask(other.id);
+  const summary = summarize("web");
+  assert.equal(summary.total, 1);
+  assert.equal(summary.completed, 0);
+  assert.equal(summary.open, 1);
+  assert.deepEqual(summary.taskIds, [web.id]);
+});
+
+test("completed and open counts are correct for the requested project", () => {
+  const { service, summarize } = createTaskBoard();
+  const a = service.createTask({ title: "A", projectId: "ops" });
+  service.createTask({ title: "B", projectId: "ops" });
+  service.createTask({ title: "C", projectId: "ops" });
+  service.createTask({ title: "Elsewhere", projectId: "labs" });
+  service.completeTask(a.id);
+  assert.deepEqual(summarize(" OPS "), {
+    projectId: "ops",
+    total: 3,
+    completed: 1,
+    open: 2,
+    taskIds: ["task-1", "task-2", "task-3"]
+  });
+});
+
+test("imported tasks join the same normalized project identity", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "Manual", projectId: "ops" });
+  service.importTasks([{ externalId: "ext-1", title: "Imported", projectId: " OPS" }]);
+  assert.equal(summarize("ops").total, 2);
+});

package/benchmarks/projects/agent-success-task-board-node/tests/title-edge.check.mjs

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_TITLE, ValidationError, createTaskBoard, validateTitle } from "../src/index.js";
+
+test("rejected titles leave the store and id sequence untouched", () => {
+  const { service, store } = createTaskBoard();
+  assert.throws(() => service.createTask({ title: "  \t " }), { code: INVALID_TITLE });
+  assert.throws(() => service.createTask({ title: "" }), { code: INVALID_TITLE });
+  assert.equal(store.list().length, 0);
+  const task = service.createTask({ title: "First" });
+  assert.equal(task.id, "task-1");
+  assert.equal(store.list().length, 1);
+});
+
+test("interior whitespace is preserved while surrounding whitespace is removed", () => {
+  const { service } = createTaskBoard();
+  assert.equal(service.createTask({ title: "  Fix  the   bug " }).title, "Fix  the   bug");
+  assert.equal(service.createTask({ title: "\n\tPlan sprint\t\n" }).title, "Plan sprint");
+});
+
+test("already-trimmed titles keep their meaning", () => {
+  const { service } = createTaskBoard();
+  assert.equal(service.createTask({ title: "Ship it" }).title, "Ship it");
+  assert.equal(service.createTask({ title: "a" }).title, "a");
+});
+
+test("validateTitle returns the normalized title and throws the documented error", () => {
+  assert.equal(validateTitle("  Triage  "), "Triage");
+  assert.throws(
+    () => validateTitle("   "),
+    (error) => error instanceof ValidationError && error.code === INVALID_TITLE
+  );
+});
+
+test("imported task titles are normalized the same way", () => {
+  const { service } = createTaskBoard();
+  const { imported } = service.importTasks([{ externalId: "ext-1", title: "  Imported item  " }]);
+  assert.equal(imported[0].title, "Imported item");
+});

package/benchmarks/projects/agent-success-task-board-node/tests/title-primary.check.mjs

@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_TITLE, ValidationError, createTaskBoard } from "../src/index.js";
+
+test("createTask trims surrounding whitespace from the title", () => {
+  const { service } = createTaskBoard();
+  const task = service.createTask({ title: "  Write release notes  " });
+  assert.equal(task.title, "Write release notes");
+});
+
+test("the stored task retains the normalized title", () => {
+  const { service, store } = createTaskBoard();
+  const task = service.createTask({ title: "\tReview backlog " });
+  assert.equal(store.get(task.id).title, "Review backlog");
+  assert.deepEqual(
+    store.list().map((entry) => entry.title),
+    ["Review backlog"]
+  );
+});
+
+test("a whitespace-only title is rejected with the invalid-title error", () => {
+  const { service } = createTaskBoard();
+  for (const title of ["   ", "\t", "\n  \r\n"]) {
+    assert.throws(
+      () => service.createTask({ title }),
+      (error) => error instanceof ValidationError && error.code === INVALID_TITLE
+    );
+  }
+});

package/dist/src/cli/help.js

@@ -156,6 +156,10 @@ export function renderExperimentRunHelp() {
         "  --case <ids>                                       Comma-separated case ids to run",
         "  --benchmark-project <ids>                          Comma-separated benchmark project ids to run",
         "",
+        "Options are accepted per plugin: every plugin rejects the options it does not support instead of ignoring them. The",
+        "generic controlled-experiment options (such as --agents, --strategies, --complexities, --max-runs and the command",
+        "templates) belong to context-strategy-comparison and warm-index-reuse only.",
+        "",
         "my-dev-kit command override (warm-index-reuse, incremental-change-staleness, context-window-scaling, retrieval-precision-recall, retrieval-query-strategy-comparison, and context-pack-generation):",
         "  --kit-command <command>                            my-dev-kit command used by the selected experiment plugin",
         "                                                      (defaults: warm-index-reuse and context-window-scaling use",
@@ -273,7 +277,40 @@ export function renderExperimentRunHelp() {
         "  --command-template-codex <template>                Command template used to invoke the codex agent",
         "  --command-template-claude <template>               Command template used to invoke the claude agent",
         "  --no-screenshot                                     Accepted for compatibility; the plugin-aware report",
-        "                                                      path does not capture screenshots yet"
+        "                                                      path does not capture screenshots yet",
+        "",
+        "agent-success-rate only:",
+        "                                                      Self-target only (the Lab itself); --target is rejected. Runs the",
+        "                                                      bundled six-task implementation corpus (two canonical benchmark",
+        "                                                      projects) with both source-context treatments, raw-full-file and",
+        "                                                      context-pack, for every selected task. The default is",
+        "                                                      deterministic-fixture mode, which validates the patch evaluation",
+        "                                                      pipeline, invokes no coding agent and needs no provider installed.",
+        "                                                      Accepted options: --out, --case, --benchmark-project,",
+        "                                                      --kit-command, --agent, --include-real-agents, --timeout-ms and",
+        "                                                      --repair-attempts; --case and --benchmark-project narrow the",
+        "                                                      catalog without reordering it.",
+        "  --agent <codex|claude>                              Select exactly one real provider. Real-agent mode is entered only",
+        "                                                      when --agent is combined with --include-real-agents; neither flag",
+        "                                                      works alone. The provider receives source-only context and returns",
+        "                                                      a unified diff; it never edits files. The Lab applies the diff in a",
+        "                                                      disposable sandbox with a guarded patch policy and decides task",
+        "                                                      success from trusted checks on the changed state.",
+        "  --include-real-agents                               Explicit authorization to invoke the selected provider. Without it",
+        "                                                      no provider is ever launched.",
+        "  --timeout-ms <ms>                                   Per provider attempt (real-agent mode only): a positive integer,",
+        "                                                      default 240000, maximum 1800000.",
+        "  --repair-attempts <0|1|2>                           Repairs allowed after the initial attempt (real-agent mode only;",
+        "                                                      default 0). At most 3 attempts per treatment. A repair runs only",
+        "                                                      after an evaluable implementation failure, in a fresh sandbox,",
+        "                                                      with fixed-category feedback and the untrusted previous diff; it",
+        "                                                      never follows a provider, sandbox or verification infrastructure",
+        "                                                      problem.",
+        "                                                      Writes report.json, report.html and report.txt plus",
+        "                                                      agent-success-rate-execution.json and",
+        "                                                      agent-success-rate-analysis.json. Initial-attempt and final success",
+        "                                                      are reported separately. Results are descriptive for the finite",
+        "                                                      corpus: no winner, ranking, composite score or significance claim."
     ].join("\n");
 }
 export function renderExperimentControlledHelp() {

package/dist/src/commands/runExperimentDescribeCommand.js

@@ -1,4 +1,5 @@
 import { createDefaultExperimentPluginRegistry } from "../experiments/index.js";
+import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/index.js";
 // ---------------------------------------------------------------------------
 // v0.4.6 Batch 4 -- reusable experiment-describe command owner.
 //
@@ -57,7 +58,27 @@ function buildDescription(plugin) {
         optionalConfigFields: readConfigFields(plugin, false),
         targetBehavior: describeTargetBehavior(plugin),
         expectedReports: describeExpectedReports(plugin),
-        examples: describeExamples(plugin)
+        examples: describeExamples(plugin),
+        ...(plugin.metadata.id === AGENT_SUCCESS_RATE_ID ? { agentSuccessRate: describeAgentSuccessRate() } : {})
+    };
+}
+const AGENT_SUCCESS_RATE_ID = "agent-success-rate";
+/** Fixed, public description of the implemented behavior. It names no task check, fact, edit scope or reference patch. */
+function describeAgentSuccessRate() {
+    return {
+        treatments: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+        corpus: "Bundled six-task implementation corpus over two canonical benchmark projects (localized, cross-module and broad-change tasks), resolved from the installed package.",
+        defaultMode: "deterministic-fixture: validates the patch evaluation pipeline and the benchmark corpus; no coding agent runs and no provider is required.",
+        realAgentMode: "Opt-in only: --agent <codex|claude> together with --include-real-agents. Neither flag alone enables it; real execution is never the default.",
+        providers: [...AGENT_SUCCESS_RATE_REAL_AGENT_IDS],
+        timeoutMs: { default: AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, maximum: AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, scope: "each provider attempt" },
+        repairAttempts: { allowed: [...AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES], default: 0, maximumTotalAttemptsPerTreatment: 3, mode: "real-agent only" },
+        agentContext: "Source-only: the provider sees the public task text and the treatment's source context, never tests, trusted checks or reference patches, and runs in a neutral working directory.",
+        patchApplication: "Guarded: the provider returns a unified diff only. The Lab validates it against a patch policy and applies it in a fresh disposable sandbox; canonical benchmark projects are never modified.",
+        taskSuccess: "Decided by trusted checks on the actual changed repository state (required behavior, no regressions, protected files intact). A provider's claim of completion is never evidence. Initial-attempt success and final success are reported separately.",
+        reportOutputs: ["report.json", "report.html", "report.txt", "agent-success-rate-execution.json", "agent-success-rate-analysis.json"],
+        targetPolicy: "Self-target only; external-local targets are rejected.",
+        interpretation: "Descriptive matched comparison of two source-context treatments on a finite corpus: no winner, ranking, composite score, causal or statistical-significance claim."
     };
 }
 function printDescription(plugin) {
@@ -84,6 +105,24 @@ function printDescription(plugin) {
     for (const example of description.examples) {
         console.log(`  ${example}`);
     }
+    const agentSuccessRate = description.agentSuccessRate; // eslint-disable-line @typescript-eslint/no-explicit-any
+    if (agentSuccessRate) {
+        console.log("");
+        console.log("Agent success rate details:");
+        console.log(`  Treatments: ${agentSuccessRate.treatments.join(", ")}`);
+        console.log(`  Corpus: ${agentSuccessRate.corpus}`);
+        console.log(`  Default mode: ${agentSuccessRate.defaultMode}`);
+        console.log(`  Real-agent mode: ${agentSuccessRate.realAgentMode}`);
+        console.log(`  Providers: ${agentSuccessRate.providers.join(", ")}`);
+        console.log(`  Timeout per attempt: default ${agentSuccessRate.timeoutMs.default} ms, maximum ${agentSuccessRate.timeoutMs.maximum} ms`);
+        console.log(`  Repair attempts: ${agentSuccessRate.repairAttempts.allowed.join(", ")} (default ${agentSuccessRate.repairAttempts.default}; at most ${agentSuccessRate.repairAttempts.maximumTotalAttemptsPerTreatment} attempts per treatment; real-agent mode only)`);
+        console.log(`  Agent context: ${agentSuccessRate.agentContext}`);
+        console.log(`  Patch application: ${agentSuccessRate.patchApplication}`);
+        console.log(`  Task success: ${agentSuccessRate.taskSuccess}`);
+        console.log(`  Report outputs: ${agentSuccessRate.reportOutputs.join(", ")}`);
+        console.log(`  Target policy: ${agentSuccessRate.targetPolicy}`);
+        console.log(`  Interpretation: ${agentSuccessRate.interpretation}`);
+    }
 }
 function printConfigSection(title, fields) {
     console.log(`${title}:`);
@@ -123,12 +162,18 @@ function readArrayField(value, key) {
     return Array.isArray(field) ? field : [];
 }
 function describePurpose(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Measure whether implementation patches resolve benchmark tasks under two source-context treatments (raw-full-file and context-pack) using trusted checks on disposable benchmark copies. Deterministic-fixture mode (default) validates the evaluation pipeline only; real-agent mode (opt-in) evaluates the patches a selected provider returns, with optional bounded repair.";
+    }
     if (plugin.metadata.id === "context-strategy-comparison") {
         return "Compare raw full-file prompts with my-dev-kit-guided retrieval prompts using the existing controlled experiment workflow.";
     }
     return plugin.metadata.description;
 }
 function describeTargetBehavior(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Self-target only: the bundled corpus is evaluated against the Lab itself and an explicit --target is rejected. Sandboxes and artifacts are written beneath the output directory (beneath the workspace when installed), never into the installed package or the canonical benchmark projects. Accepted options: --out, --case, --benchmark-project, --kit-command, --agent, --include-real-agents, --timeout-ms and --repair-attempts; every other option is rejected.";
+    }
     const supportsExternal = plugin.metadata.supportedTargets.includes("external-local");
     if (plugin.metadata.id === "context-window-scaling") {
         return "Three subject modes. Bundled (default): the bundled fixed scaling corpus against the Lab itself. Synthetic: caller-supplied deterministic synthetic repositories via --synthetic-config <path> (a SyntheticRepositoryConfigV1 JSON file, mutually exclusive with --case) written beneath the experiment output directory. External local repository: an explicitly selected local Git worktree via --target <path> together with --local-subject-config <path> (a LocalRepositorySubjectConfigV1 JSON file); the repository is never modified, the output directory must be outside it, and durable output omits the repository path, source text, and file names. Operational context-fit measurement only; no retrieval precision, recall, or ranking quality is measured.";
@@ -151,6 +196,9 @@ function describeTargetBehavior(plugin) {
     return "Runs against the current lab repository.";
 }
 function describeExpectedReports(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Writes plugin-aware report.json, report.html and report.txt with a typed agent-success-rate section (initial-attempt and final success, repair history, edit quality, provider duration and tokens, scientific limitations), plus the separate agent-success-rate-execution.json and agent-success-rate-analysis.json artifacts.";
+    }
     if (plugin.metadata.supportedOutputs.includes("html") && plugin.metadata.supportedOutputs.includes("json")) {
         return "Writes plugin-aware JSON and HTML reports with plugin, target, variant, case, metric, artifact, warning, skip, and failure metadata.";
     }
@@ -185,6 +233,9 @@ function describeExamples(plugin) {
     if (plugin.metadata.id === "context-pack-generation") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --agent codex --include-real-agents --repair-attempts 2 --out <run-dir>`);
+    }
     if (plugin.metadata.id === "context-window-scaling") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --synthetic-config <path-to-config.json> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }

package/dist/src/commands/runExperimentRunCommand.js

@@ -3,10 +3,12 @@ import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
+import { AGENT_SUCCESS_PROJECT_PROFILES_PATH, AGENT_SUCCESS_TASK_CATALOG_PATH, assertReadAgentSuccessCorpus } from "../evaluation/agentSuccess/index.js";
 import { summarizeRetrievalGroundTruthIssuesSafely, validateRetrievalPrecisionRecallCorpus } from "../evaluation/retrievalQuality/index.js";
 import { loadLocalRepositorySubject } from "../evaluation/localRepositorySubject/index.js";
 import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, retrievalPrecisionRecallPlugin, retrievalQueryStrategyComparisonPlugin, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
 import { contextPackGenerationPlugin } from "../experiments/plugins/contextPackGeneration/index.js";
+import { AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, agentSuccessRatePlugin, selectAgentSuccessTasks } from "../experiments/plugins/agentSuccessRate/index.js";
 import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
 import { projectExternalLocalTarget, projectRunForExternalLocalPersistence, } from "../experiments/plugins/contextWindowScaling/localSubjectPrivacy.js";
 import { assertWorkRootOutsideTarget } from "../experiments/plugins/contextWindowScaling/localSubjectScratch.js";
@@ -71,6 +73,27 @@ const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_ALLOWED_FLAGS = ["--experimen
 const CONTEXT_PACK_GENERATION_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
 const CONTEXT_PACK_GENERATION_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
 const CONTEXT_PACK_GENERATION_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
+// agent-success-rate (v0.9.0) owns the bundled six-task implementation corpus (task catalog and project profiles), both
+// resolved from the package root, never the cwd. It runs against the Lab itself only and accepts exactly these options;
+// every other flag is rejected, not ignored. Real providers run only with --agent together with --include-real-agents.
+const AGENT_SUCCESS_RATE_ALLOWED_FLAGS = [
+    "--experiment",
+    "--out",
+    "--case",
+    "--benchmark-project",
+    "--kit-command",
+    "--agent",
+    "--include-real-agents",
+    "--timeout-ms",
+    "--repair-attempts"
+];
+export function assertAgentSuccessRateFlags(argv) {
+    const flags = argv.filter((arg) => arg.startsWith("--"));
+    const unsupported = [...new Set(flags.filter((flag) => !AGENT_SUCCESS_RATE_ALLOWED_FLAGS.includes(flag)))];
+    if (unsupported.length > 0) {
+        throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${agentSuccessRatePlugin.metadata.id}; supported options: ${AGENT_SUCCESS_RATE_ALLOWED_FLAGS.join(", ")}.`);
+    }
+}
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
     try {
         const args = parseRunExperimentArgs(argv);
@@ -151,7 +174,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
                 `Target root: ${result.target.targetRoot}`,
                 `Output: ${String(result.metadata?.outputRoot ?? "")}`,
                 `Report JSON: ${reports.outputPaths.jsonPath}`,
-                `Report HTML: ${reports.outputPaths.htmlPath}`
+                `Report HTML: ${reports.outputPaths.htmlPath}`,
+                ...(args.experimentId === agentSuccessRatePlugin.metadata.id ? agentSuccessRateSummaryLines(result, reports.outputPaths.textPath) : [])
             ];
         // Automatic presentation (report already produced above) applies only to warm-index-reuse
         // campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
@@ -225,8 +249,14 @@ export function parseRunExperimentArgs(argv) {
     let contextBudgets;
     let syntheticConfigPath;
     let localSubjectConfigPath;
+    const agentFlagValues = [];
+    let repairAttempts;
     const seenFlags = [];
     const commandTemplates = {};
+    // The flag whitelist runs before any value is interpreted so an unsupported flag is reported as such.
+    const preScanIndex = argv.indexOf("--experiment");
+    if (preScanIndex >= 0 && argv[preScanIndex + 1] === agentSuccessRatePlugin.metadata.id)
+        assertAgentSuccessRateFlags(argv);
     for (let index = 0; index < argv.length; index += 1) {
         const arg = argv[index];
         if (arg !== undefined && arg.startsWith("--"))
@@ -264,6 +294,12 @@ export function parseRunExperimentArgs(argv) {
         else if (arg === "--benchmark-project") {
             benchmarkProjects.push(...splitList(readRequiredValue(argv, ++index, "--benchmark-project")));
         }
+        else if (arg === "--agent") {
+            agentFlagValues.push(readRequiredValue(argv, ++index, "--agent"));
+        }
+        else if (arg === "--repair-attempts") {
+            repairAttempts = parseRepairAttempts(readRequiredValue(argv, ++index, "--repair-attempts"));
+        }
         else if (arg === "--agents") {
             agents = splitList(readRequiredValue(argv, ++index, "--agents")).map((value) => parseAgentId(value));
         }
@@ -399,6 +435,49 @@ export function parseRunExperimentArgs(argv) {
             throw new Error("--benchmark-project must list at least one benchmark project id.");
         }
     }
+    if (experimentId === agentSuccessRatePlugin.metadata.id) {
+        // Mode matrix: no flags = deterministic-fixture (default); --agent together with --include-real-agents = real-agent mode.
+        // Everything else is rejected, never coerced: a provider needs both flags, and real-agent options need a provider.
+        if (agentFlagValues.length > 1) {
+            throw new Error("--agent may be supplied only once; agent-success-rate runs exactly one provider.");
+        }
+        const agentId = agentFlagValues[0];
+        if (agentId !== undefined && !AGENT_SUCCESS_RATE_REAL_AGENT_IDS.includes(agentId)) {
+            throw new Error(`--agent must be exactly one of: ${AGENT_SUCCESS_RATE_REAL_AGENT_IDS.join(", ")}.`);
+        }
+        if (agentId !== undefined && includeRealAgents !== true) {
+            throw new Error("--agent requires --include-real-agents; real providers are invoked only with explicit authorization.");
+        }
+        if (agentId === undefined && includeRealAgents === true) {
+            throw new Error("--include-real-agents requires --agent <codex|claude> for agent-success-rate.");
+        }
+        if (agentId === undefined && timeoutMs !== undefined) {
+            throw new Error("--timeout-ms is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && kitCommand !== undefined) {
+            throw new Error("--kit-command is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && repairAttempts !== undefined && repairAttempts > 0) {
+            throw new Error("--repair-attempts greater than 0 requires --agent and --include-real-agents (real-agent mode).");
+        }
+        if (timeoutMs !== undefined && timeoutMs > AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS) {
+            throw new Error(`--timeout-ms must be a positive integer no greater than ${AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
+    else {
+        if (agentFlagValues.length > 0) {
+            throw new Error(`--agent is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+        if (repairAttempts !== undefined) {
+            throw new Error(`--repair-attempts is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+    }
     if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
@@ -441,7 +520,8 @@ export function parseRunExperimentArgs(argv) {
             throw new Error(`External ${experimentId} targets require --local-subject-config.`);
         }
     }
-    if (kitCommand !== undefined && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
+    // agent-success-rate accepts --kit-command in real-agent mode; its own validation above owns that rule and message.
+    if (kitCommand !== undefined && experimentId !== agentSuccessRatePlugin.metadata.id && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
         throw new Error(`--kit-command is only supported for --experiment ${KIT_COMMAND_PLUGIN_IDS.join(" or ")}.`);
     }
     if (campaignPreset !== undefined) {
@@ -478,6 +558,8 @@ export function parseRunExperimentArgs(argv) {
             requireAgents,
             includeRealAgents,
             commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
+            agentId: agentFlagValues[0],
+            repairAttempts,
             kitCommand,
             campaignPreset,
             contextBudgets
@@ -542,8 +624,27 @@ async function loadPluginInputs(args, toolRoot, context, outputRoot) {
             ? loadRetrievalQueryStrategyComparisonLocalSubjectInputs(args, toolRoot, context, outputRoot)
             : loadRetrievalQueryStrategyComparisonInputs(args, toolRoot, context);
     }
+    if (args.experimentId === agentSuccessRatePlugin.metadata.id) {
+        return loadAgentSuccessRateInputs(args, context);
+    }
     return undefined;
 }
+// agent-success-rate: the bundled task catalog and project profiles are located through the established package-resource
+// resolver (never the cwd) and validated by the corpus reader, which also checks the canonical projects on disk. The
+// validated tasks reach the plugin through its existing programmatic input; the plugin applies --case/--benchmark-project
+// selection in catalog order and owns execution, repair, science and artifacts. No task file is accepted from the CLI.
+async function loadAgentSuccessRateInputs(args, context) {
+    const validation = agentSuccessRatePlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid agent success rate config: ${validation.errors.join("; ")}`);
+    }
+    resolvePackageResource(context, AGENT_SUCCESS_TASK_CATALOG_PATH);
+    resolvePackageResource(context, AGENT_SUCCESS_PROJECT_PROFILES_PATH);
+    const corpus = assertReadAgentSuccessCorpus(context.resourceRoot);
+    // Fail on unknown case or project ids before any output directory exists.
+    selectAgentSuccessTasks(corpus.tasks, validation.config);
+    return { agentSuccessTasks: corpus.tasks };
+}
 // context-pack-generation (bundled/self): the frozen warm-index corpus through the established resource resolver, profile
 // and corpus validation. The plugin applies --case/--benchmark-project selection and owns execution, science and artifacts.
 async function loadContextPackGenerationInputs(args, toolRoot, context) {
@@ -815,6 +916,12 @@ function readRequiredValue(argv, index, label) {
     }
     return value;
 }
+function parseRepairAttempts(value) {
+    if (!/^[0-2]$/.test(value)) {
+        throw new Error("--repair-attempts must be 0, 1 or 2.");
+    }
+    return Number(value);
+}
 function parsePositiveInteger(label, value) {
     const parsed = Number(value);
     if (!Number.isInteger(parsed) || parsed <= 0) {
@@ -822,6 +929,15 @@ function parsePositiveInteger(label, value) {
     }
     return parsed;
 }
+function agentSuccessRateSummaryLines(result, textPath) {
+    const mode = readMetadataString(result.metadata?.executionMode) ?? "deterministic-fixture";
+    const provider = readMetadataString(result.metadata?.providerId);
+    return [
+        `Report Text: ${textPath}`,
+        `Execution mode: ${mode}`,
+        ...(provider ? [`Provider: ${provider} (invoked by explicit --include-real-agents authorization)`] : ["Provider: none (deterministic-fixture; no coding agent was invoked)"])
+    ];
+}
 function readMetadataString(value) {
     return typeof value === "string" && value.trim() ? value : undefined;
 }

package/dist/src/core/processTree.js

@@ -20,6 +20,61 @@ export function requestProcessTermination(pid) {
         // The process may have exited between the liveness check and this signal.
     }
 }
+/**
+ * Opt-in, awaitable descendant-tree termination for bounded execution
+ * (see `terminateProcessTree` in `runMeasuredCommand`). The legacy
+ * `forceTerminateProcess` above is intentionally unchanged: it stays
+ * fire-and-forget and, off Windows, signals one pid only.
+ *
+ * Windows: `taskkill /pid <pid> /T /F` (shell:false) is awaited, bounded by
+ * `waitTimeoutMs`, so the caller knows the tree was asked to die before it
+ * proceeds.
+ *
+ * Other platforms: the child must have been spawned `detached` so it leads its
+ * own process group; the whole group is signalled via the negative pid. If no
+ * group exists, the single pid is signalled as a fallback.
+ *
+ * Never throws: the target may already have exited.
+ */
+export async function terminateProcessTree(pid, options = {}) {
+    if (!pid) {
+        return;
+    }
+    const platform = options.platform ?? process.platform;
+    if (platform === "win32") {
+        await new Promise((resolve) => {
+            let timer;
+            const finish = () => {
+                if (timer)
+                    clearTimeout(timer);
+                resolve();
+            };
+            const killer = spawn("taskkill", ["/pid", String(pid), "/T", "/F"], {
+                shell: false,
+                stdio: "ignore",
+                windowsHide: true
+            });
+            timer = setTimeout(finish, options.waitTimeoutMs ?? 5000);
+            killer.on("error", finish);
+            killer.on("close", finish);
+        });
+        return;
+    }
+    const signal = options.posixSignal ?? "SIGKILL";
+    try {
+        process.kill(-pid, signal);
+        return;
+    }
+    catch {
+        // No process group for this pid (or it already exited); fall back to the single pid.
+    }
+    try {
+        process.kill(pid, signal);
+    }
+    catch {
+        // The process may have exited between the liveness check and this signal.
+    }
+}
 /**
  * Forced cleanup for a process that did not stop on request.
  *