@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/benchmarks/projects/agent-success-task-board-node/tests/summary-edge.check.mjs

@@ -0,0 +1,67 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_PROJECT, createTaskBoard } from "../src/index.js";
+
+test("a project's completed count never exceeds its total", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "Solo", projectId: "alpha" });
+  for (const title of ["x", "y", "z"]) {
+    const task = service.createTask({ title, projectId: "beta" });
+    service.completeTask(task.id);
+  }
+  const summary = summarize("alpha");
+  assert.equal(summary.total, 1);
+  assert.equal(summary.completed, 0);
+  assert.ok(summary.completed <= summary.total);
+  assert.equal(summary.open, 1);
+});
+
+test("unrelated tasks in the store do not change a project's summary", () => {
+  const { service, summarize } = createTaskBoard();
+  const a = service.createTask({ title: "A", projectId: "core" });
+  service.createTask({ title: "B", projectId: "core" });
+  service.completeTask(a.id);
+  const before = summarize("core");
+  const noise = service.createTask({ title: "Noise", projectId: "core-extras" });
+  service.createTask({ title: "More noise", projectId: "CORE2" });
+  service.completeTask(noise.id);
+  service.importTasks([{ externalId: "n-1", title: "Imported noise", projectId: "c" }]);
+  assert.deepEqual(summarize("core"), before);
+});
+
+test("an unknown project has an empty, consistent summary", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "A", projectId: "known" });
+  assert.deepEqual(summarize("missing"), { projectId: "missing", total: 0, completed: 0, open: 0, taskIds: [] });
+});
+
+test("blank project identities are rejected consistently", () => {
+  const { service, store, summarize } = createTaskBoard();
+  assert.throws(() => service.createTask({ title: "Task", projectId: "   " }), { code: INVALID_PROJECT });
+  assert.throws(() => service.createTask({ title: "Task", projectId: "" }), { code: INVALID_PROJECT });
+  assert.throws(
+    () => service.importTasks([{ externalId: "ext-1", title: "Task", projectId: " " }]),
+    { code: INVALID_PROJECT }
+  );
+  assert.throws(() => summarize("  "), { code: INVALID_PROJECT });
+  assert.equal(store.list().length, 0);
+});
+
+test("summaries are deterministic and list ids in creation order", () => {
+  const build = () => {
+    const { service, summarize } = createTaskBoard();
+    service.createTask({ title: "1", projectId: "p" });
+    service.createTask({ title: "2", projectId: "q" });
+    service.createTask({ title: "3", projectId: "P" });
+    service.importTasks([{ externalId: "e-1", title: "4", projectId: "p " }]);
+    return summarize("p");
+  };
+  assert.deepEqual(build(), build());
+  assert.deepEqual(build().taskIds, ["task-1", "task-3", "task-4"]);
+});
+
+test("the default project is used for tasks created without one", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "No project given" });
+  assert.equal(summarize("inbox").total, 1);
+});

package/benchmarks/projects/agent-success-task-board-node/tests/summary-primary.check.mjs

@@ -0,0 +1,54 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { createTaskBoard } from "../src/index.js";
+
+test("tasks created with differently written project names share one project identity", () => {
+  const { service, store, summarize } = createTaskBoard();
+  service.createTask({ title: "One", projectId: "web" });
+  service.createTask({ title: "Two", projectId: "  WEB " });
+  service.createTask({ title: "Three", projectId: "Web" });
+  assert.deepEqual(
+    store.list().map((task) => task.projectId),
+    ["web", "web", "web"]
+  );
+  const summary = summarize("web");
+  assert.equal(summary.projectId, "web");
+  assert.equal(summary.total, 3);
+});
+
+test("a summary only counts tasks of the requested project", () => {
+  const { service, summarize } = createTaskBoard();
+  const web = service.createTask({ title: "Web task", projectId: "web" });
+  service.createTask({ title: "Web app task", projectId: "web-app" });
+  service.createTask({ title: "Webhook task", projectId: "webhooks" });
+  const other = service.createTask({ title: "Docs task", projectId: "docs" });
+  service.completeTask(other.id);
+  const summary = summarize("web");
+  assert.equal(summary.total, 1);
+  assert.equal(summary.completed, 0);
+  assert.equal(summary.open, 1);
+  assert.deepEqual(summary.taskIds, [web.id]);
+});
+
+test("completed and open counts are correct for the requested project", () => {
+  const { service, summarize } = createTaskBoard();
+  const a = service.createTask({ title: "A", projectId: "ops" });
+  service.createTask({ title: "B", projectId: "ops" });
+  service.createTask({ title: "C", projectId: "ops" });
+  service.createTask({ title: "Elsewhere", projectId: "labs" });
+  service.completeTask(a.id);
+  assert.deepEqual(summarize(" OPS "), {
+    projectId: "ops",
+    total: 3,
+    completed: 1,
+    open: 2,
+    taskIds: ["task-1", "task-2", "task-3"]
+  });
+});
+
+test("imported tasks join the same normalized project identity", () => {
+  const { service, summarize } = createTaskBoard();
+  service.createTask({ title: "Manual", projectId: "ops" });
+  service.importTasks([{ externalId: "ext-1", title: "Imported", projectId: " OPS" }]);
+  assert.equal(summarize("ops").total, 2);
+});

package/benchmarks/projects/agent-success-task-board-node/tests/title-edge.check.mjs

@@ -0,0 +1,39 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_TITLE, ValidationError, createTaskBoard, validateTitle } from "../src/index.js";
+
+test("rejected titles leave the store and id sequence untouched", () => {
+  const { service, store } = createTaskBoard();
+  assert.throws(() => service.createTask({ title: "  \t " }), { code: INVALID_TITLE });
+  assert.throws(() => service.createTask({ title: "" }), { code: INVALID_TITLE });
+  assert.equal(store.list().length, 0);
+  const task = service.createTask({ title: "First" });
+  assert.equal(task.id, "task-1");
+  assert.equal(store.list().length, 1);
+});
+
+test("interior whitespace is preserved while surrounding whitespace is removed", () => {
+  const { service } = createTaskBoard();
+  assert.equal(service.createTask({ title: "  Fix  the   bug " }).title, "Fix  the   bug");
+  assert.equal(service.createTask({ title: "\n\tPlan sprint\t\n" }).title, "Plan sprint");
+});
+
+test("already-trimmed titles keep their meaning", () => {
+  const { service } = createTaskBoard();
+  assert.equal(service.createTask({ title: "Ship it" }).title, "Ship it");
+  assert.equal(service.createTask({ title: "a" }).title, "a");
+});
+
+test("validateTitle returns the normalized title and throws the documented error", () => {
+  assert.equal(validateTitle("  Triage  "), "Triage");
+  assert.throws(
+    () => validateTitle("   "),
+    (error) => error instanceof ValidationError && error.code === INVALID_TITLE
+  );
+});
+
+test("imported task titles are normalized the same way", () => {
+  const { service } = createTaskBoard();
+  const { imported } = service.importTasks([{ externalId: "ext-1", title: "  Imported item  " }]);
+  assert.equal(imported[0].title, "Imported item");
+});

package/benchmarks/projects/agent-success-task-board-node/tests/title-primary.check.mjs

@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { INVALID_TITLE, ValidationError, createTaskBoard } from "../src/index.js";
+
+test("createTask trims surrounding whitespace from the title", () => {
+  const { service } = createTaskBoard();
+  const task = service.createTask({ title: "  Write release notes  " });
+  assert.equal(task.title, "Write release notes");
+});
+
+test("the stored task retains the normalized title", () => {
+  const { service, store } = createTaskBoard();
+  const task = service.createTask({ title: "\tReview backlog " });
+  assert.equal(store.get(task.id).title, "Review backlog");
+  assert.deepEqual(
+    store.list().map((entry) => entry.title),
+    ["Review backlog"]
+  );
+});
+
+test("a whitespace-only title is rejected with the invalid-title error", () => {
+  const { service } = createTaskBoard();
+  for (const title of ["   ", "\t", "\n  \r\n"]) {
+    assert.throws(
+      () => service.createTask({ title }),
+      (error) => error instanceof ValidationError && error.code === INVALID_TITLE
+    );
+  }
+});

package/dist/src/cli/help.js

@@ -156,7 +156,11 @@ export function renderExperimentRunHelp() {
         "  --case <ids>                                       Comma-separated case ids to run",
         "  --benchmark-project <ids>                          Comma-separated benchmark project ids to run",
         "",
-        "my-dev-kit command override (warm-index-reuse, incremental-change-staleness, context-window-scaling, retrieval-precision-recall, and retrieval-query-strategy-comparison):",
+        "Options are accepted per plugin: every plugin rejects the options it does not support instead of ignoring them. The",
+        "generic controlled-experiment options (such as --agents, --strategies, --complexities, --max-runs and the command",
+        "templates) belong to context-strategy-comparison and warm-index-reuse only.",
+        "",
+        "my-dev-kit command override (warm-index-reuse, incremental-change-staleness, context-window-scaling, retrieval-precision-recall, retrieval-query-strategy-comparison, and context-pack-generation):",
         "  --kit-command <command>                            my-dev-kit command used by the selected experiment plugin",
         "                                                      (defaults: warm-index-reuse and context-window-scaling use",
         "                                                      npx @dailephd/my-dev-kit@latest;",
@@ -219,6 +223,21 @@ export function renderExperimentRunHelp() {
         "                                                      strategies are reported as a tradeoff. No scalar score or total",
         "                                                      ranking exists.",
         "",
+        "context-pack-generation only:",
+        "                                                      Bundled/self mode runs the frozen 12-case corpus and accepts only",
+        "                                                      --out, --case, --benchmark-project and --kit-command (default",
+        "                                                      npx @dailephd/my-dev-kit@latest); --case and --benchmark-project",
+        "                                                      narrow the case set. Both treatments, raw-full-file and",
+        "                                                      context-pack, always run for every case; there is no treatment,",
+        "                                                      strategy, or selection-policy option. External-local mode requires",
+        "                                                      --target together with --local-subject-config and accepts only",
+        "                                                      --out, --target, --local-subject-config and --kit-command; --case",
+        "                                                      and --benchmark-project are rejected because the local-subject",
+        "                                                      config owns the case set. The repository is never modified and",
+        "                                                      durable output withholds private file, symbol, fact, semantic-node,",
+        "                                                      call-relationship, test, source-text, warning, task and case-title",
+        "                                                      identities; no context-pack body is written for an external run.",
+        "",
         "warm-index-reuse only:",
         "                                                      Legacy/default mode uses the deterministic fake agent only;",
         "                                                      agent options below are not accepted by this plugin.",
@@ -258,7 +277,40 @@ export function renderExperimentRunHelp() {
         "  --command-template-codex <template>                Command template used to invoke the codex agent",
         "  --command-template-claude <template>               Command template used to invoke the claude agent",
         "  --no-screenshot                                     Accepted for compatibility; the plugin-aware report",
-        "                                                      path does not capture screenshots yet"
+        "                                                      path does not capture screenshots yet",
+        "",
+        "agent-success-rate only:",
+        "                                                      Self-target only (the Lab itself); --target is rejected. Runs the",
+        "                                                      bundled six-task implementation corpus (two canonical benchmark",
+        "                                                      projects) with both source-context treatments, raw-full-file and",
+        "                                                      context-pack, for every selected task. The default is",
+        "                                                      deterministic-fixture mode, which validates the patch evaluation",
+        "                                                      pipeline, invokes no coding agent and needs no provider installed.",
+        "                                                      Accepted options: --out, --case, --benchmark-project,",
+        "                                                      --kit-command, --agent, --include-real-agents, --timeout-ms and",
+        "                                                      --repair-attempts; --case and --benchmark-project narrow the",
+        "                                                      catalog without reordering it.",
+        "  --agent <codex|claude>                              Select exactly one real provider. Real-agent mode is entered only",
+        "                                                      when --agent is combined with --include-real-agents; neither flag",
+        "                                                      works alone. The provider receives source-only context and returns",
+        "                                                      a unified diff; it never edits files. The Lab applies the diff in a",
+        "                                                      disposable sandbox with a guarded patch policy and decides task",
+        "                                                      success from trusted checks on the changed state.",
+        "  --include-real-agents                               Explicit authorization to invoke the selected provider. Without it",
+        "                                                      no provider is ever launched.",
+        "  --timeout-ms <ms>                                   Per provider attempt (real-agent mode only): a positive integer,",
+        "                                                      default 240000, maximum 1800000.",
+        "  --repair-attempts <0|1|2>                           Repairs allowed after the initial attempt (real-agent mode only;",
+        "                                                      default 0). At most 3 attempts per treatment. A repair runs only",
+        "                                                      after an evaluable implementation failure, in a fresh sandbox,",
+        "                                                      with fixed-category feedback and the untrusted previous diff; it",
+        "                                                      never follows a provider, sandbox or verification infrastructure",
+        "                                                      problem.",
+        "                                                      Writes report.json, report.html and report.txt plus",
+        "                                                      agent-success-rate-execution.json and",
+        "                                                      agent-success-rate-analysis.json. Initial-attempt and final success",
+        "                                                      are reported separately. Results are descriptive for the finite",
+        "                                                      corpus: no winner, ranking, composite score or significance claim."
     ].join("\n");
 }
 export function renderExperimentControlledHelp() {

package/dist/src/commands/runExperimentDescribeCommand.js

@@ -1,4 +1,5 @@
 import { createDefaultExperimentPluginRegistry } from "../experiments/index.js";
+import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/index.js";
 // ---------------------------------------------------------------------------
 // v0.4.6 Batch 4 -- reusable experiment-describe command owner.
 //
@@ -57,7 +58,27 @@ function buildDescription(plugin) {
         optionalConfigFields: readConfigFields(plugin, false),
         targetBehavior: describeTargetBehavior(plugin),
         expectedReports: describeExpectedReports(plugin),
-        examples: describeExamples(plugin)
+        examples: describeExamples(plugin),
+        ...(plugin.metadata.id === AGENT_SUCCESS_RATE_ID ? { agentSuccessRate: describeAgentSuccessRate() } : {})
+    };
+}
+const AGENT_SUCCESS_RATE_ID = "agent-success-rate";
+/** Fixed, public description of the implemented behavior. It names no task check, fact, edit scope or reference patch. */
+function describeAgentSuccessRate() {
+    return {
+        treatments: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+        corpus: "Bundled six-task implementation corpus over two canonical benchmark projects (localized, cross-module and broad-change tasks), resolved from the installed package.",
+        defaultMode: "deterministic-fixture: validates the patch evaluation pipeline and the benchmark corpus; no coding agent runs and no provider is required.",
+        realAgentMode: "Opt-in only: --agent <codex|claude> together with --include-real-agents. Neither flag alone enables it; real execution is never the default.",
+        providers: [...AGENT_SUCCESS_RATE_REAL_AGENT_IDS],
+        timeoutMs: { default: AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, maximum: AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, scope: "each provider attempt" },
+        repairAttempts: { allowed: [...AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES], default: 0, maximumTotalAttemptsPerTreatment: 3, mode: "real-agent only" },
+        agentContext: "Source-only: the provider sees the public task text and the treatment's source context, never tests, trusted checks or reference patches, and runs in a neutral working directory.",
+        patchApplication: "Guarded: the provider returns a unified diff only. The Lab validates it against a patch policy and applies it in a fresh disposable sandbox; canonical benchmark projects are never modified.",
+        taskSuccess: "Decided by trusted checks on the actual changed repository state (required behavior, no regressions, protected files intact). A provider's claim of completion is never evidence. Initial-attempt success and final success are reported separately.",
+        reportOutputs: ["report.json", "report.html", "report.txt", "agent-success-rate-execution.json", "agent-success-rate-analysis.json"],
+        targetPolicy: "Self-target only; external-local targets are rejected.",
+        interpretation: "Descriptive matched comparison of two source-context treatments on a finite corpus: no winner, ranking, composite score, causal or statistical-significance claim."
     };
 }
 function printDescription(plugin) {
@@ -84,6 +105,24 @@ function printDescription(plugin) {
     for (const example of description.examples) {
         console.log(`  ${example}`);
     }
+    const agentSuccessRate = description.agentSuccessRate; // eslint-disable-line @typescript-eslint/no-explicit-any
+    if (agentSuccessRate) {
+        console.log("");
+        console.log("Agent success rate details:");
+        console.log(`  Treatments: ${agentSuccessRate.treatments.join(", ")}`);
+        console.log(`  Corpus: ${agentSuccessRate.corpus}`);
+        console.log(`  Default mode: ${agentSuccessRate.defaultMode}`);
+        console.log(`  Real-agent mode: ${agentSuccessRate.realAgentMode}`);
+        console.log(`  Providers: ${agentSuccessRate.providers.join(", ")}`);
+        console.log(`  Timeout per attempt: default ${agentSuccessRate.timeoutMs.default} ms, maximum ${agentSuccessRate.timeoutMs.maximum} ms`);
+        console.log(`  Repair attempts: ${agentSuccessRate.repairAttempts.allowed.join(", ")} (default ${agentSuccessRate.repairAttempts.default}; at most ${agentSuccessRate.repairAttempts.maximumTotalAttemptsPerTreatment} attempts per treatment; real-agent mode only)`);
+        console.log(`  Agent context: ${agentSuccessRate.agentContext}`);
+        console.log(`  Patch application: ${agentSuccessRate.patchApplication}`);
+        console.log(`  Task success: ${agentSuccessRate.taskSuccess}`);
+        console.log(`  Report outputs: ${agentSuccessRate.reportOutputs.join(", ")}`);
+        console.log(`  Target policy: ${agentSuccessRate.targetPolicy}`);
+        console.log(`  Interpretation: ${agentSuccessRate.interpretation}`);
+    }
 }
 function printConfigSection(title, fields) {
     console.log(`${title}:`);
@@ -123,12 +162,18 @@ function readArrayField(value, key) {
     return Array.isArray(field) ? field : [];
 }
 function describePurpose(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Measure whether implementation patches resolve benchmark tasks under two source-context treatments (raw-full-file and context-pack) using trusted checks on disposable benchmark copies. Deterministic-fixture mode (default) validates the evaluation pipeline only; real-agent mode (opt-in) evaluates the patches a selected provider returns, with optional bounded repair.";
+    }
     if (plugin.metadata.id === "context-strategy-comparison") {
         return "Compare raw full-file prompts with my-dev-kit-guided retrieval prompts using the existing controlled experiment workflow.";
     }
     return plugin.metadata.description;
 }
 function describeTargetBehavior(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Self-target only: the bundled corpus is evaluated against the Lab itself and an explicit --target is rejected. Sandboxes and artifacts are written beneath the output directory (beneath the workspace when installed), never into the installed package or the canonical benchmark projects. Accepted options: --out, --case, --benchmark-project, --kit-command, --agent, --include-real-agents, --timeout-ms and --repair-attempts; every other option is rejected.";
+    }
     const supportsExternal = plugin.metadata.supportedTargets.includes("external-local");
     if (plugin.metadata.id === "context-window-scaling") {
         return "Three subject modes. Bundled (default): the bundled fixed scaling corpus against the Lab itself. Synthetic: caller-supplied deterministic synthetic repositories via --synthetic-config <path> (a SyntheticRepositoryConfigV1 JSON file, mutually exclusive with --case) written beneath the experiment output directory. External local repository: an explicitly selected local Git worktree via --target <path> together with --local-subject-config <path> (a LocalRepositorySubjectConfigV1 JSON file); the repository is never modified, the output directory must be outside it, and durable output omits the repository path, source text, and file names. Operational context-fit measurement only; no retrieval precision, recall, or ranking quality is measured.";
@@ -139,6 +184,9 @@ function describeTargetBehavior(plugin) {
     if (plugin.metadata.id === "retrieval-query-strategy-comparison") {
         return "Two subject modes. Bundled (default): the frozen 12-case warm-index corpus compares all seven retrieval query strategies and may be narrowed with --case and --benchmark-project. External local repository: an explicitly selected local Git worktree via --target <path> together with --local-subject-config <path>; the local-subject config owns the case set, one private base index is built per configured case, core strategies share that base, semantic strategies use isolated copies, the repository is never modified, and durable output withholds private file, symbol, fact, semantic-node, warning and case-title identities while preserving numeric scientific results. Bundled filters are not accepted in external-local mode.";
     }
+    if (plugin.metadata.id === "context-pack-generation") {
+        return "Two subject modes. Bundled (default): the frozen 12-case warm-index corpus against the Lab itself, narrowable with --case and --benchmark-project. External local repository: an explicitly selected local Git worktree via --target <path> together with --local-subject-config <path>; the local-subject config owns the case set, the repository is never modified, the output directory must be outside it, one private base index is built per configured case, and durable output withholds private file, symbol, fact, semantic-node, call-relationship, test, source-text, warning, task and case-title identities while preserving numeric scientific results; no context-pack body is written externally. Bundled filters are not accepted in external-local mode. Both treatments (raw-full-file and context-pack) always run for every case; there is no treatment, strategy, or selection-policy option.";
+    }
     if (supportsExternal) {
         return "Runs against the current lab repository when --target is omitted, or against an explicit local target project when --target <path> is provided. Outputs stay under lab-controlled output directories by default.";
     }
@@ -148,6 +196,9 @@ function describeTargetBehavior(plugin) {
     return "Runs against the current lab repository.";
 }
 function describeExpectedReports(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Writes plugin-aware report.json, report.html and report.txt with a typed agent-success-rate section (initial-attempt and final success, repair history, edit quality, provider duration and tokens, scientific limitations), plus the separate agent-success-rate-execution.json and agent-success-rate-analysis.json artifacts.";
+    }
     if (plugin.metadata.supportedOutputs.includes("html") && plugin.metadata.supportedOutputs.includes("json")) {
         return "Writes plugin-aware JSON and HTML reports with plugin, target, variant, case, metric, artifact, warning, skip, and failure metadata.";
     }
@@ -165,7 +216,8 @@ function describeExamples(plugin) {
     if (plugin.metadata.supportedTargets.includes("external-local") &&
         plugin.metadata.id !== "context-window-scaling" &&
         plugin.metadata.id !== "retrieval-precision-recall" &&
-        plugin.metadata.id !== "retrieval-query-strategy-comparison") {
+        plugin.metadata.id !== "retrieval-query-strategy-comparison" &&
+        plugin.metadata.id !== "context-pack-generation") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target "Z:\\Users\\newuser\\Projects\\my-dev-kit-v1"${agentOptions}${screenshotOption}`);
     }
     // Only advertise campaign examples for plugins that declare the campaignPreset config field.
@@ -178,6 +230,12 @@ function describeExamples(plugin) {
     if (plugin.metadata.id === "retrieval-query-strategy-comparison") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }
+    if (plugin.metadata.id === "context-pack-generation") {
+        examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
+    }
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --agent codex --include-real-agents --repair-attempts 2 --out <run-dir>`);
+    }
     if (plugin.metadata.id === "context-window-scaling") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --synthetic-config <path-to-config.json> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }