@dailephd/my-dev-kit-lab 0.8.2 → 0.9.1

package/dist/src/cli/help.js

@@ -156,6 +156,10 @@ export function renderExperimentRunHelp() {
         "  --case <ids>                                       Comma-separated case ids to run",
         "  --benchmark-project <ids>                          Comma-separated benchmark project ids to run",
         "",
+        "Options are accepted per plugin: every plugin rejects the options it does not support instead of ignoring them. The",
+        "generic controlled-experiment options (such as --agents, --strategies, --complexities, --max-runs and the command",
+        "templates) belong to context-strategy-comparison and warm-index-reuse only.",
+        "",
         "my-dev-kit command override (warm-index-reuse, incremental-change-staleness, context-window-scaling, retrieval-precision-recall, retrieval-query-strategy-comparison, and context-pack-generation):",
         "  --kit-command <command>                            my-dev-kit command used by the selected experiment plugin",
         "                                                      (defaults: warm-index-reuse and context-window-scaling use",
@@ -273,7 +277,45 @@ export function renderExperimentRunHelp() {
         "  --command-template-codex <template>                Command template used to invoke the codex agent",
         "  --command-template-claude <template>               Command template used to invoke the claude agent",
         "  --no-screenshot                                     Accepted for compatibility; the plugin-aware report",
-        "                                                      path does not capture screenshots yet"
+        "                                                      path does not capture screenshots yet",
+        "",
+        "agent-success-rate only:",
+        "                                                      Self-target only (the Lab itself); --target is rejected. Runs the",
+        "                                                      bundled six-task implementation corpus (two canonical benchmark",
+        "                                                      projects) with both source-context treatments, raw-full-file and",
+        "                                                      context-pack, for every selected task. The default is",
+        "                                                      deterministic-fixture mode, which validates the patch evaluation",
+        "                                                      pipeline, invokes no coding agent and needs no provider installed.",
+        "                                                      Accepted options: --out, --case, --benchmark-project,",
+        "                                                      --kit-command, --agent, --include-real-agents, --timeout-ms and",
+        "                                                      --repair-attempts; --case and --benchmark-project narrow the",
+        "                                                      catalog without reordering it.",
+        "  --agent <codex|claude>                              Select exactly one real provider. Real-agent mode is entered only",
+        "                                                      when --agent is combined with --include-real-agents; neither flag",
+        "                                                      works alone. The provider receives source-only context and returns",
+        "                                                      a unified diff; it never edits files. The Lab applies the diff in a",
+        "                                                      disposable sandbox with a guarded patch policy and decides task",
+        "                                                      success from trusted checks on the changed state.",
+        "  --include-real-agents                               Explicit authorization to invoke the selected provider. Without it",
+        "                                                      no provider is ever launched.",
+        "  --timeout-ms <ms>                                   Per provider attempt (real-agent mode only): a positive integer,",
+        "                                                      default 240000, maximum 1800000.",
+        "  --repair-attempts <0|1|2>                           Repairs allowed after the initial attempt (real-agent mode only;",
+        "                                                      default 0). At most 3 attempts per treatment. A repair runs only",
+        "                                                      after an evaluable implementation failure, in a fresh sandbox,",
+        "                                                      with fixed-category feedback and the untrusted previous diff; it",
+        "                                                      never follows a provider, sandbox or verification infrastructure",
+        "                                                      problem.",
+        "                                                      Ordinary noncampaign execution writes report.json, report.html and report.txt plus",
+        "                                                      agent-success-rate-execution.json and",
+        "                                                      agent-success-rate-analysis.json. Initial-attempt and final success",
+        "                                                      are reported separately. Results are descriptive for the finite",
+        "                                                      corpus: no winner, ranking, composite score or significance claim.",
+        "",
+        "agent-success-rate resumable campaigns and timeouts only:",
+        ...AGENT_CAMPAIGN_OPTION_HELP.map(line => `  ${line}`),
+        "Examples (replace bracketed values; explicit source-root is required when package source proof is unavailable):",
+        ...AGENT_CAMPAIGN_EXAMPLES.map(line => `  ${line}`)
     ].join("\n");
 }
 export function renderExperimentControlledHelp() {
@@ -490,3 +532,4 @@ export function renderTutorialRunHelp() {
         "  2   Invalid command usage"
     ].join("\n");
 }
+import { AGENT_CAMPAIGN_EXAMPLES, AGENT_CAMPAIGN_OPTION_HELP } from "../commands/runExperimentDescribeCommand.js";

package/dist/src/commands/runExperimentDescribeCommand.js

@@ -1,4 +1,30 @@
 import { createDefaultExperimentPluginRegistry } from "../experiments/index.js";
+import { AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES, AGENT_SUCCESS_RATE_TREATMENT_IDS } from "../experiments/plugins/agentSuccessRate/index.js";
+import { AGENT_TIMEOUT_PRESETS } from "../experiments/plugins/agentSuccessRate/timeoutPresets.js";
+/** Public command-owned operational contract, shared by human, JSON and run help. */
+export const AGENT_CAMPAIGN_OPTION_HELP = [
+    "--agent-campaign <new|resume|continue-limit>: explicit version 2 campaign; requires stable --out <directory>.",
+    "--one-case: campaign-only; process at most the first unfinished selected case, with both treatments mandatory.",
+    "--campaign-source-root <directory>: campaign-only verified current source checkout; required when installed source proof is unavailable.",
+    "--timeout-preset <standard|extended|maximum>: codex/claude real-agent mode only; 240000/900000/1800000 ms per attempt.",
+    "--timeout-ms <ms>: numeric override; mutually exclusive with --timeout-preset. Neither supplied preserves 240000 ms default.",
+    "--expected-checkpoint-revision <positive-integer>: continue-limit only; exact committed checkpoint generation.",
+    "--limit-case <case-id> --limit-treatment <raw-full-file|context-pack> --limit-epoch <epoch-id>: continue-limit only; exact terminal limit unit and epoch.",
+    "--authorize-additional-invocation: mandatory boolean with no value for continue-limit; at most two execution epochs per treatment.",
+    "Resume revalidates identity, ownership and artifact digests and skips completed work, including completed task failures.",
+    "Real campaigns require --agent <codex|claude> and --include-real-agents plus trusted local build observation bound to invocation.",
+    "Native executables and attributable restricted node entrypoints are supported; unsupported wrappers are UNVERIFIABLE. No universal installation support or transitive attestation.",
+    "Uncertain invocations block resume; no force replay, marker stealing, automatic quota waiting, polling or provider switching.",
+    "Partial campaign work is persisted without final scientific reports; complete campaigns write the existing execution/analysis v1 artifacts.",
+    "Campaign exits: 0 consistent bounded work (including one-case partial), 1 paused/blocked/failed execution, 2 invalid CLI usage. Legacy behavior is preserved.",
+    "--campaign belongs to warm-index-reuse and is rejected by agent-success-rate; other plugins reject these campaign options."
+];
+export const AGENT_CAMPAIGN_EXAMPLES = [
+    "my-dev-kit-lab experiment run --experiment agent-success-rate --agent-campaign new --one-case --out <campaign-output-root> --campaign-source-root <source-checkout>",
+    "my-dev-kit-lab experiment run --experiment agent-success-rate --agent-campaign resume --one-case --out <same-campaign-output-root> --campaign-source-root <source-checkout>",
+    "my-dev-kit-lab experiment run --experiment agent-success-rate --agent codex --include-real-agents --agent-campaign new --timeout-preset extended --out <campaign-output-root> --campaign-source-root <source-checkout>",
+    "my-dev-kit-lab experiment run --experiment agent-success-rate --agent codex --include-real-agents --agent-campaign continue-limit --out <campaign-output-root> --campaign-source-root <source-checkout> --expected-checkpoint-revision <revision> --limit-case <case-id> --limit-treatment <treatment> --limit-epoch <epoch-id> --authorize-additional-invocation"
+];
 // ---------------------------------------------------------------------------
 // v0.4.6 Batch 4 -- reusable experiment-describe command owner.
 //
@@ -57,7 +83,29 @@ function buildDescription(plugin) {
         optionalConfigFields: readConfigFields(plugin, false),
         targetBehavior: describeTargetBehavior(plugin),
         expectedReports: describeExpectedReports(plugin),
-        examples: describeExamples(plugin)
+        examples: describeExamples(plugin),
+        ...(plugin.metadata.id === AGENT_SUCCESS_RATE_ID ? { agentSuccessRate: describeAgentSuccessRate() } : {})
+    };
+}
+const AGENT_SUCCESS_RATE_ID = "agent-success-rate";
+/** Fixed, public description of the implemented behavior. It names no task check, fact, edit scope or reference patch. */
+function describeAgentSuccessRate() {
+    return {
+        treatments: [...AGENT_SUCCESS_RATE_TREATMENT_IDS],
+        corpus: "Bundled six-task implementation corpus over two canonical benchmark projects (localized, cross-module and broad-change tasks), resolved from the installed package.",
+        defaultMode: "deterministic-fixture: validates the patch evaluation pipeline and the benchmark corpus; no coding agent runs and no provider is required.",
+        realAgentMode: "Opt-in only: --agent <codex|claude> together with --include-real-agents. Neither flag alone enables it; real execution is never the default.",
+        providers: [...AGENT_SUCCESS_RATE_REAL_AGENT_IDS],
+        timeoutMs: { default: AGENT_SUCCESS_RATE_DEFAULT_AGENT_TIMEOUT_MS, maximum: AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, scope: "each provider attempt" },
+        timeoutPresets: AGENT_TIMEOUT_PRESETS,
+        campaign: { modes: ["new", "resume", "continue-limit"], options: [...AGENT_CAMPAIGN_OPTION_HELP], examples: [...AGENT_CAMPAIGN_EXAMPLES] },
+        repairAttempts: { allowed: [...AGENT_SUCCESS_RATE_REPAIR_ATTEMPT_VALUES], default: 0, maximumTotalAttemptsPerTreatment: 3, mode: "real-agent only" },
+        agentContext: "Source-only: the provider sees the public task text and the treatment's source context, never tests, trusted checks or reference patches, and runs in a neutral working directory.",
+        patchApplication: "Guarded: the provider returns a unified diff only. The Lab validates it against a patch policy and applies it in a fresh disposable sandbox; canonical benchmark projects are never modified.",
+        taskSuccess: "Decided by trusted checks on the actual changed repository state (required behavior, no regressions, protected files intact). A provider's claim of completion is never evidence. Initial-attempt success and final success are reported separately.",
+        reportOutputs: ["report.json", "report.html", "report.txt", "agent-success-rate-execution.json", "agent-success-rate-analysis.json"],
+        targetPolicy: "Self-target only; external-local targets are rejected.",
+        interpretation: "Descriptive matched comparison of two source-context treatments on a finite corpus: no winner, ranking, composite score, causal or statistical-significance claim."
     };
 }
 function printDescription(plugin) {
@@ -84,6 +132,26 @@ function printDescription(plugin) {
     for (const example of description.examples) {
         console.log(`  ${example}`);
     }
+    const agentSuccessRate = description.agentSuccessRate; // eslint-disable-line @typescript-eslint/no-explicit-any
+    if (agentSuccessRate) {
+        console.log("");
+        console.log("Agent success rate details:");
+        console.log(`  Treatments: ${agentSuccessRate.treatments.join(", ")}`);
+        console.log(`  Corpus: ${agentSuccessRate.corpus}`);
+        console.log(`  Default mode: ${agentSuccessRate.defaultMode}`);
+        console.log(`  Real-agent mode: ${agentSuccessRate.realAgentMode}`);
+        console.log(`  Providers: ${agentSuccessRate.providers.join(", ")}`);
+        console.log(`  Timeout per attempt: default ${agentSuccessRate.timeoutMs.default} ms, maximum ${agentSuccessRate.timeoutMs.maximum} ms`);
+        for (const option of agentSuccessRate.campaign.options)
+            console.log(`  ${option}`);
+        console.log(`  Repair attempts: ${agentSuccessRate.repairAttempts.allowed.join(", ")} (default ${agentSuccessRate.repairAttempts.default}; at most ${agentSuccessRate.repairAttempts.maximumTotalAttemptsPerTreatment} attempts per treatment; real-agent mode only)`);
+        console.log(`  Agent context: ${agentSuccessRate.agentContext}`);
+        console.log(`  Patch application: ${agentSuccessRate.patchApplication}`);
+        console.log(`  Task success: ${agentSuccessRate.taskSuccess}`);
+        console.log(`  Report outputs: ${agentSuccessRate.reportOutputs.join(", ")}`);
+        console.log(`  Target policy: ${agentSuccessRate.targetPolicy}`);
+        console.log(`  Interpretation: ${agentSuccessRate.interpretation}`);
+    }
 }
 function printConfigSection(title, fields) {
     console.log(`${title}:`);
@@ -123,12 +191,18 @@ function readArrayField(value, key) {
     return Array.isArray(field) ? field : [];
 }
 function describePurpose(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Measure whether implementation patches resolve benchmark tasks under two source-context treatments (raw-full-file and context-pack) using trusted checks on disposable benchmark copies. Deterministic-fixture mode (default) validates the evaluation pipeline only; real-agent mode (opt-in) evaluates the patches a selected provider returns, with optional bounded repair.";
+    }
     if (plugin.metadata.id === "context-strategy-comparison") {
         return "Compare raw full-file prompts with my-dev-kit-guided retrieval prompts using the existing controlled experiment workflow.";
     }
     return plugin.metadata.description;
 }
 function describeTargetBehavior(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Self-target only: the bundled corpus is evaluated against the Lab itself and an explicit --target is rejected. Sandboxes and artifacts are written beneath the output directory (beneath the workspace when installed), never into the installed package or the canonical benchmark projects. Accepted options: --out, --case, --benchmark-project, --kit-command, --agent, --include-real-agents, --timeout-ms and --repair-attempts; every other option is rejected.";
+    }
     const supportsExternal = plugin.metadata.supportedTargets.includes("external-local");
     if (plugin.metadata.id === "context-window-scaling") {
         return "Three subject modes. Bundled (default): the bundled fixed scaling corpus against the Lab itself. Synthetic: caller-supplied deterministic synthetic repositories via --synthetic-config <path> (a SyntheticRepositoryConfigV1 JSON file, mutually exclusive with --case) written beneath the experiment output directory. External local repository: an explicitly selected local Git worktree via --target <path> together with --local-subject-config <path> (a LocalRepositorySubjectConfigV1 JSON file); the repository is never modified, the output directory must be outside it, and durable output omits the repository path, source text, and file names. Operational context-fit measurement only; no retrieval precision, recall, or ranking quality is measured.";
@@ -151,6 +225,9 @@ function describeTargetBehavior(plugin) {
     return "Runs against the current lab repository.";
 }
 function describeExpectedReports(plugin) {
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        return "Writes plugin-aware report.json, report.html and report.txt with a typed agent-success-rate section (initial-attempt and final success, repair history, edit quality, provider duration and tokens, scientific limitations), plus the separate agent-success-rate-execution.json and agent-success-rate-analysis.json artifacts.";
+    }
     if (plugin.metadata.supportedOutputs.includes("html") && plugin.metadata.supportedOutputs.includes("json")) {
         return "Writes plugin-aware JSON and HTML reports with plugin, target, variant, case, metric, artifact, warning, skip, and failure metadata.";
     }
@@ -164,6 +241,8 @@ function describeExamples(plugin) {
         `my-dev-kit-lab experiment describe --experiment ${plugin.metadata.id}`,
         `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id}${agentOptions}`
     ];
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID)
+        examples.push(...AGENT_CAMPAIGN_EXAMPLES);
     // Only advertise an explicit --target example for plugins that support external targets.
     if (plugin.metadata.supportedTargets.includes("external-local") &&
         plugin.metadata.id !== "context-window-scaling" &&
@@ -185,6 +264,9 @@ function describeExamples(plugin) {
     if (plugin.metadata.id === "context-pack-generation") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }
+    if (plugin.metadata.id === AGENT_SUCCESS_RATE_ID) {
+        examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --case <case-id> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --agent codex --include-real-agents --repair-attempts 2 --out <run-dir>`);
+    }
     if (plugin.metadata.id === "context-window-scaling") {
         examples.push(`my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --synthetic-config <path-to-config.json> --out <run-dir>`, `my-dev-kit-lab experiment run --experiment ${plugin.metadata.id} --target <local-git-repository> --local-subject-config <path-to-local-subject-config.json> --out <run-dir-outside-the-repository>`);
     }