@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/commands/runExperimentRunCommand.js

@@ -3,9 +3,12 @@ import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
+import { AGENT_SUCCESS_PROJECT_PROFILES_PATH, AGENT_SUCCESS_TASK_CATALOG_PATH, assertReadAgentSuccessCorpus } from "../evaluation/agentSuccess/index.js";
 import { summarizeRetrievalGroundTruthIssuesSafely, validateRetrievalPrecisionRecallCorpus } from "../evaluation/retrievalQuality/index.js";
 import { loadLocalRepositorySubject } from "../evaluation/localRepositorySubject/index.js";
 import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, retrievalPrecisionRecallPlugin, retrievalQueryStrategyComparisonPlugin, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
+import { contextPackGenerationPlugin } from "../experiments/plugins/contextPackGeneration/index.js";
+import { AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS, AGENT_SUCCESS_RATE_REAL_AGENT_IDS, agentSuccessRatePlugin, selectAgentSuccessTasks } from "../experiments/plugins/agentSuccessRate/index.js";
 import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
 import { projectExternalLocalTarget, projectRunForExternalLocalPersistence, } from "../experiments/plugins/contextWindowScaling/localSubjectPrivacy.js";
 import { assertWorkRootOutsideTarget } from "../experiments/plugins/contextWindowScaling/localSubjectScratch.js";
@@ -64,6 +67,33 @@ const RETRIEVAL_PRECISION_RECALL_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--ou
 const RETRIEVAL_QUERY_STRATEGY_COMPARISON_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
 const RETRIEVAL_QUERY_STRATEGY_COMPARISON_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
 const RETRIEVAL_QUERY_STRATEGY_COMPARISON_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
+// context-pack-generation (v0.8.2) reuses the same frozen bundled corpus. Both treatments (raw-full-file and context-pack)
+// always run, so there is no treatment or strategy option. External-local mode (--target with --local-subject-config) is a
+// different subject source: the local-subject config owns the case set, so the bundled corpus filters are not accepted there.
+const CONTEXT_PACK_GENERATION_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+const CONTEXT_PACK_GENERATION_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--benchmark-project", "--kit-command"];
+const CONTEXT_PACK_GENERATION_EXTERNAL_ALLOWED_FLAGS = ["--experiment", "--out", "--target", "--local-subject-config", "--kit-command"];
+// agent-success-rate (v0.9.0) owns the bundled six-task implementation corpus (task catalog and project profiles), both
+// resolved from the package root, never the cwd. It runs against the Lab itself only and accepts exactly these options;
+// every other flag is rejected, not ignored. Real providers run only with --agent together with --include-real-agents.
+const AGENT_SUCCESS_RATE_ALLOWED_FLAGS = [
+    "--experiment",
+    "--out",
+    "--case",
+    "--benchmark-project",
+    "--kit-command",
+    "--agent",
+    "--include-real-agents",
+    "--timeout-ms",
+    "--repair-attempts"
+];
+export function assertAgentSuccessRateFlags(argv) {
+    const flags = argv.filter((arg) => arg.startsWith("--"));
+    const unsupported = [...new Set(flags.filter((flag) => !AGENT_SUCCESS_RATE_ALLOWED_FLAGS.includes(flag)))];
+    if (unsupported.length > 0) {
+        throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${agentSuccessRatePlugin.metadata.id}; supported options: ${AGENT_SUCCESS_RATE_ALLOWED_FLAGS.join(", ")}.`);
+    }
+}
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
     try {
         const args = parseRunExperimentArgs(argv);
@@ -144,7 +174,8 @@ export async function runExperimentRunCommandFromArgs(argv, options = {}) {
                 `Target root: ${result.target.targetRoot}`,
                 `Output: ${String(result.metadata?.outputRoot ?? "")}`,
                 `Report JSON: ${reports.outputPaths.jsonPath}`,
-                `Report HTML: ${reports.outputPaths.htmlPath}`
+                `Report HTML: ${reports.outputPaths.htmlPath}`,
+                ...(args.experimentId === agentSuccessRatePlugin.metadata.id ? agentSuccessRateSummaryLines(result, reports.outputPaths.textPath) : [])
             ];
         // Automatic presentation (report already produced above) applies only to warm-index-reuse
         // campaign runs; legacy fake-agent warm-index runs and context-strategy-comparison stay
@@ -218,8 +249,14 @@ export function parseRunExperimentArgs(argv) {
     let contextBudgets;
     let syntheticConfigPath;
     let localSubjectConfigPath;
+    const agentFlagValues = [];
+    let repairAttempts;
     const seenFlags = [];
     const commandTemplates = {};
+    // The flag whitelist runs before any value is interpreted so an unsupported flag is reported as such.
+    const preScanIndex = argv.indexOf("--experiment");
+    if (preScanIndex >= 0 && argv[preScanIndex + 1] === agentSuccessRatePlugin.metadata.id)
+        assertAgentSuccessRateFlags(argv);
     for (let index = 0; index < argv.length; index += 1) {
         const arg = argv[index];
         if (arg !== undefined && arg.startsWith("--"))
@@ -257,6 +294,12 @@ export function parseRunExperimentArgs(argv) {
         else if (arg === "--benchmark-project") {
             benchmarkProjects.push(...splitList(readRequiredValue(argv, ++index, "--benchmark-project")));
         }
+        else if (arg === "--agent") {
+            agentFlagValues.push(readRequiredValue(argv, ++index, "--agent"));
+        }
+        else if (arg === "--repair-attempts") {
+            repairAttempts = parseRepairAttempts(readRequiredValue(argv, ++index, "--repair-attempts"));
+        }
         else if (arg === "--agents") {
             agents = splitList(readRequiredValue(argv, ++index, "--agents")).map((value) => parseAgentId(value));
         }
@@ -315,7 +358,8 @@ export function parseRunExperimentArgs(argv) {
         incrementalChangeStalenessPlugin.metadata.id,
         contextWindowScalingPlugin.metadata.id,
         retrievalPrecisionRecallPlugin.metadata.id,
-        retrievalQueryStrategyComparisonPlugin.metadata.id
+        retrievalQueryStrategyComparisonPlugin.metadata.id,
+        contextPackGenerationPlugin.metadata.id
     ];
     if (experimentId === retrievalPrecisionRecallPlugin.metadata.id) {
         // Mode matrix, no precedence rules: neither flag = bundled; both = external-local; exactly one is rejected.
@@ -367,6 +411,73 @@ export function parseRunExperimentArgs(argv) {
             throw new Error("--benchmark-project must list at least one benchmark project id.");
         }
     }
+    if (experimentId === contextPackGenerationPlugin.metadata.id) {
+        // Mode matrix, no precedence rules: neither flag = bundled; both = external-local; exactly one is rejected.
+        if (targetPath !== undefined && localSubjectConfigPath === undefined) {
+            throw new Error(`External ${experimentId} targets require --local-subject-config.`);
+        }
+        if (localSubjectConfigPath !== undefined && targetPath === undefined) {
+            throw new Error(`--local-subject-config requires an external --target for ${experimentId}.`);
+        }
+        const externalMode = targetPath !== undefined;
+        if (externalMode && (seenFlags.includes("--case") || seenFlags.includes("--benchmark-project"))) {
+            throw new Error("--case and --benchmark-project cannot be combined with --local-subject-config; the local subject config owns the case set.");
+        }
+        const allowedFlags = externalMode ? CONTEXT_PACK_GENERATION_EXTERNAL_ALLOWED_FLAGS : CONTEXT_PACK_GENERATION_ALLOWED_FLAGS;
+        const unsupported = [...new Set(seenFlags.filter((flag) => !allowedFlags.includes(flag)))];
+        if (unsupported.length > 0) {
+            throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${experimentId}${externalMode ? " in external-local mode" : ""}; supported options: ${allowedFlags.join(", ")}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
+    if (experimentId === agentSuccessRatePlugin.metadata.id) {
+        // Mode matrix: no flags = deterministic-fixture (default); --agent together with --include-real-agents = real-agent mode.
+        // Everything else is rejected, never coerced: a provider needs both flags, and real-agent options need a provider.
+        if (agentFlagValues.length > 1) {
+            throw new Error("--agent may be supplied only once; agent-success-rate runs exactly one provider.");
+        }
+        const agentId = agentFlagValues[0];
+        if (agentId !== undefined && !AGENT_SUCCESS_RATE_REAL_AGENT_IDS.includes(agentId)) {
+            throw new Error(`--agent must be exactly one of: ${AGENT_SUCCESS_RATE_REAL_AGENT_IDS.join(", ")}.`);
+        }
+        if (agentId !== undefined && includeRealAgents !== true) {
+            throw new Error("--agent requires --include-real-agents; real providers are invoked only with explicit authorization.");
+        }
+        if (agentId === undefined && includeRealAgents === true) {
+            throw new Error("--include-real-agents requires --agent <codex|claude> for agent-success-rate.");
+        }
+        if (agentId === undefined && timeoutMs !== undefined) {
+            throw new Error("--timeout-ms is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && kitCommand !== undefined) {
+            throw new Error("--kit-command is only supported together with --agent and --include-real-agents (real-agent mode).");
+        }
+        if (agentId === undefined && repairAttempts !== undefined && repairAttempts > 0) {
+            throw new Error("--repair-attempts greater than 0 requires --agent and --include-real-agents (real-agent mode).");
+        }
+        if (timeoutMs !== undefined && timeoutMs > AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS) {
+            throw new Error(`--timeout-ms must be a positive integer no greater than ${AGENT_SUCCESS_RATE_MAX_AGENT_TIMEOUT_MS}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (seenFlags.includes("--benchmark-project") && benchmarkProjects.length === 0) {
+            throw new Error("--benchmark-project must list at least one benchmark project id.");
+        }
+    }
+    else {
+        if (agentFlagValues.length > 0) {
+            throw new Error(`--agent is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+        if (repairAttempts !== undefined) {
+            throw new Error(`--repair-attempts is only supported for --experiment ${agentSuccessRatePlugin.metadata.id}.`);
+        }
+    }
     if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
         throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
     }
@@ -376,8 +487,9 @@ export function parseRunExperimentArgs(argv) {
     if (localSubjectConfigPath !== undefined &&
         experimentId !== contextWindowScalingPlugin.metadata.id &&
         experimentId !== retrievalPrecisionRecallPlugin.metadata.id &&
-        experimentId !== retrievalQueryStrategyComparisonPlugin.metadata.id) {
-        throw new Error(`--local-subject-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}, ${retrievalPrecisionRecallPlugin.metadata.id}, or ${retrievalQueryStrategyComparisonPlugin.metadata.id}.`);
+        experimentId !== retrievalQueryStrategyComparisonPlugin.metadata.id &&
+        experimentId !== contextPackGenerationPlugin.metadata.id) {
+        throw new Error(`--local-subject-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}, ${retrievalPrecisionRecallPlugin.metadata.id}, ${retrievalQueryStrategyComparisonPlugin.metadata.id}, or ${contextPackGenerationPlugin.metadata.id}.`);
     }
     if (experimentId === contextWindowScalingPlugin.metadata.id) {
         const unsupported = [...new Set(seenFlags.filter((flag) => !CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS.includes(flag)))];
@@ -408,7 +520,8 @@ export function parseRunExperimentArgs(argv) {
             throw new Error(`External ${experimentId} targets require --local-subject-config.`);
         }
     }
-    if (kitCommand !== undefined && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
+    // agent-success-rate accepts --kit-command in real-agent mode; its own validation above owns that rule and message.
+    if (kitCommand !== undefined && experimentId !== agentSuccessRatePlugin.metadata.id && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
         throw new Error(`--kit-command is only supported for --experiment ${KIT_COMMAND_PLUGIN_IDS.join(" or ")}.`);
     }
     if (campaignPreset !== undefined) {
@@ -445,6 +558,8 @@ export function parseRunExperimentArgs(argv) {
             requireAgents,
             includeRealAgents,
             commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
+            agentId: agentFlagValues[0],
+            repairAttempts,
             kitCommand,
             campaignPreset,
             contextBudgets
@@ -499,13 +614,86 @@ async function loadPluginInputs(args, toolRoot, context, outputRoot) {
             ? loadRetrievalPrecisionRecallLocalSubjectInputs(args, toolRoot, context, outputRoot)
             : loadRetrievalPrecisionRecallInputs(args, toolRoot, context);
     }
+    if (args.experimentId === contextPackGenerationPlugin.metadata.id) {
+        return args.localSubjectConfigPath !== undefined
+            ? loadContextPackGenerationLocalSubjectInputs(args, toolRoot, context, outputRoot)
+            : loadContextPackGenerationInputs(args, toolRoot, context);
+    }
     if (args.experimentId === retrievalQueryStrategyComparisonPlugin.metadata.id) {
         return args.localSubjectConfigPath !== undefined
             ? loadRetrievalQueryStrategyComparisonLocalSubjectInputs(args, toolRoot, context, outputRoot)
             : loadRetrievalQueryStrategyComparisonInputs(args, toolRoot, context);
     }
+    if (args.experimentId === agentSuccessRatePlugin.metadata.id) {
+        return loadAgentSuccessRateInputs(args, context);
+    }
     return undefined;
 }
+// agent-success-rate: the bundled task catalog and project profiles are located through the established package-resource
+// resolver (never the cwd) and validated by the corpus reader, which also checks the canonical projects on disk. The
+// validated tasks reach the plugin through its existing programmatic input; the plugin applies --case/--benchmark-project
+// selection in catalog order and owns execution, repair, science and artifacts. No task file is accepted from the CLI.
+async function loadAgentSuccessRateInputs(args, context) {
+    const validation = agentSuccessRatePlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid agent success rate config: ${validation.errors.join("; ")}`);
+    }
+    resolvePackageResource(context, AGENT_SUCCESS_TASK_CATALOG_PATH);
+    resolvePackageResource(context, AGENT_SUCCESS_PROJECT_PROFILES_PATH);
+    const corpus = assertReadAgentSuccessCorpus(context.resourceRoot);
+    // Fail on unknown case or project ids before any output directory exists.
+    selectAgentSuccessTasks(corpus.tasks, validation.config);
+    return { agentSuccessTasks: corpus.tasks };
+}
+// context-pack-generation (bundled/self): the frozen warm-index corpus through the established resource resolver, profile
+// and corpus validation. The plugin applies --case/--benchmark-project selection and owns execution, science and artifacts.
+async function loadContextPackGenerationInputs(args, toolRoot, context) {
+    const validation = contextPackGenerationPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid context pack generation config: ${validation.errors.join("; ")}`);
+    }
+    const projectProfiles = await readBenchmarkProjectProfiles(resolvePackageResource(context, DEFAULT_PROJECT_PROFILES_RESOURCE), toolRoot);
+    const cases = await readEvaluationCases(resolvePackageResource(context, CONTEXT_PACK_GENERATION_CASES_RESOURCE), toolRoot, {
+        projectProfiles,
+        requireProjectProfileRef: true
+    });
+    const corpusErrors = validateRetrievalPrecisionRecallCorpus(cases);
+    if (corpusErrors.length > 0) {
+        throw new Error(`Invalid context-pack-generation corpus: ${corpusErrors.join(" ")}`);
+    }
+    return { cases };
+}
+// External-local context-pack-generation: same loader and safety order as the other external-local retrieval experiments,
+// with the plugin's own config validation. Ground-truth issues carry logical case ids and counts only.
+async function loadContextPackGenerationLocalSubjectInputs(args, toolRoot, context, outputRoot) {
+    const validation = contextPackGenerationPlugin.validateConfig(args.config);
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid context pack generation config: ${validation.errors.join("; ")}`);
+    }
+    if (outputRoot === undefined || args.localSubjectConfigPath === undefined) {
+        throw new Error("Internal error: external-local runs require a resolved experiment output root and config path.");
+    }
+    const target = resolveExperimentTarget(args.targetPath, toolRoot);
+    if (target.isSelf || target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target; --target must not be the Lab repository.");
+    }
+    try {
+        await assertWorkRootOutsideTarget(outputRoot, target.targetRoot);
+    }
+    catch {
+        throw new Error("Experiment output root must not be inside the external target project.");
+    }
+    const configPath = path.resolve(context.invocationCwd, args.localSubjectConfigPath);
+    const localSubjectConfig = await readJsonConfigFile("--local-subject-config", configPath, args.localSubjectConfigPath);
+    const localSubject = await loadLocalRepositorySubject({ config: localSubjectConfig, repositoryPath: target.targetRoot });
+    const groundTruthIssues = summarizeRetrievalGroundTruthIssuesSafely(localSubject.evaluationCases);
+    if (groundTruthIssues.length > 0) {
+        throw new Error(`Invalid context-pack-generation ground truth for the local subject (${groundTruthIssues
+            .map((issue) => `case ${issue.caseId}: ${issue.issueCount} issue${issue.issueCount === 1 ? "" : "s"}`)
+            .join("; ")}); details withheld.`);
+    }
+    return { cases: localSubject.evaluationCases, localSubject, env: process.env };
+}
 // External-local retrieval-query-strategy-comparison: same loader and safety order as retrieval-precision-recall, with
 // the plugin's own config validation. Ground-truth issues carry logical case ids and counts only.
 async function loadRetrievalQueryStrategyComparisonLocalSubjectInputs(args, toolRoot, context, outputRoot) {
@@ -728,6 +916,12 @@ function readRequiredValue(argv, index, label) {
     }
     return value;
 }
+function parseRepairAttempts(value) {
+    if (!/^[0-2]$/.test(value)) {
+        throw new Error("--repair-attempts must be 0, 1 or 2.");
+    }
+    return Number(value);
+}
 function parsePositiveInteger(label, value) {
     const parsed = Number(value);
     if (!Number.isInteger(parsed) || parsed <= 0) {
@@ -735,6 +929,15 @@ function parsePositiveInteger(label, value) {
     }
     return parsed;
 }
+function agentSuccessRateSummaryLines(result, textPath) {
+    const mode = readMetadataString(result.metadata?.executionMode) ?? "deterministic-fixture";
+    const provider = readMetadataString(result.metadata?.providerId);
+    return [
+        `Report Text: ${textPath}`,
+        `Execution mode: ${mode}`,
+        ...(provider ? [`Provider: ${provider} (invoked by explicit --include-real-agents authorization)`] : ["Provider: none (deterministic-fixture; no coding agent was invoked)"])
+    ];
+}
 function readMetadataString(value) {
     return typeof value === "string" && value.trim() ? value : undefined;
 }

package/dist/src/core/processTree.js

@@ -20,6 +20,61 @@ export function requestProcessTermination(pid) {
         // The process may have exited between the liveness check and this signal.
     }
 }
+/**
+ * Opt-in, awaitable descendant-tree termination for bounded execution
+ * (see `terminateProcessTree` in `runMeasuredCommand`). The legacy
+ * `forceTerminateProcess` above is intentionally unchanged: it stays
+ * fire-and-forget and, off Windows, signals one pid only.
+ *
+ * Windows: `taskkill /pid <pid> /T /F` (shell:false) is awaited, bounded by
+ * `waitTimeoutMs`, so the caller knows the tree was asked to die before it
+ * proceeds.
+ *
+ * Other platforms: the child must have been spawned `detached` so it leads its
+ * own process group; the whole group is signalled via the negative pid. If no
+ * group exists, the single pid is signalled as a fallback.
+ *
+ * Never throws: the target may already have exited.
+ */
+export async function terminateProcessTree(pid, options = {}) {
+    if (!pid) {
+        return;
+    }
+    const platform = options.platform ?? process.platform;
+    if (platform === "win32") {
+        await new Promise((resolve) => {
+            let timer;
+            const finish = () => {
+                if (timer)
+                    clearTimeout(timer);
+                resolve();
+            };
+            const killer = spawn("taskkill", ["/pid", String(pid), "/T", "/F"], {
+                shell: false,
+                stdio: "ignore",
+                windowsHide: true
+            });
+            timer = setTimeout(finish, options.waitTimeoutMs ?? 5000);
+            killer.on("error", finish);
+            killer.on("close", finish);
+        });
+        return;
+    }
+    const signal = options.posixSignal ?? "SIGKILL";
+    try {
+        process.kill(-pid, signal);
+        return;
+    }
+    catch {
+        // No process group for this pid (or it already exited); fall back to the single pid.
+    }
+    try {
+        process.kill(pid, signal);
+    }
+    catch {
+        // The process may have exited between the liveness check and this signal.
+    }
+}
 /**
  * Forced cleanup for a process that did not stop on request.
  *

package/dist/src/core/runMeasuredCommand.js

@@ -1,8 +1,9 @@
+import { existsSync } from "node:fs";
 import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { spawn } from "node:child_process";
 import { parseCommandString, serializeCommand } from "./commandLine.js";
-import { forceTerminateProcess } from "./processTree.js";
+import { forceTerminateProcess, terminateProcessTree } from "./processTree.js";
 import { buildResolvedCommandInvocation, resolveCommandInvocation } from "./resolveCommand.js";
 export { parseCommandString } from "./commandLine.js";
 export async function runMeasuredCommand(options) {
@@ -10,6 +11,11 @@ export async function runMeasuredCommand(options) {
     const named = readCommandName(options);
     const commandString = named.commandString;
     const trailingArgs = [...named.args, ...(options.extraArgs ?? [])];
+    const bounded = options.stdoutMaxBytes !== undefined ||
+        options.stderrMaxBytes !== undefined ||
+        options.inheritParentEnv === false ||
+        options.terminateProcessTree === true;
+    const childEnv = options.inheritParentEnv === false ? { ...options.env } : { ...process.env, ...options.env };
     // Windows .cmd/.bat/.ps1 argument assembly is owned by resolveCommand.ts so
     // that every spawn site (measured commands and managed long-running processes)
     // shares exactly one shim rule.
@@ -24,7 +30,7 @@ export async function runMeasuredCommand(options) {
         }, trailingArgs)
         : resolveCommandInvocation(named.executable, trailingArgs, {
             cwd: options.cwd,
-            env: { ...process.env, ...options.env },
+            env: childEnv,
             allowPowerShellShim: options.allowPowerShellShim
         });
     const resolution = invocation.resolvedCommand;
@@ -36,8 +42,9 @@ export async function runMeasuredCommand(options) {
     const startedAt = new Date().toISOString();
     const started = Date.now();
     const result = await new Promise((resolve) => {
-        let stdout = "";
-        let stderr = "";
+        const stdoutCollector = new OutputCollector(options.stdoutMaxBytes);
+        const stderrCollector = new OutputCollector(options.stderrMaxBytes);
+        let outputLimit;
         let spawnError;
         let timedOut = false;
         let timeout;
@@ -47,9 +54,12 @@ export async function runMeasuredCommand(options) {
         try {
             child = spawn(executable, args, {
                 cwd: options.cwd,
-                env: { ...process.env, ...options.env },
+                env: childEnv,
                 shell: false,
-                stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"]
+                stdio: options.stdinText !== undefined ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"],
+                // Own process group so a bounded termination can reach descendants. Windows reaches the tree
+                // through `taskkill /T`, and `detached` there would open a new console, so it is not set.
+                ...(options.terminateProcessTree === true && process.platform !== "win32" ? { detached: true } : {})
             });
         }
         catch (error) {
@@ -66,8 +76,8 @@ export async function runMeasuredCommand(options) {
                 endedAt,
                 durationMs,
                 exitCode: null,
-                stdout,
-                stderr,
+                stdout: "",
+                stderr: "",
                 stdoutPath,
                 stderrPath,
                 telemetryPath,
@@ -76,22 +86,53 @@ export async function runMeasuredCommand(options) {
                 resolvedCommand: resolution
             };
             void Promise.all([
-                writeArtifact(stdoutPath, stdout),
-                writeArtifact(stderrPath, stderr),
+                writeArtifact(stdoutPath, ""),
+                writeArtifact(stderrPath, ""),
                 writeArtifact(telemetryPath, JSON.stringify(measured, null, 2))
             ]).then(() => resolve(measured));
             return;
         }
         // stdout/stderr are always piped in both stdio branches above.
-        child.stdout.on("data", (chunk) => {
-            stdout += String(chunk);
-        });
-        child.stderr.on("data", (chunk) => {
-            stderr += String(chunk);
-        });
+        // The process-termination step is shared by the timeout and the opt-in output limit. Bounded callers
+        // get an awaited tree termination; everyone else keeps the legacy fire-and-forget single termination.
+        const KILL_GRACE_MS = 3000;
+        const terminate = () => {
+            if (options.terminateProcessTree === true) {
+                void terminateProcessTree(child.pid);
+            }
+            else {
+                forceTerminateProcess(child.pid);
+            }
+            killGraceTimeout ??= setTimeout(() => {
+                void settle(null);
+            }, KILL_GRACE_MS);
+        };
+        const collect = (collector, stream, limit) => (chunk) => {
+            if (collector.push(chunk) && limit !== undefined && outputLimit === undefined) {
+                outputLimit = { stream, limitBytes: limit };
+                spawnError = `Command ${stream} exceeded ${limit} bytes.`;
+                terminate();
+            }
+        };
+        child.stdout.on("data", collect(stdoutCollector, "stdout", options.stdoutMaxBytes));
+        child.stderr.on("data", collect(stderrCollector, "stderr", options.stderrMaxBytes));
         child.on("error", (error) => {
             spawnError = error.message;
+            // A failed spawn in a working directory that exists is not a missing executable; the OS may be refusing the path (for
+            // example a very long cwd on Windows). Report the measured length
+            // without asserting a limit of our own.
+            if (error.code === "ENOENT" && existsSync(options.cwd)) {
+                spawnError += ` (the working directory exists but the process could not be started in it; cwd length ${options.cwd.length} characters)`;
+            }
         });
+        // A failed spawn can also make the OS report the never-connected stdout/stderr pipes as errors (Windows:
+        // read ENOTCONN). They are stream events of this one failed process: record the first cause once and let
+        // the 'close' event settle the result, instead of letting an unhandled 'error' event end the whole process.
+        const recordStreamError = (stream) => (error) => {
+            spawnError = spawnError ?? `Failed to read command ${stream}: ${error.message}`;
+        };
+        child.stdout.on("error", recordStreamError("stdout"));
+        child.stderr.on("error", recordStreamError("stderr"));
         if (options.stdinText !== undefined) {
             // A child that exits or closes stdin before we write must not crash the process (EPIPE).
             child.stdin?.on("error", (error) => {
@@ -106,7 +147,7 @@ export async function runMeasuredCommand(options) {
         // intermediate shim (for example a Windows .cmd launcher) can occasionally outlive an
         // already-killed intermediate process. Without a bounded fallback, a caller whose kill silently
         // failed would await this promise forever instead of receiving the already-known timeout error.
-        const KILL_GRACE_MS = 3000;
+        // (KILL_GRACE_MS and the shared `terminate` step are declared above, beside the output collectors.)
         const settle = async (exitCode) => {
             if (settled)
                 return;
@@ -128,6 +169,8 @@ export async function runMeasuredCommand(options) {
             child.unref();
             const endedAt = new Date().toISOString();
             const durationMs = Date.now() - started;
+            const stdout = stdoutCollector.toString();
+            const stderr = stderrCollector.toString();
             await writeArtifact(stdoutPath, stdout);
             await writeArtifact(stderrPath, stderr);
             const measured = {
@@ -147,7 +190,8 @@ export async function runMeasuredCommand(options) {
                 telemetryPath,
                 ok: exitCode === 0 && !spawnError && !timedOut,
                 error: spawnError,
-                resolvedCommand: resolution
+                resolvedCommand: resolution,
+                ...(bounded ? { timedOut, ...(outputLimit ? { outputLimit } : {}) } : {})
             };
             await writeArtifact(telemetryPath, JSON.stringify(measured, null, 2));
             resolve(measured);
@@ -156,10 +200,7 @@ export async function runMeasuredCommand(options) {
             timeout = setTimeout(() => {
                 timedOut = true;
                 spawnError = `Command timed out after ${options.timeoutMs}ms.`;
-                forceTerminateProcess(child.pid);
-                killGraceTimeout = setTimeout(() => {
-                    void settle(null);
-                }, KILL_GRACE_MS);
+                terminate();
             }, options.timeoutMs);
         }
         child.on("close", (exitCode) => {
@@ -168,6 +209,47 @@ export async function runMeasuredCommand(options) {
     });
     return result;
 }
+/**
+ * Accumulates one output stream. Without a limit it reproduces the legacy `text += String(chunk)`
+ * behavior exactly. With a limit it keeps raw bytes (so the cut is on a byte boundary), stops at the
+ * limit, and reports once when more output than the limit arrived.
+ */
+class OutputCollector {
+    limit;
+    text = "";
+    chunks = [];
+    bytes = 0;
+    exceeded = false;
+    constructor(limit) {
+        this.limit = limit;
+    }
+    /** Returns true exactly once: on the chunk that first exceeds the limit. */
+    push(chunk) {
+        if (this.limit === undefined) {
+            this.text += String(chunk);
+            return false;
+        }
+        if (this.exceeded) {
+            return false;
+        }
+        const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk), "utf8");
+        const remaining = this.limit - this.bytes;
+        if (buffer.length > remaining) {
+            if (remaining > 0) {
+                this.chunks.push(buffer.subarray(0, remaining));
+            }
+            this.bytes = this.limit;
+            this.exceeded = true;
+            return true;
+        }
+        this.chunks.push(buffer);
+        this.bytes += buffer.length;
+        return false;
+    }
+    toString() {
+        return this.limit === undefined ? this.text : Buffer.concat(this.chunks).toString("utf8");
+    }
+}
 async function writeArtifact(filePath, value) {
     await mkdir(path.dirname(filePath), { recursive: true });
     await writeFile(filePath, value, "utf8");

package/dist/src/evaluation/agentSuccess/applyPatchToSandbox.js

@@ -0,0 +1,37 @@
+import { excerptGitFailure, runSandboxGit } from "../benchmarkSandbox/index.js";
+import { extractPatchCandidate } from "./extractPatchCandidate.js";
+import { parseUnifiedDiff } from "./parseUnifiedDiff.js";
+import { DEFAULT_PATCH_BOUNDS } from "./patchTypes.js";
+import { validatePatchPolicy } from "./validatePatchPolicy.js";
+const APPLY_FLAGS = ["--index", "--whitespace=nowarn", "-"];
+/**
+ * Lab-owned patch application. The order is the safety contract:
+ * extract exactly one candidate -> parse -> policy -> `git apply --check --index` -> `git apply --index`.
+ * Nothing touches the sandbox until the check succeeds, so every earlier failure leaves it unchanged.
+ * `--unsafe-paths`, `--reject` and `--3way` are never used, and nothing the agent proposed is executed.
+ */
+export async function applyPatchToSandbox(options) {
+    const bounds = options.bounds ?? DEFAULT_PATCH_BOUNDS;
+    const extracted = extractPatchCandidate(options.rawProposal, bounds);
+    if (!extracted.ok)
+        return { outcome: "parse-failure", code: extracted.code, message: extracted.message };
+    const parsed = parseUnifiedDiff(extracted.patch);
+    if (!parsed.ok)
+        return { outcome: "parse-failure", code: parsed.code, message: parsed.message };
+    const policy = validatePatchPolicy(parsed.files, { protectedFiles: options.protectedFiles, bounds });
+    if (!policy.ok)
+        return { outcome: "policy-rejection", rejections: policy.rejections };
+    const check = await runSandboxGit(options.sandbox, ["apply", "--check", ...APPLY_FLAGS], {
+        stdin: extracted.patch,
+        label: "apply-check"
+    });
+    if (!check.ok)
+        return { outcome: "git-check-failure", message: excerptGitFailure(check) };
+    const apply = await runSandboxGit(options.sandbox, ["apply", ...APPLY_FLAGS], {
+        stdin: extracted.patch,
+        label: "apply"
+    });
+    if (!apply.ok)
+        return { outcome: "git-apply-failure", message: excerptGitFailure(apply) };
+    return { outcome: "success", files: policy.files };
+}

package/dist/src/evaluation/agentSuccess/assessBaseline.js

@@ -0,0 +1,23 @@
+/**
+ * Decides whether the unmodified benchmark can evaluate an implementation.
+ *
+ * Evaluable exactly when at least one task check fails (there is something to fix) and every regression
+ * check passes (there is nothing already broken), and no check was unable to produce a verdict. A check
+ * that timed out or errored is "not evaluable" rather than "failed": an indeterminate check cannot prove
+ * the task is unsolved. An invalid baseline is a benchmark defect, never an agent failure.
+ */
+export function assessBaseline(baseline) {
+    const reasons = [];
+    const all = [...baseline.taskResults, ...baseline.regressionResults];
+    if (baseline.taskResults.length === 0 || baseline.regressionResults.length === 0)
+        reasons.push("NO_CHECKS");
+    if (all.some((result) => result.status === "timeout" || result.status === "error"))
+        reasons.push("CHECK_NOT_EVALUABLE");
+    if (baseline.taskResults.length > 0 && !baseline.taskResults.some((result) => result.status === "failed")) {
+        if (!reasons.includes("CHECK_NOT_EVALUABLE"))
+            reasons.push("ALL_TASK_CHECKS_PASSED");
+    }
+    if (baseline.regressionResults.some((result) => result.status === "failed"))
+        reasons.push("REGRESSION_CHECK_FAILED");
+    return { evaluable: reasons.length === 0, reasons };
+}