@brainbase-labs/cli 0.25.0-eng1209.8 → 0.25.0-eng1209.9

package/README.md

@@ -72,21 +72,28 @@ synchronously. Schema v1 supports:
 - `workspace_assertion`: file existence, absence, SHA-256, or content checks.
 - `sandbox_command`: a bounded argv command rooted in the workspace or hidden
   tests directory. Exit zero passes; any other exit code is a valid failed
-  verdict. Schema v1 allows one command evaluator and runs it after all
-  read-only assertions so evaluator mutations cannot change candidate verdicts.
+  verdict. Schema v1 allows up to 20 command evaluators and runs them after all
+  read-only assertions. Each command receives either the verified live workspace
+  for legacy read-only evaluation or its own isolated frozen workspace copy.
 
 Valid failed verdicts still produce a successful evaluation phase. Invalid
-specs, digest/path violations, missing environment, launch failures, timeouts,
-and output-budget violations fail the phase. Results are written atomically
+specs, digest/path violations, missing environment, phase-budget exhaustion,
+and output-budget violations fail the phase. Individual command launch failures,
+timeouts, and forced terminations are recorded as errored evaluator results so
+the remaining evaluators can still run. Results are written atomically
 with mode `0600`, include the raw spec digest and checksummed evidence/log
 input/output references with explicit `staging`, `workspace`, `tests`, or
 `logs` roots, redact declared secret values from command logs, require the
 result path to be `<logs_root>/result.json`, and treat a matching successful
 phase result as authoritative on replay.
 
-On any phase-level execution failure, MAS must tear down the task sandbox. The
-CLI kills the command process group and descendants it can observe, but the
-runtime lifecycle remains the final cleanup boundary for daemonized processes.
+On any phase-level execution failure, MAS must tear down the task sandbox.
+Failed and timed-out commands clean up the process group and descendants the
+CLI can observe. Successful evaluator commands receive the same cleanup so one
+check cannot contaminate the next. Successful hydration setup commands may
+intentionally leave services running for the agent turn; the runtime lifecycle
+remains their final cleanup boundary. Isolated evaluator workspace copies omit
+`.git` and unsafe absolute or escaping symlinks.
 
 ## Development
 

package/dist/index.js

@@ -36008,7 +36008,7 @@ function padStart(s, n) {
 // package.json
 var package_default = {
   name: "@brainbase-labs/cli",
-  version: "0.25.0-eng1209.8",
+  version: "0.25.0-eng1209.9",
   description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   type: "module",
   bin: {
@@ -78487,7 +78487,8 @@ var MAX_ARCHIVE_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024;
 var MAX_ARCHIVE_SCAN_BYTES = 2 * 1024 * 1024 * 1024;
 var MAX_SANDBOX_COMMANDS = 20;
 var MAX_CRITERIA_PER_EVALUATOR = 200;
-var MAX_CRITERIA_RESULT_BYTES = 1024 * 1024;
+var MAX_CRITERIA_RESULT_BYTES = 24 * 1024 * 1024;
+var MAX_CRITERIA_RESULT_PAYLOAD_BYTES = 20 * 1024 * 1024;
 var MAX_CRITERION_EXPLANATION_LENGTH = 16384;
 var MAX_CRITERION_EVIDENCE_IDS = 100;
 var MAX_CRITERION_EVIDENCE_ID_LENGTH = 256;
@@ -80052,6 +80053,18 @@ function markedProcessPids(marker) {
   return [];
 }
 function processExists(pid) {
+  if (process.platform === "linux") {
+    try {
+      const stat = fs81.readFileSync(path88.join("/proc", String(pid), "stat"), "utf8");
+      const commandEnd = stat.lastIndexOf(")");
+      const state = commandEnd >= 0 ? stat.slice(commandEnd + 2, commandEnd + 3) : "";
+      if (state === "Z" || state === "X")
+        return false;
+    } catch (error2) {
+      if (error2.code === "ENOENT")
+        return false;
+    }
+  }
   try {
     process.kill(pid, 0);
     return true;
@@ -80083,7 +80096,7 @@ async function terminateCommandProcesses(child, marker, observedDescendants) {
     await new Promise((resolve) => setTimeout(resolve, COMMAND_PROCESS_POLL_MS));
   }
 }
-async function runCommand(command, root, spec, context, additions = {}) {
+async function runCommand(command, root, spec, context, options = {}) {
   const cwdRel = normalizedRootRelative(command.cwd);
   assertNoSymlinkTraversal(root, cwdRel);
   const cwd2 = path88.resolve(root, cwdRel);
@@ -80107,7 +80120,7 @@ async function runCommand(command, root, spec, context, additions = {}) {
     const child = spawn4(command.argv[0], command.argv.slice(1), {
       cwd: cwd2,
       env: buildEnvironment(spec, command.secret_env, {
-        ...additions,
+        ...options.additions,
         [COMMAND_PROCESS_MARKER_ENV]: commandMarker
       }),
       stdio: ["ignore", "pipe", "pipe"],
@@ -80163,13 +80176,23 @@ async function runCommand(command, root, spec, context, additions = {}) {
       }
       target.push(chunk2);
     };
+    const finish = (code, signal) => {
+      if (observer)
+        clearInterval(observer);
+      if (code === null) {
+        reject2(new BenchmarkCommandExecutionError("command_terminated", `command was terminated by ${signal ?? "an unknown signal"}: ${command.id}`, sanitizedOutput(stdout), sanitizedOutput(stderr), Date.now() - started));
+        return;
+      }
+      resolve({
+        exitCode: code,
+        stdout: sanitizedOutput(stdout),
+        stderr: sanitizedOutput(stderr),
+        durationMs: Date.now() - started,
+        redactions: [commandMarker]
+      });
+    };
     child.stdout?.on("data", (chunk2) => capture(stdout, chunk2));
     child.stderr?.on("data", (chunk2) => capture(stderr, chunk2));
-    child.once("exit", () => {
-      if (timer)
-        clearTimeout(timer);
-      cleanupProcesses();
-    });
     child.on("error", (error2) => {
       fail(new BenchmarkCommandExecutionError("command_start_failed", `failed to start ${command.id}: ${error2.message}`, Buffer.concat(stdout), Buffer.concat(stderr), Date.now() - started));
     });
@@ -80180,24 +80203,21 @@ async function runCommand(command, root, spec, context, additions = {}) {
       }
       fail(new BenchmarkCommandExecutionError("command_timeout", `command timed out: ${command.id}`, Buffer.concat(stdout), Buffer.concat(stderr), Date.now() - started));
     }, timeoutMs2);
-    child.on("close", (code, signal) => {
+    child.on("exit", (code, signal) => {
       if (settled)
         return;
       settled = true;
       clearTimeout(timer);
-      cleanupProcesses().then(() => {
-        if (code === null) {
-          reject2(new BenchmarkCommandExecutionError("command_terminated", `command was terminated by ${signal ?? "an unknown signal"}: ${command.id}`, sanitizedOutput(stdout), sanitizedOutput(stderr), Date.now() - started));
-          return;
-        }
-        resolve({
-          exitCode: code,
-          stdout: sanitizedOutput(stdout),
-          stderr: sanitizedOutput(stderr),
-          durationMs: Date.now() - started,
-          redactions: [commandMarker]
-        });
-      }, reject2);
+      if (options.descendantCleanup === "always" || code === null || code !== 0) {
+        cleanupProcesses().then(() => finish(code, signal), reject2);
+      } else {
+        if (observer)
+          clearInterval(observer);
+        child.unref();
+        child.stdout?.unref?.();
+        child.stderr?.unref?.();
+        finish(code, signal);
+      }
     });
   });
 }
@@ -80275,7 +80295,9 @@ async function executeHydrate(spec, context) {
     assertBudget(context);
     let result2;
     try {
-      result2 = await runCommand(command, spec.workspace_root, spec, context);
+      result2 = await runCommand(command, spec.workspace_root, spec, context, {
+        descendantCleanup: "failure_only"
+      });
     } catch (error2) {
       context.steps.push({
         id: command.id,
@@ -80463,6 +80485,8 @@ function manifestDirectories(manifest, context) {
   const directories = new Set;
   for (const entry of manifest) {
     assertBudget(context);
+    if (entry.kind === "symlink")
+      continue;
     let current = path88.posix.dirname(entry.path);
     while (current !== ".") {
       assertBudget(context);
@@ -80480,9 +80504,8 @@ function candidateOutputFiles(output, manifest, context) {
       assertBudget(context);
       if (!matcher.test(entry.path))
         continue;
-      if (entry.kind === "symlink") {
-        throw new BenchmarkPhaseError("unsafe_path", `candidate output ${output.id} matches a symlink`);
-      }
+      if (entry.kind === "symlink")
+        continue;
       matched.push(entry);
     }
     return {
@@ -80490,12 +80513,6 @@ function candidateOutputFiles(output, manifest, context) {
       files: matched
     };
   }
-  for (const entry of manifest) {
-    assertBudget(context);
-    if (entry.kind === "symlink" && matcher.test(entry.path)) {
-      throw new BenchmarkPhaseError("unsafe_path", `candidate output ${output.id} matches a symlink`);
-    }
-  }
   const directories = [];
   for (const directory of manifestDirectories(manifest, context)) {
     assertBudget(context);
@@ -80510,9 +80527,8 @@ function candidateOutputFiles(output, manifest, context) {
       assertBudget(context);
       if (!entry.path.startsWith(prefix))
         continue;
-      if (entry.kind === "symlink") {
-        throw new BenchmarkPhaseError("unsafe_path", `candidate output ${output.id} contains a symlink`);
-      }
+      if (entry.kind === "symlink")
+        continue;
       selected.set(entry.path, entry);
     }
   }
@@ -80558,7 +80574,7 @@ async function copyCandidateOutput(output, manifest, spec, context) {
   }
   return copied;
 }
-async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.workspace_root, preserveUnsafeSymlinks = false) {
+async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.workspace_root, symlinkPolicy = "preserve") {
   const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-workspace-"));
   fs81.chmodSync(destinationRoot, 448);
   context.temporaryRoots.add(destinationRoot);
@@ -80572,12 +80588,9 @@ async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.wo
       if (target === null || Buffer.byteLength(target) !== frozenFile.size || sha256(target) !== frozenFile.sha256) {
         throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
       }
-      if (!preserveUnsafeSymlinks && path88.isAbsolute(target)) {
-        throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace cannot reproduce an absolute symlink: ${frozenFile.path}`);
-      }
       const resolvedTarget = path88.resolve(path88.dirname(source), target);
-      if (!preserveUnsafeSymlinks && !isWithin(sourceRoot, resolvedTarget)) {
-        throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace symlink escapes the workspace: ${frozenFile.path}`);
+      if (symlinkPolicy === "contained_relative_only" && (path88.isAbsolute(target) || !isWithin(sourceRoot, resolvedTarget))) {
+        continue;
       }
       fs81.mkdirSync(path88.dirname(destination), { recursive: true, mode: 448 });
       fs81.symlinkSync(target, destination);
@@ -80591,11 +80604,15 @@ async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.wo
   }
   return destinationRoot;
 }
-async function copyEvaluatorTests(spec, context) {
+async function copyEvaluatorTests(spec, evaluator, context) {
   const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-tests-"));
   fs81.chmodSync(destinationRoot, 448);
   context.temporaryRoots.add(destinationRoot);
-  const stack = [{ source: spec.tests_root, destination: destinationRoot }];
+  const sourceRoot = evaluatorTestsPath(evaluator, spec.tests_root);
+  const relativeRoot = evaluator.tests_path ? normalizedRootRelative(evaluator.tests_path) : ".";
+  const destinationStart = relativeRoot === "." ? destinationRoot : path88.resolve(destinationRoot, relativeRoot);
+  fs81.mkdirSync(destinationStart, { recursive: true, mode: 448 });
+  const stack = [{ source: sourceRoot, destination: destinationStart }];
   while (stack.length > 0) {
     assertBudget(context);
     const current = stack.pop();
@@ -80657,7 +80674,15 @@ var CriterionResultSchema = exports_external.object({
 });
 var CriteriaResultFileSchema = exports_external.object({
   criteria: exports_external.array(CriterionResultSchema).max(MAX_CRITERIA_PER_EVALUATOR)
-}).strict();
+}).strict().superRefine((value, context) => {
+  if (Buffer.byteLength(JSON.stringify(value)) > MAX_CRITERIA_RESULT_PAYLOAD_BYTES) {
+    context.addIssue({
+      code: exports_external.ZodIssueCode.custom,
+      path: ["criteria"],
+      message: "criterion result exceeds the 20 MiB aggregate payload limit"
+    });
+  }
+});
 function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds, spec, additionalSecrets = []) {
   let opened;
   try {
@@ -80670,7 +80695,7 @@ function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds, spe
   }
   try {
     if (opened.stat.size > MAX_CRITERIA_RESULT_BYTES) {
-      throw new BenchmarkPhaseError("invalid_criterion_result", "criterion result exceeds the 1 MiB limit");
+      throw new BenchmarkPhaseError("invalid_criterion_result", "criterion result exceeds the 24 MiB file limit");
     }
     let raw;
     try {
@@ -80766,9 +80791,12 @@ async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, final
         cwd: ".",
         secret_env: []
       }, spec.tests_root, spec, context, {
-        BB_REGEX_INPUT: frozenEvidence.finalOutputPath,
-        BB_REGEX_PATTERN_B64: Buffer.from(assertion.pattern).toString("base64"),
-        BB_REGEX_FLAGS: assertion.flags
+        additions: {
+          BB_REGEX_INPUT: frozenEvidence.finalOutputPath,
+          BB_REGEX_PATTERN_B64: Buffer.from(assertion.pattern).toString("base64"),
+          BB_REGEX_FLAGS: assertion.flags
+        },
+        descendantCleanup: "always"
       });
       if (regexResult.exitCode === 2) {
         throw new BenchmarkPhaseError("invalid_evaluator", `invalid regex in evaluator: ${evaluator.id}`);
@@ -80867,12 +80895,12 @@ async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, final
   let privateResultRoot;
   const usesLiveWorkspace = evaluator.workspace_mode === "read_only";
   try {
-    const evaluatorWorkspace = usesLiveWorkspace ? spec.workspace_root : await copyFrozenWorkspace(manifest, spec, context, frozenWorkspaceRoot);
+    const evaluatorWorkspace = usesLiveWorkspace ? spec.workspace_root : await copyFrozenWorkspace(manifest, spec, context, frozenWorkspaceRoot, "contained_relative_only");
     if (!usesLiveWorkspace)
       isolatedWorkspace = evaluatorWorkspace;
     else
       await verifyWorkspaceManifestUnchanged(manifest, spec, context);
-    isolatedTestsRoot = await copyEvaluatorTests(spec, context);
+    isolatedTestsRoot = await copyEvaluatorTests(spec, evaluator, context);
     const testsPath = evaluatorTestsPath(evaluator, isolatedTestsRoot);
     const commandRoot = evaluator.root === "workspace" ? evaluatorWorkspace : isolatedTestsRoot;
     const command = { id: evaluator.id, ...evaluator.command };
@@ -80891,7 +80919,10 @@ async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, final
       criterionResultPath = path88.join(privateResultRoot, "result.json");
       environment.BRAINBASE_BENCHMARK_CRITERIA_RESULT = criterionResultPath;
     }
-    const result2 = await runCommand(command, commandRoot, spec, context, environment);
+    const result2 = await runCommand(command, commandRoot, spec, context, {
+      additions: environment,
+      descendantCleanup: "always"
+    });
     const stdout = await writeLog(spec.logs_root, `${evaluator.id}.stdout.log`, result2.stdout, spec);
     const stderr = await writeLog(spec.logs_root, `${evaluator.id}.stderr.log`, result2.stderr, spec);
     let criterionResults;
@@ -81034,7 +81065,7 @@ async function executeEvaluate(spec, context) {
     assertEvaluateOutputBudget(spec, context);
   }
   await verifyRecordsUnchanged(manifest, spec);
-  const frozenWorkspaceRoot = await copyFrozenWorkspace(manifest, spec, context, spec.workspace_root, true);
+  const frozenWorkspaceRoot = await copyFrozenWorkspace(manifest, spec, context, spec.workspace_root);
   for (const reference of spec.references) {
     const referenceOutputs = await copyMaterial(reference, spec.staging_root, spec.tests_root, "tests", false, context);
     outputs.push(...referenceOutputs);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brainbase-labs/cli",
-  "version": "0.25.0-eng1209.8",
+  "version": "0.25.0-eng1209.9",
   "description": "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   "type": "module",
   "bin": {