@brainbase-labs/cli 0.25.0-eng1209.6 → 0.25.0-eng1209.8

package/dist/index.js

@@ -36008,7 +36008,7 @@ function padStart(s, n) {
 // package.json
 var package_default = {
   name: "@brainbase-labs/cli",
-  version: "0.25.0-eng1209.6",
+  version: "0.25.0-eng1209.8",
   description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   type: "module",
   bin: {
@@ -78492,6 +78492,9 @@ var MAX_CRITERION_EXPLANATION_LENGTH = 16384;
 var MAX_CRITERION_EVIDENCE_IDS = 100;
 var MAX_CRITERION_EVIDENCE_ID_LENGTH = 256;
 var MAX_ALLOWED_EVIDENCE_IDS = 1e4;
+var COMMAND_PROCESS_MARKER_ENV = "BRAINBASE_BENCHMARK_COMMAND_MARKER";
+var COMMAND_PROCESS_POLL_MS = 10;
+var COMMAND_PROCESS_CLEANUP_MS = 500;
 var RESERVED_WORKSPACE_PATHS = new Set([
   ".brainbase",
   ".git",
@@ -78712,6 +78715,20 @@ var BenchmarkSpecSchema = exports_external.discriminatedUnion("phase", [
   HydrateSpecSchema,
   EvaluateSpecSchema
 ]).superRefine((value, context) => {
+  if (Object.hasOwn(value.environment, COMMAND_PROCESS_MARKER_ENV)) {
+    context.addIssue({
+      code: exports_external.ZodIssueCode.custom,
+      path: ["environment", COMMAND_PROCESS_MARKER_ENV],
+      message: `${COMMAND_PROCESS_MARKER_ENV} is reserved for command supervision`
+    });
+  }
+  if (value.secret_env.includes(COMMAND_PROCESS_MARKER_ENV)) {
+    context.addIssue({
+      code: exports_external.ZodIssueCode.custom,
+      path: ["secret_env"],
+      message: `${COMMAND_PROCESS_MARKER_ENV} is reserved for command supervision`
+    });
+  }
   for (const name of Object.keys(value.environment)) {
     if (SENSITIVE_ENV_NAME_RE.test(name)) {
       context.addIssue({
@@ -79920,7 +79937,7 @@ function buildEnvironment(spec, secretNames, additions = {}) {
   Object.assign(env3, additions);
   return env3;
 }
-function redactCommandOutput(data, spec) {
+function redactCommandOutput(data, spec, additionalSecrets = []) {
   let value = data.toString("utf8");
   const sensitiveNames = new Set([
     ...spec.secret_env,
@@ -79931,6 +79948,10 @@ function redactCommandOutput(data, spec) {
     if (secret)
       value = value.split(secret).join("[REDACTED]");
   }
+  for (const secret of additionalSecrets) {
+    if (secret)
+      value = value.split(secret).join("[REDACTED]");
+  }
   return Buffer.from(value);
 }
 function descendantPids(parentPid) {
@@ -79982,6 +80003,86 @@ function terminate(child) {
     child.kill("SIGKILL");
   }
 }
+function markedProcessPids(marker) {
+  const assignment = `${COMMAND_PROCESS_MARKER_ENV}=${marker}`;
+  if (process.platform === "linux") {
+    const matches2 = [];
+    let entries;
+    try {
+      entries = fs81.readdirSync("/proc");
+    } catch {
+      return matches2;
+    }
+    for (const entry of entries) {
+      if (!/^\d+$/.test(entry))
+        continue;
+      const pid = Number(entry);
+      if (pid === process.pid)
+        continue;
+      try {
+        const environment = fs81.readFileSync(path88.join("/proc", entry, "environ"), "utf8");
+        if (environment.split("\x00").includes(assignment))
+          matches2.push(pid);
+      } catch {}
+    }
+    return matches2;
+  }
+  if (process.platform === "darwin") {
+    try {
+      const output = execFileSync2("ps", ["eww", "-axo", "pid=,command="], {
+        encoding: "utf8",
+        maxBuffer: 16 * 1024 * 1024,
+        stdio: ["ignore", "pipe", "ignore"]
+      });
+      const matches2 = [];
+      for (const line of output.split(`
+`)) {
+        const match = line.match(/^\s*(\d+)\s+(.*)$/);
+        if (!match || !match[2].includes(assignment))
+          continue;
+        const pid = Number(match[1]);
+        if (Number.isInteger(pid) && pid !== process.pid)
+          matches2.push(pid);
+      }
+      return matches2;
+    } catch {
+      return [];
+    }
+  }
+  return [];
+}
+function processExists(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error2) {
+    return error2.code !== "ESRCH";
+  }
+}
+async function terminateCommandProcesses(child, marker, observedDescendants) {
+  if (child.pid !== undefined) {
+    for (const pid of descendantPids(child.pid))
+      observedDescendants.add(pid);
+  }
+  terminate(child);
+  const deadline = Date.now() + COMMAND_PROCESS_CLEANUP_MS;
+  while (true) {
+    for (const pid of markedProcessPids(marker))
+      observedDescendants.add(pid);
+    const active = [...observedDescendants].filter(processExists);
+    for (const pid of active) {
+      try {
+        process.kill(pid, "SIGKILL");
+      } catch {}
+    }
+    if (active.length === 0)
+      return;
+    if (Date.now() >= deadline) {
+      throw new BenchmarkPhaseError("command_cleanup_failed", "command descendants remained after bounded cleanup");
+    }
+    await new Promise((resolve) => setTimeout(resolve, COMMAND_PROCESS_POLL_MS));
+  }
+}
 async function runCommand(command, root, spec, context, additions = {}) {
   const cwdRel = normalizedRootRelative(command.cwd);
   assertNoSymlinkTraversal(root, cwdRel);
@@ -80001,27 +80102,56 @@ async function runCommand(command, root, spec, context, additions = {}) {
   const reachesPhaseDeadline = command.timeout_ms === undefined || command.timeout_ms >= remainingMs;
   const timeoutMs2 = reachesPhaseDeadline ? remainingMs : command.timeout_ms;
   const started = Date.now();
+  const commandMarker = crypto6.randomBytes(32).toString("hex");
   return await new Promise((resolve, reject2) => {
     const child = spawn4(command.argv[0], command.argv.slice(1), {
       cwd: cwd2,
-      env: buildEnvironment(spec, command.secret_env, additions),
+      env: buildEnvironment(spec, command.secret_env, {
+        ...additions,
+        [COMMAND_PROCESS_MARKER_ENV]: commandMarker
+      }),
       stdio: ["ignore", "pipe", "pipe"],
       detached: process.platform !== "win32"
     });
     const stdout = [];
     const stderr = [];
+    const observedDescendants = new Set;
     let captured = 0;
     let settled = false;
     let timer;
+    let observer;
+    let cleanup;
+    const observeDescendants = () => {
+      if (child.pid === undefined)
+        return;
+      for (const pid of descendantPids(child.pid))
+        observedDescendants.add(pid);
+    };
+    const cleanupProcesses = () => {
+      if (observer)
+        clearInterval(observer);
+      cleanup ??= terminateCommandProcesses(child, commandMarker, observedDescendants);
+      return cleanup;
+    };
+    const sanitizedOutput = (chunks) => redactCommandOutput(Buffer.concat(chunks), spec, [commandMarker]);
+    const sanitizedError = (error2) => {
+      if (!(error2 instanceof BenchmarkCommandExecutionError))
+        return error2;
+      return new BenchmarkCommandExecutionError(error2.code, error2.message, sanitizedOutput(stdout), sanitizedOutput(stderr), error2.durationMs);
+    };
     const fail = (error2) => {
       if (settled)
         return;
       settled = true;
       if (timer)
         clearTimeout(timer);
-      terminate(child);
-      reject2(error2);
+      cleanupProcesses().then(() => reject2(sanitizedError(error2)), reject2);
     };
+    if (process.platform !== "linux") {
+      observeDescendants();
+      observer = setInterval(observeDescendants, COMMAND_PROCESS_POLL_MS);
+      observer.unref();
+    }
     const capture = (target, chunk2) => {
       if (settled)
         return;
@@ -80035,6 +80165,11 @@ async function runCommand(command, root, spec, context, additions = {}) {
     };
     child.stdout?.on("data", (chunk2) => capture(stdout, chunk2));
     child.stderr?.on("data", (chunk2) => capture(stderr, chunk2));
+    child.once("exit", () => {
+      if (timer)
+        clearTimeout(timer);
+      cleanupProcesses();
+    });
     child.on("error", (error2) => {
       fail(new BenchmarkCommandExecutionError("command_start_failed", `failed to start ${command.id}: ${error2.message}`, Buffer.concat(stdout), Buffer.concat(stderr), Date.now() - started));
     });
@@ -80050,16 +80185,19 @@ async function runCommand(command, root, spec, context, additions = {}) {
         return;
       settled = true;
       clearTimeout(timer);
-      if (code === null) {
-        reject2(new BenchmarkCommandExecutionError("command_terminated", `command was terminated by ${signal ?? "an unknown signal"}: ${command.id}`, Buffer.concat(stdout), Buffer.concat(stderr), Date.now() - started));
-        return;
-      }
-      resolve({
-        exitCode: code,
-        stdout: Buffer.concat(stdout),
-        stderr: Buffer.concat(stderr),
-        durationMs: Date.now() - started
-      });
+      cleanupProcesses().then(() => {
+        if (code === null) {
+          reject2(new BenchmarkCommandExecutionError("command_terminated", `command was terminated by ${signal ?? "an unknown signal"}: ${command.id}`, sanitizedOutput(stdout), sanitizedOutput(stderr), Date.now() - started));
+          return;
+        }
+        resolve({
+          exitCode: code,
+          stdout: sanitizedOutput(stdout),
+          stderr: sanitizedOutput(stderr),
+          durationMs: Date.now() - started,
+          redactions: [commandMarker]
+        });
+      }, reject2);
     });
   });
 }
@@ -80252,6 +80390,13 @@ async function workspaceManifest(spec, context) {
   }
   return records.sort((a3, b4) => a3.path.localeCompare(b4.path));
 }
+async function verifyWorkspaceManifestUnchanged(expected, spec, context) {
+  await verifyRecordsUnchanged(expected, spec);
+  const actual = await workspaceManifest(spec, context);
+  if (JSON.stringify(actual) !== JSON.stringify(expected)) {
+    throw new BenchmarkPhaseError("evidence_tampered", "candidate workspace changed during read-only evaluation");
+  }
+}
 function treeBytes(root, context) {
   let totalBytes = 0;
   const stack = [path88.resolve(root)];
@@ -80413,13 +80558,13 @@ async function copyCandidateOutput(output, manifest, spec, context) {
   }
   return copied;
 }
-async function copyFrozenWorkspace(manifest, spec, context) {
+async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.workspace_root, preserveUnsafeSymlinks = false) {
   const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-workspace-"));
   fs81.chmodSync(destinationRoot, 448);
   context.temporaryRoots.add(destinationRoot);
   for (const frozenFile of manifest) {
     assertBudget(context);
-    const source = path88.resolve(spec.workspace_root, safeRelPath(frozenFile.path));
+    const source = path88.resolve(sourceRoot, safeRelPath(frozenFile.path));
     const destination = path88.resolve(destinationRoot, safeRelPath(frozenFile.path));
     if (frozenFile.kind === "symlink") {
       const stat = fs81.lstatSync(source);
@@ -80427,18 +80572,18 @@ async function copyFrozenWorkspace(manifest, spec, context) {
       if (target === null || Buffer.byteLength(target) !== frozenFile.size || sha256(target) !== frozenFile.sha256) {
         throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
       }
-      if (path88.isAbsolute(target)) {
+      if (!preserveUnsafeSymlinks && path88.isAbsolute(target)) {
         throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace cannot reproduce an absolute symlink: ${frozenFile.path}`);
       }
       const resolvedTarget = path88.resolve(path88.dirname(source), target);
-      if (!isWithin(spec.workspace_root, resolvedTarget)) {
+      if (!preserveUnsafeSymlinks && !isWithin(sourceRoot, resolvedTarget)) {
         throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace symlink escapes the workspace: ${frozenFile.path}`);
       }
       fs81.mkdirSync(path88.dirname(destination), { recursive: true, mode: 448 });
       fs81.symlinkSync(target, destination);
       continue;
     }
-    await atomicCopy(source, destination, frozenFile.mode, spec.workspace_root);
+    await atomicCopy(source, destination, frozenFile.mode, sourceRoot);
     const copied = await recordFile(destinationRoot, destination, "workspace");
     if (copied.sha256 !== frozenFile.sha256 || copied.size !== frozenFile.size || copied.mode !== frozenFile.mode) {
       throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
@@ -80513,7 +80658,7 @@ var CriterionResultSchema = exports_external.object({
 var CriteriaResultFileSchema = exports_external.object({
   criteria: exports_external.array(CriterionResultSchema).max(MAX_CRITERIA_PER_EVALUATOR)
 }).strict();
-function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds) {
+function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds, spec, additionalSecrets = []) {
   let opened;
   try {
     opened = openRegularFileNoFollow(resultPath, "criterion result", path88.dirname(resultPath));
@@ -80557,7 +80702,13 @@ function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds) {
     if (unknownEvidenceIds.length > 0) {
       throw new BenchmarkPhaseError("invalid_criterion_result", "criterion result cites evidence outside the evaluator plan", { unknown_evidence_ids: unknownEvidenceIds });
     }
-    return criterionKeys.map((key2) => byKey.get(key2));
+    return criterionKeys.map((key2) => {
+      const criterion = byKey.get(key2);
+      return {
+        ...criterion,
+        explanation: redactCommandOutput(Buffer.from(criterion.explanation), spec, additionalSecrets).toString("utf8")
+      };
+    });
   } finally {
     fs81.closeSync(opened.fd);
   }
@@ -80580,7 +80731,7 @@ function eventType(event) {
   }
   return;
 }
-async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, frozenEvidence, context) {
+async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, finalOutput, trajectory, frozenEvidence, context) {
   const started = Date.now();
   const base2 = {
     id: evaluator.id,
@@ -80653,8 +80804,8 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
   }
   if (evaluator.type === "workspace_assertion") {
     const relative = workspaceRel(evaluator.path);
-    assertNoSymlinkTraversal(spec.workspace_root, relative);
-    const candidate = path88.resolve(spec.workspace_root, relative);
+    assertNoSymlinkTraversal(frozenWorkspaceRoot, relative);
+    const candidate = path88.resolve(frozenWorkspaceRoot, relative);
     let stat = null;
     try {
       stat = fs81.lstatSync(candidate);
@@ -80675,7 +80826,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       verdict = !exists2;
     if (evaluator.assertion.operator === "sha256") {
       if (stat?.isFile()) {
-        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
+        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, frozenWorkspaceRoot);
         try {
           actualSha256 = await sha256OfDescriptor(opened.fd);
           verdict = actualSha256 === evaluator.assertion.expected;
@@ -80686,7 +80837,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
     }
     if (evaluator.assertion.operator === "contains") {
       if (stat?.isFile()) {
-        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
+        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, frozenWorkspaceRoot);
         try {
           verdict = readDescriptor(opened.fd).toString("utf8").includes(evaluator.assertion.expected);
         } finally {
@@ -80714,9 +80865,13 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
   let isolatedWorkspace;
   let isolatedTestsRoot;
   let privateResultRoot;
+  const usesLiveWorkspace = evaluator.workspace_mode === "read_only";
   try {
-    const evaluatorWorkspace = await copyFrozenWorkspace(manifest, spec, context);
-    isolatedWorkspace = evaluatorWorkspace;
+    const evaluatorWorkspace = usesLiveWorkspace ? spec.workspace_root : await copyFrozenWorkspace(manifest, spec, context, frozenWorkspaceRoot);
+    if (!usesLiveWorkspace)
+      isolatedWorkspace = evaluatorWorkspace;
+    else
+      await verifyWorkspaceManifestUnchanged(manifest, spec, context);
     isolatedTestsRoot = await copyEvaluatorTests(spec, context);
     const testsPath = evaluatorTestsPath(evaluator, isolatedTestsRoot);
     const commandRoot = evaluator.root === "workspace" ? evaluatorWorkspace : isolatedTestsRoot;
@@ -80741,7 +80896,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
     const stderr = await writeLog(spec.logs_root, `${evaluator.id}.stderr.log`, result2.stderr, spec);
     let criterionResults;
     try {
-      criterionResults = evaluator.criterion_keys && criterionResultPath ? readCriterionResults(criterionResultPath, evaluator.criterion_keys, evaluator.allowed_evidence_ids ?? []) : undefined;
+      criterionResults = evaluator.criterion_keys && criterionResultPath ? readCriterionResults(criterionResultPath, evaluator.criterion_keys, evaluator.allowed_evidence_ids ?? [], spec, result2.redactions) : undefined;
     } catch (error2) {
       const normalized = stableError(error2);
       return {
@@ -80770,11 +80925,17 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       ...criterionResults ? { criterion_results: criterionResults } : {}
     };
   } finally {
-    for (const temporary of [privateResultRoot, isolatedTestsRoot, isolatedWorkspace]) {
-      if (!temporary)
-        continue;
-      fs81.rmSync(temporary, { recursive: true, force: true });
-      context.temporaryRoots.delete(temporary);
+    try {
+      if (usesLiveWorkspace) {
+        await verifyWorkspaceManifestUnchanged(manifest, spec, context);
+      }
+    } finally {
+      for (const temporary of [privateResultRoot, isolatedTestsRoot, isolatedWorkspace]) {
+        if (!temporary)
+          continue;
+        fs81.rmSync(temporary, { recursive: true, force: true });
+        context.temporaryRoots.delete(temporary);
+      }
     }
   }
 }
@@ -80873,12 +81034,12 @@ async function executeEvaluate(spec, context) {
     assertEvaluateOutputBudget(spec, context);
   }
   await verifyRecordsUnchanged(manifest, spec);
+  const frozenWorkspaceRoot = await copyFrozenWorkspace(manifest, spec, context, spec.workspace_root, true);
   for (const reference of spec.references) {
     const referenceOutputs = await copyMaterial(reference, spec.staging_root, spec.tests_root, "tests", false, context);
     outputs.push(...referenceOutputs);
     context.outputs.push(...referenceOutputs);
   }
-  const frozenOutputCount = context.outputs.length;
   const evaluators = [];
   const evaluatorOrder = new Map(spec.evaluators.map((evaluator, index) => [evaluator.id, index]));
   const executionOrder = [...spec.evaluators].sort((left, right) => Number(left.type === "sandbox_command") - Number(right.type === "sandbox_command"));
@@ -80886,7 +81047,7 @@ async function executeEvaluate(spec, context) {
     assertBudget(context);
     const started = Date.now();
     try {
-      const evaluated = await evaluateOne(evaluator, spec, manifest, finalOutput.buffer.toString("utf8"), trajectory, { finalOutputPath, trajectoryPath }, context);
+      const evaluated = await evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, finalOutput.buffer.toString("utf8"), trajectory, { finalOutputPath, trajectoryPath }, context);
       assertBudget(context);
       evaluators.push(evaluated);
       context.evaluators.push(evaluated);
@@ -80941,7 +81102,7 @@ async function executeEvaluate(spec, context) {
       assertEvaluateOutputBudget(spec, context);
     }
   }
-  await verifyRecordsUnchanged([...context.inputs, ...context.outputs.slice(0, frozenOutputCount)], spec);
+  await verifyRecordsUnchanged([...context.inputs, ...context.outputs], spec);
   if (!verifyOwnedDirectory(spec.tests_root, "tests", spec) || !verifyOwnedDirectory(spec.logs_root, "logs", spec)) {
     throw new BenchmarkPhaseError("evidence_tampered", "benchmark evaluator changed an owned phase directory marker");
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brainbase-labs/cli",
-  "version": "0.25.0-eng1209.6",
+  "version": "0.25.0-eng1209.8",
   "description": "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   "type": "module",
   "bin": {