@brainbase-labs/cli 0.25.0-eng1209.6 → 0.25.0-eng1209.7

package/dist/index.js

@@ -36008,7 +36008,7 @@ function padStart(s, n) {
 // package.json
 var package_default = {
   name: "@brainbase-labs/cli",
-  version: "0.25.0-eng1209.6",
+  version: "0.25.0-eng1209.7",
   description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   type: "module",
   bin: {
@@ -80035,6 +80035,9 @@ async function runCommand(command, root, spec, context, additions = {}) {
     };
     child.stdout?.on("data", (chunk2) => capture(stdout, chunk2));
     child.stderr?.on("data", (chunk2) => capture(stderr, chunk2));
+    child.once("exit", () => {
+      terminate(child);
+    });
     child.on("error", (error2) => {
       fail(new BenchmarkCommandExecutionError("command_start_failed", `failed to start ${command.id}: ${error2.message}`, Buffer.concat(stdout), Buffer.concat(stderr), Date.now() - started));
     });
@@ -80413,13 +80416,13 @@ async function copyCandidateOutput(output, manifest, spec, context) {
   }
   return copied;
 }
-async function copyFrozenWorkspace(manifest, spec, context) {
+async function copyFrozenWorkspace(manifest, spec, context, sourceRoot = spec.workspace_root, preserveUnsafeSymlinks = false) {
   const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-workspace-"));
   fs81.chmodSync(destinationRoot, 448);
   context.temporaryRoots.add(destinationRoot);
   for (const frozenFile of manifest) {
     assertBudget(context);
-    const source = path88.resolve(spec.workspace_root, safeRelPath(frozenFile.path));
+    const source = path88.resolve(sourceRoot, safeRelPath(frozenFile.path));
     const destination = path88.resolve(destinationRoot, safeRelPath(frozenFile.path));
     if (frozenFile.kind === "symlink") {
       const stat = fs81.lstatSync(source);
@@ -80427,18 +80430,18 @@ async function copyFrozenWorkspace(manifest, spec, context) {
       if (target === null || Buffer.byteLength(target) !== frozenFile.size || sha256(target) !== frozenFile.sha256) {
         throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
       }
-      if (path88.isAbsolute(target)) {
+      if (!preserveUnsafeSymlinks && path88.isAbsolute(target)) {
         throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace cannot reproduce an absolute symlink: ${frozenFile.path}`);
       }
       const resolvedTarget = path88.resolve(path88.dirname(source), target);
-      if (!isWithin(spec.workspace_root, resolvedTarget)) {
+      if (!preserveUnsafeSymlinks && !isWithin(sourceRoot, resolvedTarget)) {
         throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace symlink escapes the workspace: ${frozenFile.path}`);
       }
       fs81.mkdirSync(path88.dirname(destination), { recursive: true, mode: 448 });
       fs81.symlinkSync(target, destination);
       continue;
     }
-    await atomicCopy(source, destination, frozenFile.mode, spec.workspace_root);
+    await atomicCopy(source, destination, frozenFile.mode, sourceRoot);
     const copied = await recordFile(destinationRoot, destination, "workspace");
     if (copied.sha256 !== frozenFile.sha256 || copied.size !== frozenFile.size || copied.mode !== frozenFile.mode) {
       throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
@@ -80513,7 +80516,7 @@ var CriterionResultSchema = exports_external.object({
 var CriteriaResultFileSchema = exports_external.object({
   criteria: exports_external.array(CriterionResultSchema).max(MAX_CRITERIA_PER_EVALUATOR)
 }).strict();
-function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds) {
+function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds, spec) {
   let opened;
   try {
     opened = openRegularFileNoFollow(resultPath, "criterion result", path88.dirname(resultPath));
@@ -80557,7 +80560,13 @@ function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds) {
     if (unknownEvidenceIds.length > 0) {
       throw new BenchmarkPhaseError("invalid_criterion_result", "criterion result cites evidence outside the evaluator plan", { unknown_evidence_ids: unknownEvidenceIds });
     }
-    return criterionKeys.map((key2) => byKey.get(key2));
+    return criterionKeys.map((key2) => {
+      const criterion = byKey.get(key2);
+      return {
+        ...criterion,
+        explanation: redactCommandOutput(Buffer.from(criterion.explanation), spec).toString("utf8")
+      };
+    });
   } finally {
     fs81.closeSync(opened.fd);
   }
@@ -80580,7 +80589,7 @@ function eventType(event) {
   }
   return;
 }
-async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, frozenEvidence, context) {
+async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, finalOutput, trajectory, frozenEvidence, context) {
   const started = Date.now();
   const base2 = {
     id: evaluator.id,
@@ -80653,8 +80662,8 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
   }
   if (evaluator.type === "workspace_assertion") {
     const relative = workspaceRel(evaluator.path);
-    assertNoSymlinkTraversal(spec.workspace_root, relative);
-    const candidate = path88.resolve(spec.workspace_root, relative);
+    assertNoSymlinkTraversal(frozenWorkspaceRoot, relative);
+    const candidate = path88.resolve(frozenWorkspaceRoot, relative);
     let stat = null;
     try {
       stat = fs81.lstatSync(candidate);
@@ -80675,7 +80684,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       verdict = !exists2;
     if (evaluator.assertion.operator === "sha256") {
       if (stat?.isFile()) {
-        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
+        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, frozenWorkspaceRoot);
         try {
           actualSha256 = await sha256OfDescriptor(opened.fd);
           verdict = actualSha256 === evaluator.assertion.expected;
@@ -80686,7 +80695,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
     }
     if (evaluator.assertion.operator === "contains") {
       if (stat?.isFile()) {
-        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
+        const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, frozenWorkspaceRoot);
         try {
           verdict = readDescriptor(opened.fd).toString("utf8").includes(evaluator.assertion.expected);
         } finally {
@@ -80715,7 +80724,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
   let isolatedTestsRoot;
   let privateResultRoot;
   try {
-    const evaluatorWorkspace = await copyFrozenWorkspace(manifest, spec, context);
+    const evaluatorWorkspace = await copyFrozenWorkspace(manifest, spec, context, frozenWorkspaceRoot);
     isolatedWorkspace = evaluatorWorkspace;
     isolatedTestsRoot = await copyEvaluatorTests(spec, context);
     const testsPath = evaluatorTestsPath(evaluator, isolatedTestsRoot);
@@ -80741,7 +80750,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
     const stderr = await writeLog(spec.logs_root, `${evaluator.id}.stderr.log`, result2.stderr, spec);
     let criterionResults;
     try {
-      criterionResults = evaluator.criterion_keys && criterionResultPath ? readCriterionResults(criterionResultPath, evaluator.criterion_keys, evaluator.allowed_evidence_ids ?? []) : undefined;
+      criterionResults = evaluator.criterion_keys && criterionResultPath ? readCriterionResults(criterionResultPath, evaluator.criterion_keys, evaluator.allowed_evidence_ids ?? [], spec) : undefined;
     } catch (error2) {
       const normalized = stableError(error2);
       return {
@@ -80873,12 +80882,12 @@ async function executeEvaluate(spec, context) {
     assertEvaluateOutputBudget(spec, context);
   }
   await verifyRecordsUnchanged(manifest, spec);
+  const frozenWorkspaceRoot = await copyFrozenWorkspace(manifest, spec, context, spec.workspace_root, true);
   for (const reference of spec.references) {
     const referenceOutputs = await copyMaterial(reference, spec.staging_root, spec.tests_root, "tests", false, context);
     outputs.push(...referenceOutputs);
     context.outputs.push(...referenceOutputs);
   }
-  const frozenOutputCount = context.outputs.length;
   const evaluators = [];
   const evaluatorOrder = new Map(spec.evaluators.map((evaluator, index) => [evaluator.id, index]));
   const executionOrder = [...spec.evaluators].sort((left, right) => Number(left.type === "sandbox_command") - Number(right.type === "sandbox_command"));
@@ -80886,7 +80895,7 @@ async function executeEvaluate(spec, context) {
     assertBudget(context);
     const started = Date.now();
     try {
-      const evaluated = await evaluateOne(evaluator, spec, manifest, finalOutput.buffer.toString("utf8"), trajectory, { finalOutputPath, trajectoryPath }, context);
+      const evaluated = await evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, finalOutput.buffer.toString("utf8"), trajectory, { finalOutputPath, trajectoryPath }, context);
       assertBudget(context);
       evaluators.push(evaluated);
       context.evaluators.push(evaluated);
@@ -80941,7 +80950,7 @@ async function executeEvaluate(spec, context) {
       assertEvaluateOutputBudget(spec, context);
     }
   }
-  await verifyRecordsUnchanged([...context.inputs, ...context.outputs.slice(0, frozenOutputCount)], spec);
+  await verifyRecordsUnchanged([...context.inputs, ...context.outputs], spec);
   if (!verifyOwnedDirectory(spec.tests_root, "tests", spec) || !verifyOwnedDirectory(spec.logs_root, "logs", spec)) {
     throw new BenchmarkPhaseError("evidence_tampered", "benchmark evaluator changed an owned phase directory marker");
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brainbase-labs/cli",
-  "version": "0.25.0-eng1209.6",
+  "version": "0.25.0-eng1209.7",
   "description": "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   "type": "module",
   "bin": {