@brainbase-labs/cli 0.25.0-eng1209.3 → 0.25.0-eng1209.5

package/dist/index.js

@@ -36008,7 +36008,7 @@ function padStart(s, n) {
 // package.json
 var package_default = {
   name: "@brainbase-labs/cli",
-  version: "0.25.0-eng1209.3",
+  version: "0.25.0-eng1209.5",
   description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   type: "module",
   bin: {
@@ -79114,10 +79114,15 @@ async function verifyRecordsUnchanged(records, spec) {
     const relative = safeRelPath(record3.path);
     assertNoSymlinkTraversal(root, relative);
     const candidate = path88.resolve(root, relative);
-    if (!isWithin(root, candidate) || !fs81.existsSync(candidate)) {
+    if (!isWithin(root, candidate)) {
+      throw new BenchmarkPhaseError("evidence_tampered", `evidence was removed during evaluation: ${record3.root}:${record3.path}`);
+    }
+    let stat;
+    try {
+      stat = fs81.lstatSync(candidate);
+    } catch {
       throw new BenchmarkPhaseError("evidence_tampered", `evidence was removed during evaluation: ${record3.root}:${record3.path}`);
     }
-    const stat = fs81.lstatSync(candidate);
     if (record3.kind === "symlink") {
       const target = stat.isSymbolicLink() ? fs81.readlinkSync(candidate) : null;
       if (target === null || Buffer.byteLength(target) !== record3.size || sha256(target) !== record3.sha256) {
@@ -79212,6 +79217,15 @@ function removeRemoteHydrationInputs(spec) {
     removedSources.add(material.source);
   }
 }
+function removePrivateHydrationSourcesBeforeSetup(spec, context) {
+  removeRemoteHydrationInputs(spec);
+  for (const temporary of context.temporaryRoots) {
+    fs81.rmSync(temporary, { recursive: true, force: true });
+  }
+  context.temporaryRoots.clear();
+  context.verifiedInputs.clear();
+  context.preparedArchiveFiles.clear();
+}
 async function downloadInputReference(stagingRoot, input, context) {
   const cacheKey = inputCacheKey(input);
   const cached2 = context.verifiedInputs.get(cacheKey);
@@ -80100,6 +80114,7 @@ async function executeHydrate(spec, context) {
       throw error2;
     }
   }
+  removePrivateHydrationSourcesBeforeSetup(spec, context);
   for (const command of spec.setup_commands) {
     assertBudget(context);
     let result2;
@@ -80155,7 +80170,6 @@ async function executeHydrate(spec, context) {
   }
   outputs.splice(0, outputs.length, ...finalOutputs);
   context.outputs.splice(0, context.outputs.length, ...finalOutputs);
-  removeRemoteHydrationInputs(spec);
   assertBudget(context);
   return outputs;
 }
@@ -80220,6 +80234,42 @@ async function workspaceManifest(spec, context) {
   }
   return records.sort((a3, b4) => a3.path.localeCompare(b4.path));
 }
+function treeBytes(root, context) {
+  let totalBytes = 0;
+  const stack = [path88.resolve(root)];
+  while (stack.length > 0) {
+    const directory = stack.pop();
+    const entries = fs81.readdirSync(directory, { withFileTypes: true });
+    for (const entry of entries) {
+      assertBudget(context);
+      const candidate = path88.join(directory, entry.name);
+      const stat = fs81.lstatSync(candidate);
+      if (stat.isDirectory()) {
+        stack.push(candidate);
+        continue;
+      }
+      if (stat.isFile()) {
+        totalBytes += stat.size;
+        continue;
+      }
+      if (stat.isSymbolicLink()) {
+        totalBytes += Buffer.byteLength(fs81.readlinkSync(candidate));
+        continue;
+      }
+      throw new BenchmarkPhaseError("unsafe_path", `evaluate output contains an unsupported filesystem entry: ${path88.relative(root, candidate)}`);
+    }
+  }
+  return totalBytes;
+}
+function assertEvaluateOutputBudget(spec, context, additionalBytes = 0) {
+  const actual = treeBytes(spec.logs_root, context) + additionalBytes;
+  if (actual > spec.workspace_limits.max_total_bytes) {
+    throw new BenchmarkPhaseError("workspace_limit_exceeded", "evaluate output bytes exceed the workspace byte limit", {
+      actual,
+      max_total_bytes: spec.workspace_limits.max_total_bytes
+    });
+  }
+}
 function candidateGlob(pattern) {
   let source = "^";
   for (let index = 0;index < pattern.length; index += 1) {
@@ -80346,12 +80396,6 @@ async function copyCandidateOutput(output, manifest, spec, context) {
   return copied;
 }
 async function copyFrozenWorkspace(manifest, spec, context) {
-  for (const entry of manifest) {
-    assertBudget(context);
-    if (entry.kind === "symlink") {
-      throw new BenchmarkPhaseError("unsupported_workspace_symlink", `sandbox evaluator workspace cannot safely reproduce symlink: ${entry.path}`);
-    }
-  }
   const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-workspace-"));
   fs81.chmodSync(destinationRoot, 448);
   context.temporaryRoots.add(destinationRoot);
@@ -80359,6 +80403,23 @@ async function copyFrozenWorkspace(manifest, spec, context) {
     assertBudget(context);
     const source = path88.resolve(spec.workspace_root, safeRelPath(frozenFile.path));
     const destination = path88.resolve(destinationRoot, safeRelPath(frozenFile.path));
+    if (frozenFile.kind === "symlink") {
+      const stat = fs81.lstatSync(source);
+      const target = stat.isSymbolicLink() ? fs81.readlinkSync(source) : null;
+      if (target === null || Buffer.byteLength(target) !== frozenFile.size || sha256(target) !== frozenFile.sha256) {
+        throw new BenchmarkPhaseError("evidence_tampered", `workspace changed after it was frozen: ${frozenFile.path}`);
+      }
+      if (path88.isAbsolute(target)) {
+        throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace cannot reproduce an absolute symlink: ${frozenFile.path}`);
+      }
+      const resolvedTarget = path88.resolve(path88.dirname(source), target);
+      if (!isWithin(spec.workspace_root, resolvedTarget)) {
+        throw new BenchmarkPhaseError("unsafe_workspace_symlink", `sandbox evaluator workspace symlink escapes the workspace: ${frozenFile.path}`);
+      }
+      fs81.mkdirSync(path88.dirname(destination), { recursive: true, mode: 448 });
+      fs81.symlinkSync(target, destination);
+      continue;
+    }
     await atomicCopy(source, destination, frozenFile.mode, spec.workspace_root);
     const copied = await recordFile(destinationRoot, destination, "workspace");
     if (copied.sha256 !== frozenFile.sha256 || copied.size !== frozenFile.size || copied.mode !== frozenFile.mode) {
@@ -80367,13 +80428,48 @@ async function copyFrozenWorkspace(manifest, spec, context) {
   }
   return destinationRoot;
 }
-function evaluatorTestsPath(evaluator, spec) {
+async function copyEvaluatorTests(spec, context) {
+  const destinationRoot = fs81.mkdtempSync(path88.join(os16.tmpdir(), "brainbase-benchmark-tests-"));
+  fs81.chmodSync(destinationRoot, 448);
+  context.temporaryRoots.add(destinationRoot);
+  const stack = [{ source: spec.tests_root, destination: destinationRoot }];
+  while (stack.length > 0) {
+    assertBudget(context);
+    const current = stack.pop();
+    const entries = fs81.readdirSync(current.source, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name));
+    for (const entry of entries) {
+      assertBudget(context);
+      const source = path88.join(current.source, entry.name);
+      const destination = path88.join(current.destination, entry.name);
+      const stat = fs81.lstatSync(source);
+      if (stat.isSymbolicLink()) {
+        throw new BenchmarkPhaseError("unsafe_path", `evaluator reference tree contains a symlink: ${path88.relative(spec.tests_root, source)}`);
+      }
+      if (stat.isDirectory()) {
+        fs81.mkdirSync(destination, { recursive: true, mode: stat.mode & 511 });
+        stack.push({ source, destination });
+        continue;
+      }
+      if (!stat.isFile()) {
+        throw new BenchmarkPhaseError("unsafe_path", `evaluator reference tree contains an unsupported filesystem entry: ${path88.relative(spec.tests_root, source)}`);
+      }
+      await atomicCopy(source, destination, stat.mode & 511, spec.tests_root);
+      const sourceRecord = await recordFile(spec.tests_root, source, "tests");
+      const copiedRecord = await recordFile(destinationRoot, destination, "tests");
+      if (copiedRecord.path !== sourceRecord.path || copiedRecord.sha256 !== sourceRecord.sha256 || copiedRecord.size !== sourceRecord.size || copiedRecord.mode !== sourceRecord.mode) {
+        throw new BenchmarkPhaseError("evidence_tampered", `evaluator reference changed while it was copied: ${sourceRecord.path}`);
+      }
+    }
+  }
+  return destinationRoot;
+}
+function evaluatorTestsPath(evaluator, testsRoot) {
   if (!evaluator.tests_path)
-    return spec.tests_root;
+    return testsRoot;
   const relative = normalizedRootRelative(evaluator.tests_path);
-  assertNoSymlinkTraversal(spec.tests_root, relative);
-  const candidate = path88.resolve(spec.tests_root, relative);
-  if (!isWithin(spec.tests_root, candidate) || !fs81.existsSync(candidate)) {
+  assertNoSymlinkTraversal(testsRoot, relative);
+  const candidate = path88.resolve(testsRoot, relative);
+  if (!isWithin(testsRoot, candidate) || !fs81.existsSync(candidate)) {
     throw new BenchmarkPhaseError("invalid_tests_path", `evaluator tests_path does not exist: ${evaluator.tests_path}`);
   }
   const stat = fs81.lstatSync(candidate);
@@ -80510,7 +80606,21 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       }
       verdict = regexResult.exitCode === 0;
     }
-    return { ...base2, status: verdict ? "passed" : "failed", verdict, duration_ms: Date.now() - started };
+    return {
+      ...base2,
+      status: verdict ? "passed" : "failed",
+      verdict,
+      duration_ms: Date.now() - started,
+      details: {
+        operator: assertion.operator,
+        actual_size_bytes: Buffer.byteLength(finalOutput),
+        ...assertion.operator === "exact" ? {
+          expected_size_bytes: Buffer.byteLength(assertion.expected),
+          actual_sha256: sha256(finalOutput),
+          expected_sha256: sha256(assertion.expected)
+        } : {}
+      }
+    };
   }
   if (evaluator.type === "trajectory_assertion") {
     const count = trajectory.filter((event) => eventType(event) === evaluator.event_type).length;
@@ -80540,6 +80650,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
     }
     const exists2 = stat !== null;
     let verdict = false;
+    let actualSha256;
     if (evaluator.assertion.operator === "exists")
       verdict = exists2;
     if (evaluator.assertion.operator === "not_exists")
@@ -80548,7 +80659,8 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       if (stat?.isFile()) {
         const opened = openRegularFileNoFollow(candidate, `workspace assertion ${relative}`, spec.workspace_root);
         try {
-          verdict = await sha256OfDescriptor(opened.fd) === evaluator.assertion.expected;
+          actualSha256 = await sha256OfDescriptor(opened.fd);
+          verdict = actualSha256 === evaluator.assertion.expected;
         } finally {
           fs81.closeSync(opened.fd);
         }
@@ -80564,15 +80676,32 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
         }
       }
     }
-    return { ...base2, status: verdict ? "passed" : "failed", verdict, duration_ms: Date.now() - started };
+    return {
+      ...base2,
+      status: verdict ? "passed" : "failed",
+      verdict,
+      duration_ms: Date.now() - started,
+      details: {
+        operator: evaluator.assertion.operator,
+        path: relative,
+        exists: exists2,
+        ...stat ? {
+          actual_kind: stat.isFile() ? "file" : stat.isDirectory() ? "directory" : "other",
+          actual_size_bytes: stat.size
+        } : {},
+        ...actualSha256 ? { actual_sha256: actualSha256 } : {}
+      }
+    };
   }
   let isolatedWorkspace;
+  let isolatedTestsRoot;
   let privateResultRoot;
   try {
     const evaluatorWorkspace = await copyFrozenWorkspace(manifest, spec, context);
     isolatedWorkspace = evaluatorWorkspace;
-    const testsPath = evaluatorTestsPath(evaluator, spec);
-    const commandRoot = evaluator.root === "workspace" ? evaluatorWorkspace : spec.tests_root;
+    isolatedTestsRoot = await copyEvaluatorTests(spec, context);
+    const testsPath = evaluatorTestsPath(evaluator, isolatedTestsRoot);
+    const commandRoot = evaluator.root === "workspace" ? evaluatorWorkspace : isolatedTestsRoot;
     const command = { id: evaluator.id, ...evaluator.command };
     const environment = {
       BRAINBASE_BENCHMARK_WORKSPACE: evaluatorWorkspace,
@@ -80623,7 +80752,7 @@ async function evaluateOne(evaluator, spec, manifest, finalOutput, trajectory, f
       ...criterionResults ? { criterion_results: criterionResults } : {}
     };
   } finally {
-    for (const temporary of [privateResultRoot, isolatedWorkspace]) {
+    for (const temporary of [privateResultRoot, isolatedTestsRoot, isolatedWorkspace]) {
       if (!temporary)
         continue;
       fs81.rmSync(temporary, { recursive: true, force: true });
@@ -80672,6 +80801,7 @@ async function executeEvaluate(spec, context) {
   ];
   outputs.push(...frozenEvidenceRecords);
   context.outputs.push(...frozenEvidenceRecords);
+  assertEvaluateOutputBudget(spec, context);
   assertBudget(context);
   const manifest = await workspaceManifest(spec, context);
   const manifestPath2 = path88.join(spec.logs_root, "candidate-workspace-manifest.json");
@@ -80684,6 +80814,7 @@ async function executeEvaluate(spec, context) {
   const manifestRecord = await recordFile(spec.logs_root, manifestPath2, "logs");
   outputs.push(manifestRecord);
   context.outputs.push(manifestRecord);
+  assertEvaluateOutputBudget(spec, context);
   for (const artifactRelInput of spec.candidate_artifacts) {
     const artifactRel = workspaceRel(artifactRelInput);
     assertNoSymlinkTraversal(spec.workspace_root, artifactRel);
@@ -80701,11 +80832,13 @@ async function executeEvaluate(spec, context) {
     outputs.push(artifact);
     context.outputs.push(artifact);
   }
+  assertEvaluateOutputBudget(spec, context);
   for (const candidateOutput of spec.candidate_outputs) {
     const copied = await copyCandidateOutput(candidateOutput, manifest, spec, context);
     outputs.push(...copied);
     context.outputs.push(...copied);
   }
+  assertEvaluateOutputBudget(spec, context);
   if (spec.capture_workspace_archive) {
     const regularFiles = manifest.filter((entry) => entry.kind !== "symlink").map((entry) => entry.path);
     const archive = path88.join(spec.logs_root, "candidate-workspace.tar.gz");
@@ -80719,6 +80852,7 @@ async function executeEvaluate(spec, context) {
     const archiveRecord = await recordFile(spec.logs_root, archive, "logs");
     outputs.push(archiveRecord);
     context.outputs.push(archiveRecord);
+    assertEvaluateOutputBudget(spec, context);
   }
   await verifyRecordsUnchanged(manifest, spec);
   for (const reference of spec.references) {
@@ -80777,12 +80911,16 @@ async function executeEvaluate(spec, context) {
     }
     if (requiredResultError)
       throw requiredResultError;
+    if (evaluator.type === "sandbox_command") {
+      assertEvaluateOutputBudget(spec, context);
+    }
   }
   await verifyRecordsUnchanged([...context.inputs, ...context.outputs.slice(0, frozenOutputCount)], spec);
   if (!verifyOwnedDirectory(spec.tests_root, "tests", spec) || !verifyOwnedDirectory(spec.logs_root, "logs", spec)) {
     throw new BenchmarkPhaseError("evidence_tampered", "benchmark evaluator changed an owned phase directory marker");
   }
   assertBudget(context);
+  assertEvaluateOutputBudget(spec, context);
   evaluators.sort((left, right) => evaluatorOrder.get(left.id) - evaluatorOrder.get(right.id));
   return { outputs, evaluators };
 }
@@ -80968,6 +81106,7 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
   let status = "failed";
   let error2;
   let context;
+  let validatedSpec;
   let resultPathValidated = false;
   try {
     let bytes;
@@ -80987,6 +81126,7 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
     }
     identity2 = rawIdentity(raw);
     const spec = BenchmarkSpecSchema.parse(raw);
+    validatedSpec = spec;
     if (expectedPhase && spec.phase !== expectedPhase) {
       throw new BenchmarkPhaseError("phase_mismatch", `the ${expectedPhase} command cannot execute a ${spec.phase} spec`);
     }
@@ -81066,6 +81206,17 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
   if (!resultPathValidated || !context?.logsOwned) {
     return { exitCode: status === "succeeded" ? 0 : 1, result: result2 };
   }
+  if (status === "succeeded" && validatedSpec?.phase === "evaluate") {
+    try {
+      const resultBytes = Buffer.byteLength(`${JSON.stringify(result2, null, 2)}
+`);
+      assertEvaluateOutputBudget(validatedSpec, context, resultBytes);
+    } catch (budgetError) {
+      status = "failed";
+      result2.status = "failed";
+      result2.error = stableError(budgetError);
+    }
+  }
   try {
     writeJsonAtomic(resultPath, result2);
   } catch (writeError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brainbase-labs/cli",
-  "version": "0.25.0-eng1209.3",
+  "version": "0.25.0-eng1209.5",
   "description": "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   "type": "module",
   "bin": {