@brainbase-labs/cli 0.25.0-eng1209.9 → 0.25.0

package/dist/index.js

@@ -36008,7 +36008,7 @@ function padStart(s, n) {
 // package.json
 var package_default = {
   name: "@brainbase-labs/cli",
-  version: "0.25.0-eng1209.9",
+  version: "0.25.0",
   description: "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   type: "module",
   bin: {
@@ -78485,6 +78485,8 @@ var MAX_TRAJECTORY_BYTES = 100 * 1024 * 1024;
 var MAX_REMOTE_INPUT_BYTES = 2 * 1024 * 1024 * 1024;
 var MAX_ARCHIVE_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024;
 var MAX_ARCHIVE_SCAN_BYTES = 2 * 1024 * 1024 * 1024;
+var MAX_HYDRATE_COMMANDS = 150;
+var MAX_PHASE_RESULT_BYTES = 32 * 1024 * 1024;
 var MAX_SANDBOX_COMMANDS = 20;
 var MAX_CRITERIA_PER_EVALUATOR = 200;
 var MAX_CRITERIA_RESULT_BYTES = 24 * 1024 * 1024;
@@ -78496,6 +78498,7 @@ var MAX_ALLOWED_EVIDENCE_IDS = 1e4;
 var COMMAND_PROCESS_MARKER_ENV = "BRAINBASE_BENCHMARK_COMMAND_MARKER";
 var COMMAND_PROCESS_POLL_MS = 10;
 var COMMAND_PROCESS_CLEANUP_MS = 500;
+var JUDGE_ERROR_PREFIX = "BRAINBASE_BENCHMARK_JUDGE_ERROR_V1:";
 var RESERVED_WORKSPACE_PATHS = new Set([
   ".brainbase",
   ".git",
@@ -78662,7 +78665,7 @@ var TrajectoryEvidenceSchema = EvidenceFileSchema.extend({
 var HydrateSpecSchema = BaseSpecSchema.extend({
   phase: exports_external.literal("hydrate"),
   materials: exports_external.array(MaterialSchema).max(1e4).default([]),
-  setup_commands: exports_external.array(CommandSchema).max(128).default([])
+  setup_commands: exports_external.array(CommandSchema).max(MAX_HYDRATE_COMMANDS).default([])
 }).strict();
 var CandidateOutputSchema = exports_external.object({
   id: IdSchema,
@@ -78803,12 +78806,12 @@ var BenchmarkSpecSchema = exports_external.discriminatedUnion("phase", [
     { items: value.candidate_outputs, path: "candidate_outputs" }
   ];
   for (const { items, path: issuePath } of uniqueLists) {
-    const ids = items.map((item) => item.id);
+    const ids = items.map((item) => issuePath === "candidate_outputs" ? item.id.toLowerCase() : item.id);
     if (new Set(ids).size !== ids.length) {
       context.addIssue({
         code: exports_external.ZodIssueCode.custom,
         path: [issuePath],
-        message: `${issuePath} ids must be unique`
+        message: `${issuePath} ids must be unique${issuePath === "candidate_outputs" ? " ignoring case" : ""}`
       });
     }
   }
@@ -78871,6 +78874,7 @@ var BENCHMARK_CAPABILITIES = {
     "remote_input_references_v1",
     "archive_file_materials_v1",
     "structured_criterion_results_v1",
+    "structured_judge_errors_v1",
     "multiple_sandbox_commands_v1",
     "sandbox_command_workspace_modes_v1",
     "candidate_outputs_v1",
@@ -78884,6 +78888,8 @@ var BENCHMARK_CAPABILITIES = {
   ],
   limits: {
     max_secret_bindings: 100,
+    max_hydrate_commands: MAX_HYDRATE_COMMANDS,
+    max_phase_result_bytes: MAX_PHASE_RESULT_BYTES,
     max_evaluators: 1000,
     max_sandbox_commands: MAX_SANDBOX_COMMANDS,
     max_criteria_per_evaluator: MAX_CRITERIA_PER_EVALUATOR
@@ -80683,6 +80689,29 @@ var CriteriaResultFileSchema = exports_external.object({
     });
   }
 });
+var JudgeErrorSchema = exports_external.object({
+  code: exports_external.enum([
+    "judge_provider_quota_exhausted",
+    "judge_rate_limited",
+    "judge_provider_authentication_failed",
+    "judge_provider_unavailable",
+    "judge_request_failed"
+  ]),
+  message: exports_external.string().min(1).max(500)
+}).strict();
+function structuredJudgeError(stderr) {
+  for (const line of stderr.toString("utf8").split(/\r?\n/).reverse()) {
+    const marker = line.indexOf(JUDGE_ERROR_PREFIX);
+    if (marker < 0)
+      continue;
+    try {
+      const parsed = JudgeErrorSchema.safeParse(JSON.parse(line.slice(marker + JUDGE_ERROR_PREFIX.length)));
+      if (parsed.success)
+        return parsed.data;
+    } catch {}
+  }
+  return null;
+}
 function readCriterionResults(resultPath, criterionKeys, allowedEvidenceIds, spec, additionalSecrets = []) {
   let opened;
   try {
@@ -80929,7 +80958,13 @@ async function evaluateOne(evaluator, spec, manifest, frozenWorkspaceRoot, final
     try {
       criterionResults = evaluator.criterion_keys && criterionResultPath ? readCriterionResults(criterionResultPath, evaluator.criterion_keys, evaluator.allowed_evidence_ids ?? [], spec, result2.redactions) : undefined;
     } catch (error2) {
-      const normalized = stableError(error2);
+      let normalized = stableError(error2);
+      if (normalized?.code === "missing_criterion_result" && result2.exitCode !== 0) {
+        normalized = structuredJudgeError(result2.stderr) ?? {
+          code: "command_terminated",
+          message: "sandbox evaluator terminated before writing its criterion result"
+        };
+      }
       return {
         ...base2,
         status: "errored",
@@ -81165,6 +81200,67 @@ function stableError(error2) {
     message: error2 instanceof Error ? error2.message : "unknown benchmark phase failure"
   };
 }
+function compactOversizedResult(result2) {
+  const actualBytes = Buffer.byteLength(`${JSON.stringify(result2, null, 2)}
+`);
+  if (actualBytes <= MAX_PHASE_RESULT_BYTES)
+    return result2;
+  return {
+    schema_version: result2.schema_version,
+    cli_version: result2.cli_version,
+    phase: result2.phase,
+    attempt_id: result2.attempt_id,
+    phase_id: result2.phase_id,
+    spec_digest: result2.spec_digest,
+    status: "failed",
+    started_at: result2.started_at,
+    completed_at: result2.completed_at,
+    duration_ms: result2.duration_ms,
+    steps: [],
+    inputs: [],
+    outputs: [],
+    error: {
+      code: "result_too_large",
+      message: "benchmark phase result exceeds the 32 MiB contract limit",
+      details: {
+        actual_bytes: actualBytes,
+        max_bytes: MAX_PHASE_RESULT_BYTES,
+        step_count: result2.steps.length,
+        input_count: result2.inputs.length,
+        output_count: result2.outputs.length,
+        evaluator_count: result2.evaluators?.length ?? 0
+      }
+    }
+  };
+}
+function readCachedPhaseResult(resultPath) {
+  let opened;
+  try {
+    opened = openRegularFileNoFollow(resultPath, "cached benchmark phase result");
+  } catch {
+    return;
+  }
+  try {
+    if (opened.stat.size > MAX_PHASE_RESULT_BYTES)
+      return;
+    const chunks = [];
+    let offset = 0;
+    while (offset <= MAX_PHASE_RESULT_BYTES) {
+      const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_PHASE_RESULT_BYTES + 1 - offset));
+      const bytesRead = fs81.readSync(opened.fd, buffer, 0, buffer.length, offset);
+      if (bytesRead === 0) {
+        return JSON.parse(Buffer.concat(chunks, offset).toString("utf8"));
+      }
+      chunks.push(buffer.subarray(0, bytesRead));
+      offset += bytesRead;
+    }
+    return;
+  } catch {
+    return;
+  } finally {
+    fs81.closeSync(opened.fd);
+  }
+}
 function rawIdentity(value) {
   if (!value || typeof value !== "object") {
     return { phase: "unknown", attemptId: null, phaseId: null };
@@ -81237,13 +81333,9 @@ function prepareBenchmarkPhaseInvocation(specPathInput, resultPathInput, expecte
       timeout_ms: spec.budget.timeout_ms
     };
     let cachedResult;
-    if (fs81.existsSync(resultPath)) {
-      try {
-        const cached2 = JSON.parse(fs81.readFileSync(resultPath, "utf8"));
-        if (cached2.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
-          cachedResult = cached2;
-        }
-      } catch {}
+    const cached2 = readCachedPhaseResult(resultPath);
+    if (cached2?.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
+      cachedResult = cached2;
     }
     if (cachedResult) {
       if (spec.phase === "hydrate") {
@@ -81260,7 +81352,7 @@ function prepareBenchmarkPhaseInvocation(specPathInput, resultPathInput, expecte
   } catch (error2) {
     return {
       ok: false,
-      result: {
+      result: compactOversizedResult({
         schema_version: SCHEMA_VERSION,
         cli_version: VERSION,
         phase: identity2.phase === "unknown" ? expectedPhase : identity2.phase,
@@ -81275,7 +81367,7 @@ function prepareBenchmarkPhaseInvocation(specPathInput, resultPathInput, expecte
         inputs: [],
         outputs: [],
         error: stableError(error2)
-      }
+      })
     };
   }
 }
@@ -81355,13 +81447,9 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
     }
     resultPathValidated = true;
     let cachedResult;
-    if (fs81.existsSync(resultPath)) {
-      try {
-        const cached2 = JSON.parse(fs81.readFileSync(resultPath, "utf8"));
-        if (cached2.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
-          cachedResult = cached2;
-        }
-      } catch {}
+    const cached2 = readCachedPhaseResult(resultPath);
+    if (cached2?.schema_version === SCHEMA_VERSION && cached2.phase === spec.phase && cached2.attempt_id === spec.attempt_id && cached2.phase_id === spec.phase_id && cached2.spec_digest === digest && cached2.status === "succeeded") {
+      cachedResult = cached2;
     }
     if (cachedResult) {
       if (spec.phase === "hydrate") {
@@ -81404,7 +81492,7 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
       fs81.rmSync(temporary, { recursive: true, force: true });
     }
   }
-  const result2 = {
+  let result2 = {
     schema_version: SCHEMA_VERSION,
     cli_version: VERSION,
     phase: identity2.phase === "unknown" && expectedPhase ? expectedPhase : identity2.phase,
@@ -81422,6 +81510,7 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
     ...error2 ? { error: error2 } : {}
   };
   if (!resultPathValidated || !context?.logsOwned) {
+    result2 = compactOversizedResult(result2);
     return { exitCode: status === "succeeded" ? 0 : 1, result: result2 };
   }
   if (status === "succeeded" && validatedSpec?.phase === "evaluate") {
@@ -81435,6 +81524,8 @@ async function runBenchmarkPhase(specPathInput, resultPathInput, expectedPhase,
       result2.error = stableError(budgetError);
     }
   }
+  result2 = compactOversizedResult(result2);
+  status = result2.status;
   try {
     writeJsonAtomic(resultPath, result2);
   } catch (writeError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@brainbase-labs/cli",
-  "version": "0.25.0-eng1209.9",
+  "version": "0.25.0",
   "description": "Pack, share, and install agent templates across harnesses (Claude Code, Codex, ...).",
   "type": "module",
   "bin": {