@tangle-network/agent-eval 0.92.0 → 0.94.0

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-Y47J2LJ3.js";
+} from "./chunk-2KNZHH3P.js";
 import {
   classifyEuAiRisk,
   euAiActReport,
@@ -24,7 +24,7 @@ import {
   knowledgeReadinessTracePayload,
   scoreKnowledgeReadiness,
   userQuestionsForKnowledgeGaps
-} from "./chunk-3CKU6VGU.js";
+} from "./chunk-Q2JRAWRI.js";
 import {
   assertRecordIntegrity,
   checkRecordIntegrity,
@@ -65,12 +65,12 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-ZFIBGEOL.js";
+} from "./chunk-EGPMSBEZ.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-TWS7AZEY.js";
+} from "./chunk-2K6UUZ7P.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -92,7 +92,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-L3JOU6XM.js";
+} from "./chunk-NACM5RS7.js";
 import {
   analyzeSeries
 } from "./chunk-BOD4O7OF.js";
@@ -109,7 +109,7 @@ import {
   pytestTestParser,
   runTestGradedScenario,
   vitestTestParser
-} from "./chunk-T375SUOZ.js";
+} from "./chunk-ZOPLCJSY.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -129,11 +129,11 @@ import {
   diffFindings,
   resetLockedAppendersForTesting,
   runSemanticConceptJudge
-} from "./chunk-XY4DDNEG.js";
+} from "./chunk-MPQWFX6Y.js";
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics
-} from "./chunk-UMMZHCPB.js";
+} from "./chunk-MIFZUPEK.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -143,7 +143,7 @@ import {
   KNOWLEDGE_POISONING_KIND_SPEC,
   createTraceAnalystKind,
   renderPriorFindings
-} from "./chunk-D3V5B42D.js";
+} from "./chunk-Q5LIB7BC.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -154,25 +154,29 @@ import {
   runProposeReview,
   runProposeReviewAsControlLoop,
   scoreFromEvals
-} from "./chunk-RTWFUK6A.js";
+} from "./chunk-E4GH6USR.js";
 import {
   allCriticalPassed,
+  errorStreakDetector,
+  noProgressDetector,
   objectiveEval,
+  observeAll,
+  repeatedActionDetector,
   runAgentControlLoop,
   stopOnNoProgress,
   stopOnRepeatedAction,
   subjectiveEval
-} from "./chunk-NCRFYPS3.js";
+} from "./chunk-YEHAEDUD.js";
 import {
   assertReleaseConfidence,
   bootstrapCi,
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-4FBZZIYD.js";
+} from "./chunk-QTMYW64F.js";
 import {
   runEvalCampaign
-} from "./chunk-GSH6QNNS.js";
+} from "./chunk-KW53MSA5.js";
 import {
   LlmCallError,
   LlmClient,
@@ -184,7 +188,7 @@ import {
   isTransientLlmError,
   probeLlm,
   stripFencedJson
-} from "./chunk-CVVHBFGN.js";
+} from "./chunk-CWNP4DV4.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -195,7 +199,7 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-CY6U5S3X.js";
+} from "./chunk-URWVKRCS.js";
 import {
   attributeCounterfactuals,
   runCounterfactual
@@ -222,20 +226,26 @@ import {
   interRaterReliability,
   interpretCliffs,
   mannWhitneyU,
+  mcnemar,
+  mcnemarPower,
+  mcnemarRequiredN,
   mulberry32,
   normalizeScores,
   pairedBootstrap,
   pairedMde,
+  pairedRiskDifference,
   pairedTTest,
   partialCredit,
+  passAtK,
   positionalBias,
   requiredSampleSize,
   selfPreference,
   verbosityBias,
   weightedComposite,
   weightedMean,
-  wilcoxonSignedRank
-} from "./chunk-LMZQ2Z4U.js";
+  wilcoxonSignedRank,
+  wilson
+} from "./chunk-56GC6VYO.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -266,7 +276,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-QG2OVF2D.js";
+} from "./chunk-CTBHKLEU.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -282,7 +292,7 @@ import {
   runFailureClass,
   runsForScenario,
   toolSpans
-} from "./chunk-47X6LRCE.js";
+} from "./chunk-UFSG7ACU.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -297,7 +307,7 @@ import {
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-UHMJT4T7.js";
+} from "./chunk-TBDR6PAI.js";
 import {
   DEFAULT_TRACE_ANALYST_BUDGETS,
   OtlpFileTraceStore,
@@ -316,7 +326,7 @@ import {
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup
-} from "./chunk-QAY5UIJO.js";
+} from "./chunk-QMUEXQJS.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -883,6 +893,14 @@ function parseJudgeResponse(judgeName, resp) {
 }
 
 // src/executor.ts
+function describeShape(content, message) {
+  if (message === void 0 || message === null) return "no choices[0].message";
+  return `message.content of type ${content === null ? "null" : typeof content}`;
+}
+function errMessage(err) {
+  if (err instanceof Error) return `${err.name}: ${err.message}`;
+  return String(err);
+}
 async function executeScenario(tc, scenario, config) {
   const startTime = Date.now();
   const model = config.model ?? "gpt-4o";
@@ -903,7 +921,14 @@ async function executeScenario(tc, scenario, config) {
       temperature: 0.4,
       maxTokens: 3e3
     });
-    const content = resp.choices?.[0]?.message?.content ?? "";
+    const message = resp.choices?.[0]?.message;
+    const rawContent = message?.content;
+    if (message === void 0 || message === null || typeof rawContent !== "string") {
+      throw new CaptureIntegrityError(
+        `chat response for scenario "${scenario.id}" turn ${i} is malformed: expected choices[0].message.content to be a string, got ${describeShape(rawContent, message)}`
+      );
+    }
+    const content = rawContent;
     messages.push({ role: "assistant", content });
     const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
     let codeMatch = codeRe.exec(content);
@@ -984,26 +1009,44 @@ async function executeScenario(tc, scenario, config) {
   const judgeInput = { scenario, turns, artifacts };
   const judgeResults = [];
   let failedJudges = 0;
-  for (const judge of config.judges) {
+  const judgeFailures = [];
+  const sleep2 = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  for (let j = 0; j < config.judges.length; j++) {
+    const judge = config.judges[j];
+    const judgeName = judge.name || `judge[${j}]`;
+    let lastError;
+    let madeAttempts = 0;
     for (let attempt = 0; attempt < 3; attempt++) {
+      madeAttempts = attempt + 1;
       try {
         if (attempt > 0) {
           const wait = attempt * 1e4;
-          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
-          await new Promise((r) => setTimeout(r, wait));
+          console.log(`    ${judgeName} retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
+          await sleep2(wait);
         }
         const scores2 = await judge(tc, judgeInput);
         judgeResults.push(scores2);
-        await new Promise((r) => setTimeout(r, 3e3));
+        await sleep2(3e3);
+        lastError = void 0;
         break;
       } catch (err) {
+        lastError = err;
         if (err instanceof JudgeParseError) {
-          failedJudges++;
+          console.error(`    ${judgeName} unparseable (no retry): ${errMessage(err)}`);
           break;
         }
-        if (attempt === 2) failedJudges++;
+        console.error(`    ${judgeName} attempt ${attempt} failed: ${errMessage(err)}`);
+        if (!isTransientLlmError(err)) break;
       }
     }
+    if (lastError !== void 0) {
+      failedJudges++;
+      judgeFailures.push({
+        judge: judgeName,
+        attempts: madeAttempts,
+        reason: errMessage(lastError)
+      });
+    }
   }
   const allScores = judgeResults.flat();
   const errorScores = allScores.filter(
@@ -1023,7 +1066,7 @@ async function executeScenario(tc, scenario, config) {
       weight: weightMap.get(s.dimension) ?? 1
     }))
   );
-  return {
+  const result = {
     scenarioId: scenario.id,
     persona: scenario.persona,
     turns,
@@ -1034,6 +1077,8 @@ async function executeScenario(tc, scenario, config) {
     totalDurationMs: Date.now() - startTime,
     artifacts
   };
+  if (judgeFailures.length > 0) result.judgeFailures = judgeFailures;
+  return result;
 }
 
 // src/benchmark.ts
@@ -1170,19 +1215,67 @@ var BenchmarkRunner = class {
 };
 
 // src/client.ts
+var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
 var ProductClient = class {
   baseUrl;
   routes;
   cookies = "";
+  timeoutMs;
   constructor(config) {
     this.baseUrl = config.baseUrl.replace(/\/+$/, "");
     this.routes = config.routes;
+    this.timeoutMs = config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
   }
   route(name) {
     const path = this.routes[name];
     if (!path) throw new Error(`Route "${name}" not configured`);
     return path;
   }
+  /**
+   * Single HTTP boundary for every JSON request. Aborts after `timeoutMs`,
+   * checks `res.ok` BEFORE parsing, and throws `${method} ${path} failed:
+   * HTTP ${status} — ${body}` on a non-ok response so a 4xx/5xx error body
+   * can never be parsed as a success shape. Callers inspect the resolved
+   * value only after a successful return.
+   */
+  async request(method, path, body) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    let res;
+    try {
+      res = await fetch(`${this.baseUrl}${path}`, {
+        method,
+        headers: {
+          ...body ? { "Content-Type": "application/json" } : {},
+          Origin: this.baseUrl,
+          Cookie: this.cookies
+        },
+        ...body ? { body: JSON.stringify(body) } : {},
+        signal: controller.signal
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`${method} ${path} failed: HTTP ${res.status} \u2014 ${text.slice(0, 500)}`);
+    }
+    return res.json();
+  }
+  /**
+   * Read a required collection field. A missing/non-array field is a contract
+   * violation (wrong route, drift, partial body) — surface it loud instead of
+   * masking it as a healthy empty set. A genuinely empty `[]` passes through.
+   */
+  requireArray(res, field) {
+    const value = res[field];
+    if (!Array.isArray(value)) {
+      throw new Error(
+        `Response missing array field "${field}" (got ${typeof value}: ${JSON.stringify(value)?.slice(0, 200)})`
+      );
+    }
+    return value;
+  }
   async signup(name, email, password) {
     const res = await this.post(this.route("signup"), { name, email, password });
     const user = res.user;
@@ -1263,15 +1356,21 @@ var ProductClient = class {
   }
   async getTasks(workspaceId) {
     const res = await this.get(`${this.route("tasks")}?workspaceId=${workspaceId}`);
-    return res.tasks ?? [];
+    return this.requireArray(
+      res,
+      "tasks"
+    );
   }
   async getEvents(workspaceId) {
     const res = await this.get(`${this.route("events")}?workspaceId=${workspaceId}`);
-    return res.events ?? [];
+    return this.requireArray(res, "events");
   }
   async getApprovals(workspaceId) {
     const res = await this.get(`${this.route("approvals")}?workspaceId=${workspaceId}`);
-    return res.actions ?? [];
+    return this.requireArray(
+      res,
+      "actions"
+    );
   }
   async getVaultTree(workspaceId) {
     const res = await this.get(`${this.route("vault")}?workspaceId=${workspaceId}`);
@@ -1283,7 +1382,7 @@ var ProductClient = class {
         if (node.children) extract(node.children);
       }
     }
-    extract(res.tree ?? []);
+    extract(this.requireArray(res, "tree"));
     return paths;
   }
   async approveAction(workspaceId, id) {
@@ -1294,40 +1393,19 @@ var ProductClient = class {
   }
   async getGenerations(workspaceId) {
     const res = await this.get(`${this.route("generations")}?workspaceId=${workspaceId}`);
-    return res.generations ?? [];
+    return this.requireArray(res, "generations");
   }
   /** Generic GET for custom routes */
   async get(path) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      headers: { Cookie: this.cookies }
-    });
-    return res.json();
+    return this.request("GET", path);
   }
   /** Generic POST for custom routes */
   async post(path, body) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Origin: this.baseUrl,
-        Cookie: this.cookies
-      },
-      body: JSON.stringify(body)
-    });
-    return res.json();
+    return this.request("POST", path, body);
   }
   /** Generic PATCH for custom routes */
   async patch(path, body) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      method: "PATCH",
-      headers: {
-        "Content-Type": "application/json",
-        Origin: this.baseUrl,
-        Cookie: this.cookies
-      },
-      body: JSON.stringify(body)
-    });
-    return res.json();
+    return this.request("PATCH", path, body);
   }
 };
 async function runE2EWorkflow(client, name, workflow) {
@@ -1564,6 +1642,42 @@ Sign-off: respond with exactly "DONE" only when a ${persona.role} would act on t
 
 Output ONLY your next message to the agent \u2014 in character, first person, no meta-commentary, no stage directions.`;
 }
+function buildWorkerDriverSystemPrompt(ctx) {
+  const harness = ctx.harness ?? "a coding agent";
+  const brief = ctx.harnessBrief ? `
+This worker's harness (${harness}) can:
+${ctx.harnessBrief}
+Drive it to USE these \u2014 and never ask for a capability it lacks.
+` : `
+The worker runs in ${harness}. Drive it to exploit whatever its harness offers \u2014 parallel tool calls, sub-agents, run-to-completion \u2014 and never ask for a capability it lacks.
+`;
+  const progress = ctx.progress ? `
+What the worker has done so far:
+${ctx.progress}
+` : "\nThe worker has not started yet \u2014 your first instruction sets the whole plan.\n";
+  const context = ctx.context ? `
+Context:
+${ctx.context}
+` : "";
+  return `You are a DRIVER: a meta-agent whose entire job is to drive a capable coding worker to ACHIEVE A GOAL by writing it precise, high-signal instructions. You do not do the work yourself \u2014 you direct the worker to do it, brilliantly, and you hold it to a standard it would not hold itself to.
+
+The goal: ${ctx.goal}
+${context}${brief}${progress}
+THE BAR \u2014 non-negotiable. Every instruction you write is dense, specific, and complete. You write the message a world-class engineering lead writes to a strong report: the exact next objective, the concrete sub-steps, what to do in parallel vs in sequence, what to verify and how, what "done" looks like, and the failure modes to avoid. A thin steer \u2014 "try again", "fix the issues", a single vague sentence \u2014 is a failure on YOUR part, not the worker's. Out-drive a human power-user.
+
+HOW to drive (each turn):
+1. Read what the worker actually did (its trace/state above), not what it claims. Find the real gap between here and the goal.
+2. Decide the next objective \u2014 the largest correct step the worker can take now.
+3. Decompose it: name the sub-tasks that can run IN PARALLEL (independent files/checks/searches) and tell the worker to fan them out (sub-agents / parallel tool calls where its harness allows); name what must run in SEQUENCE and why.
+4. Make it exploit the harness: drive it to run to completion under its own autonomy, spawn sub-agents for separable work, use its tools/MCP, and not stop at the first plausible stopping point.
+5. Specify verification: the exact command / test / check that proves the step landed, and tell it to run that and report the result \u2014 never accept "done" without the check.
+6. Name the traps: the specific failure modes for THIS step (editing the wrong file, a check that passes vacuously, scope creep) and forbid them.
+7. As the task gets harder, decompose MORE, not less \u2014 more parallel branches, deeper sub-agent trees, tighter verification.
+
+COMPLETION: drive toward the goal's real acceptance check. Do not declare done \u2014 the deliverable's checker does. If the worker claims it is done, drive it to PROVE it with the check; if the check fails, drive the fix.
+
+Output ONLY your next instruction to the worker \u2014 direct, detailed, actionable, in the first person as the driver. No meta-commentary, no preamble.`;
+}
 async function decideNextUserTurn(tc, opts) {
   const { persona, state, history, productContext = "", model = "claude-sonnet-4-6" } = opts;
   const lastResponse = history.length > 0 ? history[history.length - 1].content.slice(0, 2e3) : "(no conversation yet \u2014 this is the first message)";
@@ -3485,6 +3599,7 @@ function ensembleJudge(opts) {
 }
 
 // src/judge-runner.ts
+import { cpus } from "os";
 var JudgeRunner = class {
   driver;
   constructor(driver = new SubprocessSandboxDriver()) {
@@ -3514,11 +3629,22 @@ var JudgeRunner = class {
 async function runJudgeFleet(specs, options = {}) {
   const runner = new JudgeRunner(options.driver);
   if (options.parallel === false) {
-    const results = [];
-    for (const spec of specs) results.push(await runner.run(spec));
-    return results;
+    const results2 = [];
+    for (const spec of specs) results2.push(await runner.run(spec));
+    return results2;
+  }
+  const concurrency = Math.max(1, options.concurrency ?? cpus().length);
+  const results = new Array(specs.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= specs.length) return;
+      results[i] = await runner.run(specs[i]);
+    }
   }
-  return await Promise.all(specs.map((spec) => runner.run(spec)));
+  await Promise.all(Array.from({ length: Math.min(concurrency, specs.length) }, () => worker()));
+  return results;
 }
 function compilerJudge(id, config) {
   return { id, kind: "compiler", config };
@@ -6781,8 +6907,15 @@ async function runSelfPlay(proposer, scorer, targets, options = {}) {
 
 // src/command-runner.ts
 import { spawnSync } from "child_process";
-import { existsSync as existsSync3, readdirSync as readdirSync2, readFileSync as readFileSync3, statSync as statSync2 } from "fs";
-import { join as join2 } from "path";
+import {
+  accessSync,
+  existsSync as existsSync3,
+  constants as fsConstants,
+  readdirSync as readdirSync2,
+  readFileSync as readFileSync3,
+  statSync as statSync2
+} from "fs";
+import { delimiter, join as join2 } from "path";
 var localCommandRunner = {
   name: "local",
   async run(input) {
@@ -6806,8 +6939,7 @@ var localCommandRunner = {
     };
   },
   async hasBin(name) {
-    const r = spawnSync("which", [name], { encoding: "utf8", timeout: 2e3 });
-    return r.status === 0 && (r.stdout ?? "").trim().length > 0;
+    return resolveBinOnPath(name) !== null;
   },
   async fileExists(path) {
     return existsSync3(path);
@@ -6815,16 +6947,18 @@ var localCommandRunner = {
   async readFile(path) {
     try {
       return readFileSync3(path, "utf8");
-    } catch {
-      return null;
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return null;
+      throw err;
     }
   },
   async readDir(path) {
     let entries;
     try {
       entries = readdirSync2(path);
-    } catch {
-      return [];
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return [];
+      throw err;
     }
     const out = [];
     for (const name of entries) {
@@ -6836,12 +6970,43 @@ var localCommandRunner = {
           isFile: st.isFile(),
           sizeBytes: st.isFile() ? st.size : null
         });
-      } catch {
+      } catch (err) {
+        if (isErrnoCode(err, "ENOENT")) continue;
+        throw err;
       }
     }
     return out;
   }
 };
+function isErrnoCode(err, code) {
+  return typeof err === "object" && err !== null && err.code === code;
+}
+function resolveBinOnPath(name) {
+  if (name.includes("/") || name.includes("\\")) {
+    return isExecutable(name) ? name : null;
+  }
+  const pathEnv = process.env.PATH ?? "";
+  const dirs = pathEnv.split(delimiter).filter((d) => d.length > 0);
+  const exts = process.platform === "win32" ? (process.env.PATHEXT ?? ".COM;.EXE;.BAT;.CMD").split(";").filter((e) => e.length > 0) : [""];
+  for (const dir of dirs) {
+    for (const ext of exts) {
+      const candidate = join2(dir, name + ext);
+      if (isExecutable(candidate)) return candidate;
+    }
+  }
+  return null;
+}
+function isExecutable(path) {
+  try {
+    accessSync(path, process.platform === "win32" ? fsConstants.F_OK : fsConstants.X_OK);
+    return true;
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "EACCES") || isErrnoCode(err, "ENOTDIR")) {
+      return false;
+    }
+    throw err;
+  }
+}
 
 // src/deploy-gate-layer.ts
 function deployGateLayer(input) {
@@ -8957,8 +9122,10 @@ function createSandboxPool(opts) {
   const waiters = [];
   const mutex = new Mutex();
   let nextSlotId = 0;
+  let reserving = 0;
   let totalCheckouts = 0;
   let busyMs = 0;
+  let drained = false;
   const startedAt = Date.now();
   async function acquireSlot() {
     let mintId;
@@ -8968,42 +9135,81 @@ function createSandboxPool(opts) {
         idle.busy = true;
         return idle;
       }
-      if (slots.length < opts.size) {
+      if (slots.length + reserving < opts.size) {
         mintId = `slot_${nextSlotId++}`;
+        reserving++;
         return null;
       }
       return null;
     });
     if (ready) return ready;
     if (mintId !== void 0) {
-      const resource = await opts.factory.create(mintId);
+      let resource;
+      try {
+        resource = await opts.factory.create(mintId);
+      } catch (err) {
+        await mutex.runExclusive(() => {
+          reserving--;
+        });
+        if (!drained) wakeWaiterToMint();
+        throw err;
+      }
       const state = {
         slot: { id: mintId, resource },
         busy: true
       };
       await mutex.runExclusive(() => {
         slots.push(state);
+        reserving--;
       });
       return state;
     }
-    return new Promise((resolve) => {
-      waiters.push((s) => {
-        s.busy = true;
-        resolve(s);
-      });
+    return new Promise((resolve, reject) => {
+      waiters.push({ resolve, reject });
     });
   }
+  function handOffCleanSlot(state) {
+    const next = waiters.shift();
+    if (next) {
+      state.busy = true;
+      next.resolve(state);
+    }
+  }
+  function wakeWaiterToMint() {
+    const next = waiters.shift();
+    if (!next) return;
+    acquireSlot().then(next.resolve, next.reject);
+  }
   function releaseSlot(state) {
     void (async () => {
       try {
-        if (opts.factory.reset) await opts.factory.reset(state.slot);
-      } catch (err) {
-        console.warn(`[sandbox-pool] reset failed for slot ${state.slot.id}:`, err);
+        if (opts.factory.reset) {
+          await opts.factory.reset(state.slot);
+        }
+        state.busy = false;
+        if (!drained) handOffCleanSlot(state);
+      } catch (resetErr) {
+        await mutex.runExclusive(() => {
+          const i = slots.indexOf(state);
+          if (i !== -1) slots.splice(i, 1);
+        });
+        try {
+          await opts.factory.destroy(state.slot);
+        } catch (destroyErr) {
+          console.warn(
+            `[sandbox-pool] destroy of dirty slot ${state.slot.id} failed after reset error:`,
+            destroyErr
+          );
+        }
+        console.warn(
+          `[sandbox-pool] reset failed for slot ${state.slot.id} \u2014 slot destroyed, not recycled:`,
+          resetErr
+        );
+        if (!drained) wakeWaiterToMint();
       }
-      state.busy = false;
-      const next = waiters.shift();
-      if (next) next(state);
-    })();
+    })().catch((err) => {
+      console.warn(`[sandbox-pool] release of slot ${state.slot.id} faulted:`, err);
+    });
   }
   async function checkout() {
     const state = await acquireSlot();
@@ -9026,13 +9232,15 @@ function createSandboxPool(opts) {
     }
   }
   async function drain() {
-    const snapshot = await mutex.runExclusive(() => {
+    drained = true;
+    const { snapshot, pending } = await mutex.runExclusive(() => {
       const taken = slots.splice(0, slots.length);
-      for (const w of waiters.splice(0, waiters.length)) {
-        void w;
-      }
-      return taken;
+      const queued = waiters.splice(0, waiters.length);
+      return { snapshot: taken, pending: queued };
     });
+    for (const w of pending) {
+      w.reject(new Error("sandbox pool drained while a checkout was pending"));
+    }
     await Promise.allSettled(snapshot.map((s) => opts.factory.destroy(s.slot)));
   }
   function utilization() {
@@ -10033,6 +10241,7 @@ export {
   buildTraceInsightContext,
   buildTraceInsightPrompt,
   buildTrajectory,
+  buildWorkerDriverSystemPrompt,
   byteLengthRange,
   cachedJudge,
   calibrateJudge,
@@ -10122,6 +10331,7 @@ export {
   dominates,
   eProcess,
   ensembleJudge,
+  errorStreakDetector,
   estimateCost,
   estimateTokens,
   euAiActReport,
@@ -10226,6 +10436,9 @@ export {
   mannWhitneyU,
   matchGoldens,
   matchSpan,
+  mcnemar,
+  mcnemarPower,
+  mcnemarRequiredN,
   mergeLayerResults,
   mergeSteeringBundle,
   modelDescriptionBits,
@@ -10233,15 +10446,18 @@ export {
   mulberry32,
   multiToolchainLayer,
   nistAiRmfReport,
+  noProgressDetector,
   normalizeScores,
   notBlocked,
   objectiveEval,
+  observeAll,
   otelRunCompleteHook,
   otlpToRunRecords,
   otlpToTraceRunRecords,
   pairedBootstrap,
   pairedEvalueSequence,
   pairedMde,
+  pairedRiskDifference,
   pairedTTest,
   paraphraseRobustness,
   paraphraseRobustnessScenarios,
@@ -10256,6 +10472,7 @@ export {
   parseRuntimeTrajectoryHookEvent,
   partialCredit,
   partitionHeldOut,
+  passAtK,
   passOrthogonality,
   pixelDeltaRatio,
   planTraceInsightQuestions,
@@ -10290,6 +10507,7 @@ export {
   renderPriorFindings,
   renderReleaseReport,
   renderSteeringText,
+  repeatedActionDetector,
   replayFeedbackTrajectories,
   replayFeedbackTrajectory,
   replayScorerOverCorpus,
@@ -10395,6 +10613,7 @@ export {
   welchsTTest,
   whitespaceCollapseMutator,
   wilcoxonSignedRank,
+  wilson,
   withAssignedFeedbackSplit,
   withJudgeRetry,
   withOtelPipeline,