@tangle-network/agent-eval 0.93.0 → 0.94.0

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   checkBehavioralCanary,
   checkCanaries,
   runBehavioralCanaries
-} from "./chunk-Y47J2LJ3.js";
+} from "./chunk-2KNZHH3P.js";
 import {
   classifyEuAiRisk,
   euAiActReport,
@@ -65,12 +65,12 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-ZFIBGEOL.js";
+} from "./chunk-EGPMSBEZ.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-TWS7AZEY.js";
+} from "./chunk-2K6UUZ7P.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -92,7 +92,7 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-L3JOU6XM.js";
+} from "./chunk-NACM5RS7.js";
 import {
   analyzeSeries
 } from "./chunk-BOD4O7OF.js";
@@ -109,7 +109,7 @@ import {
   pytestTestParser,
   runTestGradedScenario,
   vitestTestParser
-} from "./chunk-T375SUOZ.js";
+} from "./chunk-ZOPLCJSY.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -129,11 +129,11 @@ import {
   diffFindings,
   resetLockedAppendersForTesting,
   runSemanticConceptJudge
-} from "./chunk-XY4DDNEG.js";
+} from "./chunk-MPQWFX6Y.js";
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics
-} from "./chunk-UMMZHCPB.js";
+} from "./chunk-MIFZUPEK.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -143,7 +143,7 @@ import {
   KNOWLEDGE_POISONING_KIND_SPEC,
   createTraceAnalystKind,
   renderPriorFindings
-} from "./chunk-D3V5B42D.js";
+} from "./chunk-Q5LIB7BC.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -173,10 +173,10 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-4FBZZIYD.js";
+} from "./chunk-QTMYW64F.js";
 import {
   runEvalCampaign
-} from "./chunk-GSH6QNNS.js";
+} from "./chunk-KW53MSA5.js";
 import {
   LlmCallError,
   LlmClient,
@@ -188,7 +188,7 @@ import {
   isTransientLlmError,
   probeLlm,
   stripFencedJson
-} from "./chunk-CVVHBFGN.js";
+} from "./chunk-CWNP4DV4.js";
 import {
   evaluateInterimReleaseConfidence,
   pairedEvalueSequence
@@ -199,7 +199,7 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-CY6U5S3X.js";
+} from "./chunk-URWVKRCS.js";
 import {
   attributeCounterfactuals,
   runCounterfactual
@@ -226,20 +226,26 @@ import {
   interRaterReliability,
   interpretCliffs,
   mannWhitneyU,
+  mcnemar,
+  mcnemarPower,
+  mcnemarRequiredN,
   mulberry32,
   normalizeScores,
   pairedBootstrap,
   pairedMde,
+  pairedRiskDifference,
   pairedTTest,
   partialCredit,
+  passAtK,
   positionalBias,
   requiredSampleSize,
   selfPreference,
   verbosityBias,
   weightedComposite,
   weightedMean,
-  wilcoxonSignedRank
-} from "./chunk-LMZQ2Z4U.js";
+  wilcoxonSignedRank,
+  wilson
+} from "./chunk-56GC6VYO.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -270,7 +276,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-QG2OVF2D.js";
+} from "./chunk-CTBHKLEU.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -286,7 +292,7 @@ import {
   runFailureClass,
   runsForScenario,
   toolSpans
-} from "./chunk-47X6LRCE.js";
+} from "./chunk-UFSG7ACU.js";
 import {
   FAILURE_CLASSES,
   TRACE_SCHEMA_VERSION,
@@ -301,7 +307,7 @@ import {
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
-} from "./chunk-UHMJT4T7.js";
+} from "./chunk-TBDR6PAI.js";
 import {
   DEFAULT_TRACE_ANALYST_BUDGETS,
   OtlpFileTraceStore,
@@ -320,7 +326,7 @@ import {
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup
-} from "./chunk-QAY5UIJO.js";
+} from "./chunk-QMUEXQJS.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -887,6 +893,14 @@ function parseJudgeResponse(judgeName, resp) {
 }
 
 // src/executor.ts
+function describeShape(content, message) {
+  if (message === void 0 || message === null) return "no choices[0].message";
+  return `message.content of type ${content === null ? "null" : typeof content}`;
+}
+function errMessage(err) {
+  if (err instanceof Error) return `${err.name}: ${err.message}`;
+  return String(err);
+}
 async function executeScenario(tc, scenario, config) {
   const startTime = Date.now();
   const model = config.model ?? "gpt-4o";
@@ -907,7 +921,14 @@ async function executeScenario(tc, scenario, config) {
       temperature: 0.4,
       maxTokens: 3e3
     });
-    const content = resp.choices?.[0]?.message?.content ?? "";
+    const message = resp.choices?.[0]?.message;
+    const rawContent = message?.content;
+    if (message === void 0 || message === null || typeof rawContent !== "string") {
+      throw new CaptureIntegrityError(
+        `chat response for scenario "${scenario.id}" turn ${i} is malformed: expected choices[0].message.content to be a string, got ${describeShape(rawContent, message)}`
+      );
+    }
+    const content = rawContent;
     messages.push({ role: "assistant", content });
     const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
     let codeMatch = codeRe.exec(content);
@@ -988,26 +1009,44 @@ async function executeScenario(tc, scenario, config) {
   const judgeInput = { scenario, turns, artifacts };
   const judgeResults = [];
   let failedJudges = 0;
-  for (const judge of config.judges) {
+  const judgeFailures = [];
+  const sleep2 = config.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  for (let j = 0; j < config.judges.length; j++) {
+    const judge = config.judges[j];
+    const judgeName = judge.name || `judge[${j}]`;
+    let lastError;
+    let madeAttempts = 0;
     for (let attempt = 0; attempt < 3; attempt++) {
+      madeAttempts = attempt + 1;
       try {
         if (attempt > 0) {
           const wait = attempt * 1e4;
-          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
-          await new Promise((r) => setTimeout(r, wait));
+          console.log(`    ${judgeName} retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
+          await sleep2(wait);
         }
         const scores2 = await judge(tc, judgeInput);
         judgeResults.push(scores2);
-        await new Promise((r) => setTimeout(r, 3e3));
+        await sleep2(3e3);
+        lastError = void 0;
         break;
       } catch (err) {
+        lastError = err;
         if (err instanceof JudgeParseError) {
-          failedJudges++;
+          console.error(`    ${judgeName} unparseable (no retry): ${errMessage(err)}`);
           break;
         }
-        if (attempt === 2) failedJudges++;
+        console.error(`    ${judgeName} attempt ${attempt} failed: ${errMessage(err)}`);
+        if (!isTransientLlmError(err)) break;
       }
     }
+    if (lastError !== void 0) {
+      failedJudges++;
+      judgeFailures.push({
+        judge: judgeName,
+        attempts: madeAttempts,
+        reason: errMessage(lastError)
+      });
+    }
   }
   const allScores = judgeResults.flat();
   const errorScores = allScores.filter(
@@ -1027,7 +1066,7 @@ async function executeScenario(tc, scenario, config) {
       weight: weightMap.get(s.dimension) ?? 1
     }))
   );
-  return {
+  const result = {
     scenarioId: scenario.id,
     persona: scenario.persona,
     turns,
@@ -1038,6 +1077,8 @@ async function executeScenario(tc, scenario, config) {
     totalDurationMs: Date.now() - startTime,
     artifacts
   };
+  if (judgeFailures.length > 0) result.judgeFailures = judgeFailures;
+  return result;
 }
 
 // src/benchmark.ts
@@ -1174,19 +1215,67 @@ var BenchmarkRunner = class {
 };
 
 // src/client.ts
+var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
 var ProductClient = class {
   baseUrl;
   routes;
   cookies = "";
+  timeoutMs;
   constructor(config) {
     this.baseUrl = config.baseUrl.replace(/\/+$/, "");
     this.routes = config.routes;
+    this.timeoutMs = config.timeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
   }
   route(name) {
     const path = this.routes[name];
     if (!path) throw new Error(`Route "${name}" not configured`);
     return path;
   }
+  /**
+   * Single HTTP boundary for every JSON request. Aborts after `timeoutMs`,
+   * checks `res.ok` BEFORE parsing, and throws `${method} ${path} failed:
+   * HTTP ${status} — ${body}` on a non-ok response so a 4xx/5xx error body
+   * can never be parsed as a success shape. Callers inspect the resolved
+   * value only after a successful return.
+   */
+  async request(method, path, body) {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
+    let res;
+    try {
+      res = await fetch(`${this.baseUrl}${path}`, {
+        method,
+        headers: {
+          ...body ? { "Content-Type": "application/json" } : {},
+          Origin: this.baseUrl,
+          Cookie: this.cookies
+        },
+        ...body ? { body: JSON.stringify(body) } : {},
+        signal: controller.signal
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      throw new Error(`${method} ${path} failed: HTTP ${res.status} \u2014 ${text.slice(0, 500)}`);
+    }
+    return res.json();
+  }
+  /**
+   * Read a required collection field. A missing/non-array field is a contract
+   * violation (wrong route, drift, partial body) — surface it loud instead of
+   * masking it as a healthy empty set. A genuinely empty `[]` passes through.
+   */
+  requireArray(res, field) {
+    const value = res[field];
+    if (!Array.isArray(value)) {
+      throw new Error(
+        `Response missing array field "${field}" (got ${typeof value}: ${JSON.stringify(value)?.slice(0, 200)})`
+      );
+    }
+    return value;
+  }
   async signup(name, email, password) {
     const res = await this.post(this.route("signup"), { name, email, password });
     const user = res.user;
@@ -1267,15 +1356,21 @@ var ProductClient = class {
   }
   async getTasks(workspaceId) {
     const res = await this.get(`${this.route("tasks")}?workspaceId=${workspaceId}`);
-    return res.tasks ?? [];
+    return this.requireArray(
+      res,
+      "tasks"
+    );
   }
   async getEvents(workspaceId) {
     const res = await this.get(`${this.route("events")}?workspaceId=${workspaceId}`);
-    return res.events ?? [];
+    return this.requireArray(res, "events");
   }
   async getApprovals(workspaceId) {
     const res = await this.get(`${this.route("approvals")}?workspaceId=${workspaceId}`);
-    return res.actions ?? [];
+    return this.requireArray(
+      res,
+      "actions"
+    );
   }
   async getVaultTree(workspaceId) {
     const res = await this.get(`${this.route("vault")}?workspaceId=${workspaceId}`);
@@ -1287,7 +1382,7 @@ var ProductClient = class {
         if (node.children) extract(node.children);
       }
     }
-    extract(res.tree ?? []);
+    extract(this.requireArray(res, "tree"));
     return paths;
   }
   async approveAction(workspaceId, id) {
@@ -1298,40 +1393,19 @@ var ProductClient = class {
   }
   async getGenerations(workspaceId) {
     const res = await this.get(`${this.route("generations")}?workspaceId=${workspaceId}`);
-    return res.generations ?? [];
+    return this.requireArray(res, "generations");
   }
   /** Generic GET for custom routes */
   async get(path) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      headers: { Cookie: this.cookies }
-    });
-    return res.json();
+    return this.request("GET", path);
   }
   /** Generic POST for custom routes */
   async post(path, body) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Origin: this.baseUrl,
-        Cookie: this.cookies
-      },
-      body: JSON.stringify(body)
-    });
-    return res.json();
+    return this.request("POST", path, body);
   }
   /** Generic PATCH for custom routes */
   async patch(path, body) {
-    const res = await fetch(`${this.baseUrl}${path}`, {
-      method: "PATCH",
-      headers: {
-        "Content-Type": "application/json",
-        Origin: this.baseUrl,
-        Cookie: this.cookies
-      },
-      body: JSON.stringify(body)
-    });
-    return res.json();
+    return this.request("PATCH", path, body);
   }
 };
 async function runE2EWorkflow(client, name, workflow) {
@@ -3525,6 +3599,7 @@ function ensembleJudge(opts) {
 }
 
 // src/judge-runner.ts
+import { cpus } from "os";
 var JudgeRunner = class {
   driver;
   constructor(driver = new SubprocessSandboxDriver()) {
@@ -3554,11 +3629,22 @@ var JudgeRunner = class {
 async function runJudgeFleet(specs, options = {}) {
   const runner = new JudgeRunner(options.driver);
   if (options.parallel === false) {
-    const results = [];
-    for (const spec of specs) results.push(await runner.run(spec));
-    return results;
+    const results2 = [];
+    for (const spec of specs) results2.push(await runner.run(spec));
+    return results2;
+  }
+  const concurrency = Math.max(1, options.concurrency ?? cpus().length);
+  const results = new Array(specs.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= specs.length) return;
+      results[i] = await runner.run(specs[i]);
+    }
   }
-  return await Promise.all(specs.map((spec) => runner.run(spec)));
+  await Promise.all(Array.from({ length: Math.min(concurrency, specs.length) }, () => worker()));
+  return results;
 }
 function compilerJudge(id, config) {
   return { id, kind: "compiler", config };
@@ -6821,8 +6907,15 @@ async function runSelfPlay(proposer, scorer, targets, options = {}) {
 
 // src/command-runner.ts
 import { spawnSync } from "child_process";
-import { existsSync as existsSync3, readdirSync as readdirSync2, readFileSync as readFileSync3, statSync as statSync2 } from "fs";
-import { join as join2 } from "path";
+import {
+  accessSync,
+  existsSync as existsSync3,
+  constants as fsConstants,
+  readdirSync as readdirSync2,
+  readFileSync as readFileSync3,
+  statSync as statSync2
+} from "fs";
+import { delimiter, join as join2 } from "path";
 var localCommandRunner = {
   name: "local",
   async run(input) {
@@ -6846,8 +6939,7 @@ var localCommandRunner = {
     };
   },
   async hasBin(name) {
-    const r = spawnSync("which", [name], { encoding: "utf8", timeout: 2e3 });
-    return r.status === 0 && (r.stdout ?? "").trim().length > 0;
+    return resolveBinOnPath(name) !== null;
   },
   async fileExists(path) {
     return existsSync3(path);
@@ -6855,16 +6947,18 @@ var localCommandRunner = {
   async readFile(path) {
     try {
       return readFileSync3(path, "utf8");
-    } catch {
-      return null;
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return null;
+      throw err;
     }
   },
   async readDir(path) {
     let entries;
     try {
       entries = readdirSync2(path);
-    } catch {
-      return [];
+    } catch (err) {
+      if (isErrnoCode(err, "ENOENT")) return [];
+      throw err;
     }
     const out = [];
     for (const name of entries) {
@@ -6876,12 +6970,43 @@ var localCommandRunner = {
           isFile: st.isFile(),
           sizeBytes: st.isFile() ? st.size : null
         });
-      } catch {
+      } catch (err) {
+        if (isErrnoCode(err, "ENOENT")) continue;
+        throw err;
       }
     }
     return out;
   }
 };
+function isErrnoCode(err, code) {
+  return typeof err === "object" && err !== null && err.code === code;
+}
+function resolveBinOnPath(name) {
+  if (name.includes("/") || name.includes("\\")) {
+    return isExecutable(name) ? name : null;
+  }
+  const pathEnv = process.env.PATH ?? "";
+  const dirs = pathEnv.split(delimiter).filter((d) => d.length > 0);
+  const exts = process.platform === "win32" ? (process.env.PATHEXT ?? ".COM;.EXE;.BAT;.CMD").split(";").filter((e) => e.length > 0) : [""];
+  for (const dir of dirs) {
+    for (const ext of exts) {
+      const candidate = join2(dir, name + ext);
+      if (isExecutable(candidate)) return candidate;
+    }
+  }
+  return null;
+}
+function isExecutable(path) {
+  try {
+    accessSync(path, process.platform === "win32" ? fsConstants.F_OK : fsConstants.X_OK);
+    return true;
+  } catch (err) {
+    if (isErrnoCode(err, "ENOENT") || isErrnoCode(err, "EACCES") || isErrnoCode(err, "ENOTDIR")) {
+      return false;
+    }
+    throw err;
+  }
+}
 
 // src/deploy-gate-layer.ts
 function deployGateLayer(input) {
@@ -8997,8 +9122,10 @@ function createSandboxPool(opts) {
   const waiters = [];
   const mutex = new Mutex();
   let nextSlotId = 0;
+  let reserving = 0;
   let totalCheckouts = 0;
   let busyMs = 0;
+  let drained = false;
   const startedAt = Date.now();
   async function acquireSlot() {
     let mintId;
@@ -9008,42 +9135,81 @@ function createSandboxPool(opts) {
         idle.busy = true;
         return idle;
       }
-      if (slots.length < opts.size) {
+      if (slots.length + reserving < opts.size) {
         mintId = `slot_${nextSlotId++}`;
+        reserving++;
         return null;
       }
       return null;
     });
     if (ready) return ready;
     if (mintId !== void 0) {
-      const resource = await opts.factory.create(mintId);
+      let resource;
+      try {
+        resource = await opts.factory.create(mintId);
+      } catch (err) {
+        await mutex.runExclusive(() => {
+          reserving--;
+        });
+        if (!drained) wakeWaiterToMint();
+        throw err;
+      }
       const state = {
         slot: { id: mintId, resource },
         busy: true
       };
       await mutex.runExclusive(() => {
         slots.push(state);
+        reserving--;
       });
       return state;
     }
-    return new Promise((resolve) => {
-      waiters.push((s) => {
-        s.busy = true;
-        resolve(s);
-      });
+    return new Promise((resolve, reject) => {
+      waiters.push({ resolve, reject });
     });
   }
+  function handOffCleanSlot(state) {
+    const next = waiters.shift();
+    if (next) {
+      state.busy = true;
+      next.resolve(state);
+    }
+  }
+  function wakeWaiterToMint() {
+    const next = waiters.shift();
+    if (!next) return;
+    acquireSlot().then(next.resolve, next.reject);
+  }
   function releaseSlot(state) {
     void (async () => {
       try {
-        if (opts.factory.reset) await opts.factory.reset(state.slot);
-      } catch (err) {
-        console.warn(`[sandbox-pool] reset failed for slot ${state.slot.id}:`, err);
+        if (opts.factory.reset) {
+          await opts.factory.reset(state.slot);
+        }
+        state.busy = false;
+        if (!drained) handOffCleanSlot(state);
+      } catch (resetErr) {
+        await mutex.runExclusive(() => {
+          const i = slots.indexOf(state);
+          if (i !== -1) slots.splice(i, 1);
+        });
+        try {
+          await opts.factory.destroy(state.slot);
+        } catch (destroyErr) {
+          console.warn(
+            `[sandbox-pool] destroy of dirty slot ${state.slot.id} failed after reset error:`,
+            destroyErr
+          );
+        }
+        console.warn(
+          `[sandbox-pool] reset failed for slot ${state.slot.id} \u2014 slot destroyed, not recycled:`,
+          resetErr
+        );
+        if (!drained) wakeWaiterToMint();
       }
-      state.busy = false;
-      const next = waiters.shift();
-      if (next) next(state);
-    })();
+    })().catch((err) => {
+      console.warn(`[sandbox-pool] release of slot ${state.slot.id} faulted:`, err);
+    });
   }
   async function checkout() {
     const state = await acquireSlot();
@@ -9066,13 +9232,15 @@ function createSandboxPool(opts) {
     }
   }
   async function drain() {
-    const snapshot = await mutex.runExclusive(() => {
+    drained = true;
+    const { snapshot, pending } = await mutex.runExclusive(() => {
       const taken = slots.splice(0, slots.length);
-      for (const w of waiters.splice(0, waiters.length)) {
-        void w;
-      }
-      return taken;
+      const queued = waiters.splice(0, waiters.length);
+      return { snapshot: taken, pending: queued };
     });
+    for (const w of pending) {
+      w.reject(new Error("sandbox pool drained while a checkout was pending"));
+    }
     await Promise.allSettled(snapshot.map((s) => opts.factory.destroy(s.slot)));
   }
   function utilization() {
@@ -10268,6 +10436,9 @@ export {
   mannWhitneyU,
   matchGoldens,
   matchSpan,
+  mcnemar,
+  mcnemarPower,
+  mcnemarRequiredN,
   mergeLayerResults,
   mergeSteeringBundle,
   modelDescriptionBits,
@@ -10286,6 +10457,7 @@ export {
   pairedBootstrap,
   pairedEvalueSequence,
   pairedMde,
+  pairedRiskDifference,
   pairedTTest,
   paraphraseRobustness,
   paraphraseRobustnessScenarios,
@@ -10300,6 +10472,7 @@ export {
   parseRuntimeTrajectoryHookEvent,
   partialCredit,
   partitionHeldOut,
+  passAtK,
   passOrthogonality,
   pixelDeltaRatio,
   planTraceInsightQuestions,
@@ -10440,6 +10613,7 @@ export {
   welchsTTest,
   whitespaceCollapseMutator,
   wilcoxonSignedRank,
+  wilson,
   withAssignedFeedbackSplit,
   withJudgeRetry,
   withOtelPipeline,