@tangle-network/agent-eval 0.85.0 → 0.89.0

package/dist/index.js

@@ -1,22 +1,35 @@
+import {
+  HoldoutAuditor,
+  analyzeRuns,
+  canaryLeakView,
+  checkBehavioralCanary,
+  checkCanaries,
+  runBehavioralCanaries
+} from "./chunk-Y47J2LJ3.js";
+import {
+  classifyEuAiRisk,
+  euAiActReport,
+  nistAiRmfReport,
+  renderMarkdown,
+  soc2Report,
+  summarize
+} from "./chunk-KKHDIONI.js";
+import {
+  acquisitionPlansForKnowledgeGaps,
+  blockingKnowledgeEval,
+  knowledgeReadinessTracePayload,
+  scoreKnowledgeReadiness,
+  userQuestionsForKnowledgeGaps
+} from "./chunk-3CKU6VGU.js";
 import {
   agentProfileHash,
+  completionVerdict,
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
   extractProducedState,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-YGYXHNAQ.js";
-import {
-  parseRuntimeTrajectoryHookEvent,
-  projectRuntimeTrajectoryEvidence
-} from "./chunk-T4SQEITX.js";
-import {
-  HoldoutAuditor,
-  canaryLeakView,
-  checkBehavioralCanary,
-  checkCanaries,
-  runBehavioralCanaries
-} from "./chunk-SHTXZ4O2.js";
+} from "./chunk-FIUKOSWI.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -40,12 +53,16 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-BABOZOSN.js";
+} from "./chunk-ZFIBGEOL.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-ZZ2HOPME.js";
+} from "./chunk-TWS7AZEY.js";
+import {
+  parseRuntimeTrajectoryHookEvent,
+  projectRuntimeTrajectoryEvidence
+} from "./chunk-T4SQEITX.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -67,14 +84,14 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-RBNA5AZT.js";
+} from "./chunk-L3JOU6XM.js";
+import {
+  analyzeSeries
+} from "./chunk-BOD4O7OF.js";
 import {
   exportTrainingData,
   toNdjson
 } from "./chunk-KMPRBJK4.js";
-import {
-  buildTrajectory
-} from "./chunk-RZTMDUO7.js";
 import {
   DockerSandboxDriver,
   SandboxHarness,
@@ -85,21 +102,6 @@ import {
   runTestGradedScenario,
   vitestTestParser
 } from "./chunk-T375SUOZ.js";
-import {
-  classifyEuAiRisk,
-  euAiActReport,
-  nistAiRmfReport,
-  renderMarkdown,
-  soc2Report,
-  summarize
-} from "./chunk-KKHDIONI.js";
-import {
-  acquisitionPlansForKnowledgeGaps,
-  blockingKnowledgeEval,
-  knowledgeReadinessTracePayload,
-  scoreKnowledgeReadiness,
-  userQuestionsForKnowledgeGaps
-} from "./chunk-3CKU6VGU.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -111,9 +113,7 @@ import {
   SKILL_USAGE_ANALYST,
   SkillUsageAnalyst,
   aggregateRunScore,
-  buildDefaultAnalystRegistry,
   clamp01,
-  computeTraceMetrics,
   createAnalystAi,
   createChatClient,
   createSemanticConceptJudge,
@@ -121,7 +121,11 @@ import {
   diffFindings,
   resetLockedAppendersForTesting,
   runSemanticConceptJudge
-} from "./chunk-L5G7OUKD.js";
+} from "./chunk-XY4DDNEG.js";
+import {
+  buildDefaultAnalystRegistry,
+  computeTraceMetrics
+} from "./chunk-UMMZHCPB.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -129,11 +133,9 @@ import {
   IMPROVEMENT_KIND_SPEC,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
-  computeFindingId,
   createTraceAnalystKind,
-  makeFinding,
   renderPriorFindings
-} from "./chunk-VIDQF3F5.js";
+} from "./chunk-D3V5B42D.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -159,10 +161,10 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-FZWAFVAA.js";
+} from "./chunk-4FBZZIYD.js";
 import {
   runEvalCampaign
-} from "./chunk-WJL2NJXN.js";
+} from "./chunk-GSH6QNNS.js";
 import {
   LlmCallError,
   LlmClient,
@@ -185,7 +187,18 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-Z7VFTS2J.js";
+} from "./chunk-CY6U5S3X.js";
+import {
+  attributeCounterfactuals,
+  runCounterfactual
+} from "./chunk-REVYNR6C.js";
+import {
+  buildTrajectory
+} from "./chunk-RZTMDUO7.js";
+import {
+  computeFindingId,
+  makeFinding
+} from "./chunk-45EEMHTC.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -197,9 +210,11 @@ import {
   continuousAgreement,
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
+  eProcess,
   interRaterReliability,
   interpretCliffs,
   mannWhitneyU,
+  mulberry32,
   normalizeScores,
   pairedBootstrap,
   pairedMde,
@@ -212,7 +227,7 @@ import {
   weightedComposite,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-IDVBLYCY.js";
+} from "./chunk-LMZQ2Z4U.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -230,6 +245,9 @@ import {
   describeTraceInsightScope,
   domainEvidencePattern,
   exportRunAsOtlp,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   flattenOtlpExportToNdjson,
   inferDomainKeywords,
   iterateRawCalls,
@@ -240,7 +258,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-XQL22JDG.js";
+} from "./chunk-QG2OVF2D.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -267,16 +285,18 @@ import {
   isToolSpan
 } from "./chunk-5BKGXME7.js";
 import {
-  DEFAULT_TRACE_ANALYST_BUDGETS,
-  OtlpFileTraceStore,
-  SpanNotFoundError,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  analyzeTraces
+} from "./chunk-UHMJT4T7.js";
+import {
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  OtlpFileTraceStore,
+  SpanNotFoundError,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TraceFileMissingError,
   TraceNotFoundError,
-  analyzeTraces,
   asNumber,
   asString,
   buildTraceAnalystTools,
@@ -288,7 +308,7 @@ import {
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup
-} from "./chunk-VUINJM5M.js";
+} from "./chunk-QAY5UIJO.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -643,132 +663,337 @@ function renderCommitMessage(input) {
   return lines.join("\n").trim();
 }
 
-// src/executor.ts
-async function executeScenario(tc, scenario, config) {
-  const startTime = Date.now();
-  const model = config.model ?? "gpt-4o";
-  const systemPrompt = [config.systemPrompt, scenario.systemPromptAppend ?? ""].filter(Boolean).join("\n\n");
-  const messages = [{ role: "system", content: systemPrompt }];
-  const turns = [];
-  const allCodeBlocks = [];
-  const allBlocks = [];
-  const allToolCalls = [];
-  const blockRe = config.blockPattern ?? /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
-  for (let i = 0; i < scenario.turns.length; i++) {
-    const turn = scenario.turns[i];
-    const turnStart = Date.now();
-    messages.push({ role: "user", content: turn.user });
+// src/judges.ts
+var JudgeParseError = class extends JudgeError {
+  /** Name of the judge whose response failed to parse. */
+  judgeName;
+  /** The raw (truncated) model response that failed to parse. */
+  raw;
+  constructor(judgeName, raw, options) {
+    super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options);
+    this.judgeName = judgeName;
+    this.raw = raw;
+  }
+};
+function createDomainExpertJudge(domain) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
     const resp = await tc.chat({
-      model,
-      messages,
-      temperature: 0.4,
-      maxTokens: 3e3
-    });
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    messages.push({ role: "assistant", content });
-    const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
-    let codeMatch = codeRe.exec(content);
-    while (codeMatch !== null) {
-      allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] ?? "" });
-      codeMatch = codeRe.exec(content);
-    }
-    const turnBlocks = [];
-    const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
-    let blockMatch = blockReLocal.exec(content);
-    while (blockMatch !== null) {
-      const fields = {};
-      for (const line of (blockMatch[2] ?? "").split("\n")) {
-        const idx = line.indexOf(":");
-        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
-      }
-      const blockType = blockMatch[1] ?? "";
-      allBlocks.push({ type: blockType, fields });
-      turnBlocks.push({ type: blockType, title: fields.title ?? "" });
-      blockMatch = blockReLocal.exec(content);
-    }
-    let hasToolCall = false;
-    if (config.toolCallPatterns) {
-      for (const pattern of config.toolCallPatterns) {
-        const re = new RegExp(pattern.source, pattern.flags);
-        let toolMatch = re.exec(content);
-        while (toolMatch !== null) {
-          allToolCalls.push(toolMatch[0]);
-          hasToolCall = true;
-          toolMatch = re.exec(content);
+      model: "gpt-4o",
+      messages: [
+        {
+          role: "system",
+          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
+
+Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
+
+Evaluate:
+1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
+2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
+
+Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
+        },
+        {
+          role: "user",
+          content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
         }
-      }
-    }
-    turns.push({
-      turnIndex: i,
-      userMessage: turn.user,
-      agentResponse: content,
-      durationMs: Date.now() - turnStart,
-      blocksExtracted: turnBlocks,
-      containsCode: allCodeBlocks.length > 0,
-      containsToolCall: hasToolCall
+      ],
+      temperature: 0.1,
+      maxTokens: 800
     });
-  }
-  const artifacts = {
-    vaultFiles: [],
-    blocksExtracted: allBlocks,
-    codeBlocks: allCodeBlocks,
-    toolCalls: allToolCalls
+    return parseJudgeResponse("domain_expert", resp);
   };
-  const artifactResults = scenario.artifactChecks.map((check2) => {
-    if (config.artifactChecker) {
-      const custom = config.artifactChecker(check2, artifacts);
-      if (custom) return { check: check2, ...custom };
-    }
-    switch (check2.type) {
-      case "block_extracted": {
-        const count = allBlocks.filter((b) => b.type === check2.target).length;
-        return {
-          check: check2,
-          passed: count >= (check2.minCount ?? 1),
-          detail: `Found ${count} ${check2.target} blocks (need ${check2.minCount ?? 1})`
-        };
+}
+var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
+  const codeBlocks = artifacts.codeBlocks;
+  if (codeBlocks.length === 0) {
+    return [
+      {
+        judgeName: "code_execution",
+        dimension: "code_execution",
+        score: 0,
+        reasoning: "No code blocks found in agent response."
       }
-      case "code_valid": {
-        const hasCode = allCodeBlocks.some(
-          (b) => b.language === check2.target || b.code.includes(check2.target)
-        );
-        return { check: check2, passed: hasCode, detail: hasCode ? "Code block found" : "No matching code" };
+    ];
+  }
+  const codeText = codeBlocks.map(
+    (b, i) => `Block ${i + 1} (${b.language}):
+\`\`\`${b.language}
+${b.code.slice(0, 3e3)}
+\`\`\``
+  ).join("\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You are a principal software engineer reviewing code written by an AI agent.
+
+Score STRICTLY:
+1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
+2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
+3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
+
+Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Task: ${scenario.thesis}
+
+${codeText}`
       }
-      default:
-        return {
-          check: check2,
-          passed: false,
-          detail: `Check type "${check2.type}" requires live environment`
-        };
-    }
+    ],
+    temperature: 0.1,
+    maxTokens: 1e3
   });
-  const judgeInput = { scenario, turns, artifacts };
-  const judgeResults = [];
-  for (const judge of config.judges) {
-    let lastErr = "";
-    for (let attempt = 0; attempt < 3; attempt++) {
-      try {
-        if (attempt > 0) {
-          const wait = attempt * 1e4;
-          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
-          await new Promise((r) => setTimeout(r, wait));
-        }
-        const scores2 = await judge(tc, judgeInput);
-        judgeResults.push(scores2);
-        await new Promise((r) => setTimeout(r, 3e3));
-        break;
-      } catch (err) {
-        lastErr = err instanceof Error ? err.message : String(err);
-        if (attempt === 2) {
-          judgeResults.push([
-            {
-              judgeName: "unknown",
-              dimension: "error",
-              score: 0,
-              reasoning: `Judge failed after 3 attempts: ${lastErr.slice(0, 200)}`
-            }
-          ]);
+  return parseJudgeResponse("code_execution", resp);
+};
+var coherenceJudge = async (tc, { scenario, turns }) => {
+  if (turns.length < 2) {
+    return [];
+  }
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
+
+Score STRICTLY:
+1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
+2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
+3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
+
+Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Scenario: ${scenario.thesis}
+
+${conversation}`
+      }
+    ],
+    temperature: 0.1,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("coherence", resp);
+};
+var adversarialJudge = async (tc, { scenario, turns }) => {
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
+
+1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
+2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
+3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
+
+Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
+
+Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Persona: ${scenario.persona}
+Scenario: ${scenario.thesis}
+
+${conversation}`
+      }
+    ],
+    temperature: 0.2,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("adversarial", resp);
+};
+function createCustomJudge(name, systemPrompt, opts) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
+    const resp = await tc.chat({
+      model: opts?.model ?? "gpt-4o",
+      messages: [
+        {
+          role: "system",
+          content: systemPrompt
+        },
+        {
+          role: "user",
+          content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
+        }
+      ],
+      temperature: opts?.temperature ?? 0.1,
+      maxTokens: opts?.maxTokens ?? 1e3
+    });
+    return parseJudgeResponse(name, resp);
+  };
+}
+function defaultJudges(domain) {
+  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
+}
+function parseJudgeResponse(judgeName, resp) {
+  const content = resp.choices?.[0]?.message?.content ?? "";
+  try {
+    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
+    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+    if (arrayMatch) cleaned = arrayMatch[0];
+    const parsed = JSON.parse(cleaned);
+    return parsed.map((p) => ({
+      judgeName,
+      dimension: p.dimension,
+      score: Math.max(0, Math.min(10, p.score)),
+      reasoning: p.reasoning ?? "",
+      evidence: p.evidence
+    }));
+  } catch (err) {
+    throw new JudgeParseError(judgeName, content, { cause: err });
+  }
+}
+
+// src/executor.ts
+async function executeScenario(tc, scenario, config) {
+  const startTime = Date.now();
+  const model = config.model ?? "gpt-4o";
+  const systemPrompt = [config.systemPrompt, scenario.systemPromptAppend ?? ""].filter(Boolean).join("\n\n");
+  const messages = [{ role: "system", content: systemPrompt }];
+  const turns = [];
+  const allCodeBlocks = [];
+  const allBlocks = [];
+  const allToolCalls = [];
+  const blockRe = config.blockPattern ?? /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
+  for (let i = 0; i < scenario.turns.length; i++) {
+    const turn = scenario.turns[i];
+    const turnStart = Date.now();
+    messages.push({ role: "user", content: turn.user });
+    const resp = await tc.chat({
+      model,
+      messages,
+      temperature: 0.4,
+      maxTokens: 3e3
+    });
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    messages.push({ role: "assistant", content });
+    const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
+    let codeMatch = codeRe.exec(content);
+    while (codeMatch !== null) {
+      allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] ?? "" });
+      codeMatch = codeRe.exec(content);
+    }
+    const turnBlocks = [];
+    const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
+    let blockMatch = blockReLocal.exec(content);
+    while (blockMatch !== null) {
+      const fields = {};
+      for (const line of (blockMatch[2] ?? "").split("\n")) {
+        const idx = line.indexOf(":");
+        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+      }
+      const blockType = blockMatch[1] ?? "";
+      allBlocks.push({ type: blockType, fields });
+      turnBlocks.push({ type: blockType, title: fields.title ?? "" });
+      blockMatch = blockReLocal.exec(content);
+    }
+    let hasToolCall = false;
+    if (config.toolCallPatterns) {
+      for (const pattern of config.toolCallPatterns) {
+        const re = new RegExp(pattern.source, pattern.flags);
+        let toolMatch = re.exec(content);
+        while (toolMatch !== null) {
+          allToolCalls.push(toolMatch[0]);
+          hasToolCall = true;
+          toolMatch = re.exec(content);
+        }
+      }
+    }
+    turns.push({
+      turnIndex: i,
+      userMessage: turn.user,
+      agentResponse: content,
+      durationMs: Date.now() - turnStart,
+      blocksExtracted: turnBlocks,
+      containsCode: allCodeBlocks.length > 0,
+      containsToolCall: hasToolCall
+    });
+  }
+  const artifacts = {
+    vaultFiles: [],
+    blocksExtracted: allBlocks,
+    codeBlocks: allCodeBlocks,
+    toolCalls: allToolCalls
+  };
+  const artifactResults = scenario.artifactChecks.map((check2) => {
+    if (config.artifactChecker) {
+      const custom = config.artifactChecker(check2, artifacts);
+      if (custom) return { check: check2, ...custom };
+    }
+    switch (check2.type) {
+      case "block_extracted": {
+        const count = allBlocks.filter((b) => b.type === check2.target).length;
+        return {
+          check: check2,
+          passed: count >= (check2.minCount ?? 1),
+          detail: `Found ${count} ${check2.target} blocks (need ${check2.minCount ?? 1})`
+        };
+      }
+      case "code_valid": {
+        const hasCode = allCodeBlocks.some(
+          (b) => b.language === check2.target || b.code.includes(check2.target)
+        );
+        return { check: check2, passed: hasCode, detail: hasCode ? "Code block found" : "No matching code" };
+      }
+      default:
+        return {
+          check: check2,
+          passed: false,
+          detail: `Check type "${check2.type}" requires live environment`
+        };
+    }
+  });
+  const judgeInput = { scenario, turns, artifacts };
+  const judgeResults = [];
+  let failedJudges = 0;
+  for (const judge of config.judges) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        if (attempt > 0) {
+          const wait = attempt * 1e4;
+          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
+          await new Promise((r) => setTimeout(r, wait));
+        }
+        const scores2 = await judge(tc, judgeInput);
+        judgeResults.push(scores2);
+        await new Promise((r) => setTimeout(r, 3e3));
+        break;
+      } catch (err) {
+        if (err instanceof JudgeParseError) {
+          failedJudges++;
+          break;
         }
+        if (attempt === 2) failedJudges++;
       }
     }
   }
@@ -796,7 +1021,7 @@ async function executeScenario(tc, scenario, config) {
     turns,
     artifactResults,
     judgeScores: allScores,
-    judgeErrors: errorScores.length,
+    judgeErrors: errorScores.length + failedJudges,
     overallScore,
     totalDurationMs: Date.now() - startTime,
     artifacts
@@ -1448,11 +1673,11 @@ var FileSystemFeedbackTrajectoryStore = class {
   }
   async load() {
     if (this.loaded) return;
-    const { readFile } = await import("fs/promises");
+    const { readFile: readFile2 } = await import("fs/promises");
     const { join: join4 } = await import("path");
     const file = join4(this.dir, "feedback-trajectories.ndjson");
     try {
-      const raw = await readFile(file, "utf8");
+      const raw = await readFile2(file, "utf8");
       for (const line of raw.split("\n")) {
         if (!line.trim()) continue;
         try {
@@ -1986,217 +2211,6 @@ function assertCrossFamily(models, opts = {}) {
   return list;
 }
 
-// src/judges.ts
-function createDomainExpertJudge(domain) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
-
-Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
-
-Evaluate:
-1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
-2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
-
-Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: 0.1,
-      maxTokens: 800
-    });
-    return parseJudgeResponse("domain_expert", resp);
-  };
-}
-var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
-  const codeBlocks = artifacts.codeBlocks;
-  if (codeBlocks.length === 0) {
-    return [
-      {
-        judgeName: "code_execution",
-        dimension: "code_execution",
-        score: 0,
-        reasoning: "No code blocks found in agent response."
-      }
-    ];
-  }
-  const codeText = codeBlocks.map(
-    (b, i) => `Block ${i + 1} (${b.language}):
-\`\`\`${b.language}
-${b.code.slice(0, 3e3)}
-\`\`\``
-  ).join("\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a principal software engineer reviewing code written by an AI agent.
-
-Score STRICTLY:
-1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
-2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
-3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
-
-Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Task: ${scenario.thesis}
-
-${codeText}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 1e3
-  });
-  return parseJudgeResponse("code_execution", resp);
-};
-var coherenceJudge = async (tc, { scenario, turns }) => {
-  if (turns.length < 2) {
-    return [];
-  }
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
-
-Score STRICTLY:
-1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
-2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
-3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
-
-Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("coherence", resp);
-};
-var adversarialJudge = async (tc, { scenario, turns }) => {
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
-
-1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
-2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
-3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
-
-Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
-
-Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Persona: ${scenario.persona}
-Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.2,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("adversarial", resp);
-};
-function createCustomJudge(name, systemPrompt, opts) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: opts?.model ?? "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: systemPrompt
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: opts?.temperature ?? 0.1,
-      maxTokens: opts?.maxTokens ?? 1e3
-    });
-    return parseJudgeResponse(name, resp);
-  };
-}
-function defaultJudges(domain) {
-  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
-}
-function parseJudgeResponse(judgeName, resp) {
-  try {
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
-    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
-    if (arrayMatch) cleaned = arrayMatch[0];
-    const parsed = JSON.parse(cleaned);
-    return parsed.map((p) => ({
-      judgeName,
-      dimension: p.dimension,
-      score: Math.max(0, Math.min(10, p.score)),
-      reasoning: p.reasoning ?? "",
-      evidence: p.evidence
-    }));
-  } catch (err) {
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    console.log(
-      `    [parse_error] ${judgeName}: ${err.message?.slice(0, 50)} | response: ${content.slice(0, 100)}`
-    );
-    return [
-      {
-        judgeName,
-        dimension: "parse_error",
-        score: 0,
-        reasoning: `Parse failed: ${err.message?.slice(0, 100)}. Raw: ${content.slice(0, 200)}`
-      }
-    ];
-  }
-}
-
 // src/live-proof.ts
 async function runLiveProof(config) {
   const startedAt = Date.now();
@@ -2391,16 +2405,16 @@ function aggregatePrReviewScore(dimensions, weights = {}) {
   return (merged.recall * clamp01(dimensions.recall) + merged.precision * clamp01(dimensions.precision) + merged.actionability * clamp01(dimensions.actionability) + merged.severityCalibration * clamp01(dimensions.severityCalibration) + merged.lowNoise * clamp01(dimensions.lowNoise)) / weightSum;
 }
 function matchReferenceFindings(references, comments) {
-  const matches = [];
+  const matches2 = [];
   const usedCommentIds = /* @__PURE__ */ new Set();
   for (const reference of references) {
     const candidates = comments.filter((comment) => !usedCommentIds.has(comment.id)).map((comment) => ({ comment, score: matchScore(reference, comment) })).filter(({ score }) => score >= 0.55).sort((a, b) => b.score - a.score);
     const best = candidates[0];
     if (!best) continue;
     usedCommentIds.add(best.comment.id);
-    matches.push({ referenceId: reference.id, commentId: best.comment.id, score: best.score });
+    matches2.push({ referenceId: reference.id, commentId: best.comment.id, score: best.score });
   }
-  return matches;
+  return matches2;
 }
 function matchScore(reference, comment) {
   let score = 0;
@@ -2427,9 +2441,9 @@ function isActionableComment(comment) {
     body
   );
 }
-function isSeverityAligned(comment, references, matches) {
+function isSeverityAligned(comment, references, matches2) {
   if (!comment.severity) return false;
-  const match = matches.find((candidate) => candidate.commentId === comment.id);
+  const match = matches2.find((candidate) => candidate.commentId === comment.id);
   if (!match) return comment.severity === "nit" || comment.severity === "low";
   const reference = references.find((candidate) => candidate.id === match.referenceId);
   if (!reference) return false;
@@ -2725,28 +2739,28 @@ function analyzeAntiSlop(outputs, config) {
       }
     }
     for (const re of config.hedgingPatterns) {
-      const matches = output.match(
+      const matches2 = output.match(
         new RegExp(re, re.flags.includes("g") ? re.flags : `${re.flags}g`)
       );
-      if (matches) {
-        counts.hedging += matches.length;
+      if (matches2) {
+        counts.hedging += matches2.length;
         issues.push({
           category: "hedging",
-          detail: `${matches.length}x ${re.source}`,
-          example: matches[0]
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
         });
       }
     }
     for (const re of config.apologyPatterns) {
-      const matches = output.match(
+      const matches2 = output.match(
         new RegExp(re, re.flags.includes("g") ? re.flags : `${re.flags}g`)
       );
-      if (matches) {
-        counts.apology += matches.length;
+      if (matches2) {
+        counts.apology += matches2.length;
         issues.push({
           category: "apology",
-          detail: `${matches.length}x ${re.source}`,
-          example: matches[0]
+          detail: `${matches2.length}x ${re.source}`,
+          example: matches2[0]
         });
       }
     }
@@ -3003,6 +3017,170 @@ var DualAgentBench = class {
   }
 };
 
+// src/eval-tools.ts
+import { readFile } from "fs/promises";
+function toOpenAiTool(def) {
+  return {
+    type: "function",
+    function: { name: def.name, description: def.description, parameters: def.parameters }
+  };
+}
+function makeEvalTools(cfg) {
+  const tools = [];
+  if (cfg.judges) tools.push(runJudgesTool(cfg.judges));
+  if (cfg.completion) tools.push(verifyCompletionTool(cfg.completion.checkCorrectness));
+  if (cfg.analyze) tools.push(analyzeRunsTool(cfg.analyze));
+  return tools;
+}
+function runJudgesTool(judges) {
+  if (judges.length === 0) {
+    throw new Error("makeEvalTools: cfg.judges is empty \u2014 supply judges or omit the section");
+  }
+  const names = judges.map((j) => j.name);
+  return {
+    name: "run_judges",
+    description: `Score an artifact with the configured judges (${names.join(", ")}). Pass \`judge\` to run one judge by name; omit it to run all. Returns per-judge scores keyed by judge name.`,
+    parameters: {
+      type: "object",
+      properties: {
+        judge: {
+          type: "string",
+          enum: names,
+          description: "Run only this judge. Omit to run every configured judge."
+        },
+        artifact: {
+          description: "The artifact to score \u2014 any JSON value the judges understand."
+        },
+        scenario: {
+          description: "Optional scenario context forwarded to the judges."
+        }
+      },
+      required: ["artifact"],
+      additionalProperties: false
+    },
+    handler: async (args, ctx) => {
+      const a = requireObjectArgs("run_judges", args);
+      if (!("artifact" in a)) {
+        throw new Error("run_judges: args.artifact is required");
+      }
+      let selected = judges;
+      if (a.judge !== void 0) {
+        if (typeof a.judge !== "string") throw new Error("run_judges: args.judge must be a string");
+        selected = judges.filter((j) => j.name === a.judge);
+        if (selected.length === 0) {
+          throw new Error(
+            `run_judges: unknown judge '${a.judge}' \u2014 configured: ${names.join(", ")}`
+          );
+        }
+      }
+      const signal = ctx?.signal ?? new AbortController().signal;
+      const scenario = a.scenario;
+      const scores2 = {};
+      for (const judge of selected) {
+        if (scenario !== void 0 && judge.appliesTo && !judge.appliesTo(scenario)) continue;
+        scores2[judge.name] = await judge.score({ artifact: a.artifact, scenario, signal });
+      }
+      return { scores: scores2 };
+    }
+  };
+}
+function verifyCompletionTool(checkCorrectness) {
+  return {
+    name: "verify_completion",
+    description: "Verify produced state against a gold task spec: each gold requirement is matched to at most one produced artifact/proposal/tool-call, correctness-checked by the host, and reduced to a CompletionVerdict (completionRate, fullyComplete, per-requirement checks).",
+    parameters: {
+      type: "object",
+      properties: {
+        gold: {
+          type: "object",
+          description: "TaskGold \u2014 taskId + requirements the produced state must satisfy."
+        },
+        state: {
+          type: "object",
+          description: "ProducedState \u2014 artifacts, proposals, and tool calls the agent produced."
+        }
+      },
+      required: ["gold", "state"],
+      additionalProperties: false
+    },
+    handler: async (args) => {
+      const a = requireObjectArgs("verify_completion", args);
+      if (typeof a.gold !== "object" || a.gold === null) {
+        throw new Error("verify_completion: args.gold (TaskGold) is required");
+      }
+      if (typeof a.state !== "object" || a.state === null) {
+        throw new Error("verify_completion: args.state (ProducedState) is required");
+      }
+      return verifyCompletion(a.gold, a.state, checkCorrectness);
+    }
+  };
+}
+function analyzeRunsTool(analyzeOpts) {
+  return {
+    name: "analyze_runs",
+    description: "Run analyzeRuns over a set of RunRecords and return the InsightReport (composite distribution, per-dimension stats, lift, failure clusters, recommendations). Pass the records inline via `runs`, or `path` to a .json (array) or .jsonl (one record per line) file on the host.",
+    parameters: {
+      type: "object",
+      properties: {
+        runs: {
+          type: "array",
+          items: { type: "object" },
+          description: "RunRecord[] inline."
+        },
+        path: {
+          type: "string",
+          description: "Host path to a JSON array or JSONL file of RunRecords."
+        }
+      },
+      additionalProperties: false
+    },
+    handler: async (args) => {
+      const a = requireObjectArgs("analyze_runs", args);
+      const hasRuns = Array.isArray(a.runs);
+      const hasPath = typeof a.path === "string" && a.path.length > 0;
+      if (hasRuns === hasPath) {
+        throw new Error("analyze_runs: pass exactly one of args.runs (array) or args.path (string)");
+      }
+      const runs = hasRuns ? a.runs : await loadRunRecords(a.path);
+      if (runs.length === 0) {
+        throw new Error("analyze_runs: no runs to analyze");
+      }
+      return analyzeRuns({ ...analyzeOpts, runs });
+    }
+  };
+}
+async function loadRunRecords(path) {
+  const text = await readFile(path, "utf8");
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    throw new Error(`analyze_runs: file '${path}' is empty`);
+  }
+  if (trimmed.startsWith("[")) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error(`analyze_runs: file '${path}' did not parse to an array`);
+    }
+    return parsed;
+  }
+  return trimmed.split("\n").flatMap((line, i) => {
+    const l = line.trim();
+    if (l.length === 0) return [];
+    try {
+      return [JSON.parse(l)];
+    } catch (err) {
+      throw new Error(
+        `analyze_runs: file '${path}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  });
+}
+function requireObjectArgs(tool, args) {
+  if (typeof args !== "object" || args === null || Array.isArray(args)) {
+    throw new Error(`${tool}: args must be an object`);
+  }
+  return args;
+}
+
 // src/harness-optimizer.ts
 var DEFAULT_HARNESS_OBJECTIVES = [
   { name: "aggregate", direction: "maximize", value: (r) => r.aggregateMean },
@@ -3016,14 +3194,14 @@ async function runHarnessExperiment(config) {
   const score = config.score ?? ((trace) => critic.scoreTrace(trace));
   const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
     const trace = await config.adapter.run(request);
-    const runScore3 = await score(trace, request);
+    const runScore4 = await score(trace, request);
     const result = {
       variant: request.variant,
       scenario: request.scenario,
       trialIndex: request.trialIndex,
       trace,
-      score: runScore3,
-      aggregate: aggregateRunScore(runScore3, config.weights)
+      score: runScore4,
+      aggregate: aggregateRunScore(runScore4, config.weights)
     };
     await config.onResult?.(result);
     return result;
@@ -3124,15 +3302,22 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
   for (const d of dimensionKeys) dimAcc[d] = [];
   let rationale = "";
   let costUsd = 0;
+  const seenCount = /* @__PURE__ */ new Map();
+  const keyFor = (model) => {
+    const n = (seenCount.get(model) ?? 0) + 1;
+    seenCount.set(model, n);
+    return n === 1 ? model : `${model}#${n}`;
+  };
   for (const v of verdicts) {
     costUsd += v.costUsd ?? 0;
+    const key = keyFor(v.model);
     if (!v.perDimension) {
-      failedJudges.push(v.model);
+      failedJudges.push(key);
       continue;
     }
     const dims = {};
     for (const d of dimensionKeys) dims[d] = clamp01(Number(v.perDimension[d]));
-    perJudge[v.model] = dims;
+    perJudge[key] = dims;
     for (const d of dimensionKeys) dimAcc[d].push(dims[d]);
     if (!rationale && typeof v.rationale === "string" && v.rationale) rationale = v.rationale;
   }
@@ -3167,24 +3352,144 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
     maxDisagreement,
     failedJudges,
     costUsd,
-    rationale: rationale || "llm-judge"
+    rationale: rationale || "llm-judge",
+    verdicts: [...verdicts]
   };
 }
 
-// src/judge-runner.ts
-var JudgeRunner = class {
-  driver;
-  constructor(driver = new SubprocessSandboxDriver()) {
-    this.driver = driver;
-  }
-  async run(spec) {
-    const store = new InMemoryTraceStore();
-    const emitter = new TraceEmitter(store, { runId: `judge-${spec.id}` });
-    await emitter.startRun({
-      scenarioId: spec.id,
-      layer: "meta",
-      projectId: "judge-runner"
-    });
+// src/judge-retry.ts
+var DEFAULT_MAX_ATTEMPTS = 3;
+var DEFAULT_TIMEOUT_MS = 3e5;
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withJudgeRetry(judgeFn, policy = {}) {
+  const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const timeoutMs = policy.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const backoff = policy.backoffMs ?? backoffMs;
+  const isRetryable = policy.isRetryable ?? isTransientLlmError;
+  const models = policy.models && policy.models.length > 0 ? policy.models : [void 0];
+  let totalAttempts = 0;
+  const attemptErrors = [];
+  let lastError;
+  for (const model of models) {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      totalAttempts += 1;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(new Error("TimeoutError")), timeoutMs);
+      try {
+        const value = await judgeFn(model, controller.signal);
+        clearTimeout(timer);
+        return {
+          value,
+          succeeded: true,
+          attempts: totalAttempts,
+          modelUsed: model,
+          attemptErrors
+        };
+      } catch (err) {
+        clearTimeout(timer);
+        const errObj = err instanceof Error ? err : new Error(String(err));
+        lastError = errObj;
+        attemptErrors.push({
+          attempt: totalAttempts,
+          model: model ?? "(default)",
+          error: errObj.message
+        });
+        if (!isRetryable(errObj)) {
+          return {
+            value: null,
+            succeeded: false,
+            attempts: totalAttempts,
+            error: errObj,
+            attemptErrors
+          };
+        }
+        if (attempt < maxAttempts - 1) {
+          await sleep(backoff(attempt));
+        }
+      }
+    }
+  }
+  return {
+    value: null,
+    succeeded: false,
+    attempts: totalAttempts,
+    error: lastError,
+    attemptErrors
+  };
+}
+
+// src/judge-panel.ts
+function ensembleJudge(opts) {
+  if (opts.models.length === 0) {
+    throw new Error(`ensembleJudge '${opts.name}': models is empty \u2014 nothing to score with`);
+  }
+  if (opts.dimensions.length === 0) {
+    throw new Error(`ensembleJudge '${opts.name}': dimensions is empty \u2014 nothing to score`);
+  }
+  if (opts.crossFamily !== false) {
+    assertCrossFamily(opts.models);
+  }
+  const scoreOne = async (model, input) => {
+    if (opts.retry) {
+      const outcome = await withJudgeRetry((m) => opts.scoreWith(m, input), {
+        ...opts.retry,
+        models: [model]
+      });
+      if (!outcome.succeeded || outcome.value === null) {
+        return {
+          model,
+          perDimension: null,
+          rationale: outcome.error?.message ?? "judge failed after retries"
+        };
+      }
+      return outcome.value;
+    }
+    try {
+      return await opts.scoreWith(model, input);
+    } catch (err) {
+      return {
+        model,
+        perDimension: null,
+        rationale: err instanceof Error ? err.message : String(err)
+      };
+    }
+  };
+  return {
+    name: opts.name,
+    dimensions: opts.dimensions.map((d) => ({ key: d, description: d })),
+    async score({ artifact, scenario }) {
+      const input = { artifact, scenario };
+      const verdicts = await Promise.all(opts.models.map((model) => scoreOne(model, input)));
+      const agg = aggregateJudgeVerdicts(verdicts, opts.dimensions, opts.weights);
+      const score = {
+        dimensions: agg.perDimension,
+        composite: agg.composite,
+        notes: agg.rationale,
+        maxDisagreement: agg.maxDisagreement,
+        perJudge: agg.perJudge
+      };
+      if (agg.failedJudges.length > 0) score.failedJudges = agg.failedJudges;
+      return score;
+    }
+  };
+}
+
+// src/judge-runner.ts
+var JudgeRunner = class {
+  driver;
+  constructor(driver = new SubprocessSandboxDriver()) {
+    this.driver = driver;
+  }
+  async run(spec) {
+    const store = new InMemoryTraceStore();
+    const emitter = new TraceEmitter(store, { runId: `judge-${spec.id}` });
+    await emitter.startRun({
+      scenarioId: spec.id,
+      layer: "meta",
+      projectId: "judge-runner"
+    });
     const harness = new SandboxHarness(this.driver);
     const detail = await harness.run(spec.config, emitter);
     await emitter.endRun({ pass: detail.passed, score: detail.score, notes: `${spec.kind} judge` });
@@ -3502,7 +3807,7 @@ function createAxService(provider, apiKey, model) {
   });
 }
 function seededShuffle(items, seed) {
-  const rng = mulberry32(hashString(seed));
+  const rng = mulberry322(hashString(seed));
   const out = [...items];
   for (let i = out.length - 1; i > 0; i--) {
     const j = Math.floor(rng() * (i + 1));
@@ -3518,7 +3823,7 @@ function hashString(value) {
   }
   return h >>> 0;
 }
-function mulberry32(seed) {
+function mulberry322(seed) {
   let a = seed >>> 0;
   return () => {
     a = a + 1831565813 | 0;
@@ -3669,13 +3974,118 @@ var BudgetGuard = class {
   }
 };
 
+// src/cost-ledger.ts
+function modelPriceKey(model) {
+  return isModelPriced(model) ? model : null;
+}
+function costForUsage(model, usage) {
+  assertNonNegative(usage.inputTokens, "inputTokens");
+  assertNonNegative(usage.outputTokens, "outputTokens");
+  if (usage.cachedTokens !== void 0) assertNonNegative(usage.cachedTokens, "cachedTokens");
+  const pricing = resolveModelPricing(model);
+  if (!pricing) return { costUsd: 0, costUnknown: true };
+  const billedInput = usage.inputTokens + (usage.cachedTokens ?? 0);
+  return { costUsd: estimateCost(billedInput, usage.outputTokens, model), costUnknown: false };
+}
+var CostLedger = class {
+  entries = [];
+  completedTasks = 0;
+  /**
+   * Record one LLM call. The cost is computed from pricing unless
+   * `actualCostUsd` is supplied (a finite observed cost from the provider
+   * response), in which case `costUnknown` is false regardless of pricing.
+   */
+  record(input) {
+    const { costUsd, costUnknown } = costForUsage(input.model, input.usage);
+    const hasActual = typeof input.actualCostUsd === "number" && Number.isFinite(input.actualCostUsd);
+    if (hasActual) assertNonNegative(input.actualCostUsd, "actualCostUsd");
+    const entry = {
+      model: input.model,
+      channel: input.channel,
+      inputTokens: input.usage.inputTokens,
+      outputTokens: input.usage.outputTokens,
+      cachedTokens: input.usage.cachedTokens,
+      costUsd: hasActual ? input.actualCostUsd : costUsd,
+      costUnknown: hasActual ? false : costUnknown,
+      actualCostUsd: hasActual ? input.actualCostUsd : void 0,
+      tags: input.tags,
+      timestamp: input.timestamp ?? Date.now()
+    };
+    this.entries.push(entry);
+    return entry;
+  }
+  /** Increment the completed-task counter (used for cost-per-completed-task). */
+  markCompleted(count = 1) {
+    if (!Number.isInteger(count) || count < 0) {
+      throw new ValidationError(
+        `CostLedger.markCompleted: count must be a non-negative integer, got ${count}`
+      );
+    }
+    this.completedTasks += count;
+  }
+  list() {
+    return [...this.entries];
+  }
+  summary() {
+    const byChannel = /* @__PURE__ */ new Map();
+    const unpriced = /* @__PURE__ */ new Set();
+    let totalCost = 0;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cachedTokens = 0;
+    for (const e of this.entries) {
+      totalCost += e.costUsd;
+      inputTokens += e.inputTokens;
+      outputTokens += e.outputTokens;
+      cachedTokens += e.cachedTokens ?? 0;
+      if (e.costUnknown) unpriced.add(e.model);
+      const roll = byChannel.get(e.channel) ?? {
+        channel: e.channel,
+        calls: 0,
+        inputTokens: 0,
+        outputTokens: 0,
+        cachedTokens: 0,
+        costUsd: 0,
+        unpricedCalls: 0
+      };
+      roll.calls += 1;
+      roll.inputTokens += e.inputTokens;
+      roll.outputTokens += e.outputTokens;
+      roll.cachedTokens += e.cachedTokens ?? 0;
+      roll.costUsd += e.costUsd;
+      if (e.costUnknown) roll.unpricedCalls += 1;
+      byChannel.set(e.channel, roll);
+    }
+    return {
+      totalCalls: this.entries.length,
+      inputTokens,
+      outputTokens,
+      cachedTokens,
+      totalCostUsd: totalCost,
+      byChannel: [...byChannel.values()].sort((a, b) => a.channel.localeCompare(b.channel)),
+      unpricedModels: [...unpriced].sort(),
+      fullyPriced: unpriced.size === 0
+    };
+  }
+  /** Total spend divided by completed tasks; null when nothing completed. */
+  costPerCompletedTask() {
+    if (this.completedTasks === 0) return null;
+    return this.summary().totalCostUsd / this.completedTasks;
+  }
+};
+function assertNonNegative(n, name) {
+  if (!Number.isFinite(n) || n < 0) {
+    throw new ValidationError(`CostLedger: ${name} must be a non-negative finite number, got ${n}`);
+  }
+}
+
 // src/cost-tracker.ts
 var CostTracker = class {
   byScenario = /* @__PURE__ */ new Map();
   record(entry) {
     const full = { timestamp: entry.timestamp ?? Date.now(), ...entry };
-    assertNonNegative(full.inputTokens, "inputTokens");
-    assertNonNegative(full.outputTokens, "outputTokens");
+    assertNonNegative2(full.inputTokens, "inputTokens");
+    assertNonNegative2(full.outputTokens, "outputTokens");
     let bucket = this.byScenario.get(full.scenarioId);
     if (!bucket) {
       bucket = {
@@ -3754,12 +4164,433 @@ function costFor(entry) {
   }
   return estimateCost(entry.inputTokens, entry.outputTokens, entry.model);
 }
-function assertNonNegative(n, name) {
+function assertNonNegative2(n, name) {
   if (!Number.isFinite(n) || n < 0) {
     throw new Error(`CostTracker: ${name} must be a non-negative finite number, got ${n}`);
   }
 }
 
+// src/eval-trace-store.ts
+function runScore(record) {
+  const { holdoutScore, searchScore } = record.outcome;
+  if (typeof holdoutScore === "number") return holdoutScore;
+  if (typeof searchScore === "number") return searchScore;
+  throw new ValidationError(
+    `EvalTraceStore: run ${record.runId} has neither holdoutScore nor searchScore`
+  );
+}
+function matches(record, f) {
+  if (f.experimentId && record.experimentId !== f.experimentId) return false;
+  if (f.candidateId && record.candidateId !== f.candidateId) return false;
+  if (f.scenarioId && record.scenarioId !== f.scenarioId) return false;
+  if (f.model && record.model !== f.model) return false;
+  if (f.splitTag && record.splitTag !== f.splitTag) return false;
+  if (f.minScore !== void 0 && runScore(record) < f.minScore) return false;
+  if (f.maxScore !== void 0 && runScore(record) > f.maxScore) return false;
+  if (f.rawEquals && record.outcome.raw[f.rawEquals.key] !== f.rawEquals.value) return false;
+  if (f.where && !f.where(record)) return false;
+  return true;
+}
+function inMemoryRunRecordBackend(initial = []) {
+  const rows = initial.map((r) => validateRunRecord(r));
+  return {
+    async append(record) {
+      rows.push(record);
+    },
+    async load() {
+      return [...rows];
+    }
+  };
+}
+function jsonlRunRecordBackend(path, opts = {}) {
+  return {
+    async append(record) {
+      const fs2 = await import("fs/promises");
+      const pathMod = await import("path");
+      await fs2.mkdir(pathMod.dirname(path), { recursive: true });
+      await fs2.appendFile(path, `${JSON.stringify(record)}
+`, "utf8");
+    },
+    async load() {
+      const fs2 = await import("fs/promises");
+      let raw;
+      try {
+        raw = await fs2.readFile(path, "utf8");
+      } catch (err) {
+        if (err.code === "ENOENT") return [];
+        throw err;
+      }
+      const out = [];
+      let lineNo = 0;
+      for (const line of raw.split("\n")) {
+        lineNo++;
+        if (!line.trim()) continue;
+        let parsed;
+        try {
+          parsed = JSON.parse(line);
+        } catch (err) {
+          if (opts.skipInvalid) continue;
+          throw new ValidationError(`EvalTraceStore: ${path}:${lineNo} is not valid JSON`);
+        }
+        if (opts.skipInvalid) {
+          if (isRunRecord(parsed)) out.push(parsed);
+          continue;
+        }
+        try {
+          out.push(validateRunRecord(parsed));
+        } catch (err) {
+          throw new ValidationError(
+            `EvalTraceStore: ${path}:${lineNo} is not a valid RunRecord \u2014 ${err.message}`
+          );
+        }
+      }
+      return out;
+    }
+  };
+}
+var EvalTraceStore = class {
+  backend;
+  constructor(backend = inMemoryRunRecordBackend()) {
+    this.backend = backend;
+  }
+  /** Validate and append one run. Throws on an invalid record — the corpus
+   *  stays paper-grade. */
+  async append(record) {
+    await this.backend.append(validateRunRecord(record));
+  }
+  async all() {
+    return this.backend.load();
+  }
+  async query(filter = {}) {
+    const rows = await this.backend.load();
+    return rows.filter((r) => matches(r, filter));
+  }
+  /**
+   * Highest-scoring run for a scenario (optionally restricted to a candidate).
+   * Returns null when no run matches. Ties resolve to the earliest-appended run
+   * so the result is stable.
+   */
+  async getBest(scenarioId, opts = {}) {
+    const rows = await this.query({
+      scenarioId,
+      candidateId: opts.candidateId,
+      splitTag: opts.splitTag
+    });
+    if (rows.length === 0) return null;
+    let best = rows[0];
+    let bestScore = runScore(best);
+    for (let i = 1; i < rows.length; i++) {
+      const s = runScore(rows[i]);
+      if (s > bestScore) {
+        best = rows[i];
+        bestScore = s;
+      }
+    }
+    return best;
+  }
+  /**
+   * Compare two candidates on the scenarios they BOTH ran. When a candidate
+   * ran a scenario more than once, its best `runScore` for that scenario is
+   * used. Throws when there is no paired scenario — an unpaired "comparison" is
+   * not one.
+   */
+  async compareRuns(candidateA, candidateB) {
+    if (candidateA === candidateB) {
+      throw new ValidationError(
+        `EvalTraceStore.compareRuns: candidates must differ ("${candidateA}")`
+      );
+    }
+    const rows = await this.backend.load();
+    const bestByScenario = (candidate) => {
+      const m = /* @__PURE__ */ new Map();
+      for (const r of rows) {
+        if (r.candidateId !== candidate) continue;
+        const sid = r.scenarioId;
+        if (!sid) continue;
+        const s = runScore(r);
+        const prev = m.get(sid);
+        if (prev === void 0 || s > prev) m.set(sid, s);
+      }
+      return m;
+    };
+    const aScores = bestByScenario(candidateA);
+    const bScores = bestByScenario(candidateB);
+    const paired = [...aScores.keys()].filter((sid) => bScores.has(sid)).sort();
+    if (paired.length === 0) {
+      throw new ValidationError(
+        `EvalTraceStore.compareRuns: "${candidateA}" and "${candidateB}" share no scenario (need scenarioId on records)`
+      );
+    }
+    let sumA = 0;
+    let sumB = 0;
+    let bWins = 0;
+    let aWins = 0;
+    let ties = 0;
+    for (const sid of paired) {
+      const sa = aScores.get(sid);
+      const sb = bScores.get(sid);
+      sumA += sa;
+      sumB += sb;
+      if (sb > sa) bWins++;
+      else if (sb < sa) aWins++;
+      else ties++;
+    }
+    const meanA = sumA / paired.length;
+    const meanB = sumB / paired.length;
+    return {
+      a: candidateA,
+      b: candidateB,
+      pairedScenarioIds: paired,
+      meanA,
+      meanB,
+      meanDelta: meanB - meanA,
+      bWins,
+      ties,
+      aWins
+    };
+  }
+};
+
+// src/experiment-tracker.ts
+import { execSync } from "child_process";
+var DEFAULTS = {
+  keepThreshold: 5,
+  regressionThreshold: 5,
+  iqrUnstableAbove: 10,
+  stddevUnstableAbove: Number.POSITIVE_INFINITY,
+  minRepsForVerdict: 3
+};
+function resolveThresholds(t) {
+  const r = { ...DEFAULTS, ...t ?? {} };
+  if (r.keepThreshold < 0) {
+    throw new ValidationError(
+      `experiment-tracker: keepThreshold must be >= 0, got ${r.keepThreshold}`
+    );
+  }
+  if (r.regressionThreshold < 0) {
+    throw new ValidationError(
+      `experiment-tracker: regressionThreshold must be >= 0, got ${r.regressionThreshold}`
+    );
+  }
+  if (r.minRepsForVerdict < 1) {
+    throw new ValidationError(
+      `experiment-tracker: minRepsForVerdict must be >= 1, got ${r.minRepsForVerdict}`
+    );
+  }
+  return r;
+}
+function median(sorted) {
+  const n = sorted.length;
+  if (n === 0) return 0;
+  const mid = Math.floor(n / 2);
+  return n % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function stddev(values, mean5) {
+  if (values.length < 2) return 0;
+  const variance = values.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / values.length;
+  return Math.sqrt(variance);
+}
+function computeExperimentStats(reps, thresholds) {
+  const t = resolveThresholds(thresholds);
+  const n = reps.length;
+  if (n === 0) {
+    return {
+      median: 0,
+      mean: 0,
+      min: 0,
+      max: 0,
+      iqr: 0,
+      stddev: 0,
+      passRate: null,
+      n: 0,
+      stable: false
+    };
+  }
+  const scores2 = reps.map((r) => {
+    if (!Number.isFinite(r.score)) {
+      throw new ValidationError(`experiment-tracker: rep ${r.rep} has non-finite score ${r.score}`);
+    }
+    return r.score;
+  });
+  const sorted = [...scores2].sort((a, b) => a - b);
+  const mean5 = scores2.reduce((s, v) => s + v, 0) / n;
+  const sd = stddev(scores2, mean5);
+  const spread = iqr(scores2);
+  const rated = reps.filter((r) => typeof r.passed === "boolean");
+  const passRate = rated.length === 0 ? null : rated.filter((r) => r.passed).length / rated.length;
+  const stable = spread < t.iqrUnstableAbove && sd < t.stddevUnstableAbove;
+  return {
+    median: median(sorted),
+    mean: mean5,
+    min: sorted[0],
+    max: sorted[n - 1],
+    iqr: spread,
+    stddev: sd,
+    passRate,
+    n,
+    stable
+  };
+}
+function improvementVerdict(candidate, parent, thresholds) {
+  const t = resolveThresholds(thresholds);
+  if (!parent) {
+    return {
+      verdict: "ITERATE",
+      medianDelta: null,
+      reason: "no parent experiment to compare against"
+    };
+  }
+  if (candidate.n < t.minRepsForVerdict || parent.n < t.minRepsForVerdict) {
+    return {
+      verdict: "ITERATE",
+      medianDelta: null,
+      reason: `need >= ${t.minRepsForVerdict} reps on both sides (candidate n=${candidate.n}, parent n=${parent.n})`
+    };
+  }
+  if (!candidate.stable) {
+    return {
+      verdict: "NOISE",
+      medianDelta: candidate.median - parent.median,
+      reason: `candidate unstable (iqr=${candidate.iqr}, stddev=${candidate.stddev.toFixed(2)})`
+    };
+  }
+  const medianDelta2 = candidate.median - parent.median;
+  if (medianDelta2 > t.keepThreshold) {
+    return { verdict: "KEEP", medianDelta: medianDelta2, reason: `median +${medianDelta2} > +${t.keepThreshold}` };
+  }
+  if (medianDelta2 < -t.regressionThreshold) {
+    return {
+      verdict: "REGRESSION",
+      medianDelta: medianDelta2,
+      reason: `median ${medianDelta2} < -${t.regressionThreshold}`
+    };
+  }
+  return {
+    verdict: "NOISE",
+    medianDelta: medianDelta2,
+    reason: `median delta ${medianDelta2} inside noise band [-${t.regressionThreshold}, +${t.keepThreshold}]`
+  };
+}
+var gitProvenanceReader = () => {
+  const run = (cmd) => execSync(cmd, { encoding: "utf8" }).trim();
+  const commit = run("git rev-parse --short HEAD");
+  const message = run("git log -1 --format=%s");
+  const changedRaw = run("git diff --name-only HEAD~1");
+  const changedFiles = changedRaw.length === 0 ? [] : changedRaw.split("\n").filter(Boolean);
+  return { commit, message, changedFiles };
+};
+function inMemoryExperimentStore(initial = []) {
+  let state = initial.map((e) => structuredClone(e));
+  return {
+    async load() {
+      return state.map((e) => structuredClone(e));
+    },
+    async save(experiments) {
+      state = experiments.map((e) => structuredClone(e));
+    }
+  };
+}
+function fileExperimentStore(path) {
+  return {
+    async load() {
+      const fs2 = await import("fs/promises");
+      try {
+        const raw = await fs2.readFile(path, "utf8");
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed)) {
+          throw new ValidationError(`experiment-tracker: store at ${path} is not a JSON array`);
+        }
+        return parsed;
+      } catch (err) {
+        if (err.code === "ENOENT") return [];
+        throw err;
+      }
+    },
+    async save(experiments) {
+      const fs2 = await import("fs/promises");
+      const pathMod = await import("path");
+      await fs2.mkdir(pathMod.dirname(path), { recursive: true });
+      await fs2.writeFile(path, JSON.stringify(experiments, null, 2), "utf8");
+    }
+  };
+}
+var ExperimentTracker = class {
+  store;
+  provenanceReader;
+  thresholds;
+  now;
+  constructor(options = {}) {
+    this.store = options.store ?? inMemoryExperimentStore();
+    this.provenanceReader = options.provenanceReader ?? gitProvenanceReader;
+    this.thresholds = resolveThresholds(options.thresholds);
+    this.now = options.now ?? Date.now;
+  }
+  async create(input) {
+    const experiments = await this.store.load();
+    if (experiments.some((e) => e.id === input.id)) {
+      throw new ValidationError(`experiment-tracker: experiment id "${input.id}" already exists`);
+    }
+    if (input.parentId && !experiments.some((e) => e.id === input.parentId)) {
+      throw new ValidationError(
+        `experiment-tracker: parent experiment "${input.parentId}" not found`
+      );
+    }
+    const provenance = input.provenance ?? await this.provenanceReader();
+    const experiment = {
+      id: input.id,
+      label: input.label,
+      provenance,
+      parentId: input.parentId,
+      changeSummary: input.changeSummary,
+      reps: [],
+      stats: computeExperimentStats([], this.thresholds),
+      verdict: "ITERATE",
+      createdAt: new Date(this.now()).toISOString()
+    };
+    experiments.push(experiment);
+    await this.store.save(experiments);
+    return structuredClone(experiment);
+  }
+  /** Append a rep (its `rep` index defaults to the current rep count) and
+   *  recompute stats + verdict. Returns the updated experiment. */
+  async addRep(experimentId, rep) {
+    const experiments = await this.store.load();
+    const exp = experiments.find((e) => e.id === experimentId);
+    if (!exp)
+      throw new ValidationError(`experiment-tracker: experiment "${experimentId}" not found`);
+    const fullRep = {
+      rep: rep.rep ?? exp.reps.length,
+      score: rep.score,
+      passed: rep.passed,
+      metrics: rep.metrics,
+      timestamp: rep.timestamp ?? new Date(this.now()).toISOString()
+    };
+    exp.reps.push(fullRep);
+    exp.stats = computeExperimentStats(exp.reps, this.thresholds);
+    const parent = exp.parentId ? experiments.find((e) => e.id === exp.parentId) : void 0;
+    exp.verdict = improvementVerdict(exp.stats, parent?.stats ?? null, this.thresholds).verdict;
+    await this.store.save(experiments);
+    return structuredClone(exp);
+  }
+  async get(experimentId) {
+    const experiments = await this.store.load();
+    const found = experiments.find((e) => e.id === experimentId);
+    return found ? structuredClone(found) : void 0;
+  }
+  async list() {
+    return this.store.load();
+  }
+  /** Full verdict (not just the enum) for an experiment vs its parent. */
+  async verdictFor(experimentId) {
+    const experiments = await this.store.load();
+    const exp = experiments.find((e) => e.id === experimentId);
+    if (!exp)
+      throw new ValidationError(`experiment-tracker: experiment "${experimentId}" not found`);
+    const parent = exp.parentId ? experiments.find((e) => e.id === exp.parentId) : void 0;
+    return improvementVerdict(exp.stats, parent?.stats ?? null, this.thresholds);
+  }
+};
+
 // src/muffled-gate-scanner.ts
 import { existsSync, readdirSync, readFileSync, statSync } from "fs";
 import { join } from "path";
@@ -3872,7 +4703,7 @@ var DEFAULT_FINDERS = [
 ];
 var UNIVERSAL_FINDERS = [findConstructorCwdDropped];
 function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
-  const matches = [];
+  const matches2 = [];
   const walk = (rel) => {
     const abs = join(repoRoot, rel);
     if (!existsSync(abs)) return;
@@ -3898,12 +4729,12 @@ function autoDeriveImporters(repoRoot, roots, extensions, importsContain) {
         } catch {
           continue;
         }
-        if (text.includes(importsContain)) matches.push(sub);
+        if (text.includes(importsContain)) matches2.push(sub);
       }
     }
   };
   for (const r of roots) walk(r);
-  return matches;
+  return matches2;
 }
 function scanForMuffledGates(opts) {
   const findings = [];
@@ -4075,16 +4906,80 @@ function isObject(v) {
   return typeof v === "object" && v !== null && !Array.isArray(v);
 }
 
+// src/partition-held-out.ts
+function fnv1a32(input) {
+  let h = 2166136261;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i) & 255;
+    h = Math.imul(h, 16777619);
+  }
+  return h >>> 0;
+}
+function hashToUnit(id, seed) {
+  return fnv1a32(`${id}
+${seed}`) / 4294967296;
+}
+function assignHeldOutTag(id, options = {}) {
+  const seed = options.seed ?? "held-out-v1";
+  const holdoutFraction = options.holdoutFraction ?? 0.5;
+  assertFraction(holdoutFraction);
+  return hashToUnit(id, seed) < holdoutFraction ? "holdout" : "search";
+}
+function partitionHeldOut(ids, options = {}) {
+  const seed = options.seed ?? "held-out-v1";
+  const holdoutFraction = options.holdoutFraction ?? 0.5;
+  const minHoldout = options.minHoldout ?? 1;
+  const minSearch = options.minSearch ?? 1;
+  assertFraction(holdoutFraction);
+  if (ids.length === 0) {
+    throw new ValidationError("partitionHeldOut: no ids supplied");
+  }
+  const seen = /* @__PURE__ */ new Set();
+  const search = [];
+  const holdout = [];
+  for (const id of ids) {
+    if (typeof id !== "string" || id.length === 0) {
+      throw new ValidationError(
+        `partitionHeldOut: ids must be non-empty strings, got ${JSON.stringify(id)}`
+      );
+    }
+    if (seen.has(id)) {
+      throw new ValidationError(
+        `partitionHeldOut: duplicate id "${id}" \u2014 a paired comparison must not double-count`
+      );
+    }
+    seen.add(id);
+    if (hashToUnit(id, seed) < holdoutFraction) holdout.push(id);
+    else search.push(id);
+  }
+  if (holdout.length < minHoldout) {
+    throw new ValidationError(
+      `partitionHeldOut: holdout set has ${holdout.length} id(s), below the floor of ${minHoldout} (n=${ids.length}, holdoutFraction=${holdoutFraction}) \u2014 too few for a significant comparison`
+    );
+  }
+  if (search.length < minSearch) {
+    throw new ValidationError(
+      `partitionHeldOut: search set has ${search.length} id(s), below the floor of ${minSearch} (n=${ids.length}, holdoutFraction=${holdoutFraction})`
+    );
+  }
+  return { search, holdout, seed, holdoutFraction };
+}
+function assertFraction(f) {
+  if (!Number.isFinite(f) || f <= 0 || f >= 1) {
+    throw new ValidationError(`partitionHeldOut: holdoutFraction must be in (0, 1), got ${f}`);
+  }
+}
+
 // src/scorecard.ts
 import { appendFileSync, existsSync as existsSync2, mkdirSync, readFileSync as readFileSync2 } from "fs";
 import { dirname } from "path";
-function median(xs) {
+function median2(xs) {
   if (xs.length === 0) return 0;
   const sorted = [...xs].sort((a, b) => a - b);
   const mid = Math.floor(sorted.length / 2);
   return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
 }
-function runScore(run) {
+function runScore2(run) {
   return run.outcome.holdoutScore ?? run.outcome.searchScore;
 }
 function aggregatePerDimension(runs) {
@@ -4118,14 +5013,14 @@ function recordRuns(runs, opts) {
   }
   const lines = [];
   for (const [scenarioId, scenarioRuns] of byScenario) {
-    const scored = scenarioRuns.map((run) => ({ run, score: runScore(run) })).filter((s) => s.score !== void 0);
+    const scored = scenarioRuns.map((run) => ({ run, score: runScore2(run) })).filter((s) => s.score !== void 0);
     if (scored.length === 0) continue;
     const scores2 = scored.map((s) => s.score);
     const entry = {
       commitSha: opts.commitSha,
       timestamp,
       scores: scores2,
-      composite: median(scores2),
+      composite: median2(scores2),
       runIds: scored.map((s) => s.run.runId)
     };
     const perDimension = aggregatePerDimension(scenarioRuns);
@@ -4263,42 +5158,6 @@ function formatScorecardDiff(diff) {
   return lines.join("\n");
 }
 
-// src/series-convergence.ts
-function analyzeSeries(values, options = {}) {
-  const window = options.window ?? 5;
-  const stableCv = options.stableCv ?? 0.05;
-  const driftRun = options.driftRun ?? 3;
-  if (values.length < Math.max(2, Math.min(window, 3))) {
-    return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
-  }
-  const tail = values.slice(-window);
-  const mean5 = tail.reduce((a, b) => a + b, 0) / tail.length;
-  const variance = tail.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / tail.length;
-  const stdDev = Math.sqrt(variance);
-  const refMean = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
-  const cv = stdDev / refMean;
-  const stable = tail.length >= window && cv <= stableCv;
-  let tailRun = 0;
-  let direction = 0;
-  for (let i = values.length - 1; i > 0; i--) {
-    const delta = values[i] - values[i - 1];
-    if (delta === 0) break;
-    const dir = delta > 0 ? 1 : -1;
-    if (direction === 0) direction = dir;
-    if (dir !== direction) break;
-    tailRun += dir;
-  }
-  let state;
-  if (stable) {
-    state = "stabilized";
-  } else if (Math.abs(tailRun) >= driftRun) {
-    state = tailRun > 0 ? "drifting-up" : "drifting-down";
-  } else {
-    state = "noisy";
-  }
-  return { state, windowMean: mean5, windowCv: cv, tailRun, stable };
-}
-
 // src/slo.ts
 function checkSlos(metrics, slos) {
   const results = slos.map((slo) => check(slo, metrics[slo.metric]));
@@ -4385,70 +5244,445 @@ function scoreContinuity(pair, checks, options = {}) {
   if (checks.length === 0) {
     throw new Error("scoreContinuity: at least 1 check required");
   }
-  const passThreshold = options.passThreshold ?? 0.8;
-  const results = checks.map((c) => {
-    const raw = c.score(pair);
-    const clamped = Number.isFinite(raw) ? Math.max(0, Math.min(1, raw)) : 0;
-    return { id: c.id, description: c.description, score: clamped, pass: clamped >= passThreshold };
-  });
-  const overallScore = results.reduce((a, r) => a + r.score, 0) / results.length;
-  return { results, overallScore, pass: results.every((r) => r.pass) };
+  const passThreshold = options.passThreshold ?? 0.8;
+  const results = checks.map((c) => {
+    const raw = c.score(pair);
+    const clamped = Number.isFinite(raw) ? Math.max(0, Math.min(1, raw)) : 0;
+    return { id: c.id, description: c.description, score: clamped, pass: clamped >= passThreshold };
+  });
+  const overallScore = results.reduce((a, r) => a + r.score, 0) / results.length;
+  return { results, overallScore, pass: results.every((r) => r.pass) };
+}
+function keyPreserved(key) {
+  return {
+    id: `preserved(${key})`,
+    description: `"${key}" unchanged from before to after`,
+    score: ({ before, after }) => before[key] !== void 0 && before[key] === after[key] ? 1 : 0
+  };
+}
+function collectionPreserved(key, minRatio = 1) {
+  return {
+    id: `collection-preserved(${key})`,
+    description: `"${key}" length \u2265 ${minRatio} \xD7 prior length`,
+    score: ({ before, after }) => {
+      const b = before[key];
+      const a = after[key];
+      if (!Array.isArray(b) || !Array.isArray(a)) return 0;
+      if (b.length === 0) return a.length === 0 ? 1 : 1;
+      return Math.min(1, a.length / (b.length * minRatio));
+    }
+  };
+}
+function statusAdvanced(key, progression) {
+  return {
+    id: `status-advanced(${key})`,
+    description: `"${key}" progressed along ${progression.join("\u2192")}`,
+    score: ({ before, after }) => {
+      const bi = progression.indexOf(String(before[key]));
+      const ai2 = progression.indexOf(String(after[key]));
+      if (bi === -1 || ai2 === -1) return 0;
+      return ai2 >= bi ? 1 : 0;
+    }
+  };
+}
+
+// src/ui-finding.ts
+var UI_LENSES = [
+  "consistency",
+  "hierarchy",
+  "layout",
+  "ux-flow",
+  "duplication",
+  "accessibility",
+  "responsive",
+  "states",
+  "content",
+  "interaction",
+  "performance-perceived",
+  "other"
+];
+var UI_FINDING_SEVERITIES = [
+  "critical",
+  "high",
+  "med",
+  "low"
+];
+
+// src/trace-contracts.ts
+function isSerializedRegex(v) {
+  return typeof v === "object" && v !== null && typeof v.$regex === "string" && typeof v.flags === "string";
+}
+function matchText(actual, matcher) {
+  if (typeof actual !== "string") return false;
+  if (typeof matcher === "string") return actual === matcher;
+  if (matcher instanceof RegExp) return matcher.test(actual);
+  return new RegExp(matcher.$regex, matcher.flags).test(actual);
+}
+function resolveToolName(span) {
+  if (typeof span.toolName === "string") return span.toolName;
+  const fromAttr = span.attributes?.["tool.name"] ?? span.attributes?.["toolName"];
+  if (typeof fromAttr === "string") return fromAttr;
+  if (span.kind === "tool" && typeof span.name === "string") return span.name;
+  return void 0;
+}
+function assertPredicate(value, where) {
+  if (value === null || typeof value !== "object") {
+    throw new ValidationError(`${where}: predicate must be an object, got ${typeof value}`);
+  }
+  const p = value;
+  for (const field of ["name", "tool"]) {
+    const m = p[field];
+    if (m !== void 0 && typeof m !== "string" && !(m instanceof RegExp) && !isSerializedRegex(m)) {
+      throw new ValidationError(`${where}: "${field}" must be string | RegExp | SerializedRegex`);
+    }
+  }
+  if (p.attr !== void 0 && (p.attr === null || typeof p.attr !== "object" || Array.isArray(p.attr))) {
+    throw new ValidationError(`${where}: "attr" must be a plain object`);
+  }
+  if (p.requiresCustom && typeof p.custom !== "function") {
+    throw new ValidationError(
+      `${where}: rule was built with a custom predicate function, which does not survive JSON serialization \u2014 re-attach \`custom\` after deserializing or drop the rule`
+    );
+  }
+  const hasAttr = p.attr !== void 0 && Object.keys(p.attr).length > 0;
+  if (p.name === void 0 && p.tool === void 0 && !hasAttr && typeof p.custom !== "function") {
+    throw new ValidationError(
+      `${where}: empty predicate would match every span \u2014 specify name, tool, attr, or custom`
+    );
+  }
+}
+function matchSpan(span, predicate) {
+  assertPredicate(predicate, "matchSpan");
+  if (predicate.name !== void 0 && !matchText(span.name, predicate.name)) return false;
+  if (predicate.tool !== void 0 && !matchText(resolveToolName(span), predicate.tool))
+    return false;
+  if (predicate.attr !== void 0) {
+    for (const [key, expected] of Object.entries(predicate.attr)) {
+      const actual = span.attributes?.[key];
+      if (expected instanceof RegExp || isSerializedRegex(expected)) {
+        if (!matchText(typeof actual === "string" ? actual : void 0, expected)) {
+          return false;
+        }
+      } else if (actual !== expected) {
+        return false;
+      }
+    }
+  }
+  if (predicate.custom !== void 0 && !predicate.custom(span)) return false;
+  return true;
+}
+function describeMatcher(m) {
+  if (typeof m === "string") return m;
+  if (m instanceof RegExp) return `/${m.source}/${m.flags}`;
+  return `/${m.$regex}/${m.flags}`;
+}
+function describePredicate(p) {
+  const parts = [];
+  if (p.name !== void 0) parts.push(`name=${describeMatcher(p.name)}`);
+  if (p.tool !== void 0) parts.push(`tool=${describeMatcher(p.tool)}`);
+  if (p.attr !== void 0) {
+    for (const [k, v] of Object.entries(p.attr)) {
+      parts.push(
+        `attr.${k}=${v instanceof RegExp || isSerializedRegex(v) ? describeMatcher(v) : JSON.stringify(v)}`
+      );
+    }
+  }
+  if (typeof p.custom === "function") parts.push(`custom=${p.custom.name || "fn"}`);
+  return parts.join(",");
+}
+function normalizeMatcher(m) {
+  return m instanceof RegExp ? { $regex: m.source, flags: m.flags } : m;
+}
+function normalizePredicate(p, where) {
+  assertPredicate(p, where);
+  const out = {};
+  if (p.name !== void 0) out.name = normalizeMatcher(p.name);
+  if (p.tool !== void 0) out.tool = normalizeMatcher(p.tool);
+  if (p.attr !== void 0) {
+    out.attr = Object.fromEntries(
+      Object.entries(p.attr).map(([k, v]) => [k, v instanceof RegExp ? normalizeMatcher(v) : v])
+    );
+  }
+  if (typeof p.custom === "function") {
+    out.custom = p.custom;
+    out.requiresCustom = true;
+  }
+  return out;
+}
+var TraceContractBuilder = class {
+  constructor(name) {
+    this.name = name;
+  }
+  name;
+  rules = [];
+  /** Every span in the trace must satisfy `p`. */
+  always(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").always`);
+    return this.add({ kind: "always", label: label ?? `always(${describePredicate(np)})`, p: np });
+  }
+  /** No span in the trace may satisfy `p`. */
+  never(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").never`);
+    return this.add({ kind: "never", label: label ?? `never(${describePredicate(np)})`, p: np });
+  }
+  /** At least one span in the trace must satisfy `p`. */
+  eventually(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").eventually`);
+    return this.add({
+      kind: "eventually",
+      label: label ?? `eventually(${describePredicate(np)})`,
+      p: np
+    });
+  }
+  /** Every `b`-match must have a strictly earlier `a`-match. */
+  precedes(a, b, label) {
+    const na = normalizePredicate(a, `traceContract("${this.name}").precedes (a)`);
+    const nb = normalizePredicate(b, `traceContract("${this.name}").precedes (b)`);
+    return this.add({
+      kind: "precedes",
+      label: label ?? `precedes(${describePredicate(na)} -> ${describePredicate(nb)})`,
+      a: na,
+      b: nb
+    });
+  }
+  /** Every `p`-match is a violation unless a strictly earlier `prior`-match exists. */
+  neverUnless(p, prior, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").neverUnless (p)`);
+    const nprior = normalizePredicate(prior, `traceContract("${this.name}").neverUnless (prior)`);
+    return this.add({
+      kind: "neverUnless",
+      label: label ?? `neverUnless(${describePredicate(np)} unless ${describePredicate(nprior)})`,
+      p: np,
+      prior: nprior
+    });
+  }
+  build() {
+    if (this.rules.length === 0) {
+      throw new ValidationError(
+        `traceContract("${this.name}").build(): no rules \u2014 an empty contract would vacuously pass`
+      );
+    }
+    return { name: this.name, rules: [...this.rules] };
+  }
+  add(rule) {
+    let label = rule.label;
+    let n = 2;
+    while (this.rules.some((r) => r.label === label)) {
+      label = `${rule.label} #${n}`;
+      n += 1;
+    }
+    this.rules.push({ ...rule, label });
+    return this;
+  }
+};
+function traceContract(name) {
+  if (typeof name !== "string" || name.length === 0) {
+    throw new ValidationError("traceContract: name must be a non-empty string");
+  }
+  return new TraceContractBuilder(name);
+}
+function assertContract(contract) {
+  if (typeof contract?.name !== "string" || contract.name.length === 0) {
+    throw new ValidationError("evaluateTraceContract: contract.name must be a non-empty string");
+  }
+  if (!Array.isArray(contract.rules) || contract.rules.length === 0) {
+    throw new ValidationError(
+      `evaluateTraceContract: contract "${contract.name}" has no rules \u2014 an empty contract would vacuously pass`
+    );
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const rule of contract.rules) {
+    const where = `contract "${contract.name}" rule "${rule?.label ?? "<unlabeled>"}"`;
+    if (typeof rule?.label !== "string" || rule.label.length === 0) {
+      throw new ValidationError(`${where}: label must be a non-empty string`);
+    }
+    if (seen.has(rule.label)) {
+      throw new ValidationError(`${where}: duplicate label would collapse per-rule scores`);
+    }
+    seen.add(rule.label);
+    switch (rule.kind) {
+      case "always":
+      case "never":
+      case "eventually":
+        assertPredicate(rule.p, `${where} (p)`);
+        break;
+      case "precedes":
+        assertPredicate(rule.a, `${where} (a)`);
+        assertPredicate(rule.b, `${where} (b)`);
+        break;
+      case "neverUnless":
+        assertPredicate(rule.p, `${where} (p)`);
+        assertPredicate(rule.prior, `${where} (prior)`);
+        break;
+      default:
+        throw new ValidationError(
+          `${where}: unknown rule kind "${String(rule.kind)}"`
+        );
+    }
+  }
+}
+function orderSpans(spans) {
+  const timed = spans.filter(
+    (s) => typeof s.startedAt === "number" && Number.isFinite(s.startedAt)
+  ).length;
+  if (timed === spans.length) {
+    return spans.map((span, i) => ({ span, i })).sort((x, y) => x.span.startedAt - y.span.startedAt || x.i - y.i).map((x) => x.span);
+  }
+  if (timed === 0) return [...spans];
+  throw new ValidationError(
+    `evaluateTraceContract: ${timed}/${spans.length} spans carry a finite startedAt \u2014 mixed timestamps make ordering ambiguous; stamp every span or none`
+  );
+}
+function spanRef(span, index) {
+  return span.spanId ?? `#${index}`;
+}
+function checkGuarded(args) {
+  const out = [];
+  let guardSeen = false;
+  for (let i = 0; i < args.ordered.length; i++) {
+    const span = args.ordered[i];
+    if (!guardSeen && matchSpan(span, args.guarded)) {
+      out.push({ rule: args.label, spanId: span.spanId, detail: args.detail(spanRef(span, i)) });
+    }
+    if (matchSpan(span, args.guard)) guardSeen = true;
+  }
+  return out;
+}
+function checkRule(rule, ordered) {
+  switch (rule.kind) {
+    case "always": {
+      const out = [];
+      for (let i = 0; i < ordered.length; i++) {
+        const span = ordered[i];
+        if (!matchSpan(span, rule.p)) {
+          out.push({
+            rule: rule.label,
+            spanId: span.spanId,
+            detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") fails always(${describePredicate(rule.p)})`
+          });
+        }
+      }
+      return out;
+    }
+    case "never": {
+      const out = [];
+      for (let i = 0; i < ordered.length; i++) {
+        const span = ordered[i];
+        if (matchSpan(span, rule.p)) {
+          out.push({
+            rule: rule.label,
+            spanId: span.spanId,
+            detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
+          });
+        }
+      }
+      return out;
+    }
+    case "eventually": {
+      const hit = ordered.some((span) => matchSpan(span, rule.p));
+      return hit ? [] : [
+        {
+          rule: rule.label,
+          detail: `no span matches eventually(${describePredicate(rule.p)}) over ${ordered.length} span(s)`
+        }
+      ];
+    }
+    case "precedes":
+      return checkGuarded({
+        label: rule.label,
+        ordered,
+        guard: rule.a,
+        guarded: rule.b,
+        detail: (ref) => `span ${ref} matches ${describePredicate(rule.b)} with no earlier ${describePredicate(rule.a)} match`
+      });
+    case "neverUnless":
+      return checkGuarded({
+        label: rule.label,
+        ordered,
+        guard: rule.prior,
+        guarded: rule.p,
+        detail: (ref) => `span ${ref} matches ${describePredicate(rule.p)} with no earlier ${describePredicate(rule.prior)} match`
+      });
+  }
 }
-function keyPreserved(key) {
+function evaluateTraceContract(contract, spans) {
+  assertContract(contract);
+  const ordered = orderSpans(spans);
+  const scores2 = {};
+  const violations = [];
+  for (const rule of contract.rules) {
+    const ruleViolations = checkRule(rule, ordered);
+    scores2[rule.label] = ruleViolations.length === 0 ? 1 : 0;
+    violations.push(...ruleViolations);
+  }
+  const ruleCount = contract.rules.length;
+  const passCount = Object.values(scores2).filter((s) => s === 1).length;
   return {
-    id: `preserved(${key})`,
-    description: `"${key}" unchanged from before to after`,
-    score: ({ before, after }) => before[key] !== void 0 && before[key] === after[key] ? 1 : 0
+    contract: contract.name,
+    valid: passCount === ruleCount,
+    score: passCount / ruleCount,
+    scores: scores2,
+    violations,
+    notes: `${passCount}/${ruleCount} rules passed`
   };
 }
-function collectionPreserved(key, minRatio = 1) {
-  return {
-    id: `collection-preserved(${key})`,
-    description: `"${key}" length \u2265 ${minRatio} \xD7 prior length`,
-    score: ({ before, after }) => {
-      const b = before[key];
-      const a = after[key];
-      if (!Array.isArray(b) || !Array.isArray(a)) return 0;
-      if (b.length === 0) return a.length === 0 ? 1 : 1;
-      return Math.min(1, a.length / (b.length * minRatio));
-    }
-  };
+function checkTraceContracts(spans, contracts) {
+  if (contracts.length === 0) {
+    throw new ValidationError(
+      "checkTraceContracts: empty contract list would vacuously pass \u2014 supply at least one contract"
+    );
+  }
+  const verdicts = contracts.map((c) => evaluateTraceContract(c, spans));
+  return { verdicts, allValid: verdicts.every((v) => v.valid) };
 }
-function statusAdvanced(key, progression) {
+function contractJudge(contracts, opts) {
+  if (contracts.length === 0) {
+    throw new ValidationError("contractJudge: at least one contract required");
+  }
+  for (const c of contracts) assertContract(c);
+  const names = /* @__PURE__ */ new Set();
+  for (const c of contracts) {
+    if (names.has(c.name)) {
+      throw new ValidationError(
+        `contractJudge: duplicate contract name "${c.name}" would collapse judge dimensions`
+      );
+    }
+    names.add(c.name);
+  }
+  if (typeof opts?.spans !== "function") {
+    throw new ValidationError("contractJudge: opts.spans extraction function is required");
+  }
+  const dimensions = contracts.map((c) => ({
+    key: c.name,
+    description: c.rules.map((r) => r.label).join("; ")
+  }));
   return {
-    id: `status-advanced(${key})`,
-    description: `"${key}" progressed along ${progression.join("\u2192")}`,
-    score: ({ before, after }) => {
-      const bi = progression.indexOf(String(before[key]));
-      const ai2 = progression.indexOf(String(after[key]));
-      if (bi === -1 || ai2 === -1) return 0;
-      return ai2 >= bi ? 1 : 0;
+    name: opts.name ?? "trace-contracts",
+    dimensions,
+    score({ artifact, scenario }) {
+      const spans = opts.spans({ artifact, scenario });
+      if (!Array.isArray(spans)) {
+        throw new ValidationError(
+          `contractJudge: spans() must return a span array, got ${typeof spans}`
+        );
+      }
+      const { verdicts } = checkTraceContracts(spans, contracts);
+      const dims = {};
+      const violations = [];
+      for (const v of verdicts) {
+        dims[v.contract] = v.score;
+        violations.push(...v.violations);
+      }
+      const composite = verdicts.reduce((acc, v) => acc + v.score, 0) / verdicts.length;
+      return {
+        dimensions: dims,
+        composite,
+        notes: violations.length === 0 ? "all trace contracts satisfied" : violations.slice(0, 8).map((x) => `${x.rule}: ${x.detail}`).join("\n")
+      };
     }
   };
 }
 
-// src/ui-finding.ts
-var UI_LENSES = [
-  "consistency",
-  "hierarchy",
-  "layout",
-  "ux-flow",
-  "duplication",
-  "accessibility",
-  "responsive",
-  "states",
-  "content",
-  "interaction",
-  "performance-perceived",
-  "other"
-];
-var UI_FINDING_SEVERITIES = [
-  "critical",
-  "high",
-  "med",
-  "low"
-];
-
 // src/behavior-dsl.ts
 var BehaviorAssertion = class {
   constructor(store, runId) {
@@ -4516,6 +5750,25 @@ var BehaviorAssertion = class {
       }
     };
   }
+  /** Evaluate a finite-trace temporal contract (`traceContract(...)`) over
+   *  this run's span sequence. See `trace-contracts.ts` for the operators. */
+  toSatisfyContract(contract) {
+    return {
+      label: `agent(${this.runId}).toSatisfyContract(${contract.name})`,
+      check: async () => {
+        const spans = await this.store.spans({ runId: this.runId });
+        const verdict = evaluateTraceContract(contract, spans);
+        if (verdict.valid) {
+          return { ok: true, detail: `contract "${contract.name}": ${verdict.notes}` };
+        }
+        return {
+          ok: false,
+          detail: verdict.violations.map((v) => `${v.rule}: ${v.detail}`).join("; "),
+          evidence: verdict.violations.find((v) => v.spanId)?.spanId
+        };
+      }
+    };
+  }
   toNeverCall(toolName) {
     return {
       label: `agent(${this.runId}).toNeverCall(${toolName})`,
@@ -5118,90 +6371,6 @@ async function promptBisect(options) {
   };
 }
 
-// src/counterfactual.ts
-async function runCounterfactual(store, originalRunId, mutation, runner) {
-  const originalRun = await store.getRun(originalRunId);
-  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
-  const trajectory = await buildTrajectory(store, originalRunId);
-  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {
-    throw new ValidationError(
-      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`
-    );
-  }
-  const targetStep = trajectory.steps[mutation.at];
-  const mutatedStep = applyMutation(targetStep, mutation);
-  const cfEmitter = new TraceEmitter(store);
-  await cfEmitter.startRun({
-    scenarioId: originalRun.scenarioId,
-    variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
-    projectId: originalRun.projectId,
-    parentRunId: originalRunId,
-    layer: "meta",
-    tags: { counterfactual: "true", mutationKind: mutation.kind, mutationAt: String(mutation.at) }
-  });
-  await runner.executeFrom(
-    {
-      originalRunId,
-      originalTrajectory: trajectory,
-      prefix: trajectory.steps.slice(0, mutation.at),
-      mutation,
-      mutatedStep
-    },
-    cfEmitter
-  );
-  const counterfactual = await store.getRun(cfEmitter.runId);
-  const delta = {
-    originalOutcomeScore: originalRun.outcome?.score ?? null,
-    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
-    deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
-  };
-  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta };
-}
-function applyMutation(step, mutation) {
-  if (mutation.kind === "swap-model" && step.span.kind === "llm") {
-    const llm = step.span;
-    return { ...step, span: { ...llm, model: mutation.newModel } };
-  }
-  if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
-    const tool = step.span;
-    return { ...step, span: { ...tool, result: mutation.newResult } };
-  }
-  if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
-    const llm = step.span;
-    return {
-      ...step,
-      span: {
-        ...llm,
-        messages: [{ role: "system", content: mutation.content }, ...llm.messages]
-      }
-    };
-  }
-  if (mutation.kind === "custom") return mutation.apply(step);
-  return step;
-}
-function attributeCounterfactuals(results) {
-  const grouped = /* @__PURE__ */ new Map();
-  for (const r of results) {
-    const arr = grouped.get(r.mutation.kind) ?? [];
-    arr.push(r);
-    grouped.set(r.mutation.kind, arr);
-  }
-  const out = [];
-  for (const [kind, items] of grouped) {
-    const deltas = items.map((i) => i.delta.deltaScore).filter((d) => typeof d === "number");
-    if (deltas.length === 0) continue;
-    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length;
-    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length;
-    out.push({
-      mutationKind: kind,
-      n: deltas.length,
-      meanAbsDelta: meanAbs,
-      meanSignedDelta: meanSigned
-    });
-  }
-  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta);
-}
-
 // src/cross-trace-diff.ts
 async function crossTraceDiff(store, runA, runB, options = {}) {
   const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)]);
@@ -5873,13 +7042,13 @@ function extractErrorCount(text, opts = {}) {
     (p) => !opts.only || opts.only.includes(p.name)
   );
   for (const p of patterns) {
-    const matches = Array.from(text.matchAll(p.regex));
-    if (matches.length === 0) continue;
-    const count = p.transform ? matches.reduce((sum3, m) => sum3 + p.transform(m), 0) : matches.length;
+    const matches2 = Array.from(text.matchAll(p.regex));
+    if (matches2.length === 0) continue;
+    const count = p.transform ? matches2.reduce((sum3, m) => sum3 + p.transform(m), 0) : matches2.length;
     return {
       count,
       matched: p.name,
-      samples: matches.slice(0, 5).map((m) => m[0])
+      samples: matches2.slice(0, 5).map((m) => m[0])
     };
   }
   return { count: null, matched: null, samples: [] };
@@ -6393,14 +7562,18 @@ function aggregate(layers, results, startedAt, startedAtMs) {
     }
   }
   const finishedAtMs = Date.now();
+  const allPass = ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0;
+  const blendedScore = scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0;
   return {
     layers: results,
     passCount,
     failCount,
     skippedCount,
     errorCount,
-    allPass: ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0,
-    blendedScore: scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0,
+    allPass,
+    blendedScore,
+    valid: allPass,
+    score: blendedScore,
     durationMs: finishedAtMs - startedAtMs,
     startedAt,
     finishedAt: new Date(finishedAtMs).toISOString()
@@ -6820,7 +7993,7 @@ function scoreScenario(scenario, matcher, threshold, matchStrategy) {
 }
 function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
   const candidatesLeft = scenario.candidates.map((candidate, index) => ({ candidate, index }));
-  const matches = [];
+  const matches2 = [];
   for (const reference of scenario.references) {
     let best = null;
     for (const item of candidatesLeft) {
@@ -6833,7 +8006,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
     if (best && best.score >= threshold) {
       const matchIndex = candidatesLeft.findIndex((item) => item.index === best.index);
       if (matchIndex >= 0) candidatesLeft.splice(matchIndex, 1);
-      matches.push({
+      matches2.push({
         scenarioId: scenario.id,
         referenceId: reference.id,
         candidateId: best.candidate.id,
@@ -6843,7 +8016,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
         reason: best.reason
       });
     } else {
-      matches.push({
+      matches2.push({
         scenarioId: scenario.id,
         referenceId: reference.id,
         candidateId: best?.candidate.id ?? null,
@@ -6854,7 +8027,7 @@ function scoreScenarioReferenceOrder(scenario, matcher, threshold) {
       });
     }
   }
-  return buildScenarioScore(scenario, matches, candidatesLeft.length);
+  return buildScenarioScore(scenario, matches2, candidatesLeft.length);
 }
 function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
   const pairs = [];
@@ -6881,7 +8054,7 @@ function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
     selectedByReference.set(pair.referenceIndex, pair);
     selectedCandidates.add(pair.candidateIndex);
   }
-  const matches = scenario.references.map((reference, referenceIndex) => {
+  const matches2 = scenario.references.map((reference, referenceIndex) => {
     const weight = reference.weight ?? 1;
     const selected = selectedByReference.get(referenceIndex);
     if (selected) {
@@ -6906,7 +8079,7 @@ function scoreScenarioGlobalGreedy(scenario, matcher, threshold) {
       reason: bestRejected?.reason ?? "no candidates"
     };
   });
-  return buildScenarioScore(scenario, matches, scenario.candidates.length - selectedCandidates.size);
+  return buildScenarioScore(scenario, matches2, scenario.candidates.length - selectedCandidates.size);
 }
 function scorePair(scenario, matcher, reference, candidate) {
   const result = matcher(reference, candidate, scenario);
@@ -6917,11 +8090,11 @@ function scorePair(scenario, matcher, reference, candidate) {
   }
   return { score: clamp012(result.score), reason: result.reason ?? "" };
 }
-function buildScenarioScore(scenario, matches, falsePositives) {
-  const matched = matches.filter((match) => match.matched).length;
+function buildScenarioScore(scenario, matches2, falsePositives) {
+  const matched = matches2.filter((match) => match.matched).length;
   const total = scenario.references.length;
-  const matchedWeight = matches.filter((match) => match.matched).reduce((sum3, match) => sum3 + match.weight, 0);
-  const totalWeight = matches.reduce((sum3, match) => sum3 + match.weight, 0);
+  const matchedWeight = matches2.filter((match) => match.matched).reduce((sum3, match) => sum3 + match.weight, 0);
+  const totalWeight = matches2.reduce((sum3, match) => sum3 + match.weight, 0);
   const precision2 = ratio(matched, matched + falsePositives);
   const recall = ratio(matched, total);
   return {
@@ -6935,7 +8108,7 @@ function buildScenarioScore(scenario, matches, falsePositives) {
     precision: precision2,
     recall,
     f1: f1(precision2, recall),
-    matches
+    matches: matches2
   };
 }
 function aggregateBySplit(scores2) {
@@ -7196,7 +8369,7 @@ function createDefaultReviewer(options) {
 
 // src/description-length-gate.ts
 import { gzipSync } from "zlib";
-function runScore2(run) {
+function runScore3(run) {
   const o = run.outcome;
   const s = o.holdoutScore ?? o.searchScore ?? o.raw?.score;
   return typeof s === "number" && Number.isFinite(s) ? s : void 0;
@@ -7207,7 +8380,7 @@ function taskKey(run) {
 function perTaskMeanScore(runs) {
   const acc = /* @__PURE__ */ new Map();
   for (const run of runs) {
-    const s = runScore2(run);
+    const s = runScore3(run);
     if (s === void 0) continue;
     const key = taskKey(run);
     const cur = acc.get(key) ?? { sum: 0, n: 0 };
@@ -7359,10 +8532,10 @@ async function discoverPersonas(dir, opts = {}) {
 function matchGoldens(goldens, candidates, options = {}) {
   const extract = options.text ?? defaultExtract2;
   const haystacks = candidates.map((c) => extract(c).toLowerCase());
-  const matches = goldens.map((golden) => goldenMatched(golden, haystacks));
+  const matches2 = goldens.map((golden) => goldenMatched(golden, haystacks));
   return {
-    matches,
-    hits: matches.filter(Boolean).length,
+    matches: matches2,
+    hits: matches2.filter(Boolean).length,
     total: goldens.length
   };
 }
@@ -7615,69 +8788,6 @@ function fmt(x) {
   return x.toFixed(4);
 }
 
-// src/judge-retry.ts
-var DEFAULT_MAX_ATTEMPTS = 3;
-var DEFAULT_TIMEOUT_MS = 3e5;
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-async function withJudgeRetry(judgeFn, policy = {}) {
-  const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
-  const timeoutMs = policy.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const backoff = policy.backoffMs ?? backoffMs;
-  const isRetryable = policy.isRetryable ?? isTransientLlmError;
-  const models = policy.models && policy.models.length > 0 ? policy.models : [void 0];
-  let totalAttempts = 0;
-  const attemptErrors = [];
-  let lastError;
-  for (const model of models) {
-    for (let attempt = 0; attempt < maxAttempts; attempt++) {
-      totalAttempts += 1;
-      const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(new Error("TimeoutError")), timeoutMs);
-      try {
-        const value = await judgeFn(model, controller.signal);
-        clearTimeout(timer);
-        return {
-          value,
-          succeeded: true,
-          attempts: totalAttempts,
-          modelUsed: model,
-          attemptErrors
-        };
-      } catch (err) {
-        clearTimeout(timer);
-        const errObj = err instanceof Error ? err : new Error(String(err));
-        lastError = errObj;
-        attemptErrors.push({
-          attempt: totalAttempts,
-          model: model ?? "(default)",
-          error: errObj.message
-        });
-        if (!isRetryable(errObj)) {
-          return {
-            value: null,
-            succeeded: false,
-            attempts: totalAttempts,
-            error: errObj,
-            attemptErrors
-          };
-        }
-        if (attempt < maxAttempts - 1) {
-          await sleep(backoff(attempt));
-        }
-      }
-    }
-  }
-  return {
-    value: null,
-    succeeded: false,
-    attempts: totalAttempts,
-    error: lastError,
-    attemptErrors
-  };
-}
-
 // src/orthogonality.ts
 function passOrthogonality(input) {
   const passes = input.passes;
@@ -8056,6 +9166,7 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
     });
     try {
       const allScores = [];
+      let failedJudges = 0;
       for (let i = 0; i < judges.length; i++) {
         const judge = judges[i];
         const name = judgeNames[i] ?? `judge_${i}`;
@@ -8063,8 +9174,13 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           emitter: opts.emitter,
           parentSpanId: ensembleSpan.span.spanId
         });
-        const scores2 = await tracedFn(tc, input);
-        allScores.push(...scores2);
+        try {
+          const scores2 = await tracedFn(tc, input);
+          allScores.push(...scores2);
+        } catch (err) {
+          if (!(err instanceof JudgeParseError)) throw err;
+          failedJudges++;
+        }
       }
       const composite = allScores.length > 0 ? allScores.reduce((sum3, s) => sum3 + s.score, 0) / allScores.length : 0;
       await ensembleSpan.end({
@@ -8072,6 +9188,7 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           "judge.ensemble_size": judges.length,
           "judge.composite_score": composite,
           "judge.total_dimensions": allScores.length,
+          "judge.failed_judges": failedJudges,
           "eval.phase": "judge"
         }
       });
@@ -8501,8 +9618,256 @@ function applyDomainPatch(p, sectionId, newBody) {
 function sectionHash(section) {
   return surfaceContentHash(JSON.stringify({ title: section.title, body: section.body }));
 }
+
+// src/cost-report.ts
+function costReport(ledger) {
+  const summary = ledger.summary();
+  const perModel = /* @__PURE__ */ new Map();
+  for (const entry of ledger.list()) {
+    const roll = perModel.get(entry.model) ?? {
+      model: entry.model,
+      usd: 0,
+      entries: 0,
+      unpriced: false
+    };
+    roll.usd += entry.costUsd;
+    roll.entries += 1;
+    if (entry.costUnknown) roll.unpriced = true;
+    perModel.set(entry.model, roll);
+  }
+  return {
+    perChannel: summary.byChannel,
+    total: {
+      usd: summary.totalCostUsd,
+      unknownEntries: summary.byChannel.reduce((sum3, c) => sum3 + c.unpricedCalls, 0)
+    },
+    perModel: [...perModel.values()].sort((a, b) => a.model.localeCompare(b.model))
+  };
+}
+function attachCostToReport(report, ledger) {
+  if ("cost" in report) {
+    throw new ValidationError(
+      "attachCostToReport: report already has a 'cost' key \u2014 refusing to overwrite an existing stamp"
+    );
+  }
+  return { ...report, cost: costReport(ledger) };
+}
+
+// src/model-seats.ts
+var seatPresets = {
+  economy: {
+    worker: "kimi-k2.6",
+    judges: ["kimi-k2.6", "deepseek-v4-pro", "gpt-4.1-mini"],
+    analyst: "gpt-4.1-mini",
+    reflection: "gpt-4.1-mini",
+    verifier: "deepseek-v4-pro"
+  },
+  frontier: {}
+};
+var SeatUnsetError = class extends ConfigError {
+  constructor(seat) {
+    super(
+      `ModelSeats: seat '${seat}' is unset and no fallback was given \u2014 name a model explicitly (a model id is a budget decision, never a silent default)`
+    );
+    this.seat = seat;
+  }
+  seat;
+};
+function resolveSeat(seats, seat, fallback) {
+  const value = seats[seat];
+  if (seat === "judges") {
+    if (value !== void 0 && !Array.isArray(value)) {
+      throw new ValidationError(`ModelSeats: seat 'judges' must be a string[], got ${typeof value}`);
+    }
+    const models = Array.isArray(value) ? value : [];
+    if (models.length > 0) {
+      const blank = models.findIndex((m) => typeof m !== "string" || m.trim() === "");
+      if (blank >= 0) {
+        throw new ValidationError(
+          `ModelSeats: judges[${blank}] is blank \u2014 every panel model must be a non-empty id`
+        );
+      }
+      return [...models];
+    }
+  } else {
+    if (value !== void 0 && typeof value !== "string") {
+      throw new ValidationError(`ModelSeats: seat '${seat}' must be a string, got ${typeof value}`);
+    }
+    if (typeof value === "string" && value.trim() !== "") return value;
+  }
+  if (fallback !== void 0) {
+    if (fallback.trim() === "") {
+      throw new ValidationError(`ModelSeats: fallback for seat '${seat}' is blank`);
+    }
+    return seat === "judges" ? [fallback] : fallback;
+  }
+  throw new SeatUnsetError(seat);
+}
+
+// src/verdict-cache.ts
+import { createHash } from "crypto";
+import { appendFileSync as appendFileSync3, existsSync as existsSync5, readFileSync as readFileSync6 } from "fs";
+function canonicalizeAt(value, path) {
+  if (value === null) return "null";
+  switch (typeof value) {
+    case "boolean":
+      return value ? "true" : "false";
+    case "number":
+      if (!Number.isFinite(value)) {
+        throw new Error(
+          `canonicalJson: non-finite number (${value}) at ${path} \u2014 ambiguity is an error, not a coercion`
+        );
+      }
+      return JSON.stringify(value);
+    case "string":
+      return JSON.stringify(value);
+    case "undefined":
+    case "function":
+    case "symbol":
+      throw new Error(
+        `canonicalJson: ${typeof value} at ${path} \u2014 ambiguity is an error, not a coercion`
+      );
+    case "bigint":
+      throw new Error(`canonicalJson: bigint at ${path} \u2014 not representable in JSON`);
+    case "object":
+      break;
+  }
+  const obj = value;
+  if (typeof obj["toJSON"] === "function") {
+    return canonicalizeAt(obj.toJSON(), path);
+  }
+  if (Array.isArray(obj)) {
+    return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
+  }
+  if (obj instanceof Map || obj instanceof Set) {
+    throw new Error(
+      `canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} \u2014 would serialize as '{}'; convert to a plain object/array first`
+    );
+  }
+  const keys = Object.keys(obj).sort();
+  const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`);
+  return `{${parts.join(",")}}`;
+}
+function canonicalJson(value) {
+  return canonicalizeAt(value, "$");
+}
+function contentHash(value) {
+  return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+function inMemoryVerdictCache() {
+  const entries = /* @__PURE__ */ new Map();
+  return {
+    get: (key) => entries.get(key),
+    set: (key, score) => {
+      entries.set(key, score);
+    }
+  };
+}
+function parseCacheLine(line, path, lineNo) {
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch (err) {
+    throw new Error(
+      `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} \u2014 ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  const rec = parsed;
+  if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") {
+    throw new Error(
+      `fileVerdictCache: invalid record shape at ${path}:${lineNo} \u2014 expected {key, score:{dimensions, composite, notes}}`
+    );
+  }
+  return rec;
+}
+function fileVerdictCache(path) {
+  const entries = /* @__PURE__ */ new Map();
+  if (existsSync5(path)) {
+    const lines = readFileSync6(path, "utf8").split("\n");
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (line === void 0 || line.trim() === "") continue;
+      const rec = parseCacheLine(line, path, i + 1);
+      entries.set(rec.key, rec.score);
+    }
+  }
+  return {
+    get: (key) => entries.get(key),
+    set: (key, score) => {
+      appendFileSync3(path, `${JSON.stringify({ key, score })}
+`, "utf8");
+      entries.set(key, score);
+    }
+  };
+}
+function cachedJudge(judge, store, options) {
+  if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") {
+    throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
+  }
+  const stats = { hits: 0, misses: 0 };
+  const wrapped = {
+    name: judge.name,
+    dimensions: judge.dimensions,
+    async score(input) {
+      const key = contentHash({
+        artifact: canonicalJson(input.artifact),
+        scenarioId: input.scenario.id,
+        judgeName: judge.name,
+        dimensions: judge.dimensions,
+        judgeVersion: options.judgeVersion
+      });
+      const cached = await store.get(key);
+      if (cached !== void 0) {
+        stats.hits += 1;
+        return cached;
+      }
+      const score = await judge.score(input);
+      await store.set(key, score);
+      stats.misses += 1;
+      return score;
+    },
+    stats: () => ({ ...stats })
+  };
+  if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
+  return wrapped;
+}
+
+// src/attestation.ts
+var ATTESTATION_ALGORITHM = "sha256/canonical-json";
+function attest(report, provenance) {
+  return {
+    reportHash: contentHash(report),
+    provenance,
+    algorithm: ATTESTATION_ALGORITHM
+  };
+}
+function verifyAttestation(report, attested) {
+  if (attested.algorithm !== ATTESTATION_ALGORITHM) {
+    return {
+      valid: false,
+      reason: `unknown algorithm '${attested.algorithm}' \u2014 this verifier only checks '${ATTESTATION_ALGORITHM}'`
+    };
+  }
+  let recomputed;
+  try {
+    recomputed = contentHash(report);
+  } catch (err) {
+    return {
+      valid: false,
+      reason: `report is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  if (recomputed !== attested.reportHash) {
+    return {
+      valid: false,
+      reason: `report hash mismatch: attested ${attested.reportHash}, recomputed ${recomputed}`
+    };
+  }
+  return { valid: true };
+}
 export {
   AGENT_PROFILE_KINDS,
+  ATTESTATION_ALGORITHM,
   AgentDriver,
   AgentEvalError,
   AgentProfileCellValidationError,
@@ -8517,6 +9882,7 @@ export {
   CaptureIntegrityError,
   ConfigError,
   ConvergenceTracker,
+  CostLedger,
   CostTracker,
   CrossFamilyError,
   DEFAULT_AGENT_SLOS,
@@ -8538,6 +9904,8 @@ export {
   DockerSandboxDriver,
   DualAgentBench,
   ERROR_COUNT_PATTERNS,
+  EvalTraceStore,
+  ExperimentTracker,
   FAILURE_CLASSES,
   FAILURE_MODE_KIND_SPEC,
   FileSystemFeedbackTrajectoryStore,
@@ -8554,6 +9922,7 @@ export {
   InMemoryTraceStore,
   InMemoryWorkspaceInspector,
   JudgeError,
+  JudgeParseError,
   JudgeRunner,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
@@ -8586,6 +9955,7 @@ export {
   SKILL_USAGE_ANALYST,
   SandboxHarness,
   ScenarioRegistry,
+  SeatUnsetError,
   SingleBackendError,
   SkillUsageAnalyst,
   SpanNotFoundError,
@@ -8596,6 +9966,7 @@ export {
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TRACE_SCHEMA_VERSION,
   TokenCounter,
+  TraceContractBuilder,
   TraceEmitter,
   TraceFileMissingError,
   TraceNotFoundError,
@@ -8630,6 +10001,9 @@ export {
   assertRunCaptured,
   assertSingleBackend,
   assignFeedbackSplit,
+  assignHeldOutTag,
+  attachCostToReport,
+  attest,
   attributeCounterfactuals,
   backoffMs,
   deterministicSplit as benchmarkDeterministicSplit,
@@ -8651,17 +10025,20 @@ export {
   buildTraceInsightPrompt,
   buildTrajectory,
   byteLengthRange,
+  cachedJudge,
   calibrateJudge,
   calibrateJudgeContinuous,
   callLlm,
   callLlmJson,
   canaryLeakView,
+  canonicalJson,
   canonicalize,
   captureFetchToRawSink,
   causalAttribution,
   checkBehavioralCanary,
   checkCanaries,
   checkSlos,
+  checkTraceContracts,
   clamp01,
   classifyEuAiRisk,
   classifyFailure,
@@ -8675,20 +10052,26 @@ export {
   compareReferenceReplay,
   compareToBaseline,
   compilerJudge,
+  completionVerdict,
   composeParsers,
   composeValidators,
+  computeExperimentStats,
   computeFindingId,
   computeToolUseMetrics,
   computeTraceMetrics,
   confidenceInterval,
   containsAll,
+  contentHash,
   continuousAgreement,
+  contractJudge,
   controlFailureClassFromVerification,
   controlRunToFeedbackTrajectory,
   controlRunToRunRecord,
   convertTraceStoresToOtlp,
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
+  costForUsage,
+  costReport,
   createAnalystAi,
   createAntiSlopJudge,
   createChatClient,
@@ -8727,6 +10110,8 @@ export {
   distillPlaybook,
   domainEvidencePattern,
   dominates,
+  eProcess,
+  ensembleJudge,
   estimateCost,
   estimateTokens,
   euAiActReport,
@@ -8736,6 +10121,7 @@ export {
   evaluateInterimReleaseConfidence,
   evaluateOracles,
   evaluateReleaseConfidence,
+  evaluateTraceContract,
   executeScenario,
   expectAgent,
   exportRewardModel,
@@ -8744,6 +10130,9 @@ export {
   extractErrorCount,
   extractOtlpAttributes,
   extractProducedState,
+  extractUsage,
+  extractUsageFromResponse,
+  extractUsageFromSse,
   feedbackTrajectoriesToDatasetScenarios,
   feedbackTrajectoriesToOptimizerRows,
   feedbackTrajectoryToDatasetScenario,
@@ -8751,6 +10140,8 @@ export {
   fieldAgreement,
   fileContains,
   fileExists,
+  fileExperimentStore,
+  fileVerdictCache,
   findAutoMatchNoExpectation,
   findConstructorCwdDropped,
   findFallbackToPass,
@@ -8760,12 +10151,14 @@ export {
   firstStringAttr,
   flattenOtlpExportToNdjson,
   flowLayer,
+  fnv1a32,
   formatBenchmarkReport,
   formatDriverReport,
   formatFindings,
   formatScorecardDiff,
   gainHistogram,
   ghCliClient,
+  gitProvenanceReader,
   precision as goldenPrecision,
   gradeSemanticStatus,
   groupBy,
@@ -8773,10 +10166,15 @@ export {
   hashContent,
   hashJson,
   hashScenarios,
+  hashToUnit,
   htmlContainsElement,
   httpGithubClient,
+  improvementVerdict,
+  inMemoryExperimentStore,
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
+  inMemoryRunRecordBackend,
+  inMemoryVerdictCache,
   inferDomainKeywords,
   inferOtlpKind,
   interRaterReliability,
@@ -8797,6 +10195,7 @@ export {
   jsonShape,
   jsonlReferenceReplayStore,
   jsonlReviewStore,
+  jsonlRunRecordBackend,
   judgeFamily,
   judgeReplayGate,
   judgeSpans,
@@ -8810,12 +10209,16 @@ export {
   loadScorerFromGrader,
   localCommandRunner,
   lowercaseMutator,
+  makeEvalTools,
   makeFinding,
   mannWhitneyU,
   matchGoldens,
+  matchSpan,
   mergeLayerResults,
   mergeSteeringBundle,
   modelDescriptionBits,
+  modelPriceKey,
+  mulberry32,
   multiToolchainLayer,
   nistAiRmfReport,
   normalizeScores,
@@ -8840,6 +10243,7 @@ export {
   parseRunRecordSafe,
   parseRuntimeTrajectoryHookEvent,
   partialCredit,
+  partitionHeldOut,
   passOrthogonality,
   pixelDeltaRatio,
   planTraceInsightQuestions,
@@ -8883,6 +10287,7 @@ export {
   researchReport,
   resetLockedAppendersForTesting,
   resolveModelPricing,
+  resolveSeat,
   roundTripRunRecord,
   rowCount,
   rowWhere,
@@ -8905,6 +10310,7 @@ export {
   runProposeReview,
   runProposeReviewAsControlLoop,
   runReferenceReplay,
+  runScore,
   runSelfPlay,
   runSemanticConceptJudge,
   runTestGradedScenario,
@@ -8919,6 +10325,7 @@ export {
   scoreRedTeamOutput,
   scoreReferenceReplay,
   scoreTraceInsightReadiness,
+  seatPresets,
   securityJudge,
   selectHarnessVariant,
   selfPreference,
@@ -8944,12 +10351,14 @@ export {
   throwIfRunIncomplete,
   toAgentProfileJson,
   toLangfuseEnvelope,
+  toOpenAiTool,
   toPrometheusText,
   tokenizeDomainWords,
   toolNamesForRun,
   toolSpans,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete,
+  traceContract,
   traceJudge,
   traceJudgeEnsemble,
   tracedAnalyzeTraces,
@@ -8960,6 +10369,7 @@ export {
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,
+  verifyAttestation,
   verifyCompletion,
   verifyManifest,
   visualDiff,