@tangle-network/agent-eval 0.86.0 → 0.89.0

package/dist/index.js

@@ -1,22 +1,35 @@
+import {
+  HoldoutAuditor,
+  analyzeRuns,
+  canaryLeakView,
+  checkBehavioralCanary,
+  checkCanaries,
+  runBehavioralCanaries
+} from "./chunk-Y47J2LJ3.js";
+import {
+  classifyEuAiRisk,
+  euAiActReport,
+  nistAiRmfReport,
+  renderMarkdown,
+  soc2Report,
+  summarize
+} from "./chunk-KKHDIONI.js";
+import {
+  acquisitionPlansForKnowledgeGaps,
+  blockingKnowledgeEval,
+  knowledgeReadinessTracePayload,
+  scoreKnowledgeReadiness,
+  userQuestionsForKnowledgeGaps
+} from "./chunk-3CKU6VGU.js";
 import {
   agentProfileHash,
+  completionVerdict,
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
   extractProducedState,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-YGYXHNAQ.js";
-import {
-  parseRuntimeTrajectoryHookEvent,
-  projectRuntimeTrajectoryEvidence
-} from "./chunk-T4SQEITX.js";
-import {
-  HoldoutAuditor,
-  canaryLeakView,
-  checkBehavioralCanary,
-  checkCanaries,
-  runBehavioralCanaries
-} from "./chunk-SHTXZ4O2.js";
+} from "./chunk-FIUKOSWI.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -40,12 +53,16 @@ import {
   scoreRedTeamOutput,
   surfaceContentHash,
   toolNamesForRun
-} from "./chunk-BABOZOSN.js";
+} from "./chunk-ZFIBGEOL.js";
 import {
   BackendIntegrityError,
   assertRealBackend,
   summarizeBackendIntegrity
-} from "./chunk-ZZ2HOPME.js";
+} from "./chunk-TWS7AZEY.js";
+import {
+  parseRuntimeTrajectoryHookEvent,
+  projectRuntimeTrajectoryEvidence
+} from "./chunk-T4SQEITX.js";
 import {
   MODEL_PRICING,
   MetricsCollector,
@@ -67,14 +84,14 @@ import {
   computeToolUseMetrics,
   iqr,
   welchsTTest
-} from "./chunk-RBNA5AZT.js";
+} from "./chunk-L3JOU6XM.js";
+import {
+  analyzeSeries
+} from "./chunk-BOD4O7OF.js";
 import {
   exportTrainingData,
   toNdjson
 } from "./chunk-KMPRBJK4.js";
-import {
-  buildTrajectory
-} from "./chunk-RZTMDUO7.js";
 import {
   DockerSandboxDriver,
   SandboxHarness,
@@ -85,21 +102,6 @@ import {
   runTestGradedScenario,
   vitestTestParser
 } from "./chunk-T375SUOZ.js";
-import {
-  classifyEuAiRisk,
-  euAiActReport,
-  nistAiRmfReport,
-  renderMarkdown,
-  soc2Report,
-  summarize
-} from "./chunk-KKHDIONI.js";
-import {
-  acquisitionPlansForKnowledgeGaps,
-  blockingKnowledgeEval,
-  knowledgeReadinessTracePayload,
-  scoreKnowledgeReadiness,
-  userQuestionsForKnowledgeGaps
-} from "./chunk-3CKU6VGU.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RUN_SCORE_WEIGHTS,
@@ -111,9 +113,7 @@ import {
   SKILL_USAGE_ANALYST,
   SkillUsageAnalyst,
   aggregateRunScore,
-  buildDefaultAnalystRegistry,
   clamp01,
-  computeTraceMetrics,
   createAnalystAi,
   createChatClient,
   createSemanticConceptJudge,
@@ -121,7 +121,11 @@ import {
   diffFindings,
   resetLockedAppendersForTesting,
   runSemanticConceptJudge
-} from "./chunk-L5G7OUKD.js";
+} from "./chunk-XY4DDNEG.js";
+import {
+  buildDefaultAnalystRegistry,
+  computeTraceMetrics
+} from "./chunk-UMMZHCPB.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -129,11 +133,9 @@ import {
   IMPROVEMENT_KIND_SPEC,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
-  computeFindingId,
   createTraceAnalystKind,
-  makeFinding,
   renderPriorFindings
-} from "./chunk-VIDQF3F5.js";
+} from "./chunk-D3V5B42D.js";
 import {
   controlFailureClassFromVerification,
   controlRunToRunRecord,
@@ -159,10 +161,10 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-FZWAFVAA.js";
+} from "./chunk-4FBZZIYD.js";
 import {
   runEvalCampaign
-} from "./chunk-WJL2NJXN.js";
+} from "./chunk-GSH6QNNS.js";
 import {
   LlmCallError,
   LlmClient,
@@ -185,7 +187,18 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-Z7VFTS2J.js";
+} from "./chunk-CY6U5S3X.js";
+import {
+  attributeCounterfactuals,
+  runCounterfactual
+} from "./chunk-REVYNR6C.js";
+import {
+  buildTrajectory
+} from "./chunk-RZTMDUO7.js";
+import {
+  computeFindingId,
+  makeFinding
+} from "./chunk-45EEMHTC.js";
 import {
   benjaminiHochberg,
   bonferroni,
@@ -197,9 +210,11 @@ import {
   continuousAgreement,
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
+  eProcess,
   interRaterReliability,
   interpretCliffs,
   mannWhitneyU,
+  mulberry32,
   normalizeScores,
   pairedBootstrap,
   pairedMde,
@@ -212,7 +227,7 @@ import {
   weightedComposite,
   weightedMean,
   wilcoxonSignedRank
-} from "./chunk-IDVBLYCY.js";
+} from "./chunk-LMZQ2Z4U.js";
 import {
   FileSystemTraceStore,
   InMemoryTraceStore,
@@ -243,7 +258,7 @@ import {
   scoreTraceInsightReadiness,
   tokenizeDomainWords,
   traceAnalystOnRunComplete
-} from "./chunk-P2J6SOXT.js";
+} from "./chunk-QG2OVF2D.js";
 import {
   DEFAULT_REDACTION_RULES,
   REDACTION_VERSION,
@@ -270,16 +285,18 @@ import {
   isToolSpan
 } from "./chunk-5BKGXME7.js";
 import {
-  DEFAULT_TRACE_ANALYST_BUDGETS,
-  OtlpFileTraceStore,
-  SpanNotFoundError,
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
   TRACE_ANALYST_SUBAGENT_DESCRIPTION,
+  analyzeTraces
+} from "./chunk-UHMJT4T7.js";
+import {
+  DEFAULT_TRACE_ANALYST_BUDGETS,
+  OtlpFileTraceStore,
+  SpanNotFoundError,
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TraceFileMissingError,
   TraceNotFoundError,
-  analyzeTraces,
   asNumber,
   asString,
   buildTraceAnalystTools,
@@ -291,7 +308,7 @@ import {
   readOtlpStatus,
   stringField,
   traceAnalystFunctionGroup
-} from "./chunk-VUINJM5M.js";
+} from "./chunk-QAY5UIJO.js";
 import {
   RunIntegrityError,
   assertRunCaptured,
@@ -646,132 +663,337 @@ function renderCommitMessage(input) {
   return lines.join("\n").trim();
 }
 
-// src/executor.ts
-async function executeScenario(tc, scenario, config) {
-  const startTime = Date.now();
-  const model = config.model ?? "gpt-4o";
-  const systemPrompt = [config.systemPrompt, scenario.systemPromptAppend ?? ""].filter(Boolean).join("\n\n");
-  const messages = [{ role: "system", content: systemPrompt }];
-  const turns = [];
-  const allCodeBlocks = [];
-  const allBlocks = [];
-  const allToolCalls = [];
-  const blockRe = config.blockPattern ?? /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
-  for (let i = 0; i < scenario.turns.length; i++) {
-    const turn = scenario.turns[i];
-    const turnStart = Date.now();
-    messages.push({ role: "user", content: turn.user });
+// src/judges.ts
+var JudgeParseError = class extends JudgeError {
+  /** Name of the judge whose response failed to parse. */
+  judgeName;
+  /** The raw (truncated) model response that failed to parse. */
+  raw;
+  constructor(judgeName, raw, options) {
+    super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options);
+    this.judgeName = judgeName;
+    this.raw = raw;
+  }
+};
+function createDomainExpertJudge(domain) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
     const resp = await tc.chat({
-      model,
-      messages,
-      temperature: 0.4,
-      maxTokens: 3e3
-    });
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    messages.push({ role: "assistant", content });
-    const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
-    let codeMatch = codeRe.exec(content);
-    while (codeMatch !== null) {
-      allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] ?? "" });
-      codeMatch = codeRe.exec(content);
-    }
-    const turnBlocks = [];
-    const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
-    let blockMatch = blockReLocal.exec(content);
-    while (blockMatch !== null) {
-      const fields = {};
-      for (const line of (blockMatch[2] ?? "").split("\n")) {
-        const idx = line.indexOf(":");
-        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
-      }
-      const blockType = blockMatch[1] ?? "";
-      allBlocks.push({ type: blockType, fields });
-      turnBlocks.push({ type: blockType, title: fields.title ?? "" });
-      blockMatch = blockReLocal.exec(content);
-    }
-    let hasToolCall = false;
-    if (config.toolCallPatterns) {
-      for (const pattern of config.toolCallPatterns) {
-        const re = new RegExp(pattern.source, pattern.flags);
-        let toolMatch = re.exec(content);
-        while (toolMatch !== null) {
-          allToolCalls.push(toolMatch[0]);
-          hasToolCall = true;
-          toolMatch = re.exec(content);
+      model: "gpt-4o",
+      messages: [
+        {
+          role: "system",
+          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
+
+Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
+
+Evaluate:
+1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
+2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
+
+Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
+        },
+        {
+          role: "user",
+          content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
         }
-      }
-    }
-    turns.push({
-      turnIndex: i,
-      userMessage: turn.user,
-      agentResponse: content,
-      durationMs: Date.now() - turnStart,
-      blocksExtracted: turnBlocks,
-      containsCode: allCodeBlocks.length > 0,
-      containsToolCall: hasToolCall
+      ],
+      temperature: 0.1,
+      maxTokens: 800
     });
-  }
-  const artifacts = {
-    vaultFiles: [],
-    blocksExtracted: allBlocks,
-    codeBlocks: allCodeBlocks,
-    toolCalls: allToolCalls
+    return parseJudgeResponse("domain_expert", resp);
   };
-  const artifactResults = scenario.artifactChecks.map((check2) => {
-    if (config.artifactChecker) {
-      const custom = config.artifactChecker(check2, artifacts);
-      if (custom) return { check: check2, ...custom };
-    }
-    switch (check2.type) {
-      case "block_extracted": {
-        const count = allBlocks.filter((b) => b.type === check2.target).length;
-        return {
-          check: check2,
-          passed: count >= (check2.minCount ?? 1),
-          detail: `Found ${count} ${check2.target} blocks (need ${check2.minCount ?? 1})`
-        };
+}
+var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
+  const codeBlocks = artifacts.codeBlocks;
+  if (codeBlocks.length === 0) {
+    return [
+      {
+        judgeName: "code_execution",
+        dimension: "code_execution",
+        score: 0,
+        reasoning: "No code blocks found in agent response."
       }
-      case "code_valid": {
-        const hasCode = allCodeBlocks.some(
-          (b) => b.language === check2.target || b.code.includes(check2.target)
-        );
-        return { check: check2, passed: hasCode, detail: hasCode ? "Code block found" : "No matching code" };
+    ];
+  }
+  const codeText = codeBlocks.map(
+    (b, i) => `Block ${i + 1} (${b.language}):
+\`\`\`${b.language}
+${b.code.slice(0, 3e3)}
+\`\`\``
+  ).join("\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You are a principal software engineer reviewing code written by an AI agent.
+
+Score STRICTLY:
+1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
+2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
+3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
+
+Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Task: ${scenario.thesis}
+
+${codeText}`
       }
-      default:
-        return {
-          check: check2,
-          passed: false,
-          detail: `Check type "${check2.type}" requires live environment`
-        };
-    }
+    ],
+    temperature: 0.1,
+    maxTokens: 1e3
   });
-  const judgeInput = { scenario, turns, artifacts };
-  const judgeResults = [];
-  for (const judge of config.judges) {
-    let lastErr = "";
-    for (let attempt = 0; attempt < 3; attempt++) {
-      try {
-        if (attempt > 0) {
-          const wait = attempt * 1e4;
-          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
-          await new Promise((r) => setTimeout(r, wait));
-        }
-        const scores2 = await judge(tc, judgeInput);
-        judgeResults.push(scores2);
-        await new Promise((r) => setTimeout(r, 3e3));
-        break;
-      } catch (err) {
-        lastErr = err instanceof Error ? err.message : String(err);
-        if (attempt === 2) {
-          judgeResults.push([
-            {
-              judgeName: "unknown",
-              dimension: "error",
-              score: 0,
-              reasoning: `Judge failed after 3 attempts: ${lastErr.slice(0, 200)}`
-            }
-          ]);
+  return parseJudgeResponse("code_execution", resp);
+};
+var coherenceJudge = async (tc, { scenario, turns }) => {
+  if (turns.length < 2) {
+    return [];
+  }
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
+
+Score STRICTLY:
+1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
+2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
+3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
+
+Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Scenario: ${scenario.thesis}
+
+${conversation}`
+      }
+    ],
+    temperature: 0.1,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("coherence", resp);
+};
+var adversarialJudge = async (tc, { scenario, turns }) => {
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [
+      {
+        role: "system",
+        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
+
+1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
+2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
+3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
+
+Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
+
+Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
+      },
+      {
+        role: "user",
+        content: `Persona: ${scenario.persona}
+Scenario: ${scenario.thesis}
+
+${conversation}`
+      }
+    ],
+    temperature: 0.2,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("adversarial", resp);
+};
+function createCustomJudge(name, systemPrompt, opts) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
+    const resp = await tc.chat({
+      model: opts?.model ?? "gpt-4o",
+      messages: [
+        {
+          role: "system",
+          content: systemPrompt
+        },
+        {
+          role: "user",
+          content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
+        }
+      ],
+      temperature: opts?.temperature ?? 0.1,
+      maxTokens: opts?.maxTokens ?? 1e3
+    });
+    return parseJudgeResponse(name, resp);
+  };
+}
+function defaultJudges(domain) {
+  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
+}
+function parseJudgeResponse(judgeName, resp) {
+  const content = resp.choices?.[0]?.message?.content ?? "";
+  try {
+    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
+    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+    if (arrayMatch) cleaned = arrayMatch[0];
+    const parsed = JSON.parse(cleaned);
+    return parsed.map((p) => ({
+      judgeName,
+      dimension: p.dimension,
+      score: Math.max(0, Math.min(10, p.score)),
+      reasoning: p.reasoning ?? "",
+      evidence: p.evidence
+    }));
+  } catch (err) {
+    throw new JudgeParseError(judgeName, content, { cause: err });
+  }
+}
+
+// src/executor.ts
+async function executeScenario(tc, scenario, config) {
+  const startTime = Date.now();
+  const model = config.model ?? "gpt-4o";
+  const systemPrompt = [config.systemPrompt, scenario.systemPromptAppend ?? ""].filter(Boolean).join("\n\n");
+  const messages = [{ role: "system", content: systemPrompt }];
+  const turns = [];
+  const allCodeBlocks = [];
+  const allBlocks = [];
+  const allToolCalls = [];
+  const blockRe = config.blockPattern ?? /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
+  for (let i = 0; i < scenario.turns.length; i++) {
+    const turn = scenario.turns[i];
+    const turnStart = Date.now();
+    messages.push({ role: "user", content: turn.user });
+    const resp = await tc.chat({
+      model,
+      messages,
+      temperature: 0.4,
+      maxTokens: 3e3
+    });
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    messages.push({ role: "assistant", content });
+    const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
+    let codeMatch = codeRe.exec(content);
+    while (codeMatch !== null) {
+      allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] ?? "" });
+      codeMatch = codeRe.exec(content);
+    }
+    const turnBlocks = [];
+    const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
+    let blockMatch = blockReLocal.exec(content);
+    while (blockMatch !== null) {
+      const fields = {};
+      for (const line of (blockMatch[2] ?? "").split("\n")) {
+        const idx = line.indexOf(":");
+        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+      }
+      const blockType = blockMatch[1] ?? "";
+      allBlocks.push({ type: blockType, fields });
+      turnBlocks.push({ type: blockType, title: fields.title ?? "" });
+      blockMatch = blockReLocal.exec(content);
+    }
+    let hasToolCall = false;
+    if (config.toolCallPatterns) {
+      for (const pattern of config.toolCallPatterns) {
+        const re = new RegExp(pattern.source, pattern.flags);
+        let toolMatch = re.exec(content);
+        while (toolMatch !== null) {
+          allToolCalls.push(toolMatch[0]);
+          hasToolCall = true;
+          toolMatch = re.exec(content);
+        }
+      }
+    }
+    turns.push({
+      turnIndex: i,
+      userMessage: turn.user,
+      agentResponse: content,
+      durationMs: Date.now() - turnStart,
+      blocksExtracted: turnBlocks,
+      containsCode: allCodeBlocks.length > 0,
+      containsToolCall: hasToolCall
+    });
+  }
+  const artifacts = {
+    vaultFiles: [],
+    blocksExtracted: allBlocks,
+    codeBlocks: allCodeBlocks,
+    toolCalls: allToolCalls
+  };
+  const artifactResults = scenario.artifactChecks.map((check2) => {
+    if (config.artifactChecker) {
+      const custom = config.artifactChecker(check2, artifacts);
+      if (custom) return { check: check2, ...custom };
+    }
+    switch (check2.type) {
+      case "block_extracted": {
+        const count = allBlocks.filter((b) => b.type === check2.target).length;
+        return {
+          check: check2,
+          passed: count >= (check2.minCount ?? 1),
+          detail: `Found ${count} ${check2.target} blocks (need ${check2.minCount ?? 1})`
+        };
+      }
+      case "code_valid": {
+        const hasCode = allCodeBlocks.some(
+          (b) => b.language === check2.target || b.code.includes(check2.target)
+        );
+        return { check: check2, passed: hasCode, detail: hasCode ? "Code block found" : "No matching code" };
+      }
+      default:
+        return {
+          check: check2,
+          passed: false,
+          detail: `Check type "${check2.type}" requires live environment`
+        };
+    }
+  });
+  const judgeInput = { scenario, turns, artifacts };
+  const judgeResults = [];
+  let failedJudges = 0;
+  for (const judge of config.judges) {
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        if (attempt > 0) {
+          const wait = attempt * 1e4;
+          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
+          await new Promise((r) => setTimeout(r, wait));
+        }
+        const scores2 = await judge(tc, judgeInput);
+        judgeResults.push(scores2);
+        await new Promise((r) => setTimeout(r, 3e3));
+        break;
+      } catch (err) {
+        if (err instanceof JudgeParseError) {
+          failedJudges++;
+          break;
         }
+        if (attempt === 2) failedJudges++;
       }
     }
   }
@@ -799,7 +1021,7 @@ async function executeScenario(tc, scenario, config) {
     turns,
     artifactResults,
     judgeScores: allScores,
-    judgeErrors: errorScores.length,
+    judgeErrors: errorScores.length + failedJudges,
     overallScore,
     totalDurationMs: Date.now() - startTime,
     artifacts
@@ -1451,11 +1673,11 @@ var FileSystemFeedbackTrajectoryStore = class {
   }
   async load() {
     if (this.loaded) return;
-    const { readFile } = await import("fs/promises");
+    const { readFile: readFile2 } = await import("fs/promises");
     const { join: join4 } = await import("path");
     const file = join4(this.dir, "feedback-trajectories.ndjson");
     try {
-      const raw = await readFile(file, "utf8");
+      const raw = await readFile2(file, "utf8");
       for (const line of raw.split("\n")) {
         if (!line.trim()) continue;
         try {
@@ -1824,380 +2046,169 @@ async function preflightModels(opts) {
   return { succeeded: true, value: results, error: null };
 }
 var ModelsUnreachableError = class extends AgentEvalError {
-  constructor(message, results) {
-    super("config", message);
-    this.results = results;
-    this.name = "ModelsUnreachableError";
-  }
-  results;
-};
-function describeFailure(r) {
-  if (!r.listed) {
-    const probeNote = r.served === false ? ` (probe ${r.status}${r.detail ? `: ${r.detail}` : ""})` : "";
-    return `${r.model}: not in /models${probeNote}`;
-  }
-  return `${r.model}: listed but probe ${r.status}${r.detail ? ` \u2014 ${r.detail}` : ""}`;
-}
-async function assertModelsServed(opts) {
-  const outcome = await preflightModels(opts);
-  if (!outcome.succeeded || outcome.value === null) {
-    throw new ConfigError(
-      outcome.error ?? "assertModelsServed: preflight failed without an error message"
-    );
-  }
-  const dead = outcome.value.filter((r) => !r.listed || r.served === false);
-  if (dead.length > 0) {
-    throw new ModelsUnreachableError(
-      `assertModelsServed: ${dead.length}/${outcome.value.length} model(s) unreachable on the router \u2014 ${dead.map(describeFailure).join("; ")}`,
-      outcome.value
-    );
-  }
-  return outcome.value;
-}
-
-// src/integrity/single-backend.ts
-var SingleBackendError = class extends AgentEvalError {
-  constructor(message, report) {
-    super("backend_integrity", message);
-    this.report = report;
-    this.name = "SingleBackendError";
-  }
-  report;
-};
-function stripSlash2(url) {
-  return url.replace(/\/+$/, "");
-}
-function assertSingleBackend(agent, judge, opts = {}) {
-  const divergences = [];
-  if (agent.kind !== judge.kind) {
-    divergences.push({ field: "kind", agent: agent.kind, judge: judge.kind });
-  }
-  if (stripSlash2(agent.baseUrl) !== stripSlash2(judge.baseUrl)) {
-    divergences.push({ field: "baseUrl", agent: agent.baseUrl, judge: judge.baseUrl });
-  }
-  if (agent.model !== judge.model) {
-    divergences.push({ field: "model", agent: agent.model, judge: judge.model });
-  }
-  if (agent.provider !== judge.provider) {
-    divergences.push({ field: "provider", agent: agent.provider, judge: judge.provider });
-  }
-  const agentHasKey = Boolean(agent.apiKey);
-  const judgeHasKey = Boolean(judge.apiKey);
-  if (agentHasKey !== judgeHasKey) {
-    divergences.push({
-      field: "apiKeyPresence",
-      agent: agentHasKey ? "set" : "empty",
-      judge: judgeHasKey ? "set" : "empty"
-    });
-  }
-  const blocking = opts.strict ? divergences : divergences.filter((d) => d.field !== "model");
-  const ok = blocking.length === 0;
-  const report = { ok, divergences };
-  if (!ok) {
-    const agentLabel = opts.agentLabel ?? "agent";
-    const judgeLabel = opts.judgeLabel ?? "judge";
-    const detail = blocking.map((d) => `${d.field}: ${agentLabel}=${d.agent ?? "\u2205"} vs ${judgeLabel}=${d.judge ?? "\u2205"}`).join("; ");
-    throw new SingleBackendError(
-      `single-backend: ${agentLabel} and ${judgeLabel} backends diverge \u2014 the judge would re-route through a different backend than the agent (${detail})`,
-      report
-    );
-  }
-  return report;
-}
-
-// src/judge-families.ts
-var PROVIDER_PREFIX = {
-  anthropic: "anthropic",
-  openai: "openai",
-  "azure-openai": "openai",
-  google: "google",
-  "google-vertex": "google",
-  meta: "meta",
-  "meta-llama": "meta",
-  mistral: "mistral",
-  mistralai: "mistral",
-  deepseek: "deepseek",
-  xai: "xai",
-  qwen: "qwen",
-  alibaba: "qwen",
-  cohere: "cohere",
-  amazon: "amazon",
-  bedrock: "amazon",
-  moonshot: "moonshot",
-  moonshotai: "moonshot",
-  kimi: "moonshot",
-  "kimi-code": "moonshot",
-  zhipu: "zhipu",
-  zhipuai: "zhipu",
-  zai: "zhipu",
-  "z-ai": "zhipu",
-  glm: "zhipu"
-};
-var NAME_PATTERNS = [
-  [/claude/i, "anthropic"],
-  [/\b(gpt|davinci|babbage)\b|^o[134]\b|[-/]o[134]\b|gpt-/i, "openai"],
-  [/gemini|palm|gemma|bison/i, "google"],
-  [/llama/i, "meta"],
-  [/mi(s|x)tral|codestral|magistral/i, "mistral"],
-  [/deepseek/i, "deepseek"],
-  [/grok/i, "xai"],
-  [/qwen/i, "qwen"],
-  [/command-?(r|a)?/i, "cohere"],
-  [/\b(nova|titan)\b/i, "amazon"],
-  [/\bkimi\b|moonshot/i, "moonshot"],
-  [/\bglm\b|zhipu|\bz-?ai\b/i, "zhipu"]
-];
-function judgeFamily(modelId) {
-  const id = modelId.trim().split("@")[0].toLowerCase();
-  const slash = id.indexOf("/");
-  if (slash > 0) {
-    const prefix = id.slice(0, slash);
-    const mapped = PROVIDER_PREFIX[prefix];
-    if (mapped) return mapped;
-  }
-  for (const [pattern, family] of NAME_PATTERNS) {
-    if (pattern.test(id)) return family;
-  }
-  return "unknown";
-}
-var CrossFamilyError = class extends Error {
-  constructor(message, families, models) {
-    super(message);
-    this.families = families;
-    this.models = models;
-    this.name = "CrossFamilyError";
-  }
-  families;
-  models;
-};
-function assertCrossFamily(models, opts = {}) {
-  const minFamilies = opts.minFamilies ?? 2;
-  const families = /* @__PURE__ */ new Set();
-  for (const m of models) {
-    const f = judgeFamily(m);
-    if (f === "unknown" && !opts.allowUnknown) continue;
-    families.add(f);
-  }
-  const list = [...families].sort();
-  if (list.length < minFamilies) {
-    throw new CrossFamilyError(
-      `judge ensemble spans ${list.length} provider famil${list.length === 1 ? "y" : "ies"} (${list.join(", ") || "none"}) but ${minFamilies} required \u2014 a single-family ensemble is correlated bias, not independent signal`,
-      list,
-      models
-    );
-  }
-  return list;
-}
-
-// src/judges.ts
-function createDomainExpertJudge(domain) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
-
-Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
-
-Evaluate:
-1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
-2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
-
-Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: 0.1,
-      maxTokens: 800
-    });
-    return parseJudgeResponse("domain_expert", resp);
-  };
-}
-var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
-  const codeBlocks = artifacts.codeBlocks;
-  if (codeBlocks.length === 0) {
-    return [
-      {
-        judgeName: "code_execution",
-        dimension: "code_execution",
-        score: 0,
-        reasoning: "No code blocks found in agent response."
-      }
-    ];
-  }
-  const codeText = codeBlocks.map(
-    (b, i) => `Block ${i + 1} (${b.language}):
-\`\`\`${b.language}
-${b.code.slice(0, 3e3)}
-\`\`\``
-  ).join("\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a principal software engineer reviewing code written by an AI agent.
-
-Score STRICTLY:
-1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
-2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
-3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
-
-Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Task: ${scenario.thesis}
-
-${codeText}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 1e3
-  });
-  return parseJudgeResponse("code_execution", resp);
-};
-var coherenceJudge = async (tc, { scenario, turns }) => {
-  if (turns.length < 2) {
-    return [];
-  }
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
-
-Score STRICTLY:
-1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
-2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
-3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
-
-Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("coherence", resp);
-};
-var adversarialJudge = async (tc, { scenario, turns }) => {
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
-
-1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
-2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
-3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
-
-Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
-
-Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Persona: ${scenario.persona}
-Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.2,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("adversarial", resp);
+  constructor(message, results) {
+    super("config", message);
+    this.results = results;
+    this.name = "ModelsUnreachableError";
+  }
+  results;
 };
-function createCustomJudge(name, systemPrompt, opts) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: opts?.model ?? "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: systemPrompt
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
+function describeFailure(r) {
+  if (!r.listed) {
+    const probeNote = r.served === false ? ` (probe ${r.status}${r.detail ? `: ${r.detail}` : ""})` : "";
+    return `${r.model}: not in /models${probeNote}`;
+  }
+  return `${r.model}: listed but probe ${r.status}${r.detail ? ` \u2014 ${r.detail}` : ""}`;
+}
+async function assertModelsServed(opts) {
+  const outcome = await preflightModels(opts);
+  if (!outcome.succeeded || outcome.value === null) {
+    throw new ConfigError(
+      outcome.error ?? "assertModelsServed: preflight failed without an error message"
+    );
+  }
+  const dead = outcome.value.filter((r) => !r.listed || r.served === false);
+  if (dead.length > 0) {
+    throw new ModelsUnreachableError(
+      `assertModelsServed: ${dead.length}/${outcome.value.length} model(s) unreachable on the router \u2014 ${dead.map(describeFailure).join("; ")}`,
+      outcome.value
+    );
+  }
+  return outcome.value;
+}
 
-${conversation}`
-        }
-      ],
-      temperature: opts?.temperature ?? 0.1,
-      maxTokens: opts?.maxTokens ?? 1e3
+// src/integrity/single-backend.ts
+var SingleBackendError = class extends AgentEvalError {
+  constructor(message, report) {
+    super("backend_integrity", message);
+    this.report = report;
+    this.name = "SingleBackendError";
+  }
+  report;
+};
+function stripSlash2(url) {
+  return url.replace(/\/+$/, "");
+}
+function assertSingleBackend(agent, judge, opts = {}) {
+  const divergences = [];
+  if (agent.kind !== judge.kind) {
+    divergences.push({ field: "kind", agent: agent.kind, judge: judge.kind });
+  }
+  if (stripSlash2(agent.baseUrl) !== stripSlash2(judge.baseUrl)) {
+    divergences.push({ field: "baseUrl", agent: agent.baseUrl, judge: judge.baseUrl });
+  }
+  if (agent.model !== judge.model) {
+    divergences.push({ field: "model", agent: agent.model, judge: judge.model });
+  }
+  if (agent.provider !== judge.provider) {
+    divergences.push({ field: "provider", agent: agent.provider, judge: judge.provider });
+  }
+  const agentHasKey = Boolean(agent.apiKey);
+  const judgeHasKey = Boolean(judge.apiKey);
+  if (agentHasKey !== judgeHasKey) {
+    divergences.push({
+      field: "apiKeyPresence",
+      agent: agentHasKey ? "set" : "empty",
+      judge: judgeHasKey ? "set" : "empty"
     });
-    return parseJudgeResponse(name, resp);
-  };
+  }
+  const blocking = opts.strict ? divergences : divergences.filter((d) => d.field !== "model");
+  const ok = blocking.length === 0;
+  const report = { ok, divergences };
+  if (!ok) {
+    const agentLabel = opts.agentLabel ?? "agent";
+    const judgeLabel = opts.judgeLabel ?? "judge";
+    const detail = blocking.map((d) => `${d.field}: ${agentLabel}=${d.agent ?? "\u2205"} vs ${judgeLabel}=${d.judge ?? "\u2205"}`).join("; ");
+    throw new SingleBackendError(
+      `single-backend: ${agentLabel} and ${judgeLabel} backends diverge \u2014 the judge would re-route through a different backend than the agent (${detail})`,
+      report
+    );
+  }
+  return report;
 }
-function defaultJudges(domain) {
-  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
+
+// src/judge-families.ts
+var PROVIDER_PREFIX = {
+  anthropic: "anthropic",
+  openai: "openai",
+  "azure-openai": "openai",
+  google: "google",
+  "google-vertex": "google",
+  meta: "meta",
+  "meta-llama": "meta",
+  mistral: "mistral",
+  mistralai: "mistral",
+  deepseek: "deepseek",
+  xai: "xai",
+  qwen: "qwen",
+  alibaba: "qwen",
+  cohere: "cohere",
+  amazon: "amazon",
+  bedrock: "amazon",
+  moonshot: "moonshot",
+  moonshotai: "moonshot",
+  kimi: "moonshot",
+  "kimi-code": "moonshot",
+  zhipu: "zhipu",
+  zhipuai: "zhipu",
+  zai: "zhipu",
+  "z-ai": "zhipu",
+  glm: "zhipu"
+};
+var NAME_PATTERNS = [
+  [/claude/i, "anthropic"],
+  [/\b(gpt|davinci|babbage)\b|^o[134]\b|[-/]o[134]\b|gpt-/i, "openai"],
+  [/gemini|palm|gemma|bison/i, "google"],
+  [/llama/i, "meta"],
+  [/mi(s|x)tral|codestral|magistral/i, "mistral"],
+  [/deepseek/i, "deepseek"],
+  [/grok/i, "xai"],
+  [/qwen/i, "qwen"],
+  [/command-?(r|a)?/i, "cohere"],
+  [/\b(nova|titan)\b/i, "amazon"],
+  [/\bkimi\b|moonshot/i, "moonshot"],
+  [/\bglm\b|zhipu|\bz-?ai\b/i, "zhipu"]
+];
+function judgeFamily(modelId) {
+  const id = modelId.trim().split("@")[0].toLowerCase();
+  const slash = id.indexOf("/");
+  if (slash > 0) {
+    const prefix = id.slice(0, slash);
+    const mapped = PROVIDER_PREFIX[prefix];
+    if (mapped) return mapped;
+  }
+  for (const [pattern, family] of NAME_PATTERNS) {
+    if (pattern.test(id)) return family;
+  }
+  return "unknown";
 }
-function parseJudgeResponse(judgeName, resp) {
-  try {
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
-    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
-    if (arrayMatch) cleaned = arrayMatch[0];
-    const parsed = JSON.parse(cleaned);
-    return parsed.map((p) => ({
-      judgeName,
-      dimension: p.dimension,
-      score: Math.max(0, Math.min(10, p.score)),
-      reasoning: p.reasoning ?? "",
-      evidence: p.evidence
-    }));
-  } catch (err) {
-    const content = resp.choices?.[0]?.message?.content ?? "";
-    console.log(
-      `    [parse_error] ${judgeName}: ${err.message?.slice(0, 50)} | response: ${content.slice(0, 100)}`
+var CrossFamilyError = class extends Error {
+  constructor(message, families, models) {
+    super(message);
+    this.families = families;
+    this.models = models;
+    this.name = "CrossFamilyError";
+  }
+  families;
+  models;
+};
+function assertCrossFamily(models, opts = {}) {
+  const minFamilies = opts.minFamilies ?? 2;
+  const families = /* @__PURE__ */ new Set();
+  for (const m of models) {
+    const f = judgeFamily(m);
+    if (f === "unknown" && !opts.allowUnknown) continue;
+    families.add(f);
+  }
+  const list = [...families].sort();
+  if (list.length < minFamilies) {
+    throw new CrossFamilyError(
+      `judge ensemble spans ${list.length} provider famil${list.length === 1 ? "y" : "ies"} (${list.join(", ") || "none"}) but ${minFamilies} required \u2014 a single-family ensemble is correlated bias, not independent signal`,
+      list,
+      models
     );
-    return [
-      {
-        judgeName,
-        dimension: "parse_error",
-        score: 0,
-        reasoning: `Parse failed: ${err.message?.slice(0, 100)}. Raw: ${content.slice(0, 200)}`
-      }
-    ];
   }
+  return list;
 }
 
 // src/live-proof.ts
@@ -2994,17 +3005,181 @@ var DualAgentBench = class {
         finalScore: lastScore
       });
     }
-    const convergedResults = results.filter((r) => r.converged);
-    const convergenceRate = results.length ? convergedResults.length / results.length : 0;
-    const avgRoundsToConverge = convergedResults.length ? convergedResults.reduce((acc, r) => acc + (r.roundsToConverge ?? 0), 0) / convergedResults.length : null;
-    const avgFinalScore = results.length ? results.reduce((acc, r) => acc + r.finalScore, 0) / results.length : 0;
-    return {
-      scenarios: results,
-      aggregate: { convergenceRate, avgRoundsToConverge, avgFinalScore },
-      config: { maxRounds, convergenceThreshold: threshold }
-    };
+    const convergedResults = results.filter((r) => r.converged);
+    const convergenceRate = results.length ? convergedResults.length / results.length : 0;
+    const avgRoundsToConverge = convergedResults.length ? convergedResults.reduce((acc, r) => acc + (r.roundsToConverge ?? 0), 0) / convergedResults.length : null;
+    const avgFinalScore = results.length ? results.reduce((acc, r) => acc + r.finalScore, 0) / results.length : 0;
+    return {
+      scenarios: results,
+      aggregate: { convergenceRate, avgRoundsToConverge, avgFinalScore },
+      config: { maxRounds, convergenceThreshold: threshold }
+    };
+  }
+};
+
+// src/eval-tools.ts
+import { readFile } from "fs/promises";
+function toOpenAiTool(def) {
+  return {
+    type: "function",
+    function: { name: def.name, description: def.description, parameters: def.parameters }
+  };
+}
+function makeEvalTools(cfg) {
+  const tools = [];
+  if (cfg.judges) tools.push(runJudgesTool(cfg.judges));
+  if (cfg.completion) tools.push(verifyCompletionTool(cfg.completion.checkCorrectness));
+  if (cfg.analyze) tools.push(analyzeRunsTool(cfg.analyze));
+  return tools;
+}
+function runJudgesTool(judges) {
+  if (judges.length === 0) {
+    throw new Error("makeEvalTools: cfg.judges is empty \u2014 supply judges or omit the section");
+  }
+  const names = judges.map((j) => j.name);
+  return {
+    name: "run_judges",
+    description: `Score an artifact with the configured judges (${names.join(", ")}). Pass \`judge\` to run one judge by name; omit it to run all. Returns per-judge scores keyed by judge name.`,
+    parameters: {
+      type: "object",
+      properties: {
+        judge: {
+          type: "string",
+          enum: names,
+          description: "Run only this judge. Omit to run every configured judge."
+        },
+        artifact: {
+          description: "The artifact to score \u2014 any JSON value the judges understand."
+        },
+        scenario: {
+          description: "Optional scenario context forwarded to the judges."
+        }
+      },
+      required: ["artifact"],
+      additionalProperties: false
+    },
+    handler: async (args, ctx) => {
+      const a = requireObjectArgs("run_judges", args);
+      if (!("artifact" in a)) {
+        throw new Error("run_judges: args.artifact is required");
+      }
+      let selected = judges;
+      if (a.judge !== void 0) {
+        if (typeof a.judge !== "string") throw new Error("run_judges: args.judge must be a string");
+        selected = judges.filter((j) => j.name === a.judge);
+        if (selected.length === 0) {
+          throw new Error(
+            `run_judges: unknown judge '${a.judge}' \u2014 configured: ${names.join(", ")}`
+          );
+        }
+      }
+      const signal = ctx?.signal ?? new AbortController().signal;
+      const scenario = a.scenario;
+      const scores2 = {};
+      for (const judge of selected) {
+        if (scenario !== void 0 && judge.appliesTo && !judge.appliesTo(scenario)) continue;
+        scores2[judge.name] = await judge.score({ artifact: a.artifact, scenario, signal });
+      }
+      return { scores: scores2 };
+    }
+  };
+}
+function verifyCompletionTool(checkCorrectness) {
+  return {
+    name: "verify_completion",
+    description: "Verify produced state against a gold task spec: each gold requirement is matched to at most one produced artifact/proposal/tool-call, correctness-checked by the host, and reduced to a CompletionVerdict (completionRate, fullyComplete, per-requirement checks).",
+    parameters: {
+      type: "object",
+      properties: {
+        gold: {
+          type: "object",
+          description: "TaskGold \u2014 taskId + requirements the produced state must satisfy."
+        },
+        state: {
+          type: "object",
+          description: "ProducedState \u2014 artifacts, proposals, and tool calls the agent produced."
+        }
+      },
+      required: ["gold", "state"],
+      additionalProperties: false
+    },
+    handler: async (args) => {
+      const a = requireObjectArgs("verify_completion", args);
+      if (typeof a.gold !== "object" || a.gold === null) {
+        throw new Error("verify_completion: args.gold (TaskGold) is required");
+      }
+      if (typeof a.state !== "object" || a.state === null) {
+        throw new Error("verify_completion: args.state (ProducedState) is required");
+      }
+      return verifyCompletion(a.gold, a.state, checkCorrectness);
+    }
+  };
+}
+function analyzeRunsTool(analyzeOpts) {
+  return {
+    name: "analyze_runs",
+    description: "Run analyzeRuns over a set of RunRecords and return the InsightReport (composite distribution, per-dimension stats, lift, failure clusters, recommendations). Pass the records inline via `runs`, or `path` to a .json (array) or .jsonl (one record per line) file on the host.",
+    parameters: {
+      type: "object",
+      properties: {
+        runs: {
+          type: "array",
+          items: { type: "object" },
+          description: "RunRecord[] inline."
+        },
+        path: {
+          type: "string",
+          description: "Host path to a JSON array or JSONL file of RunRecords."
+        }
+      },
+      additionalProperties: false
+    },
+    handler: async (args) => {
+      const a = requireObjectArgs("analyze_runs", args);
+      const hasRuns = Array.isArray(a.runs);
+      const hasPath = typeof a.path === "string" && a.path.length > 0;
+      if (hasRuns === hasPath) {
+        throw new Error("analyze_runs: pass exactly one of args.runs (array) or args.path (string)");
+      }
+      const runs = hasRuns ? a.runs : await loadRunRecords(a.path);
+      if (runs.length === 0) {
+        throw new Error("analyze_runs: no runs to analyze");
+      }
+      return analyzeRuns({ ...analyzeOpts, runs });
+    }
+  };
+}
+async function loadRunRecords(path) {
+  const text = await readFile(path, "utf8");
+  const trimmed = text.trim();
+  if (trimmed.length === 0) {
+    throw new Error(`analyze_runs: file '${path}' is empty`);
+  }
+  if (trimmed.startsWith("[")) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed)) {
+      throw new Error(`analyze_runs: file '${path}' did not parse to an array`);
+    }
+    return parsed;
   }
-};
+  return trimmed.split("\n").flatMap((line, i) => {
+    const l = line.trim();
+    if (l.length === 0) return [];
+    try {
+      return [JSON.parse(l)];
+    } catch (err) {
+      throw new Error(
+        `analyze_runs: file '${path}' line ${i + 1} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  });
+}
+function requireObjectArgs(tool, args) {
+  if (typeof args !== "object" || args === null || Array.isArray(args)) {
+    throw new Error(`${tool}: args must be an object`);
+  }
+  return args;
+}
 
 // src/harness-optimizer.ts
 var DEFAULT_HARNESS_OBJECTIVES = [
@@ -3127,15 +3302,22 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
   for (const d of dimensionKeys) dimAcc[d] = [];
   let rationale = "";
   let costUsd = 0;
+  const seenCount = /* @__PURE__ */ new Map();
+  const keyFor = (model) => {
+    const n = (seenCount.get(model) ?? 0) + 1;
+    seenCount.set(model, n);
+    return n === 1 ? model : `${model}#${n}`;
+  };
   for (const v of verdicts) {
     costUsd += v.costUsd ?? 0;
+    const key = keyFor(v.model);
     if (!v.perDimension) {
-      failedJudges.push(v.model);
+      failedJudges.push(key);
       continue;
     }
     const dims = {};
     for (const d of dimensionKeys) dims[d] = clamp01(Number(v.perDimension[d]));
-    perJudge[v.model] = dims;
+    perJudge[key] = dims;
     for (const d of dimensionKeys) dimAcc[d].push(dims[d]);
     if (!rationale && typeof v.rationale === "string" && v.rationale) rationale = v.rationale;
   }
@@ -3170,7 +3352,127 @@ function aggregateJudgeVerdicts(verdicts, dimensionKeys, weights) {
     maxDisagreement,
     failedJudges,
     costUsd,
-    rationale: rationale || "llm-judge"
+    rationale: rationale || "llm-judge",
+    verdicts: [...verdicts]
+  };
+}
+
+// src/judge-retry.ts
+var DEFAULT_MAX_ATTEMPTS = 3;
+var DEFAULT_TIMEOUT_MS = 3e5;
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function withJudgeRetry(judgeFn, policy = {}) {
+  const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+  const timeoutMs = policy.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const backoff = policy.backoffMs ?? backoffMs;
+  const isRetryable = policy.isRetryable ?? isTransientLlmError;
+  const models = policy.models && policy.models.length > 0 ? policy.models : [void 0];
+  let totalAttempts = 0;
+  const attemptErrors = [];
+  let lastError;
+  for (const model of models) {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      totalAttempts += 1;
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(new Error("TimeoutError")), timeoutMs);
+      try {
+        const value = await judgeFn(model, controller.signal);
+        clearTimeout(timer);
+        return {
+          value,
+          succeeded: true,
+          attempts: totalAttempts,
+          modelUsed: model,
+          attemptErrors
+        };
+      } catch (err) {
+        clearTimeout(timer);
+        const errObj = err instanceof Error ? err : new Error(String(err));
+        lastError = errObj;
+        attemptErrors.push({
+          attempt: totalAttempts,
+          model: model ?? "(default)",
+          error: errObj.message
+        });
+        if (!isRetryable(errObj)) {
+          return {
+            value: null,
+            succeeded: false,
+            attempts: totalAttempts,
+            error: errObj,
+            attemptErrors
+          };
+        }
+        if (attempt < maxAttempts - 1) {
+          await sleep(backoff(attempt));
+        }
+      }
+    }
+  }
+  return {
+    value: null,
+    succeeded: false,
+    attempts: totalAttempts,
+    error: lastError,
+    attemptErrors
+  };
+}
+
+// src/judge-panel.ts
+function ensembleJudge(opts) {
+  if (opts.models.length === 0) {
+    throw new Error(`ensembleJudge '${opts.name}': models is empty \u2014 nothing to score with`);
+  }
+  if (opts.dimensions.length === 0) {
+    throw new Error(`ensembleJudge '${opts.name}': dimensions is empty \u2014 nothing to score`);
+  }
+  if (opts.crossFamily !== false) {
+    assertCrossFamily(opts.models);
+  }
+  const scoreOne = async (model, input) => {
+    if (opts.retry) {
+      const outcome = await withJudgeRetry((m) => opts.scoreWith(m, input), {
+        ...opts.retry,
+        models: [model]
+      });
+      if (!outcome.succeeded || outcome.value === null) {
+        return {
+          model,
+          perDimension: null,
+          rationale: outcome.error?.message ?? "judge failed after retries"
+        };
+      }
+      return outcome.value;
+    }
+    try {
+      return await opts.scoreWith(model, input);
+    } catch (err) {
+      return {
+        model,
+        perDimension: null,
+        rationale: err instanceof Error ? err.message : String(err)
+      };
+    }
+  };
+  return {
+    name: opts.name,
+    dimensions: opts.dimensions.map((d) => ({ key: d, description: d })),
+    async score({ artifact, scenario }) {
+      const input = { artifact, scenario };
+      const verdicts = await Promise.all(opts.models.map((model) => scoreOne(model, input)));
+      const agg = aggregateJudgeVerdicts(verdicts, opts.dimensions, opts.weights);
+      const score = {
+        dimensions: agg.perDimension,
+        composite: agg.composite,
+        notes: agg.rationale,
+        maxDisagreement: agg.maxDisagreement,
+        perJudge: agg.perJudge
+      };
+      if (agg.failedJudges.length > 0) score.failedJudges = agg.failedJudges;
+      return score;
+    }
   };
 }
 
@@ -3505,7 +3807,7 @@ function createAxService(provider, apiKey, model) {
   });
 }
 function seededShuffle(items, seed) {
-  const rng = mulberry32(hashString(seed));
+  const rng = mulberry322(hashString(seed));
   const out = [...items];
   for (let i = out.length - 1; i > 0; i--) {
     const j = Math.floor(rng() * (i + 1));
@@ -3521,7 +3823,7 @@ function hashString(value) {
   }
   return h >>> 0;
 }
-function mulberry32(seed) {
+function mulberry322(seed) {
   let a = seed >>> 0;
   return () => {
     a = a + 1831565813 | 0;
@@ -4856,42 +5158,6 @@ function formatScorecardDiff(diff) {
   return lines.join("\n");
 }
 
-// src/series-convergence.ts
-function analyzeSeries(values, options = {}) {
-  const window = options.window ?? 5;
-  const stableCv = options.stableCv ?? 0.05;
-  const driftRun = options.driftRun ?? 3;
-  if (values.length < Math.max(2, Math.min(window, 3))) {
-    return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
-  }
-  const tail = values.slice(-window);
-  const mean5 = tail.reduce((a, b) => a + b, 0) / tail.length;
-  const variance = tail.reduce((acc, v) => acc + (v - mean5) ** 2, 0) / tail.length;
-  const stdDev = Math.sqrt(variance);
-  const refMean = Math.abs(mean5) > 1e-9 ? Math.abs(mean5) : 1;
-  const cv = stdDev / refMean;
-  const stable = tail.length >= window && cv <= stableCv;
-  let tailRun = 0;
-  let direction = 0;
-  for (let i = values.length - 1; i > 0; i--) {
-    const delta = values[i] - values[i - 1];
-    if (delta === 0) break;
-    const dir = delta > 0 ? 1 : -1;
-    if (direction === 0) direction = dir;
-    if (dir !== direction) break;
-    tailRun += dir;
-  }
-  let state;
-  if (stable) {
-    state = "stabilized";
-  } else if (Math.abs(tailRun) >= driftRun) {
-    state = tailRun > 0 ? "drifting-up" : "drifting-down";
-  } else {
-    state = "noisy";
-  }
-  return { state, windowMean: mean5, windowCv: cv, tailRun, stable };
-}
-
 // src/slo.ts
 function checkSlos(metrics, slos) {
   const results = slos.map((slo) => check(slo, metrics[slo.metric]));
@@ -4978,70 +5244,445 @@ function scoreContinuity(pair, checks, options = {}) {
   if (checks.length === 0) {
     throw new Error("scoreContinuity: at least 1 check required");
   }
-  const passThreshold = options.passThreshold ?? 0.8;
-  const results = checks.map((c) => {
-    const raw = c.score(pair);
-    const clamped = Number.isFinite(raw) ? Math.max(0, Math.min(1, raw)) : 0;
-    return { id: c.id, description: c.description, score: clamped, pass: clamped >= passThreshold };
-  });
-  const overallScore = results.reduce((a, r) => a + r.score, 0) / results.length;
-  return { results, overallScore, pass: results.every((r) => r.pass) };
+  const passThreshold = options.passThreshold ?? 0.8;
+  const results = checks.map((c) => {
+    const raw = c.score(pair);
+    const clamped = Number.isFinite(raw) ? Math.max(0, Math.min(1, raw)) : 0;
+    return { id: c.id, description: c.description, score: clamped, pass: clamped >= passThreshold };
+  });
+  const overallScore = results.reduce((a, r) => a + r.score, 0) / results.length;
+  return { results, overallScore, pass: results.every((r) => r.pass) };
+}
+function keyPreserved(key) {
+  return {
+    id: `preserved(${key})`,
+    description: `"${key}" unchanged from before to after`,
+    score: ({ before, after }) => before[key] !== void 0 && before[key] === after[key] ? 1 : 0
+  };
+}
+function collectionPreserved(key, minRatio = 1) {
+  return {
+    id: `collection-preserved(${key})`,
+    description: `"${key}" length \u2265 ${minRatio} \xD7 prior length`,
+    score: ({ before, after }) => {
+      const b = before[key];
+      const a = after[key];
+      if (!Array.isArray(b) || !Array.isArray(a)) return 0;
+      if (b.length === 0) return a.length === 0 ? 1 : 1;
+      return Math.min(1, a.length / (b.length * minRatio));
+    }
+  };
+}
+function statusAdvanced(key, progression) {
+  return {
+    id: `status-advanced(${key})`,
+    description: `"${key}" progressed along ${progression.join("\u2192")}`,
+    score: ({ before, after }) => {
+      const bi = progression.indexOf(String(before[key]));
+      const ai2 = progression.indexOf(String(after[key]));
+      if (bi === -1 || ai2 === -1) return 0;
+      return ai2 >= bi ? 1 : 0;
+    }
+  };
+}
+
+// src/ui-finding.ts
+var UI_LENSES = [
+  "consistency",
+  "hierarchy",
+  "layout",
+  "ux-flow",
+  "duplication",
+  "accessibility",
+  "responsive",
+  "states",
+  "content",
+  "interaction",
+  "performance-perceived",
+  "other"
+];
+var UI_FINDING_SEVERITIES = [
+  "critical",
+  "high",
+  "med",
+  "low"
+];
+
+// src/trace-contracts.ts
+function isSerializedRegex(v) {
+  return typeof v === "object" && v !== null && typeof v.$regex === "string" && typeof v.flags === "string";
+}
+function matchText(actual, matcher) {
+  if (typeof actual !== "string") return false;
+  if (typeof matcher === "string") return actual === matcher;
+  if (matcher instanceof RegExp) return matcher.test(actual);
+  return new RegExp(matcher.$regex, matcher.flags).test(actual);
+}
+function resolveToolName(span) {
+  if (typeof span.toolName === "string") return span.toolName;
+  const fromAttr = span.attributes?.["tool.name"] ?? span.attributes?.["toolName"];
+  if (typeof fromAttr === "string") return fromAttr;
+  if (span.kind === "tool" && typeof span.name === "string") return span.name;
+  return void 0;
+}
+function assertPredicate(value, where) {
+  if (value === null || typeof value !== "object") {
+    throw new ValidationError(`${where}: predicate must be an object, got ${typeof value}`);
+  }
+  const p = value;
+  for (const field of ["name", "tool"]) {
+    const m = p[field];
+    if (m !== void 0 && typeof m !== "string" && !(m instanceof RegExp) && !isSerializedRegex(m)) {
+      throw new ValidationError(`${where}: "${field}" must be string | RegExp | SerializedRegex`);
+    }
+  }
+  if (p.attr !== void 0 && (p.attr === null || typeof p.attr !== "object" || Array.isArray(p.attr))) {
+    throw new ValidationError(`${where}: "attr" must be a plain object`);
+  }
+  if (p.requiresCustom && typeof p.custom !== "function") {
+    throw new ValidationError(
+      `${where}: rule was built with a custom predicate function, which does not survive JSON serialization \u2014 re-attach \`custom\` after deserializing or drop the rule`
+    );
+  }
+  const hasAttr = p.attr !== void 0 && Object.keys(p.attr).length > 0;
+  if (p.name === void 0 && p.tool === void 0 && !hasAttr && typeof p.custom !== "function") {
+    throw new ValidationError(
+      `${where}: empty predicate would match every span \u2014 specify name, tool, attr, or custom`
+    );
+  }
+}
+function matchSpan(span, predicate) {
+  assertPredicate(predicate, "matchSpan");
+  if (predicate.name !== void 0 && !matchText(span.name, predicate.name)) return false;
+  if (predicate.tool !== void 0 && !matchText(resolveToolName(span), predicate.tool))
+    return false;
+  if (predicate.attr !== void 0) {
+    for (const [key, expected] of Object.entries(predicate.attr)) {
+      const actual = span.attributes?.[key];
+      if (expected instanceof RegExp || isSerializedRegex(expected)) {
+        if (!matchText(typeof actual === "string" ? actual : void 0, expected)) {
+          return false;
+        }
+      } else if (actual !== expected) {
+        return false;
+      }
+    }
+  }
+  if (predicate.custom !== void 0 && !predicate.custom(span)) return false;
+  return true;
+}
+function describeMatcher(m) {
+  if (typeof m === "string") return m;
+  if (m instanceof RegExp) return `/${m.source}/${m.flags}`;
+  return `/${m.$regex}/${m.flags}`;
+}
+function describePredicate(p) {
+  const parts = [];
+  if (p.name !== void 0) parts.push(`name=${describeMatcher(p.name)}`);
+  if (p.tool !== void 0) parts.push(`tool=${describeMatcher(p.tool)}`);
+  if (p.attr !== void 0) {
+    for (const [k, v] of Object.entries(p.attr)) {
+      parts.push(
+        `attr.${k}=${v instanceof RegExp || isSerializedRegex(v) ? describeMatcher(v) : JSON.stringify(v)}`
+      );
+    }
+  }
+  if (typeof p.custom === "function") parts.push(`custom=${p.custom.name || "fn"}`);
+  return parts.join(",");
+}
+function normalizeMatcher(m) {
+  return m instanceof RegExp ? { $regex: m.source, flags: m.flags } : m;
+}
+function normalizePredicate(p, where) {
+  assertPredicate(p, where);
+  const out = {};
+  if (p.name !== void 0) out.name = normalizeMatcher(p.name);
+  if (p.tool !== void 0) out.tool = normalizeMatcher(p.tool);
+  if (p.attr !== void 0) {
+    out.attr = Object.fromEntries(
+      Object.entries(p.attr).map(([k, v]) => [k, v instanceof RegExp ? normalizeMatcher(v) : v])
+    );
+  }
+  if (typeof p.custom === "function") {
+    out.custom = p.custom;
+    out.requiresCustom = true;
+  }
+  return out;
+}
+var TraceContractBuilder = class {
+  constructor(name) {
+    this.name = name;
+  }
+  name;
+  rules = [];
+  /** Every span in the trace must satisfy `p`. */
+  always(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").always`);
+    return this.add({ kind: "always", label: label ?? `always(${describePredicate(np)})`, p: np });
+  }
+  /** No span in the trace may satisfy `p`. */
+  never(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").never`);
+    return this.add({ kind: "never", label: label ?? `never(${describePredicate(np)})`, p: np });
+  }
+  /** At least one span in the trace must satisfy `p`. */
+  eventually(p, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").eventually`);
+    return this.add({
+      kind: "eventually",
+      label: label ?? `eventually(${describePredicate(np)})`,
+      p: np
+    });
+  }
+  /** Every `b`-match must have a strictly earlier `a`-match. */
+  precedes(a, b, label) {
+    const na = normalizePredicate(a, `traceContract("${this.name}").precedes (a)`);
+    const nb = normalizePredicate(b, `traceContract("${this.name}").precedes (b)`);
+    return this.add({
+      kind: "precedes",
+      label: label ?? `precedes(${describePredicate(na)} -> ${describePredicate(nb)})`,
+      a: na,
+      b: nb
+    });
+  }
+  /** Every `p`-match is a violation unless a strictly earlier `prior`-match exists. */
+  neverUnless(p, prior, label) {
+    const np = normalizePredicate(p, `traceContract("${this.name}").neverUnless (p)`);
+    const nprior = normalizePredicate(prior, `traceContract("${this.name}").neverUnless (prior)`);
+    return this.add({
+      kind: "neverUnless",
+      label: label ?? `neverUnless(${describePredicate(np)} unless ${describePredicate(nprior)})`,
+      p: np,
+      prior: nprior
+    });
+  }
+  build() {
+    if (this.rules.length === 0) {
+      throw new ValidationError(
+        `traceContract("${this.name}").build(): no rules \u2014 an empty contract would vacuously pass`
+      );
+    }
+    return { name: this.name, rules: [...this.rules] };
+  }
+  add(rule) {
+    let label = rule.label;
+    let n = 2;
+    while (this.rules.some((r) => r.label === label)) {
+      label = `${rule.label} #${n}`;
+      n += 1;
+    }
+    this.rules.push({ ...rule, label });
+    return this;
+  }
+};
+function traceContract(name) {
+  if (typeof name !== "string" || name.length === 0) {
+    throw new ValidationError("traceContract: name must be a non-empty string");
+  }
+  return new TraceContractBuilder(name);
+}
+function assertContract(contract) {
+  if (typeof contract?.name !== "string" || contract.name.length === 0) {
+    throw new ValidationError("evaluateTraceContract: contract.name must be a non-empty string");
+  }
+  if (!Array.isArray(contract.rules) || contract.rules.length === 0) {
+    throw new ValidationError(
+      `evaluateTraceContract: contract "${contract.name}" has no rules \u2014 an empty contract would vacuously pass`
+    );
+  }
+  const seen = /* @__PURE__ */ new Set();
+  for (const rule of contract.rules) {
+    const where = `contract "${contract.name}" rule "${rule?.label ?? "<unlabeled>"}"`;
+    if (typeof rule?.label !== "string" || rule.label.length === 0) {
+      throw new ValidationError(`${where}: label must be a non-empty string`);
+    }
+    if (seen.has(rule.label)) {
+      throw new ValidationError(`${where}: duplicate label would collapse per-rule scores`);
+    }
+    seen.add(rule.label);
+    switch (rule.kind) {
+      case "always":
+      case "never":
+      case "eventually":
+        assertPredicate(rule.p, `${where} (p)`);
+        break;
+      case "precedes":
+        assertPredicate(rule.a, `${where} (a)`);
+        assertPredicate(rule.b, `${where} (b)`);
+        break;
+      case "neverUnless":
+        assertPredicate(rule.p, `${where} (p)`);
+        assertPredicate(rule.prior, `${where} (prior)`);
+        break;
+      default:
+        throw new ValidationError(
+          `${where}: unknown rule kind "${String(rule.kind)}"`
+        );
+    }
+  }
+}
+function orderSpans(spans) {
+  const timed = spans.filter(
+    (s) => typeof s.startedAt === "number" && Number.isFinite(s.startedAt)
+  ).length;
+  if (timed === spans.length) {
+    return spans.map((span, i) => ({ span, i })).sort((x, y) => x.span.startedAt - y.span.startedAt || x.i - y.i).map((x) => x.span);
+  }
+  if (timed === 0) return [...spans];
+  throw new ValidationError(
+    `evaluateTraceContract: ${timed}/${spans.length} spans carry a finite startedAt \u2014 mixed timestamps make ordering ambiguous; stamp every span or none`
+  );
+}
+function spanRef(span, index) {
+  return span.spanId ?? `#${index}`;
+}
+function checkGuarded(args) {
+  const out = [];
+  let guardSeen = false;
+  for (let i = 0; i < args.ordered.length; i++) {
+    const span = args.ordered[i];
+    if (!guardSeen && matchSpan(span, args.guarded)) {
+      out.push({ rule: args.label, spanId: span.spanId, detail: args.detail(spanRef(span, i)) });
+    }
+    if (matchSpan(span, args.guard)) guardSeen = true;
+  }
+  return out;
+}
+function checkRule(rule, ordered) {
+  switch (rule.kind) {
+    case "always": {
+      const out = [];
+      for (let i = 0; i < ordered.length; i++) {
+        const span = ordered[i];
+        if (!matchSpan(span, rule.p)) {
+          out.push({
+            rule: rule.label,
+            spanId: span.spanId,
+            detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") fails always(${describePredicate(rule.p)})`
+          });
+        }
+      }
+      return out;
+    }
+    case "never": {
+      const out = [];
+      for (let i = 0; i < ordered.length; i++) {
+        const span = ordered[i];
+        if (matchSpan(span, rule.p)) {
+          out.push({
+            rule: rule.label,
+            spanId: span.spanId,
+            detail: `span ${spanRef(span, i)} ("${span.name ?? ""}") matches never(${describePredicate(rule.p)})`
+          });
+        }
+      }
+      return out;
+    }
+    case "eventually": {
+      const hit = ordered.some((span) => matchSpan(span, rule.p));
+      return hit ? [] : [
+        {
+          rule: rule.label,
+          detail: `no span matches eventually(${describePredicate(rule.p)}) over ${ordered.length} span(s)`
+        }
+      ];
+    }
+    case "precedes":
+      return checkGuarded({
+        label: rule.label,
+        ordered,
+        guard: rule.a,
+        guarded: rule.b,
+        detail: (ref) => `span ${ref} matches ${describePredicate(rule.b)} with no earlier ${describePredicate(rule.a)} match`
+      });
+    case "neverUnless":
+      return checkGuarded({
+        label: rule.label,
+        ordered,
+        guard: rule.prior,
+        guarded: rule.p,
+        detail: (ref) => `span ${ref} matches ${describePredicate(rule.p)} with no earlier ${describePredicate(rule.prior)} match`
+      });
+  }
 }
-function keyPreserved(key) {
+function evaluateTraceContract(contract, spans) {
+  assertContract(contract);
+  const ordered = orderSpans(spans);
+  const scores2 = {};
+  const violations = [];
+  for (const rule of contract.rules) {
+    const ruleViolations = checkRule(rule, ordered);
+    scores2[rule.label] = ruleViolations.length === 0 ? 1 : 0;
+    violations.push(...ruleViolations);
+  }
+  const ruleCount = contract.rules.length;
+  const passCount = Object.values(scores2).filter((s) => s === 1).length;
   return {
-    id: `preserved(${key})`,
-    description: `"${key}" unchanged from before to after`,
-    score: ({ before, after }) => before[key] !== void 0 && before[key] === after[key] ? 1 : 0
+    contract: contract.name,
+    valid: passCount === ruleCount,
+    score: passCount / ruleCount,
+    scores: scores2,
+    violations,
+    notes: `${passCount}/${ruleCount} rules passed`
   };
 }
-function collectionPreserved(key, minRatio = 1) {
-  return {
-    id: `collection-preserved(${key})`,
-    description: `"${key}" length \u2265 ${minRatio} \xD7 prior length`,
-    score: ({ before, after }) => {
-      const b = before[key];
-      const a = after[key];
-      if (!Array.isArray(b) || !Array.isArray(a)) return 0;
-      if (b.length === 0) return a.length === 0 ? 1 : 1;
-      return Math.min(1, a.length / (b.length * minRatio));
-    }
-  };
+function checkTraceContracts(spans, contracts) {
+  if (contracts.length === 0) {
+    throw new ValidationError(
+      "checkTraceContracts: empty contract list would vacuously pass \u2014 supply at least one contract"
+    );
+  }
+  const verdicts = contracts.map((c) => evaluateTraceContract(c, spans));
+  return { verdicts, allValid: verdicts.every((v) => v.valid) };
 }
-function statusAdvanced(key, progression) {
+function contractJudge(contracts, opts) {
+  if (contracts.length === 0) {
+    throw new ValidationError("contractJudge: at least one contract required");
+  }
+  for (const c of contracts) assertContract(c);
+  const names = /* @__PURE__ */ new Set();
+  for (const c of contracts) {
+    if (names.has(c.name)) {
+      throw new ValidationError(
+        `contractJudge: duplicate contract name "${c.name}" would collapse judge dimensions`
+      );
+    }
+    names.add(c.name);
+  }
+  if (typeof opts?.spans !== "function") {
+    throw new ValidationError("contractJudge: opts.spans extraction function is required");
+  }
+  const dimensions = contracts.map((c) => ({
+    key: c.name,
+    description: c.rules.map((r) => r.label).join("; ")
+  }));
   return {
-    id: `status-advanced(${key})`,
-    description: `"${key}" progressed along ${progression.join("\u2192")}`,
-    score: ({ before, after }) => {
-      const bi = progression.indexOf(String(before[key]));
-      const ai2 = progression.indexOf(String(after[key]));
-      if (bi === -1 || ai2 === -1) return 0;
-      return ai2 >= bi ? 1 : 0;
+    name: opts.name ?? "trace-contracts",
+    dimensions,
+    score({ artifact, scenario }) {
+      const spans = opts.spans({ artifact, scenario });
+      if (!Array.isArray(spans)) {
+        throw new ValidationError(
+          `contractJudge: spans() must return a span array, got ${typeof spans}`
+        );
+      }
+      const { verdicts } = checkTraceContracts(spans, contracts);
+      const dims = {};
+      const violations = [];
+      for (const v of verdicts) {
+        dims[v.contract] = v.score;
+        violations.push(...v.violations);
+      }
+      const composite = verdicts.reduce((acc, v) => acc + v.score, 0) / verdicts.length;
+      return {
+        dimensions: dims,
+        composite,
+        notes: violations.length === 0 ? "all trace contracts satisfied" : violations.slice(0, 8).map((x) => `${x.rule}: ${x.detail}`).join("\n")
+      };
     }
   };
 }
 
-// src/ui-finding.ts
-var UI_LENSES = [
-  "consistency",
-  "hierarchy",
-  "layout",
-  "ux-flow",
-  "duplication",
-  "accessibility",
-  "responsive",
-  "states",
-  "content",
-  "interaction",
-  "performance-perceived",
-  "other"
-];
-var UI_FINDING_SEVERITIES = [
-  "critical",
-  "high",
-  "med",
-  "low"
-];
-
 // src/behavior-dsl.ts
 var BehaviorAssertion = class {
   constructor(store, runId) {
@@ -5109,6 +5750,25 @@ var BehaviorAssertion = class {
       }
     };
   }
+  /** Evaluate a finite-trace temporal contract (`traceContract(...)`) over
+   *  this run's span sequence. See `trace-contracts.ts` for the operators. */
+  toSatisfyContract(contract) {
+    return {
+      label: `agent(${this.runId}).toSatisfyContract(${contract.name})`,
+      check: async () => {
+        const spans = await this.store.spans({ runId: this.runId });
+        const verdict = evaluateTraceContract(contract, spans);
+        if (verdict.valid) {
+          return { ok: true, detail: `contract "${contract.name}": ${verdict.notes}` };
+        }
+        return {
+          ok: false,
+          detail: verdict.violations.map((v) => `${v.rule}: ${v.detail}`).join("; "),
+          evidence: verdict.violations.find((v) => v.spanId)?.spanId
+        };
+      }
+    };
+  }
   toNeverCall(toolName) {
     return {
       label: `agent(${this.runId}).toNeverCall(${toolName})`,
@@ -5711,90 +6371,6 @@ async function promptBisect(options) {
   };
 }
 
-// src/counterfactual.ts
-async function runCounterfactual(store, originalRunId, mutation, runner) {
-  const originalRun = await store.getRun(originalRunId);
-  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
-  const trajectory = await buildTrajectory(store, originalRunId);
-  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {
-    throw new ValidationError(
-      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`
-    );
-  }
-  const targetStep = trajectory.steps[mutation.at];
-  const mutatedStep = applyMutation(targetStep, mutation);
-  const cfEmitter = new TraceEmitter(store);
-  await cfEmitter.startRun({
-    scenarioId: originalRun.scenarioId,
-    variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
-    projectId: originalRun.projectId,
-    parentRunId: originalRunId,
-    layer: "meta",
-    tags: { counterfactual: "true", mutationKind: mutation.kind, mutationAt: String(mutation.at) }
-  });
-  await runner.executeFrom(
-    {
-      originalRunId,
-      originalTrajectory: trajectory,
-      prefix: trajectory.steps.slice(0, mutation.at),
-      mutation,
-      mutatedStep
-    },
-    cfEmitter
-  );
-  const counterfactual = await store.getRun(cfEmitter.runId);
-  const delta = {
-    originalOutcomeScore: originalRun.outcome?.score ?? null,
-    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
-    deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
-  };
-  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta };
-}
-function applyMutation(step, mutation) {
-  if (mutation.kind === "swap-model" && step.span.kind === "llm") {
-    const llm = step.span;
-    return { ...step, span: { ...llm, model: mutation.newModel } };
-  }
-  if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
-    const tool = step.span;
-    return { ...step, span: { ...tool, result: mutation.newResult } };
-  }
-  if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
-    const llm = step.span;
-    return {
-      ...step,
-      span: {
-        ...llm,
-        messages: [{ role: "system", content: mutation.content }, ...llm.messages]
-      }
-    };
-  }
-  if (mutation.kind === "custom") return mutation.apply(step);
-  return step;
-}
-function attributeCounterfactuals(results) {
-  const grouped = /* @__PURE__ */ new Map();
-  for (const r of results) {
-    const arr = grouped.get(r.mutation.kind) ?? [];
-    arr.push(r);
-    grouped.set(r.mutation.kind, arr);
-  }
-  const out = [];
-  for (const [kind, items] of grouped) {
-    const deltas = items.map((i) => i.delta.deltaScore).filter((d) => typeof d === "number");
-    if (deltas.length === 0) continue;
-    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length;
-    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length;
-    out.push({
-      mutationKind: kind,
-      n: deltas.length,
-      meanAbsDelta: meanAbs,
-      meanSignedDelta: meanSigned
-    });
-  }
-  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta);
-}
-
 // src/cross-trace-diff.ts
 async function crossTraceDiff(store, runA, runB, options = {}) {
   const [a, b] = await Promise.all([buildTrajectory(store, runA), buildTrajectory(store, runB)]);
@@ -6986,14 +7562,18 @@ function aggregate(layers, results, startedAt, startedAtMs) {
     }
   }
   const finishedAtMs = Date.now();
+  const allPass = ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0;
+  const blendedScore = scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0;
   return {
     layers: results,
     passCount,
     failCount,
     skippedCount,
     errorCount,
-    allPass: ranAnyScoredLayer && !anyScoredLayerFailed && failCount === 0 && errorCount === 0,
-    blendedScore: scoredWeightSum > 0 ? scoredWeightedTotal / scoredWeightSum : 0,
+    allPass,
+    blendedScore,
+    valid: allPass,
+    score: blendedScore,
     durationMs: finishedAtMs - startedAtMs,
     startedAt,
     finishedAt: new Date(finishedAtMs).toISOString()
@@ -8208,69 +8788,6 @@ function fmt(x) {
   return x.toFixed(4);
 }
 
-// src/judge-retry.ts
-var DEFAULT_MAX_ATTEMPTS = 3;
-var DEFAULT_TIMEOUT_MS = 3e5;
-function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-async function withJudgeRetry(judgeFn, policy = {}) {
-  const maxAttempts = policy.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
-  const timeoutMs = policy.timeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const backoff = policy.backoffMs ?? backoffMs;
-  const isRetryable = policy.isRetryable ?? isTransientLlmError;
-  const models = policy.models && policy.models.length > 0 ? policy.models : [void 0];
-  let totalAttempts = 0;
-  const attemptErrors = [];
-  let lastError;
-  for (const model of models) {
-    for (let attempt = 0; attempt < maxAttempts; attempt++) {
-      totalAttempts += 1;
-      const controller = new AbortController();
-      const timer = setTimeout(() => controller.abort(new Error("TimeoutError")), timeoutMs);
-      try {
-        const value = await judgeFn(model, controller.signal);
-        clearTimeout(timer);
-        return {
-          value,
-          succeeded: true,
-          attempts: totalAttempts,
-          modelUsed: model,
-          attemptErrors
-        };
-      } catch (err) {
-        clearTimeout(timer);
-        const errObj = err instanceof Error ? err : new Error(String(err));
-        lastError = errObj;
-        attemptErrors.push({
-          attempt: totalAttempts,
-          model: model ?? "(default)",
-          error: errObj.message
-        });
-        if (!isRetryable(errObj)) {
-          return {
-            value: null,
-            succeeded: false,
-            attempts: totalAttempts,
-            error: errObj,
-            attemptErrors
-          };
-        }
-        if (attempt < maxAttempts - 1) {
-          await sleep(backoff(attempt));
-        }
-      }
-    }
-  }
-  return {
-    value: null,
-    succeeded: false,
-    attempts: totalAttempts,
-    error: lastError,
-    attemptErrors
-  };
-}
-
 // src/orthogonality.ts
 function passOrthogonality(input) {
   const passes = input.passes;
@@ -8649,6 +9166,7 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
     });
     try {
       const allScores = [];
+      let failedJudges = 0;
       for (let i = 0; i < judges.length; i++) {
         const judge = judges[i];
         const name = judgeNames[i] ?? `judge_${i}`;
@@ -8656,8 +9174,13 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           emitter: opts.emitter,
           parentSpanId: ensembleSpan.span.spanId
         });
-        const scores2 = await tracedFn(tc, input);
-        allScores.push(...scores2);
+        try {
+          const scores2 = await tracedFn(tc, input);
+          allScores.push(...scores2);
+        } catch (err) {
+          if (!(err instanceof JudgeParseError)) throw err;
+          failedJudges++;
+        }
       }
       const composite = allScores.length > 0 ? allScores.reduce((sum3, s) => sum3 + s.score, 0) / allScores.length : 0;
       await ensembleSpan.end({
@@ -8665,6 +9188,7 @@ function traceJudgeEnsemble(judges, judgeNames, opts) {
           "judge.ensemble_size": judges.length,
           "judge.composite_score": composite,
           "judge.total_dimensions": allScores.length,
+          "judge.failed_judges": failedJudges,
           "eval.phase": "judge"
         }
       });
@@ -9094,8 +9618,256 @@ function applyDomainPatch(p, sectionId, newBody) {
 function sectionHash(section) {
   return surfaceContentHash(JSON.stringify({ title: section.title, body: section.body }));
 }
+
+// src/cost-report.ts
+function costReport(ledger) {
+  const summary = ledger.summary();
+  const perModel = /* @__PURE__ */ new Map();
+  for (const entry of ledger.list()) {
+    const roll = perModel.get(entry.model) ?? {
+      model: entry.model,
+      usd: 0,
+      entries: 0,
+      unpriced: false
+    };
+    roll.usd += entry.costUsd;
+    roll.entries += 1;
+    if (entry.costUnknown) roll.unpriced = true;
+    perModel.set(entry.model, roll);
+  }
+  return {
+    perChannel: summary.byChannel,
+    total: {
+      usd: summary.totalCostUsd,
+      unknownEntries: summary.byChannel.reduce((sum3, c) => sum3 + c.unpricedCalls, 0)
+    },
+    perModel: [...perModel.values()].sort((a, b) => a.model.localeCompare(b.model))
+  };
+}
+function attachCostToReport(report, ledger) {
+  if ("cost" in report) {
+    throw new ValidationError(
+      "attachCostToReport: report already has a 'cost' key \u2014 refusing to overwrite an existing stamp"
+    );
+  }
+  return { ...report, cost: costReport(ledger) };
+}
+
+// src/model-seats.ts
+var seatPresets = {
+  economy: {
+    worker: "kimi-k2.6",
+    judges: ["kimi-k2.6", "deepseek-v4-pro", "gpt-4.1-mini"],
+    analyst: "gpt-4.1-mini",
+    reflection: "gpt-4.1-mini",
+    verifier: "deepseek-v4-pro"
+  },
+  frontier: {}
+};
+var SeatUnsetError = class extends ConfigError {
+  constructor(seat) {
+    super(
+      `ModelSeats: seat '${seat}' is unset and no fallback was given \u2014 name a model explicitly (a model id is a budget decision, never a silent default)`
+    );
+    this.seat = seat;
+  }
+  seat;
+};
+function resolveSeat(seats, seat, fallback) {
+  const value = seats[seat];
+  if (seat === "judges") {
+    if (value !== void 0 && !Array.isArray(value)) {
+      throw new ValidationError(`ModelSeats: seat 'judges' must be a string[], got ${typeof value}`);
+    }
+    const models = Array.isArray(value) ? value : [];
+    if (models.length > 0) {
+      const blank = models.findIndex((m) => typeof m !== "string" || m.trim() === "");
+      if (blank >= 0) {
+        throw new ValidationError(
+          `ModelSeats: judges[${blank}] is blank \u2014 every panel model must be a non-empty id`
+        );
+      }
+      return [...models];
+    }
+  } else {
+    if (value !== void 0 && typeof value !== "string") {
+      throw new ValidationError(`ModelSeats: seat '${seat}' must be a string, got ${typeof value}`);
+    }
+    if (typeof value === "string" && value.trim() !== "") return value;
+  }
+  if (fallback !== void 0) {
+    if (fallback.trim() === "") {
+      throw new ValidationError(`ModelSeats: fallback for seat '${seat}' is blank`);
+    }
+    return seat === "judges" ? [fallback] : fallback;
+  }
+  throw new SeatUnsetError(seat);
+}
+
+// src/verdict-cache.ts
+import { createHash } from "crypto";
+import { appendFileSync as appendFileSync3, existsSync as existsSync5, readFileSync as readFileSync6 } from "fs";
+function canonicalizeAt(value, path) {
+  if (value === null) return "null";
+  switch (typeof value) {
+    case "boolean":
+      return value ? "true" : "false";
+    case "number":
+      if (!Number.isFinite(value)) {
+        throw new Error(
+          `canonicalJson: non-finite number (${value}) at ${path} \u2014 ambiguity is an error, not a coercion`
+        );
+      }
+      return JSON.stringify(value);
+    case "string":
+      return JSON.stringify(value);
+    case "undefined":
+    case "function":
+    case "symbol":
+      throw new Error(
+        `canonicalJson: ${typeof value} at ${path} \u2014 ambiguity is an error, not a coercion`
+      );
+    case "bigint":
+      throw new Error(`canonicalJson: bigint at ${path} \u2014 not representable in JSON`);
+    case "object":
+      break;
+  }
+  const obj = value;
+  if (typeof obj["toJSON"] === "function") {
+    return canonicalizeAt(obj.toJSON(), path);
+  }
+  if (Array.isArray(obj)) {
+    return `[${obj.map((item, i) => canonicalizeAt(item, `${path}[${i}]`)).join(",")}]`;
+  }
+  if (obj instanceof Map || obj instanceof Set) {
+    throw new Error(
+      `canonicalJson: ${obj instanceof Map ? "Map" : "Set"} at ${path} \u2014 would serialize as '{}'; convert to a plain object/array first`
+    );
+  }
+  const keys = Object.keys(obj).sort();
+  const parts = keys.map((k) => `${JSON.stringify(k)}:${canonicalizeAt(obj[k], `${path}.${k}`)}`);
+  return `{${parts.join(",")}}`;
+}
+function canonicalJson(value) {
+  return canonicalizeAt(value, "$");
+}
+function contentHash(value) {
+  return createHash("sha256").update(canonicalJson(value)).digest("hex");
+}
+function inMemoryVerdictCache() {
+  const entries = /* @__PURE__ */ new Map();
+  return {
+    get: (key) => entries.get(key),
+    set: (key, score) => {
+      entries.set(key, score);
+    }
+  };
+}
+function parseCacheLine(line, path, lineNo) {
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch (err) {
+    throw new Error(
+      `fileVerdictCache: corrupt JSONL at ${path}:${lineNo} \u2014 ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+  const rec = parsed;
+  if (typeof rec !== "object" || rec === null || typeof rec.key !== "string" || typeof rec.score !== "object" || rec.score === null || typeof rec.score.composite !== "number" || typeof rec.score.dimensions !== "object") {
+    throw new Error(
+      `fileVerdictCache: invalid record shape at ${path}:${lineNo} \u2014 expected {key, score:{dimensions, composite, notes}}`
+    );
+  }
+  return rec;
+}
+function fileVerdictCache(path) {
+  const entries = /* @__PURE__ */ new Map();
+  if (existsSync5(path)) {
+    const lines = readFileSync6(path, "utf8").split("\n");
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i];
+      if (line === void 0 || line.trim() === "") continue;
+      const rec = parseCacheLine(line, path, i + 1);
+      entries.set(rec.key, rec.score);
+    }
+  }
+  return {
+    get: (key) => entries.get(key),
+    set: (key, score) => {
+      appendFileSync3(path, `${JSON.stringify({ key, score })}
+`, "utf8");
+      entries.set(key, score);
+    }
+  };
+}
+function cachedJudge(judge, store, options) {
+  if (typeof options.judgeVersion !== "string" || options.judgeVersion.trim() === "") {
+    throw new Error("cachedJudge: judgeVersion is required and must be a non-empty string");
+  }
+  const stats = { hits: 0, misses: 0 };
+  const wrapped = {
+    name: judge.name,
+    dimensions: judge.dimensions,
+    async score(input) {
+      const key = contentHash({
+        artifact: canonicalJson(input.artifact),
+        scenarioId: input.scenario.id,
+        judgeName: judge.name,
+        dimensions: judge.dimensions,
+        judgeVersion: options.judgeVersion
+      });
+      const cached = await store.get(key);
+      if (cached !== void 0) {
+        stats.hits += 1;
+        return cached;
+      }
+      const score = await judge.score(input);
+      await store.set(key, score);
+      stats.misses += 1;
+      return score;
+    },
+    stats: () => ({ ...stats })
+  };
+  if (judge.appliesTo) wrapped.appliesTo = judge.appliesTo;
+  return wrapped;
+}
+
+// src/attestation.ts
+var ATTESTATION_ALGORITHM = "sha256/canonical-json";
+function attest(report, provenance) {
+  return {
+    reportHash: contentHash(report),
+    provenance,
+    algorithm: ATTESTATION_ALGORITHM
+  };
+}
+function verifyAttestation(report, attested) {
+  if (attested.algorithm !== ATTESTATION_ALGORITHM) {
+    return {
+      valid: false,
+      reason: `unknown algorithm '${attested.algorithm}' \u2014 this verifier only checks '${ATTESTATION_ALGORITHM}'`
+    };
+  }
+  let recomputed;
+  try {
+    recomputed = contentHash(report);
+  } catch (err) {
+    return {
+      valid: false,
+      reason: `report is not canonicalizable: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  if (recomputed !== attested.reportHash) {
+    return {
+      valid: false,
+      reason: `report hash mismatch: attested ${attested.reportHash}, recomputed ${recomputed}`
+    };
+  }
+  return { valid: true };
+}
 export {
   AGENT_PROFILE_KINDS,
+  ATTESTATION_ALGORITHM,
   AgentDriver,
   AgentEvalError,
   AgentProfileCellValidationError,
@@ -9150,6 +9922,7 @@ export {
   InMemoryTraceStore,
   InMemoryWorkspaceInspector,
   JudgeError,
+  JudgeParseError,
   JudgeRunner,
   KNOWLEDGE_GAP_KIND_SPEC,
   KNOWLEDGE_POISONING_KIND_SPEC,
@@ -9182,6 +9955,7 @@ export {
   SKILL_USAGE_ANALYST,
   SandboxHarness,
   ScenarioRegistry,
+  SeatUnsetError,
   SingleBackendError,
   SkillUsageAnalyst,
   SpanNotFoundError,
@@ -9192,6 +9966,7 @@ export {
   TRACE_ANALYST_TRUNCATION_MARKER_PREFIX,
   TRACE_SCHEMA_VERSION,
   TokenCounter,
+  TraceContractBuilder,
   TraceEmitter,
   TraceFileMissingError,
   TraceNotFoundError,
@@ -9227,6 +10002,8 @@ export {
   assertSingleBackend,
   assignFeedbackSplit,
   assignHeldOutTag,
+  attachCostToReport,
+  attest,
   attributeCounterfactuals,
   backoffMs,
   deterministicSplit as benchmarkDeterministicSplit,
@@ -9248,17 +10025,20 @@ export {
   buildTraceInsightPrompt,
   buildTrajectory,
   byteLengthRange,
+  cachedJudge,
   calibrateJudge,
   calibrateJudgeContinuous,
   callLlm,
   callLlmJson,
   canaryLeakView,
+  canonicalJson,
   canonicalize,
   captureFetchToRawSink,
   causalAttribution,
   checkBehavioralCanary,
   checkCanaries,
   checkSlos,
+  checkTraceContracts,
   clamp01,
   classifyEuAiRisk,
   classifyFailure,
@@ -9272,6 +10052,7 @@ export {
   compareReferenceReplay,
   compareToBaseline,
   compilerJudge,
+  completionVerdict,
   composeParsers,
   composeValidators,
   computeExperimentStats,
@@ -9280,7 +10061,9 @@ export {
   computeTraceMetrics,
   confidenceInterval,
   containsAll,
+  contentHash,
   continuousAgreement,
+  contractJudge,
   controlFailureClassFromVerification,
   controlRunToFeedbackTrajectory,
   controlRunToRunRecord,
@@ -9288,6 +10071,7 @@ export {
   corpusInterRaterAgreement,
   corpusInterRaterAgreementFromJudgeScores,
   costForUsage,
+  costReport,
   createAnalystAi,
   createAntiSlopJudge,
   createChatClient,
@@ -9326,6 +10110,8 @@ export {
   distillPlaybook,
   domainEvidencePattern,
   dominates,
+  eProcess,
+  ensembleJudge,
   estimateCost,
   estimateTokens,
   euAiActReport,
@@ -9335,6 +10121,7 @@ export {
   evaluateInterimReleaseConfidence,
   evaluateOracles,
   evaluateReleaseConfidence,
+  evaluateTraceContract,
   executeScenario,
   expectAgent,
   exportRewardModel,
@@ -9354,6 +10141,7 @@ export {
   fileContains,
   fileExists,
   fileExperimentStore,
+  fileVerdictCache,
   findAutoMatchNoExpectation,
   findConstructorCwdDropped,
   findFallbackToPass,
@@ -9386,6 +10174,7 @@ export {
   inMemoryReferenceReplayStore,
   inMemoryReviewStore,
   inMemoryRunRecordBackend,
+  inMemoryVerdictCache,
   inferDomainKeywords,
   inferOtlpKind,
   interRaterReliability,
@@ -9420,13 +10209,16 @@ export {
   loadScorerFromGrader,
   localCommandRunner,
   lowercaseMutator,
+  makeEvalTools,
   makeFinding,
   mannWhitneyU,
   matchGoldens,
+  matchSpan,
   mergeLayerResults,
   mergeSteeringBundle,
   modelDescriptionBits,
   modelPriceKey,
+  mulberry32,
   multiToolchainLayer,
   nistAiRmfReport,
   normalizeScores,
@@ -9495,6 +10287,7 @@ export {
   researchReport,
   resetLockedAppendersForTesting,
   resolveModelPricing,
+  resolveSeat,
   roundTripRunRecord,
   rowCount,
   rowWhere,
@@ -9532,6 +10325,7 @@ export {
   scoreRedTeamOutput,
   scoreReferenceReplay,
   scoreTraceInsightReadiness,
+  seatPresets,
   securityJudge,
   selectHarnessVariant,
   selfPreference,
@@ -9557,12 +10351,14 @@ export {
   throwIfRunIncomplete,
   toAgentProfileJson,
   toLangfuseEnvelope,
+  toOpenAiTool,
   toPrometheusText,
   tokenizeDomainWords,
   toolNamesForRun,
   toolSpans,
   traceAnalystFunctionGroup,
   traceAnalystOnRunComplete,
+  traceContract,
   traceJudge,
   traceJudgeEnsemble,
   tracedAnalyzeTraces,
@@ -9573,6 +10369,7 @@ export {
   validateRunRecord,
   verbosityBias,
   verifyAgentProfileCell,
+  verifyAttestation,
   verifyCompletion,
   verifyManifest,
   visualDiff,