@tangle-network/agent-eval 0.1.0

package/dist/index.js

@@ -0,0 +1,1293 @@
+// src/client.ts
+var ProductClient = class {
+  baseUrl;
+  routes;
+  cookies = "";
+  constructor(config) {
+    this.baseUrl = config.baseUrl.replace(/\/+$/, "");
+    this.routes = config.routes;
+  }
+  route(name) {
+    const path = this.routes[name];
+    if (!path) throw new Error(`Route "${name}" not configured`);
+    return path;
+  }
+  async signup(name, email, password) {
+    const res = await this.post(this.route("signup"), { name, email, password });
+    const user = res.user;
+    if (!user?.id) throw new Error(`Signup failed: ${JSON.stringify(res)}`);
+    return { userId: user.id };
+  }
+  async login(email, password) {
+    const res = await fetch(`${this.baseUrl}${this.route("login")}`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "Origin": this.baseUrl },
+      body: JSON.stringify({ email, password }),
+      redirect: "manual"
+    });
+    const setCookie = res.headers.get("set-cookie");
+    if (setCookie) {
+      this.cookies = setCookie.split(";")[0];
+    }
+    const body = await res.json();
+    if (!body.user) throw new Error(`Login failed: ${JSON.stringify(body)}`);
+  }
+  async createWorkspace(name, type = "project") {
+    const res = await this.post(this.route("workspaces"), { name, type });
+    const ws = res.workspace;
+    if (!ws?.id) throw new Error(`Workspace creation failed: ${JSON.stringify(res)}`);
+    return ws.id;
+  }
+  async createThread(workspaceId) {
+    const res = await this.post(this.route("threads"), { workspaceId });
+    const thread = res.thread;
+    if (!thread?.id) throw new Error(`Thread creation failed: ${JSON.stringify(res)}`);
+    return thread.id;
+  }
+  async chat(workspaceId, threadId, content, _opts) {
+    const res = await fetch(`${this.baseUrl}${this.route("chat")}`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Origin": this.baseUrl,
+        "Cookie": this.cookies
+      },
+      body: JSON.stringify({ workspaceId, threadId, content })
+    });
+    if (!res.ok || !res.body) throw new Error(`Chat failed: ${res.status}`);
+    const reader = res.body.getReader();
+    const decoder = new TextDecoder();
+    let buf = "";
+    let text = "";
+    const blocks = [];
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buf += decoder.decode(value, { stream: true });
+      const lines = buf.split("\n");
+      buf = lines.pop() ?? "";
+      for (const line of lines) {
+        if (!line.trim()) continue;
+        try {
+          const event = JSON.parse(line);
+          if (event.type === "message.part.updated" && event.data?.delta) {
+            text += event.data.delta;
+          }
+        } catch {
+        }
+      }
+    }
+    const blockRe = /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
+    let match;
+    while ((match = blockRe.exec(text)) !== null) {
+      const fields = {};
+      for (const line of match[2].split("\n")) {
+        const idx = line.indexOf(":");
+        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+      }
+      blocks.push({ type: match[1], title: fields.title ?? "" });
+    }
+    return { text, blocks };
+  }
+  async getTasks(workspaceId) {
+    const res = await this.get(`${this.route("tasks")}?workspaceId=${workspaceId}`);
+    return res.tasks ?? [];
+  }
+  async getEvents(workspaceId) {
+    const res = await this.get(`${this.route("events")}?workspaceId=${workspaceId}`);
+    return res.events ?? [];
+  }
+  async getApprovals(workspaceId) {
+    const res = await this.get(`${this.route("approvals")}?workspaceId=${workspaceId}`);
+    return res.actions ?? [];
+  }
+  async getVaultTree(workspaceId) {
+    const res = await this.get(`${this.route("vault")}?workspaceId=${workspaceId}`);
+    const paths = [];
+    function extract(nodes) {
+      for (const n of nodes) {
+        const node = n;
+        if (node.type === "file" && node.path) paths.push(node.path);
+        if (node.children) extract(node.children);
+      }
+    }
+    extract(res.tree ?? []);
+    return paths;
+  }
+  async approveAction(workspaceId, id) {
+    await this.patch(this.route("approvals"), { workspaceId, id, status: "approved" });
+  }
+  async rejectAction(workspaceId, id, reason) {
+    await this.patch(this.route("approvals"), { workspaceId, id, status: "rejected", reason });
+  }
+  async getGenerations(workspaceId) {
+    const res = await this.get(`${this.route("generations")}?workspaceId=${workspaceId}`);
+    return res.generations ?? [];
+  }
+  /** Generic GET for custom routes */
+  async get(path) {
+    const res = await fetch(`${this.baseUrl}${path}`, {
+      headers: { "Cookie": this.cookies }
+    });
+    return res.json();
+  }
+  /** Generic POST for custom routes */
+  async post(path, body) {
+    const res = await fetch(`${this.baseUrl}${path}`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Origin": this.baseUrl,
+        "Cookie": this.cookies
+      },
+      body: JSON.stringify(body)
+    });
+    return res.json();
+  }
+  /** Generic PATCH for custom routes */
+  async patch(path, body) {
+    const res = await fetch(`${this.baseUrl}${path}`, {
+      method: "PATCH",
+      headers: {
+        "Content-Type": "application/json",
+        "Origin": this.baseUrl,
+        "Cookie": this.cookies
+      },
+      body: JSON.stringify(body)
+    });
+    return res.json();
+  }
+};
+async function runE2EWorkflow(client, name, workflow) {
+  const start = Date.now();
+  const checks = [];
+  try {
+    const results = await workflow(client);
+    checks.push(...results);
+  } catch (err) {
+    checks.push({
+      name: "fatal_error",
+      passed: false,
+      expected: "no crash",
+      actual: err instanceof Error ? err.message : String(err)
+    });
+  }
+  return {
+    name,
+    passed: checks.every((c) => c.passed),
+    duration: Date.now() - start,
+    detail: `${checks.filter((c) => c.passed).length}/${checks.length} checks passed`,
+    checks
+  };
+}
+
+// src/judges.ts
+function createDomainExpertJudge(domain) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
+    const resp = await tc.chat({
+      model: "gpt-4o",
+      messages: [{
+        role: "system",
+        content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
+
+Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
+
+Evaluate:
+1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
+2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
+
+Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
+      }, {
+        role: "user",
+        content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
+      }],
+      temperature: 0.1,
+      maxTokens: 800
+    });
+    return parseJudgeResponse("domain_expert", resp);
+  };
+}
+var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
+  const codeBlocks = artifacts.codeBlocks;
+  if (codeBlocks.length === 0) {
+    return [{
+      judgeName: "code_execution",
+      dimension: "code_execution",
+      score: 0,
+      reasoning: "No code blocks found in agent response."
+    }];
+  }
+  const codeText = codeBlocks.map(
+    (b, i) => `Block ${i + 1} (${b.language}):
+\`\`\`${b.language}
+${b.code.slice(0, 3e3)}
+\`\`\``
+  ).join("\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [{
+      role: "system",
+      content: `You are a principal software engineer reviewing code written by an AI agent.
+
+Score STRICTLY:
+1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
+2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
+3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
+
+Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
+    }, {
+      role: "user",
+      content: `Task: ${scenario.thesis}
+
+${codeText}`
+    }],
+    temperature: 0.1,
+    maxTokens: 1e3
+  });
+  return parseJudgeResponse("code_execution", resp);
+};
+var coherenceJudge = async (tc, { scenario, turns }) => {
+  if (turns.length < 2) {
+    return [{
+      judgeName: "coherence",
+      dimension: "coherence",
+      score: 5,
+      reasoning: "Single-turn scenario \u2014 coherence not fully testable."
+    }];
+  }
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [{
+      role: "system",
+      content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
+
+Score STRICTLY:
+1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
+2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
+3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
+
+Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
+    }, {
+      role: "user",
+      content: `Scenario: ${scenario.thesis}
+
+${conversation}`
+    }],
+    temperature: 0.1,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("coherence", resp);
+};
+var adversarialJudge = async (tc, { scenario, turns }) => {
+  const conversation = turns.map(
+    (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 1500)}`
+  ).join("\n\n---\n\n");
+  const resp = await tc.chat({
+    model: "gpt-4o",
+    messages: [{
+      role: "system",
+      content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
+
+1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
+2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
+3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
+
+Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
+
+Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
+    }, {
+      role: "user",
+      content: `Persona: ${scenario.persona}
+Scenario: ${scenario.thesis}
+
+${conversation}`
+    }],
+    temperature: 0.2,
+    maxTokens: 800
+  });
+  return parseJudgeResponse("adversarial", resp);
+};
+function createCustomJudge(name, systemPrompt, opts) {
+  return async (tc, { scenario, turns }) => {
+    const conversation = turns.map(
+      (t, i) => `Turn ${i + 1}:
+User: ${t.userMessage}
+Agent: ${t.agentResponse.slice(0, 2e3)}`
+    ).join("\n\n---\n\n");
+    const resp = await tc.chat({
+      model: opts?.model ?? "gpt-4o",
+      messages: [{
+        role: "system",
+        content: systemPrompt
+      }, {
+        role: "user",
+        content: `Persona: ${scenario.persona} (${scenario.label})
+Scenario: ${scenario.thesis}
+
+${conversation}`
+      }],
+      temperature: opts?.temperature ?? 0.1,
+      maxTokens: opts?.maxTokens ?? 1e3
+    });
+    return parseJudgeResponse(name, resp);
+  };
+}
+function defaultJudges(domain) {
+  return [
+    createDomainExpertJudge(domain),
+    codeExecutionJudge,
+    coherenceJudge,
+    adversarialJudge
+  ];
+}
+function parseJudgeResponse(judgeName, resp) {
+  try {
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
+    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
+    if (arrayMatch) cleaned = arrayMatch[0];
+    const parsed = JSON.parse(cleaned);
+    return parsed.map((p) => ({
+      judgeName,
+      dimension: p.dimension,
+      score: Math.max(0, Math.min(10, p.score)),
+      reasoning: p.reasoning ?? "",
+      evidence: p.evidence
+    }));
+  } catch (err) {
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    console.log(`    [parse_error] ${judgeName}: ${err.message?.slice(0, 50)} | response: ${content.slice(0, 100)}`);
+    return [{
+      judgeName,
+      dimension: "parse_error",
+      score: 0,
+      reasoning: `Parse failed: ${err.message?.slice(0, 100)}. Raw: ${content.slice(0, 200)}`
+    }];
+  }
+}
+
+// src/statistics.ts
+var INVERTED_DIMENSIONS = /* @__PURE__ */ new Set([
+  "hallucination",
+  "false_confidence",
+  "worst_failure"
+]);
+function normalizeScores(scores) {
+  return scores.map((s) => {
+    if (INVERTED_DIMENSIONS.has(s.dimension)) {
+      return s;
+    }
+    return s;
+  });
+}
+function weightedMean(scores) {
+  if (scores.length === 0) return 0;
+  let totalWeight = 0;
+  let weightedSum = 0;
+  for (const { score, weight } of scores) {
+    const w = weight ?? 1;
+    weightedSum += score * w;
+    totalWeight += w;
+  }
+  return totalWeight > 0 ? weightedSum / totalWeight : 0;
+}
+function confidenceInterval(scores, confidence = 0.95) {
+  if (scores.length === 0) return { mean: 0, lower: 0, upper: 0 };
+  if (scores.length === 1) return { mean: scores[0], lower: scores[0], upper: scores[0] };
+  const n = scores.length;
+  const mean = scores.reduce((a, b) => a + b, 0) / n;
+  const B = 1e3;
+  const bootstrapMeans = [];
+  for (let i = 0; i < B; i++) {
+    let sum = 0;
+    for (let j = 0; j < n; j++) {
+      sum += scores[Math.floor(Math.random() * n)];
+    }
+    bootstrapMeans.push(sum / n);
+  }
+  bootstrapMeans.sort((a, b) => a - b);
+  const alpha = 1 - confidence;
+  const lowerIdx = Math.floor(alpha / 2 * B);
+  const upperIdx = Math.floor((1 - alpha / 2) * B) - 1;
+  return {
+    mean,
+    lower: bootstrapMeans[lowerIdx],
+    upper: bootstrapMeans[Math.min(upperIdx, B - 1)]
+  };
+}
+function interRaterReliability(judgeScores) {
+  if (judgeScores.length < 2) return 1;
+  const dimensionMap = /* @__PURE__ */ new Map();
+  for (const judgeSet of judgeScores) {
+    for (const s of judgeSet) {
+      if (!dimensionMap.has(s.dimension)) dimensionMap.set(s.dimension, []);
+      const arr = dimensionMap.get(s.dimension);
+      if (arr.length === 0 || arr[arr.length - 1].length >= judgeScores.length) {
+        arr.push([s.score]);
+      } else {
+        arr[arr.length - 1].push(s.score);
+      }
+    }
+  }
+  const allValues = [];
+  const pairDiffs = [];
+  for (const items of dimensionMap.values()) {
+    for (const ratings of items) {
+      if (ratings.length < 2) continue;
+      for (const v of ratings) allValues.push(v);
+      for (let i = 0; i < ratings.length; i++) {
+        for (let j = i + 1; j < ratings.length; j++) {
+          pairDiffs.push((ratings[i] - ratings[j]) ** 2);
+        }
+      }
+    }
+  }
+  if (pairDiffs.length === 0 || allValues.length < 2) return 1;
+  const observedDisagreement = pairDiffs.reduce((a, b) => a + b, 0) / pairDiffs.length;
+  let expectedDisagreement = 0;
+  let expectedCount = 0;
+  for (let i = 0; i < allValues.length; i++) {
+    for (let j = i + 1; j < allValues.length; j++) {
+      expectedDisagreement += (allValues[i] - allValues[j]) ** 2;
+      expectedCount++;
+    }
+  }
+  expectedDisagreement = expectedCount > 0 ? expectedDisagreement / expectedCount : 0;
+  if (expectedDisagreement === 0) return 1;
+  return 1 - observedDisagreement / expectedDisagreement;
+}
+function mannWhitneyU(a, b) {
+  if (a.length === 0 || b.length === 0) return { u: 0, p: 1 };
+  const n1 = a.length;
+  const n2 = b.length;
+  const combined = [
+    ...a.map((v) => ({ v, group: "a" })),
+    ...b.map((v) => ({ v, group: "b" }))
+  ].sort((x, y) => x.v - y.v);
+  const ranks = new Array(combined.length);
+  let i = 0;
+  while (i < combined.length) {
+    let j = i;
+    while (j < combined.length && combined[j].v === combined[i].v) j++;
+    const avgRank = (i + 1 + j) / 2;
+    for (let k = i; k < j; k++) ranks[k] = avgRank;
+    i = j;
+  }
+  let r1 = 0;
+  for (let k = 0; k < combined.length; k++) {
+    if (combined[k].group === "a") r1 += ranks[k];
+  }
+  const u1 = r1 - n1 * (n1 + 1) / 2;
+  const u2 = n1 * n2 - u1;
+  const u = Math.min(u1, u2);
+  const mu = n1 * n2 / 2;
+  const sigma = Math.sqrt(n1 * n2 * (n1 + n2 + 1) / 12);
+  if (sigma === 0) return { u, p: 1 };
+  const z = Math.abs(u - mu) / sigma;
+  const p = 2 * (1 - normalCdf(z));
+  return { u, p };
+}
+function partialCredit(current, target) {
+  if (target <= 0) return 1;
+  return Math.min(1, Math.max(0, current / target));
+}
+function normalCdf(x) {
+  const a1 = 0.254829592;
+  const a2 = -0.284496736;
+  const a3 = 1.421413741;
+  const a4 = -1.453152027;
+  const a5 = 1.061405429;
+  const p = 0.3275911;
+  const sign = x < 0 ? -1 : 1;
+  const absX = Math.abs(x);
+  const t = 1 / (1 + p * absX);
+  const y = 1 - ((((a5 * t + a4) * t + a3) * t + a2) * t + a1) * t * Math.exp(-absX * absX / 2);
+  return 0.5 * (1 + sign * y);
+}
+
+// src/executor.ts
+async function executeScenario(tc, scenario, config) {
+  const startTime = Date.now();
+  const model = config.model ?? "gpt-4o";
+  const systemPrompt = [
+    config.systemPrompt,
+    scenario.systemPromptAppend ?? ""
+  ].filter(Boolean).join("\n\n");
+  const messages = [
+    { role: "system", content: systemPrompt }
+  ];
+  const turns = [];
+  const allCodeBlocks = [];
+  const allBlocks = [];
+  const allToolCalls = [];
+  const blockRe = config.blockPattern ?? /:::(\w+)\s*\n([\s\S]*?)\n\s*:::/g;
+  for (let i = 0; i < scenario.turns.length; i++) {
+    const turn = scenario.turns[i];
+    const turnStart = Date.now();
+    messages.push({ role: "user", content: turn.user });
+    const resp = await tc.chat({
+      model,
+      messages,
+      temperature: 0.4,
+      maxTokens: 3e3
+    });
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    messages.push({ role: "assistant", content });
+    const codeRe = /```(\w+)?\n([\s\S]*?)```/g;
+    let codeMatch;
+    while ((codeMatch = codeRe.exec(content)) !== null) {
+      allCodeBlocks.push({ language: codeMatch[1] ?? "text", code: codeMatch[2] });
+    }
+    const turnBlocks = [];
+    let blockMatch;
+    const blockReLocal = new RegExp(blockRe.source, blockRe.flags);
+    while ((blockMatch = blockReLocal.exec(content)) !== null) {
+      const fields = {};
+      for (const line of blockMatch[2].split("\n")) {
+        const idx = line.indexOf(":");
+        if (idx > 0) fields[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
+      }
+      allBlocks.push({ type: blockMatch[1], fields });
+      turnBlocks.push({ type: blockMatch[1], title: fields.title ?? "" });
+    }
+    let hasToolCall = false;
+    if (config.toolCallPatterns) {
+      for (const pattern of config.toolCallPatterns) {
+        const re = new RegExp(pattern.source, pattern.flags);
+        let toolMatch;
+        while ((toolMatch = re.exec(content)) !== null) {
+          allToolCalls.push(toolMatch[0]);
+          hasToolCall = true;
+        }
+      }
+    }
+    turns.push({
+      turnIndex: i,
+      userMessage: turn.user,
+      agentResponse: content,
+      durationMs: Date.now() - turnStart,
+      blocksExtracted: turnBlocks,
+      containsCode: allCodeBlocks.length > 0,
+      containsToolCall: hasToolCall
+    });
+  }
+  const artifacts = {
+    vaultFiles: [],
+    blocksExtracted: allBlocks,
+    codeBlocks: allCodeBlocks,
+    toolCalls: allToolCalls
+  };
+  const artifactResults = scenario.artifactChecks.map((check) => {
+    if (config.artifactChecker) {
+      const custom = config.artifactChecker(check, artifacts);
+      if (custom) return { check, ...custom };
+    }
+    switch (check.type) {
+      case "block_extracted": {
+        const count = allBlocks.filter((b) => b.type === check.target).length;
+        return {
+          check,
+          passed: count >= (check.minCount ?? 1),
+          detail: `Found ${count} ${check.target} blocks (need ${check.minCount ?? 1})`
+        };
+      }
+      case "code_valid": {
+        const hasCode = allCodeBlocks.some(
+          (b) => b.language === check.target || b.code.includes(check.target)
+        );
+        return { check, passed: hasCode, detail: hasCode ? "Code block found" : "No matching code" };
+      }
+      default:
+        return { check, passed: false, detail: `Check type "${check.type}" requires live environment` };
+    }
+  });
+  const judgeInput = { scenario, turns, artifacts };
+  const judgeResults = [];
+  for (const judge of config.judges) {
+    let lastErr = "";
+    for (let attempt = 0; attempt < 3; attempt++) {
+      try {
+        if (attempt > 0) {
+          const wait = attempt * 1e4;
+          console.log(`    judge retry ${attempt}/2 (waiting ${wait / 1e3}s)`);
+          await new Promise((r) => setTimeout(r, wait));
+        }
+        const scores = await judge(tc, judgeInput);
+        judgeResults.push(scores);
+        await new Promise((r) => setTimeout(r, 3e3));
+        break;
+      } catch (err) {
+        lastErr = err instanceof Error ? err.message : String(err);
+        if (attempt === 2) {
+          judgeResults.push([{
+            judgeName: "unknown",
+            dimension: "error",
+            score: 0,
+            reasoning: `Judge failed after 3 attempts: ${lastErr.slice(0, 200)}`
+          }]);
+        }
+      }
+    }
+  }
+  const allScores = judgeResults.flat();
+  const errorScores = allScores.filter((s) => s.dimension === "parse_error" || s.dimension === "error");
+  const validScores = allScores.filter((s) => s.dimension !== "parse_error" && s.dimension !== "error");
+  const normalized = normalizeScores(validScores);
+  const weightMap = /* @__PURE__ */ new Map();
+  for (const dim of scenario.dimensions) {
+    weightMap.set(dim, 1);
+  }
+  const overallScore = weightedMean(
+    normalized.map((s) => ({
+      score: s.score,
+      weight: weightMap.get(s.dimension) ?? 1
+    }))
+  );
+  return {
+    scenarioId: scenario.id,
+    persona: scenario.persona,
+    turns,
+    artifactResults,
+    judgeScores: allScores,
+    judgeErrors: errorScores.length,
+    overallScore,
+    totalDurationMs: Date.now() - startTime,
+    artifacts
+  };
+}
+
+// src/benchmark.ts
+var BenchmarkRunner = class {
+  tc;
+  config;
+  constructor(tc, config) {
+    this.tc = tc;
+    this.config = config;
+  }
+  async run(scenarios) {
+    const toRun = scenarios ?? this.config.scenarios;
+    const passThreshold = this.config.passThreshold ?? 6;
+    console.log("=".repeat(70));
+    console.log(" AGENT EVAL \u2014 BENCHMARK");
+    console.log(" Multi-turn scenarios x Multi-judge panel");
+    console.log("=".repeat(70));
+    console.log(`Scenarios: ${toRun.length}`);
+    console.log(`Judges: ${this.config.judges.length}`);
+    console.log(`Model: ${this.config.model ?? "gpt-4o"}`);
+    console.log();
+    const results = [];
+    for (let i = 0; i < toRun.length; i++) {
+      const scenario = toRun[i];
+      console.log(`[${i + 1}/${toRun.length}] ${scenario.id} (${scenario.persona})`);
+      console.log(`  thesis: ${scenario.thesis}`);
+      console.log(`  turns: ${scenario.turns.length}`);
+      const result = await executeScenario(this.tc, scenario, {
+        systemPrompt: this.config.systemPrompt,
+        model: this.config.model,
+        judges: this.config.judges
+      });
+      results.push(result);
+      for (const turn of result.turns) {
+        const codeIcon = turn.containsCode ? "[code]" : "";
+        const toolIcon = turn.containsToolCall ? "[tool]" : "";
+        const blockCount = turn.blocksExtracted.length;
+        const blockIcon = blockCount > 0 ? `[blocks:${blockCount}]` : "";
+        console.log(`  turn ${turn.turnIndex + 1}: ${(turn.durationMs / 1e3).toFixed(1)}s ${codeIcon} ${toolIcon} ${blockIcon} (${turn.agentResponse.length} chars)`);
+      }
+      for (const ar of result.artifactResults) {
+        const icon = ar.passed ? "+" : "X";
+        console.log(`  artifact: [${icon}] ${ar.check.description} \u2014 ${ar.detail}`);
+      }
+      console.log(`  judges:`);
+      const byJudge = {};
+      for (const js of result.judgeScores) {
+        if (!byJudge[js.judgeName]) byJudge[js.judgeName] = { scores: [], dimensions: [] };
+        byJudge[js.judgeName].scores.push(js.score);
+        byJudge[js.judgeName].dimensions.push(`${js.dimension}=${js.score}`);
+      }
+      for (const [name, data] of Object.entries(byJudge)) {
+        const avg = (data.scores.reduce((a, b) => a + b, 0) / data.scores.length).toFixed(1);
+        console.log(`    ${name.padEnd(16)} avg=${avg}  [${data.dimensions.join(", ")}]`);
+      }
+      console.log(`  OVERALL: ${result.overallScore.toFixed(1)}/10 (${(result.totalDurationMs / 1e3).toFixed(0)}s)`);
+      console.log();
+    }
+    const byPersona = {};
+    const byDimension = {};
+    for (const r of results) {
+      if (!byPersona[r.persona]) byPersona[r.persona] = { avg: 0, passed: 0, total: 0 };
+      byPersona[r.persona].total++;
+      byPersona[r.persona].avg += r.overallScore;
+      if (r.overallScore >= passThreshold) byPersona[r.persona].passed++;
+      for (const js of r.judgeScores) {
+        if (!byDimension[js.dimension]) byDimension[js.dimension] = { avg: 0, scores: [] };
+        byDimension[js.dimension].scores.push(js.score);
+      }
+    }
+    for (const p of Object.values(byPersona)) {
+      p.avg = p.total > 0 ? p.avg / p.total : 0;
+    }
+    for (const d of Object.values(byDimension)) {
+      d.avg = d.scores.length > 0 ? d.scores.reduce((a, b) => a + b, 0) / d.scores.length : 0;
+    }
+    const sorted = [...results].sort((a, b) => a.overallScore - b.overallScore);
+    const weakest = sorted.slice(0, 3).map((r) => ({
+      scenario: r.scenarioId,
+      score: r.overallScore,
+      reason: r.judgeScores.filter((s) => s.score < passThreshold).map((s) => `${s.dimension}=${s.score}`).join(", ") || "close to threshold"
+    }));
+    const strongest = sorted.slice(-3).reverse().map((r) => ({
+      scenario: r.scenarioId,
+      score: r.overallScore,
+      reason: r.judgeScores.filter((s) => s.score >= 9).map((s) => `${s.dimension}=${s.score}`).join(", ") || "consistently strong"
+    }));
+    console.log("=".repeat(70));
+    console.log(" RESULTS");
+    console.log("=".repeat(70));
+    const overallAvg = results.length > 0 ? results.reduce((s, r) => s + r.overallScore, 0) / results.length : 0;
+    console.log(`Overall: ${overallAvg.toFixed(1)}/10`);
+    console.log();
+    console.log("By persona:");
+    for (const [name, data] of Object.entries(byPersona)) {
+      console.log(`  ${name.padEnd(20)} ${data.avg.toFixed(1)}/10  (${data.passed}/${data.total} passed)`);
+    }
+    console.log();
+    console.log("By dimension:");
+    const dimEntries = Object.entries(byDimension).sort((a, b) => a[1].avg - b[1].avg);
+    for (const [name, data] of dimEntries) {
+      const min = Math.min(...data.scores);
+      const max = Math.max(...data.scores);
+      console.log(`  ${name.padEnd(24)} avg=${data.avg.toFixed(1)}  range=[${min}-${max}]  n=${data.scores.length}`);
+    }
+    console.log();
+    if (weakest.length > 0) {
+      console.log("Weakest:");
+      for (const w of weakest) {
+        console.log(`  ${w.scenario}: ${w.score.toFixed(1)} \u2014 ${w.reason}`);
+      }
+      console.log();
+    }
+    return {
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      generation: this.config.generation ?? 1,
+      promptVersion: this.config.promptVersion ?? "v1",
+      scenarioCount: toRun.length,
+      results,
+      summary: { overallAvg, byPersona, byDimension, weakest, strongest }
+    };
+  }
+};
+
+// src/metrics.ts
+var MODEL_PRICING = {
+  "gpt-4o": { input: 25e-4, output: 0.01 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  "claude-sonnet-4-20250514": { input: 3e-3, output: 0.015 },
+  "claude-opus-4-20250514": { input: 0.015, output: 0.075 },
+  "claude-3-haiku-20240307": { input: 25e-5, output: 125e-5 }
+};
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function estimateCost(inputTokens, outputTokens, model) {
+  const pricing = MODEL_PRICING[model];
+  if (!pricing) return 0;
+  return inputTokens / 1e3 * pricing.input + outputTokens / 1e3 * pricing.output;
+}
+var TokenCounter = class {
+  totalInput = 0;
+  totalOutput = 0;
+  totalCost = 0;
+  model;
+  constructor(model = "gpt-4o") {
+    this.model = model;
+  }
+  /** Record tokens for a turn, returns per-turn cost */
+  record(inputTokens, outputTokens) {
+    this.totalInput += inputTokens;
+    this.totalOutput += outputTokens;
+    const cost = estimateCost(inputTokens, outputTokens, this.model);
+    this.totalCost += cost;
+    return cost;
+  }
+  /** Estimate and record from raw text */
+  recordFromText(inputText, outputText) {
+    const inputTokens = estimateTokens(inputText);
+    const outputTokens = estimateTokens(outputText);
+    const cost = this.record(inputTokens, outputTokens);
+    return { inputTokens, outputTokens, cost };
+  }
+  getTotalInput() {
+    return this.totalInput;
+  }
+  getTotalOutput() {
+    return this.totalOutput;
+  }
+  getTotalCost() {
+    return this.totalCost;
+  }
+};
+var MetricsCollector = class {
+  client;
+  workspaceId;
+  metrics = [];
+  constructor(client, workspaceId) {
+    this.client = client;
+    this.workspaceId = workspaceId;
+  }
+  /** Collect metrics after a turn completes */
+  async collect(turn, responseLatencyMs, responseChars, codeBlocksProduced, blocksExtracted, completionCriteriaMet, completionCriteriaTotal, qualityScore, inputTokens = 0, outputTokens = 0, estimatedCostUsd = 0) {
+    const state = await this.getState();
+    const m = {
+      turn,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      tasks: state.tasks,
+      events: state.events,
+      proposals: state.proposals,
+      vaultFiles: state.vaultFiles.length,
+      responseLatencyMs,
+      responseChars,
+      codeBlocksProduced,
+      blocksExtracted,
+      qualityScore,
+      inputTokens,
+      outputTokens,
+      estimatedCostUsd,
+      totalCostUsd: estimatedCostUsd,
+      completionPercent: completionCriteriaTotal > 0 ? completionCriteriaMet / completionCriteriaTotal * 100 : 0
+    };
+    this.metrics.push(m);
+    return m;
+  }
+  /** Get current product state */
+  async getState() {
+    const [tasks, events, approvals, vaultFiles] = await Promise.all([
+      this.client.getTasks(this.workspaceId),
+      this.client.getEvents(this.workspaceId),
+      this.client.getApprovals(this.workspaceId),
+      this.client.getVaultTree(this.workspaceId)
+    ]);
+    return {
+      tasks: tasks.length,
+      events: events.length,
+      proposals: {
+        pending: approvals.filter((a) => a.status === "pending").length,
+        approved: approvals.filter((a) => a.status === "approved").length,
+        rejected: approvals.filter((a) => a.status === "rejected").length
+      },
+      vaultFiles,
+      codeBlocks: 0,
+      generations: 0
+    };
+  }
+  /** Get all collected metrics */
+  getMetrics() {
+    return [...this.metrics];
+  }
+  /** Get convergence curve (completion% over turns) */
+  getConvergenceCurve() {
+    return this.metrics.map((m) => m.completionPercent);
+  }
+};
+
+// src/convergence.ts
+var ConvergenceTracker = class {
+  criteria;
+  history = [];
+  constructor(criteria) {
+    this.criteria = criteria;
+  }
+  /** Evaluate criteria against current state, record result */
+  record(turn, state) {
+    const criteriaStatus = {};
+    let totalCredit = 0;
+    for (const criterion of this.criteria) {
+      if (criterion.progress) {
+        const credit = Math.min(1, Math.max(0, criterion.progress(state)));
+        criteriaStatus[criterion.name] = credit;
+        totalCredit += credit;
+      } else {
+        const passed = criterion.check(state);
+        criteriaStatus[criterion.name] = passed;
+        totalCredit += passed ? 1 : 0;
+      }
+    }
+    const completionPercent = this.criteria.length > 0 ? totalCredit / this.criteria.length * 100 : 100;
+    this.history.push({ turn, completionPercent, criteriaStatus });
+    return {
+      completionPercent,
+      complete: totalCredit >= this.criteria.length,
+      criteriaStatus
+    };
+  }
+  /** Get convergence curve */
+  getCurve() {
+    return this.history.map((h) => h.completionPercent);
+  }
+  /** Get full history with per-criterion status */
+  getHistory() {
+    return [...this.history];
+  }
+  /** Find the turn where completion first reached 100% (or null) */
+  getTurnToCompletion() {
+    const entry = this.history.find((h) => h.completionPercent === 100);
+    return entry?.turn ?? null;
+  }
+};
+
+// src/registry.ts
+var ScenarioRegistry = class {
+  scenarios = [];
+  scenarioFiles = [];
+  /** Register scenarios from ScenarioFile format */
+  registerFiles(files) {
+    this.scenarioFiles.push(...files);
+    this.scenarios.push(...files.map(toScenario));
+  }
+  /** Register pre-built Scenario objects directly */
+  register(scenarios) {
+    this.scenarios.push(...scenarios);
+  }
+  /** Get all scenarios */
+  all() {
+    return [...this.scenarios];
+  }
+  /** Get scenarios filtered by category */
+  byCategory(category) {
+    const fromFiles = this.scenarioFiles.filter((sf) => sf.category === category).map(toScenario);
+    return fromFiles;
+  }
+  /** List all categories with counts */
+  listCategories() {
+    const counts = {};
+    for (const sf of this.scenarioFiles) {
+      counts[sf.category] = (counts[sf.category] ?? 0) + 1;
+    }
+    return Object.entries(counts).map(([category, count]) => ({ category, count }));
+  }
+  /** Get scenarios filtered by persona */
+  byPersona(persona) {
+    return this.scenarios.filter((s) => s.persona === persona);
+  }
+  /** Get a single scenario by ID */
+  byId(id) {
+    return this.scenarios.find((s) => s.id === id);
+  }
+  /** Count total scenarios */
+  get count() {
+    return this.scenarios.length;
+  }
+};
+function toScenario(sf) {
+  return {
+    id: sf.id,
+    persona: sf.persona,
+    label: sf.label,
+    thesis: sf.thesis,
+    dimensions: [],
+    turns: sf.turns,
+    artifactChecks: sf.artifactChecks,
+    systemPromptAppend: sf.isControl ? "You are a helpful AI assistant." : void 0
+  };
+}
+
+// src/driver.ts
+var AgentDriver = class {
+  tc;
+  client;
+  driverModel;
+  productContext;
+  constructor(tc, config) {
+    this.tc = tc;
+    this.client = config.client;
+    this.driverModel = config.driverModel ?? "claude-sonnet-4-6";
+    this.productContext = config.productContext ?? "";
+  }
+  /**
+   * Run a persona through the product.
+   *
+   * Returns metrics on how many turns to completion, cost curve,
+   * quality curve, and convergence curve.
+   */
+  async run(persona) {
+    const email = `eval-driver-${Date.now()}@test.agent-eval.local`;
+    await this.client.signup(`Driver ${persona.role}`, email, "eval-driver-pass");
+    await this.client.login(email, "eval-driver-pass");
+    const workspaceId = await this.client.createWorkspace(`${persona.role} Eval`);
+    const threadId = await this.client.createThread(workspaceId);
+    const metrics = new MetricsCollector(this.client, workspaceId);
+    const convergence = new ConvergenceTracker(persona.completionCriteria);
+    const turnMetrics = [];
+    const conversationHistory = [];
+    let completed = false;
+    let turnsToCompletion = null;
+    for (let turn = 1; turn <= persona.maxTurns; turn++) {
+      const state = await metrics.getState();
+      const userMessage = await this.decideNextMessage(persona, state, conversationHistory);
+      if (userMessage === "DONE") {
+        completed = true;
+        turnsToCompletion = turn - 1;
+        break;
+      }
+      const turnStart = Date.now();
+      const response = await this.client.chat(workspaceId, threadId, userMessage);
+      const latency = Date.now() - turnStart;
+      conversationHistory.push(
+        { role: "user", content: userMessage },
+        { role: "assistant", content: response.text }
+      );
+      await new Promise((r) => setTimeout(r, 2e3));
+      await this.handleApprovals(persona, workspaceId, state);
+      const postState = await metrics.getState();
+      const conv = convergence.record(turn, postState);
+      const codeBlockCount = (response.text.match(/```\w+\n/g) || []).length;
+      const m = await metrics.collect(
+        turn,
+        latency,
+        response.text.length,
+        codeBlockCount,
+        response.blocks.length,
+        Object.values(conv.criteriaStatus).filter(Boolean).length,
+        persona.completionCriteria.length
+      );
+      turnMetrics.push(m);
+      const criteriaStr = Object.entries(conv.criteriaStatus).map(([k, v]) => `${k}:${v ? "+" : "-"}`).join(" ");
+      console.log(`  [turn ${turn}] ${conv.completionPercent.toFixed(0)}% \u2014 ${criteriaStr} (${(latency / 1e3).toFixed(1)}s)`);
+      if (conv.complete) {
+        completed = true;
+        turnsToCompletion = turn;
+        console.log(`  COMPLETE at turn ${turn}`);
+        break;
+      }
+    }
+    const finalState = await metrics.getState();
+    return {
+      personaId: persona.id,
+      completed,
+      turnsToCompletion,
+      totalTurns: turnMetrics.length,
+      metrics: turnMetrics,
+      finalState,
+      convergenceCurve: convergence.getCurve(),
+      totalCostUsd: 0,
+      finalQualityScore: null
+    };
+  }
+  /** Use the driver LLM to decide what the "user" says next */
+  async decideNextMessage(persona, state, history) {
+    const lastResponse = history.length > 0 ? history[history.length - 1].content.slice(0, 2e3) : "(no conversation yet \u2014 this is the first message)";
+    const recentHistory = history.slice(-6).map(
+      (h) => `${h.role}: ${h.content.slice(0, 500)}`
+    ).join("\n\n");
+    const resp = await this.tc.chat({
+      model: this.driverModel,
+      messages: [{
+        role: "system",
+        content: `You are playing the role of a ${persona.role} testing an AI agent.
+Your goal: ${persona.goal}
+
+${this.productContext ? `Product context:
+${this.productContext}
+` : ""}
+Current state:
+- Tasks: ${state.tasks}
+- Events: ${state.events}
+- Proposals: pending=${state.proposals.pending}, approved=${state.proposals.approved}, rejected=${state.proposals.rejected}
+- Vault files: ${state.vaultFiles.length} (${state.vaultFiles.slice(0, 10).join(", ")}${state.vaultFiles.length > 10 ? "..." : ""})
+
+Completion criteria met: ${this.describeCompletion(persona, state)}
+
+Decide what to do next:
+1. If completion is 100% \u2014 respond with exactly "DONE"
+2. If a proposal is pending \u2014 approve or reject it (with reason)
+3. If the agent is on track \u2014 push for the next deliverable
+4. If the agent is off track \u2014 give specific corrective feedback
+5. If this is the first message \u2014 start with a clear, actionable request
+
+Output ONLY your next message to the agent. Be specific. Be realistic.
+Don't be patient \u2014 a real ${persona.role} wouldn't accept vague answers.`
+      }, {
+        role: "user",
+        content: recentHistory ? `Recent conversation:
+${recentHistory}
+
+The agent just said:
+${lastResponse}` : "No conversation yet. Send your opening message."
+      }],
+      temperature: 0.5,
+      maxTokens: 500
+    });
+    const content = resp.choices?.[0]?.message?.content ?? "";
+    return content.trim();
+  }
+  /** Handle pending approvals based on persona feedback patterns */
+  async handleApprovals(persona, workspaceId, _state) {
+    const approvals = await this.client.getApprovals(workspaceId);
+    const pending = approvals.filter((a) => a.status === "pending");
+    for (const action of pending) {
+      const rejection = persona.feedbackPatterns?.find((fp) => {
+        const title = action.title.toLowerCase();
+        return title.includes(fp.trigger.toLowerCase());
+      });
+      if (rejection) {
+        await this.client.rejectAction(workspaceId, action.id, rejection.response);
+        console.log(`    rejected: ${action.title} \u2014 ${rejection.response.slice(0, 60)}`);
+      } else {
+        await this.client.approveAction(workspaceId, action.id);
+        console.log(`    approved: ${action.title}`);
+      }
+    }
+  }
+  /** Describe which completion criteria are met */
+  describeCompletion(persona, state) {
+    const results = persona.completionCriteria.map((c) => {
+      const met = c.check(state);
+      return `${c.name}: ${met ? "MET" : "NOT MET"}`;
+    });
+    const metCount = results.filter((r) => r.includes("MET") && !r.includes("NOT")).length;
+    return `${metCount}/${persona.completionCriteria.length} \u2014 ${results.join(", ")}`;
+  }
+};
+
+// src/reporter.ts
+function formatBenchmarkReport(report) {
+  const lines = [];
+  lines.push(`# Benchmark Report`);
+  lines.push(``);
+  lines.push(`**Date:** ${report.timestamp}`);
+  lines.push(`**Generation:** ${report.generation}`);
+  lines.push(`**Prompt Version:** ${report.promptVersion}`);
+  lines.push(`**Scenarios:** ${report.scenarioCount}`);
+  lines.push(`**Overall Score:** ${report.summary.overallAvg.toFixed(1)}/10`);
+  lines.push(``);
+  lines.push(`## By Persona`);
+  lines.push(``);
+  lines.push(`| Persona | Avg | Passed | Total |`);
+  lines.push(`|---------|-----|--------|-------|`);
+  for (const [name, data] of Object.entries(report.summary.byPersona)) {
+    lines.push(`| ${name} | ${data.avg.toFixed(1)} | ${data.passed} | ${data.total} |`);
+  }
+  lines.push(``);
+  lines.push(`## By Dimension`);
+  lines.push(``);
+  lines.push(`| Dimension | Avg | Range | N |`);
+  lines.push(`|-----------|-----|-------|---|`);
+  const dimEntries = Object.entries(report.summary.byDimension).sort((a, b) => a[1].avg - b[1].avg);
+  for (const [name, data] of dimEntries) {
+    const min = Math.min(...data.scores);
+    const max = Math.max(...data.scores);
+    lines.push(`| ${name} | ${data.avg.toFixed(1)} | ${min}-${max} | ${data.scores.length} |`);
+  }
+  lines.push(``);
+  if (report.summary.weakest.length > 0) {
+    lines.push(`## Weakest Scenarios`);
+    lines.push(``);
+    for (const w of report.summary.weakest) {
+      lines.push(`- **${w.scenario}** (${w.score.toFixed(1)}): ${w.reason}`);
+    }
+    lines.push(``);
+  }
+  if (report.summary.strongest.length > 0) {
+    lines.push(`## Strongest Scenarios`);
+    lines.push(``);
+    for (const s of report.summary.strongest) {
+      lines.push(`- **${s.scenario}** (${s.score.toFixed(1)}): ${s.reason}`);
+    }
+    lines.push(``);
+  }
+  return lines.join("\n");
+}
+function formatDriverReport(results) {
+  const lines = [];
+  lines.push(`# Agent Driver Report`);
+  lines.push(``);
+  for (const r of results) {
+    lines.push(`## Persona: ${r.personaId}`);
+    lines.push(``);
+    lines.push(`- **Completed:** ${r.completed ? "Yes" : "No"}`);
+    lines.push(`- **Turns to completion:** ${r.turnsToCompletion ?? "N/A"}`);
+    lines.push(`- **Total turns:** ${r.totalTurns}`);
+    lines.push(`- **Final state:** ${r.finalState.tasks} tasks, ${r.finalState.events} events, ${r.finalState.vaultFiles.length} vault files`);
+    lines.push(``);
+    lines.push(`### Convergence`);
+    lines.push(``);
+    lines.push("```");
+    for (let i = 0; i < r.convergenceCurve.length; i++) {
+      const pct = r.convergenceCurve[i];
+      const bar = "#".repeat(Math.round(pct / 2));
+      lines.push(`  turn ${String(i + 1).padStart(2)}: ${bar} ${pct.toFixed(0)}%`);
+    }
+    lines.push("```");
+    lines.push(``);
+    if (r.metrics.length > 0) {
+      lines.push(`### Per-Turn Metrics`);
+      lines.push(``);
+      lines.push(`| Turn | Tasks | Events | Vault | Latency | Completion |`);
+      lines.push(`|------|-------|--------|-------|---------|------------|`);
+      for (const m of r.metrics) {
+        lines.push(`| ${m.turn} | ${m.tasks} | ${m.events} | ${m.vaultFiles} | ${(m.responseLatencyMs / 1e3).toFixed(1)}s | ${m.completionPercent.toFixed(0)}% |`);
+      }
+      lines.push(``);
+    }
+  }
+  return lines.join("\n");
+}
+function printDriverSummary(results) {
+  console.log("=".repeat(70));
+  console.log(" AGENT DRIVER \u2014 RESULTS");
+  console.log("=".repeat(70));
+  for (const r of results) {
+    const status = r.completed ? "COMPLETE" : "INCOMPLETE";
+    const turns = r.turnsToCompletion ?? r.totalTurns;
+    console.log(`  ${r.personaId.padEnd(20)} ${status.padEnd(12)} turns=${turns}  tasks=${r.finalState.tasks}  events=${r.finalState.events}  vault=${r.finalState.vaultFiles.length}`);
+  }
+  console.log();
+  const completedCount = results.filter((r) => r.completed).length;
+  console.log(`${completedCount}/${results.length} personas completed`);
+}
+export {
+  AgentDriver,
+  BenchmarkRunner,
+  ConvergenceTracker,
+  MODEL_PRICING,
+  MetricsCollector,
+  ProductClient,
+  ScenarioRegistry,
+  TokenCounter,
+  adversarialJudge,
+  codeExecutionJudge,
+  coherenceJudge,
+  confidenceInterval,
+  createCustomJudge,
+  createDomainExpertJudge,
+  defaultJudges,
+  estimateCost,
+  estimateTokens,
+  executeScenario,
+  formatBenchmarkReport,
+  formatDriverReport,
+  interRaterReliability,
+  mannWhitneyU,
+  normalizeScores,
+  partialCredit,
+  printDriverSummary,
+  runE2EWorkflow,
+  weightedMean
+};
+//# sourceMappingURL=index.js.map