openmates 0.12.0-alpha.11 → 0.12.0-alpha.13

package/dist/{chunk-2QG4XPEB.js → chunk-YHOUQRWZ.js}

@@ -986,14 +986,14 @@ var OpenMatesWsClient = class {
     });
   }
   async open(timeoutMs = 1e4) {
-    await new Promise((resolve5, reject) => {
+    await new Promise((resolve6, reject) => {
       const timeout = setTimeout(
         () => reject(new Error("WebSocket open timeout")),
         timeoutMs
       );
       this.socket.once("open", () => {
         clearTimeout(timeout);
-        resolve5();
+        resolve6();
       });
       this.socket.once("error", (error) => {
         clearTimeout(timeout);
@@ -1022,15 +1022,15 @@ var OpenMatesWsClient = class {
     this.socket.send(JSON.stringify({ type, payload }));
   }
   sendAsync(type, payload) {
-    return new Promise((resolve5, reject) => {
+    return new Promise((resolve6, reject) => {
       this.socket.send(JSON.stringify({ type, payload }), (error) => {
         if (error) reject(error);
-        else resolve5();
+        else resolve6();
       });
     });
   }
   waitForMessage(expectedType, predicate, timeoutMs = 2e4) {
-    return new Promise((resolve5, reject) => {
+    return new Promise((resolve6, reject) => {
       const onMessage = (rawData) => {
         try {
           const parsed = JSON.parse(rawData.toString());
@@ -1041,7 +1041,7 @@ var OpenMatesWsClient = class {
             return;
           }
           cleanup();
-          resolve5(parsed);
+          resolve6(parsed);
         } catch {
         }
       };
@@ -1074,14 +1074,14 @@ var OpenMatesWsClient = class {
    * Used by ensureSynced to consume the full phased-sync event stream.
    */
   collectMessages(terminatorType, timeoutMs = 9e4) {
-    return new Promise((resolve5, reject) => {
+    return new Promise((resolve6, reject) => {
       const collected = [];
       const onMessage = (rawData) => {
         try {
           const parsed = JSON.parse(rawData.toString());
           if (parsed.type === terminatorType) {
             cleanup();
-            resolve5(collected);
+            resolve6(collected);
             return;
           }
           collected.push(parsed);
@@ -1094,7 +1094,7 @@ var OpenMatesWsClient = class {
       };
       const onClose = () => {
         cleanup();
-        resolve5(collected);
+        resolve6(collected);
       };
       const timeout = setTimeout(() => {
         cleanup();
@@ -1132,7 +1132,7 @@ var OpenMatesWsClient = class {
     const timeoutMs = options?.timeoutMs ?? 9e4;
     const onStream = options?.onStream;
     const asyncEmbedWaitMs = options?.asyncEmbedWaitMs ?? 12e4;
-    return new Promise((resolve5, reject) => {
+    return new Promise((resolve6, reject) => {
       let latestContent = "";
       let messageId = null;
       let taskId = null;
@@ -1189,7 +1189,7 @@ var OpenMatesWsClient = class {
         if (waitingForUserPayload) {
           if (pendingSubChatHandlers.size > 0) return;
           cleanup();
-          resolve5({
+          resolve6({
             status: "waiting_for_user",
             messageId,
             taskId,
@@ -1209,7 +1209,7 @@ var OpenMatesWsClient = class {
         if (processingEmbedIds.size > 0 && !asyncEmbedTimer) {
           asyncEmbedTimer = setTimeout(() => {
             cleanup();
-            resolve5({
+            resolve6({
               status: "completed",
               messageId,
               taskId,
@@ -1226,7 +1226,7 @@ var OpenMatesWsClient = class {
         }
         if (processingEmbedIds.size > 0) return;
         cleanup();
-        resolve5({
+        resolve6({
           status: "completed",
           messageId,
           taskId,
@@ -1440,7 +1440,7 @@ var OpenMatesWsClient = class {
       const onClose = () => {
         if (aiResponseDone) {
           cleanup();
-          resolve5({
+          resolve6({
             status: "completed",
             messageId,
             taskId,
@@ -3681,7 +3681,11 @@ var OpenMatesClient = class _OpenMatesClient {
       messagePayload.benchmark_metadata = params.benchmarkMetadata;
     }
     if (params.incognito) {
-      messagePayload.message_history = [{
+      const providedHistory = (params.messageHistory ?? []).map((historyMessage) => ({
+        ...historyMessage,
+        chat_id: historyMessage.chat_id ?? chatId
+      }));
+      messagePayload.message_history = [...providedHistory, {
         message_id: messageId,
         chat_id: chatId,
         role: "user",
@@ -4315,7 +4319,7 @@ var OpenMatesClient = class _OpenMatesClient {
       if (response.data.status === "failed") {
         throw new Error(response.data.error ?? "Task failed");
       }
-      await new Promise((resolve5) => setTimeout(resolve5, SKILL_TASK_POLL_INTERVAL_MS));
+      await new Promise((resolve6) => setTimeout(resolve6, SKILL_TASK_POLL_INTERVAL_MS));
     }
     throw new Error(`Task ${taskId} did not complete within ${SKILL_TASK_POLL_TIMEOUT_MS / 1e3}s`);
   }
@@ -4536,7 +4540,7 @@ var OpenMatesClient = class _OpenMatesClient {
         `Rate limited by settings API; retrying in ${Math.ceil(SETTINGS_GET_RATE_LIMIT_RETRY_MS / 1e3)}s...
 `
       );
-      await new Promise((resolve5) => setTimeout(resolve5, SETTINGS_GET_RATE_LIMIT_RETRY_MS));
+      await new Promise((resolve6) => setTimeout(resolve6, SETTINGS_GET_RATE_LIMIT_RETRY_MS));
       response = await this.http.get(normalizedPath, this.getCliRequestHeaders());
     }
     if (!response.ok) {
@@ -6037,7 +6041,7 @@ function filenameFromContentDisposition(header2) {
   return plain?.trim() ?? null;
 }
 function sleep(ms) {
-  return new Promise((resolve5) => setTimeout(resolve5, ms));
+  return new Promise((resolve6) => setTimeout(resolve6, ms));
 }
 function printLogo() {
   const W = "\x1B[1;37m";
@@ -6054,8 +6058,8 @@ function printLogo() {
 // src/cli.ts
 import { createInterface as createInterface3 } from "readline/promises";
 import { realpathSync, writeFileSync as writeFileSync5 } from "fs";
-import { fileURLToPath } from "url";
-import { basename as basename3, dirname } from "path";
+import { fileURLToPath as fileURLToPath2 } from "url";
+import { basename as basename3, dirname as dirname2 } from "path";
 import WebSocket2 from "ws";
 
 // ../secret-scanner/src/registry.ts
@@ -7755,8 +7759,8 @@ async function renderRemotionShareLink(embedId, client, ln) {
   }
 }
 function generateQr(value) {
-  return new Promise((resolve5) => {
-    qrcode2.generate(value, { small: true }, (qr) => resolve5(qr));
+  return new Promise((resolve6) => {
+    qrcode2.generate(value, { small: true }, (qr) => resolve6(qr));
   });
 }
 function remotionMeta(c) {
@@ -8611,9 +8615,9 @@ function exec(cmd, cwd) {
   return execSync(cmd, { cwd, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
 }
 function runInteractive(cmd, args, cwd) {
-  return new Promise((resolve5, reject) => {
+  return new Promise((resolve6, reject) => {
     const child = nodeSpawn(cmd, args, { cwd, stdio: "inherit", shell: false });
-    child.on("close", (code) => resolve5(code ?? 1));
+    child.on("close", (code) => resolve6(code ?? 1));
     child.on("error", reject);
   });
 }
@@ -8874,10 +8878,10 @@ function warnIfMissingLlmCredentials(installPath) {
 }
 async function confirmDestructive(phrase) {
   const rl = createInterface2({ input: process.stdin, output: process.stderr });
-  return new Promise((resolve5) => {
+  return new Promise((resolve6) => {
     rl.question(`Type "${phrase}" to confirm: `, (answer) => {
       rl.close();
-      resolve5(answer.trim() === phrase);
+      resolve6(answer.trim() === phrase);
     });
   });
 }
@@ -29279,6 +29283,15 @@ Only output the final Markdown table. Do NOT include explanations, notes, or any
     copy_failed: {
       text: "Failed to copy to clipboard"
     },
+    code_file_downloaded: {
+      text: "Code file downloaded successfully"
+    },
+    code_file_download_failed: {
+      text: "Failed to download code file"
+    },
+    action_failed: {
+      text: "Failed to perform action"
+    },
     download_itinerary: {
       text: "Download itinerary"
     },
@@ -41521,26 +41534,276 @@ function buildAssistantFeedbackDecision(rating) {
 
 // src/benchmark.ts
 import { randomUUID as randomUUID3 } from "crypto";
-import { writeFileSync as writeFileSync4 } from "fs";
+import { existsSync as existsSync6, mkdtempSync, readFileSync as readFileSync6, readdirSync, writeFileSync as writeFileSync4 } from "fs";
+import { tmpdir } from "os";
+import { dirname, join as join4, resolve as resolve5 } from "path";
+import { fileURLToPath } from "url";
 var DEFAULT_JUDGE_MODEL = "google/gemini-3-flash-preview";
-var BENCHMARK_CASES = [
+var DEFAULT_EXTENSIVE_SIZE = 10;
+var DEFAULT_PARALLEL = 4;
+var FIXTURE_IMAGE_SVG = `<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="1200" height="800" viewBox="0 0 1200 800">
+  <rect width="1200" height="800" fill="#d8ecff"/>
+  <rect y="560" width="1200" height="240" fill="#d7c39a"/>
+  <text x="600" y="88" text-anchor="middle" font-family="Arial, sans-serif" font-size="44" font-weight="700" fill="#23344d">Brandenburger Tor, Berlin</text>
+  <g transform="translate(160 170)" fill="#c9aa6a" stroke="#5d4522" stroke-width="8">
+    <rect x="80" y="160" width="800" height="58"/>
+    <rect x="120" y="218" width="720" height="48"/>
+    <rect x="150" y="266" width="660" height="42"/>
+    <g fill="#d9bd7d">
+      <rect x="170" y="308" width="54" height="250"/>
+      <rect x="285" y="308" width="54" height="250"/>
+      <rect x="400" y="308" width="54" height="250"/>
+      <rect x="515" y="308" width="54" height="250"/>
+      <rect x="630" y="308" width="54" height="250"/>
+      <rect x="745" y="308" width="54" height="250"/>
+    </g>
+    <rect x="130" y="558" width="700" height="50"/>
+    <path d="M480 30 C530 72 620 88 682 48 L720 84 C652 142 530 124 456 78 Z" fill="#3e6f5f"/>
+    <circle cx="510" cy="92" r="22" fill="#3e6f5f"/>
+    <circle cx="625" cy="92" r="22" fill="#3e6f5f"/>
+    <path d="M565 38 l26 78 h-52 z" fill="#3e6f5f"/>
+  </g>
+  <text x="600" y="740" text-anchor="middle" font-family="Arial, sans-serif" font-size="32" fill="#23344d">Neoclassical gate with Quadriga on top</text>
+</svg>
+`;
+var QUICK_CASES = [
   {
-    id: "smoke-exact-token",
-    suite: "smoke",
+    id: "quick-exact-token",
+    suite: "quick",
+    title: "Exact token smoke test",
     prompt: "Reply with exactly this token and no extra text: BENCHMARK_SMOKE_OK",
-    expectedIncludes: "BENCHMARK_SMOKE_OK"
+    complexity: "basic",
+    category: "smoke",
+    expectedIncludes: "BENCHMARK_SMOKE_OK",
+    judge: true,
+    estimatedInputTokens: 12e3,
+    estimatedOutputTokens: 64
   },
   {
-    id: "arithmetic-direct",
-    suite: "tools",
+    id: "quick-arithmetic",
+    suite: "quick",
+    title: "Arithmetic direct answer",
     prompt: "Compute 19 * 23. Reply with only the integer result.",
-    expectedIncludes: "437"
+    complexity: "basic",
+    category: "math",
+    expectedIncludes: "437",
+    judge: true,
+    estimatedInputTokens: 12e3,
+    estimatedOutputTokens: 64
   },
   {
-    id: "quality-concise-explanation",
-    suite: "quality",
-    prompt: "In four concise sentences, explain why deterministic benchmarks still need human-readable evaluation notes.",
-    needsJudge: true
+    id: "quick-code",
+    suite: "quick",
+    title: "Small code generation",
+    prompt: "Write a TypeScript function isPalindrome(input: string): boolean that ignores spaces, punctuation, and case. Include only the function and one short usage example.",
+    complexity: "medium",
+    category: "coding",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 650
+  },
+  {
+    id: "quick-image-brandenburger-tor",
+    suite: "quick",
+    title: "Default image understanding",
+    prompt: "Look at the attached image. What landmark is shown, when was it built, and who designed it? Answer in three concise bullet points.",
+    complexity: "medium",
+    category: "image",
+    image: "default",
+    expectedIncludes: "Brandenburg",
+    judge: true,
+    estimatedInputTokens: 13500,
+    estimatedOutputTokens: 350
+  },
+  {
+    id: "quick-followup-continuity",
+    suite: "quick",
+    title: "Short multi-turn continuity",
+    prompt: "Create a three-step plan for evaluating whether a new AI model is ready for production use.",
+    complexity: "medium",
+    category: "multi_turn",
+    judge: true,
+    estimatedInputTokens: 14e3,
+    estimatedOutputTokens: 900,
+    followUps: [
+      { prompt: "Now make step 2 more concrete with two measurable checks." },
+      { prompt: "Summarize the final plan in one sentence." }
+    ]
+  }
+];
+var EXTENSIVE_CASES = [
+  ...QUICK_CASES,
+  {
+    id: "extensive-coding-debug",
+    suite: "extensive",
+    title: "Debug a JavaScript bug",
+    prompt: "A JavaScript function returns NaN when summing prices from [{price: '12.50'}, {price: undefined}]. Explain the bug and write a corrected function.",
+    complexity: "medium",
+    category: "coding",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 850
+  },
+  {
+    id: "extensive-coding-api-design",
+    suite: "extensive",
+    title: "Design a small API contract",
+    prompt: "Design a minimal JSON API for creating and listing benchmark runs. Include request/response examples and one validation error.",
+    complexity: "advanced",
+    category: "coding",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 1e3
+  },
+  {
+    id: "extensive-reasoning-tradeoffs",
+    suite: "extensive",
+    title: "Reason about benchmark tradeoffs",
+    prompt: "Compare deterministic assertions and LLM-as-judge evaluation for model benchmarks. Give two strengths and two risks for each.",
+    complexity: "medium",
+    category: "reasoning",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 800
+  },
+  {
+    id: "extensive-planning",
+    suite: "extensive",
+    title: "Operational rollout plan",
+    prompt: "Create a rollout checklist for switching a production chatbot from one model to another. Include monitoring, rollback, and user-visible risk checks.",
+    complexity: "advanced",
+    category: "synthesis",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 950
+  },
+  {
+    id: "extensive-long-context-followup",
+    suite: "extensive",
+    title: "Prebuilt 20-message long chat follow-up",
+    prompt: "Based on the earlier discussion, choose the best launch strategy and explain why in five bullets.",
+    complexity: "advanced",
+    category: "long_context",
+    longContext: true,
+    judge: true,
+    estimatedInputTokens: 18500,
+    estimatedOutputTokens: 900
+  },
+  {
+    id: "extensive-policy-summary",
+    suite: "extensive",
+    title: "Policy summarization",
+    prompt: "Summarize why privacy-preserving benchmark logs should avoid raw user prompts. Include a concrete safer alternative.",
+    complexity: "medium",
+    category: "reasoning",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 650
+  },
+  {
+    id: "extensive-structured-output",
+    suite: "extensive",
+    title: "Structured JSON output",
+    prompt: "Return only JSON with keys risk, mitigation, and confidence for the risk: benchmark results are biased by prompt wording.",
+    complexity: "medium",
+    category: "synthesis",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 350
+  },
+  {
+    id: "extensive-creative-constraint",
+    suite: "extensive",
+    title: "Creative constrained response",
+    prompt: "Write a six-line product note announcing model comparisons. Each line must be under 70 characters and avoid hype words like revolutionary or magical.",
+    complexity: "medium",
+    category: "synthesis",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 500
+  },
+  {
+    id: "extensive-data-reasoning",
+    suite: "extensive",
+    title: "Interpret metrics",
+    prompt: "A benchmark has pass rates 8/10, 7/10, and 9/10 across three runs. Explain what you can and cannot conclude from this sample.",
+    complexity: "medium",
+    category: "reasoning",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 600
+  },
+  {
+    id: "extensive-security-review",
+    suite: "extensive",
+    title: "Security review",
+    prompt: "Review this benchmark design for security risks: it logs prompts, outputs, model ids, and usage costs to a shared file. List risks and safer defaults.",
+    complexity: "advanced",
+    category: "reasoning",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 850
+  },
+  {
+    id: "extensive-followup-requirements",
+    suite: "extensive",
+    title: "Three-turn requirements refinement",
+    prompt: "Draft acceptance criteria for a CLI benchmark comparison feature.",
+    complexity: "advanced",
+    category: "multi_turn",
+    judge: true,
+    estimatedInputTokens: 14500,
+    estimatedOutputTokens: 1100,
+    followUps: [
+      { prompt: "Add one criterion about cost estimation before live runs." },
+      { prompt: "Add one criterion about partial results after interruption." },
+      { prompt: "Now compress the criteria to five bullets total." }
+    ]
+  },
+  {
+    id: "extensive-coding-tests",
+    suite: "extensive",
+    title: "Write tests for parser behavior",
+    prompt: "Write Node.js test cases for a function parseSuites(value) that accepts quick, extensive, all, and comma-separated lists, and rejects unknown suites.",
+    complexity: "medium",
+    category: "coding",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 950
+  },
+  {
+    id: "extensive-coding-refactor",
+    suite: "extensive",
+    title: "Refactor duplicated code",
+    prompt: "Given two duplicated TypeScript loops that build arrays of result objects, explain when to extract a helper and write the helper signature.",
+    complexity: "medium",
+    category: "coding",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 750
+  },
+  {
+    id: "extensive-comparison-analysis",
+    suite: "extensive",
+    title: "Compare two model outputs",
+    prompt: "Explain how you would compare two model outputs when one is concise but misses caveats and the other is verbose but complete.",
+    complexity: "medium",
+    category: "reasoning",
+    judge: true,
+    estimatedInputTokens: 12200,
+    estimatedOutputTokens: 650
+  },
+  {
+    id: "extensive-failure-mode",
+    suite: "extensive",
+    title: "Failure-mode analysis",
+    prompt: "List five failure modes for image-understanding benchmarks and one mitigation for each.",
+    complexity: "advanced",
+    category: "image",
+    judge: true,
+    estimatedInputTokens: 12300,
+    estimatedOutputTokens: 900
   }
 ];
 async function handleBenchmark(client, subcommand, rest, flags) {
@@ -41551,122 +41814,105 @@ async function handleBenchmark(client, subcommand, rest, flags) {
   if (subcommand !== "model") {
     throw new Error(`Unknown benchmark command '${subcommand}'. Run 'openmates benchmark --help'.`);
   }
-  const targetModel = rest[0];
-  if (!targetModel) {
-    throw new Error("Missing target model. Usage: openmates benchmark model <provider/model> --confirm-spend-credits");
+  const targetModels = rest.filter((arg) => !arg.startsWith("--"));
+  if (targetModels.length === 0) {
+    throw new Error("Missing target model. Usage: openmates benchmark model <provider/model> [model-b] --confirm-spend-credits");
+  }
+  const compare = flags.compare === true;
+  if (targetModels.length > 1 && !compare) {
+    throw new Error("Multiple target models require --compare.");
+  }
+  if (compare && targetModels.length < 2) {
+    throw new Error("--compare requires at least two target models.");
   }
   const judgeModel = typeof flags["judge-model"] === "string" ? flags["judge-model"] : DEFAULT_JUDGE_MODEL;
   const suites = parseSuites(flags.suite);
   const runs = parseRuns(flags.runs);
+  const extensiveSize = parseExtensiveSize(flags["extensive-size"]);
+  const parallel = parseParallel(flags.parallel);
+  const caseIds = parseCaseIds(flags.case);
   const dryRun = flags["dry-run"] === true;
   const output = typeof flags.output === "string" ? flags.output : void 0;
   const runId = typeof flags["run-id"] === "string" ? flags["run-id"] : randomUUID3();
+  const imagePath = typeof flags.image === "string" ? resolve5(flags.image) : defaultImageFixturePath();
   if (!dryRun && flags["confirm-spend-credits"] !== true) {
     throw new Error(
       "Benchmark runs spend real credits from the logged-in account. Rerun with --confirm-spend-credits, or use --dry-run to preview the plan."
     );
   }
-  const cases = expandCases(suites, runs);
-  const baseResult = {
-    command: "benchmark model",
-    status: dryRun ? "planned" : "completed",
+  const cases = filterCases(expandCases(suites, runs, extensiveSize), caseIds);
+  const pricing = loadPricingForModels([...targetModels, judgeModel]);
+  const estimate = estimateCredits(cases, targetModels, judgeModel, pricing);
+  const result = makeBaseResult({
     runId,
-    targetModel,
+    targetModels,
     judgeModel,
     suites,
     runs,
-    spendsCredits: !dryRun,
-    cases: [],
-    summary: { total: cases.length, passed: 0, failed: 0 }
-  };
+    compare,
+    parallel,
+    extensiveSize,
+    dryRun,
+    estimate,
+    totalJobs: cases.length * targetModels.length
+  });
   if (dryRun) {
-    writeBenchmarkResult(baseResult, flags, output);
+    writeBenchmarkResult(result, flags, output);
     return;
   }
   if (!client.hasSession()) {
     throw new Error("Benchmark runs require login. Run 'openmates login' first.");
   }
-  for (const benchmarkCase of cases) {
-    const startedAt = Date.now();
-    const targetResponse = await client.sendMessage({
-      message: `${modelMention(targetModel)} ${benchmarkCase.prompt}`,
-      incognito: true,
-      autoApproveSubChats: true,
-      benchmarkMetadata: benchmarkMetadata({
-        runId,
-        suite: benchmarkCase.suite,
-        caseId: benchmarkCase.id,
-        targetModel,
-        judgeModel
-      }),
-      precollectResponse: true
+  let interrupted = false;
+  const onInterrupt = () => {
+    interrupted = true;
+  };
+  process.once("SIGINT", onInterrupt);
+  try {
+    const jobs = cases.flatMap((benchmarkCase) => targetModels.map((model) => ({ model, benchmarkCase })));
+    await runPool(jobs, parallel, async (job) => {
+      if (interrupted) return;
+      const caseResult = await runCaseJob({ client, job, judgeModel, runId, imagePath });
+      result.cases.push(caseResult);
+      recomputeResult(result, jobs.length, interrupted);
     });
-    const caseResult = {
-      id: benchmarkCase.id,
-      suite: benchmarkCase.suite,
-      run: benchmarkCase.run,
-      prompt: benchmarkCase.prompt,
-      assistant: targetResponse.assistant,
-      modelName: targetResponse.modelName,
-      passed: benchmarkCase.expectedIncludes ? targetResponse.assistant.includes(benchmarkCase.expectedIncludes) : true,
-      durationMs: Date.now() - startedAt,
-      expectedIncludes: benchmarkCase.expectedIncludes
-    };
-    if (benchmarkCase.needsJudge) {
-      const judgeResponse = await client.sendMessage({
-        message: `${modelMention(judgeModel)} ${judgePrompt(benchmarkCase.prompt, targetResponse.assistant)}`,
-        incognito: true,
-        autoApproveSubChats: true,
-        benchmarkMetadata: benchmarkMetadata({
-          runId,
-          suite: benchmarkCase.suite,
-          caseId: `${benchmarkCase.id}:judge`,
-          targetModel,
-          judgeModel
-        }),
-        precollectResponse: true
-      });
-      const judgment = parseJudgment(judgeResponse.assistant);
-      caseResult.judge = {
-        model: judgeModel,
-        score: judgment.score,
-        reason: judgment.reason,
-        raw: judgeResponse.assistant
-      };
-      caseResult.passed = judgment.score !== null && judgment.score >= 4;
-    }
-    baseResult.cases.push(caseResult);
+  } finally {
+    process.off("SIGINT", onInterrupt);
   }
-  baseResult.summary.passed = baseResult.cases.filter((result) => result.passed).length;
-  baseResult.summary.failed = baseResult.cases.length - baseResult.summary.passed;
-  writeBenchmarkResult(baseResult, flags, output);
+  recomputeResult(result, cases.length * targetModels.length, interrupted);
+  writeBenchmarkResult(result, flags, output);
 }
 function printBenchmarkHelp() {
   console.log(`Benchmark commands:
-  openmates benchmark model <provider/model> --confirm-spend-credits [--suite smoke|tools|quality|all] [--runs <n>] [--json]
+  openmates benchmark model <provider/model> [provider/model...] --confirm-spend-credits [--compare] [--suite quick|extensive|all] [--json]
 
 Runs real incognito chat requests through the OpenMates product path. Live runs
 spend the logged-in user's credits and usage entries are grouped as benchmark spend.
 
 Options:
   --confirm-spend-credits       Required for live benchmark runs
-  --dry-run                     Preview the benchmark plan without login or spend
-  --suite <list>                Comma-separated suites: smoke, tools, quality, all (default: smoke)
+  --dry-run                     Preview the benchmark plan without inference or spend
+  --compare                     Compare two or more target models
+  --suite <list>                Comma-separated suites: quick, extensive, all (default: quick)
+  --case <id[,id...]>           Run only specific case id(s) from the selected suites
+  --extensive-size <n>          Extensive cases to run: 5, 10, or 20 (default: ${DEFAULT_EXTENSIVE_SIZE})
   --runs <n>                    Repeat each selected case (default: 1)
-  --judge-model <provider/model> Judge for quality cases (default: ${DEFAULT_JUDGE_MODEL})
+  --parallel <n>                Concurrent target case requests (default: ${DEFAULT_PARALLEL})
+  --judge-model <provider/model> Judge for evaluated cases (default: ${DEFAULT_JUDGE_MODEL})
+  --image <path>                Override default Brandenburger Tor image fixture
   --run-id <id>                 Reuse a benchmark run id for grouping
   --output <path>               Save JSON result to a file
   --json                        Print JSON result`);
 }
 function parseSuites(value) {
-  if (value === void 0 || value === false) return ["smoke"];
+  if (value === void 0 || value === false) return ["quick"];
   if (value === true) throw new Error("--suite requires a value");
   const suites = value.split(",").map((suite) => suite.trim()).filter(Boolean);
-  if (suites.includes("all")) return ["smoke", "tools", "quality"];
-  const allowed = /* @__PURE__ */ new Set(["smoke", "tools", "quality"]);
+  if (suites.includes("all")) return ["quick", "extensive"];
+  const allowed = /* @__PURE__ */ new Set(["quick", "extensive"]);
   const invalid = suites.filter((suite) => !allowed.has(suite));
   if (invalid.length > 0 || suites.length === 0) {
-    throw new Error("Invalid --suite. Use smoke, tools, quality, or all.");
+    throw new Error("Invalid --suite. Use quick, extensive, or all.");
   }
   return [...new Set(suites)];
 }
@@ -41679,14 +41925,331 @@ function parseRuns(value) {
   }
   return parsed;
 }
-function expandCases(suites, runs) {
-  const selected = BENCHMARK_CASES.filter((benchmarkCase) => suites.includes(benchmarkCase.suite));
+function parseExtensiveSize(value) {
+  if (value === void 0 || value === false) return DEFAULT_EXTENSIVE_SIZE;
+  if (value === true) throw new Error("--extensive-size requires a value");
+  const parsed = Number.parseInt(value, 10);
+  if (![5, 10, 20].includes(parsed)) {
+    throw new Error("--extensive-size must be 5, 10, or 20");
+  }
+  return parsed;
+}
+function parseParallel(value) {
+  if (value === void 0 || value === false) return DEFAULT_PARALLEL;
+  if (value === true) throw new Error("--parallel requires a value");
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 1 || parsed > 20) {
+    throw new Error("--parallel must be an integer from 1 to 20");
+  }
+  return parsed;
+}
+function parseCaseIds(value) {
+  if (value === void 0 || value === false) return [];
+  if (value === true) throw new Error("--case requires a case id");
+  const caseIds = value.split(",").map((caseId) => caseId.trim()).filter(Boolean);
+  if (caseIds.length === 0) throw new Error("--case requires at least one case id");
+  return [...new Set(caseIds)];
+}
+function filterCases(cases, caseIds) {
+  if (caseIds.length === 0) return cases;
+  const availableIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
+  const missing = caseIds.filter((caseId) => !availableIds.has(caseId));
+  if (missing.length > 0) {
+    throw new Error(
+      `Unknown benchmark case id(s): ${missing.join(", ")}. Available in selected suite(s): ${[...availableIds].sort().join(", ")}`
+    );
+  }
+  return cases.filter((benchmarkCase) => caseIds.includes(benchmarkCase.id));
+}
+function expandCases(suites, runs, extensiveSize) {
+  const selected = [];
+  if (suites.includes("quick")) selected.push(...QUICK_CASES);
+  if (suites.includes("extensive")) selected.push(...selectExtensiveCases(extensiveSize));
+  const uniqueSelected = dedupeCases(selected);
   const expanded = [];
   for (let run = 1; run <= runs; run += 1) {
-    for (const benchmarkCase of selected) expanded.push({ ...benchmarkCase, run });
+    for (const benchmarkCase of uniqueSelected) expanded.push({ ...benchmarkCase, run });
   }
   return expanded;
 }
+function selectExtensiveCases(size) {
+  const cases = dedupeCases(EXTENSIVE_CASES).slice(0, size);
+  const minimumCoding = Math.ceil(size * 0.15);
+  const codingCount = cases.filter((benchmarkCase) => benchmarkCase.category === "coding").length;
+  if (codingCount >= minimumCoding) return cases;
+  const selectedIds = new Set(cases.map((benchmarkCase) => benchmarkCase.id));
+  const codingBackfill = EXTENSIVE_CASES.filter(
+    (benchmarkCase) => benchmarkCase.category === "coding" && !selectedIds.has(benchmarkCase.id)
+  );
+  const result = [...cases];
+  for (const codingCase of codingBackfill) {
+    let replaceIndex = -1;
+    for (let index = result.length - 1; index >= 0; index -= 1) {
+      if (result[index]?.category !== "coding") {
+        replaceIndex = index;
+        break;
+      }
+    }
+    if (replaceIndex === -1) break;
+    result[replaceIndex] = codingCase;
+    if (result.filter((benchmarkCase) => benchmarkCase.category === "coding").length >= minimumCoding) break;
+  }
+  return result;
+}
+function dedupeCases(cases) {
+  const seen = /* @__PURE__ */ new Set();
+  const result = [];
+  for (const benchmarkCase of cases) {
+    if (seen.has(benchmarkCase.id)) continue;
+    seen.add(benchmarkCase.id);
+    result.push(benchmarkCase);
+  }
+  return result;
+}
+async function runCaseJob(params) {
+  const { client, job, judgeModel, runId, imagePath } = params;
+  const { model, benchmarkCase } = job;
+  const startedAt = Date.now();
+  const turns = [];
+  const history = benchmarkCase.longContext ? buildLongContextHistory() : [];
+  let chatId;
+  try {
+    const initialPrompt = await buildPromptWithAttachments(client, benchmarkCase, model, imagePath);
+    const targetResponse = await sendBenchmarkTurn({
+      client,
+      model,
+      judgeModel,
+      runId,
+      benchmarkCase,
+      prompt: initialPrompt.message,
+      chatId,
+      history,
+      preparedEmbeds: initialPrompt.embeds,
+      caseId: benchmarkCase.id
+    });
+    chatId = targetResponse.chatId;
+    turns.push(targetResponse.turn);
+    appendHistory(history, "user", initialPrompt.message);
+    appendHistory(history, "assistant", targetResponse.turn.assistant);
+    for (const [index, followUp] of (benchmarkCase.followUps ?? []).entries()) {
+      const response = await sendBenchmarkTurn({
+        client,
+        model,
+        judgeModel,
+        runId,
+        benchmarkCase,
+        prompt: `${modelMention(model)} ${followUp.prompt}`,
+        chatId,
+        history,
+        caseId: `${benchmarkCase.id}:followup-${index + 1}`
+      });
+      chatId = response.chatId;
+      turns.push(response.turn);
+      appendHistory(history, "user", response.rawPrompt);
+      appendHistory(history, "assistant", response.turn.assistant);
+    }
+    const assistant = turns.at(-1)?.assistant ?? "";
+    const caseResult = {
+      id: benchmarkCase.id,
+      suite: benchmarkCase.suite,
+      title: benchmarkCase.title,
+      model,
+      run: benchmarkCase.run,
+      complexity: benchmarkCase.complexity,
+      category: benchmarkCase.category,
+      prompt: benchmarkCase.prompt,
+      assistant,
+      modelName: turns.at(-1)?.modelName ?? null,
+      passed: benchmarkCase.expectedIncludes ? assistant.includes(benchmarkCase.expectedIncludes) : true,
+      durationMs: Date.now() - startedAt,
+      expectedIncludes: benchmarkCase.expectedIncludes,
+      turns
+    };
+    if (benchmarkCase.judge) {
+      caseResult.judge = await judgeCase({ client, judgeModel, targetModel: model, benchmarkCase, caseResult, runId });
+      caseResult.passed = caseResult.judge.score !== null && caseResult.judge.score >= 4 && caseResult.passed;
+    }
+    return caseResult;
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return {
+      id: benchmarkCase.id,
+      suite: benchmarkCase.suite,
+      title: benchmarkCase.title,
+      model,
+      run: benchmarkCase.run,
+      complexity: benchmarkCase.complexity,
+      category: benchmarkCase.category,
+      prompt: benchmarkCase.prompt,
+      assistant: turns.at(-1)?.assistant ?? "",
+      modelName: turns.at(-1)?.modelName ?? null,
+      passed: false,
+      durationMs: Date.now() - startedAt,
+      expectedIncludes: benchmarkCase.expectedIncludes,
+      turns,
+      error: message
+    };
+  }
+}
+async function sendBenchmarkTurn(params) {
+  const startedAt = Date.now();
+  const response = await params.client.sendMessage({
+    message: params.prompt,
+    chatId: params.chatId,
+    incognito: true,
+    autoApproveSubChats: true,
+    benchmarkMetadata: benchmarkMetadata({
+      runId: params.runId,
+      suite: params.benchmarkCase.suite,
+      caseId: params.caseId,
+      targetModel: params.model,
+      judgeModel: params.judgeModel
+    }),
+    messageHistory: params.history,
+    preparedEmbeds: params.preparedEmbeds,
+    precollectResponse: true
+  });
+  return {
+    chatId: response.chatId,
+    rawPrompt: params.prompt,
+    turn: {
+      prompt: params.prompt,
+      assistant: response.assistant,
+      modelName: response.modelName,
+      durationMs: Date.now() - startedAt
+    }
+  };
+}
+async function buildPromptWithAttachments(client, benchmarkCase, model, imagePath) {
+  const baseMessage = `${modelMention(model)} ${benchmarkCase.prompt}`;
+  if (benchmarkCase.image !== "default") return { message: baseMessage };
+  const attachment = await prepareImageAttachment(client, imagePath);
+  return { message: `${baseMessage}
+
+${attachment.messageSuffix}`, embeds: attachment.embeds };
+}
+async function prepareImageAttachment(client, imagePath) {
+  if (!existsSync6(imagePath)) throw new Error(`Benchmark image not found: ${imagePath}`);
+  const processed = processFiles([imagePath], null);
+  if (processed.blocked.length > 0 || processed.errors.length > 0 || processed.embeds.length === 0) {
+    const reason = [...processed.blocked, ...processed.errors].map((entry) => entry.error).join("; ") || "no image embed produced";
+    throw new Error(`Failed to prepare benchmark image: ${reason}`);
+  }
+  const fileEmbed = processed.embeds[0];
+  if (!fileEmbed.requiresUpload || !fileEmbed.localPath) {
+    return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
+  }
+  await uploadBenchmarkImage(client, fileEmbed);
+  return { messageSuffix: fileEmbed.referenceBlock, embeds: [fileEmbed.embed] };
+}
+async function uploadBenchmarkImage(client, fileEmbed) {
+  if (!fileEmbed.localPath) return;
+  const uploadResult = await uploadFile(fileEmbed.localPath, client.getSession());
+  const embedRef = fileEmbed.embed.embedRef ?? `benchmark-image-${uploadResult.embed_id.slice(0, 8)}`;
+  fileEmbed.embed.embedRef = embedRef;
+  fileEmbed.embed.content = toonEncodeContent({
+    type: "image",
+    app_id: "images",
+    skill_id: "upload",
+    status: "finished",
+    filename: fileEmbed.displayName,
+    embed_ref: embedRef,
+    content_hash: uploadResult.content_hash,
+    s3_base_url: uploadResult.s3_base_url,
+    files: uploadResult.files,
+    aes_key: uploadResult.aes_key,
+    aes_nonce: uploadResult.aes_nonce,
+    vault_wrapped_aes_key: uploadResult.vault_wrapped_aes_key,
+    ai_detection: uploadResult.ai_detection
+  });
+  fileEmbed.embed.status = "finished";
+  fileEmbed.embed.contentHash = uploadResult.content_hash;
+  fileEmbed.embed.embedId = uploadResult.embed_id;
+  fileEmbed.referenceBlock = createBenchmarkEmbedReferenceBlock(fileEmbed.embed.embedId, fileEmbed.embed.type);
+}
+function createBenchmarkEmbedReferenceBlock(embedId, embedType) {
+  return `
+
+\`\`\`json
+${JSON.stringify({ type: embedType, embed_id: embedId })}
+\`\`\``;
+}
+async function judgeCase(params) {
+  const startedAt = Date.now();
+  const judgeResponse = await params.client.sendMessage({
+    message: `${modelMention(params.judgeModel)} ${judgePrompt(params.targetModel, params.benchmarkCase, params.caseResult)}`,
+    incognito: true,
+    autoApproveSubChats: true,
+    benchmarkMetadata: benchmarkMetadata({
+      runId: params.runId,
+      suite: params.benchmarkCase.suite,
+      caseId: `${params.benchmarkCase.id}:judge:${params.targetModel}`,
+      targetModel: params.targetModel,
+      judgeModel: params.judgeModel
+    }),
+    precollectResponse: true
+  });
+  const judgment = parseJudgment(judgeResponse.assistant);
+  return {
+    model: params.judgeModel,
+    score: judgment.score,
+    reason: judgment.reason,
+    raw: judgeResponse.assistant,
+    durationMs: Date.now() - startedAt
+  };
+}
+async function runPool(items, parallel, worker) {
+  let index = 0;
+  const workers = Array.from({ length: Math.min(parallel, items.length) }, async () => {
+    while (index < items.length) {
+      const item = items[index];
+      index += 1;
+      await worker(item);
+    }
+  });
+  await Promise.all(workers);
+}
+function buildLongContextHistory() {
+  const now = Math.floor(Date.now() / 1e3) - 2e3;
+  const topics = [
+    ["user", "We need to launch a CLI benchmark for model comparisons."],
+    ["assistant", "The first goal should be a quick suite with deterministic checks."],
+    ["user", "The benchmark also needs image inference."],
+    ["assistant", "Use a public fixture image and ask a factual visual question."],
+    ["user", "We should avoid wasting credits."],
+    ["assistant", "Run a pricing preflight and require explicit spend confirmation."],
+    ["user", "What about longer conversations?"],
+    ["assistant", "Add a 20-message predefined history and a dependent follow-up."],
+    ["user", "The extensive suite should not be too small."],
+    ["assistant", "Default to 10 cases and allow 5 or 20 as alternatives."],
+    ["user", "Coding quality matters."],
+    ["assistant", "Reserve at least 15 percent of extensive cases for coding prompts."],
+    ["user", "We also need comparison mode."],
+    ["assistant", "Accept multiple models with --compare and run target jobs in parallel."],
+    ["user", "How should judging work?"],
+    ["assistant", "Judge each completed case immediately with Gemini so partial results remain useful."],
+    ["user", "What if the process is interrupted?"],
+    ["assistant", "Print or write a partial summary with completed judgments and skipped counts."],
+    ["user", "What is the best launch strategy?"],
+    ["assistant", "Ship quick and comparison first, then use extensive for slower releases."]
+  ];
+  return topics.map(([role, content], index) => ({
+    message_id: `benchmark-history-${index + 1}`,
+    role,
+    sender_name: role === "user" ? "User" : "Assistant",
+    content,
+    created_at: now + index * 30
+  }));
+}
+function appendHistory(history, role, content) {
+  history.push({
+    message_id: randomUUID3(),
+    role,
+    sender_name: role === "user" ? "User" : "Assistant",
+    content,
+    created_at: Math.floor(Date.now() / 1e3)
+  });
+}
 function modelMention(model) {
   const separator = model.indexOf("/");
   if (separator === -1) return `@ai-model:${model}`;
@@ -41705,16 +42268,28 @@ function benchmarkMetadata(params) {
     benchmark_judge_model: params.judgeModel
   };
 }
-function judgePrompt(prompt, answer) {
+function judgePrompt(targetModel, benchmarkCase, result) {
   return [
-    "You are judging a model benchmark response.",
-    'Return only JSON with shape {"score": number, "reason": string}.',
-    "Score from 1 to 5 for correctness, clarity, and following instructions.",
-    `Benchmark prompt: ${JSON.stringify(prompt)}`,
-    `Candidate answer: ${JSON.stringify(answer)}`
+    "You are judging a real OpenMates model benchmark response.",
+    "Return exactly two plain-text lines, with no markdown, no code block, and no tool use.",
+    "Line 1 format: BENCHMARK_SCORE=<integer from 1 to 5>",
+    "Line 2 format: BENCHMARK_REASON=<one short sentence>",
+    "Score for correctness, instruction-following, usefulness, and continuity where relevant.",
+    `Target model: ${targetModel}`,
+    `Benchmark case: ${benchmarkCase.id} (${benchmarkCase.category}, ${benchmarkCase.complexity})`,
+    `Initial prompt: ${JSON.stringify(benchmarkCase.prompt)}`,
+    `Turns: ${JSON.stringify(result.turns.map((turn) => ({ prompt: turn.prompt, assistant: turn.assistant })))}`
   ].join("\n");
 }
 function parseJudgment(answer) {
+  const markerScore = answer.match(/BENCHMARK_SCORE\s*=\s*([1-5])/i);
+  if (markerScore) {
+    const reasonMatch = answer.match(/BENCHMARK_REASON\s*=\s*(.+)/i);
+    return {
+      score: Number.parseInt(markerScore[1], 10),
+      reason: reasonMatch?.[1]?.trim() ?? null
+    };
+  }
   const jsonText = extractJsonObject(answer);
   if (!jsonText) return { score: null, reason: null };
   try {
@@ -41734,6 +42309,220 @@ function extractJsonObject(text) {
   if (start === -1 || end === -1 || end <= start) return null;
   return text.slice(start, end + 1);
 }
+function loadPricingForModels(models) {
+  const availablePricing = loadProviderPricing();
+  const pricing = /* @__PURE__ */ new Map();
+  const missing = [];
+  for (const model of [...new Set(models)]) {
+    const key = normalizeModelKey(model);
+    const modelPricing = availablePricing.get(key);
+    if (!modelPricing) {
+      missing.push(model);
+      continue;
+    }
+    pricing.set(model, modelPricing);
+  }
+  if (missing.length > 0) {
+    throw new Error(
+      `Cannot estimate benchmark cost because pricing metadata is unavailable for: ${missing.join(", ")}. Use provider/model ids with backend provider pricing metadata.`
+    );
+  }
+  return pricing;
+}
+function loadProviderPricing() {
+  const providersDir = findProvidersDir();
+  const pricing = /* @__PURE__ */ new Map();
+  if (!providersDir) return pricing;
+  for (const fileName of readdirSync(providersDir)) {
+    if (!fileName.endsWith(".yml")) continue;
+    const filePath = join4(providersDir, fileName);
+    const text = readFileSync6(filePath, "utf-8");
+    const provider = parseProviderId(text) ?? fileName.replace(/\.yml$/, "");
+    for (const modelPricing of parseModelPricing(text, provider)) {
+      pricing.set(`${modelPricing.provider}/${modelPricing.modelId}`, modelPricing);
+      pricing.set(modelPricing.modelId, modelPricing);
+    }
+  }
+  return pricing;
+}
+function parseProviderId(text) {
+  const match = text.match(/^provider_id:\s*["']?([^"'\n]+)["']?/m);
+  return match?.[1]?.trim() ?? null;
+}
+function parseModelPricing(text, provider) {
+  const lines = text.split("\n");
+  const results = [];
+  let modelId = null;
+  let inModel = false;
+  let inputTokensPerCredit = null;
+  let outputTokensPerCredit = null;
+  for (const line of lines) {
+    const modelMatch = line.match(/^\s{2}-\s+id:\s*["']?([^"'\n#]+)["']?/);
+    if (modelMatch) {
+      if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
+        results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
+      }
+      inModel = true;
+      modelId = modelMatch[1].trim();
+      inputTokensPerCredit = null;
+      outputTokensPerCredit = null;
+      continue;
+    }
+    if (!inModel) continue;
+    const inputMatch = line.match(/^\s{10}per_credit_unit:\s*(\d+)/);
+    if (inputMatch && inputTokensPerCredit === null) {
+      inputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
+      continue;
+    }
+    if (inputMatch && inputTokensPerCredit !== null && outputTokensPerCredit === null) {
+      outputTokensPerCredit = Number.parseInt(inputMatch[1], 10);
+    }
+  }
+  if (inModel && modelId && inputTokensPerCredit && outputTokensPerCredit) {
+    results.push({ provider, modelId, inputTokensPerCredit, outputTokensPerCredit });
+  }
+  return results;
+}
+function normalizeModelKey(model) {
+  return model.includes("/") ? model : model;
+}
+function findProvidersDir() {
+  const currentFile = fileURLToPath(import.meta.url);
+  let current = dirname(currentFile);
+  for (let index = 0; index < 8; index += 1) {
+    const candidate = join4(current, "backend", "providers");
+    if (existsSync6(candidate)) return candidate;
+    const parentCandidate = join4(current, "..", "..", "backend", "providers");
+    if (existsSync6(parentCandidate)) return resolve5(parentCandidate);
+    const next = dirname(current);
+    if (next === current) break;
+    current = next;
+  }
+  return null;
+}
+function estimateCredits(cases, targetModels, judgeModel, pricing) {
+  let targetCredits = 0;
+  let judgeCredits = 0;
+  let targetInputTokens = 0;
+  let targetOutputTokens = 0;
+  let judgeInputTokens = 0;
+  let judgeOutputTokens = 0;
+  for (const benchmarkCase of cases) {
+    const turnCount = 1 + (benchmarkCase.followUps?.length ?? 0);
+    for (const model of targetModels) {
+      const modelPricing = pricing.get(model);
+      if (!modelPricing) continue;
+      const input = benchmarkCase.estimatedInputTokens * turnCount;
+      const output = benchmarkCase.estimatedOutputTokens * turnCount;
+      targetInputTokens += input;
+      targetOutputTokens += output;
+      targetCredits += creditsFor(modelPricing, input, output);
+      if (benchmarkCase.judge) {
+        const judgePricing = pricing.get(judgeModel);
+        if (!judgePricing) continue;
+        const judgeInput = Math.max(2e3, Math.ceil(output * 1.5));
+        const judgeOutput = 350;
+        judgeInputTokens += judgeInput;
+        judgeOutputTokens += judgeOutput;
+        judgeCredits += creditsFor(judgePricing, judgeInput, judgeOutput);
+      }
+    }
+  }
+  return {
+    targetCredits,
+    judgeCredits,
+    totalCredits: targetCredits + judgeCredits,
+    assumptions: { targetInputTokens, targetOutputTokens, judgeInputTokens, judgeOutputTokens }
+  };
+}
+function creditsFor(pricing, inputTokens, outputTokens) {
+  return Math.ceil(inputTokens / pricing.inputTokensPerCredit) + Math.ceil(outputTokens / pricing.outputTokensPerCredit);
+}
+function makeBaseResult(params) {
+  return {
+    command: "benchmark model",
+    status: params.dryRun ? "planned" : "completed",
+    runId: params.runId,
+    targetModel: params.targetModels[0],
+    targetModels: params.targetModels,
+    judgeModel: params.judgeModel,
+    suites: params.suites,
+    runs: params.runs,
+    compare: params.compare,
+    parallel: params.parallel,
+    extensiveSize: params.extensiveSize,
+    spendsCredits: !params.dryRun,
+    estimatedCredits: params.estimate,
+    cases: [],
+    modelSummaries: params.targetModels.map((model) => ({
+      model,
+      total: 0,
+      passed: 0,
+      failed: 0,
+      averageJudgeScore: null,
+      averageDurationMs: null
+    })),
+    summary: {
+      total: params.totalJobs,
+      completed: 0,
+      passed: 0,
+      failed: 0,
+      skipped: params.dryRun ? params.totalJobs : 0,
+      interrupted: false
+    }
+  };
+}
+function recomputeResult(result, totalJobs, interrupted) {
+  const completed = result.cases.length;
+  const passed = result.cases.filter((caseResult) => caseResult.passed).length;
+  const failed = result.cases.filter((caseResult) => !caseResult.passed).length;
+  result.summary = {
+    total: totalJobs,
+    completed,
+    passed,
+    failed,
+    skipped: Math.max(0, totalJobs - completed),
+    interrupted
+  };
+  result.status = interrupted || completed < totalJobs ? "partial" : "completed";
+  result.modelSummaries = result.targetModels.map((model) => summarizeModel(model, result.cases));
+  if (result.compare) result.comparison = buildComparison(result.modelSummaries);
+}
+function summarizeModel(model, cases) {
+  const modelCases = cases.filter((caseResult) => caseResult.model === model);
+  const scores = modelCases.map((caseResult) => caseResult.judge?.score).filter((score) => typeof score === "number" && Number.isFinite(score));
+  const durations = modelCases.map((caseResult) => caseResult.durationMs).filter((value) => value > 0);
+  return {
+    model,
+    total: modelCases.length,
+    passed: modelCases.filter((caseResult) => caseResult.passed).length,
+    failed: modelCases.filter((caseResult) => !caseResult.passed).length,
+    averageJudgeScore: scores.length > 0 ? round2(scores.reduce((sum, score) => sum + score, 0) / scores.length) : null,
+    averageDurationMs: durations.length > 0 ? Math.round(durations.reduce((sum, value) => sum + value, 0) / durations.length) : null
+  };
+}
+function buildComparison(summaries) {
+  const ranking = [...summaries].sort((a, b) => (b.averageJudgeScore ?? -1) - (a.averageJudgeScore ?? -1) || b.passed - a.passed).map((summary) => ({
+    model: summary.model,
+    averageJudgeScore: summary.averageJudgeScore,
+    passed: summary.passed,
+    total: summary.total
+  }));
+  const notes = ranking.length > 0 ? [`Top model so far: ${ranking[0].model} (${ranking[0].passed}/${ranking[0].total} passed).`] : [];
+  return { ranking, notes };
+}
+function round2(value) {
+  return Math.round(value * 100) / 100;
+}
+function defaultImageFixturePath() {
+  const fixtureDir = join4(dirname(fileURLToPath(import.meta.url)), "..", "fixtures");
+  const fixturePath = join4(fixtureDir, "brandenburger-tor.png");
+  if (existsSync6(fixturePath)) return fixturePath;
+  const tempDir = mkdtempSync(join4(tmpdir(), "openmates-benchmark-"));
+  const tempPath = join4(tempDir, "brandenburger-tor.svg");
+  writeFileSync4(tempPath, FIXTURE_IMAGE_SVG, "utf-8");
+  return tempPath;
+}
 function writeBenchmarkResult(result, flags, output) {
   const json = `${JSON.stringify(result, null, 2)}
 `;
@@ -41742,17 +42531,19 @@ function writeBenchmarkResult(result, flags, output) {
     process.stdout.write(json);
     return;
   }
-  console.log(`Benchmark ${result.status}: ${result.targetModel}`);
+  console.log(`Benchmark ${result.status}: ${result.targetModels.join(", ")}`);
   console.log(`Run ID: ${result.runId}`);
   console.log(`Suites: ${result.suites.join(", ")}`);
   console.log(`Judge: ${result.judgeModel}`);
+  console.log(`Estimated credits: ${result.estimatedCredits.totalCredits}`);
   console.log(`Spend credits: ${result.spendsCredits ? "yes" : "no"}`);
-  if (result.status === "completed") {
-    console.log(`Passed: ${result.summary.passed}/${result.summary.total}`);
+  if (result.status !== "planned") {
+    console.log(`Passed: ${result.summary.passed}/${result.summary.completed} completed (${result.summary.skipped} skipped)`);
     for (const benchmarkCase of result.cases) {
       const mark = benchmarkCase.passed ? "PASS" : "FAIL";
-      const judge = benchmarkCase.judge?.score !== void 0 ? ` judge=${benchmarkCase.judge.score ?? "unparsed"}` : "";
-      console.log(`${mark} ${benchmarkCase.suite}/${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${judge}`);
+      const judge = benchmarkCase.judge ? ` judge=${benchmarkCase.judge.score ?? "unparsed"}` : "";
+      const error = benchmarkCase.error ? ` error=${benchmarkCase.error}` : "";
+      console.log(`${mark} ${benchmarkCase.model} ${benchmarkCase.suite}/${benchmarkCase.id} (${benchmarkCase.durationMs}ms)${judge}${error}`);
     }
   }
 }
@@ -42141,10 +42932,10 @@ Run 'openmates chats show ` + chatId + "' to check if suggestions have been save
         input: process.stdin,
         output: process.stdout
       });
-      const answer = await new Promise((resolve5) => {
+      const answer = await new Promise((resolve6) => {
         iface.question(
           `Delete ${resolved.length} chat(s)? This cannot be undone. [y/N] `,
-          resolve5
+          resolve6
         );
       });
       iface.close();
@@ -42304,16 +43095,16 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
       }
     }
     const { mkdir, writeFile } = await import("fs/promises");
-    const { join: join4 } = await import("path");
+    const { join: join5 } = await import("path");
     if (useZip) {
-      const tmpDir = join4(outputDir, `.${filenameBase}_tmp`);
+      const tmpDir = join5(outputDir, `.${filenameBase}_tmp`);
       await mkdir(tmpDir, { recursive: true });
-      await writeFile(join4(tmpDir, `${filenameBase}.yml`), yamlContent);
-      await writeFile(join4(tmpDir, `${filenameBase}.md`), mdContent);
+      await writeFile(join5(tmpDir, `${filenameBase}.yml`), yamlContent);
+      await writeFile(join5(tmpDir, `${filenameBase}.md`), mdContent);
       if (codeEmbeds.length > 0) {
         for (const ce of codeEmbeds) {
           const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
-          const fullPath = join4(tmpDir, "code", fpath);
+          const fullPath = join5(tmpDir, "code", fpath);
           await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
             recursive: true
           });
@@ -42321,13 +43112,13 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
         }
       }
       if (transcriptEmbeds.length > 0) {
-        const tDir = join4(tmpDir, "transcripts");
+        const tDir = join5(tmpDir, "transcripts");
         await mkdir(tDir, { recursive: true });
         for (const te of transcriptEmbeds) {
-          await writeFile(join4(tDir, te.filename), te.content);
+          await writeFile(join5(tDir, te.filename), te.content);
         }
       }
-      const zipPath = join4(outputDir, `${filenameBase}.zip`);
+      const zipPath = join5(outputDir, `${filenameBase}.zip`);
       const { execSync: execSync2 } = await import("child_process");
       try {
         execSync2(`cd "${tmpDir}" && zip -r "${zipPath}" .`, { stdio: "pipe" });
@@ -42342,17 +43133,17 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
         );
       }
     } else {
-      const chatDir = join4(outputDir, filenameBase);
+      const chatDir = join5(outputDir, filenameBase);
       await mkdir(chatDir, { recursive: true });
       const written = [];
-      await writeFile(join4(chatDir, `${filenameBase}.yml`), yamlContent);
+      await writeFile(join5(chatDir, `${filenameBase}.yml`), yamlContent);
       written.push(`${filenameBase}.yml`);
-      await writeFile(join4(chatDir, `${filenameBase}.md`), mdContent);
+      await writeFile(join5(chatDir, `${filenameBase}.md`), mdContent);
       written.push(`${filenameBase}.md`);
       if (codeEmbeds.length > 0) {
         for (const ce of codeEmbeds) {
           const fpath = ce.filePath ?? ce.filename ?? `${ce.embedId.slice(0, 8)}.${getExtForLang(ce.language)}`;
-          const fullPath = join4(chatDir, "code", fpath);
+          const fullPath = join5(chatDir, "code", fpath);
           await mkdir(fullPath.substring(0, fullPath.lastIndexOf("/")), {
             recursive: true
           });
@@ -42361,10 +43152,10 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
         }
       }
       if (transcriptEmbeds.length > 0) {
-        const tDir = join4(chatDir, "transcripts");
+        const tDir = join5(chatDir, "transcripts");
         await mkdir(tDir, { recursive: true });
         for (const te of transcriptEmbeds) {
-          await writeFile(join4(tDir, te.filename), te.content);
+          await writeFile(join5(tDir, te.filename), te.content);
           written.push(`transcripts/${te.filename}`);
         }
       }
@@ -42400,7 +43191,7 @@ ${deleted}/${resolved.length} chat(s) deleted.`);
       printJson2({
         chat_id: chat.id,
         title: chat.title,
-        output_dir: useZip ? join4(outputDir, `${filenameBase}.zip`) : join4(outputDir, filenameBase),
+        output_dir: useZip ? join5(outputDir, `${filenameBase}.zip`) : join5(outputDir, filenameBase),
         files,
         code_embeds: codeEmbeds.length,
         transcript_embeds: transcriptEmbeds.length
@@ -42921,7 +43712,7 @@ async function handleCodeRun(client, flags, apiKey) {
   }
 }
 async function streamCodeRunToTerminal(url, jsonMode) {
-  return await new Promise((resolve5, reject) => {
+  return await new Promise((resolve6, reject) => {
     const ws = new WebSocket2(url);
     let lastStatus = {};
     ws.on("message", (data) => {
@@ -42940,7 +43731,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
           const status = String(payload.status ?? "");
           if (["finished", "failed", "timeout", "cancelled"].includes(status)) {
             ws.close();
-            resolve5(lastStatus);
+            resolve6(lastStatus);
           }
         }
       } catch (err) {
@@ -42950,7 +43741,7 @@ async function streamCodeRunToTerminal(url, jsonMode) {
     });
     ws.on("error", () => reject(new Error("Code Run stream failed.")));
     ws.on("close", () => {
-      if (Object.keys(lastStatus).length > 0) resolve5(lastStatus);
+      if (Object.keys(lastStatus).length > 0) resolve6(lastStatus);
     });
   });
 }
@@ -42961,7 +43752,7 @@ async function pollCodeRunStatus(client, statusPath, apiKey, jsonMode) {
     if (!jsonMode && value) process.stderr.write(`Code Run status: ${value}
 `);
     if (["finished", "failed", "timeout", "cancelled"].includes(value)) return status;
-    await new Promise((resolve5) => setTimeout(resolve5, 1e3));
+    await new Promise((resolve6) => setTimeout(resolve6, 1e3));
   }
 }
 function buildSkillInput(flags, inlineTokens, schemaParams) {
@@ -43445,11 +44236,11 @@ function parseYamlScalar(value) {
 }
 async function saveDownloadedDocument(document, output) {
   const { mkdir, writeFile } = await import("fs/promises");
-  const { join: join4, basename: basename4, dirname: dirname2 } = await import("path");
+  const { join: join5, basename: basename4, dirname: dirname3 } = await import("path");
   const target = typeof output === "string" ? output : ".";
   const filename = basename4(document.filename || "document.pdf");
-  const filePath = target.endsWith(".pdf") ? target : join4(target, filename);
-  await mkdir(dirname2(filePath), { recursive: true });
+  const filePath = target.endsWith(".pdf") ? target : join5(target, filename);
+  await mkdir(dirname3(filePath), { recursive: true });
   await writeFile(filePath, document.data);
   return filePath;
 }
@@ -43477,7 +44268,7 @@ function printMateInfo(mateId, json) {
 async function confirmOrExit(question) {
   const rl = await import("readline");
   const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
-  const answer = await new Promise((resolve5) => iface.question(question, resolve5));
+  const answer = await new Promise((resolve6) => iface.question(question, resolve6));
   iface.close();
   if (answer.trim().toLowerCase() !== "y") {
     console.log("Aborted.");
@@ -43487,7 +44278,7 @@ async function confirmOrExit(question) {
 async function promptLine(question) {
   const rl = await import("readline");
   const iface = rl.createInterface({ input: process.stdin, output: process.stdout });
-  const answer = await new Promise((resolve5) => iface.question(question, resolve5));
+  const answer = await new Promise((resolve6) => iface.question(question, resolve6));
   iface.close();
   return answer.trim();
 }
@@ -43495,7 +44286,7 @@ async function promptSecret(question) {
   if (!process.stdin.isTTY) {
     return promptLine(question);
   }
-  return new Promise((resolve5) => {
+  return new Promise((resolve6) => {
     const stdin2 = process.stdin;
     const wasRaw = stdin2.isRaw;
     let value = "";
@@ -43508,7 +44299,7 @@ async function promptSecret(question) {
         stdin2.off("data", onData);
         stdin2.setRawMode(wasRaw);
         process.stdout.write("\n");
-        resolve5(value);
+        resolve6(value);
         return;
       }
       if (char === "") {
@@ -43528,7 +44319,7 @@ async function promptSecret(question) {
 }
 async function writeSecretFile(filePath, content, force = false) {
   const { mkdir, writeFile, stat: stat2 } = await import("fs/promises");
-  const { dirname: dirname2 } = await import("path");
+  const { dirname: dirname3 } = await import("path");
   try {
     await stat2(filePath);
     if (!force) throw new Error(`${filePath} already exists. Use --force to overwrite.`);
@@ -43538,7 +44329,7 @@ async function writeSecretFile(filePath, content, force = false) {
     }
     if (error instanceof Error && !("code" in error)) throw error;
   }
-  await mkdir(dirname2(filePath), { recursive: true });
+  await mkdir(dirname3(filePath), { recursive: true });
   await writeFile(filePath, content, { mode: 384 });
   return filePath;
 }
@@ -46496,7 +47287,7 @@ async function handleDocs(client, subcommand, rest, flags) {
   }
   if (subcommand === "download") {
     const { writeFile, mkdir } = await import("fs/promises");
-    const { join: join4, dirname: dirname2 } = await import("path");
+    const { join: join5, dirname: dirname3 } = await import("path");
     if (flags.all === true) {
       const outputDir = typeof flags.output === "string" ? flags.output : "./openmates-docs";
       const tree = await client.listDocs();
@@ -46505,8 +47296,8 @@ async function handleDocs(client, subcommand, rest, flags) {
       let count = 0;
       for (const slug2 of slugs) {
         const content2 = await client.getDoc(slug2);
-        const filePath = join4(outputDir, `${slug2}.md`);
-        await mkdir(dirname2(filePath), { recursive: true });
+        const filePath = join5(outputDir, `${slug2}.md`);
+        await mkdir(dirname3(filePath), { recursive: true });
         await writeFile(filePath, content2, "utf-8");
         count++;
         process.stderr.write(`\r  Downloaded ${count}/${slugs.length}`);
@@ -46578,8 +47369,8 @@ function isCliEntrypoint() {
   if (!entrypoint) return false;
   try {
     const invokedPath = realpathSync(entrypoint);
-    const modulePath = realpathSync(fileURLToPath(import.meta.url));
-    return invokedPath === modulePath || basename3(invokedPath) === "cli.js" && dirname(invokedPath) === dirname(modulePath);
+    const modulePath = realpathSync(fileURLToPath2(import.meta.url));
+    return invokedPath === modulePath || basename3(invokedPath) === "cli.js" && dirname2(invokedPath) === dirname2(modulePath);
   } catch {
     return false;
   }