agent-duelist 0.1.2 → 0.2.1

package/dist/cli.js

@@ -38,343 +38,6 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   mod
 ));
 
-// src/reporter/console.ts
-var console_exports = {};
-__export(console_exports, {
-  consoleReporter: () => consoleReporter
-});
-function bold(s5) {
-  return `${boldCode}${s5}${reset}`;
-}
-function dim(s5) {
-  return `${dimCode}${s5}${reset}`;
-}
-function colorScore(value) {
-  const pct = Math.round(value * 100);
-  const str = `${pct}%`;
-  if (value >= 0.8) return `${green}${str}${reset}`;
-  if (value >= 0.5) return `${yellow}${str}${reset}`;
-  return `${red}${str}${reset}`;
-}
-function consoleReporter(results) {
-  if (results.length === 0) {
-    console.log("\nNo results to display.\n");
-    return;
-  }
-  const tasks = [...new Set(results.map((r3) => r3.taskName))];
-  const providers = [...new Set(results.map((r3) => r3.providerId))];
-  const scorerNames = [...new Set(results.flatMap((r3) => r3.scores.map((s5) => s5.name)))];
-  const hasCost = scorerNames.includes("cost");
-  const hasErrors = results.some((r3) => r3.error);
-  const runsPerCell = Math.max(...results.map((r3) => r3.run));
-  const runLabel = runsPerCell > 1 ? ` (${runsPerCell} runs each)` : "";
-  console.log("");
-  console.log(`  ${bold(`\u2B21 Agent Duelist Results${runLabel}`)}`);
-  console.log(`  ${dim("\u2500".repeat(70))}`);
-  console.log("");
-  for (const task of tasks) {
-    console.log(`  ${bold(`Task: ${task}`)}`);
-    const cols = [{ label: "Provider", width: 22, align: "left" }];
-    for (const name of scorerNames) {
-      if (name === "latency") cols.push({ label: "Latency", width: 10, align: "right" });
-      else if (name === "cost") {
-        cols.push({ label: "Cost", width: 12, align: "right" });
-        cols.push({ label: "Tokens", width: 9, align: "right" });
-      } else if (name === "correctness") cols.push({ label: "Match", width: 8, align: "right" });
-      else if (name === "schema-correctness") cols.push({ label: "Schema", width: 8, align: "right" });
-      else if (name === "fuzzy-similarity") cols.push({ label: "Fuzzy", width: 8, align: "right" });
-      else if (name === "llm-judge-correctness") cols.push({ label: "Judge", width: 8, align: "right" });
-      else if (name === "tool-usage") cols.push({ label: "Tool", width: 8, align: "right" });
-      else cols.push({ label: name, width: 10, align: "right" });
-    }
-    if (hasErrors) cols.push({ label: "Status", width: 8, align: "left" });
-    const totalWidth = cols.reduce((sum, c3) => sum + c3.width + 2, 0);
-    console.log(`  ${dim(cols.map((c3) => pad(c3.label, c3.width + 2, c3.align)).join(""))}`);
-    console.log(`  ${dim("\u2500".repeat(totalWidth))}`);
-    for (const provider of providers) {
-      const taskResults = results.filter(
-        (r3) => r3.taskName === task && r3.providerId === provider
-      );
-      const errorResults2 = taskResults.filter((r3) => r3.error);
-      const successResults = taskResults.filter((r3) => !r3.error);
-      if (successResults.length === 0 && errorResults2.length > 0) {
-        const cells2 = [pad(provider, 24, "left")];
-        for (const name of scorerNames) {
-          if (name === "cost") {
-            cells2.push(pad("\u2014", 14, "right"));
-            cells2.push(pad("\u2014", 11, "right"));
-          } else cells2.push(pad("\u2014", cols.find((c3) => c3.label !== "Provider").width + 2, "right"));
-        }
-        if (hasErrors) cells2.push(`  ${red}FAIL${reset}`);
-        console.log(`  ${cells2.join("")}`);
-        continue;
-      }
-      const avgScores = averageScores(successResults);
-      const avgDetails = averageDetails(successResults);
-      const latencyMs = average(successResults.map((r3) => r3.raw.latencyMs));
-      const cells = [pad(provider, 24, "left")];
-      for (const name of scorerNames) {
-        if (name === "latency") {
-          cells.push(pad(latencyMs !== void 0 ? `${Math.round(latencyMs)}ms` : "\u2014", 12, "right"));
-        } else if (name === "cost") {
-          cells.push(pad(formatCost(avgDetails.costUsd), 14, "right"));
-          cells.push(pad(avgDetails.totalTokens !== void 0 ? `${avgDetails.totalTokens}` : "\u2014", 11, "right"));
-        } else {
-          const val = avgScores[name];
-          if (val === void 0) cells.push(pad("\u2014", 10, "right"));
-          else cells.push(pad(colorScore(val), 10 + colorLen(colorScore(val)), "right"));
-        }
-      }
-      if (hasErrors) {
-        const failCount = errorResults2.length;
-        cells.push(failCount > 0 ? `  ${yellow}${failCount} err${reset}` : `  ${green}OK${reset}`);
-      }
-      console.log(`  ${cells.join("")}`);
-    }
-    console.log("");
-  }
-  printSummary(results, providers);
-  const errorResults = results.filter((r3) => r3.error);
-  if (errorResults.length > 0) {
-    console.log(`  ${bold("Errors")}`);
-    console.log(`  ${dim("\u2500".repeat(70))}`);
-    const seen = /* @__PURE__ */ new Set();
-    for (const r3 of errorResults) {
-      const key = `${r3.providerId}::${r3.error}`;
-      if (seen.has(key)) continue;
-      seen.add(key);
-      const count = errorResults.filter((e5) => e5.providerId === r3.providerId && e5.error === r3.error).length;
-      const suffix = count > 1 ? ` (\xD7${count})` : "";
-      console.log(`  ${red}\u2717${reset} ${r3.providerId}: ${r3.error}${suffix}`);
-      const hint = apiKeyHint(r3.providerId, r3.error ?? "");
-      if (hint) console.log(`    ${dim(hint)}`);
-    }
-    console.log("");
-  }
-  if (hasCost) {
-    console.log(dim(`  Costs estimated from OpenRouter pricing catalog. Run npx tsx scripts/update-pricing.ts to refresh.`));
-    console.log("");
-  }
-}
-function printSummary(results, providers) {
-  const successResults = results.filter((r3) => !r3.error);
-  if (successResults.length === 0) return;
-  console.log(`  ${dim("\u2500".repeat(70))}`);
-  console.log(`  ${bold("Summary")}`);
-  console.log("");
-  const single = providers.length === 1;
-  const correctnessKey = successResults.some((r3) => r3.scores.some((s5) => s5.name === "llm-judge-correctness" && s5.value >= 0)) ? "llm-judge-correctness" : "correctness";
-  const byCorrectness = rankProviders(successResults, providers, correctnessKey);
-  if (byCorrectness) {
-    const label = single ? "Avg correctness" : `Most correct: ${bold(byCorrectness.id)} ${dim(providerLabel(byCorrectness.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${colorScore(byCorrectness.avg)})`);
-  }
-  const byLatency = providers.map((id) => {
-    const runs = successResults.filter((r3) => r3.providerId === id);
-    const avg = average(runs.map((r3) => r3.raw.latencyMs));
-    return { id, avg: avg ?? Infinity };
-  }).sort((a7, b3) => a7.avg - b3.avg)[0];
-  if (byLatency && byLatency.avg !== Infinity) {
-    const label = single ? "Avg latency" : `Fastest: ${bold(byLatency.id)} ${dim(providerLabel(byLatency.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${Math.round(byLatency.avg)}ms)`);
-  }
-  const byCost = providers.map((id) => {
-    const runs = successResults.filter((r3) => r3.providerId === id);
-    const costs = runs.map((r3) => {
-      const s5 = r3.scores.find((s6) => s6.name === "cost");
-      return s5 && s5.value >= 0 ? s5.value : void 0;
-    }).filter((c3) => c3 !== void 0);
-    const avg = costs.length > 0 ? costs.reduce((a7, b3) => a7 + b3, 0) / costs.length : void 0;
-    return { id, avg };
-  }).filter((p5) => p5.avg !== void 0).sort((a7, b3) => a7.avg - b3.avg)[0];
-  if (byCost?.avg !== void 0) {
-    const label = single ? "Avg cost" : `Cheapest: ${bold(byCost.id)} ${dim(providerLabel(byCost.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${formatCost(byCost.avg)})`);
-  }
-  console.log("");
-}
-function rankProviders(results, providers, scorerName) {
-  const ranked = providers.map((id) => {
-    const runs = results.filter((r3) => r3.providerId === id);
-    const scores = runs.flatMap((r3) => r3.scores.filter((s5) => s5.name === scorerName && s5.value >= 0)).map((s5) => s5.value);
-    const avg = scores.length > 0 ? scores.reduce((a7, b3) => a7 + b3, 0) / scores.length : void 0;
-    return { id, avg };
-  }).filter((p5) => p5.avg !== void 0).sort((a7, b3) => b3.avg - a7.avg);
-  return ranked[0] ? { id: ranked[0].id, avg: ranked[0].avg } : void 0;
-}
-function averageScores(results) {
-  const sums = {};
-  const counts = {};
-  for (const result of results) {
-    for (const score of result.scores) {
-      if (score.value < 0) continue;
-      sums[score.name] = (sums[score.name] ?? 0) + score.value;
-      counts[score.name] = (counts[score.name] ?? 0) + 1;
-    }
-  }
-  const avgs = {};
-  for (const name of Object.keys(sums)) {
-    avgs[name] = sums[name] / counts[name];
-  }
-  return avgs;
-}
-function averageDetails(results) {
-  let costSum = 0;
-  let costCount = 0;
-  let tokenSum = 0;
-  let tokenCount = 0;
-  for (const result of results) {
-    const costScore = result.scores.find((s5) => s5.name === "cost");
-    const details = costScore?.details;
-    if (details?.estimatedUsd != null) {
-      costSum += details.estimatedUsd;
-      costCount++;
-    }
-    if (details?.totalTokens != null) {
-      tokenSum += details.totalTokens;
-      tokenCount++;
-    }
-  }
-  return {
-    costUsd: costCount > 0 ? costSum / costCount : void 0,
-    totalTokens: tokenCount > 0 ? Math.round(tokenSum / tokenCount) : void 0
-  };
-}
-function average(nums) {
-  if (nums.length === 0) return void 0;
-  return nums.reduce((a7, b3) => a7 + b3, 0) / nums.length;
-}
-function formatCost(usd) {
-  if (usd === void 0) return "\u2014";
-  if (usd === 0) return "$0.00";
-  if (usd >= 0.01) return `~$${usd.toFixed(2)}`;
-  const digits = Math.max(4, -Math.floor(Math.log10(usd)) + 1);
-  return `~$${usd.toFixed(digits).replace(/0+$/, "")}`;
-}
-function pad(str, width, align) {
-  if (align === "right") return str.padStart(width);
-  return str.padEnd(width);
-}
-function colorLen(str) {
-  const stripped = str.replace(/\x1b\[[0-9;]*m/g, "");
-  return str.length - stripped.length;
-}
-function apiKeyHint(providerId, error) {
-  const lower = error.toLowerCase();
-  const isAuthError = lower.includes("api key") || lower.includes("401") || lower.includes("unauthorized") || lower.includes("authentication") || lower.includes("incorrect api key") || lower.includes("apikey");
-  if (!isAuthError) return void 0;
-  const prefix = providerId.split("/")[0];
-  switch (prefix) {
-    case "openai":
-      return "Set: export OPENAI_API_KEY=sk-...";
-    case "azure":
-      return "Set: export AZURE_OPENAI_API_KEY=... and AZURE_OPENAI_ENDPOINT=...";
-    case "anthropic":
-      return "Set: export ANTHROPIC_API_KEY=sk-ant-...";
-    case "google":
-      return "Set: export GOOGLE_API_KEY=...";
-    default:
-      return `Check the API key for ${providerId}`;
-  }
-}
-function providerLabel(providerId) {
-  const prefix = providerId.split("/")[0];
-  switch (prefix) {
-    case "azure":
-      return "(OpenAI via Azure)";
-    case "openai":
-      return "(OpenAI)";
-    case "anthropic":
-      return "(Anthropic)";
-    case "google":
-      return "(Google)";
-    case "mistral":
-      return "(Mistral)";
-    case "meta":
-      return "(Meta)";
-    case "deepseek":
-      return "(DeepSeek)";
-    case "cohere":
-      return "(Cohere)";
-    case "qwen":
-      return "(Qwen)";
-    case "xai":
-      return "(xAI)";
-    case "minimax":
-      return "(MiniMax)";
-    case "moonshot":
-      return "(Moonshot / Kimi)";
-    case "perplexity":
-      return "(Perplexity)";
-    case "amazon":
-      return "(Amazon)";
-    case "nvidia":
-      return "(NVIDIA)";
-    case "microsoft":
-      return "(Microsoft)";
-    case "ai21":
-      return "(AI21 Labs)";
-    case "bytedance":
-      return "(ByteDance)";
-    case "together":
-      return "(Together AI)";
-    case "fireworks":
-      return "(Fireworks AI)";
-    case "groq":
-      return "(Groq)";
-    case "cerebras":
-      return "(Cerebras)";
-    default:
-      return `(${prefix})`;
-  }
-}
-var reset, boldCode, dimCode, green, red, yellow, cyan;
-var init_console = __esm({
-  "src/reporter/console.ts"() {
-    "use strict";
-    reset = "\x1B[0m";
-    boldCode = "\x1B[1m";
-    dimCode = "\x1B[2m";
-    green = "\x1B[32m";
-    red = "\x1B[31m";
-    yellow = "\x1B[33m";
-    cyan = "\x1B[36m";
-  }
-});
-
-// src/reporter/json.ts
-var json_exports = {};
-__export(json_exports, {
-  jsonReporter: () => jsonReporter
-});
-function jsonReporter(results) {
-  return JSON.stringify(
-    {
-      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
-      summary: buildSummary(results),
-      results
-    },
-    null,
-    2
-  );
-}
-function buildSummary(results) {
-  const tasks = [...new Set(results.map((r3) => r3.taskName))];
-  const providers = [...new Set(results.map((r3) => r3.providerId))];
-  return {
-    totalBenchmarks: results.length,
-    tasks: tasks.length,
-    providers: providers.length,
-    providerIds: providers,
-    taskNames: tasks
-  };
-}
-var init_json = __esm({
-  "src/reporter/json.ts"() {
-    "use strict";
-  }
-});
-
 // node_modules/tsx/dist/temporary-directory-CwHp0_NW.mjs
 import r from "path";
 import o from "os";
@@ -6530,44 +6193,1795 @@ var init_api = __esm({
 // src/cli.ts
 import "dotenv/config";
 import { Command } from "commander";
-import { readFileSync, writeFileSync, existsSync } from "fs";
-import { resolve, join, dirname } from "path";
+import { readFileSync as readFileSync3, writeFileSync as writeFileSync2, mkdirSync as mkdirSync2, existsSync } from "fs";
+import { resolve, join, dirname as dirname2 } from "path";
 import { pathToFileURL, fileURLToPath } from "url";
-var __dirname2 = dirname(fileURLToPath(import.meta.url));
-var program = new Command();
-program.name("duelist").description("Pit LLM providers against each other on agent tasks.").version(getVersion());
-program.command("init").description("Scaffold an arena.config.ts in the current directory").option("--force", "Overwrite existing config file").action((opts) => {
-  const target = resolve("arena.config.ts");
-  if (existsSync(target) && !opts.force) {
-    console.error("arena.config.ts already exists. Use --force to overwrite.");
-    process.exit(1);
+
+// src/utils/format.ts
+var MAX_FRACTION_DIGITS = 100;
+function formatCost(usd) {
+  if (usd === void 0) return "\u2014";
+  if (usd === 0) return "$0.00";
+  if (usd >= 0.01) return `~$${usd.toFixed(2)}`;
+  const digits = Math.min(
+    MAX_FRACTION_DIGITS,
+    Math.max(4, -Math.floor(Math.log10(Math.abs(usd))) + 1)
+  );
+  return `~$${usd.toFixed(digits).replace(/0+$/, "")}`;
+}
+function formatDelta(delta, precision = 4) {
+  const sign = delta >= 0 ? "+" : "";
+  return `${sign}${delta.toFixed(precision)}`;
+}
+
+// src/reporter/shared.ts
+function groupResults(results) {
+  const taskSet = /* @__PURE__ */ new Set();
+  const providerSet = /* @__PURE__ */ new Set();
+  const scorerSet = /* @__PURE__ */ new Set();
+  const grouped = /* @__PURE__ */ new Map();
+  const byProvider = /* @__PURE__ */ new Map();
+  let hasErrors = false;
+  let maxRun = 0;
+  for (const r3 of results) {
+    taskSet.add(r3.taskName);
+    providerSet.add(r3.providerId);
+    for (const s5 of r3.scores) scorerSet.add(s5.name);
+    if (r3.error) hasErrors = true;
+    if (r3.run > maxRun) maxRun = r3.run;
+    const key = `${r3.taskName}::${r3.providerId}`;
+    let group = grouped.get(key);
+    if (!group) {
+      group = [];
+      grouped.set(key, group);
+    }
+    group.push(r3);
+    let provGroup = byProvider.get(r3.providerId);
+    if (!provGroup) {
+      provGroup = [];
+      byProvider.set(r3.providerId, provGroup);
+    }
+    provGroup.push(r3);
+  }
+  return {
+    tasks: [...taskSet],
+    providers: [...providerSet],
+    scorerNames: [...scorerSet],
+    grouped,
+    byProvider,
+    hasErrors,
+    maxRun
+  };
+}
+function aggregateProviderTask(providerId, grouped, task) {
+  const taskResults = grouped.get(`${task}::${providerId}`) ?? [];
+  const errorResults = taskResults.filter((r3) => r3.error);
+  const successResults = taskResults.filter((r3) => !r3.error);
+  if (successResults.length === 0) {
+    return {
+      providerId,
+      avgScores: {},
+      avgDetails: { costUsd: void 0, totalTokens: void 0 },
+      latencyMs: void 0,
+      allErrors: errorResults.length > 0,
+      errorCount: errorResults.length
+    };
+  }
+  return {
+    providerId,
+    avgScores: averageScores(successResults),
+    avgDetails: averageDetails(successResults),
+    latencyMs: average(successResults.map((r3) => r3.raw.latencyMs)),
+    allErrors: false,
+    errorCount: errorResults.length
+  };
+}
+function averageScores(results) {
+  const sums = {};
+  const counts = {};
+  for (const result of results) {
+    for (const score of result.scores) {
+      if (score.value < 0) continue;
+      sums[score.name] = (sums[score.name] ?? 0) + score.value;
+      counts[score.name] = (counts[score.name] ?? 0) + 1;
+    }
+  }
+  const avgs = {};
+  for (const name of Object.keys(sums)) {
+    avgs[name] = sums[name] / counts[name];
+  }
+  return avgs;
+}
+function averageDetails(results) {
+  let costSum = 0;
+  let costCount = 0;
+  let tokenSum = 0;
+  let tokenCount = 0;
+  for (const result of results) {
+    const costScore = result.scores.find((s5) => s5.name === "cost");
+    const details = costScore?.details;
+    if (details?.estimatedUsd != null) {
+      costSum += details.estimatedUsd;
+      costCount++;
+    }
+    if (details?.totalTokens != null) {
+      tokenSum += details.totalTokens;
+      tokenCount++;
+    }
+  }
+  return {
+    costUsd: costCount > 0 ? costSum / costCount : void 0,
+    totalTokens: tokenCount > 0 ? Math.round(tokenSum / tokenCount) : void 0
+  };
+}
+function average(nums) {
+  if (nums.length === 0) return void 0;
+  return nums.reduce((a7, b3) => a7 + b3, 0) / nums.length;
+}
+function computeColumnStats(providerData, scorerNames) {
+  const stats = /* @__PURE__ */ new Map();
+  const valid = providerData.filter((p5) => !p5.allErrors);
+  if (scorerNames.includes("latency")) {
+    const values = /* @__PURE__ */ new Map();
+    for (const p5 of providerData) {
+      values.set(p5.providerId, p5.allErrors ? void 0 : p5.latencyMs);
+    }
+    const nums = valid.map((p5) => p5.latencyMs).filter((v4) => v4 !== void 0);
+    stats.set("latency", {
+      values,
+      best: nums.length > 0 ? Math.min(...nums) : void 0,
+      worst: nums.length > 0 ? Math.max(...nums) : void 0
+    });
+  }
+  if (scorerNames.includes("cost")) {
+    const costValues = /* @__PURE__ */ new Map();
+    const tokenValues = /* @__PURE__ */ new Map();
+    for (const p5 of providerData) {
+      costValues.set(p5.providerId, p5.allErrors ? void 0 : p5.avgDetails.costUsd);
+      tokenValues.set(p5.providerId, p5.allErrors ? void 0 : p5.avgDetails.totalTokens);
+    }
+    const costNums = valid.map((p5) => p5.avgDetails.costUsd).filter((v4) => v4 !== void 0);
+    const tokenNums = valid.map((p5) => p5.avgDetails.totalTokens).filter((v4) => v4 !== void 0);
+    stats.set("cost", {
+      values: costValues,
+      best: costNums.length > 0 ? Math.min(...costNums) : void 0,
+      worst: costNums.length > 0 ? Math.max(...costNums) : void 0
+    });
+    stats.set("tokens", {
+      values: tokenValues,
+      best: tokenNums.length > 0 ? Math.min(...tokenNums) : void 0,
+      worst: tokenNums.length > 0 ? Math.max(...tokenNums) : void 0
+    });
+  }
+  for (const name of scorerNames) {
+    if (name === "latency" || name === "cost") continue;
+    const values = /* @__PURE__ */ new Map();
+    for (const p5 of providerData) {
+      values.set(p5.providerId, p5.allErrors ? void 0 : p5.avgScores[name]);
+    }
+    const nums = valid.map((p5) => p5.avgScores[name]).filter((v4) => v4 !== void 0);
+    stats.set(name, {
+      values,
+      best: nums.length > 0 ? Math.max(...nums) : void 0,
+      worst: nums.length > 0 ? Math.min(...nums) : void 0
+    });
+  }
+  return stats;
+}
+function computeMedals(columnStats, providerIds) {
+  const medals = /* @__PURE__ */ new Map();
+  if (providerIds.length < 2) {
+    for (const id of providerIds) medals.set(id, "none");
+    return medals;
+  }
+  const wins = /* @__PURE__ */ new Map();
+  for (const id of providerIds) wins.set(id, 0);
+  for (const [, colStats] of columnStats) {
+    if (colStats.best === void 0) continue;
+    const bestProviders = [...colStats.values.entries()].filter(([, v4]) => v4 !== void 0 && v4 === colStats.best);
+    if (bestProviders.length === 1) {
+      wins.set(bestProviders[0][0], (wins.get(bestProviders[0][0]) ?? 0) + 1);
+    }
+  }
+  const totalWins = [...wins.values()].reduce((a7, b3) => a7 + b3, 0);
+  if (totalWins === 0) {
+    for (const id of providerIds) medals.set(id, "none");
+    return medals;
+  }
+  const sorted = [...wins.entries()].sort(
+    (a7, b3) => b3[1] - a7[1] || a7[0].localeCompare(b3[0])
+  );
+  const medalList = ["gold", "silver", "bronze"];
+  let rank = 0;
+  for (let i7 = 0; i7 < sorted.length; i7++) {
+    if (i7 > 0 && sorted[i7][1] < sorted[i7 - 1][1]) {
+      rank = i7;
+    }
+    const hasWins = sorted[i7][1] > 0;
+    medals.set(sorted[i7][0], hasWins && rank < medalList.length ? medalList[rank] : "none");
+  }
+  return medals;
+}
+function providerLabel(providerId) {
+  const prefix = providerId.split("/")[0];
+  switch (prefix) {
+    case "azure":
+      return "(OpenAI via Azure)";
+    case "openai":
+      return "(OpenAI)";
+    case "anthropic":
+      return "(Anthropic)";
+    case "google":
+      return "(Google)";
+    case "mistral":
+      return "(Mistral)";
+    case "meta":
+      return "(Meta)";
+    case "deepseek":
+      return "(DeepSeek)";
+    case "cohere":
+      return "(Cohere)";
+    case "qwen":
+      return "(Qwen)";
+    case "xai":
+      return "(xAI)";
+    case "minimax":
+      return "(MiniMax)";
+    case "moonshot":
+      return "(Moonshot / Kimi)";
+    case "perplexity":
+      return "(Perplexity)";
+    case "amazon":
+      return "(Amazon)";
+    case "nvidia":
+      return "(NVIDIA)";
+    case "microsoft":
+      return "(Microsoft)";
+    case "ai21":
+      return "(AI21 Labs)";
+    case "bytedance":
+      return "(ByteDance)";
+    case "together":
+      return "(Together AI)";
+    case "fireworks":
+      return "(Fireworks AI)";
+    case "groq":
+      return "(Groq)";
+    case "cerebras":
+      return "(Cerebras)";
+    default:
+      return `(${prefix})`;
+  }
+}
+function apiKeyHint(providerId, error) {
+  const lower = error.toLowerCase();
+  const isAuthError = lower.includes("api key") || lower.includes("401") || lower.includes("unauthorized") || lower.includes("authentication") || lower.includes("incorrect api key") || lower.includes("apikey");
+  if (!isAuthError) return void 0;
+  const prefix = providerId.split("/")[0];
+  switch (prefix) {
+    case "openai":
+      return "Set: export OPENAI_API_KEY=sk-...";
+    case "azure":
+      return "Set: export AZURE_OPENAI_API_KEY=... and AZURE_OPENAI_ENDPOINT=...";
+    case "anthropic":
+      return "Set: export ANTHROPIC_API_KEY=sk-ant-...";
+    case "google":
+      return "Set: export GOOGLE_API_KEY=...";
+    default:
+      return `Check the API key for ${providerId}`;
+  }
+}
+function rankProviders(successByProvider, providers, scorerName) {
+  const ranked = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const scores = runs.flatMap((r3) => r3.scores.filter((s5) => s5.name === scorerName && s5.value >= 0)).map((s5) => s5.value);
+    const avg = scores.length > 0 ? scores.reduce((a7, b3) => a7 + b3, 0) / scores.length : void 0;
+    return { id, avg };
+  }).filter((p5) => p5.avg !== void 0).sort((a7, b3) => b3.avg - a7.avg);
+  return ranked[0] ? { id: ranked[0].id, avg: ranked[0].avg } : void 0;
+}
+function scorerLabel(name) {
+  switch (name) {
+    case "correctness":
+      return "Match";
+    case "schema-correctness":
+      return "Schema";
+    case "fuzzy-similarity":
+      return "Fuzzy";
+    case "llm-judge-correctness":
+      return "Judge";
+    case "tool-usage":
+      return "Tool";
+    default:
+      return name;
+  }
+}
+function medalEmoji(medal) {
+  switch (medal) {
+    case "gold":
+      return "\u{1F947}";
+    case "silver":
+      return "\u{1F948}";
+    case "bronze":
+      return "\u{1F949}";
+    case "none":
+      return "";
+  }
+}
+
+// src/reporter/console.ts
+var reset = "\x1B[0m";
+var boldCode = "\x1B[1m";
+var dimCode = "\x1B[2m";
+var green = "\x1B[32m";
+var red = "\x1B[31m";
+var yellow = "\x1B[33m";
+var cyan = "\x1B[36m";
+var brightGreen = "\x1B[92m";
+var brightWhite = "\x1B[97m";
+function bold(s5) {
+  return `${boldCode}${s5}${reset}`;
+}
+function dim(s5) {
+  return `${dimCode}${s5}${reset}`;
+}
+function stripAnsi(s5) {
+  return s5.replace(/\x1b\[[0-9;]*m/g, "");
+}
+function displayWidth(s5) {
+  const stripped = stripAnsi(s5);
+  let width = 0;
+  for (const ch of stripped) {
+    const code = ch.codePointAt(0) ?? 0;
+    if (code >= 126976) width += 2;
+    else if (code >= 9728 && code <= 10175) width += 2;
+    else width += 1;
+  }
+  return width;
+}
+function padCell(str, targetWidth, align) {
+  const dw = displayWidth(str);
+  const padding = Math.max(0, targetWidth - dw);
+  if (align === "right") return " ".repeat(padding) + str;
+  return str + " ".repeat(padding);
+}
+function sparkBar(ratio, width = 8) {
+  const clamped = Math.max(0, Math.min(1, ratio));
+  const fillLen = Math.round(clamped * width);
+  const fill = "\u2593".repeat(fillLen);
+  const track = "\u2591".repeat(width - fillLen);
+  return { fill, track };
+}
+function drawTableLine(widths, position) {
+  const totalInner = widths.reduce((sum, w4) => sum + w4 + 2, 0) + widths.length - 1;
+  if (position === "bottom") {
+    return dim(`\u2514${"\u2500".repeat(totalInner)}\u2518`);
+  }
+  if (position === "merge") {
+    return dim(`\u251C${"\u2500".repeat(totalInner)}\u2524`);
+  }
+  const segments = widths.map((w4) => "\u2500".repeat(w4 + 2));
+  if (position === "top") {
+    return dim(`\u250C${segments.join("\u252C")}\u2510`);
+  }
+  return dim(`\u251C${segments.join("\u253C")}\u2524`);
+}
+function drawTableRow(cells, widths, aligns) {
+  const parts = cells.map(
+    (cell, i7) => " " + padCell(cell, widths[i7], aligns[i7]) + " "
+  );
+  return dim("\u2502") + parts.join(dim("\u2502")) + dim("\u2502");
+}
+function drawSpanRow(content, widths) {
+  const totalInner = widths.reduce((sum, w4) => sum + w4 + 2, 0) + widths.length - 1;
+  const dw = displayWidth(content);
+  const padding = Math.max(0, totalInner - dw - 1);
+  return dim("\u2502") + " " + content + " ".repeat(padding) + dim("\u2502");
+}
+function colorByRank(text, value, colStats, providerCount) {
+  if (value === void 0) return dim("\u2014");
+  if (providerCount < 2) return text;
+  if (colStats.best === void 0 || colStats.worst === void 0) return text;
+  if (colStats.best === colStats.worst) return text;
+  if (value === colStats.best) return `${brightGreen}${boldCode}${text}${reset}`;
+  if (value === colStats.worst) return `${red}${text}${reset}`;
+  return `${yellow}${text}${reset}`;
+}
+function consoleReporter(results, options) {
+  const showSparklines = options?.sparklines ?? true;
+  if (results.length === 0) {
+    console.log("\nNo results to display.\n");
+    return;
+  }
+  const { tasks, providers, scorerNames, grouped, byProvider, hasErrors, maxRun } = groupResults(results);
+  const hasCost = scorerNames.includes("cost");
+  const multi = providers.length >= 2;
+  const runsPerCell = maxRun;
+  const runLabel = runsPerCell > 1 ? `  ${dim(`(${runsPerCell} runs each)`)}` : "";
+  console.log("");
+  console.log(`  ${brightWhite}${boldCode}\u2B21  Agent Duelist${reset}${runLabel}`);
+  console.log(`  ${dim("\u2501".repeat(72))}`);
+  console.log("");
+  for (const task of tasks) {
+    console.log(`  ${bold(`Task: ${task}`)}`);
+    console.log("");
+    const providerData = providers.map(
+      (providerId) => aggregateProviderTask(providerId, grouped, task)
+    );
+    const columnStats = computeColumnStats(providerData, scorerNames);
+    const medals = computeMedals(columnStats, providers);
+    const maxProviderLen = Math.max(...providers.map((id) => id.length));
+    const providerWidth = Math.min(35, Math.max(22, maxProviderLen + 5));
+    const cols = [
+      { label: "Provider", width: providerWidth, align: "left" }
+    ];
+    for (const name of scorerNames) {
+      if (name === "latency") {
+        cols.push({ label: "Latency", width: 10, align: "right", statsKey: "latency" });
+      } else if (name === "cost") {
+        cols.push({ label: "Cost", width: 12, align: "right", statsKey: "cost" });
+        cols.push({ label: "Tokens", width: 9, align: "right", statsKey: "tokens" });
+      } else {
+        cols.push({ label: scorerLabel(name), width: showSparklines ? 15 : 8, align: "right", statsKey: name });
+      }
+    }
+    if (hasErrors) {
+      cols.push({ label: "Status", width: 8, align: "left" });
+    }
+    const widths = cols.map((c3) => c3.width);
+    const aligns = cols.map((c3) => c3.align);
+    console.log(`  ${drawTableLine(widths, "top")}`);
+    const headerCells = cols.map((c3) => bold(c3.label));
+    console.log(`  ${drawTableRow(headerCells, widths, aligns)}`);
+    console.log(`  ${drawTableLine(widths, "header")}`);
+    for (const pd of providerData) {
+      const medal = medalEmoji(medals.get(pd.providerId) ?? "none");
+      const providerCell = medal ? `${medal} ${pd.providerId}` : pd.providerId;
+      const cells = [providerCell];
+      if (pd.allErrors) {
+        for (const col of cols.slice(1)) {
+          if (col.label === "Status") {
+            cells.push(`${red}FAIL${reset}`);
+          } else {
+            cells.push(dim("\u2014"));
+          }
+        }
+      } else {
+        for (const col of cols.slice(1)) {
+          if (col.label === "Status") {
+            cells.push(
+              pd.errorCount > 0 ? `${yellow}${pd.errorCount} err${reset}` : `${green}OK${reset}`
+            );
+            continue;
+          }
+          const statsKey = col.statsKey;
+          const colStats = columnStats.get(statsKey);
+          if (statsKey === "latency") {
+            const ms = pd.latencyMs;
+            if (ms === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = `${Math.round(ms)}ms`;
+              cells.push(colStats ? colorByRank(text, ms, colStats, providers.length) : text);
+            }
+          } else if (statsKey === "cost") {
+            const cost = pd.avgDetails.costUsd;
+            if (cost === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = formatCost(cost);
+              cells.push(colStats ? colorByRank(text, cost, colStats, providers.length) : text);
+            }
+          } else if (statsKey === "tokens") {
+            const tokens = pd.avgDetails.totalTokens;
+            if (tokens === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = `${tokens}`;
+              cells.push(colStats ? colorByRank(text, tokens, colStats, providers.length) : text);
+            }
+          } else {
+            const val = pd.avgScores[statsKey];
+            if (val === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const pctStr = `${Math.round(val * 100)}%`.padStart(4);
+              let coloredPct;
+              if (multi && colStats) {
+                coloredPct = colorByRank(pctStr, val, colStats, providers.length);
+              } else {
+                if (val >= 0.8) coloredPct = `${green}${pctStr}${reset}`;
+                else if (val >= 0.5) coloredPct = `${yellow}${pctStr}${reset}`;
+                else coloredPct = `${red}${pctStr}${reset}`;
+              }
+              if (showSparklines) {
+                const { fill, track } = sparkBar(val);
+                const barColor = val >= 0.8 ? green : val >= 0.5 ? yellow : red;
+                cells.push(`${coloredPct} ${barColor}${fill}${reset}${dim(track)}`);
+              } else {
+                cells.push(coloredPct);
+              }
+            }
+          }
+        }
+      }
+      console.log(`  ${drawTableRow(cells, widths, aligns)}`);
+    }
+    if (multi && providerData.some((p5) => !p5.allErrors)) {
+      const winnerId = [...medals.entries()].find(([, m8]) => m8 === "gold")?.[0];
+      if (winnerId) {
+        console.log(`  ${drawTableLine(widths, "merge")}`);
+        const winnerText = `${brightGreen}${boldCode}\u{1F3C6}  Winner: ${winnerId}${reset} ${dim(providerLabel(winnerId))}`;
+        console.log(`  ${drawSpanRow(winnerText, widths)}`);
+      }
+    }
+    console.log(`  ${drawTableLine(widths, "bottom")}`);
+    console.log("");
+  }
+  printSummary(results, providers, byProvider);
+  const errorResults = results.filter((r3) => r3.error);
+  if (errorResults.length > 0) {
+    console.log(`  ${bold("Errors")}`);
+    console.log(`  ${dim("\u2501".repeat(72))}`);
+    const seen = /* @__PURE__ */ new Set();
+    for (const r3 of errorResults) {
+      const key = `${r3.providerId}::${r3.error}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      const count = errorResults.filter((e5) => e5.providerId === r3.providerId && e5.error === r3.error).length;
+      const suffix = count > 1 ? ` (\xD7${count})` : "";
+      console.log(`  ${red}\u2716${reset} ${r3.providerId}: ${r3.error}${suffix}`);
+      const hint = apiKeyHint(r3.providerId, r3.error ?? "");
+      if (hint) console.log(`    ${dim(hint)}`);
+    }
+    console.log("");
+  }
+  if (hasCost) {
+    console.log(dim(`  Costs estimated from OpenRouter pricing catalog. Run npx tsx scripts/update-pricing.ts to refresh.`));
+    console.log("");
+  }
+}
+function printSummary(results, providers, byProvider) {
+  const successResults = results.filter((r3) => !r3.error);
+  if (successResults.length === 0) return;
+  const successByProvider = /* @__PURE__ */ new Map();
+  for (const id of providers) {
+    successByProvider.set(id, (byProvider.get(id) ?? []).filter((r3) => !r3.error));
+  }
+  console.log(`  ${bold("Summary")}`);
+  console.log(`  ${dim("\u2501".repeat(72))}`);
+  console.log("");
+  const single = providers.length === 1;
+  const correctnessKey = successResults.some((r3) => r3.scores.some((s5) => s5.name === "llm-judge-correctness" && s5.value >= 0)) ? "llm-judge-correctness" : "correctness";
+  const byCorrectness = rankProviders(successByProvider, providers, correctnessKey);
+  if (byCorrectness) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const pctStr = `${Math.round(byCorrectness.avg * 100)}%`;
+    if (single) {
+      console.log(`  ${medal} Avg correctness:  ${brightGreen}${boldCode}${pctStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Most correct:  ${bold(byCorrectness.id)} ${dim(providerLabel(byCorrectness.id))}  ${brightGreen}${boldCode}${pctStr}${reset}`);
+    }
+  }
+  const byLatency = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const avg = average(runs.map((r3) => r3.raw.latencyMs));
+    return { id, avg: avg ?? Infinity };
+  }).sort((a7, b3) => a7.avg - b3.avg)[0];
+  if (byLatency && byLatency.avg !== Infinity) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const msStr = `${Math.round(byLatency.avg)}ms`;
+    if (single) {
+      console.log(`  ${medal} Avg latency:     ${brightGreen}${boldCode}${msStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Fastest:       ${bold(byLatency.id)} ${dim(providerLabel(byLatency.id))}  ${brightGreen}${boldCode}${msStr}${reset}`);
+    }
+  }
+  const byCost = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const costs = runs.map((r3) => {
+      const s5 = r3.scores.find((s6) => s6.name === "cost");
+      return s5 && s5.value >= 0 ? s5.value : void 0;
+    }).filter((c3) => c3 !== void 0);
+    const avg = costs.length > 0 ? costs.reduce((a7, b3) => a7 + b3, 0) / costs.length : void 0;
+    return { id, avg };
+  }).filter((p5) => p5.avg !== void 0).sort((a7, b3) => a7.avg - b3.avg)[0];
+  if (byCost?.avg !== void 0) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const costStr = formatCost(byCost.avg);
+    if (single) {
+      console.log(`  ${medal} Avg cost:        ${brightGreen}${boldCode}${costStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Cheapest:      ${bold(byCost.id)} ${dim(providerLabel(byCost.id))}  ${brightGreen}${boldCode}${costStr}${reset}`);
+    }
+  }
+  if (!single) {
+    const wins = /* @__PURE__ */ new Map();
+    for (const id of providers) wins.set(id, 0);
+    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
+    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
+    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
+    const maxWins = Math.max(...wins.values());
+    if (maxWins > 0) {
+      const topProviders = [...wins.entries()].filter(([, w4]) => w4 === maxWins);
+      console.log("");
+      if (topProviders.length === 1) {
+        const [winnerId, winCount] = topProviders[0];
+        console.log(`  \u{1F3C6} Overall:      ${brightGreen}${boldCode}${winnerId}${reset} ${dim(providerLabel(winnerId))}  ${dim(`(${winCount}/3 categories)`)}`);
+      } else {
+        const names = topProviders.map(([id]) => bold(id)).join(dim(", "));
+        console.log(`  \u{1F3C6} Overall:      ${names}  ${dim(`(tied at ${maxWins}/3)`)}`);
+      }
+    }
+  }
+  console.log("");
+}
+
+// src/reporter/json.ts
+function jsonReporter(results) {
+  return JSON.stringify(
+    {
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      summary: buildSummary(results),
+      results
+    },
+    null,
+    2
+  );
+}
+function buildSummary(results) {
+  const tasks = [...new Set(results.map((r3) => r3.taskName))];
+  const providers = [...new Set(results.map((r3) => r3.providerId))];
+  return {
+    totalBenchmarks: results.length,
+    tasks: tasks.length,
+    providers: providers.length,
+    providerIds: providers,
+    taskNames: tasks
+  };
+}
+
+// src/reporter/markdown.ts
+var COMMENT_MARKER = "<!-- duelist-ci-report -->";
+function markdownReporter(report, _current) {
+  const lines = [COMMENT_MARKER, ""];
+  const status = report.failed ? "\u{1F534} Failed" : "\u{1F7E2} Passed";
+  lines.push(`## \u2B21 Agent Duelist CI \u2014 ${status}`);
+  lines.push("");
+  if (report.comparisons.length > 0) {
+    lines.push(markdownComparisonTable(report.comparisons));
+    lines.push("");
+  }
+  if (report.cost.totalUsd > 0 || report.cost.budget !== void 0) {
+    lines.push(markdownCostSummary(report.cost));
+    lines.push("");
+  }
+  if (report.flakyResults.length > 0) {
+    lines.push("### \u26A0\uFE0F Flaky Results");
+    lines.push("");
+    lines.push("These scorer/task combinations have high variance (CV > 0.3). Consider increasing `runs` or tightening prompts.");
+    lines.push("");
+    for (const f6 of report.flakyResults) {
+      lines.push(`- **${f6.providerId}** \xD7 ${f6.taskName} \u2192 ${f6.scorerName} (CV = ${f6.current.cv.toFixed(2)})`);
+    }
+    lines.push("");
+  }
+  if (report.failureReasons.length > 0) {
+    lines.push("### Failure Reasons");
+    lines.push("");
+    for (const reason of report.failureReasons) {
+      lines.push(`- ${reason}`);
+    }
+    lines.push("");
+  }
+  lines.push("---");
+  lines.push("*Generated by [agent-duelist](https://github.com/DataGobes/agent-duelist) CI*");
+  return lines.join("\n");
+}
+function markdownComparisonTable(comparisons) {
+  const lines = [];
+  lines.push("| Provider | Task | Scorer | Baseline | Current | Delta | Status |");
+  lines.push("|----------|------|--------|----------|---------|-------|--------|");
+  for (const c3 of comparisons) {
+    const baselineStr = c3.baseline ? formatStats(c3.baseline) : "\u2014";
+    const currentStr = formatStats(c3.current);
+    const deltaStr = c3.delta !== null ? formatDelta(c3.delta, 3) : "\u2014";
+    const status = statusIndicator(c3);
+    lines.push(`| ${c3.providerId} | ${c3.taskName} | ${c3.scorerName} | ${baselineStr} | ${currentStr} | ${deltaStr} | ${status} |`);
+  }
+  return lines.join("\n");
+}
+function markdownCostSummary(cost) {
+  const lines = [];
+  lines.push("### \u{1F4B0} Cost Summary");
+  lines.push("");
+  lines.push(`**Total:** $${cost.totalUsd.toFixed(4)}`);
+  if (cost.budget !== void 0) {
+    const pct = cost.budget > 0 ? (cost.totalUsd / cost.budget * 100).toFixed(0) : "\u221E";
+    const indicator = cost.overBudget ? "\u{1F534}" : "\u{1F7E2}";
+    lines.push(`**Budget:** $${cost.budget.toFixed(2)} (${pct}% used) ${indicator}`);
+  }
+  if (cost.perProvider.size > 1) {
+    lines.push("");
+    lines.push("| Provider | Cost |");
+    lines.push("|----------|------|");
+    for (const [provider, usd] of cost.perProvider) {
+      lines.push(`| ${provider} | $${usd.toFixed(4)} |`);
+    }
+  }
+  return lines.join("\n");
+}
+function formatStats(stats) {
+  if (stats.n > 1) {
+    const margin = (stats.ci95Upper - stats.ci95Lower) / 2;
+    return `${stats.mean.toFixed(3)} \xB1 ${margin.toFixed(3)}`;
+  }
+  return stats.mean.toFixed(3);
+}
+function statusIndicator(c3) {
+  if (c3.regressed) return "\u{1F534} regressed";
+  if (c3.improved) return "\u{1F7E2} improved";
+  if (c3.baseline === null) return "\u{1F195} new";
+  return "\u26AA unchanged";
+}
+
+// src/reporter/html.ts
+function esc(s5) {
+  return s5.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function htmlReporter(results) {
+  if (results.length === 0) {
+    return emptyReport();
+  }
+  const { tasks, providers, scorerNames, grouped, byProvider, maxRun } = groupResults(results);
+  const hasCost = scorerNames.includes("cost");
+  const multi = providers.length >= 2;
+  const runsLabel = maxRun > 1 ? `${maxRun} runs each` : "1 run";
+  const taskSections = tasks.map((task) => {
+    const providerData = providers.map((id) => aggregateProviderTask(id, grouped, task));
+    const columnStats = computeColumnStats(providerData, scorerNames);
+    const medals = computeMedals(columnStats, providers);
+    const winnerId = multi ? [...medals.entries()].find(([, m8]) => m8 === "gold")?.[0] : void 0;
+    return { task, providerData, columnStats, medals, winnerId };
+  });
+  const successResults = results.filter((r3) => !r3.error);
+  const successByProvider = /* @__PURE__ */ new Map();
+  for (const id of providers) {
+    successByProvider.set(id, (byProvider.get(id) ?? []).filter((r3) => !r3.error));
+  }
+  const correctnessKey = successResults.some(
+    (r3) => r3.scores.some((s5) => s5.name === "llm-judge-correctness" && s5.value >= 0)
+  ) ? "llm-judge-correctness" : "correctness";
+  const byCorrectness = rankProviders(successByProvider, providers, correctnessKey);
+  const byLatency = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const avg = average(runs.map((r3) => r3.raw.latencyMs));
+    return { id, avg: avg ?? Infinity };
+  }).sort((a7, b3) => a7.avg - b3.avg)[0];
+  const byCost = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const costs = runs.map((r3) => {
+      const s5 = r3.scores.find((s6) => s6.name === "cost");
+      return s5 && s5.value >= 0 ? s5.value : void 0;
+    }).filter((c3) => c3 !== void 0);
+    const avg = costs.length > 0 ? costs.reduce((a7, b3) => a7 + b3, 0) / costs.length : void 0;
+    return { id, avg };
+  }).filter((p5) => p5.avg !== void 0).sort((a7, b3) => a7.avg - b3.avg)[0];
+  let overallWinner;
+  if (multi) {
+    const wins = /* @__PURE__ */ new Map();
+    for (const id of providers) wins.set(id, 0);
+    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
+    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
+    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
+    const maxWins = Math.max(...wins.values());
+    if (maxWins > 0) {
+      const tops = [...wins.entries()].filter(([, w4]) => w4 === maxWins);
+      if (tops.length === 1) overallWinner = tops[0][0];
+    }
+  }
+  const errorResults = results.filter((r3) => r3.error);
+  const deduped = dedupeErrors(errorResults);
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Agent Duelist Report</title>
+<meta name="description" content="LLM provider benchmark results \u2014 ${providers.length} provider${providers.length !== 1 ? "s" : ""}, ${tasks.length} task${tasks.length !== 1 ? "s" : ""}">
+<meta property="og:title" content="Agent Duelist Report">
+<meta property="og:description" content="LLM provider benchmark: ${providers.map(esc).join(" vs ")}">
+<meta property="og:type" content="website">
+${renderStyle()}
+</head>
+<body>
+<div class="bg-mesh"><div class="bg-mesh-extra"></div></div>
+<div class="report">
+
+${renderHeader(runsLabel, providers.length, tasks.length)}
+
+${tasks.length > 1 ? renderTabs(tasks) : ""}
+
+<main>
+${taskSections.map((s5, i7) => renderTaskSection(
+    s5.task,
+    s5.providerData,
+    s5.columnStats,
+    s5.medals,
+    s5.winnerId,
+    scorerNames,
+    hasCost,
+    multi,
+    i7
+  )).join("\n")}
+</main>
+
+${renderSummary(byCorrectness, byLatency, byCost, overallWinner, multi)}
+
+${deduped.length > 0 ? renderErrors(deduped) : ""}
+
+${renderFooter()}
+
+</div>
+${renderScript(tasks.length)}
+</body>
+</html>`;
+}
+function emptyReport() {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Agent Duelist Report</title>
+${renderStyle()}
+</head>
+<body>
+<div class="bg-mesh"><div class="bg-mesh-extra"></div></div>
+<div class="report">
+${renderHeader("0 runs", 0, 0)}
+<main><p class="empty-msg">No results to display.</p></main>
+${renderFooter()}
+</div>
+</body>
+</html>`;
+}
+function dedupeErrors(errorResults) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const r3 of errorResults) {
+    const key = `${r3.providerId}::${r3.error}`;
+    const existing = seen.get(key);
+    if (existing) {
+      existing.count++;
+    } else {
+      seen.set(key, {
+        providerId: r3.providerId,
+        error: r3.error ?? "Unknown error",
+        count: 1,
+        hint: apiKeyHint(r3.providerId, r3.error ?? "")
+      });
+    }
+  }
+  return [...seen.values()];
+}
+function renderStyle() {
+  return `<style>
+:root {
+  --bg: #0f172a;
+  --bg-deep: #020617;
+  --panel: rgba(15, 23, 42, 0.85);
+  --accent: #f59e0b;
+  --accent-soft: rgba(245, 158, 11, 0.15);
+  --text: #e2e8f0;
+  --muted: #94a3b8;
+  --border: rgba(148, 163, 184, 0.15);
+  --green: #22c55e;
+  --red: #ef4444;
+  --yellow: #eab308;
+  --radius: 12px;
+  --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Monaco, Consolas, monospace;
+  --sans: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+html, body {
+  font-family: var(--sans);
+  background: var(--bg);
+  color: var(--text);
+  min-height: 100vh;
+}
+body { padding: 24px; display: flex; justify-content: center; }
+
+/* Animated gradient mesh */
+.bg-mesh {
+  position: fixed; inset: 0; z-index: 0;
+  overflow: hidden; pointer-events: none;
+}
+.bg-mesh::before, .bg-mesh::after {
+  content: ""; position: absolute; border-radius: 50%;
+  filter: blur(120px); opacity: 0.4;
+}
+.bg-mesh::before {
+  width: 600px; height: 600px;
+  background: radial-gradient(circle, rgba(245,158,11,0.18), transparent 70%);
+  top: -10%; left: -5%;
+  animation: meshDrift1 18s ease-in-out infinite alternate;
+}
+.bg-mesh::after {
+  width: 500px; height: 500px;
+  background: radial-gradient(circle, rgba(139,92,246,0.12), transparent 70%);
+  bottom: -10%; right: -5%;
+  animation: meshDrift2 22s ease-in-out infinite alternate;
+}
+.bg-mesh-extra {
+  position: absolute; width: 400px; height: 400px;
+  border-radius: 50%; filter: blur(100px); opacity: 0.3;
+  background: radial-gradient(circle, rgba(56,189,248,0.12), transparent 70%);
+  top: 50%; left: 60%;
+  animation: meshDrift3 15s ease-in-out infinite alternate;
+}
+@keyframes meshDrift1 { from { transform: translate(0,0) scale(1); } to { transform: translate(80px,60px) scale(1.15); } }
+@keyframes meshDrift2 { from { transform: translate(0,0) scale(1); } to { transform: translate(-60px,-50px) scale(1.1); } }
+@keyframes meshDrift3 { from { transform: translate(0,0) scale(1); } to { transform: translate(-40px,40px) scale(1.2); } }
+
+/* Report container */
+.report {
+  position: relative; z-index: 1;
+  width: 100%; max-width: 960px;
+}
+
+/* Header */
+.report-header {
+  display: flex; justify-content: space-between; align-items: center;
+  padding: 20px 0; margin-bottom: 8px;
+}
+.report-brand {
+  display: flex; align-items: center; gap: 10px;
+  text-decoration: none; color: var(--muted);
+  font-weight: 600; font-size: 14px;
+  letter-spacing: 0.04em; text-transform: uppercase;
+}
+.report-brand:hover { color: var(--text); }
+.brand-icon {
+  width: 32px; height: 32px; border-radius: 8px;
+  background: linear-gradient(135deg, var(--accent-soft), rgba(245,158,11,0.05));
+  border: 1px solid rgba(245,158,11,0.3);
+  display: flex; align-items: center; justify-content: center;
+  font-size: 16px;
+}
+.report-meta {
+  font-size: 12px; color: var(--muted);
+  text-align: right; line-height: 1.6;
+}
+
+/* Task tabs */
+.task-tabs {
+  display: flex; gap: 6px; margin-bottom: 16px; flex-wrap: wrap;
+}
+.task-tab {
+  padding: 6px 16px; border-radius: 999px;
+  border: 1px solid var(--border);
+  background: transparent; color: var(--muted);
+  font-size: 13px; font-weight: 500; cursor: pointer;
+  transition: all 150ms ease;
+}
+.task-tab:hover { border-color: rgba(245,158,11,0.3); color: var(--text); }
+.task-tab.active {
+  background: var(--accent-soft);
+  border-color: rgba(245,158,11,0.4);
+  color: var(--accent);
+}
+
+/* Task sections */
+.task-section { display: none; }
+.task-section.active { display: block; }
+.task-name {
+  font-size: 18px; font-weight: 600;
+  margin-bottom: 12px; letter-spacing: -0.01em;
+}
+
+/* Results table */
+.results-table {
+  width: 100%; border-collapse: collapse;
+  font-size: 13px; margin-bottom: 16px;
+  border-radius: var(--radius); overflow: hidden;
+  border: 1px solid var(--border);
+}
+.results-table th, .results-table td {
+  padding: 10px 14px;
+  text-align: left;
+  border-bottom: 1px solid var(--border);
+}
+.results-table th {
+  background: rgba(0,0,0,0.3);
+  font-size: 11px; font-weight: 600;
+  text-transform: uppercase; letter-spacing: 0.05em;
+  color: var(--muted); cursor: pointer;
+  user-select: none; white-space: nowrap;
+}
+.results-table th:hover { color: var(--text); }
+.results-table th .sort-arrow { margin-left: 4px; font-size: 10px; }
+.results-table tbody tr {
+  background: var(--panel);
+  transition: background 120ms ease;
+}
+.results-table tbody tr:hover { background: rgba(15,23,42,0.95); }
+.results-table tbody tr:last-child td { border-bottom: none; }
+
+/* Score cell with progress bar */
+.score-cell { position: relative; min-width: 90px; }
+.score-bar {
+  position: absolute; left: 0; bottom: 0;
+  height: 3px; border-radius: 2px;
+  transition: width 300ms ease;
+}
+.score-val { position: relative; z-index: 1; font-family: var(--mono); font-size: 12px; }
+
+/* Color ranking */
+.rank-best { color: var(--green); font-weight: 600; }
+.rank-worst { color: var(--red); }
+.rank-mid { color: var(--yellow); }
+.rank-neutral { color: var(--text); }
+.rank-error { color: var(--muted); }
+
+/* Winner banner */
+.task-winner {
+  display: flex; align-items: center; gap: 10px;
+  padding: 12px 18px; margin-bottom: 20px;
+  border-radius: var(--radius);
+  background: linear-gradient(135deg, rgba(34,197,94,0.08), rgba(34,197,94,0.02));
+  border: 1px solid rgba(34,197,94,0.2);
+  font-size: 14px; font-weight: 500;
+}
+.task-winner .trophy { font-size: 20px; }
+.task-winner .winner-name { color: var(--green); font-weight: 600; }
+.task-winner .winner-label { color: var(--muted); font-size: 12px; margin-left: 4px; }
+
+/* Summary cards */
+.summary-section { margin-top: 32px; }
+.summary-title {
+  font-size: 16px; font-weight: 600;
+  margin-bottom: 12px; color: var(--text);
+}
+.summary-cards {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+  gap: 12px;
+}
+.summary-card {
+  padding: 16px; border-radius: var(--radius);
+  border: 1px solid var(--border);
+  background: var(--panel);
+}
+.summary-card .card-label {
+  font-size: 11px; font-weight: 600;
+  text-transform: uppercase; letter-spacing: 0.05em;
+  color: var(--muted); margin-bottom: 6px;
+}
+.summary-card .card-value {
+  font-size: 20px; font-weight: 700;
+  color: var(--green); font-family: var(--mono);
+}
+.summary-card .card-provider {
+  font-size: 12px; color: var(--muted); margin-top: 4px;
+}
+
+/* Errors */
+.errors-section { margin-top: 24px; }
+.errors-title {
+  font-size: 16px; font-weight: 600;
+  margin-bottom: 8px; color: var(--red);
+  cursor: pointer;
+}
+.errors-list {
+  border-radius: var(--radius);
+  border: 1px solid rgba(239,68,68,0.2);
+  background: rgba(239,68,68,0.04);
+  overflow: hidden;
+}
+.error-item {
+  padding: 10px 16px;
+  border-bottom: 1px solid rgba(239,68,68,0.1);
+  font-size: 13px;
+}
+.error-item:last-child { border-bottom: none; }
+.error-provider { font-weight: 600; color: var(--text); }
+.error-msg { color: var(--muted); margin-left: 8px; }
+.error-count { color: var(--muted); font-size: 11px; }
+.error-hint { color: var(--muted); font-size: 12px; margin-top: 4px; font-style: italic; }
+
+/* Footer */
+.report-footer {
+  margin-top: 40px; padding: 20px 0;
+  border-top: 1px solid var(--border);
+  display: flex; justify-content: space-between; align-items: center;
+  flex-wrap: wrap; gap: 12px;
+}
+.footer-brand {
+  font-size: 13px; color: var(--muted);
+}
+.footer-brand a {
+  color: var(--accent); text-decoration: none; font-weight: 500;
+}
+.footer-brand a:hover { text-decoration: underline; }
+.footer-cta {
+  display: inline-flex; align-items: center; gap: 6px;
+  padding: 6px 14px; border-radius: 8px;
+  background: var(--accent-soft);
+  border: 1px solid rgba(245,158,11,0.3);
+  color: var(--accent); font-size: 12px; font-weight: 500;
+  text-decoration: none;
+  transition: transform 120ms ease, box-shadow 120ms ease;
+}
+.footer-cta:hover { transform: translateY(-1px); box-shadow: 0 4px 12px rgba(245,158,11,0.2); }
+
+/* Empty state */
+.empty-msg {
+  text-align: center; color: var(--muted);
+  padding: 60px 20px; font-size: 16px;
+}
+
+/* Responsive */
+@media (max-width: 640px) {
+  body { padding: 12px; }
+  .report-header { flex-direction: column; align-items: flex-start; gap: 8px; }
+  .report-meta { text-align: left; }
+  .summary-cards { grid-template-columns: 1fr; }
+  .results-table { font-size: 12px; }
+  .results-table th, .results-table td { padding: 8px 10px; }
+  .report-footer { flex-direction: column; align-items: flex-start; }
+}
+</style>`;
+}
+function renderHeader(runsLabel, providerCount, taskCount) {
+  const now = (/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19) + " UTC";
+  return `<header class="report-header">
+  <a class="report-brand" href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">
+    <div class="brand-icon">&#x2B21;</div>
+    <span>Agent Duelist</span>
+  </a>
+  <div class="report-meta">
+    ${providerCount} provider${providerCount !== 1 ? "s" : ""} &middot;
+    ${taskCount} task${taskCount !== 1 ? "s" : ""} &middot;
+    ${esc(runsLabel)}<br>
+    ${esc(now)}
+  </div>
+</header>`;
+}
+function renderTabs(tasks) {
+  const buttons = tasks.map(
+    (t3, i7) => `<button class="task-tab${i7 === 0 ? " active" : ""}" data-task="${i7}">${esc(t3)}</button>`
+  ).join("\n    ");
+  return `<nav class="task-tabs">
+    ${buttons}
+  </nav>`;
+}
+function renderTaskSection(task, providerData, columnStats, medals, winnerId, scorerNames, _hasCost, multi, index) {
+  const cols = [
+    { label: "Provider", key: "provider", isScore: false }
+  ];
+  for (const name of scorerNames) {
+    if (name === "latency") {
+      cols.push({ label: "Latency", key: "latency", isScore: false });
+    } else if (name === "cost") {
+      cols.push({ label: "Cost", key: "cost", isScore: false });
+      cols.push({ label: "Tokens", key: "tokens", isScore: false });
+    } else {
+      cols.push({ label: scorerLabel(name), key: name, isScore: true });
+    }
+  }
+  const ths = cols.map(
+    (c3) => `<th data-col="${esc(c3.key)}">${esc(c3.label)}<span class="sort-arrow"></span></th>`
+  ).join("");
+  const rows = providerData.map((pd) => {
+    const medal = medalEmoji(medals.get(pd.providerId) ?? "none");
+    const cells = [];
+    const medalHtml = medal ? `${medal} ` : "";
+    cells.push(`<td>${medalHtml}${esc(pd.providerId)}</td>`);
+    if (pd.allErrors) {
+      for (let ci = 1; ci < cols.length; ci++) {
+        cells.push(`<td class="rank-error">&mdash;</td>`);
+      }
+    } else {
+      for (const col of cols.slice(1)) {
+        cells.push(renderDataCell(col.key, col.isScore, pd, columnStats, multi));
+      }
+    }
+    return `<tr>${cells.join("")}</tr>`;
+  }).join("\n");
+  const winnerHtml = winnerId ? `<div class="task-winner">
+    <span class="trophy">&#x1F3C6;</span>
+    <span>Winner: <span class="winner-name">${esc(winnerId)}</span>
+    <span class="winner-label">${esc(providerLabel(winnerId))}</span></span>
+  </div>` : "";
+  return `<section class="task-section${index === 0 ? " active" : ""}" data-task-idx="${index}">
+  <h2 class="task-name">${esc(task)}</h2>
+  <table class="results-table">
+    <thead><tr>${ths}</tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+  ${winnerHtml}
+</section>`;
+}
+function renderDataCell(key, _isScore, pd, columnStats, multi) {
+  const colStats = columnStats.get(key);
+  if (key === "latency") {
+    const ms = pd.latencyMs;
+    if (ms === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(ms, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${ms}">${Math.round(ms)}ms</td>`;
+  }
+  if (key === "cost") {
+    const cost = pd.avgDetails.costUsd;
+    if (cost === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(cost, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${cost}">${esc(formatCost(cost))}</td>`;
+  }
+  if (key === "tokens") {
+    const tokens = pd.avgDetails.totalTokens;
+    if (tokens === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(tokens, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${tokens}">${tokens}</td>`;
+  }
+  const val = pd.avgScores[key];
+  if (val === void 0) return `<td class="rank-error">&mdash;</td>`;
+  const pct = Math.round(val * 100);
+  let rankCls;
+  if (multi && colStats) {
+    rankCls = rankClass_(val, colStats);
+  } else {
+    rankCls = val >= 0.8 ? "rank-best" : val >= 0.5 ? "rank-mid" : "rank-worst";
+  }
+  const barColor = val >= 0.8 ? "var(--green)" : val >= 0.5 ? "var(--yellow)" : "var(--red)";
+  return `<td class="score-cell ${rankCls}" data-sort-val="${val}">
+    <span class="score-val">${pct}%</span>
+    <div class="score-bar" style="width:${pct}%;background:${barColor}"></div>
+  </td>`;
+}
+function rankClass_(value, colStats) {
+  if (colStats.best === void 0 || colStats.worst === void 0) return "rank-neutral";
+  if (colStats.best === colStats.worst) return "rank-neutral";
+  if (value === colStats.best) return "rank-best";
+  if (value === colStats.worst) return "rank-worst";
+  return "rank-mid";
+}
+function renderSummary(byCorrectness, byLatency, byCost, overallWinner, multi) {
+  const cards = [];
+  if (byCorrectness) {
+    const pct = `${Math.round(byCorrectness.avg * 100)}%`;
+    const provider = multi ? `<div class="card-provider">${esc(byCorrectness.id)} ${esc(providerLabel(byCorrectness.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Most Correct" : "Avg Correctness"}</div>
+      <div class="card-value">${pct}</div>
+      ${provider}
+    </div>`);
+  }
+  if (byLatency && byLatency.avg !== Infinity) {
+    const ms = `${Math.round(byLatency.avg)}ms`;
+    const provider = multi ? `<div class="card-provider">${esc(byLatency.id)} ${esc(providerLabel(byLatency.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Fastest" : "Avg Latency"}</div>
+      <div class="card-value">${ms}</div>
+      ${provider}
+    </div>`);
+  }
+  if (byCost?.avg !== void 0) {
+    const cost = esc(formatCost(byCost.avg));
+    const provider = multi ? `<div class="card-provider">${esc(byCost.id)} ${esc(providerLabel(byCost.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Cheapest" : "Avg Cost"}</div>
+      <div class="card-value">${cost}</div>
+      ${provider}
+    </div>`);
+  }
+  if (overallWinner) {
+    cards.push(`<div class="summary-card">
+      <div class="card-label">Overall Winner</div>
+      <div class="card-value">&#x1F3C6;</div>
+      <div class="card-provider">${esc(overallWinner)} ${esc(providerLabel(overallWinner))}</div>
+    </div>`);
+  }
+  if (cards.length === 0) return "";
+  return `<section class="summary-section">
+  <h2 class="summary-title">Summary</h2>
+  <div class="summary-cards">
+    ${cards.join("\n    ")}
+  </div>
+</section>`;
+}
+function renderErrors(errors) {
+  const items = errors.map((e5) => {
+    const suffix = e5.count > 1 ? ` <span class="error-count">(&times;${e5.count})</span>` : "";
+    const hint = e5.hint ? `<div class="error-hint">${esc(e5.hint)}</div>` : "";
+    return `<div class="error-item">
+      <span class="error-provider">${esc(e5.providerId)}:</span>
+      <span class="error-msg">${esc(e5.error)}</span>${suffix}
+      ${hint}
+    </div>`;
+  }).join("\n");
+  return `<section class="errors-section">
+  <h2 class="errors-title" onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'block'">Errors</h2>
+  <div class="errors-list">
+    ${items}
+  </div>
+</section>`;
+}
+function renderFooter() {
+  return `<footer class="report-footer">
+  <div class="footer-brand">
+    Powered by <a href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">Agent Duelist</a>
+  </div>
+  <a class="footer-cta" href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">
+    &#x2B50; Star on GitHub
+  </a>
+</footer>`;
+}
+function renderScript(taskCount) {
+  return `<script>
+(function() {
+  /* Tab switching */
+  ${taskCount > 1 ? `
+  var tabs = document.querySelectorAll('.task-tab');
+  var sections = document.querySelectorAll('.task-section');
+  tabs.forEach(function(tab) {
+    tab.addEventListener('click', function() {
+      var idx = parseInt(tab.getAttribute('data-task'));
+      tabs.forEach(function(t) { t.classList.remove('active'); });
+      sections.forEach(function(s) { s.classList.remove('active'); });
+      tab.classList.add('active');
+      sections[idx].classList.add('active');
+    });
+  });` : ""}
+
+  /* Column sorting */
+  document.querySelectorAll('.results-table th').forEach(function(th, colIdx) {
+    var table = th.closest('table');
+    var asc = true;
+    th.addEventListener('click', function() {
+      var tbody = table.querySelector('tbody');
+      var rows = Array.from(tbody.querySelectorAll('tr'));
+      rows.sort(function(a, b) {
+        var aCell = a.children[colIdx];
+        var bCell = b.children[colIdx];
+        var aVal = aCell.getAttribute('data-sort-val');
+        var bVal = bCell.getAttribute('data-sort-val');
+        if (aVal !== null && bVal !== null) {
+          return asc ? parseFloat(aVal) - parseFloat(bVal) : parseFloat(bVal) - parseFloat(aVal);
+        }
+        var aText = aCell.textContent || '';
+        var bText = bCell.textContent || '';
+        return asc ? aText.localeCompare(bText) : bText.localeCompare(aText);
+      });
+      rows.forEach(function(row) { tbody.appendChild(row); });
+
+      /* Update sort arrows */
+      table.querySelectorAll('th .sort-arrow').forEach(function(a) { a.textContent = ''; });
+      th.querySelector('.sort-arrow').textContent = asc ? ' \\u25B2' : ' \\u25BC';
+      asc = !asc;
+    });
+  });
+})();
+</script>`;
+}
+
+// src/ci.ts
+import { readFileSync, writeFileSync, mkdirSync } from "fs";
+import { dirname } from "path";
+var LOWER_IS_BETTER = /* @__PURE__ */ new Set(["cost"]);
+var FLAKY_CV_THRESHOLD = 0.3;
+var T_CRITICAL_95 = {
+  1: 12.706,
+  2: 4.303,
+  3: 3.182,
+  4: 2.776,
+  5: 2.571,
+  6: 2.447,
+  7: 2.365,
+  8: 2.306,
+  9: 2.262,
+  10: 2.228,
+  15: 2.131,
+  20: 2.086,
+  25: 2.06,
+  30: 2.042
+};
+var T_CRITICAL_KEYS = Object.keys(T_CRITICAL_95).map(Number).sort((a7, b3) => a7 - b3);
+function tCritical(df) {
+  if (df <= 0) return 1.96;
+  if (T_CRITICAL_95[df] !== void 0) return T_CRITICAL_95[df];
+  const keys = T_CRITICAL_KEYS;
+  if (df > keys[keys.length - 1]) return 1.96;
+  for (let i7 = 0; i7 < keys.length - 1; i7++) {
+    if (df > keys[i7] && df < keys[i7 + 1]) {
+      const low = keys[i7], high = keys[i7 + 1];
+      const ratio = (df - low) / (high - low);
+      return T_CRITICAL_95[low] + ratio * (T_CRITICAL_95[high] - T_CRITICAL_95[low]);
+    }
+  }
+  return 1.96;
+}
+function computeScorerStats(samples) {
+  const n3 = samples.length;
+  if (n3 === 0) {
+    return { mean: 0, stddev: 0, cv: 0, n: 0, ci95Lower: 0, ci95Upper: 0 };
+  }
+  const mean = samples.reduce((a7, b3) => a7 + b3, 0) / n3;
+  if (n3 === 1) {
+    return { mean, stddev: 0, cv: 0, n: 1, ci95Lower: mean, ci95Upper: mean };
+  }
+  const variance = samples.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (n3 - 1);
+  const stddev = Math.sqrt(variance);
+  const cv = mean !== 0 ? stddev / Math.abs(mean) : 0;
+  const se4 = stddev / Math.sqrt(n3);
+  const t3 = tCritical(n3 - 1);
+  return {
+    mean,
+    stddev,
+    cv,
+    n: n3,
+    ci95Lower: mean - t3 * se4,
+    ci95Upper: mean + t3 * se4
+  };
+}
+function groupKey(providerId, taskName, scorerName) {
+  return `${providerId}::${taskName}::${scorerName}`;
+}
+function computeStats(results) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const r3 of results) {
+    if (r3.error) continue;
+    for (const score of r3.scores) {
+      if (score.value < 0) continue;
+      const key = groupKey(r3.providerId, r3.taskName, score.name);
+      if (!grouped.has(key)) grouped.set(key, []);
+      grouped.get(key).push(score.value);
+    }
+  }
+  const stats = /* @__PURE__ */ new Map();
+  for (const [key, samples] of grouped) {
+    stats.set(key, computeScorerStats(samples));
+  }
+  return stats;
+}
+function computeCostSummary(results, budget) {
+  let totalUsd = 0;
+  const perProvider = /* @__PURE__ */ new Map();
+  for (const r3 of results) {
+    if (r3.error) continue;
+    const costScore = r3.scores.find((s5) => s5.name === "cost");
+    if (!costScore || costScore.value < 0) continue;
+    const details = costScore.details;
+    const usd = details?.estimatedUsd ?? 0;
+    if (usd <= 0) continue;
+    totalUsd += usd;
+    perProvider.set(r3.providerId, (perProvider.get(r3.providerId) ?? 0) + usd);
+  }
+  return {
+    totalUsd,
+    perProvider,
+    budget,
+    overBudget: budget !== void 0 && totalUsd > budget
+  };
+}
+function compareResults(baselineStats, currentStats, thresholds, budget, currentResults) {
+  const comparisons = [];
+  const failureReasons = [];
+  for (const [key, current] of currentStats) {
+    const [providerId, taskName, scorerName] = key.split("::");
+    const baseline = baselineStats?.get(key) ?? null;
+    let delta = null;
+    let regressed = false;
+    let improved = false;
+    if (baseline) {
+      delta = current.mean - baseline.mean;
+      const threshold = thresholds.get(scorerName);
+      if (threshold !== void 0) {
+        const lowerIsBetter = LOWER_IS_BETTER.has(scorerName);
+        regressed = detectRegression(baseline, current, threshold, lowerIsBetter);
+        improved = detectImprovement(baseline, current, threshold, lowerIsBetter);
+      }
+    }
+    const flaky = current.n > 1 && current.cv > FLAKY_CV_THRESHOLD;
+    comparisons.push({
+      providerId,
+      taskName,
+      scorerName,
+      baseline,
+      current,
+      delta,
+      regressed,
+      improved,
+      flaky
+    });
+  }
+  const cost = computeCostSummary(currentResults ?? [], budget);
+  const regressions = comparisons.filter((c3) => c3.regressed);
+  if (regressions.length > 0) {
+    for (const r3 of regressions) {
+      failureReasons.push(
+        `${r3.providerId} \xD7 ${r3.taskName}: ${r3.scorerName} regressed by ${formatDelta(r3.delta)}`
+      );
+    }
+  }
+  if (cost.overBudget) {
+    failureReasons.push(
+      `Total cost $${cost.totalUsd.toFixed(4)} exceeds budget $${cost.budget.toFixed(2)}`
+    );
+  }
+  const flakyResults = comparisons.filter((c3) => c3.flaky);
+  const failed = failureReasons.length > 0;
+  return { comparisons, cost, failed, flakyResults, failureReasons };
+}
+function detectRegression(baseline, current, threshold, lowerIsBetter) {
+  if (baseline.n === 1 && current.n === 1) {
+    const delta = current.mean - baseline.mean;
+    if (lowerIsBetter) return delta > threshold;
+    return delta < -threshold;
+  }
+  if (lowerIsBetter) {
+    return current.ci95Lower - baseline.ci95Upper > threshold;
+  }
+  return baseline.ci95Upper - current.ci95Lower > threshold && current.mean < baseline.mean;
+}
+function detectImprovement(baseline, current, threshold, lowerIsBetter) {
+  if (baseline.n === 1 && current.n === 1) {
+    const delta = current.mean - baseline.mean;
+    if (lowerIsBetter) return delta < -threshold;
+    return delta > threshold;
+  }
+  if (lowerIsBetter) {
+    return baseline.ci95Lower - current.ci95Upper > threshold;
+  }
+  return current.ci95Lower - baseline.ci95Upper > threshold;
+}
+function loadBaseline(path) {
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const data = JSON.parse(raw);
+    const results = data.results ?? data;
+    if (!Array.isArray(results)) return null;
+    return {
+      timestamp: data.timestamp ?? "unknown",
+      results
+    };
+  } catch {
+    return null;
+  }
+}
+function saveBaseline(path, results) {
+  mkdirSync(dirname(path), { recursive: true });
+  const data = {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    results
+  };
+  writeFileSync(path, JSON.stringify(data, null, 2));
+}
+
+// src/github.ts
+import { readFileSync as readFileSync2 } from "fs";
+function detectGitHubContext() {
+  const token = process.env.GITHUB_TOKEN;
+  const repository = process.env.GITHUB_REPOSITORY;
+  const eventPath = process.env.GITHUB_EVENT_PATH;
+  if (!token || !repository) return null;
+  const [owner, repo] = repository.split("/");
+  if (!owner || !repo) return null;
+  let prNumber;
+  if (eventPath) {
+    try {
+      const event = JSON.parse(readFileSync2(eventPath, "utf-8"));
+      if (event.pull_request && typeof event.pull_request === "object") {
+        const pr = event.pull_request;
+        prNumber = pr.number;
+      }
+      if (!prNumber && event.issue && typeof event.issue === "object") {
+        const issue = event.issue;
+        if (issue.pull_request) {
+          prNumber = issue.number;
+        }
+      }
+    } catch {
+    }
+  }
+  if (!prNumber && process.env.DUELIST_PR_NUMBER) {
+    prNumber = parseInt(process.env.DUELIST_PR_NUMBER, 10);
+  }
+  if (!prNumber) return null;
+  return { token, owner, repo, prNumber };
+}
+var API_BASE = "https://api.github.com";
+function ghHeaders(token, extra) {
+  return {
+    Authorization: `Bearer ${token}`,
+    Accept: "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+    ...extra
+  };
+}
+async function findExistingComment(ctx, marker) {
+  let page = 1;
+  const perPage = 50;
+  while (true) {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=${perPage}&page=${page}`;
+    const res = await fetch(url, { headers: ghHeaders(ctx.token) });
+    if (!res.ok) return null;
+    const comments = await res.json();
+    if (comments.length === 0) break;
+    for (const comment of comments) {
+      if (comment.body?.includes(marker)) {
+        return comment.id;
+      }
+    }
+    if (comments.length < perPage) break;
+    page++;
+  }
+  return null;
+}
+async function upsertPrComment(ctx, body, marker) {
+  const existingId = await findExistingComment(ctx, marker);
+  if (existingId) {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/comments/${existingId}`;
+    const res = await fetch(url, {
+      method: "PATCH",
+      headers: ghHeaders(ctx.token, { "Content-Type": "application/json" }),
+      body: JSON.stringify({ body })
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      console.warn(`Failed to update PR comment: ${res.status} ${text}`);
+    }
+  } else {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
+    const res = await fetch(url, {
+      method: "POST",
+      headers: ghHeaders(ctx.token, { "Content-Type": "application/json" }),
+      body: JSON.stringify({ body })
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      console.warn(`Failed to create PR comment: ${res.status} ${text}`);
+    }
+  }
+}
+
+// src/cli.ts
+var __dirname2 = dirname2(fileURLToPath(import.meta.url));
+var program = new Command();
+program.name("duelist").description("Pit LLM providers against each other on agent tasks.").version(getVersion());
+program.command("init").description("Scaffold an arena.config.ts in the current directory").option("--force", "Overwrite existing config file").action((opts) => {
+  const target = resolve("arena.config.ts");
+  if (existsSync(target) && !opts.force) {
+    console.error("arena.config.ts already exists. Use --force to overwrite.");
+    process.exit(1);
   }
   const templatePath = join(__dirname2, "..", "templates", "arena.config.ts");
   let template;
   if (existsSync(templatePath)) {
-    template = readFileSync(templatePath, "utf-8");
+    template = readFileSync3(templatePath, "utf-8");
   } else {
     template = DEFAULT_TEMPLATE;
   }
-  writeFileSync(target, template);
+  writeFileSync2(target, template);
   console.log(existsSync(target) && opts.force ? "Overwrote arena.config.ts" : "Created arena.config.ts");
   console.log("");
   console.log("Next steps:");
   console.log("  1. export OPENAI_API_KEY=sk-...");
   console.log("  2. npx duelist run");
 });
-program.command("run").description("Run benchmarks defined in your arena config").option("-c, --config <path>", "Path to config file", "arena.config.ts").option("--reporter <type>", "Output format: console or json", "console").option("-q, --quiet", "Suppress per-result progress (show only final report)").action(async (opts) => {
-  const configPath = resolve(opts.config);
+program.command("run").description("Run benchmarks defined in your arena config").option("-c, --config <path>", "Path to config file", "arena.config.ts").option("--reporter <type>", "Output format: console, json, or html", "console").option("--output <path>", "Output file path (used with html reporter)", "duelist-report.html").option("-q, --quiet", "Suppress per-result progress (show only final report)").action(async (opts) => {
+  if (!["console", "json", "html"].includes(opts.reporter)) {
+    console.error(`Unknown reporter "${opts.reporter}". Use "console", "json", or "html".`);
+    process.exit(1);
+  }
+  const typedArena = await loadArenaConfig(opts.config);
+  try {
+    const showProgress = opts.reporter !== "json" && !opts.quiet;
+    const onResult = showProgress ? logResult : void 0;
+    const results = await typedArena.run({ onResult });
+    if (opts.reporter === "json") {
+      console.log(jsonReporter(results));
+    } else if (opts.reporter === "html") {
+      const html = htmlReporter(results);
+      const outPath = resolve(opts.output);
+      mkdirSync2(dirname2(outPath), { recursive: true });
+      writeFileSync2(outPath, html);
+      console.log(`
+HTML report written to ${outPath}`);
+    } else {
+      console.log("");
+      consoleReporter(results, { sparklines: typedArena.config?.sparklines });
+    }
+    const allFailed = results.length > 0 && results.every((r3) => r3.error);
+    if (allFailed) process.exit(1);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`Benchmark failed: ${message}`);
+    process.exit(1);
+  }
+});
+function collectThreshold(value, previous) {
+  const [scorer, delta] = value.split("=");
+  if (!scorer || delta === void 0 || isNaN(Number(delta))) {
+    console.error(`Invalid threshold format: "${value}". Expected scorer=delta (e.g., correctness=0.1)`);
+    process.exit(1);
+  }
+  previous.set(scorer, Number(delta));
+  return previous;
+}
+program.command("ci").description("Run benchmarks, compare against baseline, and enforce quality gates").option("-c, --config <path>", "Path to config file", "arena.config.ts").option("--baseline <path>", "Baseline JSON file", ".duelist/baseline.json").option("--budget <dollars>", "Max total cost in USD", parseFloat).option("--threshold <scorer=delta>", "Regression threshold (repeatable)", collectThreshold, /* @__PURE__ */ new Map()).option("--update-baseline", "Save results as new baseline after passing").option("--comment", "Post results as GitHub PR comment").option("-q, --quiet", "Suppress per-result progress").action(async (opts) => {
+  const ciOpts = {
+    configPath: opts.config,
+    baselinePath: resolve(opts.baseline),
+    budget: opts.budget,
+    thresholds: opts.threshold,
+    updateBaseline: opts.updateBaseline ?? false,
+    comment: opts.comment ?? false,
+    quiet: opts.quiet ?? false
+  };
+  const typedArena = await loadArenaConfig(ciOpts.configPath);
+  console.log("Running benchmarks...");
+  const onResult = ciOpts.quiet ? void 0 : logResult;
+  let results;
+  try {
+    results = await typedArena.run({ onResult });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`Benchmark failed: ${message}`);
+    process.exit(1);
+  }
+  const baseline = loadBaseline(ciOpts.baselinePath);
+  const baselineStats = baseline ? computeStats(baseline.results) : null;
+  if (baseline) {
+    console.log(`Loaded baseline from ${ciOpts.baselinePath} (${baseline.timestamp})`);
+  } else {
+    console.log("No baseline found \u2014 this run establishes the first baseline.");
+  }
+  const currentStats = computeStats(results);
+  const report = compareResults(baselineStats, currentStats, ciOpts.thresholds, ciOpts.budget, results);
+  console.log("");
+  consoleReporter(results, { sparklines: typedArena.config?.sparklines ?? true });
+  if (report.flakyResults.length > 0) {
+    console.log(`\u26A0  ${report.flakyResults.length} flaky result(s) detected (high variance)`);
+  }
+  if (report.cost.overBudget) {
+    console.log(`\u{1F534} Budget exceeded: $${report.cost.totalUsd.toFixed(4)} > $${report.cost.budget.toFixed(2)}`);
+  }
+  for (const reason of report.failureReasons) {
+    console.log(`\u{1F534} ${reason}`);
+  }
+  if (!report.failed) {
+    console.log("\u{1F7E2} CI passed");
+  }
+  if (ciOpts.comment) {
+    const ghCtx = detectGitHubContext();
+    if (ghCtx) {
+      const markdown = markdownReporter(report, results);
+      try {
+        await upsertPrComment(ghCtx, markdown, COMMENT_MARKER);
+        console.log("Posted results to PR comment.");
+      } catch (err) {
+        console.warn(`Failed to post PR comment: ${err instanceof Error ? err.message : err}`);
+      }
+    } else {
+      console.warn("--comment: not in a GitHub Actions PR context, skipping.");
+    }
+  }
+  if (ciOpts.updateBaseline && !report.failed) {
+    saveBaseline(ciOpts.baselinePath, results);
+    console.log(`Baseline saved to ${ciOpts.baselinePath}`);
+  } else if (ciOpts.updateBaseline && report.failed) {
+    console.log("Baseline not updated (CI failed).");
+  }
+  process.exit(report.failed ? 1 : 0);
+});
+program.parse();
+async function loadArenaConfig(configOpt) {
+  const configPath = resolve(configOpt);
   if (!existsSync(configPath)) {
     console.error(`Config not found: ${configPath}`);
     console.error("");
     console.error("Create one with: npx duelist init");
     process.exit(1);
   }
-  if (!["console", "json"].includes(opts.reporter)) {
-    console.error(`Unknown reporter "${opts.reporter}". Use "console" or "json".`);
-    process.exit(1);
-  }
   let mod;
   try {
     if (configPath.endsWith(".ts")) {
@@ -6591,35 +8005,21 @@ program.command("run").description("Run benchmarks defined in your arena config"
     console.error(`Loaded from: ${configPath}`);
     process.exit(1);
   }
-  try {
-    const typedArena = arena;
-    const showProgress = opts.reporter === "console" && !opts.quiet;
-    const onResult = showProgress ? (result) => {
-      if (result.error) {
-        console.log(`  ${result.providerId} \xD7 ${result.taskName}: ERROR ${result.error}`);
-      } else {
-        const scores = result.scores.map((s5) => `${s5.name}=${formatScoreForLog(s5)}`).join(" ");
-        console.log(`  ${result.providerId} \xD7 ${result.taskName}: ${scores}`);
-      }
-    } : void 0;
-    const results = await typedArena.run({ onResult });
-    const { consoleReporter: consoleReporter2 } = await Promise.resolve().then(() => (init_console(), console_exports));
-    const { jsonReporter: jsonReporter2 } = await Promise.resolve().then(() => (init_json(), json_exports));
-    if (opts.reporter === "json") {
-      console.log(jsonReporter2(results));
-    } else {
-      console.log("");
-      consoleReporter2(results);
-    }
-    const allFailed = results.length > 0 && results.every((r3) => r3.error);
-    if (allFailed) process.exit(1);
-  } catch (err) {
-    const message = err instanceof Error ? err.message : String(err);
-    console.error(`Benchmark failed: ${message}`);
-    process.exit(1);
+  const maybeConfig = arena.config;
+  if (maybeConfig === void 0 || maybeConfig === null || typeof maybeConfig !== "object") {
+    ;
+    arena.config = {};
   }
-});
-program.parse();
+  return arena;
+}
+function logResult(result) {
+  if (result.error) {
+    console.log(`  ${result.providerId} \xD7 ${result.taskName}: ERROR ${result.error}`);
+  } else {
+    const scores = result.scores.map((s5) => `${s5.name}=${formatScoreForLog(s5)}`).join(" ");
+    console.log(`  ${result.providerId} \xD7 ${result.taskName}: ${scores}`);
+  }
+}
 async function importTypeScript(filePath) {
   try {
     await Promise.resolve().then(() => (init_api(), api_exports));
@@ -6642,17 +8042,16 @@ function formatScoreForLog(s5) {
     return `${Math.round(details.ms)}ms`;
   }
   if (s5.name === "cost" && details?.estimatedUsd != null) {
-    const usd = details.estimatedUsd;
-    if (usd === 0) return "$0.00";
-    if (usd >= 0.01) return `~$${usd.toFixed(2)}`;
-    const digits = Math.max(4, -Math.floor(Math.log10(usd)) + 1);
-    return `~$${usd.toFixed(digits).replace(/0+$/, "")}`;
+    return formatCost(details.estimatedUsd);
+  }
+  if (s5.value < 0 && details?.reason) {
+    return `SKIP (${details.reason})`;
   }
   return String(s5.value);
 }
 function getVersion() {
   try {
-    const pkg = readFileSync(join(__dirname2, "..", "package.json"), "utf-8");
+    const pkg = readFileSync3(join(__dirname2, "..", "package.json"), "utf-8");
     return JSON.parse(pkg).version ?? "0.0.0";
   } catch {
     return "0.0.0";
@@ -6674,12 +8073,12 @@ import { z } from 'zod'
 
 export default defineArena({
   providers: [
-    openai('gpt-4o-mini'),
+    openai('gpt-5-mini'),
     // Add more providers to compare:
-    // openai('gpt-4o'),
-    // azureOpenai('gpt-4o-mini'),
-    // anthropic('claude-sonnet-4-20250514'),
-    // gemini('gemini-2.5-flash'),
+    // openai('gpt-5.2'),
+    // azureOpenai('gpt-5-mini'),
+    // anthropic('claude-sonnet-4.6'),
+    // gemini('gemini-3-flash-preview'),
   ],
 
   tasks: [