agent-duelist 0.1.2 → 0.2.1

package/dist/index.js

@@ -1410,11 +1410,13 @@ var fuzzySimilarityScorer = ({ task, result }) => {
   }
   const a = stringify(task.expected);
   const b = stringify(result.output);
-  const similarity = jaccardSimilarity(tokenize(a), tokenize(b));
+  const setA = tokenize(a);
+  const setB = tokenize(b);
+  const similarity = jaccardSimilarity(setA, setB);
   return {
     name: "fuzzy-similarity",
     value: Math.round(similarity * 100) / 100,
-    details: { method: "jaccard", expectedTokens: tokenize(a).size, actualTokens: tokenize(b).size }
+    details: { method: "jaccard", expectedTokens: setA.size, actualTokens: setB.size }
   };
 };
 function stringify(value) {
@@ -1435,7 +1437,163 @@ function jaccardSimilarity(a, b) {
 }
 
 // src/scorers/llm-judge.ts
+import OpenAI2, { AzureOpenAI as AzureOpenAI2 } from "openai";
+
+// src/providers/openai.ts
 import OpenAI, { AzureOpenAI } from "openai";
+import { zodToJsonSchema } from "zod-to-json-schema";
+
+// src/providers/shared.ts
+var SCHEMA_SYSTEM_MESSAGE = "Respond with valid JSON matching the requested schema.";
+function parseSchemaOutput(rawContent, hasSchema) {
+  if (!hasSchema) return rawContent;
+  try {
+    return JSON.parse(rawContent);
+  } catch {
+    return rawContent;
+  }
+}
+
+// src/providers/openai.ts
+var REQUEST_TIMEOUT_MS = 6e4;
+function openai(model, options) {
+  const client = new OpenAI({
+    apiKey: options?.apiKey ?? process.env.OPENAI_API_KEY,
+    baseURL: options?.baseURL,
+    timeout: options?.timeoutMs ?? REQUEST_TIMEOUT_MS
+  });
+  return makeProvider(`openai/${model}`, "OpenAI", model, client, model);
+}
+function openaiCompatible(options) {
+  const apiKey = options.apiKey ?? (options.apiKeyEnv ? process.env[options.apiKeyEnv] : void 0) ?? "no-key";
+  const client = new OpenAI({
+    apiKey,
+    baseURL: options.baseURL,
+    timeout: options.timeoutMs ?? REQUEST_TIMEOUT_MS
+  });
+  if (options.free) {
+    registerPricing(options.id, { inputPerToken: 0, outputPerToken: 0 });
+  }
+  return makeProvider(options.id, options.name, options.model, client, options.model, options.stripThinking);
+}
+function azureOpenai(model, options) {
+  const deployment = options?.deployment ?? model;
+  const client = new AzureOpenAI({
+    apiKey: options?.apiKey ?? process.env.AZURE_OPENAI_API_KEY,
+    endpoint: options?.endpoint ?? process.env.AZURE_OPENAI_ENDPOINT,
+    apiVersion: options?.apiVersion ?? process.env.AZURE_OPENAI_API_VERSION ?? "2024-12-01-preview",
+    deployment,
+    timeout: options?.timeoutMs ?? REQUEST_TIMEOUT_MS
+  });
+  return makeProvider(`azure/${model}`, "Azure OpenAI", model, client, deployment);
+}
+function makeProvider(id, name, model, client, requestModel, stripThinking) {
+  return {
+    id,
+    name,
+    model,
+    async run(input) {
+      const start = Date.now();
+      const params = {
+        model: requestModel,
+        messages: [{ role: "user", content: input.prompt }]
+      };
+      if (input.schema) {
+        params.response_format = { type: "json_object" };
+        params.messages = [
+          { role: "system", content: SCHEMA_SYSTEM_MESSAGE },
+          ...params.messages
+        ];
+      }
+      if (input.tools?.length) {
+        params.tools = input.tools.map(toolDefToOpenAI);
+        params.tool_choice = "auto";
+      }
+      const response = await client.chat.completions.create(params, { signal: input.signal });
+      let totalPromptTokens = response.usage?.prompt_tokens ?? 0;
+      let totalCompletionTokens = response.usage?.completion_tokens ?? 0;
+      const choice = response.choices[0];
+      const toolCallsRaw = choice?.message?.tool_calls;
+      const collectedToolCalls = [];
+      let finalResponse = response;
+      if (toolCallsRaw?.length && input.tools?.length) {
+        const toolMessages = [
+          ...params.messages,
+          choice.message
+        ];
+        for (const tc of toolCallsRaw) {
+          const toolDef = input.tools.find((t) => t.name === tc.function.name);
+          let args;
+          try {
+            args = JSON.parse(tc.function.arguments);
+          } catch {
+            args = tc.function.arguments;
+          }
+          let result;
+          if (toolDef?.handler) {
+            result = await toolDef.handler(args);
+          }
+          collectedToolCalls.push({ name: tc.function.name, arguments: args, result });
+          toolMessages.push({
+            role: "tool",
+            tool_call_id: tc.id,
+            content: JSON.stringify(result ?? {})
+          });
+        }
+        const followUp = await client.chat.completions.create({
+          model: requestModel,
+          messages: toolMessages
+        }, { signal: input.signal });
+        totalPromptTokens += followUp.usage?.prompt_tokens ?? 0;
+        totalCompletionTokens += followUp.usage?.completion_tokens ?? 0;
+        finalResponse = followUp;
+      }
+      const latencyMs = Date.now() - start;
+      const finalChoice = finalResponse.choices[0];
+      let rawContent = finalChoice?.message?.content ?? "";
+      if (stripThinking) {
+        rawContent = rawContent.replace(/<think>[\s\S]*?<\/think>\s*/, "");
+      }
+      const output = parseSchemaOutput(rawContent, !!input.schema);
+      return {
+        output,
+        usage: {
+          promptTokens: totalPromptTokens || void 0,
+          completionTokens: totalCompletionTokens || void 0
+        },
+        latencyMs,
+        raw: finalResponse,
+        toolCalls: collectedToolCalls.length > 0 ? collectedToolCalls : void 0
+      };
+    }
+  };
+}
+function gemini(model, options) {
+  const apiKey = options?.apiKey ?? process.env.GOOGLE_API_KEY;
+  if (!apiKey) {
+    throw new Error(
+      `Missing API key for google/${model}. Set GOOGLE_API_KEY or pass apiKey option.`
+    );
+  }
+  const client = new OpenAI({
+    apiKey,
+    baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
+    timeout: options?.timeoutMs ?? REQUEST_TIMEOUT_MS
+  });
+  return makeProvider(`google/${model}`, "Google AI", model, client, model);
+}
+function toolDefToOpenAI(tool) {
+  return {
+    type: "function",
+    function: {
+      name: tool.name,
+      description: tool.description,
+      parameters: zodToJsonSchema(tool.parameters, { target: "openAi" })
+    }
+  };
+}
+
+// src/scorers/llm-judge.ts
 var JUDGE_PROMPT = `You are a strict scoring judge. Evaluate the actual output against the expected output on three criteria. Score each from 0.0 to 1.0 using the full range (not just 0, 0.5, 1).
 
 Criteria:
@@ -1451,40 +1609,42 @@ conciseness: <number>
 Task: {task}
 Expected: {expected}
 Actual: {actual}`;
-function resolveJudgeClient(configModel) {
-  const model = configModel ?? process.env.DUELIST_JUDGE_MODEL ?? "gpt-4o-mini";
+function resolveJudgeClient(configModel, timeoutMs = REQUEST_TIMEOUT_MS) {
+  const model = configModel ?? process.env.DUELIST_JUDGE_MODEL ?? "gpt-5-mini";
   if (model.startsWith("gemini") && process.env.GOOGLE_API_KEY) {
     return {
-      client: new OpenAI({
+      client: new OpenAI2({
         apiKey: process.env.GOOGLE_API_KEY,
-        baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/"
+        baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
+        timeout: timeoutMs
       }),
       model
     };
   }
   if (!process.env.OPENAI_API_KEY && process.env.AZURE_OPENAI_API_KEY) {
     return {
-      client: new AzureOpenAI({
+      client: new AzureOpenAI2({
         apiKey: process.env.AZURE_OPENAI_API_KEY,
         endpoint: process.env.AZURE_OPENAI_ENDPOINT,
         apiVersion: process.env.AZURE_OPENAI_API_VERSION ?? "2024-12-01-preview",
-        deployment: model
+        deployment: model,
+        timeout: timeoutMs
       }),
       model
     };
   }
   const apiKey = process.env.OPENAI_API_KEY;
   if (!apiKey) return void 0;
-  return { client: new OpenAI({ apiKey }), model };
+  return { client: new OpenAI2({ apiKey, timeout: timeoutMs }), model };
 }
-function createLlmJudgeScorer(judgeModel) {
+function createLlmJudgeScorer(judgeModel, timeoutMs = REQUEST_TIMEOUT_MS) {
   let cached = void 0;
   return async ({ task, result }) => {
     if (task.expected === void 0) {
       return { name: "llm-judge-correctness", value: -1, details: { reason: "no expected value" } };
     }
     if (cached === void 0) {
-      cached = resolveJudgeClient(judgeModel) ?? null;
+      cached = resolveJudgeClient(judgeModel, timeoutMs) ?? null;
     }
     if (!cached) {
       return {
@@ -1499,8 +1659,7 @@ function createLlmJudgeScorer(judgeModel) {
       const response = await client.chat.completions.create({
         model,
         messages: [{ role: "user", content: prompt }],
-        temperature: 0,
-        max_tokens: 2048
+        max_completion_tokens: 2048
       });
       const content = response.choices[0]?.message?.content?.trim() ?? "";
       const parsed = {};
@@ -1557,10 +1716,10 @@ var staticScorers = {
   "fuzzy-similarity": fuzzySimilarityScorer,
   "tool-usage": toolUsageScorer
 };
-function resolveScorers(names, judgeModel) {
+function resolveScorers(names, judgeModel, timeoutMs) {
   return names.map((name) => {
     if (name === "llm-judge-correctness") {
-      return createLlmJudgeScorer(judgeModel);
+      return createLlmJudgeScorer(judgeModel, timeoutMs);
     }
     const scorer = staticScorers[name];
     if (!scorer) {
@@ -1571,219 +1730,156 @@ function resolveScorers(names, judgeModel) {
 }
 
 // src/runner.ts
+var DEFAULT_TIMEOUT_MS = 6e4;
+function withTimeout(run, ms) {
+  return new Promise((resolve, reject) => {
+    const controller = new AbortController();
+    const timer = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`Request timed out after ${ms}ms`));
+    }, ms);
+    run(controller.signal).then(
+      (v) => {
+        clearTimeout(timer);
+        resolve(v);
+      },
+      (e) => {
+        clearTimeout(timer);
+        reject(e);
+      }
+    );
+  });
+}
 async function runBenchmarks(options) {
   const { providers, tasks, scorers, runs, onResult } = options;
+  const timeout = options.timeout ?? DEFAULT_TIMEOUT_MS;
   const results = [];
   for (const task of tasks) {
-    for (const provider of providers) {
-      for (let run = 1; run <= runs; run++) {
-        let result;
-        try {
-          const taskResult = await provider.run({
-            prompt: task.prompt,
-            schema: task.schema,
-            tools: task.tools
-          });
-          const scores = await Promise.all(
-            scorers.map((scorer) => scorer({ task, result: taskResult }, provider.id))
-          );
-          result = {
-            providerId: provider.id,
-            taskName: task.name,
-            run,
-            scores,
-            raw: {
-              output: taskResult.output,
-              latencyMs: taskResult.latencyMs,
-              usage: taskResult.usage,
-              toolCalls: taskResult.toolCalls
-            }
-          };
-        } catch (err) {
-          const message = err instanceof Error ? err.message : String(err);
-          result = {
-            providerId: provider.id,
-            taskName: task.name,
-            run,
-            scores: [],
-            error: message,
-            raw: { output: "", latencyMs: 0 }
-          };
-        }
-        results.push(result);
-        onResult?.(result);
-      }
+    for (let run = 1; run <= runs; run++) {
+      const runResults = await Promise.all(
+        providers.map(async (provider) => {
+          let result;
+          try {
+            const taskResult = await withTimeout((signal) => provider.run({
+              prompt: task.prompt,
+              schema: task.schema,
+              tools: task.tools,
+              signal
+            }), timeout);
+            const scores = await Promise.all(
+              scorers.map((scorer) => scorer({ task, result: taskResult }, provider.id))
+            );
+            result = {
+              providerId: provider.id,
+              taskName: task.name,
+              run,
+              scores,
+              raw: {
+                output: taskResult.output,
+                latencyMs: taskResult.latencyMs,
+                usage: taskResult.usage,
+                toolCalls: taskResult.toolCalls
+              }
+            };
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            result = {
+              providerId: provider.id,
+              taskName: task.name,
+              run,
+              scores: [],
+              error: message,
+              raw: { output: "", latencyMs: 0 }
+            };
+          }
+          onResult?.(result);
+          return result;
+        })
+      );
+      results.push(...runResults);
     }
   }
   return results;
 }
 
-// src/reporter/console.ts
-var reset = "\x1B[0m";
-var boldCode = "\x1B[1m";
-var dimCode = "\x1B[2m";
-var green = "\x1B[32m";
-var red = "\x1B[31m";
-var yellow = "\x1B[33m";
-var cyan = "\x1B[36m";
-function bold(s) {
-  return `${boldCode}${s}${reset}`;
-}
-function dim(s) {
-  return `${dimCode}${s}${reset}`;
+// src/utils/format.ts
+var MAX_FRACTION_DIGITS = 100;
+function formatCost(usd) {
+  if (usd === void 0) return "\u2014";
+  if (usd === 0) return "$0.00";
+  if (usd >= 0.01) return `~$${usd.toFixed(2)}`;
+  const digits = Math.min(
+    MAX_FRACTION_DIGITS,
+    Math.max(4, -Math.floor(Math.log10(Math.abs(usd))) + 1)
+  );
+  return `~$${usd.toFixed(digits).replace(/0+$/, "")}`;
 }
-function colorScore(value) {
-  const pct = Math.round(value * 100);
-  const str = `${pct}%`;
-  if (value >= 0.8) return `${green}${str}${reset}`;
-  if (value >= 0.5) return `${yellow}${str}${reset}`;
-  return `${red}${str}${reset}`;
+function formatDelta(delta, precision = 4) {
+  const sign = delta >= 0 ? "+" : "";
+  return `${sign}${delta.toFixed(precision)}`;
 }
-function consoleReporter(results) {
-  if (results.length === 0) {
-    console.log("\nNo results to display.\n");
-    return;
-  }
-  const tasks = [...new Set(results.map((r) => r.taskName))];
-  const providers = [...new Set(results.map((r) => r.providerId))];
-  const scorerNames = [...new Set(results.flatMap((r) => r.scores.map((s) => s.name)))];
-  const hasCost = scorerNames.includes("cost");
-  const hasErrors = results.some((r) => r.error);
-  const runsPerCell = Math.max(...results.map((r) => r.run));
-  const runLabel = runsPerCell > 1 ? ` (${runsPerCell} runs each)` : "";
-  console.log("");
-  console.log(`  ${bold(`\u2B21 Agent Duelist Results${runLabel}`)}`);
-  console.log(`  ${dim("\u2500".repeat(70))}`);
-  console.log("");
-  for (const task of tasks) {
-    console.log(`  ${bold(`Task: ${task}`)}`);
-    const cols = [{ label: "Provider", width: 22, align: "left" }];
-    for (const name of scorerNames) {
-      if (name === "latency") cols.push({ label: "Latency", width: 10, align: "right" });
-      else if (name === "cost") {
-        cols.push({ label: "Cost", width: 12, align: "right" });
-        cols.push({ label: "Tokens", width: 9, align: "right" });
-      } else if (name === "correctness") cols.push({ label: "Match", width: 8, align: "right" });
-      else if (name === "schema-correctness") cols.push({ label: "Schema", width: 8, align: "right" });
-      else if (name === "fuzzy-similarity") cols.push({ label: "Fuzzy", width: 8, align: "right" });
-      else if (name === "llm-judge-correctness") cols.push({ label: "Judge", width: 8, align: "right" });
-      else if (name === "tool-usage") cols.push({ label: "Tool", width: 8, align: "right" });
-      else cols.push({ label: name, width: 10, align: "right" });
-    }
-    if (hasErrors) cols.push({ label: "Status", width: 8, align: "left" });
-    const totalWidth = cols.reduce((sum, c) => sum + c.width + 2, 0);
-    console.log(`  ${dim(cols.map((c) => pad(c.label, c.width + 2, c.align)).join(""))}`);
-    console.log(`  ${dim("\u2500".repeat(totalWidth))}`);
-    for (const provider of providers) {
-      const taskResults = results.filter(
-        (r) => r.taskName === task && r.providerId === provider
-      );
-      const errorResults2 = taskResults.filter((r) => r.error);
-      const successResults = taskResults.filter((r) => !r.error);
-      if (successResults.length === 0 && errorResults2.length > 0) {
-        const cells2 = [pad(provider, 24, "left")];
-        for (const name of scorerNames) {
-          if (name === "cost") {
-            cells2.push(pad("\u2014", 14, "right"));
-            cells2.push(pad("\u2014", 11, "right"));
-          } else cells2.push(pad("\u2014", cols.find((c) => c.label !== "Provider").width + 2, "right"));
-        }
-        if (hasErrors) cells2.push(`  ${red}FAIL${reset}`);
-        console.log(`  ${cells2.join("")}`);
-        continue;
-      }
-      const avgScores = averageScores(successResults);
-      const avgDetails = averageDetails(successResults);
-      const latencyMs = average(successResults.map((r) => r.raw.latencyMs));
-      const cells = [pad(provider, 24, "left")];
-      for (const name of scorerNames) {
-        if (name === "latency") {
-          cells.push(pad(latencyMs !== void 0 ? `${Math.round(latencyMs)}ms` : "\u2014", 12, "right"));
-        } else if (name === "cost") {
-          cells.push(pad(formatCost(avgDetails.costUsd), 14, "right"));
-          cells.push(pad(avgDetails.totalTokens !== void 0 ? `${avgDetails.totalTokens}` : "\u2014", 11, "right"));
-        } else {
-          const val = avgScores[name];
-          if (val === void 0) cells.push(pad("\u2014", 10, "right"));
-          else cells.push(pad(colorScore(val), 10 + colorLen(colorScore(val)), "right"));
-        }
-      }
-      if (hasErrors) {
-        const failCount = errorResults2.length;
-        cells.push(failCount > 0 ? `  ${yellow}${failCount} err${reset}` : `  ${green}OK${reset}`);
-      }
-      console.log(`  ${cells.join("")}`);
+
+// src/reporter/shared.ts
+function groupResults(results) {
+  const taskSet = /* @__PURE__ */ new Set();
+  const providerSet = /* @__PURE__ */ new Set();
+  const scorerSet = /* @__PURE__ */ new Set();
+  const grouped = /* @__PURE__ */ new Map();
+  const byProvider = /* @__PURE__ */ new Map();
+  let hasErrors = false;
+  let maxRun = 0;
+  for (const r of results) {
+    taskSet.add(r.taskName);
+    providerSet.add(r.providerId);
+    for (const s of r.scores) scorerSet.add(s.name);
+    if (r.error) hasErrors = true;
+    if (r.run > maxRun) maxRun = r.run;
+    const key = `${r.taskName}::${r.providerId}`;
+    let group = grouped.get(key);
+    if (!group) {
+      group = [];
+      grouped.set(key, group);
     }
-    console.log("");
-  }
-  printSummary(results, providers);
-  const errorResults = results.filter((r) => r.error);
-  if (errorResults.length > 0) {
-    console.log(`  ${bold("Errors")}`);
-    console.log(`  ${dim("\u2500".repeat(70))}`);
-    const seen = /* @__PURE__ */ new Set();
-    for (const r of errorResults) {
-      const key = `${r.providerId}::${r.error}`;
-      if (seen.has(key)) continue;
-      seen.add(key);
-      const count = errorResults.filter((e) => e.providerId === r.providerId && e.error === r.error).length;
-      const suffix = count > 1 ? ` (\xD7${count})` : "";
-      console.log(`  ${red}\u2717${reset} ${r.providerId}: ${r.error}${suffix}`);
-      const hint = apiKeyHint(r.providerId, r.error ?? "");
-      if (hint) console.log(`    ${dim(hint)}`);
+    group.push(r);
+    let provGroup = byProvider.get(r.providerId);
+    if (!provGroup) {
+      provGroup = [];
+      byProvider.set(r.providerId, provGroup);
     }
-    console.log("");
-  }
-  if (hasCost) {
-    console.log(dim(`  Costs estimated from OpenRouter pricing catalog. Run npx tsx scripts/update-pricing.ts to refresh.`));
-    console.log("");
+    provGroup.push(r);
   }
+  return {
+    tasks: [...taskSet],
+    providers: [...providerSet],
+    scorerNames: [...scorerSet],
+    grouped,
+    byProvider,
+    hasErrors,
+    maxRun
+  };
 }
-function printSummary(results, providers) {
-  const successResults = results.filter((r) => !r.error);
-  if (successResults.length === 0) return;
-  console.log(`  ${dim("\u2500".repeat(70))}`);
-  console.log(`  ${bold("Summary")}`);
-  console.log("");
-  const single = providers.length === 1;
-  const correctnessKey = successResults.some((r) => r.scores.some((s) => s.name === "llm-judge-correctness" && s.value >= 0)) ? "llm-judge-correctness" : "correctness";
-  const byCorrectness = rankProviders(successResults, providers, correctnessKey);
-  if (byCorrectness) {
-    const label = single ? "Avg correctness" : `Most correct: ${bold(byCorrectness.id)} ${dim(providerLabel(byCorrectness.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${colorScore(byCorrectness.avg)})`);
-  }
-  const byLatency = providers.map((id) => {
-    const runs = successResults.filter((r) => r.providerId === id);
-    const avg = average(runs.map((r) => r.raw.latencyMs));
-    return { id, avg: avg ?? Infinity };
-  }).sort((a, b) => a.avg - b.avg)[0];
-  if (byLatency && byLatency.avg !== Infinity) {
-    const label = single ? "Avg latency" : `Fastest: ${bold(byLatency.id)} ${dim(providerLabel(byLatency.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${Math.round(byLatency.avg)}ms)`);
-  }
-  const byCost = providers.map((id) => {
-    const runs = successResults.filter((r) => r.providerId === id);
-    const costs = runs.map((r) => {
-      const s = r.scores.find((s2) => s2.name === "cost");
-      return s && s.value >= 0 ? s.value : void 0;
-    }).filter((c) => c !== void 0);
-    const avg = costs.length > 0 ? costs.reduce((a, b) => a + b, 0) / costs.length : void 0;
-    return { id, avg };
-  }).filter((p) => p.avg !== void 0).sort((a, b) => a.avg - b.avg)[0];
-  if (byCost?.avg !== void 0) {
-    const label = single ? "Avg cost" : `Cheapest: ${bold(byCost.id)} ${dim(providerLabel(byCost.id))}`;
-    console.log(`  ${cyan}\u25C6${reset} ${label} (avg ${formatCost(byCost.avg)})`);
+function aggregateProviderTask(providerId, grouped, task) {
+  const taskResults = grouped.get(`${task}::${providerId}`) ?? [];
+  const errorResults = taskResults.filter((r) => r.error);
+  const successResults = taskResults.filter((r) => !r.error);
+  if (successResults.length === 0) {
+    return {
+      providerId,
+      avgScores: {},
+      avgDetails: { costUsd: void 0, totalTokens: void 0 },
+      latencyMs: void 0,
+      allErrors: errorResults.length > 0,
+      errorCount: errorResults.length
+    };
   }
-  console.log("");
-}
-function rankProviders(results, providers, scorerName) {
-  const ranked = providers.map((id) => {
-    const runs = results.filter((r) => r.providerId === id);
-    const scores = runs.flatMap((r) => r.scores.filter((s) => s.name === scorerName && s.value >= 0)).map((s) => s.value);
-    const avg = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : void 0;
-    return { id, avg };
-  }).filter((p) => p.avg !== void 0).sort((a, b) => b.avg - a.avg);
-  return ranked[0] ? { id: ranked[0].id, avg: ranked[0].avg } : void 0;
+  return {
+    providerId,
+    avgScores: averageScores(successResults),
+    avgDetails: averageDetails(successResults),
+    latencyMs: average(successResults.map((r) => r.raw.latencyMs)),
+    allErrors: false,
+    errorCount: errorResults.length
+  };
 }
 function averageScores(results) {
   const sums = {};
@@ -1827,38 +1923,89 @@ function average(nums) {
   if (nums.length === 0) return void 0;
   return nums.reduce((a, b) => a + b, 0) / nums.length;
 }
-function formatCost(usd) {
-  if (usd === void 0) return "\u2014";
-  if (usd === 0) return "$0.00";
-  if (usd >= 0.01) return `~$${usd.toFixed(2)}`;
-  const digits = Math.max(4, -Math.floor(Math.log10(usd)) + 1);
-  return `~$${usd.toFixed(digits).replace(/0+$/, "")}`;
+function computeColumnStats(providerData, scorerNames) {
+  const stats = /* @__PURE__ */ new Map();
+  const valid = providerData.filter((p) => !p.allErrors);
+  if (scorerNames.includes("latency")) {
+    const values = /* @__PURE__ */ new Map();
+    for (const p of providerData) {
+      values.set(p.providerId, p.allErrors ? void 0 : p.latencyMs);
+    }
+    const nums = valid.map((p) => p.latencyMs).filter((v) => v !== void 0);
+    stats.set("latency", {
+      values,
+      best: nums.length > 0 ? Math.min(...nums) : void 0,
+      worst: nums.length > 0 ? Math.max(...nums) : void 0
+    });
+  }
+  if (scorerNames.includes("cost")) {
+    const costValues = /* @__PURE__ */ new Map();
+    const tokenValues = /* @__PURE__ */ new Map();
+    for (const p of providerData) {
+      costValues.set(p.providerId, p.allErrors ? void 0 : p.avgDetails.costUsd);
+      tokenValues.set(p.providerId, p.allErrors ? void 0 : p.avgDetails.totalTokens);
+    }
+    const costNums = valid.map((p) => p.avgDetails.costUsd).filter((v) => v !== void 0);
+    const tokenNums = valid.map((p) => p.avgDetails.totalTokens).filter((v) => v !== void 0);
+    stats.set("cost", {
+      values: costValues,
+      best: costNums.length > 0 ? Math.min(...costNums) : void 0,
+      worst: costNums.length > 0 ? Math.max(...costNums) : void 0
+    });
+    stats.set("tokens", {
+      values: tokenValues,
+      best: tokenNums.length > 0 ? Math.min(...tokenNums) : void 0,
+      worst: tokenNums.length > 0 ? Math.max(...tokenNums) : void 0
+    });
+  }
+  for (const name of scorerNames) {
+    if (name === "latency" || name === "cost") continue;
+    const values = /* @__PURE__ */ new Map();
+    for (const p of providerData) {
+      values.set(p.providerId, p.allErrors ? void 0 : p.avgScores[name]);
+    }
+    const nums = valid.map((p) => p.avgScores[name]).filter((v) => v !== void 0);
+    stats.set(name, {
+      values,
+      best: nums.length > 0 ? Math.max(...nums) : void 0,
+      worst: nums.length > 0 ? Math.min(...nums) : void 0
+    });
+  }
+  return stats;
 }
-function pad(str, width, align) {
-  if (align === "right") return str.padStart(width);
-  return str.padEnd(width);
-}
-function colorLen(str) {
-  const stripped = str.replace(/\x1b\[[0-9;]*m/g, "");
-  return str.length - stripped.length;
-}
-function apiKeyHint(providerId, error) {
-  const lower = error.toLowerCase();
-  const isAuthError = lower.includes("api key") || lower.includes("401") || lower.includes("unauthorized") || lower.includes("authentication") || lower.includes("incorrect api key") || lower.includes("apikey");
-  if (!isAuthError) return void 0;
-  const prefix = providerId.split("/")[0];
-  switch (prefix) {
-    case "openai":
-      return "Set: export OPENAI_API_KEY=sk-...";
-    case "azure":
-      return "Set: export AZURE_OPENAI_API_KEY=... and AZURE_OPENAI_ENDPOINT=...";
-    case "anthropic":
-      return "Set: export ANTHROPIC_API_KEY=sk-ant-...";
-    case "google":
-      return "Set: export GOOGLE_API_KEY=...";
-    default:
-      return `Check the API key for ${providerId}`;
-  }
+function computeMedals(columnStats, providerIds) {
+  const medals = /* @__PURE__ */ new Map();
+  if (providerIds.length < 2) {
+    for (const id of providerIds) medals.set(id, "none");
+    return medals;
+  }
+  const wins = /* @__PURE__ */ new Map();
+  for (const id of providerIds) wins.set(id, 0);
+  for (const [, colStats] of columnStats) {
+    if (colStats.best === void 0) continue;
+    const bestProviders = [...colStats.values.entries()].filter(([, v]) => v !== void 0 && v === colStats.best);
+    if (bestProviders.length === 1) {
+      wins.set(bestProviders[0][0], (wins.get(bestProviders[0][0]) ?? 0) + 1);
+    }
+  }
+  const totalWins = [...wins.values()].reduce((a, b) => a + b, 0);
+  if (totalWins === 0) {
+    for (const id of providerIds) medals.set(id, "none");
+    return medals;
+  }
+  const sorted = [...wins.entries()].sort(
+    (a, b) => b[1] - a[1] || a[0].localeCompare(b[0])
+  );
+  const medalList = ["gold", "silver", "bronze"];
+  let rank = 0;
+  for (let i = 0; i < sorted.length; i++) {
+    if (i > 0 && sorted[i][1] < sorted[i - 1][1]) {
+      rank = i;
+    }
+    const hasWins = sorted[i][1] > 0;
+    medals.set(sorted[i][0], hasWins && rank < medalList.length ? medalList[rank] : "none");
+  }
+  return medals;
 }
 function providerLabel(providerId) {
   const prefix = providerId.split("/")[0];
@@ -1911,6 +2058,369 @@ function providerLabel(providerId) {
       return `(${prefix})`;
   }
 }
+function apiKeyHint(providerId, error) {
+  const lower = error.toLowerCase();
+  const isAuthError = lower.includes("api key") || lower.includes("401") || lower.includes("unauthorized") || lower.includes("authentication") || lower.includes("incorrect api key") || lower.includes("apikey");
+  if (!isAuthError) return void 0;
+  const prefix = providerId.split("/")[0];
+  switch (prefix) {
+    case "openai":
+      return "Set: export OPENAI_API_KEY=sk-...";
+    case "azure":
+      return "Set: export AZURE_OPENAI_API_KEY=... and AZURE_OPENAI_ENDPOINT=...";
+    case "anthropic":
+      return "Set: export ANTHROPIC_API_KEY=sk-ant-...";
+    case "google":
+      return "Set: export GOOGLE_API_KEY=...";
+    default:
+      return `Check the API key for ${providerId}`;
+  }
+}
+function rankProviders(successByProvider, providers, scorerName) {
+  const ranked = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const scores = runs.flatMap((r) => r.scores.filter((s) => s.name === scorerName && s.value >= 0)).map((s) => s.value);
+    const avg = scores.length > 0 ? scores.reduce((a, b) => a + b, 0) / scores.length : void 0;
+    return { id, avg };
+  }).filter((p) => p.avg !== void 0).sort((a, b) => b.avg - a.avg);
+  return ranked[0] ? { id: ranked[0].id, avg: ranked[0].avg } : void 0;
+}
+function scorerLabel(name) {
+  switch (name) {
+    case "correctness":
+      return "Match";
+    case "schema-correctness":
+      return "Schema";
+    case "fuzzy-similarity":
+      return "Fuzzy";
+    case "llm-judge-correctness":
+      return "Judge";
+    case "tool-usage":
+      return "Tool";
+    default:
+      return name;
+  }
+}
+function medalEmoji(medal) {
+  switch (medal) {
+    case "gold":
+      return "\u{1F947}";
+    case "silver":
+      return "\u{1F948}";
+    case "bronze":
+      return "\u{1F949}";
+    case "none":
+      return "";
+  }
+}
+
+// src/reporter/console.ts
+var reset = "\x1B[0m";
+var boldCode = "\x1B[1m";
+var dimCode = "\x1B[2m";
+var green = "\x1B[32m";
+var red = "\x1B[31m";
+var yellow = "\x1B[33m";
+var cyan = "\x1B[36m";
+var brightGreen = "\x1B[92m";
+var brightWhite = "\x1B[97m";
+function bold(s) {
+  return `${boldCode}${s}${reset}`;
+}
+function dim(s) {
+  return `${dimCode}${s}${reset}`;
+}
+function stripAnsi(s) {
+  return s.replace(/\x1b\[[0-9;]*m/g, "");
+}
+function displayWidth(s) {
+  const stripped = stripAnsi(s);
+  let width = 0;
+  for (const ch of stripped) {
+    const code = ch.codePointAt(0) ?? 0;
+    if (code >= 126976) width += 2;
+    else if (code >= 9728 && code <= 10175) width += 2;
+    else width += 1;
+  }
+  return width;
+}
+function padCell(str, targetWidth, align) {
+  const dw = displayWidth(str);
+  const padding = Math.max(0, targetWidth - dw);
+  if (align === "right") return " ".repeat(padding) + str;
+  return str + " ".repeat(padding);
+}
+function sparkBar(ratio, width = 8) {
+  const clamped = Math.max(0, Math.min(1, ratio));
+  const fillLen = Math.round(clamped * width);
+  const fill = "\u2593".repeat(fillLen);
+  const track = "\u2591".repeat(width - fillLen);
+  return { fill, track };
+}
+function drawTableLine(widths, position) {
+  const totalInner = widths.reduce((sum, w) => sum + w + 2, 0) + widths.length - 1;
+  if (position === "bottom") {
+    return dim(`\u2514${"\u2500".repeat(totalInner)}\u2518`);
+  }
+  if (position === "merge") {
+    return dim(`\u251C${"\u2500".repeat(totalInner)}\u2524`);
+  }
+  const segments = widths.map((w) => "\u2500".repeat(w + 2));
+  if (position === "top") {
+    return dim(`\u250C${segments.join("\u252C")}\u2510`);
+  }
+  return dim(`\u251C${segments.join("\u253C")}\u2524`);
+}
+function drawTableRow(cells, widths, aligns) {
+  const parts = cells.map(
+    (cell, i) => " " + padCell(cell, widths[i], aligns[i]) + " "
+  );
+  return dim("\u2502") + parts.join(dim("\u2502")) + dim("\u2502");
+}
+function drawSpanRow(content, widths) {
+  const totalInner = widths.reduce((sum, w) => sum + w + 2, 0) + widths.length - 1;
+  const dw = displayWidth(content);
+  const padding = Math.max(0, totalInner - dw - 1);
+  return dim("\u2502") + " " + content + " ".repeat(padding) + dim("\u2502");
+}
+function colorByRank(text, value, colStats, providerCount) {
+  if (value === void 0) return dim("\u2014");
+  if (providerCount < 2) return text;
+  if (colStats.best === void 0 || colStats.worst === void 0) return text;
+  if (colStats.best === colStats.worst) return text;
+  if (value === colStats.best) return `${brightGreen}${boldCode}${text}${reset}`;
+  if (value === colStats.worst) return `${red}${text}${reset}`;
+  return `${yellow}${text}${reset}`;
+}
+function consoleReporter(results, options) {
+  const showSparklines = options?.sparklines ?? true;
+  if (results.length === 0) {
+    console.log("\nNo results to display.\n");
+    return;
+  }
+  const { tasks, providers, scorerNames, grouped, byProvider, hasErrors, maxRun } = groupResults(results);
+  const hasCost = scorerNames.includes("cost");
+  const multi = providers.length >= 2;
+  const runsPerCell = maxRun;
+  const runLabel = runsPerCell > 1 ? `  ${dim(`(${runsPerCell} runs each)`)}` : "";
+  console.log("");
+  console.log(`  ${brightWhite}${boldCode}\u2B21  Agent Duelist${reset}${runLabel}`);
+  console.log(`  ${dim("\u2501".repeat(72))}`);
+  console.log("");
+  for (const task of tasks) {
+    console.log(`  ${bold(`Task: ${task}`)}`);
+    console.log("");
+    const providerData = providers.map(
+      (providerId) => aggregateProviderTask(providerId, grouped, task)
+    );
+    const columnStats = computeColumnStats(providerData, scorerNames);
+    const medals = computeMedals(columnStats, providers);
+    const maxProviderLen = Math.max(...providers.map((id) => id.length));
+    const providerWidth = Math.min(35, Math.max(22, maxProviderLen + 5));
+    const cols = [
+      { label: "Provider", width: providerWidth, align: "left" }
+    ];
+    for (const name of scorerNames) {
+      if (name === "latency") {
+        cols.push({ label: "Latency", width: 10, align: "right", statsKey: "latency" });
+      } else if (name === "cost") {
+        cols.push({ label: "Cost", width: 12, align: "right", statsKey: "cost" });
+        cols.push({ label: "Tokens", width: 9, align: "right", statsKey: "tokens" });
+      } else {
+        cols.push({ label: scorerLabel(name), width: showSparklines ? 15 : 8, align: "right", statsKey: name });
+      }
+    }
+    if (hasErrors) {
+      cols.push({ label: "Status", width: 8, align: "left" });
+    }
+    const widths = cols.map((c) => c.width);
+    const aligns = cols.map((c) => c.align);
+    console.log(`  ${drawTableLine(widths, "top")}`);
+    const headerCells = cols.map((c) => bold(c.label));
+    console.log(`  ${drawTableRow(headerCells, widths, aligns)}`);
+    console.log(`  ${drawTableLine(widths, "header")}`);
+    for (const pd of providerData) {
+      const medal = medalEmoji(medals.get(pd.providerId) ?? "none");
+      const providerCell = medal ? `${medal} ${pd.providerId}` : pd.providerId;
+      const cells = [providerCell];
+      if (pd.allErrors) {
+        for (const col of cols.slice(1)) {
+          if (col.label === "Status") {
+            cells.push(`${red}FAIL${reset}`);
+          } else {
+            cells.push(dim("\u2014"));
+          }
+        }
+      } else {
+        for (const col of cols.slice(1)) {
+          if (col.label === "Status") {
+            cells.push(
+              pd.errorCount > 0 ? `${yellow}${pd.errorCount} err${reset}` : `${green}OK${reset}`
+            );
+            continue;
+          }
+          const statsKey = col.statsKey;
+          const colStats = columnStats.get(statsKey);
+          if (statsKey === "latency") {
+            const ms = pd.latencyMs;
+            if (ms === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = `${Math.round(ms)}ms`;
+              cells.push(colStats ? colorByRank(text, ms, colStats, providers.length) : text);
+            }
+          } else if (statsKey === "cost") {
+            const cost = pd.avgDetails.costUsd;
+            if (cost === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = formatCost(cost);
+              cells.push(colStats ? colorByRank(text, cost, colStats, providers.length) : text);
+            }
+          } else if (statsKey === "tokens") {
+            const tokens = pd.avgDetails.totalTokens;
+            if (tokens === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const text = `${tokens}`;
+              cells.push(colStats ? colorByRank(text, tokens, colStats, providers.length) : text);
+            }
+          } else {
+            const val = pd.avgScores[statsKey];
+            if (val === void 0) {
+              cells.push(dim("\u2014"));
+            } else {
+              const pctStr = `${Math.round(val * 100)}%`.padStart(4);
+              let coloredPct;
+              if (multi && colStats) {
+                coloredPct = colorByRank(pctStr, val, colStats, providers.length);
+              } else {
+                if (val >= 0.8) coloredPct = `${green}${pctStr}${reset}`;
+                else if (val >= 0.5) coloredPct = `${yellow}${pctStr}${reset}`;
+                else coloredPct = `${red}${pctStr}${reset}`;
+              }
+              if (showSparklines) {
+                const { fill, track } = sparkBar(val);
+                const barColor = val >= 0.8 ? green : val >= 0.5 ? yellow : red;
+                cells.push(`${coloredPct} ${barColor}${fill}${reset}${dim(track)}`);
+              } else {
+                cells.push(coloredPct);
+              }
+            }
+          }
+        }
+      }
+      console.log(`  ${drawTableRow(cells, widths, aligns)}`);
+    }
+    if (multi && providerData.some((p) => !p.allErrors)) {
+      const winnerId = [...medals.entries()].find(([, m]) => m === "gold")?.[0];
+      if (winnerId) {
+        console.log(`  ${drawTableLine(widths, "merge")}`);
+        const winnerText = `${brightGreen}${boldCode}\u{1F3C6}  Winner: ${winnerId}${reset} ${dim(providerLabel(winnerId))}`;
+        console.log(`  ${drawSpanRow(winnerText, widths)}`);
+      }
+    }
+    console.log(`  ${drawTableLine(widths, "bottom")}`);
+    console.log("");
+  }
+  printSummary(results, providers, byProvider);
+  const errorResults = results.filter((r) => r.error);
+  if (errorResults.length > 0) {
+    console.log(`  ${bold("Errors")}`);
+    console.log(`  ${dim("\u2501".repeat(72))}`);
+    const seen = /* @__PURE__ */ new Set();
+    for (const r of errorResults) {
+      const key = `${r.providerId}::${r.error}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      const count = errorResults.filter((e) => e.providerId === r.providerId && e.error === r.error).length;
+      const suffix = count > 1 ? ` (\xD7${count})` : "";
+      console.log(`  ${red}\u2716${reset} ${r.providerId}: ${r.error}${suffix}`);
+      const hint = apiKeyHint(r.providerId, r.error ?? "");
+      if (hint) console.log(`    ${dim(hint)}`);
+    }
+    console.log("");
+  }
+  if (hasCost) {
+    console.log(dim(`  Costs estimated from OpenRouter pricing catalog. Run npx tsx scripts/update-pricing.ts to refresh.`));
+    console.log("");
+  }
+}
+function printSummary(results, providers, byProvider) {
+  const successResults = results.filter((r) => !r.error);
+  if (successResults.length === 0) return;
+  const successByProvider = /* @__PURE__ */ new Map();
+  for (const id of providers) {
+    successByProvider.set(id, (byProvider.get(id) ?? []).filter((r) => !r.error));
+  }
+  console.log(`  ${bold("Summary")}`);
+  console.log(`  ${dim("\u2501".repeat(72))}`);
+  console.log("");
+  const single = providers.length === 1;
+  const correctnessKey = successResults.some((r) => r.scores.some((s) => s.name === "llm-judge-correctness" && s.value >= 0)) ? "llm-judge-correctness" : "correctness";
+  const byCorrectness = rankProviders(successByProvider, providers, correctnessKey);
+  if (byCorrectness) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const pctStr = `${Math.round(byCorrectness.avg * 100)}%`;
+    if (single) {
+      console.log(`  ${medal} Avg correctness:  ${brightGreen}${boldCode}${pctStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Most correct:  ${bold(byCorrectness.id)} ${dim(providerLabel(byCorrectness.id))}  ${brightGreen}${boldCode}${pctStr}${reset}`);
+    }
+  }
+  const byLatency = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const avg = average(runs.map((r) => r.raw.latencyMs));
+    return { id, avg: avg ?? Infinity };
+  }).sort((a, b) => a.avg - b.avg)[0];
+  if (byLatency && byLatency.avg !== Infinity) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const msStr = `${Math.round(byLatency.avg)}ms`;
+    if (single) {
+      console.log(`  ${medal} Avg latency:     ${brightGreen}${boldCode}${msStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Fastest:       ${bold(byLatency.id)} ${dim(providerLabel(byLatency.id))}  ${brightGreen}${boldCode}${msStr}${reset}`);
+    }
+  }
+  const byCost = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const costs = runs.map((r) => {
+      const s = r.scores.find((s2) => s2.name === "cost");
+      return s && s.value >= 0 ? s.value : void 0;
+    }).filter((c) => c !== void 0);
+    const avg = costs.length > 0 ? costs.reduce((a, b) => a + b, 0) / costs.length : void 0;
+    return { id, avg };
+  }).filter((p) => p.avg !== void 0).sort((a, b) => a.avg - b.avg)[0];
+  if (byCost?.avg !== void 0) {
+    const medal = single ? `${cyan}\u25C6${reset}` : "\u{1F947}";
+    const costStr = formatCost(byCost.avg);
+    if (single) {
+      console.log(`  ${medal} Avg cost:        ${brightGreen}${boldCode}${costStr}${reset}`);
+    } else {
+      console.log(`  ${medal} Cheapest:      ${bold(byCost.id)} ${dim(providerLabel(byCost.id))}  ${brightGreen}${boldCode}${costStr}${reset}`);
+    }
+  }
+  if (!single) {
+    const wins = /* @__PURE__ */ new Map();
+    for (const id of providers) wins.set(id, 0);
+    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
+    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
+    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
+    const maxWins = Math.max(...wins.values());
+    if (maxWins > 0) {
+      const topProviders = [...wins.entries()].filter(([, w]) => w === maxWins);
+      console.log("");
+      if (topProviders.length === 1) {
+        const [winnerId, winCount] = topProviders[0];
+        console.log(`  \u{1F3C6} Overall:      ${brightGreen}${boldCode}${winnerId}${reset} ${dim(providerLabel(winnerId))}  ${dim(`(${winCount}/3 categories)`)}`);
+      } else {
+        const names = topProviders.map(([id]) => bold(id)).join(dim(", "));
+        console.log(`  \u{1F3C6} Overall:      ${names}  ${dim(`(tied at ${maxWins}/3)`)}`);
+      }
+    }
+  }
+  console.log("");
+}
 
 // src/reporter/json.ts
 function jsonReporter(results) {
@@ -1945,7 +2455,7 @@ function defineArena(config) {
     throw new Error("At least one task is required");
   }
   const scorerNames = config.scorers ?? ["latency", "cost", "correctness"];
-  const scorerFns = resolveScorers(scorerNames, config.judgeModel);
+  const scorerFns = resolveScorers(scorerNames, config.judgeModel, config.timeout);
   const runs = config.runs ?? 1;
   return {
     config,
@@ -1955,141 +2465,13 @@ function defineArena(config) {
         tasks: config.tasks,
         scorers: scorerFns,
         runs,
+        timeout: config.timeout,
         onResult: options?.onResult
       });
     }
   };
 }
 
-// src/providers/openai.ts
-import OpenAI2, { AzureOpenAI as AzureOpenAI2 } from "openai";
-import { zodToJsonSchema } from "zod-to-json-schema";
-function openai(model, options) {
-  const client = new OpenAI2({
-    apiKey: options?.apiKey ?? process.env.OPENAI_API_KEY,
-    baseURL: options?.baseURL
-  });
-  return makeProvider(`openai/${model}`, "OpenAI", model, client, model);
-}
-function openaiCompatible(options) {
-  const apiKey = options.apiKey ?? (options.apiKeyEnv ? process.env[options.apiKeyEnv] : void 0) ?? "no-key";
-  const client = new OpenAI2({
-    apiKey,
-    baseURL: options.baseURL
-  });
-  if (options.free) {
-    registerPricing(options.id, { inputPerToken: 0, outputPerToken: 0 });
-  }
-  return makeProvider(options.id, options.name, options.model, client, options.model, options.stripThinking);
-}
-function azureOpenai(model, options) {
-  const deployment = options?.deployment ?? model;
-  const client = new AzureOpenAI2({
-    apiKey: options?.apiKey ?? process.env.AZURE_OPENAI_API_KEY,
-    endpoint: options?.endpoint ?? process.env.AZURE_OPENAI_ENDPOINT,
-    apiVersion: options?.apiVersion ?? process.env.AZURE_OPENAI_API_VERSION ?? "2024-12-01-preview",
-    deployment
-  });
-  return makeProvider(`azure/${model}`, "Azure OpenAI", model, client, deployment);
-}
-function makeProvider(id, name, model, client, requestModel, stripThinking) {
-  return {
-    id,
-    name,
-    model,
-    async run(input) {
-      const start = Date.now();
-      const params = {
-        model: requestModel,
-        messages: [{ role: "user", content: input.prompt }]
-      };
-      if (input.schema) {
-        params.response_format = { type: "json_object" };
-        params.messages = [
-          { role: "system", content: "Respond with valid JSON matching the requested schema." },
-          ...params.messages
-        ];
-      }
-      if (input.tools?.length) {
-        params.tools = input.tools.map(toolDefToOpenAI);
-        params.tool_choice = "auto";
-      }
-      const response = await client.chat.completions.create(params);
-      let totalPromptTokens = response.usage?.prompt_tokens ?? 0;
-      let totalCompletionTokens = response.usage?.completion_tokens ?? 0;
-      const choice = response.choices[0];
-      const toolCallsRaw = choice?.message?.tool_calls;
-      const collectedToolCalls = [];
-      let finalResponse = response;
-      if (toolCallsRaw?.length && input.tools?.length) {
-        const toolMessages = [
-          ...params.messages,
-          choice.message
-        ];
-        for (const tc of toolCallsRaw) {
-          const toolDef = input.tools.find((t) => t.name === tc.function.name);
-          let args;
-          try {
-            args = JSON.parse(tc.function.arguments);
-          } catch {
-            args = tc.function.arguments;
-          }
-          let result;
-          if (toolDef?.handler) {
-            result = await toolDef.handler(args);
-          }
-          collectedToolCalls.push({ name: tc.function.name, arguments: args, result });
-          toolMessages.push({
-            role: "tool",
-            tool_call_id: tc.id,
-            content: JSON.stringify(result ?? {})
-          });
-        }
-        const followUp = await client.chat.completions.create({
-          model: requestModel,
-          messages: toolMessages
-        });
-        totalPromptTokens += followUp.usage?.prompt_tokens ?? 0;
-        totalCompletionTokens += followUp.usage?.completion_tokens ?? 0;
-        finalResponse = followUp;
-      }
-      const latencyMs = Date.now() - start;
-      const finalChoice = finalResponse.choices[0];
-      let rawContent = finalChoice?.message?.content ?? "";
-      if (stripThinking) {
-        rawContent = rawContent.replace(/<think>[\s\S]*?<\/think>\s*/, "");
-      }
-      let output = rawContent;
-      if (input.schema) {
-        try {
-          output = JSON.parse(rawContent);
-        } catch {
-        }
-      }
-      return {
-        output,
-        usage: {
-          promptTokens: totalPromptTokens || void 0,
-          completionTokens: totalCompletionTokens || void 0
-        },
-        latencyMs,
-        raw: finalResponse,
-        toolCalls: collectedToolCalls.length > 0 ? collectedToolCalls : void 0
-      };
-    }
-  };
-}
-function toolDefToOpenAI(tool) {
-  return {
-    type: "function",
-    function: {
-      name: tool.name,
-      description: tool.description,
-      parameters: zodToJsonSchema(tool.parameters, { target: "openAi" })
-    }
-  };
-}
-
 // src/providers/anthropic.ts
 import Anthropic from "@anthropic-ai/sdk";
 function anthropic(model, options) {
@@ -2103,23 +2485,17 @@ function anthropic(model, options) {
     model,
     async run(input) {
       const start = Date.now();
-      const systemMessage = input.schema ? "Respond with valid JSON matching the requested schema." : void 0;
+      const systemMessage = input.schema ? SCHEMA_SYSTEM_MESSAGE : void 0;
       const response = await client.messages.create({
         model,
         max_tokens: maxTokens,
         system: systemMessage,
         messages: [{ role: "user", content: input.prompt }]
-      });
+      }, { signal: input.signal });
       const latencyMs = Date.now() - start;
       const textBlock = response.content.find((b) => b.type === "text");
       const rawContent = textBlock?.type === "text" ? textBlock.text : "";
-      let output = rawContent;
-      if (input.schema) {
-        try {
-          output = JSON.parse(rawContent);
-        } catch {
-        }
-      }
+      const output = parseSchemaOutput(rawContent, !!input.schema);
       return {
         output,
         usage: {
@@ -2133,30 +2509,1024 @@ function anthropic(model, options) {
   };
 }
 
-// src/providers/gemini.ts
-import OpenAI3 from "openai";
-function gemini(model, options) {
-  const apiKey = options?.apiKey ?? process.env.GOOGLE_API_KEY;
-  if (!apiKey) {
-    throw new Error(
-      `Missing API key for google/${model}. Set GOOGLE_API_KEY or pass apiKey option.`
-    );
+// src/reporter/markdown.ts
+var COMMENT_MARKER = "<!-- duelist-ci-report -->";
+function markdownReporter(report, _current) {
+  const lines = [COMMENT_MARKER, ""];
+  const status = report.failed ? "\u{1F534} Failed" : "\u{1F7E2} Passed";
+  lines.push(`## \u2B21 Agent Duelist CI \u2014 ${status}`);
+  lines.push("");
+  if (report.comparisons.length > 0) {
+    lines.push(markdownComparisonTable(report.comparisons));
+    lines.push("");
   }
-  const client = new OpenAI3({
-    apiKey,
-    baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/"
+  if (report.cost.totalUsd > 0 || report.cost.budget !== void 0) {
+    lines.push(markdownCostSummary(report.cost));
+    lines.push("");
+  }
+  if (report.flakyResults.length > 0) {
+    lines.push("### \u26A0\uFE0F Flaky Results");
+    lines.push("");
+    lines.push("These scorer/task combinations have high variance (CV > 0.3). Consider increasing `runs` or tightening prompts.");
+    lines.push("");
+    for (const f of report.flakyResults) {
+      lines.push(`- **${f.providerId}** \xD7 ${f.taskName} \u2192 ${f.scorerName} (CV = ${f.current.cv.toFixed(2)})`);
+    }
+    lines.push("");
+  }
+  if (report.failureReasons.length > 0) {
+    lines.push("### Failure Reasons");
+    lines.push("");
+    for (const reason of report.failureReasons) {
+      lines.push(`- ${reason}`);
+    }
+    lines.push("");
+  }
+  lines.push("---");
+  lines.push("*Generated by [agent-duelist](https://github.com/DataGobes/agent-duelist) CI*");
+  return lines.join("\n");
+}
+function markdownComparisonTable(comparisons) {
+  const lines = [];
+  lines.push("| Provider | Task | Scorer | Baseline | Current | Delta | Status |");
+  lines.push("|----------|------|--------|----------|---------|-------|--------|");
+  for (const c of comparisons) {
+    const baselineStr = c.baseline ? formatStats(c.baseline) : "\u2014";
+    const currentStr = formatStats(c.current);
+    const deltaStr = c.delta !== null ? formatDelta(c.delta, 3) : "\u2014";
+    const status = statusIndicator(c);
+    lines.push(`| ${c.providerId} | ${c.taskName} | ${c.scorerName} | ${baselineStr} | ${currentStr} | ${deltaStr} | ${status} |`);
+  }
+  return lines.join("\n");
+}
+function markdownCostSummary(cost) {
+  const lines = [];
+  lines.push("### \u{1F4B0} Cost Summary");
+  lines.push("");
+  lines.push(`**Total:** $${cost.totalUsd.toFixed(4)}`);
+  if (cost.budget !== void 0) {
+    const pct = cost.budget > 0 ? (cost.totalUsd / cost.budget * 100).toFixed(0) : "\u221E";
+    const indicator = cost.overBudget ? "\u{1F534}" : "\u{1F7E2}";
+    lines.push(`**Budget:** $${cost.budget.toFixed(2)} (${pct}% used) ${indicator}`);
+  }
+  if (cost.perProvider.size > 1) {
+    lines.push("");
+    lines.push("| Provider | Cost |");
+    lines.push("|----------|------|");
+    for (const [provider, usd] of cost.perProvider) {
+      lines.push(`| ${provider} | $${usd.toFixed(4)} |`);
+    }
+  }
+  return lines.join("\n");
+}
+function formatStats(stats) {
+  if (stats.n > 1) {
+    const margin = (stats.ci95Upper - stats.ci95Lower) / 2;
+    return `${stats.mean.toFixed(3)} \xB1 ${margin.toFixed(3)}`;
+  }
+  return stats.mean.toFixed(3);
+}
+function statusIndicator(c) {
+  if (c.regressed) return "\u{1F534} regressed";
+  if (c.improved) return "\u{1F7E2} improved";
+  if (c.baseline === null) return "\u{1F195} new";
+  return "\u26AA unchanged";
+}
+
+// src/reporter/html.ts
+function esc(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function htmlReporter(results) {
+  if (results.length === 0) {
+    return emptyReport();
+  }
+  const { tasks, providers, scorerNames, grouped, byProvider, maxRun } = groupResults(results);
+  const hasCost = scorerNames.includes("cost");
+  const multi = providers.length >= 2;
+  const runsLabel = maxRun > 1 ? `${maxRun} runs each` : "1 run";
+  const taskSections = tasks.map((task) => {
+    const providerData = providers.map((id) => aggregateProviderTask(id, grouped, task));
+    const columnStats = computeColumnStats(providerData, scorerNames);
+    const medals = computeMedals(columnStats, providers);
+    const winnerId = multi ? [...medals.entries()].find(([, m]) => m === "gold")?.[0] : void 0;
+    return { task, providerData, columnStats, medals, winnerId };
   });
-  return makeProvider(`google/${model}`, "Google AI", model, client, model);
+  const successResults = results.filter((r) => !r.error);
+  const successByProvider = /* @__PURE__ */ new Map();
+  for (const id of providers) {
+    successByProvider.set(id, (byProvider.get(id) ?? []).filter((r) => !r.error));
+  }
+  const correctnessKey = successResults.some(
+    (r) => r.scores.some((s) => s.name === "llm-judge-correctness" && s.value >= 0)
+  ) ? "llm-judge-correctness" : "correctness";
+  const byCorrectness = rankProviders(successByProvider, providers, correctnessKey);
+  const byLatency = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const avg = average(runs.map((r) => r.raw.latencyMs));
+    return { id, avg: avg ?? Infinity };
+  }).sort((a, b) => a.avg - b.avg)[0];
+  const byCost = providers.map((id) => {
+    const runs = successByProvider.get(id) ?? [];
+    const costs = runs.map((r) => {
+      const s = r.scores.find((s2) => s2.name === "cost");
+      return s && s.value >= 0 ? s.value : void 0;
+    }).filter((c) => c !== void 0);
+    const avg = costs.length > 0 ? costs.reduce((a, b) => a + b, 0) / costs.length : void 0;
+    return { id, avg };
+  }).filter((p) => p.avg !== void 0).sort((a, b) => a.avg - b.avg)[0];
+  let overallWinner;
+  if (multi) {
+    const wins = /* @__PURE__ */ new Map();
+    for (const id of providers) wins.set(id, 0);
+    if (byCorrectness) wins.set(byCorrectness.id, (wins.get(byCorrectness.id) ?? 0) + 1);
+    if (byLatency && byLatency.avg !== Infinity) wins.set(byLatency.id, (wins.get(byLatency.id) ?? 0) + 1);
+    if (byCost?.avg !== void 0) wins.set(byCost.id, (wins.get(byCost.id) ?? 0) + 1);
+    const maxWins = Math.max(...wins.values());
+    if (maxWins > 0) {
+      const tops = [...wins.entries()].filter(([, w]) => w === maxWins);
+      if (tops.length === 1) overallWinner = tops[0][0];
+    }
+  }
+  const errorResults = results.filter((r) => r.error);
+  const deduped = dedupeErrors(errorResults);
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Agent Duelist Report</title>
+<meta name="description" content="LLM provider benchmark results \u2014 ${providers.length} provider${providers.length !== 1 ? "s" : ""}, ${tasks.length} task${tasks.length !== 1 ? "s" : ""}">
+<meta property="og:title" content="Agent Duelist Report">
+<meta property="og:description" content="LLM provider benchmark: ${providers.map(esc).join(" vs ")}">
+<meta property="og:type" content="website">
+${renderStyle()}
+</head>
+<body>
+<div class="bg-mesh"><div class="bg-mesh-extra"></div></div>
+<div class="report">
+
+${renderHeader(runsLabel, providers.length, tasks.length)}
+
+${tasks.length > 1 ? renderTabs(tasks) : ""}
+
+<main>
+${taskSections.map((s, i) => renderTaskSection(
+    s.task,
+    s.providerData,
+    s.columnStats,
+    s.medals,
+    s.winnerId,
+    scorerNames,
+    hasCost,
+    multi,
+    i
+  )).join("\n")}
+</main>
+
+${renderSummary(byCorrectness, byLatency, byCost, overallWinner, multi)}
+
+${deduped.length > 0 ? renderErrors(deduped) : ""}
+
+${renderFooter()}
+
+</div>
+${renderScript(tasks.length)}
+</body>
+</html>`;
+}
+function emptyReport() {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Agent Duelist Report</title>
+${renderStyle()}
+</head>
+<body>
+<div class="bg-mesh"><div class="bg-mesh-extra"></div></div>
+<div class="report">
+${renderHeader("0 runs", 0, 0)}
+<main><p class="empty-msg">No results to display.</p></main>
+${renderFooter()}
+</div>
+</body>
+</html>`;
+}
+function dedupeErrors(errorResults) {
+  const seen = /* @__PURE__ */ new Map();
+  for (const r of errorResults) {
+    const key = `${r.providerId}::${r.error}`;
+    const existing = seen.get(key);
+    if (existing) {
+      existing.count++;
+    } else {
+      seen.set(key, {
+        providerId: r.providerId,
+        error: r.error ?? "Unknown error",
+        count: 1,
+        hint: apiKeyHint(r.providerId, r.error ?? "")
+      });
+    }
+  }
+  return [...seen.values()];
+}
+function renderStyle() {
+  return `<style>
+:root {
+  --bg: #0f172a;
+  --bg-deep: #020617;
+  --panel: rgba(15, 23, 42, 0.85);
+  --accent: #f59e0b;
+  --accent-soft: rgba(245, 158, 11, 0.15);
+  --text: #e2e8f0;
+  --muted: #94a3b8;
+  --border: rgba(148, 163, 184, 0.15);
+  --green: #22c55e;
+  --red: #ef4444;
+  --yellow: #eab308;
+  --radius: 12px;
+  --mono: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Monaco, Consolas, monospace;
+  --sans: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+}
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+html, body {
+  font-family: var(--sans);
+  background: var(--bg);
+  color: var(--text);
+  min-height: 100vh;
+}
+body { padding: 24px; display: flex; justify-content: center; }
+
+/* Animated gradient mesh */
+.bg-mesh {
+  position: fixed; inset: 0; z-index: 0;
+  overflow: hidden; pointer-events: none;
+}
+.bg-mesh::before, .bg-mesh::after {
+  content: ""; position: absolute; border-radius: 50%;
+  filter: blur(120px); opacity: 0.4;
+}
+.bg-mesh::before {
+  width: 600px; height: 600px;
+  background: radial-gradient(circle, rgba(245,158,11,0.18), transparent 70%);
+  top: -10%; left: -5%;
+  animation: meshDrift1 18s ease-in-out infinite alternate;
+}
+.bg-mesh::after {
+  width: 500px; height: 500px;
+  background: radial-gradient(circle, rgba(139,92,246,0.12), transparent 70%);
+  bottom: -10%; right: -5%;
+  animation: meshDrift2 22s ease-in-out infinite alternate;
+}
+.bg-mesh-extra {
+  position: absolute; width: 400px; height: 400px;
+  border-radius: 50%; filter: blur(100px); opacity: 0.3;
+  background: radial-gradient(circle, rgba(56,189,248,0.12), transparent 70%);
+  top: 50%; left: 60%;
+  animation: meshDrift3 15s ease-in-out infinite alternate;
+}
+@keyframes meshDrift1 { from { transform: translate(0,0) scale(1); } to { transform: translate(80px,60px) scale(1.15); } }
+@keyframes meshDrift2 { from { transform: translate(0,0) scale(1); } to { transform: translate(-60px,-50px) scale(1.1); } }
+@keyframes meshDrift3 { from { transform: translate(0,0) scale(1); } to { transform: translate(-40px,40px) scale(1.2); } }
+
+/* Report container */
+.report {
+  position: relative; z-index: 1;
+  width: 100%; max-width: 960px;
+}
+
+/* Header */
+.report-header {
+  display: flex; justify-content: space-between; align-items: center;
+  padding: 20px 0; margin-bottom: 8px;
+}
+.report-brand {
+  display: flex; align-items: center; gap: 10px;
+  text-decoration: none; color: var(--muted);
+  font-weight: 600; font-size: 14px;
+  letter-spacing: 0.04em; text-transform: uppercase;
+}
+.report-brand:hover { color: var(--text); }
+.brand-icon {
+  width: 32px; height: 32px; border-radius: 8px;
+  background: linear-gradient(135deg, var(--accent-soft), rgba(245,158,11,0.05));
+  border: 1px solid rgba(245,158,11,0.3);
+  display: flex; align-items: center; justify-content: center;
+  font-size: 16px;
+}
+.report-meta {
+  font-size: 12px; color: var(--muted);
+  text-align: right; line-height: 1.6;
+}
+
+/* Task tabs */
+.task-tabs {
+  display: flex; gap: 6px; margin-bottom: 16px; flex-wrap: wrap;
+}
+.task-tab {
+  padding: 6px 16px; border-radius: 999px;
+  border: 1px solid var(--border);
+  background: transparent; color: var(--muted);
+  font-size: 13px; font-weight: 500; cursor: pointer;
+  transition: all 150ms ease;
+}
+.task-tab:hover { border-color: rgba(245,158,11,0.3); color: var(--text); }
+.task-tab.active {
+  background: var(--accent-soft);
+  border-color: rgba(245,158,11,0.4);
+  color: var(--accent);
+}
+
+/* Task sections */
+.task-section { display: none; }
+.task-section.active { display: block; }
+.task-name {
+  font-size: 18px; font-weight: 600;
+  margin-bottom: 12px; letter-spacing: -0.01em;
+}
+
+/* Results table */
+.results-table {
+  width: 100%; border-collapse: collapse;
+  font-size: 13px; margin-bottom: 16px;
+  border-radius: var(--radius); overflow: hidden;
+  border: 1px solid var(--border);
+}
+.results-table th, .results-table td {
+  padding: 10px 14px;
+  text-align: left;
+  border-bottom: 1px solid var(--border);
+}
+.results-table th {
+  background: rgba(0,0,0,0.3);
+  font-size: 11px; font-weight: 600;
+  text-transform: uppercase; letter-spacing: 0.05em;
+  color: var(--muted); cursor: pointer;
+  user-select: none; white-space: nowrap;
+}
+.results-table th:hover { color: var(--text); }
+.results-table th .sort-arrow { margin-left: 4px; font-size: 10px; }
+.results-table tbody tr {
+  background: var(--panel);
+  transition: background 120ms ease;
+}
+.results-table tbody tr:hover { background: rgba(15,23,42,0.95); }
+.results-table tbody tr:last-child td { border-bottom: none; }
+
+/* Score cell with progress bar */
+.score-cell { position: relative; min-width: 90px; }
+.score-bar {
+  position: absolute; left: 0; bottom: 0;
+  height: 3px; border-radius: 2px;
+  transition: width 300ms ease;
+}
+.score-val { position: relative; z-index: 1; font-family: var(--mono); font-size: 12px; }
+
+/* Color ranking */
+.rank-best { color: var(--green); font-weight: 600; }
+.rank-worst { color: var(--red); }
+.rank-mid { color: var(--yellow); }
+.rank-neutral { color: var(--text); }
+.rank-error { color: var(--muted); }
+
+/* Winner banner */
+.task-winner {
+  display: flex; align-items: center; gap: 10px;
+  padding: 12px 18px; margin-bottom: 20px;
+  border-radius: var(--radius);
+  background: linear-gradient(135deg, rgba(34,197,94,0.08), rgba(34,197,94,0.02));
+  border: 1px solid rgba(34,197,94,0.2);
+  font-size: 14px; font-weight: 500;
+}
+.task-winner .trophy { font-size: 20px; }
+.task-winner .winner-name { color: var(--green); font-weight: 600; }
+.task-winner .winner-label { color: var(--muted); font-size: 12px; margin-left: 4px; }
+
+/* Summary cards */
+.summary-section { margin-top: 32px; }
+.summary-title {
+  font-size: 16px; font-weight: 600;
+  margin-bottom: 12px; color: var(--text);
+}
+.summary-cards {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(200px, 1fr));
+  gap: 12px;
+}
+.summary-card {
+  padding: 16px; border-radius: var(--radius);
+  border: 1px solid var(--border);
+  background: var(--panel);
+}
+.summary-card .card-label {
+  font-size: 11px; font-weight: 600;
+  text-transform: uppercase; letter-spacing: 0.05em;
+  color: var(--muted); margin-bottom: 6px;
+}
+.summary-card .card-value {
+  font-size: 20px; font-weight: 700;
+  color: var(--green); font-family: var(--mono);
+}
+.summary-card .card-provider {
+  font-size: 12px; color: var(--muted); margin-top: 4px;
+}
+
+/* Errors */
+.errors-section { margin-top: 24px; }
+.errors-title {
+  font-size: 16px; font-weight: 600;
+  margin-bottom: 8px; color: var(--red);
+  cursor: pointer;
+}
+.errors-list {
+  border-radius: var(--radius);
+  border: 1px solid rgba(239,68,68,0.2);
+  background: rgba(239,68,68,0.04);
+  overflow: hidden;
+}
+.error-item {
+  padding: 10px 16px;
+  border-bottom: 1px solid rgba(239,68,68,0.1);
+  font-size: 13px;
+}
+.error-item:last-child { border-bottom: none; }
+.error-provider { font-weight: 600; color: var(--text); }
+.error-msg { color: var(--muted); margin-left: 8px; }
+.error-count { color: var(--muted); font-size: 11px; }
+.error-hint { color: var(--muted); font-size: 12px; margin-top: 4px; font-style: italic; }
+
+/* Footer */
+.report-footer {
+  margin-top: 40px; padding: 20px 0;
+  border-top: 1px solid var(--border);
+  display: flex; justify-content: space-between; align-items: center;
+  flex-wrap: wrap; gap: 12px;
+}
+.footer-brand {
+  font-size: 13px; color: var(--muted);
+}
+.footer-brand a {
+  color: var(--accent); text-decoration: none; font-weight: 500;
+}
+.footer-brand a:hover { text-decoration: underline; }
+.footer-cta {
+  display: inline-flex; align-items: center; gap: 6px;
+  padding: 6px 14px; border-radius: 8px;
+  background: var(--accent-soft);
+  border: 1px solid rgba(245,158,11,0.3);
+  color: var(--accent); font-size: 12px; font-weight: 500;
+  text-decoration: none;
+  transition: transform 120ms ease, box-shadow 120ms ease;
+}
+.footer-cta:hover { transform: translateY(-1px); box-shadow: 0 4px 12px rgba(245,158,11,0.2); }
+
+/* Empty state */
+.empty-msg {
+  text-align: center; color: var(--muted);
+  padding: 60px 20px; font-size: 16px;
+}
+
+/* Responsive */
+@media (max-width: 640px) {
+  body { padding: 12px; }
+  .report-header { flex-direction: column; align-items: flex-start; gap: 8px; }
+  .report-meta { text-align: left; }
+  .summary-cards { grid-template-columns: 1fr; }
+  .results-table { font-size: 12px; }
+  .results-table th, .results-table td { padding: 8px 10px; }
+  .report-footer { flex-direction: column; align-items: flex-start; }
+}
+</style>`;
+}
+function renderHeader(runsLabel, providerCount, taskCount) {
+  const now = (/* @__PURE__ */ new Date()).toISOString().replace("T", " ").slice(0, 19) + " UTC";
+  return `<header class="report-header">
+  <a class="report-brand" href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">
+    <div class="brand-icon">&#x2B21;</div>
+    <span>Agent Duelist</span>
+  </a>
+  <div class="report-meta">
+    ${providerCount} provider${providerCount !== 1 ? "s" : ""} &middot;
+    ${taskCount} task${taskCount !== 1 ? "s" : ""} &middot;
+    ${esc(runsLabel)}<br>
+    ${esc(now)}
+  </div>
+</header>`;
+}
+function renderTabs(tasks) {
+  const buttons = tasks.map(
+    (t, i) => `<button class="task-tab${i === 0 ? " active" : ""}" data-task="${i}">${esc(t)}</button>`
+  ).join("\n    ");
+  return `<nav class="task-tabs">
+    ${buttons}
+  </nav>`;
+}
+function renderTaskSection(task, providerData, columnStats, medals, winnerId, scorerNames, _hasCost, multi, index) {
+  const cols = [
+    { label: "Provider", key: "provider", isScore: false }
+  ];
+  for (const name of scorerNames) {
+    if (name === "latency") {
+      cols.push({ label: "Latency", key: "latency", isScore: false });
+    } else if (name === "cost") {
+      cols.push({ label: "Cost", key: "cost", isScore: false });
+      cols.push({ label: "Tokens", key: "tokens", isScore: false });
+    } else {
+      cols.push({ label: scorerLabel(name), key: name, isScore: true });
+    }
+  }
+  const ths = cols.map(
+    (c) => `<th data-col="${esc(c.key)}">${esc(c.label)}<span class="sort-arrow"></span></th>`
+  ).join("");
+  const rows = providerData.map((pd) => {
+    const medal = medalEmoji(medals.get(pd.providerId) ?? "none");
+    const cells = [];
+    const medalHtml = medal ? `${medal} ` : "";
+    cells.push(`<td>${medalHtml}${esc(pd.providerId)}</td>`);
+    if (pd.allErrors) {
+      for (let ci = 1; ci < cols.length; ci++) {
+        cells.push(`<td class="rank-error">&mdash;</td>`);
+      }
+    } else {
+      for (const col of cols.slice(1)) {
+        cells.push(renderDataCell(col.key, col.isScore, pd, columnStats, multi));
+      }
+    }
+    return `<tr>${cells.join("")}</tr>`;
+  }).join("\n");
+  const winnerHtml = winnerId ? `<div class="task-winner">
+    <span class="trophy">&#x1F3C6;</span>
+    <span>Winner: <span class="winner-name">${esc(winnerId)}</span>
+    <span class="winner-label">${esc(providerLabel(winnerId))}</span></span>
+  </div>` : "";
+  return `<section class="task-section${index === 0 ? " active" : ""}" data-task-idx="${index}">
+  <h2 class="task-name">${esc(task)}</h2>
+  <table class="results-table">
+    <thead><tr>${ths}</tr></thead>
+    <tbody>${rows}</tbody>
+  </table>
+  ${winnerHtml}
+</section>`;
+}
+function renderDataCell(key, _isScore, pd, columnStats, multi) {
+  const colStats = columnStats.get(key);
+  if (key === "latency") {
+    const ms = pd.latencyMs;
+    if (ms === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(ms, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${ms}">${Math.round(ms)}ms</td>`;
+  }
+  if (key === "cost") {
+    const cost = pd.avgDetails.costUsd;
+    if (cost === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(cost, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${cost}">${esc(formatCost(cost))}</td>`;
+  }
+  if (key === "tokens") {
+    const tokens = pd.avgDetails.totalTokens;
+    if (tokens === void 0) return `<td class="rank-error">&mdash;</td>`;
+    const rankClass = multi && colStats ? rankClass_(tokens, colStats) : "rank-neutral";
+    return `<td class="${rankClass}" data-sort-val="${tokens}">${tokens}</td>`;
+  }
+  const val = pd.avgScores[key];
+  if (val === void 0) return `<td class="rank-error">&mdash;</td>`;
+  const pct = Math.round(val * 100);
+  let rankCls;
+  if (multi && colStats) {
+    rankCls = rankClass_(val, colStats);
+  } else {
+    rankCls = val >= 0.8 ? "rank-best" : val >= 0.5 ? "rank-mid" : "rank-worst";
+  }
+  const barColor = val >= 0.8 ? "var(--green)" : val >= 0.5 ? "var(--yellow)" : "var(--red)";
+  return `<td class="score-cell ${rankCls}" data-sort-val="${val}">
+    <span class="score-val">${pct}%</span>
+    <div class="score-bar" style="width:${pct}%;background:${barColor}"></div>
+  </td>`;
+}
+function rankClass_(value, colStats) {
+  if (colStats.best === void 0 || colStats.worst === void 0) return "rank-neutral";
+  if (colStats.best === colStats.worst) return "rank-neutral";
+  if (value === colStats.best) return "rank-best";
+  if (value === colStats.worst) return "rank-worst";
+  return "rank-mid";
+}
+function renderSummary(byCorrectness, byLatency, byCost, overallWinner, multi) {
+  const cards = [];
+  if (byCorrectness) {
+    const pct = `${Math.round(byCorrectness.avg * 100)}%`;
+    const provider = multi ? `<div class="card-provider">${esc(byCorrectness.id)} ${esc(providerLabel(byCorrectness.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Most Correct" : "Avg Correctness"}</div>
+      <div class="card-value">${pct}</div>
+      ${provider}
+    </div>`);
+  }
+  if (byLatency && byLatency.avg !== Infinity) {
+    const ms = `${Math.round(byLatency.avg)}ms`;
+    const provider = multi ? `<div class="card-provider">${esc(byLatency.id)} ${esc(providerLabel(byLatency.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Fastest" : "Avg Latency"}</div>
+      <div class="card-value">${ms}</div>
+      ${provider}
+    </div>`);
+  }
+  if (byCost?.avg !== void 0) {
+    const cost = esc(formatCost(byCost.avg));
+    const provider = multi ? `<div class="card-provider">${esc(byCost.id)} ${esc(providerLabel(byCost.id))}</div>` : "";
+    cards.push(`<div class="summary-card">
+      <div class="card-label">${multi ? "Cheapest" : "Avg Cost"}</div>
+      <div class="card-value">${cost}</div>
+      ${provider}
+    </div>`);
+  }
+  if (overallWinner) {
+    cards.push(`<div class="summary-card">
+      <div class="card-label">Overall Winner</div>
+      <div class="card-value">&#x1F3C6;</div>
+      <div class="card-provider">${esc(overallWinner)} ${esc(providerLabel(overallWinner))}</div>
+    </div>`);
+  }
+  if (cards.length === 0) return "";
+  return `<section class="summary-section">
+  <h2 class="summary-title">Summary</h2>
+  <div class="summary-cards">
+    ${cards.join("\n    ")}
+  </div>
+</section>`;
+}
+function renderErrors(errors) {
+  const items = errors.map((e) => {
+    const suffix = e.count > 1 ? ` <span class="error-count">(&times;${e.count})</span>` : "";
+    const hint = e.hint ? `<div class="error-hint">${esc(e.hint)}</div>` : "";
+    return `<div class="error-item">
+      <span class="error-provider">${esc(e.providerId)}:</span>
+      <span class="error-msg">${esc(e.error)}</span>${suffix}
+      ${hint}
+    </div>`;
+  }).join("\n");
+  return `<section class="errors-section">
+  <h2 class="errors-title" onclick="this.nextElementSibling.style.display=this.nextElementSibling.style.display==='none'?'block':'block'">Errors</h2>
+  <div class="errors-list">
+    ${items}
+  </div>
+</section>`;
+}
+function renderFooter() {
+  return `<footer class="report-footer">
+  <div class="footer-brand">
+    Powered by <a href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">Agent Duelist</a>
+  </div>
+  <a class="footer-cta" href="https://github.com/DataGobes/agent-duelist" target="_blank" rel="noopener">
+    &#x2B50; Star on GitHub
+  </a>
+</footer>`;
+}
+function renderScript(taskCount) {
+  return `<script>
+(function() {
+  /* Tab switching */
+  ${taskCount > 1 ? `
+  var tabs = document.querySelectorAll('.task-tab');
+  var sections = document.querySelectorAll('.task-section');
+  tabs.forEach(function(tab) {
+    tab.addEventListener('click', function() {
+      var idx = parseInt(tab.getAttribute('data-task'));
+      tabs.forEach(function(t) { t.classList.remove('active'); });
+      sections.forEach(function(s) { s.classList.remove('active'); });
+      tab.classList.add('active');
+      sections[idx].classList.add('active');
+    });
+  });` : ""}
+
+  /* Column sorting */
+  document.querySelectorAll('.results-table th').forEach(function(th, colIdx) {
+    var table = th.closest('table');
+    var asc = true;
+    th.addEventListener('click', function() {
+      var tbody = table.querySelector('tbody');
+      var rows = Array.from(tbody.querySelectorAll('tr'));
+      rows.sort(function(a, b) {
+        var aCell = a.children[colIdx];
+        var bCell = b.children[colIdx];
+        var aVal = aCell.getAttribute('data-sort-val');
+        var bVal = bCell.getAttribute('data-sort-val');
+        if (aVal !== null && bVal !== null) {
+          return asc ? parseFloat(aVal) - parseFloat(bVal) : parseFloat(bVal) - parseFloat(aVal);
+        }
+        var aText = aCell.textContent || '';
+        var bText = bCell.textContent || '';
+        return asc ? aText.localeCompare(bText) : bText.localeCompare(aText);
+      });
+      rows.forEach(function(row) { tbody.appendChild(row); });
+
+      /* Update sort arrows */
+      table.querySelectorAll('th .sort-arrow').forEach(function(a) { a.textContent = ''; });
+      th.querySelector('.sort-arrow').textContent = asc ? ' \\u25B2' : ' \\u25BC';
+      asc = !asc;
+    });
+  });
+})();
+</script>`;
+}
+
+// src/ci.ts
+import { readFileSync, writeFileSync, mkdirSync } from "fs";
+import { dirname } from "path";
+var LOWER_IS_BETTER = /* @__PURE__ */ new Set(["cost"]);
+var FLAKY_CV_THRESHOLD = 0.3;
+var T_CRITICAL_95 = {
+  1: 12.706,
+  2: 4.303,
+  3: 3.182,
+  4: 2.776,
+  5: 2.571,
+  6: 2.447,
+  7: 2.365,
+  8: 2.306,
+  9: 2.262,
+  10: 2.228,
+  15: 2.131,
+  20: 2.086,
+  25: 2.06,
+  30: 2.042
+};
+var T_CRITICAL_KEYS = Object.keys(T_CRITICAL_95).map(Number).sort((a, b) => a - b);
+function tCritical(df) {
+  if (df <= 0) return 1.96;
+  if (T_CRITICAL_95[df] !== void 0) return T_CRITICAL_95[df];
+  const keys = T_CRITICAL_KEYS;
+  if (df > keys[keys.length - 1]) return 1.96;
+  for (let i = 0; i < keys.length - 1; i++) {
+    if (df > keys[i] && df < keys[i + 1]) {
+      const low = keys[i], high = keys[i + 1];
+      const ratio = (df - low) / (high - low);
+      return T_CRITICAL_95[low] + ratio * (T_CRITICAL_95[high] - T_CRITICAL_95[low]);
+    }
+  }
+  return 1.96;
+}
+function computeScorerStats(samples) {
+  const n = samples.length;
+  if (n === 0) {
+    return { mean: 0, stddev: 0, cv: 0, n: 0, ci95Lower: 0, ci95Upper: 0 };
+  }
+  const mean = samples.reduce((a, b) => a + b, 0) / n;
+  if (n === 1) {
+    return { mean, stddev: 0, cv: 0, n: 1, ci95Lower: mean, ci95Upper: mean };
+  }
+  const variance = samples.reduce((sum, x) => sum + (x - mean) ** 2, 0) / (n - 1);
+  const stddev = Math.sqrt(variance);
+  const cv = mean !== 0 ? stddev / Math.abs(mean) : 0;
+  const se = stddev / Math.sqrt(n);
+  const t = tCritical(n - 1);
+  return {
+    mean,
+    stddev,
+    cv,
+    n,
+    ci95Lower: mean - t * se,
+    ci95Upper: mean + t * se
+  };
+}
+function groupKey(providerId, taskName, scorerName) {
+  return `${providerId}::${taskName}::${scorerName}`;
+}
+function computeStats(results) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    if (r.error) continue;
+    for (const score of r.scores) {
+      if (score.value < 0) continue;
+      const key = groupKey(r.providerId, r.taskName, score.name);
+      if (!grouped.has(key)) grouped.set(key, []);
+      grouped.get(key).push(score.value);
+    }
+  }
+  const stats = /* @__PURE__ */ new Map();
+  for (const [key, samples] of grouped) {
+    stats.set(key, computeScorerStats(samples));
+  }
+  return stats;
+}
+function computeCostSummary(results, budget) {
+  let totalUsd = 0;
+  const perProvider = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    if (r.error) continue;
+    const costScore = r.scores.find((s) => s.name === "cost");
+    if (!costScore || costScore.value < 0) continue;
+    const details = costScore.details;
+    const usd = details?.estimatedUsd ?? 0;
+    if (usd <= 0) continue;
+    totalUsd += usd;
+    perProvider.set(r.providerId, (perProvider.get(r.providerId) ?? 0) + usd);
+  }
+  return {
+    totalUsd,
+    perProvider,
+    budget,
+    overBudget: budget !== void 0 && totalUsd > budget
+  };
+}
+function compareResults(baselineStats, currentStats, thresholds, budget, currentResults) {
+  const comparisons = [];
+  const failureReasons = [];
+  for (const [key, current] of currentStats) {
+    const [providerId, taskName, scorerName] = key.split("::");
+    const baseline = baselineStats?.get(key) ?? null;
+    let delta = null;
+    let regressed = false;
+    let improved = false;
+    if (baseline) {
+      delta = current.mean - baseline.mean;
+      const threshold = thresholds.get(scorerName);
+      if (threshold !== void 0) {
+        const lowerIsBetter = LOWER_IS_BETTER.has(scorerName);
+        regressed = detectRegression(baseline, current, threshold, lowerIsBetter);
+        improved = detectImprovement(baseline, current, threshold, lowerIsBetter);
+      }
+    }
+    const flaky = current.n > 1 && current.cv > FLAKY_CV_THRESHOLD;
+    comparisons.push({
+      providerId,
+      taskName,
+      scorerName,
+      baseline,
+      current,
+      delta,
+      regressed,
+      improved,
+      flaky
+    });
+  }
+  const cost = computeCostSummary(currentResults ?? [], budget);
+  const regressions = comparisons.filter((c) => c.regressed);
+  if (regressions.length > 0) {
+    for (const r of regressions) {
+      failureReasons.push(
+        `${r.providerId} \xD7 ${r.taskName}: ${r.scorerName} regressed by ${formatDelta(r.delta)}`
+      );
+    }
+  }
+  if (cost.overBudget) {
+    failureReasons.push(
+      `Total cost $${cost.totalUsd.toFixed(4)} exceeds budget $${cost.budget.toFixed(2)}`
+    );
+  }
+  const flakyResults = comparisons.filter((c) => c.flaky);
+  const failed = failureReasons.length > 0;
+  return { comparisons, cost, failed, flakyResults, failureReasons };
+}
+function detectRegression(baseline, current, threshold, lowerIsBetter) {
+  if (baseline.n === 1 && current.n === 1) {
+    const delta = current.mean - baseline.mean;
+    if (lowerIsBetter) return delta > threshold;
+    return delta < -threshold;
+  }
+  if (lowerIsBetter) {
+    return current.ci95Lower - baseline.ci95Upper > threshold;
+  }
+  return baseline.ci95Upper - current.ci95Lower > threshold && current.mean < baseline.mean;
+}
+function detectImprovement(baseline, current, threshold, lowerIsBetter) {
+  if (baseline.n === 1 && current.n === 1) {
+    const delta = current.mean - baseline.mean;
+    if (lowerIsBetter) return delta < -threshold;
+    return delta > threshold;
+  }
+  if (lowerIsBetter) {
+    return baseline.ci95Lower - current.ci95Upper > threshold;
+  }
+  return current.ci95Lower - baseline.ci95Upper > threshold;
+}
+function loadBaseline(path) {
+  try {
+    const raw = readFileSync(path, "utf-8");
+    const data = JSON.parse(raw);
+    const results = data.results ?? data;
+    if (!Array.isArray(results)) return null;
+    return {
+      timestamp: data.timestamp ?? "unknown",
+      results
+    };
+  } catch {
+    return null;
+  }
+}
+function saveBaseline(path, results) {
+  mkdirSync(dirname(path), { recursive: true });
+  const data = {
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    results
+  };
+  writeFileSync(path, JSON.stringify(data, null, 2));
+}
+
+// src/github.ts
+import { readFileSync as readFileSync2 } from "fs";
+function detectGitHubContext() {
+  const token = process.env.GITHUB_TOKEN;
+  const repository = process.env.GITHUB_REPOSITORY;
+  const eventPath = process.env.GITHUB_EVENT_PATH;
+  if (!token || !repository) return null;
+  const [owner, repo] = repository.split("/");
+  if (!owner || !repo) return null;
+  let prNumber;
+  if (eventPath) {
+    try {
+      const event = JSON.parse(readFileSync2(eventPath, "utf-8"));
+      if (event.pull_request && typeof event.pull_request === "object") {
+        const pr = event.pull_request;
+        prNumber = pr.number;
+      }
+      if (!prNumber && event.issue && typeof event.issue === "object") {
+        const issue = event.issue;
+        if (issue.pull_request) {
+          prNumber = issue.number;
+        }
+      }
+    } catch {
+    }
+  }
+  if (!prNumber && process.env.DUELIST_PR_NUMBER) {
+    prNumber = parseInt(process.env.DUELIST_PR_NUMBER, 10);
+  }
+  if (!prNumber) return null;
+  return { token, owner, repo, prNumber };
+}
+var API_BASE = "https://api.github.com";
+function ghHeaders(token, extra) {
+  return {
+    Authorization: `Bearer ${token}`,
+    Accept: "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28",
+    ...extra
+  };
+}
+async function findExistingComment(ctx, marker) {
+  let page = 1;
+  const perPage = 50;
+  while (true) {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=${perPage}&page=${page}`;
+    const res = await fetch(url, { headers: ghHeaders(ctx.token) });
+    if (!res.ok) return null;
+    const comments = await res.json();
+    if (comments.length === 0) break;
+    for (const comment of comments) {
+      if (comment.body?.includes(marker)) {
+        return comment.id;
+      }
+    }
+    if (comments.length < perPage) break;
+    page++;
+  }
+  return null;
+}
+async function upsertPrComment(ctx, body, marker) {
+  const existingId = await findExistingComment(ctx, marker);
+  if (existingId) {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/comments/${existingId}`;
+    const res = await fetch(url, {
+      method: "PATCH",
+      headers: ghHeaders(ctx.token, { "Content-Type": "application/json" }),
+      body: JSON.stringify({ body })
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      console.warn(`Failed to update PR comment: ${res.status} ${text}`);
+    }
+  } else {
+    const url = `${API_BASE}/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
+    const res = await fetch(url, {
+      method: "POST",
+      headers: ghHeaders(ctx.token, { "Content-Type": "application/json" }),
+      body: JSON.stringify({ body })
+    });
+    if (!res.ok) {
+      const text = await res.text();
+      console.warn(`Failed to create PR comment: ${res.status} ${text}`);
+    }
+  }
 }
 export {
   anthropic,
   azureOpenai,
+  compareResults,
+  computeStats,
   consoleReporter,
   defineArena,
+  detectGitHubContext,
   gemini,
+  htmlReporter,
   jsonReporter,
+  loadBaseline,
+  markdownReporter,
   openai,
   openaiCompatible,
-  registerPricing
+  registerPricing,
+  saveBaseline,
+  upsertPrComment
 };
 //# sourceMappingURL=index.js.map