bridgebench 3.1.0-alpha.0

package/dist/chunk-NJTYVNP4.cjs

@@ -0,0 +1,1467 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+var _chunk4TWPCPRPcjs = require('./chunk-4TWPCPRP.cjs');
+
+
+
+
+
+
+
+
+
+var _chunkCIXITJW6cjs = require('./chunk-CIXITJW6.cjs');
+
+// src/elo.ts
+var ELO_INITIAL = 1e3;
+var ELO_K = 32;
+function expectedScore(ratingA, ratingB) {
+  return 1 / (1 + Math.pow(10, (ratingB - ratingA) / 400));
+}
+function applyEloWin(ratingA, ratingB, winner) {
+  const expectedA = expectedScore(ratingA, ratingB);
+  const scoreA = winner === "a" ? 1 : 0;
+  return {
+    ratingA: ratingA + ELO_K * (scoreA - expectedA),
+    ratingB: ratingB + ELO_K * (1 - scoreA - (1 - expectedA))
+  };
+}
+
+// src/version.ts
+var _fs = require('fs');
+var _path = require('path'); var _path2 = _interopRequireDefault(_path);
+var _zod = require('zod');
+var PackageMetadataSchema = _zod.z.object({
+  name: _zod.z.string().min(1),
+  version: _zod.z.string().min(1)
+});
+var metadata = PackageMetadataSchema.parse(
+  JSON.parse(_fs.readFileSync.call(void 0, _path2.default.join(_chunk4TWPCPRPcjs.packageRoot.call(void 0, _chunkCIXITJW6cjs.importMetaUrl), "package.json"), "utf8"))
+);
+var PACKAGE_NAME = metadata.name;
+var ENGINE_VERSION = metadata.version;
+
+// src/run-manifest.ts
+var _crypto = require('crypto');
+
+var RUN_MANIFEST_VERSION = "1.0.0";
+var ManifestModelSchema = _zod.z.object({
+  id: _zod.z.string().min(1),
+  canonicalSlug: _zod.z.string().min(1),
+  role: _zod.z.enum(["competitor", "judge"]),
+  request: _zod.z.object({
+    maxTokens: _zod.z.number().int().positive(),
+    temperature: _zod.z.number().finite(),
+    reasoningEffort: _zod.z.enum(["high", "medium", "low"]),
+    excludeReasoning: _zod.z.boolean(),
+    timeoutMs: _zod.z.number().int().positive()
+  })
+});
+var ManifestCompetitorSchema = ManifestModelSchema.extend({
+  role: _zod.z.literal("competitor")
+});
+var ManifestJudgeSchema = ManifestModelSchema.extend({
+  role: _zod.z.literal("judge")
+});
+var RunManifestSchema = _zod.z.object({
+  version: _zod.z.literal(RUN_MANIFEST_VERSION),
+  methodologyVersion: _zod.z.string().min(1),
+  engineVersion: _zod.z.string().min(1),
+  category: _chunkCIXITJW6cjs.BenchmarkCategorySchema,
+  seed: _zod.z.string(),
+  matches: _zod.z.number().int().positive(),
+  competitors: _zod.z.array(ManifestCompetitorSchema).min(2).superRefine((models, context) => {
+    if (new Set(models.map((model) => model.id)).size !== models.length) {
+      context.addIssue({
+        code: _zod.z.ZodIssueCode.custom,
+        message: "Competitor manifest entries must be unique"
+      });
+    }
+  }),
+  judges: _zod.z.array(ManifestJudgeSchema).length(3).superRefine((models, context) => {
+    if (new Set(models.map((model) => model.id)).size !== models.length) {
+      context.addIssue({
+        code: _zod.z.ZodIssueCode.custom,
+        message: "Judge manifest entries must be unique"
+      });
+    }
+  }),
+  tasks: _zod.z.array(
+    _zod.z.object({
+      id: _zod.z.string().min(1),
+      version: _zod.z.string().min(1),
+      publicHash: _zod.z.string().min(1),
+      privateHash: _zod.z.string().min(1)
+    })
+  ),
+  promptPolicyHashes: _zod.z.object({
+    competitor: _zod.z.string().regex(/^[a-f0-9]{64}$/),
+    judge: _zod.z.string().regex(/^[a-f0-9]{64}$/)
+  })
+});
+function manifestModel(model) {
+  return {
+    id: model.id,
+    canonicalSlug: model.canonicalSlug,
+    role: model.role,
+    request: { ...model.request }
+  };
+}
+function canonicalize(value) {
+  if (Array.isArray(value)) return value.map(canonicalize);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value).sort(([left], [right]) => left.localeCompare(right)).map(([key, child]) => [key, canonicalize(child)])
+    );
+  }
+  return value;
+}
+function canonicalJson(value) {
+  return JSON.stringify(canonicalize(value));
+}
+function runManifestHash(manifest) {
+  return _crypto.createHash.call(void 0, "sha256").update(canonicalJson(manifest)).digest("hex");
+}
+function runIdFromManifest(manifest) {
+  return `run-${runManifestHash(manifest).slice(0, 20)}`;
+}
+function createRunManifest(config, tasks, resolvedCompetitors = _chunk4TWPCPRPcjs.resolveCompetitorRoster.call(void 0, 
+  config.competitorIds
+)) {
+  const canonicalCompetitors = _chunk4TWPCPRPcjs.resolveCompetitorRoster.call(void 0, 
+    resolvedCompetitors.map((model) => model.id)
+  );
+  const competitors = canonicalCompetitors.map((model) => ({ ...manifestModel(model), role: "competitor" })).sort((left, right) => left.id.localeCompare(right.id));
+  const judges = _chunk4TWPCPRPcjs.listModels.call(void 0, "judge").map((model) => ({ ...manifestModel(model), role: "judge" })).sort((left, right) => left.id.localeCompare(right.id));
+  const manifest = {
+    version: RUN_MANIFEST_VERSION,
+    methodologyVersion: _chunkCIXITJW6cjs.METHODOLOGY_VERSION,
+    engineVersion: ENGINE_VERSION,
+    category: config.category,
+    seed: config.seed,
+    matches: config.matches,
+    competitors,
+    judges,
+    tasks: tasks.map((task) => ({
+      id: task.public.id,
+      version: task.public.version,
+      publicHash: task.publicHash,
+      privateHash: task.privateHash
+    })).sort((left, right) => left.id.localeCompare(right.id)),
+    promptPolicyHashes: {
+      competitor: _chunk4TWPCPRPcjs.competitorPromptPolicyHash.call(void 0, config.category),
+      judge: _chunk4TWPCPRPcjs.judgePromptPolicyHash.call(void 0, config.category)
+    }
+  };
+  return RunManifestSchema.parse(manifest);
+}
+
+// src/verification.ts
+var LEGACY_METHODOLOGY_VERSION = "reasoning-arena-v0.2.0";
+var SUPPORTED_METHODOLOGY_VERSIONS = /* @__PURE__ */ new Set([LEGACY_METHODOLOGY_VERSION, _chunkCIXITJW6cjs.METHODOLOGY_VERSION]);
+var EPSILON = 1e-9;
+function fail(message) {
+  throw new Error(message);
+}
+function equalNumber(actual, expected, label) {
+  if (actual === void 0 || Math.abs(actual - expected) > EPSILON) {
+    fail(`${label} expected ${expected}, found ${_nullishCoalesce(actual, () => ( "<missing>"))}`);
+  }
+}
+function assertRecordKeys(record, expected, label) {
+  const actual = Object.keys(record).sort();
+  const wanted = [...expected].sort();
+  if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
+    fail(`${label} keys must be exactly ${wanted.join(", ")}`);
+  }
+}
+function expectedPanel(panel, modelA, modelB) {
+  const votesByModel = { [modelA]: 0, [modelB]: 0 };
+  let validVotes = 0;
+  for (const vote of panel.votes) {
+    if (vote.winnerModelId === null) continue;
+    if (vote.winnerModelId !== modelA && vote.winnerModelId !== modelB) {
+      fail(`judge ${vote.judgeModelId} resolved an unknown competitor`);
+    }
+    validVotes += 1;
+    votesByModel[vote.winnerModelId] = (_nullishCoalesce(votesByModel[vote.winnerModelId], () => ( 0))) + 1;
+  }
+  const winnerModelId = _nullishCoalesce(_optionalChain([Object, 'access', _ => _.entries, 'call', _2 => _2(votesByModel), 'access', _3 => _3.find, 'call', _4 => _4(([, count]) => count >= 2), 'optionalAccess', _5 => _5[0]]), () => ( null));
+  const winnerVotes = winnerModelId ? _nullishCoalesce(votesByModel[winnerModelId], () => ( 0)) : 0;
+  return {
+    winnerModelId,
+    validVotes,
+    votesByModel,
+    agreement: winnerVotes === 3 ? "unanimous" : winnerVotes === 2 ? "split" : "insufficient"
+  };
+}
+function verifyPanel(panel, modelA, modelB) {
+  const expected = expectedPanel(panel, modelA, modelB);
+  if (panel.validVotes !== expected.validVotes) {
+    fail(`panel validVotes expected ${expected.validVotes}, found ${panel.validVotes}`);
+  }
+  assertRecordKeys(panel.votesByModel, [modelA, modelB], "panel votesByModel");
+  equalNumber(
+    panel.votesByModel[modelA],
+    _nullishCoalesce(expected.votesByModel[modelA], () => ( 0)),
+    `votes for ${modelA}`
+  );
+  equalNumber(
+    panel.votesByModel[modelB],
+    _nullishCoalesce(expected.votesByModel[modelB], () => ( 0)),
+    `votes for ${modelB}`
+  );
+  if (panel.winnerModelId !== expected.winnerModelId) {
+    fail(
+      `panel winner expected ${_nullishCoalesce(expected.winnerModelId, () => ( "none"))}, found ${_nullishCoalesce(panel.winnerModelId, () => ( "none"))}`
+    );
+  }
+  if (panel.agreement !== expected.agreement) {
+    fail(`panel agreement expected ${expected.agreement}, found ${panel.agreement}`);
+  }
+  return expected.winnerModelId;
+}
+function verifyOutcome(match) {
+  const { modelA, modelB, responseA, responseB } = match.competitors;
+  if (modelA === modelB) fail("a competitor cannot face itself");
+  if (responseA.modelId !== modelA || responseB.modelId !== modelB) {
+    fail("competitor response identity does not match its scheduled side");
+  }
+  let expectedWinner = null;
+  let expectedOutcome;
+  if (responseA.success !== responseB.success) {
+    expectedOutcome = "forfeit";
+    expectedWinner = responseA.success ? modelA : modelB;
+    if (match.panel !== null) fail("forfeit must not contain a judge panel");
+  } else if (!responseA.success && !responseB.success) {
+    expectedOutcome = "no-contest";
+    if (match.panel !== null) fail("double failure must not contain a judge panel");
+  } else {
+    if (match.panel === null) fail("two successful responses require a judge panel");
+    expectedWinner = verifyPanel(match.panel, modelA, modelB);
+    expectedOutcome = expectedWinner ? "judged" : "no-contest";
+  }
+  if (match.outcome !== expectedOutcome) {
+    fail(`outcome expected ${expectedOutcome}, found ${match.outcome}`);
+  }
+  if (match.winnerModelId !== expectedWinner) {
+    fail(`winner expected ${_nullishCoalesce(expectedWinner, () => ( "none"))}, found ${_nullishCoalesce(match.winnerModelId, () => ( "none"))}`);
+  }
+  if (match.pointAwarded !== (expectedWinner !== null)) {
+    fail(`pointAwarded must be ${expectedWinner !== null}`);
+  }
+  const judgeCost = _nullishCoalesce(_optionalChain([match, 'access', _6 => _6.panel, 'optionalAccess', _7 => _7.votes, 'access', _8 => _8.reduce, 'call', _9 => _9((sum, vote) => sum + (_nullishCoalesce(_optionalChain([vote, 'access', _10 => _10.completion, 'optionalAccess', _11 => _11.costUsd]), () => ( 0))), 0)]), () => ( 0));
+  equalNumber(
+    match.matchCostUsd,
+    _chunkCIXITJW6cjs.competitorCost.call(void 0, responseA) + _chunkCIXITJW6cjs.competitorCost.call(void 0, responseB) + judgeCost,
+    "matchCostUsd"
+  );
+}
+function verifyManifest(match, manifest, category) {
+  const hash = runManifestHash(manifest);
+  if (match.runManifestHash !== hash) {
+    fail(`run manifest hash expected ${hash}, found ${_nullishCoalesce(match.runManifestHash, () => ( "<missing>"))}`);
+  }
+  if (match.runId !== runIdFromManifest(manifest)) {
+    fail(`runId does not match the canonical run manifest`);
+  }
+  if (manifest.category !== category || manifest.seed !== match.seed || manifest.methodologyVersion !== match.methodologyVersion) {
+    fail("run manifest category, seed, or methodology does not match the journal line");
+  }
+  if (match.scheduleIndex >= manifest.matches) {
+    fail(`scheduleIndex ${match.scheduleIndex} exceeds manifest match count ${manifest.matches}`);
+  }
+  const competitorIds = new Set(manifest.competitors.map((model) => model.id));
+  if (!competitorIds.has(match.competitors.modelA) || !competitorIds.has(match.competitors.modelB)) {
+    fail("journal competitor is absent from the run manifest");
+  }
+  const task = manifest.tasks.find((candidate) => candidate.id === match.task.id);
+  if (!task || task.version !== match.task.version || task.publicHash !== match.task.publicHash || task.privateHash !== match.task.privateHash) {
+    fail(`task ${match.task.id} does not match the run manifest`);
+  }
+  const judgeIds = new Set(manifest.judges.map((model) => model.id));
+  for (const vote of _nullishCoalesce(_optionalChain([match, 'access', _12 => _12.panel, 'optionalAccess', _13 => _13.votes]), () => ( []))) {
+    if (!judgeIds.has(vote.judgeModelId)) {
+      fail(`judge ${vote.judgeModelId} is absent from the run manifest`);
+    }
+  }
+}
+function verifyJournal(input, category, options = {}) {
+  const ratings = {};
+  const points = {};
+  const matches = [];
+  const matchIds = /* @__PURE__ */ new Set();
+  const runs = /* @__PURE__ */ new Set();
+  const lastScheduleIndex = /* @__PURE__ */ new Map();
+  const manifestHashes = /* @__PURE__ */ new Map();
+  const warnings = /* @__PURE__ */ new Set();
+  let methodologyVersion = null;
+  for (const [offset, original] of input.entries()) {
+    const line = offset + 1;
+    try {
+      if (!SUPPORTED_METHODOLOGY_VERSIONS.has(original.methodologyVersion)) {
+        fail(`unsupported methodology version ${original.methodologyVersion}`);
+      }
+      methodologyVersion ??= original.methodologyVersion;
+      if (methodologyVersion !== original.methodologyVersion) {
+        fail(`mixed methodology versions ${methodologyVersion} and ${original.methodologyVersion}`);
+      }
+      const match = {
+        ...original,
+        task: {
+          ...original.task,
+          category: _nullishCoalesce(original.task.category, () => ( category))
+        }
+      };
+      if (match.task.category !== category) {
+        fail(`task category ${match.task.category} does not match ${category}`);
+      }
+      if (matchIds.has(match.matchId)) fail(`duplicate matchId ${match.matchId}`);
+      matchIds.add(match.matchId);
+      const previousIndex = lastScheduleIndex.get(match.runId);
+      if (previousIndex !== void 0 && match.scheduleIndex <= previousIndex) {
+        fail(`scheduleIndex ${match.scheduleIndex} is not increasing within run ${match.runId}`);
+      }
+      lastScheduleIndex.set(match.runId, match.scheduleIndex);
+      runs.add(match.runId);
+      if (match.runManifestHash) {
+        const priorHash = manifestHashes.get(match.runId);
+        if (priorHash && priorHash !== match.runManifestHash) {
+          fail(`run ${match.runId} references multiple manifest hashes`);
+        }
+        manifestHashes.set(match.runId, match.runManifestHash);
+        const manifest = _nullishCoalesce(_optionalChain([options, 'access', _14 => _14.manifestForRun, 'optionalCall', _15 => _15(match.runId)]), () => ( null));
+        if (manifest) {
+          verifyManifest(match, manifest, category);
+        } else if (options.requireManifests) {
+          fail(`run manifest ${match.runId}.json is required but missing`);
+        } else {
+          warnings.add(`Run ${match.runId} references a manifest that was not supplied.`);
+        }
+      } else {
+        warnings.add(`Run ${match.runId} is a legacy journal without a manifest hash.`);
+      }
+      verifyOutcome(match);
+      const { modelA, modelB } = match.competitors;
+      const ratingA = _nullishCoalesce(ratings[modelA], () => ( ELO_INITIAL));
+      const ratingB = _nullishCoalesce(ratings[modelB], () => ( ELO_INITIAL));
+      assertRecordKeys(match.eloBefore, [modelA, modelB], "eloBefore");
+      assertRecordKeys(match.eloAfter, [modelA, modelB], "eloAfter");
+      equalNumber(match.eloBefore[modelA], ratingA, `eloBefore.${modelA}`);
+      equalNumber(match.eloBefore[modelB], ratingB, `eloBefore.${modelB}`);
+      let nextA = ratingA;
+      let nextB = ratingB;
+      if (match.winnerModelId) {
+        const update = applyEloWin(ratingA, ratingB, match.winnerModelId === modelA ? "a" : "b");
+        nextA = update.ratingA;
+        nextB = update.ratingB;
+        points[match.winnerModelId] = (_nullishCoalesce(points[match.winnerModelId], () => ( 0))) + 1;
+      }
+      equalNumber(match.eloAfter[modelA], nextA, `eloAfter.${modelA}`);
+      equalNumber(match.eloAfter[modelB], nextB, `eloAfter.${modelB}`);
+      ratings[modelA] = nextA;
+      ratings[modelB] = nextB;
+      matches.push(match);
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : String(error);
+      throw new Error(`Journal line ${line}: ${reason}`, { cause: error });
+    }
+  }
+  return {
+    category,
+    methodologyVersion,
+    matches,
+    ratings,
+    points,
+    runs: [...runs],
+    warnings: [...warnings]
+  };
+}
+function verifiedEloState(matches, category) {
+  const verified = verifyJournal(matches, category);
+  return { ratings: verified.ratings, points: verified.points };
+}
+
+// src/report.ts
+function round(value, digits = 2) {
+  return Number(value.toFixed(digits));
+}
+function emptyEntry(modelId, displayName) {
+  return {
+    modelId,
+    displayName,
+    elo: ELO_INITIAL,
+    points: 0,
+    wins: 0,
+    losses: 0,
+    forfeits: 0,
+    matches: 0,
+    winRate: 0,
+    unanimousWins: 0,
+    totalCostUsd: 0,
+    byCluster: {}
+  };
+}
+function ensureEntry(entries, modelId) {
+  const existing = entries.get(modelId);
+  if (existing) return existing;
+  let displayName = modelId;
+  try {
+    displayName = _chunk4TWPCPRPcjs.getModel.call(void 0, modelId).displayName;
+  } catch (e) {
+  }
+  const created = emptyEntry(modelId, displayName);
+  entries.set(modelId, created);
+  return created;
+}
+function buildSnapshot(matches, category, options = {}) {
+  const verified = verifyJournal(matches, category, options);
+  const entries = /* @__PURE__ */ new Map();
+  const competitorIds = _nullishCoalesce(options.competitorIds, () => ( _chunk4TWPCPRPcjs.listModels.call(void 0, "competitor").map((model) => model.id)));
+  for (const modelId of competitorIds) {
+    ensureEntry(entries, modelId);
+  }
+  for (const match of verified.matches) {
+    const modelA = ensureEntry(entries, match.competitors.modelA);
+    const modelB = ensureEntry(entries, match.competitors.modelB);
+    modelA.elo = _nullishCoalesce(match.eloAfter[modelA.modelId], () => ( modelA.elo));
+    modelB.elo = _nullishCoalesce(match.eloAfter[modelB.modelId], () => ( modelB.elo));
+    modelA.totalCostUsd += _chunkCIXITJW6cjs.competitorCost.call(void 0, match.competitors.responseA);
+    modelB.totalCostUsd += _chunkCIXITJW6cjs.competitorCost.call(void 0, match.competitors.responseB);
+    if (match.outcome === "no-contest" || !match.winnerModelId) continue;
+    modelA.matches += 1;
+    modelB.matches += 1;
+    const winner = ensureEntry(entries, match.winnerModelId);
+    const loser = match.winnerModelId === modelA.modelId ? modelB : modelA;
+    winner.points += 1;
+    winner.wins += 1;
+    loser.losses += 1;
+    if (match.outcome === "forfeit") loser.forfeits += 1;
+    if (_optionalChain([match, 'access', _16 => _16.panel, 'optionalAccess', _17 => _17.agreement]) === "unanimous") winner.unanimousWins += 1;
+    const cluster = match.task.cluster;
+    winner.byCluster[cluster] = incrementCluster(winner.byCluster[cluster], "wins");
+    loser.byCluster[cluster] = incrementCluster(loser.byCluster[cluster], "losses");
+  }
+  const leaderboard = [...entries.values()].map((entry) => ({
+    ...entry,
+    elo: round(_nullishCoalesce(verified.ratings[entry.modelId], () => ( ELO_INITIAL))),
+    winRate: entry.matches === 0 ? 0 : round(entry.wins / entry.matches * 100, 1),
+    totalCostUsd: round(entry.totalCostUsd, 6)
+  })).sort(
+    (a, b) => b.elo - a.elo || b.points - a.points || a.displayName.localeCompare(b.displayName)
+  ).map((entry, index) => ({ rank: index + 1, ...entry }));
+  return {
+    version: "0.2.0",
+    methodologyVersion: _nullishCoalesce(verified.methodologyVersion, () => ( _chunkCIXITJW6cjs.METHODOLOGY_VERSION)),
+    category,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    initialElo: ELO_INITIAL,
+    kFactor: ELO_K,
+    leaderboard,
+    matches: verified.matches
+  };
+}
+function incrementCluster(current, field) {
+  const next = _nullishCoalesce(current, () => ( { wins: 0, losses: 0 }));
+  return { ...next, [field]: next[field] + 1 };
+}
+function renderMarkdown(snapshot) {
+  const meta = _chunkCIXITJW6cjs.CATEGORY_META[snapshot.category];
+  const modelNames = new Map(
+    snapshot.leaderboard.map((entry) => [entry.modelId, entry.displayName])
+  );
+  const rows = snapshot.leaderboard.map(
+    (entry) => `| ${entry.rank} | ${entry.displayName} | ${entry.elo.toFixed(2)} | ${entry.points} | ${entry.wins}-${entry.losses} | ${entry.forfeits} | ${entry.winRate.toFixed(1)}% | $${entry.totalCostUsd.toFixed(4)} |`
+  );
+  const recent = snapshot.matches.slice(-20).reverse().map((match) => {
+    const a = _nullishCoalesce(modelNames.get(match.competitors.modelA), () => ( match.competitors.modelA));
+    const b = _nullishCoalesce(modelNames.get(match.competitors.modelB), () => ( match.competitors.modelB));
+    const winner = match.winnerModelId ? _nullishCoalesce(modelNames.get(match.winnerModelId), () => ( match.winnerModelId)) : "No contest";
+    return `| ${match.task.id} | ${a} vs ${b} | ${winner} | ${match.outcome} | $${match.matchCostUsd.toFixed(4)} |`;
+  });
+  return `# BridgeBench V3 ${meta.label} Arena
+
+${meta.tagline}
+
+Generated ${snapshot.generatedAt}. Ratings start at ${snapshot.initialElo} with K=${snapshot.kFactor}.
+
+## Leaderboard
+
+| Rank | Model | Elo | Points | W-L | Forfeits | Win rate | Competitor cost |
+|---:|---|---:|---:|---:|---:|---:|---:|
+${rows.join("\n")}
+
+## Recent matches
+
+| Task | Matchup | Winner | Outcome | Total cost |
+|---|---|---|---|---:|
+${recent.join("\n") || "| \u2014 | \u2014 | \u2014 | \u2014 | \u2014 |"}
+`;
+}
+function writeReports(store, options = {}) {
+  const snapshot = buildSnapshot(store.readAll(), store.category, {
+    ...options,
+    manifestForRun: store.readRunManifest ? (runId) => store.readRunManifest(runId) : void 0,
+    requireManifests: Boolean(store.readRunManifest)
+  });
+  store.writeSnapshot(snapshot);
+  store.writeMarkdown(renderMarkdown(snapshot));
+  return snapshot;
+}
+
+// src/scheduler.ts
+
+function hashSeed(seed) {
+  return Number.parseInt(_crypto.createHash.call(void 0, "sha256").update(seed).digest("hex").slice(0, 8), 16);
+}
+function mulberry32(seed) {
+  return () => {
+    seed |= 0;
+    seed = seed + 1831565813 | 0;
+    let value = Math.imul(seed ^ seed >>> 15, 1 | seed);
+    value = value + Math.imul(value ^ value >>> 7, 61 | value) ^ value;
+    return ((value ^ value >>> 14) >>> 0) / 4294967296;
+  };
+}
+function stableId(parts) {
+  return _crypto.createHash.call(void 0, "sha256").update(parts.join("|")).digest("hex").slice(0, 20);
+}
+function pairKey(candidate) {
+  return `${candidate.left}|${candidate.right}`;
+}
+function candidateScore(candidate, modelExposure, taskExposure, pairExposure) {
+  const left = _nullishCoalesce(modelExposure.get(candidate.left), () => ( 0));
+  const right = _nullishCoalesce(modelExposure.get(candidate.right), () => ( 0));
+  return [
+    Math.max(left, right),
+    left + right,
+    _nullishCoalesce(taskExposure.get(candidate.taskId), () => ( 0)),
+    _nullishCoalesce(pairExposure.get(pairKey(candidate)), () => ( 0))
+  ];
+}
+function compareScores(left, right) {
+  for (let index = 0; index < left.length; index += 1) {
+    const difference = left[index] - right[index];
+    if (difference !== 0) return difference;
+  }
+  return 0;
+}
+function scheduleMatches(input) {
+  if (!Number.isInteger(input.count) || input.count < 1)
+    throw new Error("Match count must be a positive integer");
+  if (input.modelIds.length < 2) throw new Error("At least two competitor models are required");
+  if (new Set(input.modelIds).size !== input.modelIds.length)
+    throw new Error("Competitor model IDs must be unique");
+  const random = mulberry32(hashSeed(`${input.category}|${input.seed}`));
+  const runId = _nullishCoalesce(input.runId, () => ( `run-${stableId([
+    _chunkCIXITJW6cjs.METHODOLOGY_VERSION,
+    input.category,
+    input.seed,
+    String(input.count),
+    ...[...input.modelIds].sort(),
+    ...input.tasks.map((task) => task.publicHash)
+  ])}`));
+  const modelExposure = new Map(input.modelIds.map((id) => [id, 0]));
+  const taskExposure = new Map(input.tasks.map((task) => [task.public.id, 0]));
+  const pairExposure = /* @__PURE__ */ new Map();
+  const used = /* @__PURE__ */ new Set();
+  const combinations = [];
+  for (let i = 0; i < input.modelIds.length; i += 1) {
+    for (let j = i + 1; j < input.modelIds.length; j += 1) {
+      for (const task of input.tasks) {
+        combinations.push({
+          left: input.modelIds[i],
+          right: input.modelIds[j],
+          taskId: task.public.id
+        });
+      }
+    }
+  }
+  const schedule = [];
+  for (let index = 0; index < input.count; index += 1) {
+    if (used.size === combinations.length) used.clear();
+    const candidates = combinations.filter(
+      (candidate) => !used.has(`${candidate.left}|${candidate.right}|${candidate.taskId}`)
+    );
+    candidates.sort(
+      (left, right) => compareScores(
+        candidateScore(left, modelExposure, taskExposure, pairExposure),
+        candidateScore(right, modelExposure, taskExposure, pairExposure)
+      )
+    );
+    const best = candidates[0];
+    const bestScore = candidateScore(best, modelExposure, taskExposure, pairExposure);
+    const tied = candidates.filter(
+      (candidate) => compareScores(
+        candidateScore(candidate, modelExposure, taskExposure, pairExposure),
+        bestScore
+      ) === 0
+    );
+    const selected = tied[Math.floor(random() * tied.length)];
+    used.add(`${selected.left}|${selected.right}|${selected.taskId}`);
+    modelExposure.set(selected.left, modelExposure.get(selected.left) + 1);
+    modelExposure.set(selected.right, modelExposure.get(selected.right) + 1);
+    taskExposure.set(selected.taskId, taskExposure.get(selected.taskId) + 1);
+    const selectedPairKey = pairKey(selected);
+    pairExposure.set(selectedPairKey, (_nullishCoalesce(pairExposure.get(selectedPairKey), () => ( 0))) + 1);
+    const swap = random() >= 0.5;
+    const modelA = swap ? selected.right : selected.left;
+    const modelB = swap ? selected.left : selected.right;
+    schedule.push({
+      id: `match-${stableId([runId, String(index), selected.taskId, modelA, modelB])}`,
+      runId,
+      index,
+      seed: input.seed,
+      category: input.category,
+      taskId: selected.taskId,
+      modelA,
+      modelB
+    });
+  }
+  return schedule;
+}
+
+// src/triage.ts
+var FAST_RESPONSE_MS = 5e3;
+var LOW_OUTPUT_TOKENS = 150;
+function classifyError(message) {
+  if (/premature close|invalid response body/i.test(message)) return "premature-close";
+  if (/timed out|timeout|ETIMEDOUT/i.test(message)) return "timeout";
+  if (/429|rate.?limit/i.test(message)) return "rate-limit";
+  if (/5\d\d|overloaded/i.test(message)) return "server-error";
+  if (/401|403|unauthorized|forbidden/i.test(message)) return "auth";
+  if (/empty completion/i.test(message)) return "empty-completion";
+  return "other";
+}
+function detectResponseAnomalies(response) {
+  if (!response.success) return ["failed"];
+  const flags = [];
+  if (response.latencyMs < FAST_RESPONSE_MS) flags.push("fast-response");
+  if (response.outputTokens < LOW_OUTPUT_TOKENS) flags.push("low-output");
+  if (response.finishReason === "length") flags.push("truncated");
+  if (!response.reasoningTokens) flags.push("reasoning-unreported");
+  if (response.costUsd === 0) flags.push("zero-cost");
+  return flags;
+}
+function describeFlags(response) {
+  if (!response.success) return response.error;
+  return `${response.latencyMs}ms, ${response.outputTokens} output tokens, reasoning ${_nullishCoalesce(response.reasoningTokens, () => ( "unreported"))}, finish ${response.finishReason}, $${response.costUsd.toFixed(4)}`;
+}
+function triageJournal(results) {
+  const byRun = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    const bucket = _nullishCoalesce(byRun.get(result.runId), () => ( []));
+    bucket.push(result);
+    byRun.set(result.runId, bucket);
+  }
+  const reports = [];
+  for (const [runId, runResults] of byRun) {
+    const report = {
+      runId,
+      seed: runResults[0].seed,
+      firstTimestamp: runResults[0].timestamp,
+      matches: runResults.length,
+      outcomes: { judged: 0, forfeit: 0, "no-contest": 0 },
+      totalCostUsd: 0,
+      errorClasses: {},
+      models: {},
+      judge: { validVotes: 0, abstentions: 0, unanimous: 0, split: 0, avgConfidence: null },
+      anomalies: []
+    };
+    const confidences = [];
+    for (const result of runResults) {
+      report.outcomes[result.outcome] += 1;
+      report.totalCostUsd += result.matchCostUsd;
+      for (const response of [result.competitors.responseA, result.competitors.responseB]) {
+        const model = report.models[response.modelId] ??= {
+          matches: 0,
+          failures: 0,
+          wins: 0,
+          avgLatencyMs: null,
+          avgOutputTokens: null,
+          reasoningReported: 0
+        };
+        model.matches += 1;
+        if (!response.success) {
+          model.failures += 1;
+          const errorClass = classifyError(_nullishCoalesce(response.error, () => ( "")));
+          report.errorClasses[errorClass] = (_nullishCoalesce(report.errorClasses[errorClass], () => ( 0))) + 1;
+        } else {
+          model.avgLatencyMs = ((_nullishCoalesce(model.avgLatencyMs, () => ( 0))) * (model.matches - model.failures - 1) + response.latencyMs) / (model.matches - model.failures);
+          model.avgOutputTokens = ((_nullishCoalesce(model.avgOutputTokens, () => ( 0))) * (model.matches - model.failures - 1) + response.outputTokens) / (model.matches - model.failures);
+          if (response.reasoningTokens) model.reasoningReported += 1;
+        }
+        const flags = detectResponseAnomalies(response);
+        for (const flag of flags) {
+          report.anomalies.push({
+            matchId: result.matchId,
+            scheduleIndex: result.scheduleIndex,
+            taskId: result.task.id,
+            modelId: response.modelId,
+            flag,
+            detail: describeFlags(response)
+          });
+        }
+      }
+      if (result.winnerModelId && report.models[result.winnerModelId]) {
+        report.models[result.winnerModelId].wins += 1;
+      }
+      if (result.panel) {
+        if (result.panel.agreement === "unanimous") report.judge.unanimous += 1;
+        if (result.panel.agreement === "split") report.judge.split += 1;
+        for (const vote of result.panel.votes) {
+          if (vote.verdict) {
+            report.judge.validVotes += 1;
+            confidences.push(vote.verdict.confidence);
+          } else {
+            report.judge.abstentions += 1;
+            report.anomalies.push({
+              matchId: result.matchId,
+              scheduleIndex: result.scheduleIndex,
+              taskId: result.task.id,
+              modelId: vote.judgeModelId,
+              flag: "judge-abstained",
+              detail: _nullishCoalesce(vote.error, () => ( "no valid verdict"))
+            });
+          }
+        }
+      }
+      if (result.outcome === "judged" && result.matchCostUsd === 0) {
+        report.anomalies.push({
+          matchId: result.matchId,
+          scheduleIndex: result.scheduleIndex,
+          taskId: result.task.id,
+          modelId: null,
+          flag: "zero-cost-match",
+          detail: "judged match reported $0 total cost"
+        });
+      }
+    }
+    if (confidences.length > 0) {
+      report.judge.avgConfidence = confidences.reduce((sum, value) => sum + value, 0) / confidences.length;
+    }
+    reports.push(report);
+  }
+  return reports.sort((left, right) => left.firstTimestamp.localeCompare(right.firstTimestamp));
+}
+var HEALTH_ADVICE = {
+  failed: "Request failed \u2014 check the run log for the attempt-level error and whether retries were exhausted.",
+  "fast-response": `Response landed under ${FAST_RESPONSE_MS / 1e3}s \u2014 the task may be too easy for this model, or reasoning effort was not applied.`,
+  "low-output": `Fewer than ${LOW_OUTPUT_TOKENS} output tokens \u2014 likely a shallow answer; consider hardening the task.`,
+  truncated: "finish_reason=length \u2014 raise maxTokens or shorten the prompt.",
+  "reasoning-unreported": "Provider reported no reasoning tokens \u2014 verify the reasoning parameter reaches the model (arena generation <id>).",
+  "zero-cost": "Zero cost on a successful response \u2014 usage accounting may be broken.",
+  "zero-cost-match": "A judged match with $0 spend means usage accounting is broken end to end.",
+  "judge-abstained": "A judge returned no valid structured verdict twice \u2014 inspect judge.verdict-parse-failed entries in the run log."
+};
+function formatTriage(reports) {
+  const lines = [];
+  for (const report of reports) {
+    lines.push(`run ${report.runId} (seed "${report.seed}", ${report.firstTimestamp})`);
+    lines.push(
+      `  matches ${report.matches} | judged ${report.outcomes.judged}, forfeit ${report.outcomes.forfeit}, no-contest ${report.outcomes["no-contest"]} | spend $${report.totalCostUsd.toFixed(4)}`
+    );
+    if (Object.keys(report.errorClasses).length > 0) {
+      lines.push(
+        `  request failures: ${Object.entries(report.errorClasses).map(([errorClass, count]) => `${errorClass}\xD7${count}`).join(", ")}`
+      );
+    }
+    lines.push(
+      `  judges: ${report.judge.validVotes} valid votes, ${report.judge.abstentions} abstentions, ${report.judge.unanimous} unanimous / ${report.judge.split} split panels` + (report.judge.avgConfidence === null ? "" : `, avg confidence ${report.judge.avgConfidence.toFixed(2)}`)
+    );
+    for (const [modelId, model] of Object.entries(report.models)) {
+      const latency = model.avgLatencyMs === null ? "\u2014" : `${Math.round(model.avgLatencyMs)}ms`;
+      const tokens = model.avgOutputTokens === null ? "\u2014" : `${Math.round(model.avgOutputTokens)} tok`;
+      lines.push(
+        `    ${modelId.padEnd(30)} matches ${model.matches}, wins ${model.wins}, failures ${model.failures}, avg ${latency} / ${tokens}, reasoning reported ${model.reasoningReported}/${model.matches - model.failures}`
+      );
+    }
+    if (report.anomalies.length === 0) {
+      lines.push("  no anomalies detected");
+    } else {
+      lines.push(`  anomalies (${report.anomalies.length}):`);
+      const byFlag = /* @__PURE__ */ new Map();
+      for (const anomaly of report.anomalies) {
+        const bucket = _nullishCoalesce(byFlag.get(anomaly.flag), () => ( []));
+        bucket.push(anomaly);
+        byFlag.set(anomaly.flag, bucket);
+      }
+      for (const [flag, anomalies] of byFlag) {
+        lines.push(`    ${flag} \xD7${anomalies.length} \u2014 ${_nullishCoalesce(HEALTH_ADVICE[flag], () => ( ""))}`);
+        for (const anomaly of anomalies.slice(0, 4)) {
+          lines.push(
+            `      [${anomaly.scheduleIndex}] ${anomaly.taskId} ${_nullishCoalesce(anomaly.modelId, () => ( ""))}: ${anomaly.detail.slice(0, 140)}`
+          );
+        }
+        if (anomalies.length > 4) lines.push(`      \u2026 ${anomalies.length - 4} more`);
+      }
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+// src/arena.ts
+var HEALTH_STOP_MIN_MATCHES = 4;
+var HEALTH_STOP_FAILURE_RATE = 0.5;
+function failedResponse(modelId, error, latencyMs) {
+  return {
+    modelId,
+    success: false,
+    error: _chunk4TWPCPRPcjs.sanitizeError.call(void 0, error),
+    latencyMs
+  };
+}
+var ArenaRunner = class {
+  constructor(gateway, store, onEvent, logger = _chunk4TWPCPRPcjs.noopLogger) {
+    this.gateway = gateway;
+    this.store = store;
+    this.onEvent = onEvent;
+    this.logger = logger;
+    this.judges = new (0, _chunk4TWPCPRPcjs.JudgePanel)(gateway, onEvent, logger);
+  }
+  
+  
+  
+  
+  
+  emit(event) {
+    _optionalChain([this, 'access', _18 => _18.onEvent, 'optionalCall', _19 => _19({ ...event, timestamp: (/* @__PURE__ */ new Date()).toISOString() })]);
+  }
+  observeCancellation(runId, progress, signal) {
+    let emitted = false;
+    const requested = () => {
+      if (emitted) return;
+      emitted = true;
+      this.logger.info("run.cancellation-requested", {
+        runId,
+        completed: progress.completed,
+        costUsd: progress.costUsd
+      });
+      this.emit({
+        id: `${runId}-cancellation-requested`,
+        type: "run.cancellation-requested",
+        data: {
+          runId,
+          completed: progress.completed,
+          costUsd: progress.costUsd
+        }
+      });
+    };
+    if (_optionalChain([signal, 'optionalAccess', _20 => _20.aborted])) requested();
+    else _optionalChain([signal, 'optionalAccess', _21 => _21.addEventListener, 'call', _22 => _22("abort", requested, { once: true })]);
+    return {
+      requested,
+      dispose: () => _optionalChain([signal, 'optionalAccess', _23 => _23.removeEventListener, 'call', _24 => _24("abort", requested)])
+    };
+  }
+  cancelRun(runId, progress) {
+    this.logger.info("run.cancelled", {
+      runId,
+      completed: progress.completed,
+      costUsd: progress.costUsd
+    });
+    this.emit({
+      id: `${runId}-cancelled`,
+      type: "run.cancelled",
+      data: {
+        runId,
+        completed: progress.completed,
+        costUsd: progress.costUsd
+      }
+    });
+    return {
+      runId,
+      completed: progress.completed,
+      costUsd: progress.costUsd,
+      stoppedForBudget: progress.stoppedForBudget,
+      cancelled: true
+    };
+  }
+  async prepareRun(config, tasks, signal) {
+    const competitors = _chunk4TWPCPRPcjs.resolveCompetitorRoster.call(void 0, config.competitorIds);
+    const competitorIds = competitors.map((model) => model.id);
+    const mismatched = tasks.filter((task) => task.public.category !== config.category);
+    if (mismatched.length > 0) {
+      throw new Error(
+        `Run category ${config.category} received tasks from another arena: ${mismatched.map((task) => task.public.id).join(", ")}`
+      );
+    }
+    const journal = this.store.readAll();
+    const incompatible = journal.find((match) => match.methodologyVersion !== _chunkCIXITJW6cjs.METHODOLOGY_VERSION);
+    if (incompatible) {
+      throw new Error(
+        `Cannot append ${_chunkCIXITJW6cjs.METHODOLOGY_VERSION} matches to a ${incompatible.methodologyVersion} journal; archive it and start a new journal`
+      );
+    }
+    this.store.rebuildEloState(competitorIds);
+    const manifest = createRunManifest(config, tasks, competitors);
+    const manifestHash = runManifestHash(manifest);
+    const runId = runIdFromManifest(manifest);
+    const schedule = scheduleMatches({
+      category: config.category,
+      seed: config.seed,
+      count: config.matches,
+      modelIds: competitorIds,
+      tasks,
+      runId
+    });
+    const existing = journal.filter((result) => result.runId === runId);
+    const prepared = {
+      runId,
+      manifestHash,
+      competitors,
+      schedule,
+      tasksById: new Map(tasks.map((task) => [task.public.id, task])),
+      completedIds: new Set(journal.map((result) => result.matchId)),
+      progress: {
+        costUsd: existing.reduce((sum, result) => sum + result.matchCostUsd, 0),
+        completed: 0,
+        matchesWithFailures: 0,
+        stoppedForBudget: false
+      }
+    };
+    if (_optionalChain([signal, 'optionalAccess', _25 => _25.aborted])) return prepared;
+    try {
+      await Promise.all(
+        [...competitors, ..._chunk4TWPCPRPcjs.listModels.call(void 0, "judge")].map(
+          (model) => this.gateway.validateModel(model, signal)
+        )
+      );
+    } catch (error) {
+      if (_optionalChain([signal, 'optionalAccess', _26 => _26.aborted])) return prepared;
+      throw error;
+    }
+    if (_optionalChain([signal, 'optionalAccess', _27 => _27.aborted])) return prepared;
+    this.store.writeRunManifest(runId, manifest);
+    return prepared;
+  }
+  async run(config, tasks, execution = {}) {
+    const { signal, onMatchResult } = execution;
+    const { runId, manifestHash, competitors, schedule, tasksById, completedIds, progress } = await this.prepareRun(config, tasks, signal);
+    const competitorIds = competitors.map((model) => model.id);
+    const cancellation = this.observeCancellation(runId, progress, signal);
+    try {
+      if (_optionalChain([signal, 'optionalAccess', _28 => _28.aborted])) return this.cancelRun(runId, progress);
+      this.logger.info("run.started", {
+        runId,
+        category: config.category,
+        seed: config.seed,
+        matches: config.matches,
+        maxCostUsd: config.maxCostUsd,
+        resume: config.resume,
+        healthStop: config.healthStop !== false,
+        competitorIds
+      });
+      this.emit({
+        id: `${runId}-started`,
+        type: "run.started",
+        data: {
+          runId,
+          category: config.category,
+          seed: config.seed,
+          matches: config.matches,
+          maxCostUsd: config.maxCostUsd,
+          competitorIds
+        }
+      });
+      for (const match of schedule) {
+        if (_optionalChain([signal, 'optionalAccess', _29 => _29.aborted])) return this.cancelRun(runId, progress);
+        if (completedIds.has(match.id)) {
+          if (config.resume) continue;
+          throw new Error(
+            `Match ${match.id} is already journaled; rerun with --resume or choose a new seed`
+          );
+        }
+        if (progress.costUsd >= config.maxCostUsd) {
+          progress.stoppedForBudget = true;
+          this.emit({
+            id: `${match.runId}-budget-stopped`,
+            type: "run.budget-stopped",
+            data: {
+              runId: match.runId,
+              completed: progress.completed,
+              costUsd: progress.costUsd,
+              maxCostUsd: config.maxCostUsd
+            }
+          });
+          break;
+        }
+        const task = tasksById.get(match.taskId);
+        if (!task) throw new Error(`Scheduled task not found: ${match.taskId}`);
+        this.emit({
+          id: `${match.id}-started`,
+          type: "match.started",
+          data: {
+            matchId: match.id,
+            index: match.index,
+            total: schedule.length,
+            category: match.category,
+            taskId: match.taskId,
+            taskTitle: task.public.title,
+            modelA: match.modelA,
+            modelB: match.modelB
+          }
+        });
+        let result;
+        try {
+          result = await this.runMatch(match, task, manifestHash, competitorIds, signal);
+        } catch (error) {
+          if (_optionalChain([signal, 'optionalAccess', _30 => _30.aborted]) || _chunk4TWPCPRPcjs.isArenaCancellationError.call(void 0, error)) {
+            cancellation.requested();
+            return this.cancelRun(runId, progress);
+          }
+          throw error;
+        }
+        if (_optionalChain([signal, 'optionalAccess', _31 => _31.aborted])) return this.cancelRun(runId, progress);
+        this.store.append(result);
+        writeReports(this.store, { competitorIds });
+        progress.costUsd += result.matchCostUsd;
+        progress.completed += 1;
+        if (!result.competitors.responseA.success || !result.competitors.responseB.success) {
+          progress.matchesWithFailures += 1;
+        }
+        this.emit({
+          id: `${match.id}-completed`,
+          type: "match.completed",
+          data: {
+            matchId: match.id,
+            taskId: match.taskId,
+            winnerModelId: result.winnerModelId,
+            outcome: result.outcome,
+            costUsd: result.matchCostUsd,
+            eloAfter: result.eloAfter,
+            completed: progress.completed,
+            total: schedule.length
+          }
+        });
+        if (onMatchResult) await onMatchResult(result);
+        if (_optionalChain([signal, 'optionalAccess', _32 => _32.aborted])) return this.cancelRun(runId, progress);
+        if (config.healthStop !== false && progress.completed >= HEALTH_STOP_MIN_MATCHES && progress.matchesWithFailures / progress.completed >= HEALTH_STOP_FAILURE_RATE) {
+          this.logger.error("run.health-stopped", {
+            runId,
+            completed: progress.completed,
+            matchesWithFailures: progress.matchesWithFailures,
+            failureRate: progress.matchesWithFailures / progress.completed
+          });
+          throw new Error(
+            `Run halted after ${progress.completed} matches: ${progress.matchesWithFailures} had failed competitor responses. The provider path is unhealthy, so continuing would journal junk matches. Inspect the run log${this.logger.filePath ? ` at ${this.logger.filePath}` : ""}, fix the cause, then rerun with --resume (or pass --no-health-stop to override).`
+          );
+        }
+      }
+      if (_optionalChain([signal, 'optionalAccess', _33 => _33.aborted])) return this.cancelRun(runId, progress);
+      writeReports(this.store, { competitorIds });
+      this.logger.info("run.completed", {
+        runId,
+        completed: progress.completed,
+        matchesWithFailures: progress.matchesWithFailures,
+        costUsd: progress.costUsd,
+        stoppedForBudget: progress.stoppedForBudget,
+        cancelled: false
+      });
+      this.emit({
+        id: `${runId}-completed`,
+        type: "run.completed",
+        data: {
+          runId,
+          completed: progress.completed,
+          costUsd: progress.costUsd,
+          stoppedForBudget: progress.stoppedForBudget
+        }
+      });
+      return {
+        runId,
+        completed: progress.completed,
+        costUsd: progress.costUsd,
+        stoppedForBudget: progress.stoppedForBudget,
+        cancelled: false
+      };
+    } finally {
+      cancellation.dispose();
+    }
+  }
+  async runMatch(match, task, manifestHash, competitorIds, signal) {
+    _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+    const state = this.store.rebuildEloState(competitorIds);
+    const prompt = _chunk4TWPCPRPcjs.buildCompetitorPrompt.call(void 0, task);
+    const [responseA, responseB] = await Promise.all([
+      this.runCompetitor(match, "A", prompt, signal),
+      this.runCompetitor(match, "B", prompt, signal)
+    ]);
+    _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+    this.inspectResponse(match, responseA);
+    this.inspectResponse(match, responseB);
+    this.emit({
+      id: `${match.id}-competitors-completed`,
+      type: "competitors.completed",
+      data: {
+        matchId: match.id,
+        modelA: {
+          success: responseA.success,
+          latencyMs: responseA.latencyMs,
+          costUsd: _chunkCIXITJW6cjs.competitorCost.call(void 0, responseA),
+          outputTokens: _chunkCIXITJW6cjs.competitorOutputTokens.call(void 0, responseA),
+          reasoningTokens: _nullishCoalesce(_chunkCIXITJW6cjs.competitorReasoningTokens.call(void 0, responseA), () => ( null))
+        },
+        modelB: {
+          success: responseB.success,
+          latencyMs: responseB.latencyMs,
+          costUsd: _chunkCIXITJW6cjs.competitorCost.call(void 0, responseB),
+          outputTokens: _chunkCIXITJW6cjs.competitorOutputTokens.call(void 0, responseB),
+          reasoningTokens: _nullishCoalesce(_chunkCIXITJW6cjs.competitorReasoningTokens.call(void 0, responseB), () => ( null))
+        }
+      }
+    });
+    _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+    const eloBefore = {
+      [match.modelA]: _nullishCoalesce(state.ratings[match.modelA], () => ( ELO_INITIAL)),
+      [match.modelB]: _nullishCoalesce(state.ratings[match.modelB], () => ( ELO_INITIAL))
+    };
+    const eloAfter = { ...eloBefore };
+    let outcome = "no-contest";
+    let winnerModelId = null;
+    let panel = null;
+    if (responseA.success !== responseB.success) {
+      outcome = "forfeit";
+      winnerModelId = responseA.success ? match.modelA : match.modelB;
+    } else if (responseA.success && responseB.success) {
+      this.emit({
+        id: `${match.id}-judging-started`,
+        type: "judging.started",
+        data: { matchId: match.id, judges: _chunk4TWPCPRPcjs.listModels.call(void 0, "judge").map((model) => model.id) }
+      });
+      panel = await this.judges.judge({ match, task, responseA, responseB }, signal);
+      _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+      if (panel.winnerModelId) {
+        outcome = "judged";
+        winnerModelId = panel.winnerModelId;
+      }
+    }
+    if (winnerModelId) {
+      const update = applyEloWin(
+        eloBefore[match.modelA],
+        eloBefore[match.modelB],
+        winnerModelId === match.modelA ? "a" : "b"
+      );
+      eloAfter[match.modelA] = update.ratingA;
+      eloAfter[match.modelB] = update.ratingB;
+    }
+    const judgeCost = _nullishCoalesce(_optionalChain([panel, 'optionalAccess', _34 => _34.votes, 'access', _35 => _35.reduce, 'call', _36 => _36((sum, vote) => sum + (_nullishCoalesce(_optionalChain([vote, 'access', _37 => _37.completion, 'optionalAccess', _38 => _38.costUsd]), () => ( 0))), 0)]), () => ( 0));
+    return {
+      methodologyVersion: _chunkCIXITJW6cjs.METHODOLOGY_VERSION,
+      runId: match.runId,
+      runManifestHash: manifestHash,
+      matchId: match.id,
+      scheduleIndex: match.index,
+      seed: match.seed,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      task: {
+        id: task.public.id,
+        version: task.public.version,
+        category: task.public.category,
+        cluster: task.public.cluster,
+        publicHash: task.publicHash,
+        privateHash: task.privateHash
+      },
+      competitors: { modelA: match.modelA, modelB: match.modelB, responseA, responseB },
+      outcome,
+      winnerModelId,
+      panel,
+      eloBefore,
+      eloAfter,
+      pointAwarded: winnerModelId !== null,
+      matchCostUsd: _chunkCIXITJW6cjs.competitorCost.call(void 0, responseA) + _chunkCIXITJW6cjs.competitorCost.call(void 0, responseB) + judgeCost
+    };
+  }
+  /** Surface signals that make a journaled response untrustworthy or a task too easy. */
+  inspectResponse(match, response) {
+    if (!response.success) {
+      this.logger.warn("competitor.failed", {
+        matchId: match.id,
+        taskId: match.taskId,
+        modelId: response.modelId,
+        latencyMs: response.latencyMs,
+        error: _nullishCoalesce(response.error, () => ( "unknown"))
+      });
+      return;
+    }
+    const flags = detectResponseAnomalies(response);
+    if (flags.length > 0) {
+      this.logger.warn("competitor.suspicious", {
+        matchId: match.id,
+        taskId: match.taskId,
+        modelId: response.modelId,
+        flags,
+        latencyMs: response.latencyMs,
+        outputTokens: response.outputTokens,
+        reasoningTokens: _nullishCoalesce(response.reasoningTokens, () => ( null)),
+        costUsd: response.costUsd,
+        finishReason: response.finishReason
+      });
+    }
+  }
+  async runCompetitor(match, side, prompt, signal) {
+    const modelId = side === "A" ? match.modelA : match.modelB;
+    const startedAt = Date.now();
+    let sequence = 0;
+    const emitDelta = (text, done, success = true) => {
+      sequence += 1;
+      this.emit({
+        id: `${match.id}-delta-${side}-${sequence}`,
+        type: "competitor.delta",
+        data: { matchId: match.id, modelId, side, text, done, success }
+      });
+    };
+    try {
+      _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+      const completion = await this.gateway.complete({
+        model: _chunk4TWPCPRPcjs.getModel.call(void 0, modelId),
+        ...prompt,
+        signal,
+        onDelta: (text) => emitDelta(text, false)
+      });
+      _chunk4TWPCPRPcjs.throwIfCancelled.call(void 0, signal);
+      emitDelta(completion.content, true);
+      return { modelId, success: true, ...completion };
+    } catch (error) {
+      if (_optionalChain([signal, 'optionalAccess', _39 => _39.aborted]) || _chunk4TWPCPRPcjs.isArenaCancellationError.call(void 0, error)) {
+        throw new (0, _chunk4TWPCPRPcjs.ArenaCancellationError)();
+      }
+      const failed = failedResponse(modelId, error, Date.now() - startedAt);
+      emitDelta(failed.error, true, false);
+      return failed;
+    }
+  }
+};
+
+// src/mock-gateway.ts
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function baseCompletion(content, generationId) {
+  return {
+    generationId,
+    content,
+    inputTokens: 120,
+    outputTokens: Math.max(1, Math.ceil(content.length / 4)),
+    costUsd: 0.02,
+    latencyMs: 250,
+    finishReason: "stop",
+    attempts: 1
+  };
+}
+var MockOpenRouterGateway = class {
+  constructor(options = {}) {
+    this.options = options;
+  }
+  
+  async validateModel(_model) {
+  }
+  async complete(request) {
+    if (_optionalChain([request, 'access', _40 => _40.signal, 'optionalAccess', _41 => _41.aborted])) {
+      throw new Error("Mock completion aborted");
+    }
+    if (request.structured) {
+      const winner = _nullishCoalesce(this.options.judgeWinner, () => ( "MODEL_A"));
+      const payload = {
+        winner,
+        confidence: 0.9,
+        rationale: "The selected mock response is better grounded.",
+        criteria: {
+          correctness: "Correct.",
+          grounding: "Grounded.",
+          constraintHandling: "Complete.",
+          completeness: "Complete."
+        },
+        violations: []
+      };
+      void _chunk4TWPCPRPcjs.judgeVerdictJsonSchema.call(void 0, );
+      return baseCompletion(JSON.stringify(payload), `mock-judge-${request.model.id}`);
+    }
+    const variantA = [
+      `The terminal state follows the cancellation edge before retry exhaustion.`,
+      `The journal records a cancel-requested transition before the final attempt window closed, which rules out budget-stop and health-stop.`,
+      `The discriminating evidence is the cancellation timestamp preceding the last retry.`
+    ];
+    const variantB = [
+      `Working backwards from the final journal entry, the run ends on the cancellation path rather than retry exhaustion.`,
+      `Spend stayed below the budget threshold and no health probes failed in the window, so those edges are excluded.`,
+      `The single strongest signal is the cancel-requested event landing before the retry counter reached its limit.`
+    ];
+    const checksum = [...request.model.id].reduce((sum, ch) => sum + ch.charCodeAt(0), 0);
+    const text = _nullishCoalesce(this.options.competitorText, () => ( (checksum % 2 === 0 ? variantA : variantB).join(" ")));
+    const delayMs = Number(_nullishCoalesce(process.env.MOCK_CHUNK_DELAY_MS, () => ( ""))) || this.options.chunkDelayMs || 40;
+    const parts = _nullishCoalesce(text.match(/.{1,24}/g), () => ( [text]));
+    let cumulative = "";
+    for (const part of parts) {
+      if (_optionalChain([request, 'access', _42 => _42.signal, 'optionalAccess', _43 => _43.aborted])) {
+        throw new Error("Mock completion aborted");
+      }
+      cumulative += part;
+      _optionalChain([request, 'access', _44 => _44.onDelta, 'optionalCall', _45 => _45(cumulative)]);
+      if (delayMs > 0) await sleep(delayMs);
+    }
+    return baseCompletion(cumulative, `mock-competitor-${request.model.id}`);
+  }
+};
+
+// src/store.ts
+
+
+
+
+
+
+
+
+
+function categoryStoreConfig(category, resultsRoot = process.env.BRIDGEBENCH_RESULTS_DIR) {
+  const base = resultsRoot ? _path2.default.resolve(process.cwd(), resultsRoot) : _path2.default.join(process.cwd(), "results");
+  const root = _path2.default.join(base, category);
+  return {
+    category,
+    journalPath: _path2.default.join(root, "journal.jsonl"),
+    snapshotPath: _path2.default.join(root, "snapshot.json"),
+    markdownPath: _path2.default.join(root, "leaderboard.md"),
+    runsDir: _path2.default.join(root, "runs")
+  };
+}
+var ArenaStore = class {
+  constructor(config) {
+    this.config = config;
+    if (config.readOnly) return;
+    for (const file of [config.journalPath, config.snapshotPath, config.markdownPath]) {
+      _fs.mkdirSync.call(void 0, _path2.default.dirname(file), { recursive: true });
+    }
+    _fs.mkdirSync.call(void 0, this.runsDir, { recursive: true });
+  }
+  
+  get category() {
+    return this.config.category;
+  }
+  get runsDir() {
+    return _nullishCoalesce(this.config.runsDir, () => ( _path2.default.join(_path2.default.dirname(this.config.journalPath), "runs")));
+  }
+  append(result) {
+    _fs.appendFileSync.call(void 0, this.config.journalPath, `${JSON.stringify(result)}
+`, {
+      encoding: "utf8",
+      mode: 384
+    });
+  }
+  readAll() {
+    if (!_fs.existsSync.call(void 0, this.config.journalPath)) return [];
+    return _fs.readFileSync.call(void 0, this.config.journalPath, "utf8").split("\n").filter(Boolean).map((line, index) => {
+      let decoded;
+      try {
+        decoded = JSON.parse(line);
+      } catch (e2) {
+        throw new Error(`Malformed arena journal line ${index + 1}; refusing to continue`);
+      }
+      const parsed = _chunkCIXITJW6cjs.MatchResultSchema.safeParse(decoded);
+      if (!parsed.success) {
+        const issue = parsed.error.issues[0];
+        const location = _optionalChain([issue, 'optionalAccess', _46 => _46.path, 'access', _47 => _47.length]) ? ` at ${issue.path.join(".")}` : "";
+        throw new Error(
+          `Invalid arena journal line ${index + 1}${location}: ${_nullishCoalesce(_optionalChain([issue, 'optionalAccess', _48 => _48.message]), () => ( "schema mismatch"))}`
+        );
+      }
+      return parsed.data;
+    });
+  }
+  writeRunManifest(runId, manifest) {
+    const validated = RunManifestSchema.parse(manifest);
+    const target = _path2.default.join(this.runsDir, `${runId}.json`);
+    const content = `${canonicalJson(validated)}
+`;
+    if (_fs.existsSync.call(void 0, target)) {
+      const existing = _fs.readFileSync.call(void 0, target, "utf8");
+      if (existing !== content) {
+        throw new Error(`Run manifest ${runId} already exists with different inputs`);
+      }
+      return;
+    }
+    this.atomicWrite(target, content);
+  }
+  readRunManifest(runId) {
+    const target = _path2.default.join(this.runsDir, `${runId}.json`);
+    if (!_fs.existsSync.call(void 0, target)) return null;
+    try {
+      return RunManifestSchema.parse(JSON.parse(_fs.readFileSync.call(void 0, target, "utf8")));
+    } catch (error) {
+      const reason = error instanceof Error ? error.message : String(error);
+      throw new Error(`Invalid run manifest ${runId}: ${reason}`, { cause: error });
+    }
+  }
+  completedMatchIds() {
+    return new Set(this.readAll().map((result) => result.matchId));
+  }
+  rebuildEloState(competitorIds = _chunk4TWPCPRPcjs.listModels.call(void 0, "competitor").map((model) => model.id)) {
+    const ratings = Object.fromEntries(competitorIds.map((modelId) => [modelId, ELO_INITIAL]));
+    const points = Object.fromEntries(competitorIds.map((modelId) => [modelId, 0]));
+    const verified = verifyJournal(this.readAll(), this.category, {
+      manifestForRun: (runId) => this.readRunManifest(runId),
+      requireManifests: true
+    });
+    Object.assign(ratings, verified.ratings);
+    Object.assign(points, verified.points);
+    return { ratings, points };
+  }
+  writeSnapshot(snapshot) {
+    this.atomicWrite(this.config.snapshotPath, `${JSON.stringify(snapshot, null, 2)}
+`);
+  }
+  writeMarkdown(markdown) {
+    this.atomicWrite(this.config.markdownPath, markdown);
+  }
+  atomicWrite(target, content) {
+    const temporary = `${target}.tmp`;
+    _fs.writeFileSync.call(void 0, temporary, content, { encoding: "utf8", mode: 384 });
+    _fs.renameSync.call(void 0, temporary, target);
+  }
+};
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+exports.ELO_INITIAL = ELO_INITIAL; exports.ELO_K = ELO_K; exports.expectedScore = expectedScore; exports.applyEloWin = applyEloWin; exports.PACKAGE_NAME = PACKAGE_NAME; exports.ENGINE_VERSION = ENGINE_VERSION; exports.RUN_MANIFEST_VERSION = RUN_MANIFEST_VERSION; exports.RunManifestSchema = RunManifestSchema; exports.canonicalJson = canonicalJson; exports.runManifestHash = runManifestHash; exports.runIdFromManifest = runIdFromManifest; exports.createRunManifest = createRunManifest; exports.verifyJournal = verifyJournal; exports.verifiedEloState = verifiedEloState; exports.buildSnapshot = buildSnapshot; exports.renderMarkdown = renderMarkdown; exports.writeReports = writeReports; exports.scheduleMatches = scheduleMatches; exports.FAST_RESPONSE_MS = FAST_RESPONSE_MS; exports.LOW_OUTPUT_TOKENS = LOW_OUTPUT_TOKENS; exports.classifyError = classifyError; exports.detectResponseAnomalies = detectResponseAnomalies; exports.triageJournal = triageJournal; exports.formatTriage = formatTriage; exports.ArenaRunner = ArenaRunner; exports.MockOpenRouterGateway = MockOpenRouterGateway; exports.categoryStoreConfig = categoryStoreConfig; exports.ArenaStore = ArenaStore;
+//# sourceMappingURL=chunk-NJTYVNP4.cjs.map