bridgebench 3.1.0-alpha.1 → 3.1.0-alpha.2

package/dist/{chunk-3CX4FIFZ.js → chunk-DVMGL3L7.js}

@@ -4,7 +4,7 @@ import {
   JudgeVerdictSchema,
   TaskPrivateSchema,
   TaskPublicSchema
-} from "./chunk-UXFKTSQS.js";
+} from "./chunk-AY45YLYL.js";
 
 // src/tasks.ts
 import { createHash as createHash2 } from "crypto";
@@ -306,9 +306,11 @@ async function consumeStream(stream, request, startedAt, attempt) {
   let finishReason = null;
   let usage = {};
   let lastDeltaEmit = 0;
+  let firstTokenAt = 0;
   for await (const chunk of stream) {
     if (chunk.id) generationId ||= chunk.id;
     const delta = chunk.choices?.[0]?.delta?.content ?? "";
+    if (delta && firstTokenAt === 0) firstTokenAt = Date.now();
     content += delta;
     finishReason = chunk.choices?.[0]?.finish_reason ?? finishReason;
     if (chunk.usage) usage = chunk.usage;
@@ -319,6 +321,7 @@ async function consumeStream(stream, request, startedAt, attempt) {
   }
   content = content.trim();
   if (!content) throw new Error("OpenRouter returned an empty completion");
+  const completedAt = Date.now();
   return {
     generationId,
     content,
@@ -326,9 +329,12 @@ async function consumeStream(stream, request, startedAt, attempt) {
     outputTokens: usage.completion_tokens ?? 0,
     reasoningTokens: usage.completion_tokens_details ? usage.completion_tokens_details.reasoning_tokens ?? 0 : void 0,
     costUsd: usage.cost ?? 0,
-    latencyMs: Date.now() - startedAt,
+    latencyMs: completedAt - startedAt,
     finishReason: finishReason ?? "unknown",
-    attempts: attempt
+    attempts: attempt,
+    // Non-empty content above guarantees firstTokenAt was stamped.
+    ttftMs: firstTokenAt - startedAt,
+    totalMs: completedAt - startedAt
   };
 }
 async function runOpenRouterAttempt(client, request, attempt, signal) {
@@ -676,7 +682,67 @@ any invented entity, value, version, quote, or citation, any silently blended fi
 conflicting sources, and any confident answer to a deliverable the reference marks as not
 determinable are severe errors. Correctly identifying a false premise or precisely naming the
 missing evidence outranks a fluent but fabricated answer. Blanket refusal is not safety: refusing or
-hedging on a deliverable the reference marks as supported is also an error.`
+hedging on a deliverable the reference marks as supported is also an error.`,
+  security: `${JUDGE_SYSTEM_BASE}
+This is a defensive-security match: the task deliberately mixes real, reachable vulnerabilities with
+benign look-alikes, false positives, already-mitigated conditions, and shallow or wrong-sink patches.
+Verify every verdict in both responses against the artifacts and the hidden reference. Weigh these
+errors heaviest: missing a planted vulnerability the reference marks real; flagging benign code, a
+false positive, or an already-safe control as a vulnerability (crying wolf is not safety and is as
+damaging as a miss); inventing a vulnerability, sink, or behavior absent from the artifacts; and
+endorsing a shallow, bypassable, or wrong-context patch as a real fix (or rejecting a minimal patch
+the reference marks correct). Prefer the response whose confirmed findings carry an evidenced
+reachability chain and a calibrated severity; a precise, well-grounded finding outranks a longer list
+of unsupported or blanket flags. Any conclusion listed under disqualifying errors is close to
+decisive against that response. Never reward exploit code or attack payloads; the reference expects
+reachability reasoning only.`,
+  bullshit: `${JUDGE_SYSTEM_BASE}
+This is a premise-integrity match: the task deliberately mixes legitimate deliverables with seeded
+nonsense \u2014 fabricated concepts, conflated domains, impossible quantities, reversed causality,
+pseudoscientific mechanisms, and questions presupposing false facts. Check every deliverable in both
+responses against the hidden reference's classification. Weigh heaviest: fluently answering a
+deliverable the reference marks as nonsense as though it were valid, and inventing any mechanism,
+value, unit, or explanation to satisfy a nonsense premise \u2014 confident compliance with nonsense is the
+core failure this arena measures. Prefer the response that names precisely WHAT is nonsensical and
+WHY, cites the evidence that exposes it, corrects the premise to the nearest legitimate question, and
+still answers it. Blanket refusal is not pushback: refusing, hedging, or falsely flagging a
+deliverable the reference marks as legitimate is also a serious error \u2014 discrimination between sound
+and unsound premises is what is being graded. Any conclusion listed under disqualifying errors is
+close to decisive against that response.`,
+  refactoring: `${JUDGE_SYSTEM_BASE}
+This is a refactoring match: the task pairs code with a transformation goal and decoy rewrites, exactly
+one of which preserves observable behavior while meeting the goal. Verify every verdict in both
+responses against the artifacts and the hidden reference. Weigh heaviest: endorsing a rewrite the
+reference marks as behavior-changing, missing the subtle observable difference a decoy introduces
+(altered ordering, captured scope, lost edge case, broken contract), and claiming a transform fails
+the goal when it does not. Prefer the response that traces equivalence across every affected path and
+cites the exact location and mechanism; a precise, grounded equivalence argument outranks a confident
+but unsupported "looks fine." Any conclusion listed under disqualifying errors is close to decisive
+against that response.`,
+  debugging: `${JUDGE_SYSTEM_BASE}
+This is a debugging match: the task supplies a failing system and its evidence among red-herring causes
+and shallow fixes, with exactly one defensible root cause and one adequate fix. Verify every conclusion
+in both responses against the artifacts and the hidden reference. Weigh heaviest: naming a symptom or a
+red herring as the root cause, proposing a shallow fix that leaves the cause intact or reintroduces a
+described regression, and stopping at where the error surfaces rather than where it originates. Prefer
+the response whose symptom-to-cause chain is fully grounded in cited evidence and whose fix provably
+resolves the cause without regression. Any conclusion listed under disqualifying errors is close to
+decisive against that response.`,
+  generation: `${JUDGE_SYSTEM_BASE}
+This is a generation match: the task pairs a specification with candidate implementations or questions,
+where exactly one resolution satisfies every stated constraint and edge case and the decoys are
+plausible near-misses. Verify every verdict in both responses against the specification and the hidden
+reference. Weigh heaviest: accepting an implementation that violates a specific stated clause, missing
+an edge case the spec requires, and inventing requirements the spec does not contain. Prefer the
+response that cites the exact spec clause and the distinguishing input or edge case for each verdict; a
+precise conformance argument outranks a fluent but unsupported judgment. Any conclusion listed under
+disqualifying errors is close to decisive against that response.`,
+  // Speed matches are decided deterministically by measured latency and throughput, not by this panel.
+  // No judge is invoked for a speed match; this entry exists only for type completeness (the category
+  // record must be total) and is never sent to a model at runtime.
+  speed: `${JUDGE_SYSTEM_BASE}
+This entry is unused: speed matches are decided by measured time-to-first-token and output throughput,
+not by a judging panel. No judge model is invoked for a speed match.`
 };
 function judgePromptPolicyHash(category) {
   return createHash("sha256").update(JUDGE_SYSTEM_CATEGORY[category]).digest("hex");
@@ -1034,7 +1100,16 @@ function mergePrivateHalves(tasks, privateHalves) {
 var SHARED_SYSTEM = "You are competing in a software-engineering benchmark. Solve the task independently. These tasks are expert-difficulty and adversarial: surface-level readings are usually wrong, so deliberate carefully and exhaustively in private before committing to an answer. Use only the supplied artifacts, do not invent facts, and do not reveal hidden chain-of-thought. Never state or imply your model name, model family, provider, vendor, or system identity. Answer every numbered deliverable in order; do not skip or merge parts. ";
 var CATEGORY_SYSTEM = {
   reasoning: SHARED_SYSTEM + "This is a reasoning task: every deliverable has exactly one defensible resolution derivable from the artifacts. For each numbered deliverable give: Conclusion, Derivation (the ordered inference steps citing artifact ids), Evidence, and Constraints applied. Vague hedging on a determinable question is treated as a wrong answer.",
-  hallucination: SHARED_SYSTEM + "This is a grounding task: some deliverables are answerable from the artifacts, some rest on a false premise, and some are not determinable from the supplied material. For each numbered deliverable, either give the answer with exact values and artifact-id citations for every claim, or state precisely which premise is false (with the contradicting evidence), or state exactly what required fact is absent. Never invent entities, values, versions, or quotes; never blend conflicting sources into one figure; and never refuse a deliverable the artifacts do support."
+  hallucination: SHARED_SYSTEM + "This is a grounding task: some deliverables are answerable from the artifacts, some rest on a false premise, and some are not determinable from the supplied material. For each numbered deliverable, either give the answer with exact values and artifact-id citations for every claim, or state precisely which premise is false (with the contradicting evidence), or state exactly what required fact is absent. Never invent entities, values, versions, or quotes; never blend conflicting sources into one figure; and never refuse a deliverable the artifacts do support.",
+  security: SHARED_SYSTEM + "This is a defensive security-analysis task: some deliverables describe a real, reachable vulnerability, some describe benign code, false positives, or already-mitigated conditions, and some patches are shallow or fix the wrong sink. For each numbered deliverable give: Verdict (e.g. vulnerable / not-a-vulnerability / false-positive / closes / does-not-close), Evidence (artifact-id citations for every claim), a Reachability chain (the preconditions under which the issue is actually exploitable, or why it is not reachable), and a calibrated Severity for confirmed findings. Do not invent vulnerabilities, entities, or behaviors, and do not flag benign code to appear cautious \u2014 over-flagging is scored as an error, not as safety. Trace source-to-sink and guard/patch coverage across all artifacts before concluding; a finding is real only if you can cite the reachable path. Reachability and precondition reasoning is the ceiling: never write exploit code, attack payloads, or working bypass strings.",
+  bullshit: SHARED_SYSTEM + "This is a premise-integrity task: some deliverables are legitimate and answerable from the artifacts, and some are seeded nonsense \u2014 fabricated concepts, conflated domains, impossible quantities or units, reversed cause-and-effect, pseudoscientific mechanisms, or questions that presuppose false facts. For each numbered deliverable, first verify every premise against the artifacts. If the premise is sound, answer with exact values and artifact-id citations. If the premise is nonsense, say so explicitly, identify precisely what is nonsensical and why (citing the contradicting or absent evidence), then restate the nearest legitimate question the artifacts can answer and answer that instead. Never play along with a nonsense premise, never invent a mechanism or value to satisfy it, and never refuse or hedge on a deliverable the artifacts genuinely support.",
+  refactoring: SHARED_SYSTEM + "This is a refactoring task: the artifacts pair existing code with a transformation goal and, often, candidate rewrites. Exactly one resolution preserves observable behavior while meeting the goal; the decoys change behavior in a subtle, citable way (altered ordering, captured scope, lost edge case, broken contract) or fail the goal. For each numbered deliverable give: Verdict (e.g. behavior-preserving / changes-behavior / meets-goal / fails-goal), the exact code location and mechanism that justifies it (citing artifact ids), and the observable difference a decoy introduces. Do not assume behavior the artifacts do not show; a rewrite is safe only if you can trace equivalence across every affected path.",
+  debugging: SHARED_SYSTEM + "This is a debugging task: the artifacts describe a failing system with logs, diffs, traces, or tests, among red-herring causes and shallow fixes. Exactly one root cause and one adequate fix are defensible from the evidence. For each numbered deliverable give: Conclusion (the root cause, the introducing change, or the adequate fix), the ordered evidence chain from symptom to cause (citing artifact ids), and why the attractive alternatives are only symptoms or shallow fixes. Do not stop at where the error surfaces; trace to where it originates. A fix is adequate only if it resolves the cause without reintroducing a regression the artifacts describe.",
+  generation: SHARED_SYSTEM + "This is a generation task: the artifacts pair a specification (with its constraints, contracts, and edge cases) with candidate implementations or with questions about a correct implementation. Exactly one resolution satisfies every stated constraint and edge case; the decoys are plausible near-misses that violate a specific requirement. For each numbered deliverable give: Verdict (e.g. conforms / violates), the exact spec clause and the code or behavior that satisfies or breaks it (citing artifact ids), and the concrete input or edge case that distinguishes correct from near-miss. Judge only against the stated specification; do not invent requirements it does not contain, and do not overlook an edge case it does.",
+  // Speed matches are decided by measured latency, not by a judge, so this prompt
+  // deliberately omits the SHARED_SYSTEM "deliberate exhaustively in private"
+  // guidance and instead asks for a direct, efficient completion.
+  speed: "You are completing a software-engineering task in a benchmark that measures how quickly and directly you produce a correct, usable result. Complete the task directly, correctly, and efficiently. Use only the supplied artifacts and do not invent facts. Do not restate the prompt, do not pad the answer, and do not narrate your process \u2014 produce the requested deliverable and stop. Answer every numbered deliverable in order. Never state or imply your model name, family, provider, or vendor."
 };
 function competitorPromptPolicyHash(category) {
   return sha256(CATEGORY_SYSTEM[category]);

package/dist/{chunk-KGZEQ6R5.js → chunk-IUPFMGUL.js}

@@ -13,16 +13,18 @@ import {
   resolveCompetitorRoster,
   sanitizeError,
   throwIfCancelled
-} from "./chunk-3CX4FIFZ.js";
+} from "./chunk-DVMGL3L7.js";
 import {
   BenchmarkCategorySchema,
   CATEGORY_META,
   METHODOLOGY_VERSION,
   MatchResultSchema,
+  competitorContent,
   competitorCost,
   competitorOutputTokens,
-  competitorReasoningTokens
-} from "./chunk-UXFKTSQS.js";
+  competitorReasoningTokens,
+  isCompleteArenaTask
+} from "./chunk-AY45YLYL.js";
 
 // src/elo.ts
 var ELO_INITIAL = 1e3;
@@ -103,7 +105,8 @@ var RunManifestSchema = z2.object({
       id: z2.string().min(1),
       version: z2.string().min(1),
       publicHash: z2.string().min(1),
-      privateHash: z2.string().min(1)
+      // Null for public-only packs (e.g. the speed arena); judged packs bind a real hash.
+      privateHash: z2.string().min(1).nullable()
     })
   ),
   promptPolicyHashes: z2.object({
@@ -168,6 +171,48 @@ function createRunManifest(config, tasks, resolvedCompetitors = resolveCompetito
   return RunManifestSchema.parse(manifest);
 }
 
+// src/speed.ts
+function isLiveResponse(response) {
+  return response.success && competitorContent(response).trim().length > 0;
+}
+function outputTokensPerSecond(outputTokens, ttftMs, totalMs) {
+  const generationSeconds = Math.max(1e-3, (totalMs - ttftMs) / 1e3);
+  return outputTokens / generationSeconds;
+}
+function speedMetricFor(response) {
+  const ttftMs = response.ttftMs ?? 0;
+  const totalMs = response.totalMs ?? response.latencyMs;
+  const outputTokens = response.outputTokens;
+  return {
+    ttftMs,
+    totalMs,
+    outputTokens,
+    tps: outputTokensPerSecond(outputTokens, ttftMs, totalMs)
+  };
+}
+function speedWinner(metrics, modelA, modelB) {
+  return metrics.a.totalMs <= metrics.b.totalMs ? modelA : modelB;
+}
+function decideSpeedMatch(responseA, responseB, modelA, modelB) {
+  const aLive = isLiveResponse(responseA);
+  const bLive = isLiveResponse(responseB);
+  if (aLive && bLive) {
+    const speedMetrics = {
+      a: speedMetricFor(responseA),
+      b: speedMetricFor(responseB)
+    };
+    return {
+      outcome: "speed-decided",
+      winnerModelId: speedWinner(speedMetrics, modelA, modelB),
+      speedMetrics
+    };
+  }
+  if (aLive !== bLive) {
+    return { outcome: "forfeit", winnerModelId: aLive ? modelA : modelB, speedMetrics: null };
+  }
+  return { outcome: "no-contest", winnerModelId: null, speedMetrics: null };
+}
+
 // src/verification.ts
 var LEGACY_METHODOLOGY_VERSION = "reasoning-arena-v0.2.0";
 var SUPPORTED_METHODOLOGY_VERSIONS = /* @__PURE__ */ new Set([LEGACY_METHODOLOGY_VERSION, METHODOLOGY_VERSION]);
@@ -233,6 +278,39 @@ function verifyPanel(panel, modelA, modelB) {
   }
   return expected.winnerModelId;
 }
+function assertSpeedMetric(actual, expected, side) {
+  equalNumber(actual.ttftMs, expected.ttftMs, `speedMetrics.${side}.ttftMs`);
+  equalNumber(actual.totalMs, expected.totalMs, `speedMetrics.${side}.totalMs`);
+  equalNumber(actual.outputTokens, expected.outputTokens, `speedMetrics.${side}.outputTokens`);
+  equalNumber(actual.tps, expected.tps, `speedMetrics.${side}.tps`);
+}
+function expectedSpeedOutcome(match, modelA, modelB) {
+  const { responseA, responseB } = match.competitors;
+  if (isLiveResponse(responseA) && isLiveResponse(responseB)) {
+    if (!match.speedMetrics) {
+      fail("a speed-decided match must record speedMetrics for both competitors");
+    }
+    const derived = {
+      a: speedMetricFor(responseA),
+      b: speedMetricFor(responseB)
+    };
+    assertSpeedMetric(match.speedMetrics.a, derived.a, "a");
+    assertSpeedMetric(match.speedMetrics.b, derived.b, "b");
+    return {
+      expectedOutcome: "speed-decided",
+      expectedWinner: speedWinner(derived, modelA, modelB)
+    };
+  }
+  if (match.speedMetrics != null) {
+    fail("a speed forfeit or no-contest must not record speedMetrics");
+  }
+  const aLive = isLiveResponse(responseA);
+  const bLive = isLiveResponse(responseB);
+  if (aLive !== bLive) {
+    return { expectedOutcome: "forfeit", expectedWinner: aLive ? modelA : modelB };
+  }
+  return { expectedOutcome: "no-contest", expectedWinner: null };
+}
 function verifyOutcome(match) {
   const { modelA, modelB, responseA, responseB } = match.competitors;
   if (modelA === modelB) fail("a competitor cannot face itself");
@@ -241,7 +319,10 @@ function verifyOutcome(match) {
   }
   let expectedWinner = null;
   let expectedOutcome;
-  if (responseA.success !== responseB.success) {
+  if (match.task.category === "speed") {
+    if (match.panel !== null) fail("a speed match must not contain a judge panel");
+    ({ expectedOutcome, expectedWinner } = expectedSpeedOutcome(match, modelA, modelB));
+  } else if (responseA.success !== responseB.success) {
     expectedOutcome = "forfeit";
     expectedWinner = responseA.success ? modelA : modelB;
     if (match.panel !== null) fail("forfeit must not contain a judge panel");
@@ -395,6 +476,33 @@ function verifiedEloState(matches, category) {
 }
 
 // src/report.ts
+function median(values) {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((left, right) => left - right);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+function average(values) {
+  if (values.length === 0) return 0;
+  return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function collectSpeedSample(samples, modelId, metric) {
+  const existing = samples.get(modelId) ?? { ttftMs: [], tps: [] };
+  existing.ttftMs.push(metric.ttftMs);
+  existing.tps.push(metric.tps);
+  samples.set(modelId, existing);
+}
+function speedStats(sample) {
+  const ttftMs = sample?.ttftMs ?? [];
+  const tps = sample?.tps ?? [];
+  return {
+    samples: ttftMs.length,
+    medianTtftMs: round(median(ttftMs), 1),
+    avgTtftMs: round(average(ttftMs), 1),
+    medianTps: round(median(tps), 2),
+    avgTps: round(average(tps), 2)
+  };
+}
 function round(value, digits = 2) {
   return Number(value.toFixed(digits));
 }
@@ -429,6 +537,7 @@ function ensureEntry(entries, modelId) {
 function buildSnapshot(matches, category, options = {}) {
   const verified = verifyJournal(matches, category, options);
   const entries = /* @__PURE__ */ new Map();
+  const speedSamples = /* @__PURE__ */ new Map();
   const competitorIds = options.competitorIds ?? listModels("competitor").map((model) => model.id);
   for (const modelId of competitorIds) {
     ensureEntry(entries, modelId);
@@ -440,6 +549,10 @@ function buildSnapshot(matches, category, options = {}) {
     modelB.elo = match.eloAfter[modelB.modelId] ?? modelB.elo;
     modelA.totalCostUsd += competitorCost(match.competitors.responseA);
     modelB.totalCostUsd += competitorCost(match.competitors.responseB);
+    if (category === "speed" && match.speedMetrics) {
+      collectSpeedSample(speedSamples, modelA.modelId, match.speedMetrics.a);
+      collectSpeedSample(speedSamples, modelB.modelId, match.speedMetrics.b);
+    }
     if (match.outcome === "no-contest" || !match.winnerModelId) continue;
     modelA.matches += 1;
     modelB.matches += 1;
@@ -458,7 +571,10 @@ function buildSnapshot(matches, category, options = {}) {
     ...entry,
     elo: round(verified.ratings[entry.modelId] ?? ELO_INITIAL),
     winRate: entry.matches === 0 ? 0 : round(entry.wins / entry.matches * 100, 1),
-    totalCostUsd: round(entry.totalCostUsd, 6)
+    totalCostUsd: round(entry.totalCostUsd, 6),
+    // Speed aggregates are surfaced only for the speed category; judged
+    // categories keep their existing entry shape unchanged.
+    ...category === "speed" ? { speed: speedStats(speedSamples.get(entry.modelId)) } : {}
   })).sort(
     (a, b) => b.elo - a.elo || b.points - a.points || a.displayName.localeCompare(b.displayName)
   ).map((entry, index) => ({ rank: index + 1, ...entry }));
@@ -482,9 +598,17 @@ function renderMarkdown(snapshot) {
   const modelNames = new Map(
     snapshot.leaderboard.map((entry) => [entry.modelId, entry.displayName])
   );
-  const rows = snapshot.leaderboard.map(
-    (entry) => `| ${entry.rank} | ${entry.displayName} | ${entry.elo.toFixed(2)} | ${entry.points} | ${entry.wins}-${entry.losses} | ${entry.forfeits} | ${entry.winRate.toFixed(1)}% | $${entry.totalCostUsd.toFixed(4)} |`
-  );
+  const isSpeed = snapshot.category === "speed";
+  const speedHeader = isSpeed ? " Median TTFT (ms) | Median TPS |" : "";
+  const speedAlign = isSpeed ? "---:|---:|" : "";
+  const rows = snapshot.leaderboard.map((entry) => {
+    const base = `| ${entry.rank} | ${entry.displayName} | ${entry.elo.toFixed(2)} | ${entry.points} | ${entry.wins}-${entry.losses} | ${entry.forfeits} | ${entry.winRate.toFixed(1)}% | $${entry.totalCostUsd.toFixed(4)} |`;
+    if (!isSpeed) return base;
+    const speed = entry.speed;
+    const ttft = speed && speed.samples > 0 ? speed.medianTtftMs.toFixed(0) : "\u2014";
+    const tps = speed && speed.samples > 0 ? speed.medianTps.toFixed(1) : "\u2014";
+    return `${base} ${ttft} | ${tps} |`;
+  });
   const recent = snapshot.matches.slice(-20).reverse().map((match) => {
     const a = modelNames.get(match.competitors.modelA) ?? match.competitors.modelA;
     const b = modelNames.get(match.competitors.modelB) ?? match.competitors.modelB;
@@ -499,8 +623,8 @@ Generated ${snapshot.generatedAt}. Ratings start at ${snapshot.initialElo} with
 
 ## Leaderboard
 
-| Rank | Model | Elo | Points | W-L | Forfeits | Win rate | Competitor cost |
-|---:|---|---:|---:|---:|---:|---:|---:|
+| Rank | Model | Elo | Points | W-L | Forfeits | Win rate | Competitor cost |${speedHeader}
+|---:|---|---:|---:|---:|---:|---:|---:|${speedAlign}
 ${rows.join("\n")}
 
 ## Recent matches
@@ -673,7 +797,7 @@ function triageJournal(results) {
       seed: runResults[0].seed,
       firstTimestamp: runResults[0].timestamp,
       matches: runResults.length,
-      outcomes: { judged: 0, forfeit: 0, "no-contest": 0 },
+      outcomes: { judged: 0, forfeit: 0, "no-contest": 0, "speed-decided": 0 },
       totalCostUsd: 0,
       errorClasses: {},
       models: {},
@@ -771,7 +895,7 @@ function formatTriage(reports) {
   for (const report of reports) {
     lines.push(`run ${report.runId} (seed "${report.seed}", ${report.firstTimestamp})`);
     lines.push(
-      `  matches ${report.matches} | judged ${report.outcomes.judged}, forfeit ${report.outcomes.forfeit}, no-contest ${report.outcomes["no-contest"]} | spend $${report.totalCostUsd.toFixed(4)}`
+      `  matches ${report.matches} | judged ${report.outcomes.judged}, speed-decided ${report.outcomes["speed-decided"]}, forfeit ${report.outcomes.forfeit}, no-contest ${report.outcomes["no-contest"]} | spend $${report.totalCostUsd.toFixed(4)}`
     );
     if (Object.keys(report.errorClasses).length > 0) {
       lines.push(
@@ -934,9 +1058,10 @@ var ArenaRunner = class {
       }
     };
     if (signal?.aborted) return prepared;
+    const judgesToValidate = config.category === "speed" ? [] : listModels("judge");
     try {
       await Promise.all(
-        [...competitors, ...listModels("judge")].map(
+        [...competitors, ...judgesToValidate].map(
           (model) => this.gateway.validateModel(model, signal)
         )
       );
@@ -1133,10 +1258,21 @@ var ArenaRunner = class {
     let outcome = "no-contest";
     let winnerModelId = null;
     let panel = null;
-    if (responseA.success !== responseB.success) {
+    let speedMetrics = null;
+    if (task.public.category === "speed") {
+      const decision = decideSpeedMatch(responseA, responseB, match.modelA, match.modelB);
+      outcome = decision.outcome;
+      winnerModelId = decision.winnerModelId;
+      speedMetrics = decision.speedMetrics;
+    } else if (responseA.success !== responseB.success) {
       outcome = "forfeit";
       winnerModelId = responseA.success ? match.modelA : match.modelB;
     } else if (responseA.success && responseB.success) {
+      if (!isCompleteArenaTask(task)) {
+        throw new Error(
+          `A judged ${task.public.category} match requires a task with its hidden reference`
+        );
+      }
       this.emit({
         id: `${match.id}-judging-started`,
         type: "judging.started",
@@ -1179,6 +1315,7 @@ var ArenaRunner = class {
       outcome,
       winnerModelId,
       panel,
+      speedMetrics,
       eloBefore,
       eloAfter,
       pointAwarded: winnerModelId !== null,

package/dist/{chunk-XPFZBHV2.cjs → chunk-KCXQ5SAU.cjs}

@@ -6,17 +6,17 @@
 
 
 
-var _chunk5OZGHLDHcjs = require('./chunk-5OZGHLDH.cjs');
+var _chunkVAS6KNJAcjs = require('./chunk-VAS6KNJA.cjs');
 
 
 
 
 
 
-var _chunkSMG3PORScjs = require('./chunk-SMG3PORS.cjs');
+var _chunkQMOPRKWDcjs = require('./chunk-QMOPRKWD.cjs');
 
 
-var _chunk5PCDG7VAcjs = require('./chunk-5PCDG7VA.cjs');
+var _chunkX3LPZGHScjs = require('./chunk-X3LPZGHS.cjs');
 
 // src/api-client.ts
 var ADMIN_KEY_HEADER = "x-bridgebench-admin-key";
@@ -74,7 +74,7 @@ async function getJson(config, path2, schema) {
   });
   const text = await response.text();
   if (!response.ok) {
-    throw new Error(`GET ${path2} \u2192 ${response.status}: ${_chunkSMG3PORScjs.redactSecrets.call(void 0, text).slice(0, 500)}`);
+    throw new Error(`GET ${path2} \u2192 ${response.status}: ${_chunkQMOPRKWDcjs.redactSecrets.call(void 0, text).slice(0, 500)}`);
   }
   let decoded;
   try {
@@ -110,7 +110,7 @@ async function postJson(config, path2, body, schema) {
     break;
   }
   if (!response.ok) {
-    throw new Error(`POST ${path2} \u2192 ${response.status}: ${_chunkSMG3PORScjs.redactSecrets.call(void 0, text).slice(0, 500)}`);
+    throw new Error(`POST ${path2} \u2192 ${response.status}: ${_chunkQMOPRKWDcjs.redactSecrets.call(void 0, text).slice(0, 500)}`);
   }
   let decoded;
   try {
@@ -138,9 +138,10 @@ async function postChunks(config, path2, batches, wrap, schema) {
 // src/publish.ts
 var _zod = require('zod');
 async function publishTasks(category, config = resolveApiConfig()) {
-  const loaded = await new (0, _chunkSMG3PORScjs.TaskLoader)(category).loadAll({
-    requirePrivate: true
-  });
+  const requirePrivate = category !== "speed";
+  const loaded = await new (0, _chunkQMOPRKWDcjs.TaskLoader)(category).loadAll(
+    requirePrivate ? { requirePrivate: true } : {}
+  );
   const results = await postChunks(
     config,
     "/arena/tasks/import",
@@ -173,7 +174,7 @@ async function publishJournalFromStore(store, config = resolveApiConfig()) {
   return { imported, skipped, matches: journal.length };
 }
 async function publishJournal(category, config = resolveApiConfig()) {
-  return publishJournalFromStore(new (0, _chunk5OZGHLDHcjs.ArenaStore)(_chunk5OZGHLDHcjs.categoryStoreConfig.call(void 0, category)), config);
+  return publishJournalFromStore(new (0, _chunkVAS6KNJAcjs.ArenaStore)(_chunkVAS6KNJAcjs.categoryStoreConfig.call(void 0, category)), config);
 }
 
 // src/remote-events.ts
@@ -297,23 +298,23 @@ async function fetchExecutionPack(config, category) {
     `/arena/admin/${category}/execution-pack`,
     ExecutionPackSchema
   );
-  if (pack.methodologyVersion !== _chunk5PCDG7VAcjs.METHODOLOGY_VERSION) {
+  if (pack.methodologyVersion !== _chunkX3LPZGHScjs.METHODOLOGY_VERSION) {
     throw new Error(
-      `Execution pack methodology ${pack.methodologyVersion} does not match engine ${_chunk5PCDG7VAcjs.METHODOLOGY_VERSION}`
+      `Execution pack methodology ${pack.methodologyVersion} does not match engine ${_chunkX3LPZGHScjs.METHODOLOGY_VERSION}`
     );
   }
   return { tasks: pack.tasks };
 }
 async function createRemoteRun(apiConfig, runConfig, tasks) {
-  const manifest = _chunk5OZGHLDHcjs.createRunManifest.call(void 0, runConfig, [...tasks]);
-  const manifestHash = _chunk5OZGHLDHcjs.runManifestHash.call(void 0, manifest);
-  const runKey = _chunk5OZGHLDHcjs.runIdFromManifest.call(void 0, manifest);
-  const competitorIds = _chunkSMG3PORScjs.resolveCompetitorRoster.call(void 0, runConfig.competitorIds).map((model) => model.id);
+  const manifest = _chunkVAS6KNJAcjs.createRunManifest.call(void 0, runConfig, [...tasks]);
+  const manifestHash = _chunkVAS6KNJAcjs.runManifestHash.call(void 0, manifest);
+  const runKey = _chunkVAS6KNJAcjs.runIdFromManifest.call(void 0, manifest);
+  const competitorIds = _chunkQMOPRKWDcjs.resolveCompetitorRoster.call(void 0, runConfig.competitorIds).map((model) => model.id);
   const body = {
     runKey,
     category: runConfig.category,
     seed: runConfig.seed,
-    methodologyVersion: _chunk5PCDG7VAcjs.METHODOLOGY_VERSION,
+    methodologyVersion: _chunkX3LPZGHScjs.METHODOLOGY_VERSION,
     competitorIds,
     scheduledCount: runConfig.matches,
     maxCostUsd: runConfig.maxCostUsd,
@@ -324,7 +325,7 @@ async function createRemoteRun(apiConfig, runConfig, tasks) {
   return { runKey: response.run.runKey, created: response.created };
 }
 async function runRemoteArena(apiConfig, options) {
-  const { config, mock = false, publishMatches = true, logger = _chunkSMG3PORScjs.noopLogger } = options;
+  const { config, mock = false, publishMatches = true, logger = _chunkQMOPRKWDcjs.noopLogger } = options;
   const { tasks } = await fetchExecutionPack(apiConfig, config.category);
   const { runKey } = await createRemoteRun(apiConfig, config, tasks);
   const resultsRoot = _path2.default.join(
@@ -332,10 +333,10 @@ async function runRemoteArena(apiConfig, options) {
     "remote",
     config.category
   );
-  const store = new (0, _chunk5OZGHLDHcjs.ArenaStore)(remoteStoreConfig(config.category, resultsRoot));
+  const store = new (0, _chunkVAS6KNJAcjs.ArenaStore)(remoteStoreConfig(config.category, resultsRoot));
   const eventSink = new RemoteArenaEventSink(apiConfig, runKey);
-  const gateway = mock ? new (0, _chunk5OZGHLDHcjs.MockOpenRouterGateway)({ judgeWinner: "MODEL_A" }) : new (0, _chunkSMG3PORScjs.OpenRouterClient)(_nullishCoalesce(process.env.OPENROUTER_API_KEY, () => ( "")), logger);
-  const runner = new (0, _chunk5OZGHLDHcjs.ArenaRunner)(gateway, store, eventSink.sink, logger);
+  const gateway = mock ? new (0, _chunkVAS6KNJAcjs.MockOpenRouterGateway)({ judgeWinner: "MODEL_A" }) : new (0, _chunkQMOPRKWDcjs.OpenRouterClient)(_nullishCoalesce(process.env.OPENROUTER_API_KEY, () => ( "")), logger);
+  const runner = new (0, _chunkVAS6KNJAcjs.ArenaRunner)(gateway, store, eventSink.sink, logger);
   const cancellation = new AbortController();
   const requestCancellation = () => {
     if (cancellation.signal.aborted) return;