bridgebench 3.1.0-alpha.1 → 3.1.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +141 -173
  2. package/dist/{chunk-UXFKTSQS.js → chunk-AY45YLYL.js} +105 -4
  3. package/dist/{chunk-MBC4C3VA.js → chunk-CJGHBY54.js} +7 -6
  4. package/dist/{chunk-3CX4FIFZ.js → chunk-DVMGL3L7.js} +80 -5
  5. package/dist/{chunk-KGZEQ6R5.js → chunk-IUPFMGUL.js} +152 -15
  6. package/dist/{chunk-XPFZBHV2.cjs → chunk-KCXQ5SAU.cjs} +21 -20
  7. package/dist/{chunk-SMG3PORS.cjs → chunk-QMOPRKWD.cjs} +89 -14
  8. package/dist/{chunk-5OZGHLDH.cjs → chunk-VAS6KNJA.cjs} +216 -79
  9. package/dist/{chunk-5PCDG7VA.cjs → chunk-X3LPZGHS.cjs} +106 -5
  10. package/dist/cli.cjs +51 -48
  11. package/dist/cli.js +9 -6
  12. package/dist/client.cjs +5 -5
  13. package/dist/client.d.cts +3 -3
  14. package/dist/client.d.ts +3 -3
  15. package/dist/client.js +4 -4
  16. package/dist/contracts/index.cjs +6 -2
  17. package/dist/contracts/index.d.cts +2 -2
  18. package/dist/contracts/index.d.ts +2 -2
  19. package/dist/contracts/index.js +7 -3
  20. package/dist/index.cjs +8 -4
  21. package/dist/index.d.cts +9 -8
  22. package/dist/index.d.ts +9 -8
  23. package/dist/index.js +8 -4
  24. package/dist/{logger-QJU7SBDz.d.ts → logger-BByta-7V.d.cts} +23 -23
  25. package/dist/{logger-CCR9Mg1c.d.cts → logger-BQf29BLe.d.ts} +23 -23
  26. package/dist/{reports-4CejmOHf.d.cts → reports-B8TCJtPr.d.ts} +57 -13
  27. package/dist/{reports-s2CTnGN8.d.ts → reports-DPpOoOux.d.cts} +57 -13
  28. package/dist/{tasks-CpaCJ6JE.d.cts → tasks-BmhWuMBD.d.cts} +24 -22
  29. package/dist/{tasks-CpaCJ6JE.d.ts → tasks-BmhWuMBD.d.ts} +24 -22
  30. package/dist/tasks.cjs +3 -3
  31. package/dist/tasks.d.cts +1 -1
  32. package/dist/tasks.d.ts +1 -1
  33. package/dist/tasks.js +2 -2
  34. package/docs/README.md +32 -12
  35. package/docs/methodology.md +14 -0
  36. package/docs/operator-guide.md +178 -0
  37. package/docs/replay-elo.md +12 -2
  38. package/docs/reviewing-bridgebench.md +173 -0
  39. package/docs/task-authoring.md +79 -1
  40. package/package.json +5 -3
  41. package/tasks/bullshit/public/crossed-metric-properties.yaml +192 -0
  42. package/tasks/bullshit/public/crossed-release-checksums.yaml +199 -0
  43. package/tasks/bullshit/public/fabricated-config-keys.yaml +243 -0
  44. package/tasks/bullshit/public/fabricated-protocol-features.yaml +272 -0
  45. package/tasks/bullshit/public/impossible-capacity-math.yaml +197 -0
  46. package/tasks/bullshit/public/impossible-latency-allocation.yaml +215 -0
  47. package/tasks/bullshit/public/loaded-approval-bypass.yaml +229 -0
  48. package/tasks/bullshit/public/loaded-migration-fallout.yaml +221 -0
  49. package/tasks/bullshit/public/pseudo-gc-heap-tuning.yaml +206 -0
  50. package/tasks/bullshit/public/pseudo-network-tuning.yaml +204 -0
  51. package/tasks/bullshit/public/reversed-alert-cascade.yaml +251 -0
  52. package/tasks/bullshit/public/reversed-dependency-failure.yaml +268 -0
  53. package/tasks/debugging/public/deadlock-lock-order-inversion.yaml +131 -0
  54. package/tasks/debugging/public/error-propagation-config-swallow.yaml +174 -0
  55. package/tasks/debugging/public/error-propagation-retry-mask.yaml +167 -0
  56. package/tasks/debugging/public/fix-adequacy-cursor-pagination.yaml +166 -0
  57. package/tasks/debugging/public/fix-adequacy-idempotency-race.yaml +169 -0
  58. package/tasks/debugging/public/keepalive-502-connection-reuse.yaml +162 -0
  59. package/tasks/debugging/public/pool-exhaustion-held-connection.yaml +142 -0
  60. package/tasks/debugging/public/race-oversell-reserve-counter.yaml +133 -0
  61. package/tasks/debugging/public/regression-multipart-filesize-cap.yaml +135 -0
  62. package/tasks/debugging/public/regression-pagination-tiebreak.yaml +112 -0
  63. package/tasks/debugging/public/state-corruption-index-ghost.yaml +160 -0
  64. package/tasks/debugging/public/state-corruption-ledger-balance.yaml +170 -0
  65. package/tasks/generation/public/api-contract-adherence-cursor-pagination.yaml +257 -0
  66. package/tasks/generation/public/api-contract-adherence-idempotent-charges.yaml +261 -0
  67. package/tasks/generation/public/array-rotate-left-normalization.yaml +166 -0
  68. package/tasks/generation/public/cache-interface-dropin.yaml +178 -0
  69. package/tasks/generation/public/edge-case-coverage-cache-loader.yaml +264 -0
  70. package/tasks/generation/public/edge-case-coverage-ledger-tally.yaml +231 -0
  71. package/tasks/generation/public/event-envelope-wire-compat.yaml +149 -0
  72. package/tasks/generation/public/kadane-linear-constant-space.yaml +175 -0
  73. package/tasks/generation/public/lower-bound-insertion-point.yaml +176 -0
  74. package/tasks/generation/public/rolling-checksum-single-pass-pure.yaml +186 -0
  75. package/tasks/generation/public/spec-conformance-password-policy.yaml +190 -0
  76. package/tasks/generation/public/spec-conformance-slug-normalizer.yaml +177 -0
  77. package/tasks/refactoring/public/api-migration-http-retry-client.yaml +208 -0
  78. package/tasks/refactoring/public/api-migration-orm-query-builder.yaml +187 -0
  79. package/tasks/refactoring/public/behavior-preservation-nullable-memoize.yaml +136 -0
  80. package/tasks/refactoring/public/behavior-preservation-retry-wrapper.yaml +187 -0
  81. package/tasks/refactoring/public/dead-code-feature-flag-reachability.yaml +162 -0
  82. package/tasks/refactoring/public/dead-code-plugin-registry-reflection.yaml +125 -0
  83. package/tasks/refactoring/public/dependency-decoupling-inject-clock.yaml +237 -0
  84. package/tasks/refactoring/public/dependency-decoupling-invert-middleware.yaml +177 -0
  85. package/tasks/refactoring/public/extract-and-inline-closure-capture.yaml +132 -0
  86. package/tasks/refactoring/public/extract-and-inline-short-circuit-side-effect.yaml +120 -0
  87. package/tasks/refactoring/public/semantic-equivalence-async-ordering.yaml +157 -0
  88. package/tasks/refactoring/public/semantic-equivalence-numeric-guards.yaml +115 -0
  89. package/tasks/security/public/authz-guard-chain-exposure.yaml +224 -0
  90. package/tasks/security/public/authz-object-scope-idor.yaml +228 -0
  91. package/tasks/security/public/patch-mass-assignment-privesc.yaml +226 -0
  92. package/tasks/security/public/patch-sqli-candidate-fixes.yaml +243 -0
  93. package/tasks/security/public/supply-lockfile-tamper-trace.yaml +227 -0
  94. package/tasks/security/public/supply-secrets-leak-forensics.yaml +205 -0
  95. package/tasks/security/public/taint-export-template-render.yaml +207 -0
  96. package/tasks/security/public/taint-webhook-outbound-fetch.yaml +214 -0
  97. package/tasks/security/public/triage-dependency-advisories.yaml +189 -0
  98. package/tasks/security/public/triage-sast-false-positives.yaml +257 -0
  99. package/tasks/security/public/vuln-path-sanitizer-escape.yaml +165 -0
  100. package/tasks/security/public/vuln-token-alg-confusion.yaml +252 -0
  101. package/tasks/speed/public/speed-api-diff-changelog.yaml +42 -0
  102. package/tasks/speed/public/speed-callbacks-to-async.yaml +39 -0
  103. package/tasks/speed/public/speed-class-to-hooks.yaml +47 -0
  104. package/tasks/speed/public/speed-config-schema-json.yaml +34 -0
  105. package/tasks/speed/public/speed-csv-report-generator.yaml +38 -0
  106. package/tasks/speed/public/speed-debounce-hook.yaml +28 -0
  107. package/tasks/speed/public/speed-log-triage-summary.yaml +35 -0
  108. package/tasks/speed/public/speed-openapi-fragment.yaml +38 -0
  109. package/tasks/speed/public/speed-pagination-cursor.yaml +40 -0
  110. package/tasks/speed/public/speed-rate-limiter-design.yaml +43 -0
  111. package/tasks/speed/public/speed-rest-client-module.yaml +42 -0
  112. package/tasks/speed/public/speed-slugify-helper.yaml +30 -0
@@ -1,4 +1,4 @@
1
- import { B as BenchmarkCategory } from './tasks-CpaCJ6JE.cjs';
1
+ import { B as BenchmarkCategory } from './tasks-BmhWuMBD.js';
2
2
  import { z } from 'zod';
3
3
 
4
4
  interface ResponseSummary {
@@ -57,7 +57,7 @@ interface ArenaEventDataMap {
57
57
  matchId: string;
58
58
  taskId: string;
59
59
  winnerModelId: string | null;
60
- outcome: 'judged' | 'forfeit' | 'no-contest';
60
+ outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
61
61
  costUsd: number;
62
62
  eloAfter: Record<string, number>;
63
63
  completed: number;
@@ -133,9 +133,13 @@ declare const ModelCompletionSchema: z.ZodObject<{
133
133
  latencyMs: z.ZodNumber;
134
134
  finishReason: z.ZodString;
135
135
  attempts: z.ZodOptional<z.ZodNumber>;
136
+ /** Time from request start to the first non-empty content delta (speed arena metric). */
137
+ ttftMs: z.ZodOptional<z.ZodNumber>;
138
+ /** Request start to stream completion for the successful attempt (speed arena metric). */
139
+ totalMs: z.ZodOptional<z.ZodNumber>;
136
140
  }, "strip", z.ZodTypeAny, {
137
- generationId: string;
138
141
  content: string;
142
+ generationId: string;
139
143
  inputTokens: number;
140
144
  outputTokens: number;
141
145
  costUsd: number;
@@ -143,9 +147,11 @@ declare const ModelCompletionSchema: z.ZodObject<{
143
147
  finishReason: string;
144
148
  reasoningTokens?: number | undefined;
145
149
  attempts?: number | undefined;
150
+ ttftMs?: number | undefined;
151
+ totalMs?: number | undefined;
146
152
  }, {
147
- generationId: string;
148
153
  content: string;
154
+ generationId: string;
149
155
  inputTokens: number;
150
156
  outputTokens: number;
151
157
  costUsd: number;
@@ -153,6 +159,8 @@ declare const ModelCompletionSchema: z.ZodObject<{
153
159
  finishReason: string;
154
160
  reasoningTokens?: number | undefined;
155
161
  attempts?: number | undefined;
162
+ ttftMs?: number | undefined;
163
+ totalMs?: number | undefined;
156
164
  }>;
157
165
  type ModelCompletion = z.infer<typeof ModelCompletionSchema>;
158
166
  interface ChatRequest {
@@ -218,12 +226,14 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
218
226
  latencyMs: z.ZodNumber;
219
227
  finishReason: z.ZodString;
220
228
  attempts: z.ZodOptional<z.ZodNumber>;
229
+ ttftMs: z.ZodOptional<z.ZodNumber>;
230
+ totalMs: z.ZodOptional<z.ZodNumber>;
221
231
  } & {
222
232
  modelId: z.ZodString;
223
233
  success: z.ZodLiteral<true>;
224
234
  }, "strip", z.ZodTypeAny, {
225
- generationId: string;
226
235
  content: string;
236
+ generationId: string;
227
237
  inputTokens: number;
228
238
  outputTokens: number;
229
239
  costUsd: number;
@@ -233,9 +243,11 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
233
243
  success: true;
234
244
  reasoningTokens?: number | undefined;
235
245
  attempts?: number | undefined;
246
+ ttftMs?: number | undefined;
247
+ totalMs?: number | undefined;
236
248
  }, {
237
- generationId: string;
238
249
  content: string;
250
+ generationId: string;
239
251
  inputTokens: number;
240
252
  outputTokens: number;
241
253
  costUsd: number;
@@ -245,6 +257,8 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
245
257
  success: true;
246
258
  reasoningTokens?: number | undefined;
247
259
  attempts?: number | undefined;
260
+ ttftMs?: number | undefined;
261
+ totalMs?: number | undefined;
248
262
  }>;
249
263
  type CompetitorSuccess = z.infer<typeof CompetitorSuccessSchema>;
250
264
  declare const CompetitorFailureSchema: z.ZodObject<{
@@ -280,14 +294,14 @@ declare const JudgeVerdictSchema: z.ZodObject<{
280
294
  completeness: z.ZodString;
281
295
  }, "strip", z.ZodTypeAny, {
282
296
  correctness: string;
283
- grounding: string;
284
297
  constraintHandling: string;
285
298
  completeness: string;
299
+ grounding: string;
286
300
  }, {
287
301
  correctness: string;
288
- grounding: string;
289
302
  constraintHandling: string;
290
303
  completeness: string;
304
+ grounding: string;
291
305
  }>;
292
306
  violations: z.ZodArray<z.ZodString, "many">;
293
307
  }, "strip", z.ZodTypeAny, {
@@ -296,9 +310,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
296
310
  rationale: string;
297
311
  criteria: {
298
312
  correctness: string;
299
- grounding: string;
300
313
  constraintHandling: string;
301
314
  completeness: string;
315
+ grounding: string;
302
316
  };
303
317
  violations: string[];
304
318
  }, {
@@ -307,9 +321,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
307
321
  rationale: string;
308
322
  criteria: {
309
323
  correctness: string;
310
- grounding: string;
311
324
  constraintHandling: string;
312
325
  completeness: string;
326
+ grounding: string;
313
327
  };
314
328
  violations: string[];
315
329
  }>;
@@ -382,6 +396,22 @@ interface EloState {
382
396
  ratings: Record<string, number>;
383
397
  points: Record<string, number>;
384
398
  }
399
+ /** Per-competitor latency measurements the speed arena decides matches by. */
400
+ interface SpeedMetric {
401
+ /** Time to first token: ms from request start to the first non-empty content delta. */
402
+ ttftMs: number;
403
+ /** Output tokens per second over the generation window: outputTokens / max(1e-3, (totalMs - ttftMs)/1000). */
404
+ tps: number;
405
+ /** Total wall-clock ms from request start to stream completion — the value that decides the winner. */
406
+ totalMs: number;
407
+ outputTokens: number;
408
+ }
409
+ /** Present only on 'speed-decided' matches; `a` is modelA's measurement, `b` is modelB's. */
410
+ interface SpeedMetrics {
411
+ a: SpeedMetric;
412
+ b: SpeedMetric;
413
+ }
414
+ declare const SpeedMetricsSchema: z.ZodType<SpeedMetrics>;
385
415
  interface MatchResult {
386
416
  methodologyVersion: string;
387
417
  runId: string;
@@ -398,7 +428,8 @@ interface MatchResult {
398
428
  category?: BenchmarkCategory;
399
429
  cluster: string;
400
430
  publicHash: string;
401
- privateHash: string;
431
+ /** Null for public-only packs (e.g. the speed arena), which have no hidden half. */
432
+ privateHash: string | null;
402
433
  };
403
434
  competitors: {
404
435
  modelA: string;
@@ -406,9 +437,11 @@ interface MatchResult {
406
437
  responseA: CompetitorResponse;
407
438
  responseB: CompetitorResponse;
408
439
  };
409
- outcome: 'judged' | 'forfeit' | 'no-contest';
440
+ outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
410
441
  winnerModelId: string | null;
411
442
  panel: PanelDecision | null;
443
+ /** Present only when outcome is 'speed-decided'; null (or absent) otherwise. */
444
+ speedMetrics?: SpeedMetrics | null;
412
445
  eloBefore: Record<string, number>;
413
446
  eloAfter: Record<string, number>;
414
447
  pointAwarded: boolean;
@@ -422,6 +455,15 @@ declare function competitorOutputTokens(response: CompetitorResponse): number;
422
455
  declare function competitorReasoningTokens(response: CompetitorResponse): number | undefined;
423
456
  declare function competitorContent(response: CompetitorResponse): string;
424
457
 
458
+ /** Per-model latency aggregates surfaced by the speed arena, derived from decided matches. */
459
+ interface SpeedStats {
460
+ /** Number of speed-decided matches this model participated in. */
461
+ samples: number;
462
+ medianTtftMs: number;
463
+ avgTtftMs: number;
464
+ medianTps: number;
465
+ avgTps: number;
466
+ }
425
467
  interface LeaderboardEntry {
426
468
  rank: number;
427
469
  modelId: string;
@@ -439,6 +481,8 @@ interface LeaderboardEntry {
439
481
  wins: number;
440
482
  losses: number;
441
483
  }>>;
484
+ /** Present only for the speed category; omitted for judged categories so their output is unchanged. */
485
+ speed?: SpeedStats;
442
486
  }
443
487
  interface ArenaSnapshot {
444
488
  version: '0.2.0';
@@ -451,4 +495,4 @@ interface ArenaSnapshot {
451
495
  matches: MatchResult[];
452
496
  }
453
497
 
454
- export { type ArenaEvent as A, competitorOutputTokens as B, type ChatRequest as C, competitorReasoningTokens as D, type EloState as E, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, competitorContent as y, competitorCost as z };
498
+ export { type ArenaEvent as A, SpeedMetricsSchema as B, type ChatRequest as C, type SpeedStats as D, type EloState as E, competitorContent as F, competitorCost as G, competitorOutputTokens as H, competitorReasoningTokens as I, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, type SpeedMetric as y, type SpeedMetrics as z };
@@ -1,4 +1,4 @@
1
- import { B as BenchmarkCategory } from './tasks-CpaCJ6JE.js';
1
+ import { B as BenchmarkCategory } from './tasks-BmhWuMBD.cjs';
2
2
  import { z } from 'zod';
3
3
 
4
4
  interface ResponseSummary {
@@ -57,7 +57,7 @@ interface ArenaEventDataMap {
57
57
  matchId: string;
58
58
  taskId: string;
59
59
  winnerModelId: string | null;
60
- outcome: 'judged' | 'forfeit' | 'no-contest';
60
+ outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
61
61
  costUsd: number;
62
62
  eloAfter: Record<string, number>;
63
63
  completed: number;
@@ -133,9 +133,13 @@ declare const ModelCompletionSchema: z.ZodObject<{
133
133
  latencyMs: z.ZodNumber;
134
134
  finishReason: z.ZodString;
135
135
  attempts: z.ZodOptional<z.ZodNumber>;
136
+ /** Time from request start to the first non-empty content delta (speed arena metric). */
137
+ ttftMs: z.ZodOptional<z.ZodNumber>;
138
+ /** Request start to stream completion for the successful attempt (speed arena metric). */
139
+ totalMs: z.ZodOptional<z.ZodNumber>;
136
140
  }, "strip", z.ZodTypeAny, {
137
- generationId: string;
138
141
  content: string;
142
+ generationId: string;
139
143
  inputTokens: number;
140
144
  outputTokens: number;
141
145
  costUsd: number;
@@ -143,9 +147,11 @@ declare const ModelCompletionSchema: z.ZodObject<{
143
147
  finishReason: string;
144
148
  reasoningTokens?: number | undefined;
145
149
  attempts?: number | undefined;
150
+ ttftMs?: number | undefined;
151
+ totalMs?: number | undefined;
146
152
  }, {
147
- generationId: string;
148
153
  content: string;
154
+ generationId: string;
149
155
  inputTokens: number;
150
156
  outputTokens: number;
151
157
  costUsd: number;
@@ -153,6 +159,8 @@ declare const ModelCompletionSchema: z.ZodObject<{
153
159
  finishReason: string;
154
160
  reasoningTokens?: number | undefined;
155
161
  attempts?: number | undefined;
162
+ ttftMs?: number | undefined;
163
+ totalMs?: number | undefined;
156
164
  }>;
157
165
  type ModelCompletion = z.infer<typeof ModelCompletionSchema>;
158
166
  interface ChatRequest {
@@ -218,12 +226,14 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
218
226
  latencyMs: z.ZodNumber;
219
227
  finishReason: z.ZodString;
220
228
  attempts: z.ZodOptional<z.ZodNumber>;
229
+ ttftMs: z.ZodOptional<z.ZodNumber>;
230
+ totalMs: z.ZodOptional<z.ZodNumber>;
221
231
  } & {
222
232
  modelId: z.ZodString;
223
233
  success: z.ZodLiteral<true>;
224
234
  }, "strip", z.ZodTypeAny, {
225
- generationId: string;
226
235
  content: string;
236
+ generationId: string;
227
237
  inputTokens: number;
228
238
  outputTokens: number;
229
239
  costUsd: number;
@@ -233,9 +243,11 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
233
243
  success: true;
234
244
  reasoningTokens?: number | undefined;
235
245
  attempts?: number | undefined;
246
+ ttftMs?: number | undefined;
247
+ totalMs?: number | undefined;
236
248
  }, {
237
- generationId: string;
238
249
  content: string;
250
+ generationId: string;
239
251
  inputTokens: number;
240
252
  outputTokens: number;
241
253
  costUsd: number;
@@ -245,6 +257,8 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
245
257
  success: true;
246
258
  reasoningTokens?: number | undefined;
247
259
  attempts?: number | undefined;
260
+ ttftMs?: number | undefined;
261
+ totalMs?: number | undefined;
248
262
  }>;
249
263
  type CompetitorSuccess = z.infer<typeof CompetitorSuccessSchema>;
250
264
  declare const CompetitorFailureSchema: z.ZodObject<{
@@ -280,14 +294,14 @@ declare const JudgeVerdictSchema: z.ZodObject<{
280
294
  completeness: z.ZodString;
281
295
  }, "strip", z.ZodTypeAny, {
282
296
  correctness: string;
283
- grounding: string;
284
297
  constraintHandling: string;
285
298
  completeness: string;
299
+ grounding: string;
286
300
  }, {
287
301
  correctness: string;
288
- grounding: string;
289
302
  constraintHandling: string;
290
303
  completeness: string;
304
+ grounding: string;
291
305
  }>;
292
306
  violations: z.ZodArray<z.ZodString, "many">;
293
307
  }, "strip", z.ZodTypeAny, {
@@ -296,9 +310,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
296
310
  rationale: string;
297
311
  criteria: {
298
312
  correctness: string;
299
- grounding: string;
300
313
  constraintHandling: string;
301
314
  completeness: string;
315
+ grounding: string;
302
316
  };
303
317
  violations: string[];
304
318
  }, {
@@ -307,9 +321,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
307
321
  rationale: string;
308
322
  criteria: {
309
323
  correctness: string;
310
- grounding: string;
311
324
  constraintHandling: string;
312
325
  completeness: string;
326
+ grounding: string;
313
327
  };
314
328
  violations: string[];
315
329
  }>;
@@ -382,6 +396,22 @@ interface EloState {
382
396
  ratings: Record<string, number>;
383
397
  points: Record<string, number>;
384
398
  }
399
+ /** Per-competitor latency measurements the speed arena decides matches by. */
400
+ interface SpeedMetric {
401
+ /** Time to first token: ms from request start to the first non-empty content delta. */
402
+ ttftMs: number;
403
+ /** Output tokens per second over the generation window: outputTokens / max(1e-3, (totalMs - ttftMs)/1000). */
404
+ tps: number;
405
+ /** Total wall-clock ms from request start to stream completion — the value that decides the winner. */
406
+ totalMs: number;
407
+ outputTokens: number;
408
+ }
409
+ /** Present only on 'speed-decided' matches; `a` is modelA's measurement, `b` is modelB's. */
410
+ interface SpeedMetrics {
411
+ a: SpeedMetric;
412
+ b: SpeedMetric;
413
+ }
414
+ declare const SpeedMetricsSchema: z.ZodType<SpeedMetrics>;
385
415
  interface MatchResult {
386
416
  methodologyVersion: string;
387
417
  runId: string;
@@ -398,7 +428,8 @@ interface MatchResult {
398
428
  category?: BenchmarkCategory;
399
429
  cluster: string;
400
430
  publicHash: string;
401
- privateHash: string;
431
+ /** Null for public-only packs (e.g. the speed arena), which have no hidden half. */
432
+ privateHash: string | null;
402
433
  };
403
434
  competitors: {
404
435
  modelA: string;
@@ -406,9 +437,11 @@ interface MatchResult {
406
437
  responseA: CompetitorResponse;
407
438
  responseB: CompetitorResponse;
408
439
  };
409
- outcome: 'judged' | 'forfeit' | 'no-contest';
440
+ outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
410
441
  winnerModelId: string | null;
411
442
  panel: PanelDecision | null;
443
+ /** Present only when outcome is 'speed-decided'; null (or absent) otherwise. */
444
+ speedMetrics?: SpeedMetrics | null;
412
445
  eloBefore: Record<string, number>;
413
446
  eloAfter: Record<string, number>;
414
447
  pointAwarded: boolean;
@@ -422,6 +455,15 @@ declare function competitorOutputTokens(response: CompetitorResponse): number;
422
455
  declare function competitorReasoningTokens(response: CompetitorResponse): number | undefined;
423
456
  declare function competitorContent(response: CompetitorResponse): string;
424
457
 
458
+ /** Per-model latency aggregates surfaced by the speed arena, derived from decided matches. */
459
+ interface SpeedStats {
460
+ /** Number of speed-decided matches this model participated in. */
461
+ samples: number;
462
+ medianTtftMs: number;
463
+ avgTtftMs: number;
464
+ medianTps: number;
465
+ avgTps: number;
466
+ }
425
467
  interface LeaderboardEntry {
426
468
  rank: number;
427
469
  modelId: string;
@@ -439,6 +481,8 @@ interface LeaderboardEntry {
439
481
  wins: number;
440
482
  losses: number;
441
483
  }>>;
484
+ /** Present only for the speed category; omitted for judged categories so their output is unchanged. */
485
+ speed?: SpeedStats;
442
486
  }
443
487
  interface ArenaSnapshot {
444
488
  version: '0.2.0';
@@ -451,4 +495,4 @@ interface ArenaSnapshot {
451
495
  matches: MatchResult[];
452
496
  }
453
497
 
454
- export { type ArenaEvent as A, competitorOutputTokens as B, type ChatRequest as C, competitorReasoningTokens as D, type EloState as E, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, competitorContent as y, competitorCost as z };
498
+ export { type ArenaEvent as A, SpeedMetricsSchema as B, type ChatRequest as C, type SpeedStats as D, type EloState as E, competitorContent as F, competitorCost as G, competitorOutputTokens as H, competitorReasoningTokens as I, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, type SpeedMetric as y, type SpeedMetrics as z };
@@ -1,9 +1,9 @@
1
1
  import { z } from 'zod';
2
2
 
3
3
  declare const METHODOLOGY_VERSION = "arena-v0.3.0";
4
- declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination"]>;
4
+ declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"]>;
5
5
  type BenchmarkCategory = z.infer<typeof BenchmarkCategorySchema>;
6
- declare const CATEGORIES: ["reasoning", "hallucination"];
6
+ declare const CATEGORIES: ["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"];
7
7
  declare const CATEGORY_CLUSTERS: Record<BenchmarkCategory, readonly string[]>;
8
8
  declare const CATEGORY_META: Record<BenchmarkCategory, {
9
9
  label: string;
@@ -16,21 +16,21 @@ declare const TaskArtifactSchema: z.ZodObject<{
16
16
  label: z.ZodString;
17
17
  content: z.ZodString;
18
18
  }, "strict", z.ZodTypeAny, {
19
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
20
19
  id: string;
21
- content: string;
20
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
22
21
  label: string;
22
+ content: string;
23
23
  }, {
24
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
25
24
  id: string;
26
- content: string;
25
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
27
26
  label: string;
27
+ content: string;
28
28
  }>;
29
29
  type TaskArtifact = z.infer<typeof TaskArtifactSchema>;
30
30
  declare const TaskPublicSchema: z.ZodObject<{
31
31
  id: z.ZodString;
32
32
  version: z.ZodString;
33
- category: z.ZodEnum<["reasoning", "hallucination"]>;
33
+ category: z.ZodEnum<["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"]>;
34
34
  cluster: z.ZodString;
35
35
  difficulty: z.ZodEnum<["hard", "expert"]>;
36
36
  title: z.ZodString;
@@ -42,47 +42,47 @@ declare const TaskPublicSchema: z.ZodObject<{
42
42
  label: z.ZodString;
43
43
  content: z.ZodString;
44
44
  }, "strict", z.ZodTypeAny, {
45
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
46
45
  id: string;
47
- content: string;
46
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
48
47
  label: string;
48
+ content: string;
49
49
  }, {
50
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
51
50
  id: string;
52
- content: string;
51
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
53
52
  label: string;
53
+ content: string;
54
54
  }>, "many">;
55
55
  tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
56
56
  }, "strict", z.ZodTypeAny, {
57
57
  id: string;
58
58
  version: string;
59
- category: "reasoning" | "hallucination";
59
+ category: "reasoning" | "hallucination" | "security" | "bullshit" | "refactoring" | "debugging" | "generation" | "speed";
60
60
  cluster: string;
61
61
  difficulty: "hard" | "expert";
62
62
  title: string;
63
63
  summary: string;
64
64
  prompt: string;
65
65
  artifacts: {
66
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
67
66
  id: string;
68
- content: string;
67
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
69
68
  label: string;
69
+ content: string;
70
70
  }[];
71
71
  tags: string[];
72
72
  }, {
73
73
  id: string;
74
74
  version: string;
75
- category: "reasoning" | "hallucination";
75
+ category: "reasoning" | "hallucination" | "security" | "bullshit" | "refactoring" | "debugging" | "generation" | "speed";
76
76
  cluster: string;
77
77
  difficulty: "hard" | "expert";
78
78
  title: string;
79
79
  summary: string;
80
80
  prompt: string;
81
81
  artifacts: {
82
- type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
83
82
  id: string;
84
- content: string;
83
+ type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
85
84
  label: string;
85
+ content: string;
86
86
  }[];
87
87
  tags?: string[] | undefined;
88
88
  }>;
@@ -100,14 +100,14 @@ declare const TaskPrivateSchema: z.ZodObject<{
100
100
  completeness: z.ZodString;
101
101
  }, "strict", z.ZodTypeAny, {
102
102
  correctness: string;
103
+ evidenceGrounding: string;
103
104
  constraintHandling: string;
104
105
  completeness: string;
105
- evidenceGrounding: string;
106
106
  }, {
107
107
  correctness: string;
108
+ evidenceGrounding: string;
108
109
  constraintHandling: string;
109
110
  completeness: string;
110
- evidenceGrounding: string;
111
111
  }>;
112
112
  }, "strict", z.ZodTypeAny, {
113
113
  id: string;
@@ -117,9 +117,9 @@ declare const TaskPrivateSchema: z.ZodObject<{
117
117
  disqualifyingErrors: string[];
118
118
  rubric: {
119
119
  correctness: string;
120
+ evidenceGrounding: string;
120
121
  constraintHandling: string;
121
122
  completeness: string;
122
- evidenceGrounding: string;
123
123
  };
124
124
  }, {
125
125
  id: string;
@@ -128,9 +128,9 @@ declare const TaskPrivateSchema: z.ZodObject<{
128
128
  requiredEvidence: string[];
129
129
  rubric: {
130
130
  correctness: string;
131
+ evidenceGrounding: string;
131
132
  constraintHandling: string;
132
133
  completeness: string;
133
- evidenceGrounding: string;
134
134
  };
135
135
  disqualifyingErrors?: string[] | undefined;
136
136
  }>;
@@ -147,5 +147,7 @@ type CompleteArenaTask = ArenaTask & {
147
147
  private: TaskPrivate;
148
148
  privateHash: string;
149
149
  };
150
+ /** Narrow a task to one carrying its hidden reference (present for every judged category). */
151
+ declare function isCompleteArenaTask(task: ArenaTask): task is CompleteArenaTask;
150
152
 
151
- export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i };
153
+ export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i, isCompleteArenaTask as j };