bridgebench 3.1.0-alpha.0 → 3.1.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -173
- package/dist/{chunk-UECBSKTD.js → chunk-AY45YLYL.js} +105 -5
- package/dist/{chunk-LFKEV2YL.js → chunk-CJGHBY54.js} +7 -7
- package/dist/{chunk-JTVNKSMO.js → chunk-DVMGL3L7.js} +80 -6
- package/dist/{chunk-EQHRUV2I.js → chunk-IUPFMGUL.js} +152 -16
- package/dist/{chunk-7YCJSOK7.cjs → chunk-KCXQ5SAU.cjs} +21 -21
- package/dist/{chunk-4TWPCPRP.cjs → chunk-QMOPRKWD.cjs} +89 -15
- package/dist/{chunk-NJTYVNP4.cjs → chunk-VAS6KNJA.cjs} +216 -80
- package/dist/{chunk-CIXITJW6.cjs → chunk-X3LPZGHS.cjs} +106 -6
- package/dist/cli.cjs +51 -49
- package/dist/cli.js +9 -7
- package/dist/client.cjs +5 -6
- package/dist/client.d.cts +3 -3
- package/dist/client.d.ts +3 -3
- package/dist/client.js +4 -5
- package/dist/contracts/index.cjs +6 -3
- package/dist/contracts/index.d.cts +2 -2
- package/dist/contracts/index.d.ts +2 -2
- package/dist/contracts/index.js +7 -4
- package/dist/index.cjs +8 -5
- package/dist/index.d.cts +9 -8
- package/dist/index.d.ts +9 -8
- package/dist/index.js +8 -5
- package/dist/{logger-CCR9Mg1c.d.cts → logger-BByta-7V.d.cts} +23 -23
- package/dist/{logger-QJU7SBDz.d.ts → logger-BQf29BLe.d.ts} +23 -23
- package/dist/{reports-s2CTnGN8.d.ts → reports-B8TCJtPr.d.ts} +57 -13
- package/dist/{reports-4CejmOHf.d.cts → reports-DPpOoOux.d.cts} +57 -13
- package/dist/{tasks-CpaCJ6JE.d.ts → tasks-BmhWuMBD.d.cts} +24 -22
- package/dist/{tasks-CpaCJ6JE.d.cts → tasks-BmhWuMBD.d.ts} +24 -22
- package/dist/tasks.cjs +3 -4
- package/dist/tasks.d.cts +1 -1
- package/dist/tasks.d.ts +1 -1
- package/dist/tasks.js +2 -3
- package/docs/README.md +32 -12
- package/docs/methodology.md +14 -0
- package/docs/operator-guide.md +178 -0
- package/docs/replay-elo.md +12 -2
- package/docs/reviewing-bridgebench.md +173 -0
- package/docs/task-authoring.md +79 -1
- package/package.json +5 -3
- package/tasks/bullshit/public/crossed-metric-properties.yaml +192 -0
- package/tasks/bullshit/public/crossed-release-checksums.yaml +199 -0
- package/tasks/bullshit/public/fabricated-config-keys.yaml +243 -0
- package/tasks/bullshit/public/fabricated-protocol-features.yaml +272 -0
- package/tasks/bullshit/public/impossible-capacity-math.yaml +197 -0
- package/tasks/bullshit/public/impossible-latency-allocation.yaml +215 -0
- package/tasks/bullshit/public/loaded-approval-bypass.yaml +229 -0
- package/tasks/bullshit/public/loaded-migration-fallout.yaml +221 -0
- package/tasks/bullshit/public/pseudo-gc-heap-tuning.yaml +206 -0
- package/tasks/bullshit/public/pseudo-network-tuning.yaml +204 -0
- package/tasks/bullshit/public/reversed-alert-cascade.yaml +251 -0
- package/tasks/bullshit/public/reversed-dependency-failure.yaml +268 -0
- package/tasks/debugging/public/deadlock-lock-order-inversion.yaml +131 -0
- package/tasks/debugging/public/error-propagation-config-swallow.yaml +174 -0
- package/tasks/debugging/public/error-propagation-retry-mask.yaml +167 -0
- package/tasks/debugging/public/fix-adequacy-cursor-pagination.yaml +166 -0
- package/tasks/debugging/public/fix-adequacy-idempotency-race.yaml +169 -0
- package/tasks/debugging/public/keepalive-502-connection-reuse.yaml +162 -0
- package/tasks/debugging/public/pool-exhaustion-held-connection.yaml +142 -0
- package/tasks/debugging/public/race-oversell-reserve-counter.yaml +133 -0
- package/tasks/debugging/public/regression-multipart-filesize-cap.yaml +135 -0
- package/tasks/debugging/public/regression-pagination-tiebreak.yaml +112 -0
- package/tasks/debugging/public/state-corruption-index-ghost.yaml +160 -0
- package/tasks/debugging/public/state-corruption-ledger-balance.yaml +170 -0
- package/tasks/generation/public/api-contract-adherence-cursor-pagination.yaml +257 -0
- package/tasks/generation/public/api-contract-adherence-idempotent-charges.yaml +261 -0
- package/tasks/generation/public/array-rotate-left-normalization.yaml +166 -0
- package/tasks/generation/public/cache-interface-dropin.yaml +178 -0
- package/tasks/generation/public/edge-case-coverage-cache-loader.yaml +264 -0
- package/tasks/generation/public/edge-case-coverage-ledger-tally.yaml +231 -0
- package/tasks/generation/public/event-envelope-wire-compat.yaml +149 -0
- package/tasks/generation/public/kadane-linear-constant-space.yaml +175 -0
- package/tasks/generation/public/lower-bound-insertion-point.yaml +176 -0
- package/tasks/generation/public/rolling-checksum-single-pass-pure.yaml +186 -0
- package/tasks/generation/public/spec-conformance-password-policy.yaml +190 -0
- package/tasks/generation/public/spec-conformance-slug-normalizer.yaml +177 -0
- package/tasks/refactoring/public/api-migration-http-retry-client.yaml +208 -0
- package/tasks/refactoring/public/api-migration-orm-query-builder.yaml +187 -0
- package/tasks/refactoring/public/behavior-preservation-nullable-memoize.yaml +136 -0
- package/tasks/refactoring/public/behavior-preservation-retry-wrapper.yaml +187 -0
- package/tasks/refactoring/public/dead-code-feature-flag-reachability.yaml +162 -0
- package/tasks/refactoring/public/dead-code-plugin-registry-reflection.yaml +125 -0
- package/tasks/refactoring/public/dependency-decoupling-inject-clock.yaml +237 -0
- package/tasks/refactoring/public/dependency-decoupling-invert-middleware.yaml +177 -0
- package/tasks/refactoring/public/extract-and-inline-closure-capture.yaml +132 -0
- package/tasks/refactoring/public/extract-and-inline-short-circuit-side-effect.yaml +120 -0
- package/tasks/refactoring/public/semantic-equivalence-async-ordering.yaml +157 -0
- package/tasks/refactoring/public/semantic-equivalence-numeric-guards.yaml +115 -0
- package/tasks/security/public/authz-guard-chain-exposure.yaml +224 -0
- package/tasks/security/public/authz-object-scope-idor.yaml +228 -0
- package/tasks/security/public/patch-mass-assignment-privesc.yaml +226 -0
- package/tasks/security/public/patch-sqli-candidate-fixes.yaml +243 -0
- package/tasks/security/public/supply-lockfile-tamper-trace.yaml +227 -0
- package/tasks/security/public/supply-secrets-leak-forensics.yaml +205 -0
- package/tasks/security/public/taint-export-template-render.yaml +207 -0
- package/tasks/security/public/taint-webhook-outbound-fetch.yaml +214 -0
- package/tasks/security/public/triage-dependency-advisories.yaml +189 -0
- package/tasks/security/public/triage-sast-false-positives.yaml +257 -0
- package/tasks/security/public/vuln-path-sanitizer-escape.yaml +165 -0
- package/tasks/security/public/vuln-token-alg-confusion.yaml +252 -0
- package/tasks/speed/public/speed-api-diff-changelog.yaml +42 -0
- package/tasks/speed/public/speed-callbacks-to-async.yaml +39 -0
- package/tasks/speed/public/speed-class-to-hooks.yaml +47 -0
- package/tasks/speed/public/speed-config-schema-json.yaml +34 -0
- package/tasks/speed/public/speed-csv-report-generator.yaml +38 -0
- package/tasks/speed/public/speed-debounce-hook.yaml +28 -0
- package/tasks/speed/public/speed-log-triage-summary.yaml +35 -0
- package/tasks/speed/public/speed-openapi-fragment.yaml +38 -0
- package/tasks/speed/public/speed-pagination-cursor.yaml +40 -0
- package/tasks/speed/public/speed-rate-limiter-design.yaml +43 -0
- package/tasks/speed/public/speed-rest-client-module.yaml +42 -0
- package/tasks/speed/public/speed-slugify-helper.yaml +30 -0
- package/dist/chunk-4TWPCPRP.cjs.map +0 -1
- package/dist/chunk-7YCJSOK7.cjs.map +0 -1
- package/dist/chunk-CIXITJW6.cjs.map +0 -1
- package/dist/chunk-EQHRUV2I.js.map +0 -1
- package/dist/chunk-JTVNKSMO.js.map +0 -1
- package/dist/chunk-LFKEV2YL.js.map +0 -1
- package/dist/chunk-NJTYVNP4.cjs.map +0 -1
- package/dist/chunk-UECBSKTD.js.map +0 -1
- package/dist/cli.cjs.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/client.cjs.map +0 -1
- package/dist/client.js.map +0 -1
- package/dist/contracts/index.cjs.map +0 -1
- package/dist/contracts/index.js.map +0 -1
- package/dist/index.cjs.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/tasks.cjs.map +0 -1
- package/dist/tasks.js.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { B as BenchmarkCategory } from './tasks-
|
|
1
|
+
import { B as BenchmarkCategory } from './tasks-BmhWuMBD.js';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
|
|
4
4
|
interface ResponseSummary {
|
|
@@ -57,7 +57,7 @@ interface ArenaEventDataMap {
|
|
|
57
57
|
matchId: string;
|
|
58
58
|
taskId: string;
|
|
59
59
|
winnerModelId: string | null;
|
|
60
|
-
outcome: 'judged' | 'forfeit' | 'no-contest';
|
|
60
|
+
outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
|
|
61
61
|
costUsd: number;
|
|
62
62
|
eloAfter: Record<string, number>;
|
|
63
63
|
completed: number;
|
|
@@ -133,9 +133,13 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
133
133
|
latencyMs: z.ZodNumber;
|
|
134
134
|
finishReason: z.ZodString;
|
|
135
135
|
attempts: z.ZodOptional<z.ZodNumber>;
|
|
136
|
+
/** Time from request start to the first non-empty content delta (speed arena metric). */
|
|
137
|
+
ttftMs: z.ZodOptional<z.ZodNumber>;
|
|
138
|
+
/** Request start to stream completion for the successful attempt (speed arena metric). */
|
|
139
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
136
140
|
}, "strip", z.ZodTypeAny, {
|
|
137
|
-
generationId: string;
|
|
138
141
|
content: string;
|
|
142
|
+
generationId: string;
|
|
139
143
|
inputTokens: number;
|
|
140
144
|
outputTokens: number;
|
|
141
145
|
costUsd: number;
|
|
@@ -143,9 +147,11 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
143
147
|
finishReason: string;
|
|
144
148
|
reasoningTokens?: number | undefined;
|
|
145
149
|
attempts?: number | undefined;
|
|
150
|
+
ttftMs?: number | undefined;
|
|
151
|
+
totalMs?: number | undefined;
|
|
146
152
|
}, {
|
|
147
|
-
generationId: string;
|
|
148
153
|
content: string;
|
|
154
|
+
generationId: string;
|
|
149
155
|
inputTokens: number;
|
|
150
156
|
outputTokens: number;
|
|
151
157
|
costUsd: number;
|
|
@@ -153,6 +159,8 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
153
159
|
finishReason: string;
|
|
154
160
|
reasoningTokens?: number | undefined;
|
|
155
161
|
attempts?: number | undefined;
|
|
162
|
+
ttftMs?: number | undefined;
|
|
163
|
+
totalMs?: number | undefined;
|
|
156
164
|
}>;
|
|
157
165
|
type ModelCompletion = z.infer<typeof ModelCompletionSchema>;
|
|
158
166
|
interface ChatRequest {
|
|
@@ -218,12 +226,14 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
218
226
|
latencyMs: z.ZodNumber;
|
|
219
227
|
finishReason: z.ZodString;
|
|
220
228
|
attempts: z.ZodOptional<z.ZodNumber>;
|
|
229
|
+
ttftMs: z.ZodOptional<z.ZodNumber>;
|
|
230
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
221
231
|
} & {
|
|
222
232
|
modelId: z.ZodString;
|
|
223
233
|
success: z.ZodLiteral<true>;
|
|
224
234
|
}, "strip", z.ZodTypeAny, {
|
|
225
|
-
generationId: string;
|
|
226
235
|
content: string;
|
|
236
|
+
generationId: string;
|
|
227
237
|
inputTokens: number;
|
|
228
238
|
outputTokens: number;
|
|
229
239
|
costUsd: number;
|
|
@@ -233,9 +243,11 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
233
243
|
success: true;
|
|
234
244
|
reasoningTokens?: number | undefined;
|
|
235
245
|
attempts?: number | undefined;
|
|
246
|
+
ttftMs?: number | undefined;
|
|
247
|
+
totalMs?: number | undefined;
|
|
236
248
|
}, {
|
|
237
|
-
generationId: string;
|
|
238
249
|
content: string;
|
|
250
|
+
generationId: string;
|
|
239
251
|
inputTokens: number;
|
|
240
252
|
outputTokens: number;
|
|
241
253
|
costUsd: number;
|
|
@@ -245,6 +257,8 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
245
257
|
success: true;
|
|
246
258
|
reasoningTokens?: number | undefined;
|
|
247
259
|
attempts?: number | undefined;
|
|
260
|
+
ttftMs?: number | undefined;
|
|
261
|
+
totalMs?: number | undefined;
|
|
248
262
|
}>;
|
|
249
263
|
type CompetitorSuccess = z.infer<typeof CompetitorSuccessSchema>;
|
|
250
264
|
declare const CompetitorFailureSchema: z.ZodObject<{
|
|
@@ -280,14 +294,14 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
280
294
|
completeness: z.ZodString;
|
|
281
295
|
}, "strip", z.ZodTypeAny, {
|
|
282
296
|
correctness: string;
|
|
283
|
-
grounding: string;
|
|
284
297
|
constraintHandling: string;
|
|
285
298
|
completeness: string;
|
|
299
|
+
grounding: string;
|
|
286
300
|
}, {
|
|
287
301
|
correctness: string;
|
|
288
|
-
grounding: string;
|
|
289
302
|
constraintHandling: string;
|
|
290
303
|
completeness: string;
|
|
304
|
+
grounding: string;
|
|
291
305
|
}>;
|
|
292
306
|
violations: z.ZodArray<z.ZodString, "many">;
|
|
293
307
|
}, "strip", z.ZodTypeAny, {
|
|
@@ -296,9 +310,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
296
310
|
rationale: string;
|
|
297
311
|
criteria: {
|
|
298
312
|
correctness: string;
|
|
299
|
-
grounding: string;
|
|
300
313
|
constraintHandling: string;
|
|
301
314
|
completeness: string;
|
|
315
|
+
grounding: string;
|
|
302
316
|
};
|
|
303
317
|
violations: string[];
|
|
304
318
|
}, {
|
|
@@ -307,9 +321,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
307
321
|
rationale: string;
|
|
308
322
|
criteria: {
|
|
309
323
|
correctness: string;
|
|
310
|
-
grounding: string;
|
|
311
324
|
constraintHandling: string;
|
|
312
325
|
completeness: string;
|
|
326
|
+
grounding: string;
|
|
313
327
|
};
|
|
314
328
|
violations: string[];
|
|
315
329
|
}>;
|
|
@@ -382,6 +396,22 @@ interface EloState {
|
|
|
382
396
|
ratings: Record<string, number>;
|
|
383
397
|
points: Record<string, number>;
|
|
384
398
|
}
|
|
399
|
+
/** Per-competitor latency measurements the speed arena decides matches by. */
|
|
400
|
+
interface SpeedMetric {
|
|
401
|
+
/** Time to first token: ms from request start to the first non-empty content delta. */
|
|
402
|
+
ttftMs: number;
|
|
403
|
+
/** Output tokens per second over the generation window: outputTokens / max(1e-3, (totalMs - ttftMs)/1000). */
|
|
404
|
+
tps: number;
|
|
405
|
+
/** Total wall-clock ms from request start to stream completion — the value that decides the winner. */
|
|
406
|
+
totalMs: number;
|
|
407
|
+
outputTokens: number;
|
|
408
|
+
}
|
|
409
|
+
/** Present only on 'speed-decided' matches; `a` is modelA's measurement, `b` is modelB's. */
|
|
410
|
+
interface SpeedMetrics {
|
|
411
|
+
a: SpeedMetric;
|
|
412
|
+
b: SpeedMetric;
|
|
413
|
+
}
|
|
414
|
+
declare const SpeedMetricsSchema: z.ZodType<SpeedMetrics>;
|
|
385
415
|
interface MatchResult {
|
|
386
416
|
methodologyVersion: string;
|
|
387
417
|
runId: string;
|
|
@@ -398,7 +428,8 @@ interface MatchResult {
|
|
|
398
428
|
category?: BenchmarkCategory;
|
|
399
429
|
cluster: string;
|
|
400
430
|
publicHash: string;
|
|
401
|
-
|
|
431
|
+
/** Null for public-only packs (e.g. the speed arena), which have no hidden half. */
|
|
432
|
+
privateHash: string | null;
|
|
402
433
|
};
|
|
403
434
|
competitors: {
|
|
404
435
|
modelA: string;
|
|
@@ -406,9 +437,11 @@ interface MatchResult {
|
|
|
406
437
|
responseA: CompetitorResponse;
|
|
407
438
|
responseB: CompetitorResponse;
|
|
408
439
|
};
|
|
409
|
-
outcome: 'judged' | 'forfeit' | 'no-contest';
|
|
440
|
+
outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
|
|
410
441
|
winnerModelId: string | null;
|
|
411
442
|
panel: PanelDecision | null;
|
|
443
|
+
/** Present only when outcome is 'speed-decided'; null (or absent) otherwise. */
|
|
444
|
+
speedMetrics?: SpeedMetrics | null;
|
|
412
445
|
eloBefore: Record<string, number>;
|
|
413
446
|
eloAfter: Record<string, number>;
|
|
414
447
|
pointAwarded: boolean;
|
|
@@ -422,6 +455,15 @@ declare function competitorOutputTokens(response: CompetitorResponse): number;
|
|
|
422
455
|
declare function competitorReasoningTokens(response: CompetitorResponse): number | undefined;
|
|
423
456
|
declare function competitorContent(response: CompetitorResponse): string;
|
|
424
457
|
|
|
458
|
+
/** Per-model latency aggregates surfaced by the speed arena, derived from decided matches. */
|
|
459
|
+
interface SpeedStats {
|
|
460
|
+
/** Number of speed-decided matches this model participated in. */
|
|
461
|
+
samples: number;
|
|
462
|
+
medianTtftMs: number;
|
|
463
|
+
avgTtftMs: number;
|
|
464
|
+
medianTps: number;
|
|
465
|
+
avgTps: number;
|
|
466
|
+
}
|
|
425
467
|
interface LeaderboardEntry {
|
|
426
468
|
rank: number;
|
|
427
469
|
modelId: string;
|
|
@@ -439,6 +481,8 @@ interface LeaderboardEntry {
|
|
|
439
481
|
wins: number;
|
|
440
482
|
losses: number;
|
|
441
483
|
}>>;
|
|
484
|
+
/** Present only for the speed category; omitted for judged categories so their output is unchanged. */
|
|
485
|
+
speed?: SpeedStats;
|
|
442
486
|
}
|
|
443
487
|
interface ArenaSnapshot {
|
|
444
488
|
version: '0.2.0';
|
|
@@ -451,4 +495,4 @@ interface ArenaSnapshot {
|
|
|
451
495
|
matches: MatchResult[];
|
|
452
496
|
}
|
|
453
497
|
|
|
454
|
-
export { type ArenaEvent as A,
|
|
498
|
+
export { type ArenaEvent as A, SpeedMetricsSchema as B, type ChatRequest as C, type SpeedStats as D, type EloState as E, competitorContent as F, competitorCost as G, competitorOutputTokens as H, competitorReasoningTokens as I, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, type SpeedMetric as y, type SpeedMetrics as z };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { B as BenchmarkCategory } from './tasks-
|
|
1
|
+
import { B as BenchmarkCategory } from './tasks-BmhWuMBD.cjs';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
|
|
4
4
|
interface ResponseSummary {
|
|
@@ -57,7 +57,7 @@ interface ArenaEventDataMap {
|
|
|
57
57
|
matchId: string;
|
|
58
58
|
taskId: string;
|
|
59
59
|
winnerModelId: string | null;
|
|
60
|
-
outcome: 'judged' | 'forfeit' | 'no-contest';
|
|
60
|
+
outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
|
|
61
61
|
costUsd: number;
|
|
62
62
|
eloAfter: Record<string, number>;
|
|
63
63
|
completed: number;
|
|
@@ -133,9 +133,13 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
133
133
|
latencyMs: z.ZodNumber;
|
|
134
134
|
finishReason: z.ZodString;
|
|
135
135
|
attempts: z.ZodOptional<z.ZodNumber>;
|
|
136
|
+
/** Time from request start to the first non-empty content delta (speed arena metric). */
|
|
137
|
+
ttftMs: z.ZodOptional<z.ZodNumber>;
|
|
138
|
+
/** Request start to stream completion for the successful attempt (speed arena metric). */
|
|
139
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
136
140
|
}, "strip", z.ZodTypeAny, {
|
|
137
|
-
generationId: string;
|
|
138
141
|
content: string;
|
|
142
|
+
generationId: string;
|
|
139
143
|
inputTokens: number;
|
|
140
144
|
outputTokens: number;
|
|
141
145
|
costUsd: number;
|
|
@@ -143,9 +147,11 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
143
147
|
finishReason: string;
|
|
144
148
|
reasoningTokens?: number | undefined;
|
|
145
149
|
attempts?: number | undefined;
|
|
150
|
+
ttftMs?: number | undefined;
|
|
151
|
+
totalMs?: number | undefined;
|
|
146
152
|
}, {
|
|
147
|
-
generationId: string;
|
|
148
153
|
content: string;
|
|
154
|
+
generationId: string;
|
|
149
155
|
inputTokens: number;
|
|
150
156
|
outputTokens: number;
|
|
151
157
|
costUsd: number;
|
|
@@ -153,6 +159,8 @@ declare const ModelCompletionSchema: z.ZodObject<{
|
|
|
153
159
|
finishReason: string;
|
|
154
160
|
reasoningTokens?: number | undefined;
|
|
155
161
|
attempts?: number | undefined;
|
|
162
|
+
ttftMs?: number | undefined;
|
|
163
|
+
totalMs?: number | undefined;
|
|
156
164
|
}>;
|
|
157
165
|
type ModelCompletion = z.infer<typeof ModelCompletionSchema>;
|
|
158
166
|
interface ChatRequest {
|
|
@@ -218,12 +226,14 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
218
226
|
latencyMs: z.ZodNumber;
|
|
219
227
|
finishReason: z.ZodString;
|
|
220
228
|
attempts: z.ZodOptional<z.ZodNumber>;
|
|
229
|
+
ttftMs: z.ZodOptional<z.ZodNumber>;
|
|
230
|
+
totalMs: z.ZodOptional<z.ZodNumber>;
|
|
221
231
|
} & {
|
|
222
232
|
modelId: z.ZodString;
|
|
223
233
|
success: z.ZodLiteral<true>;
|
|
224
234
|
}, "strip", z.ZodTypeAny, {
|
|
225
|
-
generationId: string;
|
|
226
235
|
content: string;
|
|
236
|
+
generationId: string;
|
|
227
237
|
inputTokens: number;
|
|
228
238
|
outputTokens: number;
|
|
229
239
|
costUsd: number;
|
|
@@ -233,9 +243,11 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
233
243
|
success: true;
|
|
234
244
|
reasoningTokens?: number | undefined;
|
|
235
245
|
attempts?: number | undefined;
|
|
246
|
+
ttftMs?: number | undefined;
|
|
247
|
+
totalMs?: number | undefined;
|
|
236
248
|
}, {
|
|
237
|
-
generationId: string;
|
|
238
249
|
content: string;
|
|
250
|
+
generationId: string;
|
|
239
251
|
inputTokens: number;
|
|
240
252
|
outputTokens: number;
|
|
241
253
|
costUsd: number;
|
|
@@ -245,6 +257,8 @@ declare const CompetitorSuccessSchema: z.ZodObject<{
|
|
|
245
257
|
success: true;
|
|
246
258
|
reasoningTokens?: number | undefined;
|
|
247
259
|
attempts?: number | undefined;
|
|
260
|
+
ttftMs?: number | undefined;
|
|
261
|
+
totalMs?: number | undefined;
|
|
248
262
|
}>;
|
|
249
263
|
type CompetitorSuccess = z.infer<typeof CompetitorSuccessSchema>;
|
|
250
264
|
declare const CompetitorFailureSchema: z.ZodObject<{
|
|
@@ -280,14 +294,14 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
280
294
|
completeness: z.ZodString;
|
|
281
295
|
}, "strip", z.ZodTypeAny, {
|
|
282
296
|
correctness: string;
|
|
283
|
-
grounding: string;
|
|
284
297
|
constraintHandling: string;
|
|
285
298
|
completeness: string;
|
|
299
|
+
grounding: string;
|
|
286
300
|
}, {
|
|
287
301
|
correctness: string;
|
|
288
|
-
grounding: string;
|
|
289
302
|
constraintHandling: string;
|
|
290
303
|
completeness: string;
|
|
304
|
+
grounding: string;
|
|
291
305
|
}>;
|
|
292
306
|
violations: z.ZodArray<z.ZodString, "many">;
|
|
293
307
|
}, "strip", z.ZodTypeAny, {
|
|
@@ -296,9 +310,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
296
310
|
rationale: string;
|
|
297
311
|
criteria: {
|
|
298
312
|
correctness: string;
|
|
299
|
-
grounding: string;
|
|
300
313
|
constraintHandling: string;
|
|
301
314
|
completeness: string;
|
|
315
|
+
grounding: string;
|
|
302
316
|
};
|
|
303
317
|
violations: string[];
|
|
304
318
|
}, {
|
|
@@ -307,9 +321,9 @@ declare const JudgeVerdictSchema: z.ZodObject<{
|
|
|
307
321
|
rationale: string;
|
|
308
322
|
criteria: {
|
|
309
323
|
correctness: string;
|
|
310
|
-
grounding: string;
|
|
311
324
|
constraintHandling: string;
|
|
312
325
|
completeness: string;
|
|
326
|
+
grounding: string;
|
|
313
327
|
};
|
|
314
328
|
violations: string[];
|
|
315
329
|
}>;
|
|
@@ -382,6 +396,22 @@ interface EloState {
|
|
|
382
396
|
ratings: Record<string, number>;
|
|
383
397
|
points: Record<string, number>;
|
|
384
398
|
}
|
|
399
|
+
/** Per-competitor latency measurements the speed arena decides matches by. */
|
|
400
|
+
interface SpeedMetric {
|
|
401
|
+
/** Time to first token: ms from request start to the first non-empty content delta. */
|
|
402
|
+
ttftMs: number;
|
|
403
|
+
/** Output tokens per second over the generation window: outputTokens / max(1e-3, (totalMs - ttftMs)/1000). */
|
|
404
|
+
tps: number;
|
|
405
|
+
/** Total wall-clock ms from request start to stream completion — the value that decides the winner. */
|
|
406
|
+
totalMs: number;
|
|
407
|
+
outputTokens: number;
|
|
408
|
+
}
|
|
409
|
+
/** Present only on 'speed-decided' matches; `a` is modelA's measurement, `b` is modelB's. */
|
|
410
|
+
interface SpeedMetrics {
|
|
411
|
+
a: SpeedMetric;
|
|
412
|
+
b: SpeedMetric;
|
|
413
|
+
}
|
|
414
|
+
declare const SpeedMetricsSchema: z.ZodType<SpeedMetrics>;
|
|
385
415
|
interface MatchResult {
|
|
386
416
|
methodologyVersion: string;
|
|
387
417
|
runId: string;
|
|
@@ -398,7 +428,8 @@ interface MatchResult {
|
|
|
398
428
|
category?: BenchmarkCategory;
|
|
399
429
|
cluster: string;
|
|
400
430
|
publicHash: string;
|
|
401
|
-
|
|
431
|
+
/** Null for public-only packs (e.g. the speed arena), which have no hidden half. */
|
|
432
|
+
privateHash: string | null;
|
|
402
433
|
};
|
|
403
434
|
competitors: {
|
|
404
435
|
modelA: string;
|
|
@@ -406,9 +437,11 @@ interface MatchResult {
|
|
|
406
437
|
responseA: CompetitorResponse;
|
|
407
438
|
responseB: CompetitorResponse;
|
|
408
439
|
};
|
|
409
|
-
outcome: 'judged' | 'forfeit' | 'no-contest';
|
|
440
|
+
outcome: 'judged' | 'forfeit' | 'no-contest' | 'speed-decided';
|
|
410
441
|
winnerModelId: string | null;
|
|
411
442
|
panel: PanelDecision | null;
|
|
443
|
+
/** Present only when outcome is 'speed-decided'; null (or absent) otherwise. */
|
|
444
|
+
speedMetrics?: SpeedMetrics | null;
|
|
412
445
|
eloBefore: Record<string, number>;
|
|
413
446
|
eloAfter: Record<string, number>;
|
|
414
447
|
pointAwarded: boolean;
|
|
@@ -422,6 +455,15 @@ declare function competitorOutputTokens(response: CompetitorResponse): number;
|
|
|
422
455
|
declare function competitorReasoningTokens(response: CompetitorResponse): number | undefined;
|
|
423
456
|
declare function competitorContent(response: CompetitorResponse): string;
|
|
424
457
|
|
|
458
|
+
/** Per-model latency aggregates surfaced by the speed arena, derived from decided matches. */
|
|
459
|
+
interface SpeedStats {
|
|
460
|
+
/** Number of speed-decided matches this model participated in. */
|
|
461
|
+
samples: number;
|
|
462
|
+
medianTtftMs: number;
|
|
463
|
+
avgTtftMs: number;
|
|
464
|
+
medianTps: number;
|
|
465
|
+
avgTps: number;
|
|
466
|
+
}
|
|
425
467
|
interface LeaderboardEntry {
|
|
426
468
|
rank: number;
|
|
427
469
|
modelId: string;
|
|
@@ -439,6 +481,8 @@ interface LeaderboardEntry {
|
|
|
439
481
|
wins: number;
|
|
440
482
|
losses: number;
|
|
441
483
|
}>>;
|
|
484
|
+
/** Present only for the speed category; omitted for judged categories so their output is unchanged. */
|
|
485
|
+
speed?: SpeedStats;
|
|
442
486
|
}
|
|
443
487
|
interface ArenaSnapshot {
|
|
444
488
|
version: '0.2.0';
|
|
@@ -451,4 +495,4 @@ interface ArenaSnapshot {
|
|
|
451
495
|
matches: MatchResult[];
|
|
452
496
|
}
|
|
453
497
|
|
|
454
|
-
export { type ArenaEvent as A,
|
|
498
|
+
export { type ArenaEvent as A, SpeedMetricsSchema as B, type ChatRequest as C, type SpeedStats as D, type EloState as E, competitorContent as F, competitorCost as G, competitorOutputTokens as H, competitorReasoningTokens as I, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, type SpeedMetric as y, type SpeedMetrics as z };
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
|
|
3
3
|
declare const METHODOLOGY_VERSION = "arena-v0.3.0";
|
|
4
|
-
declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination"]>;
|
|
4
|
+
declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"]>;
|
|
5
5
|
type BenchmarkCategory = z.infer<typeof BenchmarkCategorySchema>;
|
|
6
|
-
declare const CATEGORIES: ["reasoning", "hallucination"];
|
|
6
|
+
declare const CATEGORIES: ["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"];
|
|
7
7
|
declare const CATEGORY_CLUSTERS: Record<BenchmarkCategory, readonly string[]>;
|
|
8
8
|
declare const CATEGORY_META: Record<BenchmarkCategory, {
|
|
9
9
|
label: string;
|
|
@@ -16,21 +16,21 @@ declare const TaskArtifactSchema: z.ZodObject<{
|
|
|
16
16
|
label: z.ZodString;
|
|
17
17
|
content: z.ZodString;
|
|
18
18
|
}, "strict", z.ZodTypeAny, {
|
|
19
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
20
19
|
id: string;
|
|
21
|
-
|
|
20
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
22
21
|
label: string;
|
|
22
|
+
content: string;
|
|
23
23
|
}, {
|
|
24
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
25
24
|
id: string;
|
|
26
|
-
|
|
25
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
27
26
|
label: string;
|
|
27
|
+
content: string;
|
|
28
28
|
}>;
|
|
29
29
|
type TaskArtifact = z.infer<typeof TaskArtifactSchema>;
|
|
30
30
|
declare const TaskPublicSchema: z.ZodObject<{
|
|
31
31
|
id: z.ZodString;
|
|
32
32
|
version: z.ZodString;
|
|
33
|
-
category: z.ZodEnum<["reasoning", "hallucination"]>;
|
|
33
|
+
category: z.ZodEnum<["reasoning", "hallucination", "security", "bullshit", "refactoring", "debugging", "generation", "speed"]>;
|
|
34
34
|
cluster: z.ZodString;
|
|
35
35
|
difficulty: z.ZodEnum<["hard", "expert"]>;
|
|
36
36
|
title: z.ZodString;
|
|
@@ -42,47 +42,47 @@ declare const TaskPublicSchema: z.ZodObject<{
|
|
|
42
42
|
label: z.ZodString;
|
|
43
43
|
content: z.ZodString;
|
|
44
44
|
}, "strict", z.ZodTypeAny, {
|
|
45
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
46
45
|
id: string;
|
|
47
|
-
|
|
46
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
48
47
|
label: string;
|
|
48
|
+
content: string;
|
|
49
49
|
}, {
|
|
50
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
51
50
|
id: string;
|
|
52
|
-
|
|
51
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
53
52
|
label: string;
|
|
53
|
+
content: string;
|
|
54
54
|
}>, "many">;
|
|
55
55
|
tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
|
|
56
56
|
}, "strict", z.ZodTypeAny, {
|
|
57
57
|
id: string;
|
|
58
58
|
version: string;
|
|
59
|
-
category: "reasoning" | "hallucination";
|
|
59
|
+
category: "reasoning" | "hallucination" | "security" | "bullshit" | "refactoring" | "debugging" | "generation" | "speed";
|
|
60
60
|
cluster: string;
|
|
61
61
|
difficulty: "hard" | "expert";
|
|
62
62
|
title: string;
|
|
63
63
|
summary: string;
|
|
64
64
|
prompt: string;
|
|
65
65
|
artifacts: {
|
|
66
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
67
66
|
id: string;
|
|
68
|
-
|
|
67
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
69
68
|
label: string;
|
|
69
|
+
content: string;
|
|
70
70
|
}[];
|
|
71
71
|
tags: string[];
|
|
72
72
|
}, {
|
|
73
73
|
id: string;
|
|
74
74
|
version: string;
|
|
75
|
-
category: "reasoning" | "hallucination";
|
|
75
|
+
category: "reasoning" | "hallucination" | "security" | "bullshit" | "refactoring" | "debugging" | "generation" | "speed";
|
|
76
76
|
cluster: string;
|
|
77
77
|
difficulty: "hard" | "expert";
|
|
78
78
|
title: string;
|
|
79
79
|
summary: string;
|
|
80
80
|
prompt: string;
|
|
81
81
|
artifacts: {
|
|
82
|
-
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
83
82
|
id: string;
|
|
84
|
-
|
|
83
|
+
type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
|
|
85
84
|
label: string;
|
|
85
|
+
content: string;
|
|
86
86
|
}[];
|
|
87
87
|
tags?: string[] | undefined;
|
|
88
88
|
}>;
|
|
@@ -100,14 +100,14 @@ declare const TaskPrivateSchema: z.ZodObject<{
|
|
|
100
100
|
completeness: z.ZodString;
|
|
101
101
|
}, "strict", z.ZodTypeAny, {
|
|
102
102
|
correctness: string;
|
|
103
|
+
evidenceGrounding: string;
|
|
103
104
|
constraintHandling: string;
|
|
104
105
|
completeness: string;
|
|
105
|
-
evidenceGrounding: string;
|
|
106
106
|
}, {
|
|
107
107
|
correctness: string;
|
|
108
|
+
evidenceGrounding: string;
|
|
108
109
|
constraintHandling: string;
|
|
109
110
|
completeness: string;
|
|
110
|
-
evidenceGrounding: string;
|
|
111
111
|
}>;
|
|
112
112
|
}, "strict", z.ZodTypeAny, {
|
|
113
113
|
id: string;
|
|
@@ -117,9 +117,9 @@ declare const TaskPrivateSchema: z.ZodObject<{
|
|
|
117
117
|
disqualifyingErrors: string[];
|
|
118
118
|
rubric: {
|
|
119
119
|
correctness: string;
|
|
120
|
+
evidenceGrounding: string;
|
|
120
121
|
constraintHandling: string;
|
|
121
122
|
completeness: string;
|
|
122
|
-
evidenceGrounding: string;
|
|
123
123
|
};
|
|
124
124
|
}, {
|
|
125
125
|
id: string;
|
|
@@ -128,9 +128,9 @@ declare const TaskPrivateSchema: z.ZodObject<{
|
|
|
128
128
|
requiredEvidence: string[];
|
|
129
129
|
rubric: {
|
|
130
130
|
correctness: string;
|
|
131
|
+
evidenceGrounding: string;
|
|
131
132
|
constraintHandling: string;
|
|
132
133
|
completeness: string;
|
|
133
|
-
evidenceGrounding: string;
|
|
134
134
|
};
|
|
135
135
|
disqualifyingErrors?: string[] | undefined;
|
|
136
136
|
}>;
|
|
@@ -147,5 +147,7 @@ type CompleteArenaTask = ArenaTask & {
|
|
|
147
147
|
private: TaskPrivate;
|
|
148
148
|
privateHash: string;
|
|
149
149
|
};
|
|
150
|
+
/** Narrow a task to one carrying its hidden reference (present for every judged category). */
|
|
151
|
+
declare function isCompleteArenaTask(task: ArenaTask): task is CompleteArenaTask;
|
|
150
152
|
|
|
151
|
-
export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i };
|
|
153
|
+
export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i, isCompleteArenaTask as j };
|