bridgebench 3.1.0-alpha.0

package/dist/reports-s2CTnGN8.d.ts

@@ -0,0 +1,454 @@
+import { B as BenchmarkCategory } from './tasks-CpaCJ6JE.js';
+import { z } from 'zod';
+
+interface ResponseSummary {
+    success: boolean;
+    latencyMs: number;
+    costUsd: number;
+    outputTokens: number;
+    reasoningTokens: number | null;
+}
+interface ArenaEventDataMap {
+    'run.started': {
+        runId: string;
+        category: BenchmarkCategory;
+        seed: string;
+        matches: number;
+        maxCostUsd: number;
+        competitorIds: string[];
+    };
+    'match.started': {
+        matchId: string;
+        index: number;
+        total: number;
+        category: BenchmarkCategory;
+        taskId: string;
+        taskTitle: string;
+        modelA: string;
+        modelB: string;
+    };
+    'competitor.delta': {
+        matchId: string;
+        modelId: string;
+        side: 'A' | 'B';
+        text: string;
+        done: boolean;
+        success: boolean;
+    };
+    'competitors.completed': {
+        matchId: string;
+        modelA: ResponseSummary;
+        modelB: ResponseSummary;
+    };
+    'judging.started': {
+        matchId: string;
+        judges: string[];
+    };
+    'judge.completed': {
+        matchId: string;
+        judgeModelId: string;
+        anonymousWinner: 'MODEL_A' | 'MODEL_B' | null;
+        votedFor: string | null;
+        confidence: number | null;
+        valid: boolean;
+        error: string | null;
+    };
+    'match.completed': {
+        matchId: string;
+        taskId: string;
+        winnerModelId: string | null;
+        outcome: 'judged' | 'forfeit' | 'no-contest';
+        costUsd: number;
+        eloAfter: Record<string, number>;
+        completed: number;
+        total: number;
+    };
+    'run.budget-stopped': {
+        runId: string;
+        completed: number;
+        costUsd: number;
+        maxCostUsd: number;
+    };
+    'run.cancellation-requested': {
+        runId: string;
+        completed: number;
+        costUsd: number;
+    };
+    'run.cancelled': {
+        runId: string;
+        completed: number;
+        costUsd: number;
+    };
+    'run.completed': {
+        runId: string;
+        completed: number;
+        costUsd: number;
+        stoppedForBudget: boolean;
+    };
+    'run.failed': {
+        error: string;
+    };
+}
+type ArenaEventType = keyof ArenaEventDataMap;
+type ArenaEvent = {
+    [Type in ArenaEventType]: {
+        id: string;
+        type: Type;
+        timestamp: string;
+        data: ArenaEventDataMap[Type];
+    };
+}[ArenaEventType];
+type ArenaEventInput = {
+    [Type in ArenaEventType]: Omit<Extract<ArenaEvent, {
+        type: Type;
+    }>, 'timestamp'>;
+}[ArenaEventType];
+type ArenaEventSink = (event: ArenaEvent) => void;
+
+declare const ModelRoleSchema: z.ZodEnum<["competitor", "judge"]>;
+type ModelRole = z.infer<typeof ModelRoleSchema>;
+interface ModelRequestPolicy {
+    maxTokens: number;
+    temperature: number;
+    reasoningEffort: 'high' | 'medium' | 'low';
+    excludeReasoning: boolean;
+    timeoutMs: number;
+}
+interface ModelRegistryEntry {
+    id: string;
+    canonicalSlug: string;
+    displayName: string;
+    vendor: string;
+    role: ModelRole;
+    enabled: boolean;
+    request: ModelRequestPolicy;
+}
+declare const ModelCompletionSchema: z.ZodObject<{
+    generationId: z.ZodString;
+    content: z.ZodString;
+    inputTokens: z.ZodNumber;
+    outputTokens: z.ZodNumber;
+    reasoningTokens: z.ZodOptional<z.ZodNumber>;
+    costUsd: z.ZodNumber;
+    latencyMs: z.ZodNumber;
+    finishReason: z.ZodString;
+    attempts: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    generationId: string;
+    content: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    latencyMs: number;
+    finishReason: string;
+    reasoningTokens?: number | undefined;
+    attempts?: number | undefined;
+}, {
+    generationId: string;
+    content: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    latencyMs: number;
+    finishReason: string;
+    reasoningTokens?: number | undefined;
+    attempts?: number | undefined;
+}>;
+type ModelCompletion = z.infer<typeof ModelCompletionSchema>;
+interface ChatRequest {
+    model: ModelRegistryEntry;
+    system: string;
+    user: string;
+    structured?: boolean;
+    /** Execution-only cancellation; never serialized into a run manifest or provider payload. */
+    signal?: AbortSignal;
+    /** Called with the accumulated visible text as it streams in (throttled). */
+    onDelta?: (text: string) => void;
+}
+interface OpenRouterGateway {
+    complete(request: ChatRequest): Promise<ModelCompletion>;
+    validateModel(model: ModelRegistryEntry, signal?: AbortSignal): Promise<void>;
+}
+
+interface ArenaRunConfig {
+    category: BenchmarkCategory;
+    seed: string;
+    matches: number;
+    maxCostUsd: number;
+    resume: boolean;
+    /** Omit to use every enabled competitor in the registry. */
+    competitorIds?: readonly string[];
+    /** Abort early when most matches contain a failed response. */
+    healthStop?: boolean;
+}
+interface ArenaExecutionOptions {
+    /** Execution-only cancellation; deliberately excluded from the canonical manifest. */
+    signal?: AbortSignal;
+    /**
+     * Invoked after each match result is journaled, before the next match
+     * starts. Used to publish results incrementally instead of only at run end.
+     * Must not throw; handle and log failures internally.
+     */
+    onMatchResult?: (result: MatchResult) => Promise<void> | void;
+}
+interface ArenaRunResult {
+    runId: string;
+    completed: number;
+    costUsd: number;
+    stoppedForBudget: boolean;
+    cancelled: boolean;
+}
+interface ScheduledMatch {
+    id: string;
+    runId: string;
+    index: number;
+    seed: string;
+    category: BenchmarkCategory;
+    taskId: string;
+    modelA: string;
+    modelB: string;
+}
+declare const CompetitorSuccessSchema: z.ZodObject<{
+    generationId: z.ZodString;
+    content: z.ZodString;
+    inputTokens: z.ZodNumber;
+    outputTokens: z.ZodNumber;
+    reasoningTokens: z.ZodOptional<z.ZodNumber>;
+    costUsd: z.ZodNumber;
+    latencyMs: z.ZodNumber;
+    finishReason: z.ZodString;
+    attempts: z.ZodOptional<z.ZodNumber>;
+} & {
+    modelId: z.ZodString;
+    success: z.ZodLiteral<true>;
+}, "strip", z.ZodTypeAny, {
+    generationId: string;
+    content: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    latencyMs: number;
+    finishReason: string;
+    modelId: string;
+    success: true;
+    reasoningTokens?: number | undefined;
+    attempts?: number | undefined;
+}, {
+    generationId: string;
+    content: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    latencyMs: number;
+    finishReason: string;
+    modelId: string;
+    success: true;
+    reasoningTokens?: number | undefined;
+    attempts?: number | undefined;
+}>;
+type CompetitorSuccess = z.infer<typeof CompetitorSuccessSchema>;
+declare const CompetitorFailureSchema: z.ZodObject<{
+    modelId: z.ZodString;
+    success: z.ZodLiteral<false>;
+    error: z.ZodString;
+    latencyMs: z.ZodNumber;
+    attempts: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    latencyMs: number;
+    modelId: string;
+    success: false;
+    error: string;
+    attempts?: number | undefined;
+}, {
+    latencyMs: number;
+    modelId: string;
+    success: false;
+    error: string;
+    attempts?: number | undefined;
+}>;
+type CompetitorFailure = z.infer<typeof CompetitorFailureSchema>;
+type CompetitorResponse = CompetitorSuccess | CompetitorFailure;
+declare const CompetitorResponseSchema: z.ZodType<CompetitorResponse>;
+declare const JudgeVerdictSchema: z.ZodObject<{
+    winner: z.ZodEnum<["MODEL_A", "MODEL_B"]>;
+    confidence: z.ZodNumber;
+    rationale: z.ZodString;
+    criteria: z.ZodObject<{
+        correctness: z.ZodString;
+        grounding: z.ZodString;
+        constraintHandling: z.ZodString;
+        completeness: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        correctness: string;
+        grounding: string;
+        constraintHandling: string;
+        completeness: string;
+    }, {
+        correctness: string;
+        grounding: string;
+        constraintHandling: string;
+        completeness: string;
+    }>;
+    violations: z.ZodArray<z.ZodString, "many">;
+}, "strip", z.ZodTypeAny, {
+    winner: "MODEL_A" | "MODEL_B";
+    confidence: number;
+    rationale: string;
+    criteria: {
+        correctness: string;
+        grounding: string;
+        constraintHandling: string;
+        completeness: string;
+    };
+    violations: string[];
+}, {
+    winner: "MODEL_A" | "MODEL_B";
+    confidence: number;
+    rationale: string;
+    criteria: {
+        correctness: string;
+        grounding: string;
+        constraintHandling: string;
+        completeness: string;
+    };
+    violations: string[];
+}>;
+type JudgeVerdict = z.infer<typeof JudgeVerdictSchema>;
+/**
+ * Provider-safe JSON Schema subset generated from the same verdict shape.
+ * Runtime Zod parsing above enforces the local string and array limits.
+ */
+declare const JUDGE_VERDICT_TRANSPORT_SCHEMA: {
+    readonly type: "object";
+    readonly additionalProperties: false;
+    readonly required: readonly ["winner", "confidence", "rationale", "criteria", "violations"];
+    readonly properties: {
+        readonly winner: {
+            readonly type: "string";
+            readonly enum: readonly ["MODEL_A", "MODEL_B"];
+        };
+        readonly confidence: {
+            readonly type: "number";
+            readonly minimum: 0;
+            readonly maximum: 1;
+        };
+        readonly rationale: {
+            readonly type: "string";
+        };
+        readonly criteria: {
+            readonly type: "object";
+            readonly additionalProperties: false;
+            readonly required: readonly ["correctness", "grounding", "constraintHandling", "completeness"];
+            readonly properties: {
+                readonly correctness: {
+                    readonly type: "string";
+                };
+                readonly grounding: {
+                    readonly type: "string";
+                };
+                readonly constraintHandling: {
+                    readonly type: "string";
+                };
+                readonly completeness: {
+                    readonly type: "string";
+                };
+            };
+        };
+        readonly violations: {
+            readonly type: "array";
+            readonly items: {
+                readonly type: "string";
+            };
+        };
+    };
+};
+interface JudgeVote {
+    judgeModelId: string;
+    modelAIdentity: string;
+    modelBIdentity: string;
+    verdict: JudgeVerdict | null;
+    winnerModelId: string | null;
+    completion: ModelCompletion | null;
+    error?: string;
+}
+interface PanelDecision {
+    winnerModelId: string | null;
+    validVotes: number;
+    votesByModel: Record<string, number>;
+    agreement: 'unanimous' | 'split' | 'insufficient';
+    votes: JudgeVote[];
+}
+interface EloState {
+    ratings: Record<string, number>;
+    points: Record<string, number>;
+}
+interface MatchResult {
+    methodologyVersion: string;
+    runId: string;
+    /** Absent on legacy journal lines written before run manifests. */
+    runManifestHash?: string;
+    matchId: string;
+    scheduleIndex: number;
+    seed: string;
+    timestamp: string;
+    task: {
+        id: string;
+        version: string;
+        /** Absent on reasoning-only legacy lines. */
+        category?: BenchmarkCategory;
+        cluster: string;
+        publicHash: string;
+        privateHash: string;
+    };
+    competitors: {
+        modelA: string;
+        modelB: string;
+        responseA: CompetitorResponse;
+        responseB: CompetitorResponse;
+    };
+    outcome: 'judged' | 'forfeit' | 'no-contest';
+    winnerModelId: string | null;
+    panel: PanelDecision | null;
+    eloBefore: Record<string, number>;
+    eloAfter: Record<string, number>;
+    pointAwarded: boolean;
+    matchCostUsd: number;
+}
+declare const JudgeVoteSchema: z.ZodType<JudgeVote>;
+declare const PanelDecisionSchema: z.ZodType<PanelDecision>;
+declare const MatchResultSchema: z.ZodType<MatchResult>;
+declare function competitorCost(response: CompetitorResponse): number;
+declare function competitorOutputTokens(response: CompetitorResponse): number;
+declare function competitorReasoningTokens(response: CompetitorResponse): number | undefined;
+declare function competitorContent(response: CompetitorResponse): string;
+
+interface LeaderboardEntry {
+    rank: number;
+    modelId: string;
+    displayName: string;
+    elo: number;
+    points: number;
+    wins: number;
+    losses: number;
+    forfeits: number;
+    matches: number;
+    winRate: number;
+    unanimousWins: number;
+    totalCostUsd: number;
+    byCluster: Partial<Record<string, {
+        wins: number;
+        losses: number;
+    }>>;
+}
+interface ArenaSnapshot {
+    version: '0.2.0';
+    methodologyVersion: string;
+    category: BenchmarkCategory;
+    generatedAt: string;
+    initialElo: number;
+    kFactor: number;
+    leaderboard: LeaderboardEntry[];
+    matches: MatchResult[];
+}
+
+export { type ArenaEvent as A, competitorOutputTokens as B, type ChatRequest as C, competitorReasoningTokens as D, type EloState as E, JUDGE_VERDICT_TRANSPORT_SCHEMA as J, type LeaderboardEntry as L, type MatchResult as M, type OpenRouterGateway as O, type PanelDecision as P, type ScheduledMatch as S, type ArenaEventDataMap as a, type ArenaEventInput as b, type ArenaEventSink as c, type ArenaEventType as d, type ArenaExecutionOptions as e, type ArenaRunConfig as f, type ArenaRunResult as g, type ArenaSnapshot as h, type CompetitorFailure as i, type CompetitorResponse as j, CompetitorResponseSchema as k, type CompetitorSuccess as l, type JudgeVerdict as m, JudgeVerdictSchema as n, type JudgeVote as o, JudgeVoteSchema as p, MatchResultSchema as q, type ModelCompletion as r, ModelCompletionSchema as s, type ModelRegistryEntry as t, type ModelRequestPolicy as u, type ModelRole as v, ModelRoleSchema as w, PanelDecisionSchema as x, competitorContent as y, competitorCost as z };

package/dist/tasks-CpaCJ6JE.d.cts

@@ -0,0 +1,151 @@
+import { z } from 'zod';
+
+declare const METHODOLOGY_VERSION = "arena-v0.3.0";
+declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination"]>;
+type BenchmarkCategory = z.infer<typeof BenchmarkCategorySchema>;
+declare const CATEGORIES: ["reasoning", "hallucination"];
+declare const CATEGORY_CLUSTERS: Record<BenchmarkCategory, readonly string[]>;
+declare const CATEGORY_META: Record<BenchmarkCategory, {
+    label: string;
+    tagline: string;
+}>;
+
+declare const TaskArtifactSchema: z.ZodObject<{
+    id: z.ZodString;
+    type: z.ZodEnum<["code", "log", "config", "spec", "diff", "table", "note"]>;
+    label: z.ZodString;
+    content: z.ZodString;
+}, "strict", z.ZodTypeAny, {
+    type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+    id: string;
+    content: string;
+    label: string;
+}, {
+    type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+    id: string;
+    content: string;
+    label: string;
+}>;
+type TaskArtifact = z.infer<typeof TaskArtifactSchema>;
+declare const TaskPublicSchema: z.ZodObject<{
+    id: z.ZodString;
+    version: z.ZodString;
+    category: z.ZodEnum<["reasoning", "hallucination"]>;
+    cluster: z.ZodString;
+    difficulty: z.ZodEnum<["hard", "expert"]>;
+    title: z.ZodString;
+    summary: z.ZodString;
+    prompt: z.ZodString;
+    artifacts: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        type: z.ZodEnum<["code", "log", "config", "spec", "diff", "table", "note"]>;
+        label: z.ZodString;
+        content: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }, {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }>, "many">;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+}, "strict", z.ZodTypeAny, {
+    id: string;
+    version: string;
+    category: "reasoning" | "hallucination";
+    cluster: string;
+    difficulty: "hard" | "expert";
+    title: string;
+    summary: string;
+    prompt: string;
+    artifacts: {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }[];
+    tags: string[];
+}, {
+    id: string;
+    version: string;
+    category: "reasoning" | "hallucination";
+    cluster: string;
+    difficulty: "hard" | "expert";
+    title: string;
+    summary: string;
+    prompt: string;
+    artifacts: {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }[];
+    tags?: string[] | undefined;
+}>;
+type TaskPublic = z.infer<typeof TaskPublicSchema>;
+declare const TaskPrivateSchema: z.ZodObject<{
+    id: z.ZodString;
+    version: z.ZodString;
+    expectedResolution: z.ZodString;
+    requiredEvidence: z.ZodArray<z.ZodString, "many">;
+    disqualifyingErrors: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    rubric: z.ZodObject<{
+        correctness: z.ZodString;
+        evidenceGrounding: z.ZodString;
+        constraintHandling: z.ZodString;
+        completeness: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    }, {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    }>;
+}, "strict", z.ZodTypeAny, {
+    id: string;
+    version: string;
+    expectedResolution: string;
+    requiredEvidence: string[];
+    disqualifyingErrors: string[];
+    rubric: {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    };
+}, {
+    id: string;
+    version: string;
+    expectedResolution: string;
+    requiredEvidence: string[];
+    rubric: {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    };
+    disqualifyingErrors?: string[] | undefined;
+}>;
+type TaskPrivate = z.infer<typeof TaskPrivateSchema>;
+interface ArenaTask {
+    public: TaskPublic;
+    /** Null when loaded without a private overlay. */
+    private: TaskPrivate | null;
+    publicHash: string;
+    privateHash: string | null;
+}
+/** A task whose hidden reference is present — required for judged matches. */
+type CompleteArenaTask = ArenaTask & {
+    private: TaskPrivate;
+    privateHash: string;
+};
+
+export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i };