bridgebench 3.1.0-alpha.0

package/dist/tasks-CpaCJ6JE.d.ts

@@ -0,0 +1,151 @@
+import { z } from 'zod';
+
+declare const METHODOLOGY_VERSION = "arena-v0.3.0";
+declare const BenchmarkCategorySchema: z.ZodEnum<["reasoning", "hallucination"]>;
+type BenchmarkCategory = z.infer<typeof BenchmarkCategorySchema>;
+declare const CATEGORIES: ["reasoning", "hallucination"];
+declare const CATEGORY_CLUSTERS: Record<BenchmarkCategory, readonly string[]>;
+declare const CATEGORY_META: Record<BenchmarkCategory, {
+    label: string;
+    tagline: string;
+}>;
+
+declare const TaskArtifactSchema: z.ZodObject<{
+    id: z.ZodString;
+    type: z.ZodEnum<["code", "log", "config", "spec", "diff", "table", "note"]>;
+    label: z.ZodString;
+    content: z.ZodString;
+}, "strict", z.ZodTypeAny, {
+    type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+    id: string;
+    content: string;
+    label: string;
+}, {
+    type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+    id: string;
+    content: string;
+    label: string;
+}>;
+type TaskArtifact = z.infer<typeof TaskArtifactSchema>;
+declare const TaskPublicSchema: z.ZodObject<{
+    id: z.ZodString;
+    version: z.ZodString;
+    category: z.ZodEnum<["reasoning", "hallucination"]>;
+    cluster: z.ZodString;
+    difficulty: z.ZodEnum<["hard", "expert"]>;
+    title: z.ZodString;
+    summary: z.ZodString;
+    prompt: z.ZodString;
+    artifacts: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        type: z.ZodEnum<["code", "log", "config", "spec", "diff", "table", "note"]>;
+        label: z.ZodString;
+        content: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }, {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }>, "many">;
+    tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+}, "strict", z.ZodTypeAny, {
+    id: string;
+    version: string;
+    category: "reasoning" | "hallucination";
+    cluster: string;
+    difficulty: "hard" | "expert";
+    title: string;
+    summary: string;
+    prompt: string;
+    artifacts: {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }[];
+    tags: string[];
+}, {
+    id: string;
+    version: string;
+    category: "reasoning" | "hallucination";
+    cluster: string;
+    difficulty: "hard" | "expert";
+    title: string;
+    summary: string;
+    prompt: string;
+    artifacts: {
+        type: "code" | "log" | "config" | "spec" | "diff" | "table" | "note";
+        id: string;
+        content: string;
+        label: string;
+    }[];
+    tags?: string[] | undefined;
+}>;
+type TaskPublic = z.infer<typeof TaskPublicSchema>;
+declare const TaskPrivateSchema: z.ZodObject<{
+    id: z.ZodString;
+    version: z.ZodString;
+    expectedResolution: z.ZodString;
+    requiredEvidence: z.ZodArray<z.ZodString, "many">;
+    disqualifyingErrors: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    rubric: z.ZodObject<{
+        correctness: z.ZodString;
+        evidenceGrounding: z.ZodString;
+        constraintHandling: z.ZodString;
+        completeness: z.ZodString;
+    }, "strict", z.ZodTypeAny, {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    }, {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    }>;
+}, "strict", z.ZodTypeAny, {
+    id: string;
+    version: string;
+    expectedResolution: string;
+    requiredEvidence: string[];
+    disqualifyingErrors: string[];
+    rubric: {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    };
+}, {
+    id: string;
+    version: string;
+    expectedResolution: string;
+    requiredEvidence: string[];
+    rubric: {
+        correctness: string;
+        constraintHandling: string;
+        completeness: string;
+        evidenceGrounding: string;
+    };
+    disqualifyingErrors?: string[] | undefined;
+}>;
+type TaskPrivate = z.infer<typeof TaskPrivateSchema>;
+interface ArenaTask {
+    public: TaskPublic;
+    /** Null when loaded without a private overlay. */
+    private: TaskPrivate | null;
+    publicHash: string;
+    privateHash: string | null;
+}
+/** A task whose hidden reference is present — required for judged matches. */
+type CompleteArenaTask = ArenaTask & {
+    private: TaskPrivate;
+    privateHash: string;
+};
+
+export { type ArenaTask as A, type BenchmarkCategory as B, CATEGORIES as C, METHODOLOGY_VERSION as M, type TaskArtifact as T, BenchmarkCategorySchema as a, CATEGORY_CLUSTERS as b, CATEGORY_META as c, type CompleteArenaTask as d, TaskArtifactSchema as e, type TaskPrivate as f, TaskPrivateSchema as g, type TaskPublic as h, TaskPublicSchema as i };

package/dist/tasks.cjs

@@ -0,0 +1,22 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true});
+
+
+
+
+
+
+
+
+var _chunk4TWPCPRPcjs = require('./chunk-4TWPCPRP.cjs');
+require('./chunk-CIXITJW6.cjs');
+
+
+
+
+
+
+
+
+
+exports.TASKS_PER_CATEGORY = _chunk4TWPCPRPcjs.TASKS_PER_CATEGORY; exports.TASKS_PER_CLUSTER = _chunk4TWPCPRPcjs.TASKS_PER_CLUSTER; exports.TaskLoader = _chunk4TWPCPRPcjs.TaskLoader; exports.buildCompetitorPrompt = _chunk4TWPCPRPcjs.buildCompetitorPrompt; exports.competitorPromptPolicyHash = _chunk4TWPCPRPcjs.competitorPromptPolicyHash; exports.defaultTaskRoot = _chunk4TWPCPRPcjs.defaultTaskRoot; exports.mergePrivateHalves = _chunk4TWPCPRPcjs.mergePrivateHalves; exports.validatePublicTaskFile = _chunk4TWPCPRPcjs.validatePublicTaskFile;
+//# sourceMappingURL=tasks.cjs.map

package/dist/tasks.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/matthewmiller/Desktop/bridgemind/bridgebench/dist/tasks.cjs"],"names":[],"mappings":"AAAA;AACE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACF,wDAA6B;AAC7B,gCAA6B;AAC7B;AACE;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACF,6hBAAC","file":"/Users/matthewmiller/Desktop/bridgemind/bridgebench/dist/tasks.cjs"}

package/dist/tasks.d.cts

@@ -0,0 +1,39 @@
+import { f as TaskPrivate, B as BenchmarkCategory, d as CompleteArenaTask, A as ArenaTask } from './tasks-CpaCJ6JE.cjs';
+import 'zod';
+
+declare const TASKS_PER_CATEGORY = 12;
+declare const TASKS_PER_CLUSTER = 2;
+declare function defaultTaskRoot(category: BenchmarkCategory): string;
+declare function validatePublicTaskFile(filePath: string): Promise<ArenaTask>;
+declare class TaskLoader {
+    readonly category: BenchmarkCategory;
+    private readonly root;
+    private readonly privateDir;
+    constructor(category: BenchmarkCategory, root?: string, privateRoot?: string);
+    get hasPrivate(): boolean;
+    loadAll(options: {
+        requirePrivate: true;
+    }): Promise<CompleteArenaTask[]>;
+    loadAll(options?: {
+        requirePrivate?: boolean;
+    }): Promise<ArenaTask[]>;
+}
+interface PrivateHalf {
+    value: TaskPrivate;
+    /** SHA-256 of the exact private YAML the judges receive — the journal's task identity. */
+    hash: string;
+}
+/**
+ * Completes public tasks with private halves supplied by an external store
+ * (e.g. the API's rubric table) instead of the on-disk overlay. Pairing
+ * rules match the disk loader: matching id/version and every
+ * requiredEvidence entry naming a real public artifact.
+ */
+declare function mergePrivateHalves(tasks: readonly ArenaTask[], privateHalves: readonly PrivateHalf[]): CompleteArenaTask[];
+declare function competitorPromptPolicyHash(category: BenchmarkCategory): string;
+declare function buildCompetitorPrompt(task: ArenaTask): {
+    system: string;
+    user: string;
+};
+
+export { type PrivateHalf, TASKS_PER_CATEGORY, TASKS_PER_CLUSTER, TaskLoader, buildCompetitorPrompt, competitorPromptPolicyHash, defaultTaskRoot, mergePrivateHalves, validatePublicTaskFile };

package/dist/tasks.d.ts

@@ -0,0 +1,39 @@
+import { f as TaskPrivate, B as BenchmarkCategory, d as CompleteArenaTask, A as ArenaTask } from './tasks-CpaCJ6JE.js';
+import 'zod';
+
+declare const TASKS_PER_CATEGORY = 12;
+declare const TASKS_PER_CLUSTER = 2;
+declare function defaultTaskRoot(category: BenchmarkCategory): string;
+declare function validatePublicTaskFile(filePath: string): Promise<ArenaTask>;
+declare class TaskLoader {
+    readonly category: BenchmarkCategory;
+    private readonly root;
+    private readonly privateDir;
+    constructor(category: BenchmarkCategory, root?: string, privateRoot?: string);
+    get hasPrivate(): boolean;
+    loadAll(options: {
+        requirePrivate: true;
+    }): Promise<CompleteArenaTask[]>;
+    loadAll(options?: {
+        requirePrivate?: boolean;
+    }): Promise<ArenaTask[]>;
+}
+interface PrivateHalf {
+    value: TaskPrivate;
+    /** SHA-256 of the exact private YAML the judges receive — the journal's task identity. */
+    hash: string;
+}
+/**
+ * Completes public tasks with private halves supplied by an external store
+ * (e.g. the API's rubric table) instead of the on-disk overlay. Pairing
+ * rules match the disk loader: matching id/version and every
+ * requiredEvidence entry naming a real public artifact.
+ */
+declare function mergePrivateHalves(tasks: readonly ArenaTask[], privateHalves: readonly PrivateHalf[]): CompleteArenaTask[];
+declare function competitorPromptPolicyHash(category: BenchmarkCategory): string;
+declare function buildCompetitorPrompt(task: ArenaTask): {
+    system: string;
+    user: string;
+};
+
+export { type PrivateHalf, TASKS_PER_CATEGORY, TASKS_PER_CLUSTER, TaskLoader, buildCompetitorPrompt, competitorPromptPolicyHash, defaultTaskRoot, mergePrivateHalves, validatePublicTaskFile };

package/dist/tasks.js

@@ -0,0 +1,22 @@
+import {
+  TASKS_PER_CATEGORY,
+  TASKS_PER_CLUSTER,
+  TaskLoader,
+  buildCompetitorPrompt,
+  competitorPromptPolicyHash,
+  defaultTaskRoot,
+  mergePrivateHalves,
+  validatePublicTaskFile
+} from "./chunk-JTVNKSMO.js";
+import "./chunk-UECBSKTD.js";
+export {
+  TASKS_PER_CATEGORY,
+  TASKS_PER_CLUSTER,
+  TaskLoader,
+  buildCompetitorPrompt,
+  competitorPromptPolicyHash,
+  defaultTaskRoot,
+  mergePrivateHalves,
+  validatePublicTaskFile
+};
+//# sourceMappingURL=tasks.js.map

package/dist/tasks.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/docs/README.md

@@ -0,0 +1,25 @@
+# BridgeBench documentation
+
+Start with the contract, then follow the workflow you need.
+
+## Protocol
+
+- [Methodology](methodology.md) — scheduling, blind judging, outcomes, and Elo.
+- [Replay Elo](replay-elo.md) — verify a journal and reproduce its ladder.
+- [Private packs](private-packs.md) — hidden-reference data flow and contamination controls.
+
+## Contributing
+
+- [Task authoring](task-authoring.md) — public schema, private schema, clusters, and validation.
+- [Glossary](glossary.md) — the terms used by the engine, CLI, and published reports.
+- [Repository contribution guide](../CONTRIBUTING.md) — local checks and pull-request expectations.
+
+## Canonical sources
+
+- Model roster and request policy: [`src/models.ts`](../src/models.ts)
+- Category and methodology constants: [`src/types.ts`](../src/types.ts)
+- Task-pack invariants: [`src/tasks.ts`](../src/tasks.ts)
+- Journal verification: [`src/verification.ts`](../src/verification.ts)
+- CLI workflows: `npm run arena -- --help` and `npm run tasks -- --help`
+
+When prose and executable validation disagree, treat the code as the current behavior and open an issue to repair the documentation.

package/docs/glossary.md

@@ -0,0 +1,49 @@
+# Glossary
+
+## Arena
+
+One independent benchmark category with its own task pack, journal, Elo ladder, and leaderboard. Ratings never cross arena boundaries.
+
+## Competitor
+
+A model answering the public task context. Competitors receive no hidden reference and do not know their opponent.
+
+## Judge
+
+A model comparing two anonymized competitor responses against the hidden reference. Judges are fixed, independent, and ineligible as competitors.
+
+## Public task
+
+The prompt and artifacts visible to competitors and committed under `tasks/<category>/public/`.
+
+## Hidden reference
+
+The expected resolution, required evidence, disqualifying errors, and rubric used by judges. Active references live outside the public repository.
+
+## Private overlay
+
+The separate checkout containing active hidden references. `BRIDGEBENCH_PRIVATE_DIR` points the engine to it.
+
+## Run manifest
+
+The canonical inputs that define one deterministic schedule: methodology, engine version, category, seed, match count, model roster and policy, task hashes, and prompt-policy hashes.
+
+## Journal
+
+The append-only local execution record. Each JSONL line contains one completed match and references its run manifest.
+
+## Snapshot
+
+A verified report derived from a journal replay. Snapshots are replaceable; journals are not.
+
+## Point
+
+One win awarded for a judged majority or forfeit. No-contests award no point.
+
+## Elo
+
+The per-arena rating derived in journal order. Ratings start at 1000 and use K=32.
+
+## Published replica
+
+The bridgebench.ai API copy of verified engine output. Publishing does not replace the local journal as the execution authority.

package/docs/methodology.md

@@ -0,0 +1,58 @@
+# Methodology — the arena contract
+
+Every ranking BridgeBench publishes is the fold of an append-only match journal. This document specifies the full protocol that produces a journal line, so any published ladder can be audited or reproduced. The engine stamps every line with `methodologyVersion` (currently `arena-v0.3.0`, `src/contracts/categories.ts`); results from different methodology versions never mix silently.
+
+## Run identity and scheduling
+
+A **run** is deterministic from a versioned manifest:
+
+- The manifest binds the category, seed, match count, engine and methodology versions, sorted competitor and judge slugs, request policies, task hashes, and competitor/judge prompt-policy hashes. Runs may select an explicit roster of at least two unique, enabled competitors; omitting one selects every enabled competitor.
+- The run ID is a SHA-256 prefix of the canonical manifest. Every journal line records the full manifest hash.
+- The scheduler builds all `(competitorA, competitorB, task)` combinations and greedily picks the next match by lowest exposure, ordered by: max per-model exposure → summed model exposure → task exposure → pair exposure. Ties break by a seeded mulberry32 PRNG, so balance is reproducible, not incidental (`src/scheduler.ts`).
+- Which competitor sits on side A is randomized per match from the same seeded stream.
+- Match IDs are stable hashes; re-running a schedule with `--resume` skips journaled match IDs exactly. Repeating a completed schedule without `--resume` is rejected.
+
+## Competitor execution
+
+- Both competitors receive byte-identical context: a category-specific system prompt plus the task title, summary, prompt, and all public artifacts inline (`src/tasks.ts`). They are never told they are in a match, and the system prompt forbids revealing model identity.
+- Requests run concurrently over OpenRouter with pinned slugs (`latest` aliases prohibited). Before any paid run the CLI re-validates the selected competitors and all three judges against the live catalog, and confirms judges still support structured output.
+- Transport is fail-closed: 3 attempts on retryable errors, per-request watchdog timeouts, and a hard prompt-size cap. A competitor that exhausts retries **forfeits** (the survivor wins); two failures make a **no-contest** (no point, no Elo movement).
+
+## Blind three-judge panel
+
+Judged matches go to a fixed, cross-vendor panel of three model judges. Independence is enforced structurally:
+
+1. **Anonymization.** Before an answer crosses into a judge prompt, explicit competitor identity terms (IDs, canonical slugs, display names, vendors) and family names (Claude, GPT, Kimi, Opus, …) are redacted (`src/judges.ts`). Judges see only `Model A` and `Model B`.
+2. **Per-judge order permutation.** Whether A/B are swapped for a given judge is a deterministic hash of `matchId|judgeId`, so position bias can't systematically favor one side and the permutation is replayable.
+3. **Isolation.** A judge receives the task, the hidden reference (expected resolution, required evidence, disqualifying errors, rubric), and the two anonymous answers — never identities, ratings, costs, or the other judges' votes.
+4. **Forced choice, structured output.** Verdicts are JSON-schema enforced (`MODEL_A` or `MODEL_B`, confidence, rationale, four-criteria commentary, violations). A malformed verdict gets one retry, then the judge **abstains**.
+5. **Majority.** Two valid votes for the same side decide the match. Agreement is recorded as `unanimous` (three winner votes), `split` (exactly two winner votes, including a 2–0 panel with one abstention), or `insufficient` (no majority → no-contest).
+
+Judge prompts are category-specific: the reasoning panel treats hedging on a determinable deliverable as an error and disqualifying errors as near-decisive; the hallucination panel weighs fabrication heaviest and also penalizes refusing deliverables the artifacts do support.
+
+The hidden reference is **judging context, not an oracle** — no deterministic task score picks the winner, and no model-generated code is ever executed.
+
+## Scoring
+
+- A win awards exactly one point and one Elo update. Elo starts at 1000 with K=32 (`src/elo.ts`). Forfeit wins score identically to judged wins; no-contests change nothing.
+- Each arena keeps its own ladder. Reasoning Elo and hallucination Elo never mix.
+
+## The journal is the source of truth
+
+- Every completed match is appended to `results/<category>/journal.jsonl` **before** any report is rebuilt. Snapshots and leaderboards are derived atomically and may be deleted and rebuilt at any time.
+- Every line records: methodology version, run ID and manifest hash, seed, schedule index, the task's ID/version/cluster and the SHA-256 of **both** task halves (`publicHash`, `privateHash`), both full competitor responses with token/cost/latency accounting, the panel's votes and rationales, `eloBefore`/`eloAfter` for both models, and the match cost.
+- The task hashes make drift externally detectable: if a task file changes after a match ran, the recorded hash no longer matches the file. When a retired pack's private halves are published, `privateHash` proves they are byte-identical to what the judges actually used.
+- OpenRouter generation IDs are journaled per response; `npm run arena -- generation <id>` fetches the provider's own token/cost record for independent cross-checking.
+
+`npm run arena -- verify` schema-validates every line, checks outcome and panel invariants, replays points and Elo, and validates referenced run manifests. Reports and resume state use the same verified fold. See [replay-elo.md](replay-elo.md) for the complete workflow.
+
+## Adversarial-input posture
+
+Candidate answers are treated as untrusted data end to end: judge prompts explicitly refuse instructions embedded in answers, verdicts are schema-validated, the dashboard renders model output only as escaped text, structured logs redact API keys, and journal/log/snapshot files are written mode 0600.
+
+## Operational guards
+
+- **Budget stop**: spend is checked before each match against `--max-cost-usd` (default $25); a stopped run resumes deterministically with `--resume`.
+- **Cancellation**: Ctrl-C or the dashboard cancel action aborts active competitor and judge requests, emits `run.cancellation-requested` followed by terminal `run.cancelled`, and stops before the next match. Completed journal lines remain resumable; a partial match is never appended.
+- **Health stop**: once ≥4 matches complete and ≥50% contain a failed competitor response, the run halts instead of journaling a batch of silent no-contests (`--no-health-stop` overrides).
+- **Triage**: every run auto-prints an anomaly report (failed requests, suspiciously fast responses, low output, truncation, unreported reasoning tokens, zero-cost accounting, judge abstentions) — `npm run triage` re-analyzes any journal.

package/docs/private-packs.md

@@ -0,0 +1,74 @@
+# Private packs — the hidden-reference overlay
+
+Task prompts are public. Active expected resolutions, trap inventories, disqualifying-error lists, and rubrics are withheld from the public repository to reduce contamination risk. This document defines where those hidden references travel, who stores them, and how they are retired.
+
+## Layout
+
+The overlay is a separate, private checkout that mirrors this repo's `tasks/` layout:
+
+```text
+<private-checkout>/
+└── tasks/
+    ├── reasoning/private/<task-id>.yaml       # one per public task, same filename
+    └── hallucination/private/<task-id>.yaml
+```
+
+Point the engine at it with:
+
+```bash
+export BRIDGEBENCH_PRIVATE_DIR=/path/to/private-checkout
+```
+
+Resolution order in `src/tasks.ts`: an explicit `privateRoot` constructor argument → the `BRIDGEBENCH_PRIVATE_DIR` overlay → a repo-local `tasks/<category>/private/` directory (maintainer setups only; the public repo never contains one).
+
+## What works without the overlay
+
+| Works from public halves alone                                   | Needs the overlay                                         |
+| ---------------------------------------------------------------- | --------------------------------------------------------- |
+| `npm run tasks -- validate` (public schema + pack balance)       | `npm run arena -- run` (judges need the hidden reference) |
+| `npm run report` (rebuild snapshots/leaderboards from a journal) | Starting a run from the dashboard                         |
+| `npm run triage`, `arena generation <id>`                        | Authoring/validating private halves                       |
+| `npm run arena -- publish` (journal sync; admin key required)    | `npm run tasks -- publish` (the API stores both halves)   |
+| The dashboard's Leaderboard and Matches views                    |                                                           |
+| `npm test` (offline suites synthesize a stand-in reference)      |                                                           |
+
+When a judged run is attempted without the overlay, the loader fails closed with a pointer to this document.
+
+## Data flow and trust boundaries
+
+An active hidden reference is not confined to the overlay during a run:
+
+1. `TaskLoader` reads it into the local operator process.
+2. `JudgePanel` includes it in requests to the three configured model judges.
+3. The local journal stores its SHA-256 hash, not its contents.
+4. `tasks publish` may send it to the configured BridgeBench API, where it is stored outside public read endpoints.
+
+The operator machine, configured model providers, transport provider, and private API store are therefore inside the active-pack trust boundary. Their retention and access policies matter. Withholding references from Git history and public issues reduces exposure; it cannot prove that a third party never retains or trains on transmitted content.
+
+If an active reference leaks:
+
+1. stop paid runs for that pack;
+2. record the affected task IDs and first-known exposure time;
+3. rotate the compromised tasks and bump their versions;
+4. start a new run manifest and journal;
+5. disclose the incident with the retired references when doing so no longer exposes live tasks.
+
+## Contamination guard
+
+Three mechanisms keep hidden references out of the public repo:
+
+- `.gitignore` ignores `tasks/*/private/`, so a casually dropped-in overlay can't be staged by `git add -A`.
+- CI fails if any tracked path matches `tasks/*/private/`.
+- CI parses tracked task YAML and rejects every private-only top-level field, including `expectedResolution`, `requiredEvidence`, `disqualifyingErrors`, and `rubric`.
+
+When both halves load, `tasks validate` also verifies pairing: matching `id`/`version` and that every `requiredEvidence` entry names a real public artifact.
+
+## Publish-at-retirement
+
+Hidden references are withheld only while their pack is live:
+
+1. **While live** — private halves are absent from the public repo. They remain in the overlay and the private systems named in the trust boundary above. Every journal line records `privateHash`, the SHA-256 of the exact private YAML the judges received.
+2. **At retirement** — when a pack rotates out, its private halves are published in full alongside the pack's final journal.
+3. **Verification** — anyone can hash the published private halves and compare them with each journaled `privateHash`. `arena verify` checks the journal's match and Elo invariants; the hash comparison checks task identity.
+
+If a published score doesn't reproduce, open an issue.

package/docs/replay-elo.md

@@ -0,0 +1,79 @@
+# Replay the Elo yourself
+
+A BridgeBench ladder is not a claim — it is the fold of a journal you can re-run. This is the recipe.
+
+## What you need
+
+- The arena's journal: `results/<category>/journal.jsonl`, one JSON object per completed match, in append order. Published season journals are downloadable from [bridgebench.ai](https://bridgebench.ai).
+- The matching `runs/` directory for journals that reference versioned run manifests.
+- No API key and no private overlay.
+
+Verify the included synthetic journal:
+
+```bash
+npm run arena -- verify \
+  --category reasoning \
+  --journal test/fixtures/journals/valid.jsonl
+```
+
+For a downloaded journal, place its run manifests in a sibling `runs/` directory or pass `--manifests-dir <path>`.
+
+## The journal line, abridged
+
+```jsonc
+{
+  "methodologyVersion": "arena-v0.3.0",
+  "runId": "run-…", "matchId": "…", "scheduleIndex": 3, "seed": "…",
+  "task": { "id": "…", "version": "2.0.0", "category": "reasoning",
+            "cluster": "…", "publicHash": "sha256…", "privateHash": "sha256…" },
+  "competitors": { "modelA": "…", "modelB": "…", "responseA": { /* full text + tokens + cost */ }, "responseB": { … } },
+  "outcome": "judged",              // judged | forfeit | no-contest
+  "winnerModelId": "…",             // null on no-contest
+  "panel": { "votes": [ /* 3 votes: verdict, rationale, resolved winner */ ], "agreement": "unanimous" },
+  "eloBefore": { "modelA-id": 1000, "modelB-id": 1016 },
+  "eloAfter":  { "modelA-id": 1016.4, "modelB-id": 999.6 },
+  "pointAwarded": true,
+  "matchCostUsd": 0.42
+}
+```
+
+## Replay algorithm
+
+Elo is classic: initial rating **1000**, **K = 32**, one update per decided match, applied in journal order.
+
+```
+expected(a, b) = 1 / (1 + 10^((b − a) / 400))
+
+for each line in journal order:
+    if winnerModelId is null: continue          # no-contest — nothing moves
+    a, b   = ratings of modelA, modelB          # default 1000 if unseen
+    ea     = expected(a, b)
+    scoreA = 1 if winnerModelId == modelA else 0
+    ratings[modelA] = a + 32 * (scoreA − ea)
+    ratings[modelB] = b + 32 * ((1 − scoreA) − (1 − ea))
+    points[winnerModelId] += 1
+```
+
+Forfeit wins update Elo exactly like judged wins; `no-contest` lines change nothing. Reasoning and hallucination journals fold separately and never share ratings.
+
+The rating formula lives in [`src/elo.ts`](../src/elo.ts). [`src/verification.ts`](../src/verification.ts) owns the validated fold used by reports, resume state, and the CLI verifier.
+
+## Verification is per-line, not just end-state
+
+Every line records `eloBefore` and `eloAfter` for both models. So you don't just check that your final ladder matches the published one — you can assert, at every single match, that:
+
+1. `eloBefore` equals your running ratings at that point;
+2. `eloAfter` equals your computed update;
+3. the winner is consistent with the panel's votes (two valid votes for the same side), or with a forfeit (exactly one failed response).
+
+`arena verify` rejects a reordered line, edited verdict, inconsistent outcome, wrong point, altered cost, or nudged rating at the first failing line. New journal lines also bind to a versioned run manifest covering the roster, task hashes, prompt policies, methodology, and engine version.
+
+With the repo checked out, `npm run report` verifies the journal before rebuilding `snapshot.json` and `leaderboard.md`; diff them against published snapshots.
+
+## Auditing beyond the math
+
+- **Task drift**: hash the public task file (`sha256` of the raw YAML) and compare with the journaled `publicHash`. For retired packs, do the same with the published private halves against `privateHash` — see [private-packs.md](private-packs.md).
+- **Judging**: each vote carries the judge's rationale and resolved winner; `agreement` must match the winner-vote count (three `unanimous`, exactly two `split`).
+- **Cost/token accounting**: every response carries its OpenRouter generation ID; `npm run arena -- generation <id>` fetches the provider's own record.
+
+If any of it doesn't reproduce, open an issue with the line number.

package/docs/task-authoring.md

@@ -0,0 +1,80 @@
+# Task authoring
+
+Every task is split into a **public half** (what competitors and everyone else sees, in this repo) and a **private half** (the hidden reference judges use, kept in the private overlay — see [private-packs.md](private-packs.md)). Both halves share a filename, ID, and version; the loader refuses mismatches.
+
+## Domain invariant
+
+**Every task is a coding / software-engineering scenario.** BridgeBench is a vibe-coding benchmark, so all artifacts are software artifacts — source code, diffs, git history, CI logs and configs, package manifests, API specs, database migrations, deploy records, service telemetry, AI-coding-agent sessions — and every deliverable is a question a coding agent would face. No generic business ops (invoices, staffing, org charts). All content is fictional: invented companies, services, and people only.
+
+## Pack invariants (enforced by the loader and `npm run tasks -- validate`)
+
+- Exactly **12 tasks per category**, **2 per cluster**, unique IDs.
+- Public and private halves must agree on `id` and `version`.
+- Every `requiredEvidence` entry in the private half must name an existing public artifact ID.
+- Public artifact IDs must be unique and the YAML filename must match the task ID.
+- Rendered competitor and worst-case judge payloads must fit the engine's prompt budget.
+- A task's `category` must match its pack directory and its `cluster` must belong to that category.
+
+Use `npm run tasks -- validate --file <path>` to validate one proposed public task without enforcing full-pack balance.
+
+## Public half schema (`tasks/<category>/public/<id>.yaml`)
+
+| Field              | Constraints                                                                                     |
+| ------------------ | ----------------------------------------------------------------------------------------------- |
+| `id`               | kebab-case, regex-validated                                                                     |
+| `version`          | semver; bump on any content change (hashes are journaled)                                       |
+| `category`         | `reasoning` \| `hallucination`                                                                  |
+| `cluster`          | one of the category's six clusters (below)                                                      |
+| `difficulty`       | `hard` \| `expert` (current packs are all `expert`)                                             |
+| `title`, `summary` | summary ≤ 500 chars                                                                             |
+| `prompt`           | ≤ 10k chars — the numbered deliverables live here                                               |
+| `artifacts[]`      | 1–20 of `{id, type, label, content ≤ 40k}`; `type` ∈ code, log, config, spec, diff, table, note |
+| `tags[]`           | optional, ≤ 60 chars each                                                                       |
+
+Tasks are deliberately heavyweight: five to eight interlocking artifacts (~9–18k characters) and four to ten numbered deliverables, so a match exercises real deliberation budget instead of a one-screen skim.
+
+## Private half schema (overlay: `tasks/<category>/private/<id>.yaml`)
+
+| Field                   | Purpose                                                              |
+| ----------------------- | -------------------------------------------------------------------- |
+| `id`, `version`         | must match the public half                                           |
+| `expectedResolution`    | ≤ 10k prose — the defensible resolution for every deliverable        |
+| `requiredEvidence[]`    | public artifact IDs a strong answer must ground itself in            |
+| `disqualifyingErrors[]` | the planted decoy conclusions, named explicitly                      |
+| `rubric`                | `{correctness, evidenceGrounding, constraintHandling, completeness}` |
+
+## Reasoning clusters
+
+Fully determinable; decoy paths planted and listed as disqualifying errors in the hidden reference:
+
+- **stateful-execution** — trace state through interleaved operations;
+- **constraint-reconciliation** — many constraints, one consistent assignment;
+- **root-cause-reasoning** — the incident has one defensible root cause and several attractive red herrings;
+- **multi-artifact-synthesis** — the answer only exists across artifacts, never inside one;
+- **formal-counterexample** — find the input/schedule that breaks the claimed property;
+- **uncertainty-adversarial** — noisy signals, but one best-supported conclusion.
+
+**Authoring rule:** every decoy you plant must appear in `disqualifyingErrors`, and the true chain must be derivable from `requiredEvidence` alone. If two resolutions survive honest scrutiny, the task is broken — fix the artifacts, not the rubric.
+
+## Hallucination clusters
+
+Traps are internal to the artifacts, so ground truth stays verifiable; every task mixes supported deliverables with traps so blanket refusal loses as clearly as fabrication:
+
+- **false-premise** — deliverables embed claims the artifacts contradict; the premise must be corrected with citations;
+- **missing-evidence** — the key fact is deliberately absent while tempting near-substitutes exist;
+- **entity-fabrication** — plausible endpoints, functions, tools, files, or config keys that are never defined, next to near-miss twins that are;
+- **knowledge-boundary** — audit-style question sets where roughly half the items are determinable and half are not;
+- **conflicting-sources** — artifacts disagree; silent blending or unacknowledged picking is the failure;
+- **citation-fidelity** — exact values, quotes, and attributions under dense near-duplicate detail.
+
+**Authoring rule:** the private half classifies every deliverable as _supported_, _false-premise_, or _not-determinable_, with the full trap inventory. Where a task tests absence (entity fabrication, missing evidence), an artifact should self-declare exhaustiveness (e.g. a spec that states it lists the complete API surface), so "doesn't exist" is provable from the material.
+
+## Process
+
+1. Author both halves locally (public in your fork, private in the overlay checkout).
+2. `npm run tasks -- validate` until clean — with the overlay present it validates pairing and evidence; without it, the public schema and pack balance.
+3. Bump `version` on any edit to either half; the journal hashes both, so silent drift is detectable.
+4. External contributors: validate and propose the public half through the task-proposal issue form. Do not include a hidden reference.
+5. After a public task is accepted, a maintainer starts a private handoff through the GitHub account attached to the issue. The private half never appears in the public issue or pull request.
+6. Replacing a live task requires a pack-rotation plan that preserves the 12-task, two-per-cluster invariant.
+7. Private halves are published when their pack retires (see [private-packs.md](private-packs.md)).