@lythos/skill-arena 0.9.1 → 0.9.3

package/src/comparative-judge.ts

@@ -0,0 +1,166 @@
+import { zodToJsonSchema } from 'zod-to-json-schema'
+import { ComparativeReport, ScoreCell, ParetoEntry } from '@lythos/test-utils/schema'
+import type { AgentAdapter } from '@lythos/test-utils/agents'
+import type { ArenaManifest } from '@lythos/test-utils/schema'
+
+// ── Pareto Frontier (deterministic algorithm) ──────────────────────────────
+
+export interface ScoreVector {
+  participant_id: string
+  scores: Record<string, number>
+  dominated: boolean
+  dominated_by: string[]
+}
+
+/**
+ * Compute Pareto frontier from score vectors.
+ * Participant A dominates B if A >= B in all criteria AND A > B in at least one.
+ * This is deterministic — never delegated to LLM.
+ */
+export function computePareto(vectors: { participant_id: string; scores: Record<string, number> }[]): ParetoEntry[] {
+  const result: ParetoEntry[] = vectors.map(v => ({
+    participant_id: v.participant_id,
+    scores: { ...v.scores },
+    dominated: false,
+    dominated_by: [] as string[],
+  }))
+
+  // Union of all criteria across all participants
+  const allCriteria = [...new Set(vectors.flatMap(v => Object.keys(v.scores)))]
+
+  if (allCriteria.length === 0) return result
+
+  for (let i = 0; i < result.length; i++) {
+    for (let j = 0; j < result.length; j++) {
+      if (i === j) continue
+      const a = vectors[i].scores
+      const b = vectors[j].scores
+
+      const allGe = allCriteria.every(k => (a[k] ?? 0) >= (b[k] ?? 0))
+      const anyGt = allCriteria.some(k => (a[k] ?? 0) > (b[k] ?? 0))
+
+      if (allGe && anyGt) {
+        // i dominates j
+        result[j].dominated = true
+        if (!result[j].dominated_by.includes(result[i].participant_id)) {
+          result[j].dominated_by.push(result[i].participant_id)
+        }
+      }
+    }
+  }
+
+  return result
+}
+
+// ── Comparative Judge Prompt ──────────────────────────────────────────────
+
+function buildComparativePrompt(opts: {
+  manifest: ArenaManifest
+  verdicts: { participantId: string; verdict: unknown }[]
+}): string {
+  const criteriaDesc = opts.manifest.criteria.join(', ')
+  const participants = opts.manifest.participants
+    .map(p => `- ${p.id}: ${p.name} (${p.description || 'no description'})`)
+    .join('\n')
+
+  return `You are a comparative judge evaluating ${opts.manifest.participants.length} participants against shared criteria.
+
+## Task
+${opts.manifest.task}
+
+## Participants
+${participants}
+
+## Criteria
+${criteriaDesc}
+
+## Your Job
+For each participant, score them 1-5 on each criterion. Provide a brief rationale.
+Score meanings: 1=poor, 3=acceptable, 5=excellent.
+
+Use the submit_scores tool to return your structured evaluation.`
+}
+
+const SCORE_TOOL = {
+  name: 'submit_scores',
+  description: 'Submit per-participant scores for each criterion with rationales',
+  input_schema: zodToJsonSchema(ComparativeReport.pick({ score_matrix: true, key_findings: true, recommendations: true })) as Record<string, unknown>,
+}
+
+function toScoreMatrix(
+  manifest: ArenaManifest,
+  scores: { participant_id: string; criterion: string; weight: number; score: number; rationale: string }[]
+): typeof ScoreCell._output[] {
+  return scores.map(s => ScoreCell.parse(s))
+}
+
+// ── Comparative Judge ─────────────────────────────────────────────────────
+
+export async function runComparativeJudge(opts: {
+  manifest: ArenaManifest
+  verdicts: { participantId: string; verdict: unknown }[]
+  judge: AgentAdapter
+  workdir: string
+}): Promise<typeof ComparativeReport._output> {
+  const { manifest, verdicts, judge, workdir } = opts
+
+  const prompt = buildComparativePrompt({ manifest, verdicts })
+
+  let raw: string
+  let parsed: unknown
+
+  if (judge.invokeTool) {
+    parsed = await judge.invokeTool({
+      tool: SCORE_TOOL,
+      prompt,
+      cwd: workdir,
+      timeoutMs: 120000,
+    })
+    raw = JSON.stringify(parsed)
+  } else {
+    const result = await judge.spawn({ cwd: workdir, brief: prompt, timeoutMs: 120000 })
+    raw = result.stdout
+    const fenceMatch = raw.match(/```(?:json)?\s*([\s\S]*?)\s*```/)
+    const jsonStr = fenceMatch ? fenceMatch[1].trim() : raw.trim()
+    parsed = JSON.parse(jsonStr)
+  }
+
+  // Validate LLM output
+  const llmResult = ComparativeReport.pick({
+    score_matrix: true,
+    key_findings: true,
+    recommendations: true,
+  }).parse(parsed)
+
+  const scoreMatrix = toScoreMatrix(manifest, llmResult.score_matrix)
+
+  // Pareto: deterministic, never delegated to LLM
+  const participantScores = manifest.participants.map(p => {
+    const pScores: Record<string, number> = {}
+    for (const cell of scoreMatrix) {
+      if (cell.participant_id === p.id) {
+        pScores[cell.criterion] = cell.score
+      }
+    }
+    return { participant_id: p.id, scores: pScores }
+  })
+
+  const pareto = computePareto(participantScores)
+
+  // Weighted totals (equal weight by default)
+  const weightedTotals: Record<string, number> = {}
+  for (const p of manifest.participants) {
+    const pCells = scoreMatrix.filter(c => c.participant_id === p.id)
+    weightedTotals[p.id] = pCells.reduce((sum, c) => sum + c.score * c.weight, 0) / (pCells.length || 1)
+  }
+
+  return ComparativeReport.parse({
+    arena_id: manifest.id,
+    generated_at: new Date().toISOString(),
+    score_matrix: scoreMatrix,
+    weighted_totals: weightedTotals,
+    pareto,
+    key_findings: llmResult.key_findings ?? [],
+    recommendations: llmResult.recommendations ?? [],
+  })
+}

package/src/player.test.ts

@@ -0,0 +1,95 @@
+import { describe, test, expect } from 'bun:test'
+import { resolvePlayer, resolveSides, groupBySide, totalRuns } from './player'
+import { parseArenaToml } from './arena-toml'
+
+const toml = parseArenaToml(`
+[arena]
+task = "Test task"
+criteria = ["a", "b"]
+runs_per_side = 3
+
+[[side]]
+name = "minimal"
+player = "claude-code"
+deck = "./decks/minimal.toml"
+
+[[side]]
+name = "rich"
+player = "expert-architect"
+deck = "./decks/rich.toml"
+`)
+
+describe('resolvePlayer', () => {
+  test('maps claude-code → claude', () => {
+    expect(resolvePlayer('claude-code')).toBe('claude')
+  })
+
+  test('maps Claude → claude (case insensitive)', () => {
+    expect(resolvePlayer('Claude')).toBe('claude')
+  })
+
+  test('maps kimi → kimi', () => {
+    expect(resolvePlayer('kimi')).toBe('kimi')
+  })
+
+  test('passes through unknown player names', () => {
+    expect(resolvePlayer('expert-architect')).toBe('expert-architect')
+  })
+
+  test('trims whitespace', () => {
+    expect(resolvePlayer('  claude-code  ')).toBe('claude')
+  })
+})
+
+describe('resolveSides', () => {
+  test('resolves all sides in arena.toml', () => {
+    const sides = resolveSides(toml)
+    expect(sides).toHaveLength(2)
+    expect(sides[0].platform).toBe('claude')
+    expect(sides[1].platform).toBe('expert-architect')
+    expect(sides[0].playerName).toBe('claude-code')
+  })
+
+  test('preserves side config', () => {
+    const sides = resolveSides(toml)
+    expect(sides[0].side.name).toBe('minimal')
+    expect(sides[0].side.deck).toBe('./decks/minimal.toml')
+  })
+})
+
+describe('groupBySide', () => {
+  test('groups by side name with run count', () => {
+    const groups = groupBySide(toml)
+    expect(groups).toHaveLength(2)
+    expect(groups[0].runs).toBe(3) // runs_per_side
+    expect(groups[1].runs).toBe(3)
+    expect(groups[0].platform).toBe('claude')
+  })
+
+  test('control flag preserved', () => {
+    const controlToml = parseArenaToml(`
+[arena]
+task = "x"
+criteria = ["a"]
+
+[[side]]
+name = "test"
+player = "claude-code"
+deck = "a.toml"
+
+[[side]]
+name = "baseline"
+player = "claude-code"
+deck = "b.toml"
+control = true
+`)
+    const groups = groupBySide(controlToml)
+    expect(groups[1].control).toBe(true)
+  })
+})
+
+describe('totalRuns', () => {
+  test('calculates sides × runs_per_side', () => {
+    expect(totalRuns(toml)).toBe(6) // 2 sides × 3 runs
+  })
+})

package/src/player.ts

@@ -0,0 +1,71 @@
+import type { Side, ArenaToml } from './arena-toml'
+
+// ── Player reference resolution (pure function) ────────────────────────────
+// Maps arena.toml player names → platform identifiers.
+// AgentAdapter creation is the IO layer's job (T4), not ours.
+
+export interface ResolvedSide {
+  side: Side
+  platform: string                  // resolved platform for useAgent()
+  playerName: string                // original player reference
+}
+
+/** Built-in player registry. Player names that map directly to useAgent platforms. */
+const BUILTIN_PLAYERS: Record<string, string> = {
+  'claude': 'claude',
+  'claude-code': 'claude',
+  'kimi': 'kimi',
+  'cursor': 'cursor',
+  'gemini': 'gemini',
+}
+
+/**
+ * Resolve a player reference to its platform identifier.
+ * - Built-in names (claude, kimi, cursor) map directly
+ * - Unknown names are passed through (assumed to be useAgent-compatible)
+ * - Future: custom player.toml files will override built-in mappings
+ */
+export function resolvePlayer(name: string): string {
+  const normalized = name.toLowerCase().trim()
+  return BUILTIN_PLAYERS[normalized] ?? normalized
+}
+
+/**
+ * Map arena.toml sides to resolved side configs.
+ * Pure function — no IO, no agent creation.
+ */
+export function resolveSides(toml: ArenaToml): ResolvedSide[] {
+  return toml.side.map(side => ({
+    side,
+    platform: resolvePlayer(side.player),
+    playerName: side.player,
+  }))
+}
+
+// ── Side grouping (for per-side aggregation in T3) ─────────────────────────
+
+export interface SideGroup {
+  sideName: string
+  player: string
+  deck: string
+  control: boolean
+  runs: number
+  platform: string
+}
+
+/** Group resolved sides by name for per-side statistical aggregation */
+export function groupBySide(toml: ArenaToml): SideGroup[] {
+  return resolveSides(toml).map(rs => ({
+    sideName: rs.side.name,
+    player: rs.playerName,
+    deck: rs.side.deck,
+    control: rs.side.control,
+    runs: toml.arena.runs_per_side,
+    platform: rs.platform,
+  }))
+}
+
+/** Get total run count from arena.toml (sides × runs_per_side) */
+export function totalRuns(toml: ArenaToml): number {
+  return toml.side.length * toml.arena.runs_per_side
+}

package/src/runner.ts

@@ -0,0 +1,250 @@
+import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { runAgentScenario, type AgentScenario } from '@lythos/test-utils/agent-bdd'
+import { useAgent } from '@lythos/test-utils/agents'
+import { ArenaManifest, Player } from '@lythos/test-utils/schema'
+import type { ArenaManifest as ArenaManifestType, JudgeVerdict } from '@lythos/test-utils/schema'
+import { runComparativeJudge } from './comparative-judge'
+import { parseArenaToml, buildExecutionPlan, type ArenaToml } from './arena-toml'
+import { resolvePlayer, resolveSides } from './player'
+import { aggregateAllStats } from './stats'
+import type { SideStats } from './stats'
+
+// ── Helpers ───────────────────────────────────────────────────────────────
+
+function stamp(): string {
+  const d = new Date()
+  return `${d.getFullYear()}${String(d.getMonth() + 1).padStart(2, '0')}${String(d.getDate()).padStart(2, '0')}-${String(d.getHours()).padStart(2, '0')}${String(d.getMinutes()).padStart(2, '0')}${String(d.getSeconds()).padStart(2, '0')}`
+}
+
+// ── Declarative runner (arena.toml → execute) ─────────────────────────────
+
+export interface ArenaResult {
+  manifest: ArenaManifestType
+  report: unknown
+  stats: SideStats[]
+  artifactsDir: string
+}
+
+export async function runArenaFromToml(opts: {
+  toml: ArenaToml
+  taskPath: string
+  outDir?: string
+  dryRun?: boolean
+}): Promise<ArenaResult | { plan: ReturnType<typeof buildExecutionPlan> }> {
+  const { toml, taskPath, outDir, dryRun } = opts
+
+  const plan = buildExecutionPlan(toml)
+
+  // dry-run: return plan without executing
+  if (dryRun) {
+    return { plan }
+  }
+
+  const arenaId = `arena-${stamp()}`
+  const artifactsDir = outDir || join(process.cwd(), 'runs', arenaId)
+  const resolved = resolveSides(toml)
+
+  // Build manifest
+  const manifest = ArenaManifest.parse({
+    id: arenaId,
+    created_at: new Date().toISOString(),
+    task: readFileSync(resolve(taskPath), 'utf-8').slice(0, 200),
+    mode: 'decks',
+    participants: [...new Map(resolved.map(r => [r.side.name, r])).values()].map(r => ({
+      id: r.side.name,
+      name: r.side.name,
+      player: r.platform,
+      deck: r.side.deck,
+      description: `${r.playerName} × ${r.side.deck}`,
+    })),
+    criteria: toml.arena.criteria,
+    status: 'running',
+  })
+
+  mkdirSync(artifactsDir, { recursive: true })
+  writeFileSync(join(artifactsDir, 'arena.json'), JSON.stringify(manifest, null, 2) + '\n')
+
+  // Execute plan: per-cell agent run
+  const verdictsBySide = new Map<string, JudgeVerdict[]>()
+
+  for (const cell of plan.cells) {
+    const cellDir = join(artifactsDir, 'runs', cell.side, `run-${cell.run}`)
+    mkdirSync(cellDir, { recursive: true })
+
+    try {
+      const agent = useAgent(resolvePlayer(cell.player))
+      const result = await runAgentScenario({
+        scenarioPath: resolve(taskPath),
+        agent,
+        setupWorkdir(_scenario: AgentScenario, workdir: string) {
+          mkdirSync(workdir, { recursive: true })
+          const deckContent = readFileSync(resolve(cell.deck), 'utf-8')
+          writeFileSync(join(workdir, 'skill-deck.toml'), deckContent)
+        },
+        baseDir: join(artifactsDir, 'runs', cell.side),
+      })
+
+      const v = (result.verdict ?? {
+        verdict: 'ERROR' as const,
+        reason: 'No verdict returned',
+        criteria: [],
+      }) as JudgeVerdict
+
+      if (!verdictsBySide.has(cell.side)) verdictsBySide.set(cell.side, [])
+      verdictsBySide.get(cell.side)!.push(v)
+    } catch (e) {
+      if (!verdictsBySide.has(cell.side)) verdictsBySide.set(cell.side, [])
+      verdictsBySide.get(cell.side)!.push({
+        verdict: 'ERROR' as const,
+        reason: `Runner exception: ${e instanceof Error ? e.message : String(e)}`,
+        criteria: [],
+      })
+    }
+  }
+
+  // Aggregate stats
+  const stats = aggregateAllStats(verdictsBySide)
+
+  // Comparative judge
+  const flatVerdicts: { participantId: string; verdict: unknown }[] = []
+  for (const [side, verdicts] of verdictsBySide) {
+    // Use the first run's verdict for comparative judge (or aggregate into one)
+    if (verdicts.length > 0) {
+      flatVerdicts.push({ participantId: side, verdict: verdicts[0] })
+    }
+  }
+
+  const judge = useAgent(resolved[0]?.platform ?? 'claude')
+  const report = await runComparativeJudge({
+    manifest,
+    verdicts: flatVerdicts,
+    judge,
+    workdir: artifactsDir,
+  })
+
+  // Write report
+  writeReport(artifactsDir, manifest, report, stats)
+
+  // Update manifest
+  const finalManifest = ArenaManifest.parse({ ...manifest, status: 'completed' })
+  writeFileSync(join(artifactsDir, 'arena.json'), JSON.stringify(finalManifest, null, 2) + '\n')
+
+  return { manifest: finalManifest, report, stats, artifactsDir }
+}
+
+// ── Backward compat: CLI-flag style runner ─────────────────────────────────
+
+export async function runArena(opts: {
+  taskPath: string
+  playerPaths: string[]
+  deckPaths: string[]
+  criteria: string[]
+  outDir: string
+}): Promise<{ manifest: ArenaManifestType; report: unknown; artifactsDir: string }> {
+  const { taskPath, playerPaths, deckPaths, criteria, outDir } = opts
+
+  // Convert CLI flags to ArenaToml internally
+  const toml: ArenaToml = {
+    arena: {
+      task: readFileSync(resolve(taskPath), 'utf-8').slice(0, 200),
+      criteria,
+      runs_per_side: 1,
+      max_participants: Math.min(playerPaths.length, deckPaths.length),
+    },
+    side: playerPaths.flatMap((playerPath, pi) =>
+      deckPaths.map((deckPath, di) => ({
+        name: `run-${String(pi * deckPaths.length + di + 1).padStart(2, '0')}`,
+        player: Player.parse(JSON.parse(readFileSync(resolve(playerPath), 'utf-8'))).platform,
+        deck: deckPath,
+      }))
+    ),
+  }
+
+  const result = await runArenaFromToml({ toml, taskPath, outDir })
+  const { manifest, report, artifactsDir } = result as ArenaResult
+  return { manifest, report, artifactsDir }
+}
+
+// ── Report renderer ────────────────────────────────────────────────────────
+
+function writeReport(dir: string, manifest: ArenaManifestType, report: unknown & { score_matrix?: { participant_id: string; criterion: string; weight: number; score: number; rationale: string }[]; pareto?: { participant_id: string; dominated: boolean; dominated_by: string[] }[]; key_findings?: string[]; recommendations?: { audience: string; recommendation: string }[] }, stats: SideStats[]): void {
+  const lines: string[] = [
+    `# Arena Report: ${manifest.id}`,
+    '',
+    `**Task**: ${manifest.task}`,
+    `**Criteria**: ${manifest.criteria.join(', ')}`,
+    `**Date**: ${new Date().toISOString()}`,
+    '',
+    '## Score Matrix',
+    '',
+    renderScoreMatrix(report),
+    '',
+    '## Per-Side Statistics',
+    '',
+    renderStatsTable(stats),
+    '',
+    '## Pareto Frontier',
+    '',
+    renderPareto(report),
+    '',
+    '## Key Findings',
+    '',
+    ...(report.key_findings ?? []).map((f: string) => `- ${f}`),
+    '',
+    '## Recommendations',
+    '',
+    ...(report.recommendations ?? []).map((r: { audience: string; recommendation: string }) => `- **${r.audience}**: ${r.recommendation}`),
+  ]
+
+  writeFileSync(join(dir, 'report.md'), lines.join('\n') + '\n')
+}
+
+function renderStatsTable(stats: SideStats[]): string {
+  if (stats.length === 0) return 'No statistics available.\n'
+
+  let table = `| Side | Runs | Pass Rate | Mean Confidence | Criteria |\n`
+  table += `|------|------|-----------|-----------------|----------|\n`
+
+  for (const s of stats) {
+    const confStr = s.meanConfidence != null ? `${s.meanConfidence.toFixed(0)}%` : '-'
+    const criteriaStr = s.criteria.map(c => `${c.name}: ${(c.mean * 100).toFixed(0)}%`).join(', ')
+    table += `| ${s.sideName} | ${s.runs} | ${(s.passRate * 100).toFixed(0)}% | ${confStr} | ${criteriaStr} |\n`
+  }
+
+  return table
+}
+
+function renderScoreMatrix(report: unknown & { score_matrix?: { participant_id: string; criterion: string; weight: number; score: number; rationale: string }[] }): string {
+  if (!report.score_matrix?.length) return 'No scores available.\n'
+
+  const participants = [...new Set(report.score_matrix.map(s => s.participant_id))]
+  const criteria = [...new Set(report.score_matrix.map(s => s.criterion))]
+
+  let table = `| Criterion | Weight | ${participants.join(' | ')} |\n`
+  table += `|${'---|'.repeat(2 + participants.length)}\n`
+
+  for (const c of criteria) {
+    table += `| ${c} | 25% | ${participants.map(p => {
+      const cell = report.score_matrix!.find(s => s.participant_id === p && s.criterion === c)
+      return `**${cell?.score ?? '?'}**`
+    }).join(' | ')} |\n`
+  }
+
+  table += `| **Weighted Total** | 100% | ${participants.map(p => {
+    const pScores = report.score_matrix!.filter(s => s.participant_id === p)
+    const avg = pScores.length ? pScores.reduce((sum, s) => sum + s.score, 0) / pScores.length : 0
+    return `**${avg.toFixed(1)}**`
+  }).join(' | ')} |\n`
+
+  return table
+}
+
+function renderPareto(report: unknown & { pareto?: { participant_id: string; dominated: boolean; dominated_by: string[] }[] }): string {
+  if (!report.pareto?.length) return 'No Pareto analysis.\n'
+  return report.pareto.map(p =>
+    p.dominated
+      ? `- **${p.participant_id}**: dominated by ${p.dominated_by.join(', ')}`
+      : `- **${p.participant_id}**: Pareto-optimal (non-dominated)`
+  ).join('\n')
+}

package/src/stats.test.ts

@@ -0,0 +1,111 @@
+import { describe, test, expect } from 'bun:test'
+import { aggregateSideStats, aggregateAllStats } from './stats'
+import type { JudgeVerdict } from '@lythos/test-utils/schema'
+
+function makeVerdict(overrides?: Partial<JudgeVerdict>): JudgeVerdict {
+  return {
+    verdict: 'PASS',
+    reason: 'OK',
+    criteria: [{ name: 'correctness', passed: true }],
+    ...overrides,
+  }
+}
+
+// ── aggregateSideStats ─────────────────────────────────────────────────────
+
+describe('aggregateSideStats', () => {
+  test('single run: passRate=1, no variance', () => {
+    const stats = aggregateSideStats('test', [makeVerdict()])
+    expect(stats.sideName).toBe('test')
+    expect(stats.runs).toBe(1)
+    expect(stats.passRate).toBe(1)
+    expect(stats.failRate).toBe(0)
+    expect(stats.errorRate).toBe(0)
+  })
+
+  test('3 runs: 2 PASS, 1 FAIL', () => {
+    const verdicts = [
+      makeVerdict(),
+      makeVerdict(),
+      makeVerdict({ verdict: 'FAIL', reason: 'bad' }),
+    ]
+    const stats = aggregateSideStats('test', verdicts)
+    expect(stats.passRate).toBeCloseTo(2 / 3)
+    expect(stats.failRate).toBeCloseTo(1 / 3)
+  })
+
+  test('confidence: mean across runs', () => {
+    const verdicts = [
+      makeVerdict({ confidence: 90 }),
+      makeVerdict({ confidence: 80 }),
+      makeVerdict({ confidence: 70 }),
+    ]
+    const stats = aggregateSideStats('test', verdicts)
+    expect(stats.meanConfidence).toBeCloseTo(80)
+    expect(stats.confidenceVariance).toBeCloseTo(100) // (100+0+100)/2 = 100
+  })
+
+  test('confidence: null when no verdict has it', () => {
+    const stats = aggregateSideStats('test', [makeVerdict(), makeVerdict()])
+    expect(stats.meanConfidence).toBeNull()
+    expect(stats.confidenceVariance).toBeNull()
+  })
+
+  test('per-criterion pass rate', () => {
+    const verdicts = [
+      makeVerdict({ criteria: [{ name: 'accuracy', passed: true }] }),
+      makeVerdict({ criteria: [{ name: 'accuracy', passed: false }] }),
+      makeVerdict({ criteria: [{ name: 'accuracy', passed: true }] }),
+    ]
+    const stats = aggregateSideStats('test', verdicts)
+    expect(stats.criteria).toHaveLength(1)
+    expect(stats.criteria[0].name).toBe('accuracy')
+    expect(stats.criteria[0].mean).toBeCloseTo(2 / 3)
+  })
+
+  test('per-criterion scores: mean and variance', () => {
+    const verdicts = [
+      makeVerdict({ scores: { coverage: 5, relevance: 4 } }),
+      makeVerdict({ scores: { coverage: 3, relevance: 4 } }),
+      makeVerdict({ scores: { coverage: 4, relevance: 4 } }),
+    ]
+    const stats = aggregateSideStats('test', verdicts)
+    expect(stats.scoreByCriterion.coverage.mean).toBeCloseTo(4)
+    expect(stats.scoreByCriterion.relevance.mean).toBeCloseTo(4)
+    expect(stats.scoreByCriterion.relevance.variance).toBe(0) // all 4s
+  })
+
+  test('zero runs: all zeros', () => {
+    const stats = aggregateSideStats('empty', [])
+    expect(stats.runs).toBe(0)
+    expect(stats.passRate).toBe(0)
+    expect(stats.meanConfidence).toBeNull()
+  })
+
+  test('handles ERROR verdicts correctly', () => {
+    const verdicts = [
+      makeVerdict(),
+      makeVerdict({ verdict: 'ERROR', reason: 'parse failed' }),
+    ]
+    const stats = aggregateSideStats('test', verdicts)
+    expect(stats.passRate).toBe(0.5)
+    expect(stats.errorRate).toBe(0.5)
+  })
+})
+
+// ── aggregateAllStats ──────────────────────────────────────────────────────
+
+describe('aggregateAllStats', () => {
+  test('aggregates multiple sides', () => {
+    const map = new Map<string, JudgeVerdict[]>()
+    map.set('side-a', [makeVerdict(), makeVerdict()])
+    map.set('side-b', [makeVerdict({ verdict: 'FAIL', reason: 'nope' })])
+
+    const stats = aggregateAllStats(map)
+    expect(stats).toHaveLength(2)
+    expect(stats[0].sideName).toBe('side-a')
+    expect(stats[0].passRate).toBe(1)
+    expect(stats[1].sideName).toBe('side-b')
+    expect(stats[1].passRate).toBe(0)
+  })
+})