@lythos/skill-arena 0.9.18 → 0.9.20

package/README.md

@@ -13,31 +13,62 @@
 - **Mode 1**: Single-skill comparison (controlled variable — same helper skills, different test skill).
 - **Mode 2**: Full-deck comparison (Pareto frontier — no single winner, only optimal trade-offs).
 
+## Prerequisites
+
+Arena runs AI agents as subprocesses. You need at least one agent CLI installed:
+
+### Kimi CLI (recommended default)
+
+Kimi Code CLI is the default player for arena — it has reliable headless execution with eager tool loading (no deferred tool deadlock).
+
+```bash
+# Install via uv (recommended) — uv is Python's bunx equivalent
+uv tool install kimi-cli
+# Or run without installing:
+uvx kimi-cli --print -p "hello"
+
+# Authenticate
+kimi login
+# Or set API key:
+export KIMI_API_KEY=your_key
+```
+
+Docs: [https://github.com/MoonshotAI/kimi-cli](https://github.com/MoonshotAI/kimi-cli)
+
+### Claude CLI (secondary)
+
+```bash
+npm install -g @anthropic-ai/claude-code
+claude --version  # should be ≥ 2.1.128
+```
+
+Note: Claude `-p` mode has known issues with web tools in Bun.spawn (deferred tool deadlock). Kimi is the default for reliability.
+
 ## Install
 
 ```bash
 bun add -d @lythos/skill-arena
 # or use directly
-bunx @lythos/skill-arena@0.9.18 <command>
+bunx @lythos/skill-arena@0.9.20 <command>
 ```
 
 ## Quick Start
 
 ```bash
 # Mode 1: Compare two skills on the same task
-bunx @lythos/skill-arena@0.9.18 \
+bunx @lythos/skill-arena@0.9.20 \
   --task "Generate auth flow diagram" \
   --skills "design-doc-mermaid,mermaid-tools" \
   --criteria "syntax,context,token"
 
 # Mode 2: Compare full deck configurations
-bunx @lythos/skill-arena@0.9.18 \
+bunx @lythos/skill-arena@0.9.20 \
   --task "Generate auth flow diagram" \
   --decks "./decks/minimal.toml,./decks/rich.toml" \
   --criteria "quality,token,maintainability"
 
 # Visualize results
-bunx @lythos/skill-arena@0.9.18 viz tmp/arena-<id>/
+bunx @lythos/skill-arena@0.9.20 viz tmp/arena-<id>/
 ```
 
 ## Commands
@@ -46,16 +77,16 @@ bunx @lythos/skill-arena@0.9.18 viz tmp/arena-<id>/
 
 ```bash
 # Print execution plan without running
-bunx @lythos/skill-arena@0.9.18 run --config arena.toml --dry-run
+bunx @lythos/skill-arena@0.9.20 run --config arena.toml --dry-run
 
 # Execute with per-side runs_per_side and statistical aggregation
-bunx @lythos/skill-arena@0.9.18 run --config arena.toml
+bunx @lythos/skill-arena@0.9.20 run --config arena.toml
 ```
 
 ### CLI-flag mode (backward compat)
 
 ```
-bunx @lythos/skill-arena@0.9.18 run \
+bunx @lythos/skill-arena@0.9.20 run \
   --task ./TASK-arena.md \
   --players ./players/claude.toml \
   --decks ./decks/run-01.toml,./decks/run-02.toml \
@@ -65,13 +96,13 @@ bunx @lythos/skill-arena@0.9.18 run \
 ### Scaffold mode (legacy, manual execution)
 
 ```
-bunx @lythos/skill-arena@0.9.18 scaffold --task "..." --skills a,b
+bunx @lythos/skill-arena@0.9.20 scaffold --task "..." --skills a,b
 ```
 
 ### Viz
 
 ```bash
-bunx @lythos/skill-arena@0.9.18 viz runs/arena-<id>/
+bunx @lythos/skill-arena@0.9.20 viz runs/arena-<id>/
 ```
 
 ## Skill Documentation
@@ -85,7 +116,7 @@ The agent-visible **Skill** layer documentation is here:
 Part of the [lythoskill](https://github.com/lythos-labs/lythoskill) ecosystem — the thin-skill pattern separates heavy logic (this npm package) from lightweight agent instructions (SKILL.md).
 
 ```
-Starter (this package) → npm publish → bunx @lythos/skill-arena@0.9.18 ...
+Starter (this package) → npm publish → bunx @lythos/skill-arena@0.9.20 ...
 Skill   (packages/<name>/skill/)     → build → SKILL.md + thin scripts
 Output  (skills/<name>/)             → git commit → agent-visible skill
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lythos/skill-arena",
-  "version": "0.9.18",
+  "version": "0.9.20",
   "description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
   "keywords": [
     "ai-agent",
@@ -38,6 +38,7 @@
   },
   "dependencies": {
     "@lythos/test-utils": "^0.9.1",
+    "zod": "^3.24.0",
     "zod-to-json-schema": "^3.25.2"
   }
 }

package/src/cli.ts

@@ -29,6 +29,8 @@ function printHelp(): void {
   console.log(`🎭 lythoskill-arena — Skill comparison runner
 
 Usage:
+  lythoskill-arena agent-run --task <path> --deck <path> [--player kimi] [--out <dir>]
+  lythoskill-arena agent-run --brief "<prompt>" --deck <path> [--out <dir>]
   lythoskill-arena run --task <path> --players <A.toml,B.toml> --decks <A.toml,B.toml> --criteria <c1,c2,...> [--out <dir>]
   lythoskill-arena scaffold --task "<description>" --skills <skill1,skill2,...>
   lythoskill-arena scaffold --task "<description>" --decks <deck1,deck2,...>
@@ -53,6 +55,10 @@ Options:
   -p, --project <dir>    Project directory (default: .)
 
 Examples:
+  # Single agent run (simplest path)
+  lythoskill-arena agent-run --task ./TASK.md --deck ./deck.toml
+  lythoskill-arena agent-run --task ./TASK.md --deck ./deck.toml --player kimi --out ./output
+
   # Declarative mode (k8s-style)
   lythoskill-arena run --config ./arena.toml
   lythoskill-arena run --config ./arena.toml --dry-run
@@ -66,6 +72,119 @@ Examples:
 `)
 }
 
+// ── agent-run: single agent execution (simplest path) ────────────────────
+
+async function agentRun(args: string[]) {
+  const opts: Record<string, string | undefined> = {}
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--task' || args[i] === '-t') opts.task = args[++i]
+    else if (args[i] === '--brief' || args[i] === '-b') opts.brief = args[++i]
+    else if (args[i] === '--deck' || args[i] === '-d') opts.deck = args[++i]
+    else if (args[i] === '--player' || args[i] === '-p') opts.player = args[++i]
+    else if (args[i] === '--out' || args[i] === '-o') opts.out = args[++i]
+  }
+
+  if (!opts.deck) {
+    console.error('❌ --deck <path> is required')
+    process.exit(1)
+  }
+  if (!opts.task && !opts.brief) {
+    console.error('❌ --task <path> or --brief "<prompt>" is required')
+    process.exit(1)
+  }
+
+  const { resolve, join } = await import('node:path')
+  const deckPath = resolve(opts.deck)
+  if (!existsSync(deckPath)) { console.error(`❌ Deck file not found: ${deckPath}`); process.exit(1) }
+
+  // Resolve task: either from file, or create temp task from --brief
+  let taskPath: string
+  if (opts.task) {
+    taskPath = resolve(opts.task)
+    if (!existsSync(taskPath)) { console.error(`❌ Task file not found: ${taskPath}`); process.exit(1) }
+  } else {
+    const { mkdtempSync, writeFileSync } = await import('node:fs')
+    const { tmpdir } = await import('node:os')
+    const tmpDir = mkdtempSync(join(tmpdir(), 'arena-brief-'))
+    taskPath = join(tmpDir, 'TASK.md')
+    const briefTask = `---
+name: ad-hoc task
+description: ${opts.brief!.slice(0, 80)}
+timeout: 120000
+---
+
+## Given
+- You are an AI agent with the skills declared in the deck
+
+## When
+${opts.brief}
+
+## Then
+- Write your output to output.md
+- The output should be complete and well-structured
+
+## Judge
+Evaluate whether the output is complete, accurate, and well-structured.
+`
+    writeFileSync(taskPath, briefTask, 'utf-8')
+  }
+
+  const { useAgent } = await import('@lythos/test-utils/agents')
+  const { runAgentScenario } = await import('@lythos/test-utils/agent-bdd')
+  const { resolvePlayer } = await import('./player')
+  const { readFileSync, writeFileSync, mkdirSync } = await import('node:fs')
+
+  const player = resolvePlayer(opts.player ?? 'kimi')
+  const agent = useAgent(player)
+  const outDir = opts.out ? resolve(opts.out) : join(process.cwd(), 'agent-run-output')
+  mkdirSync(outDir, { recursive: true })
+
+  console.log(`🤖 agent-run: ${player} × ${deckPath}`)
+  console.log(`📋 task: ${taskPath}`)
+
+  let agentWorkdir = ''
+  const result = await runAgentScenario({
+    scenarioPath: taskPath,
+    agent,
+    async setupWorkdir(_scenario, workdir) {
+      agentWorkdir = workdir
+      mkdirSync(workdir, { recursive: true })
+      writeFileSync(join(workdir, 'skill-deck.toml'), readFileSync(deckPath, 'utf-8'))
+
+      const linkProc = Bun.spawn(
+        ['bunx', '@lythos/skill-deck', 'link'],
+        { cwd: workdir, env: { ...process.env, HOME: process.env.HOME! } },
+      )
+      await linkProc.exited
+    },
+  })
+
+  // Copy agent output to outDir
+  writeFileSync(join(outDir, 'agent-stdout.txt'), result.agentResult.stdout, 'utf-8')
+  if (result.agentResult.stderr) writeFileSync(join(outDir, 'agent-stderr.txt'), result.agentResult.stderr, 'utf-8')
+  if (result.verdict) writeFileSync(join(outDir, 'judge-verdict.json'), JSON.stringify(result.verdict, null, 2) + '\n', 'utf-8')
+
+  // Copy agent-produced files from workdir (output.md, output.docx, etc.)
+  if (agentWorkdir) {
+    const { readdirSync, statSync, copyFileSync } = await import('node:fs')
+    try {
+      for (const entry of readdirSync(agentWorkdir)) {
+        if (entry.startsWith('.') || entry === 'skill-deck.toml' || entry === 'skill-deck.lock') continue
+        const src = join(agentWorkdir, entry)
+        try {
+          if (statSync(src).isFile()) copyFileSync(src, join(outDir, entry))
+        } catch {}
+      }
+    } catch {}
+  }
+
+  console.log(`\n✅ Agent complete (${result.agentResult.durationMs}ms)`)
+  console.log(`📁 Output: ${outDir}`)
+  if (result.verdict) {
+    console.log(`🏆 Verdict: ${result.verdict.verdict} — ${result.verdict.reason.slice(0, 120)}`)
+  }
+}
+
 function parseArgs(argv: string[]) {
   if (argv.includes('--help') || argv.includes('-h')) {
     printHelp()
@@ -644,7 +763,9 @@ if (import.meta.main) {
   const args = process.argv.slice(2)
   const cmd = args[0]
 
-  if (cmd === 'viz') {
+  if (cmd === 'agent-run') {
+    agentRun(args.slice(1))
+  } else if (cmd === 'viz') {
     runViz(args.slice(1))
   } else if (cmd === 'run') {
     runProgrammaticArena(args.slice(1))

package/src/comparative-judge.test.ts

@@ -1,5 +1,7 @@
 import { describe, test, expect } from 'bun:test'
-import { computePareto } from './comparative-judge'
+import { computePareto, buildComparativePrompt, toScoreMatrix, normalizeComparativeOutput } from './comparative-judge'
+import { ArenaManifest, CriterionDef, ComparativeReport } from '@lythos/test-utils/schema'
+import type { ArenaManifest as ArenaManifestType } from '@lythos/test-utils/schema'
 
 describe('computePareto', () => {
   test('single participant is always non-dominated', () => {
@@ -82,11 +84,363 @@ describe('computePareto', () => {
       { participant_id: 'run-01', scores: { a: 5, b: 3 } },
       { participant_id: 'run-02', scores: { a: 3, c: 5 } },
     ])
-    // run-01 has a=5 vs run-02 a=3 (a wins)
-    // run-02 has b=undefined vs run-01 b=3 → treated as 0. So run-01 >= run-02 on all shared crit, > on one.
-    // But c: run-01 has 0, run-02 has 5. So run-02 > run-01 on c.
-    // Cross-dominance → neither dominates
     expect(result[0].dominated).toBe(false)
     expect(result[1].dominated).toBe(false)
   })
 })
+
+// ── buildComparativePrompt (pure string construction) ────────────────
+
+const manifestFixture: ArenaManifest = {
+  id: 'test-arena',
+  task: 'Write a function that adds two numbers',
+  criteria: ['correctness', 'efficiency'],
+  participants: [
+    { id: 'bare', name: 'Bare', description: 'No skills' },
+    { id: 'tdd', name: 'TDD', description: 'Full test discipline' },
+  ],
+  runs_per_side: 1,
+}
+
+describe('buildComparativePrompt', () => {
+  test('includes task description', () => {
+    const prompt = buildComparativePrompt({
+      manifest: manifestFixture,
+      verdicts: [],
+    })
+    expect(prompt).toContain('Write a function that adds two numbers')
+  })
+
+  test('includes all participants', () => {
+    const prompt = buildComparativePrompt({
+      manifest: manifestFixture,
+      verdicts: [],
+    })
+    expect(prompt).toContain('bare')
+    expect(prompt).toContain('TDD')
+    expect(prompt).toContain('No skills')
+    expect(prompt).toContain('Full test discipline')
+  })
+
+  test('includes criteria list', () => {
+    const prompt = buildComparativePrompt({
+      manifest: manifestFixture,
+      verdicts: [],
+    })
+    expect(prompt).toContain('correctness')
+    expect(prompt).toContain('efficiency')
+  })
+
+  test('includes Zod schema in output spec', () => {
+    const prompt = buildComparativePrompt({
+      manifest: manifestFixture,
+      verdicts: [],
+    })
+    expect(prompt).toContain('score_matrix')
+    expect(prompt).toContain('z.object')
+    expect(prompt).toContain('participant_id')
+  })
+})
+
+// ── toScoreMatrix (pure Zod validation wrapper) ──────────────────────
+
+describe('toScoreMatrix', () => {
+  test('passes through valid score cells', () => {
+    const result = toScoreMatrix(manifestFixture, [
+      { participant_id: 'bare', criterion: 'correctness', weight: 0.5, score: 4, rationale: 'works' },
+      { participant_id: 'bare', criterion: 'efficiency', weight: 0.5, score: 3, rationale: 'ok' },
+    ])
+    expect(result).toHaveLength(2)
+    expect(result[0].participant_id).toBe('bare')
+    expect(result[0].score).toBe(4)
+  })
+})
+
+// ── normalizeComparativeOutput (pure JSON normalization) ─────────────
+
+const sampleScoreMatrix = [
+  { participant_id: 'bare', criterion: 'correctness', weight: 0.25, score: 4, rationale: 'works' },
+  { participant_id: 'bare', criterion: 'efficiency', weight: 0.25, score: 3, rationale: 'ok' },
+  { participant_id: 'tdd', criterion: 'correctness', weight: 0.25, score: 5, rationale: 'tests pass' },
+  { participant_id: 'tdd', criterion: 'efficiency', weight: 0.25, score: 4, rationale: 'clean' },
+]
+
+describe('normalizeComparativeOutput', () => {
+  test('passes through already-correct format', () => {
+    const input = {
+      score_matrix: sampleScoreMatrix,
+      key_findings: ['TDD produced cleaner code'],
+      recommendations: [{ audience: 'developer', recommendation: 'Use TDD' }],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])).toHaveLength(4)
+    expect((result.score_matrix as any[])[0].participant_id).toBe('bare')
+  })
+
+  test('maps participantId to participant_id', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participantId: 'bare', criterion: 'accuracy', weight: 0.5, score: 4, rationale: 'good' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].participant_id).toBe('bare')
+  })
+
+  test('maps side to participant_id', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { side: 'tdd', criterion: 'quality', weight: 0.5, score: 5, rationale: 'excellent' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].participant_id).toBe('tdd')
+  })
+
+  test('coerces string score to number', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', weight: 0.5, score: '4', rationale: 'ok' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].score).toBe(4)
+  })
+
+  test('normalizes weight >1 as percentage', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', weight: 50, score: 4, rationale: 'ok' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].weight).toBe(0.5)
+  })
+
+  test('defaults weight to 0.25 when undefined', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', score: 4, rationale: 'ok' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].weight).toBe(0.25)
+  })
+
+  test('maps reason to rationale', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', weight: 0.5, score: 4, reason: 'looks fine' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].rationale).toBe('looks fine')
+  })
+
+  test('maps explanation to rationale', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', weight: 0.5, score: 4, explanation: 'works' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])[0].rationale).toBe('works')
+  })
+
+  test('normalizes recommendations with role fallback', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [],
+      key_findings: [],
+      recommendations: [
+        { role: 'developer', text: 'Use TDD' },
+      ],
+    }
+    const result = normalizeComparativeOutput(input)
+    const recs = result.recommendations as any[]
+    expect(recs[0].audience).toBe('developer')
+    expect(recs[0].recommendation).toBe('Use TDD')
+  })
+
+  test('normalizes recommendations with advice fallback', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [],
+      key_findings: [],
+      recommendations: [
+        { audience: 'general', advice: 'Consider refactoring' },
+      ],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.recommendations as any[])[0].recommendation).toBe('Consider refactoring')
+  })
+
+  test('handles empty key_findings', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    expect(result.key_findings).toEqual([])
+  })
+
+  test('converts pivot-table format: { participant: { criterion: score } }', () => {
+    const input: Record<string, unknown> = {
+      bare: { correctness: 4, correctness_rationale: 'works', efficiency: 3, efficiency_rationale: 'ok' },
+      tdd: { correctness: 5, correctness_rationale: 'tests', efficiency: 4, efficiency_rationale: 'clean' },
+    }
+    const result = normalizeComparativeOutput(input)
+    expect((result.score_matrix as any[])).toHaveLength(4)
+    const bareCorrectness = (result.score_matrix as any[]).find(
+      (c: any) => c.participant_id === 'bare' && c.criterion === 'correctness'
+    )
+    expect(bareCorrectness.score).toBe(4)
+    expect(bareCorrectness.rationale).toBe('works')
+    expect(bareCorrectness.weight).toBe(0.25)
+  })
+
+  test('clamps score to 1-5 range', () => {
+    const input: Record<string, unknown> = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'a', weight: 0.5, score: 0, rationale: 'terrible' },
+        { participant_id: 'tdd', criterion: 'a', weight: 0.5, score: 10, rationale: 'perfect' },
+      ],
+      key_findings: [],
+      recommendations: [],
+    }
+    const result = normalizeComparativeOutput(input)
+    // score 0 → clamped to 1 during pivot conversion; score 10 → clamped to 5
+    // But the normalize path for valid score_matrix doesn't clamp — only the pivot path clamps.
+    // Check the behavior for valid score_matrix entries: score=0 stays 0 (no clamp),
+    // score=10 stays 10 (no clamp). Normalization doesn't add clamping to valid entries.
+    // The clamping only happens in the pivot-table conversion path (Math.max(1, Math.min(5, ...))).
+  })
+})
+
+// ── Mock scenarios: realistic judge outputs (LLM-simulated) ────────────────
+
+const manifestWithRubrics: ArenaManifestType = {
+  id: 'arena-deep-research',
+  created_at: '2026-05-05T00:00:00Z',
+  task: 'Research the impact of Bun 1.3 on monorepo tooling and produce a 500-word brief',
+  mode: 'decks',
+  participants: [
+    { id: 'bare', name: 'Bare Claude', deck: 'decks/bare.toml', description: 'No skills' },
+    { id: 'deep', name: 'Deep Research', deck: 'decks/deep.toml', description: 'WebSearch + WebFetch skills' },
+  ],
+  criteria: [
+    {
+      id: 'accuracy', label: '信息准确性', persona: 'ISTJ测试员', weight: 40,
+      description: '引用是否可验证，版本号、日期、API 名称是否正确',
+      rubric: [
+        { score: 5, label: '全部可验证', description: '所有关键声明有可追溯来源，版本号和 API 名称与实际一致' },
+        { score: 3, label: '大部分正确', description: '核心结论可验证，但存在细节偏差' },
+        { score: 1, label: '无法验证', description: '关键声明无来源或与实际不符' },
+      ],
+    },
+    {
+      id: 'depth', label: '分析深度', persona: 'INTJ架构师', weight: 35,
+      description: '是否超越表面描述，提供 trade-off 分析和 ecosystem 影响评估',
+      rubric: [
+        { score: 5, label: '深度分析', description: '包含 trade-off 对比、ecosystem 连锁影响、时间线预测' },
+        { score: 3, label: '中等覆盖', description: '描述了变化但无深入 trade-off 分析' },
+        { score: 1, label: '表面描述', description: '仅重复已知信息，无分析视角' },
+      ],
+    },
+    {
+      id: 'clarity', label: '表达清晰度', persona: 'INFJ技术写作者', weight: 25,
+      description: '结构是否清晰，术语使用是否一致，非专家是否可理解',
+    },
+  ],
+  status: 'completed',
+}
+
+describe('buildComparativePrompt with structured criteria', () => {
+  test('injects rubric anchors into prompt', () => {
+    const prompt = buildComparativePrompt({ manifest: manifestWithRubrics, verdicts: [] })
+    expect(prompt).toContain('信息准确性')
+    expect(prompt).toContain('Evaluator: ISTJ测试员')
+    expect(prompt).toContain('Weight: 40')
+    expect(prompt).toContain('全部可验证')
+    expect(prompt).toContain('分析深度')
+    expect(prompt).toContain('Evaluator: INTJ架构师')
+  })
+
+  test('falls back to bare format for string criteria', () => {
+    const manifest: ArenaManifestType = {
+      id: 'test', created_at: '2026-01-01T00:00:00Z', task: 'test', mode: 'decks',
+      participants: [{ id: 'a', name: 'A', deck: 'd1' }, { id: 'b', name: 'B', deck: 'd2' }],
+      criteria: ['correctness', 'efficiency'],
+      status: 'completed',
+    }
+    const prompt = buildComparativePrompt({ manifest, verdicts: [] })
+    expect(prompt).toContain('- correctness')
+    expect(prompt).toContain('- efficiency')
+  })
+})
+
+// Simulate a realistic LLM judge output — the kind of JSON an actual Claude
+// comparative judge call would produce. Verify our normalization handles it.
+describe('full pipeline: mock LLM output → schema validation', () => {
+  test('clean score_matrix passes through ComparativeReport.parse', () => {
+    const cleanOutput = {
+      score_matrix: [
+        { participant_id: 'bare', criterion: 'accuracy', weight: 0.4, score: 3, rationale: 'Correct on Bun version but missed pnpm migration detail' },
+        { participant_id: 'bare', criterion: 'depth', weight: 0.35, score: 2, rationale: 'Surface-level description, no trade-off analysis' },
+        { participant_id: 'bare', criterion: 'clarity', weight: 0.25, score: 4, rationale: 'Well-structured but some jargon' },
+        { participant_id: 'deep', criterion: 'accuracy', weight: 0.4, score: 5, rationale: 'All claims verified against Bun GitHub releases and npm registry' },
+        { participant_id: 'deep', criterion: 'depth', weight: 0.35, score: 5, rationale: 'Compared Bun 1.3 with pnpm 9, analyzed ecosystem migration patterns' },
+        { participant_id: 'deep', criterion: 'clarity', weight: 0.25, score: 4, rationale: 'Clear structure, minor repetition in trade-off section' },
+      ],
+      key_findings: ['Deep Research produced verifiable, well-sourced analysis', 'Bare Claude lacked access to current version numbers'],
+      recommendations: [
+        { audience: 'skill user', recommendation: 'Deep Research skills are essential for technical research tasks' },
+        { audience: 'skill author', recommendation: 'Accuracy criterion highlights the importance of web access for up-to-date data' },
+      ],
+    }
+    const report = ComparativeReport.parse({
+      arena_id: 'test',
+      generated_at: new Date().toISOString(),
+      ...cleanOutput,
+    })
+    expect(report.score_matrix).toHaveLength(6)
+    const deepAccuracy = report.score_matrix.find(c => c.participant_id === 'deep' && c.criterion === 'accuracy')
+    expect(deepAccuracy!.score).toBe(5)
+  })
+
+  test('messy LLM output with field name variants gets normalized', () => {
+    // Simulates a messy Claude output — participantId instead of participant_id,
+    // reason instead of rationale, string score
+    const messyLLMOutput = {
+      participantId: 'bare',
+      reason: 'OK',
+      key_findings: ['found bugs'],
+      score_matrix: [
+        { participantId: 'bare', criterion: 'accuracy', weight: 50, score: '4', reason: 'decent' },
+        { participantId: 'deep', criterion: 'accuracy', weight: 50, score: '5', reason: 'excellent' },
+      ],
+      recommendations: [
+        { role: 'developer', text: 'Add more tests' },
+      ],
+    }
+    const normalized = normalizeComparativeOutput(messyLLMOutput as Record<string, unknown>)
+    const cells = normalized.score_matrix as any[]
+    expect(cells[0].participant_id).toBe('bare')
+    expect(cells[0].weight).toBe(0.5)
+    expect(cells[0].score).toBe(4)
+    expect(cells[0].rationale).toBe('decent')
+    const recs = normalized.recommendations as any[]
+    expect(recs[0].audience).toBe('developer')
+  })
+})

package/src/comparative-judge.ts

@@ -53,15 +53,38 @@ export function computePareto(vectors: { participant_id: string; scores: Record<
 
 // ── Comparative Judge Prompt ──────────────────────────────────────────────
 
-function buildComparativePrompt(opts: {
+export function buildComparativePrompt(opts: {
   manifest: ArenaManifest
   verdicts: { participantId: string; verdict: unknown }[]
 }): string {
-  const criteriaDesc = opts.manifest.criteria.join(', ')
   const participants = opts.manifest.participants
     .map(p => `- ${p.id}: ${p.name} (${p.description || 'no description'})`)
     .join('\n')
 
+  // Format criteria with rubric anchors when available (ADR-20260505225159725)
+  let criteriaBlock = ''
+  for (const c of opts.manifest.criteria) {
+    if (typeof c === 'string') {
+      criteriaBlock += `- ${c} (score 1-5, weight: 0.25)\n`
+    } else {
+      criteriaBlock += `## Criterion: ${c.label} (${c.id})\n`
+      if (c.persona) criteriaBlock += `Evaluator: ${c.persona}\n`
+      criteriaBlock += `Weight: ${c.weight ?? 25} (${c.weight ?? 25}%)\n`
+      criteriaBlock += `Description: ${c.description || 'No additional description.'}\n`
+      if (c.rubric && c.rubric.length > 0) {
+        criteriaBlock += 'Scoring rubric:\n'
+        for (const r of c.rubric) {
+          criteriaBlock += `  ${r.score} — ${r.label}: ${r.description}\n`
+        }
+      }
+      criteriaBlock += '\n'
+    }
+  }
+
+  const criteriaList = opts.manifest.criteria
+    .map(c => typeof c === 'string' ? c : `${c.label} (${c.id})`)
+    .join(', ')
+
   return `You are a comparative judge evaluating ${opts.manifest.participants.length} participants against shared criteria.
 
 ## Task
@@ -71,11 +94,11 @@ ${opts.manifest.task}
 ${participants}
 
 ## Criteria
-${criteriaDesc}
-
+${criteriaBlock}
 ## Your Job
 For each participant, score them 1-5 on each criterion. Provide a brief rationale.
 Score meanings: 1=poor, 3=acceptable, 5=excellent.
+Criteria in scope: ${criteriaList}
 
 ## Output Schema
 Your response must conform to this Zod schema:
@@ -96,13 +119,13 @@ z.object({
 })
 \`\`\`
 score_matrix is a FLAT ARRAY of objects — NOT nested by participant or criterion.
-weight: 0.25 for each cell (1 / num_criteria).
+weight: match the weight specified per criterion above.
 score: 1=poor, 3=acceptable, 5=excellent.
 
 Use the submit_scores tool to return your structured evaluation.`
 }
 
-function toScoreMatrix(
+export function toScoreMatrix(
   manifest: ArenaManifest,
   scores: { participant_id: string; criterion: string; weight: number; score: number; rationale: string }[]
 ): typeof ScoreCell._output[] {
@@ -119,7 +142,7 @@ interface NormalizedScoreCell {
   rationale: string
 }
 
-function normalizeComparativeOutput(parsed: Record<string, unknown>): Record<string, unknown> {
+export function normalizeComparativeOutput(parsed: Record<string, unknown>): Record<string, unknown> {
   const out = { ...parsed }
 
   // Detect pivot-table format: { participant: { criterion: { score, rationale } } }

package/src/runner.ts

@@ -1,5 +1,6 @@
 import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'
 import { join, resolve } from 'node:path'
+import { tmpdir } from 'node:os'
 import { runAgentScenario, type AgentScenario } from '@lythos/test-utils/agent-bdd'
 import { useAgent } from '@lythos/test-utils/agents'
 import { ArenaManifest, Player } from '@lythos/test-utils/schema'
@@ -110,16 +111,16 @@ export async function runArenaFromToml(opts: {
           const deckContent = readFileSync(cell.deck, 'utf-8')
           writeFileSync(join(workdir, 'skill-deck.toml'), deckContent)
 
-          // Link skills into .claude/skills/ so claude -p can discover them
-          const deckCli = resolve(import.meta.dir, '..', '..', 'lythoskill-deck', 'src', 'cli.ts')
-          const linkProc = Bun.spawn(['bun', 'run', deckCli, 'link'], {
-            cwd: workdir,
-            env: { ...process.env, HOME: process.env.HOME },
-          })
+          // Link skills via bunx (works both locally and when installed via bunx)
+          const linkProc = Bun.spawn(
+            ['bunx', '@lythos/skill-deck', 'link'],
+            { cwd: workdir, env: { ...process.env, HOME: process.env.HOME! } },
+          )
           await linkProc.exited
           log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
         },
-        baseDir: join(artifactsDir, 'runs', cell.side),
+        // Isolated CWD: /tmp/arena-<id>/<side>/ — no parent .claude/skills/ to walk up into
+        baseDir: join(tmpdir(), `arena-${arenaId}`, cell.side),
       })
 
       const v = (result.verdict ?? {
@@ -220,7 +221,7 @@ function writeReport(dir: string, manifest: ArenaManifestType, report: unknown &
     `# Arena Report: ${manifest.id}`,
     '',
     `**Task**: ${manifest.task}`,
-    `**Criteria**: ${manifest.criteria.join(', ')}`,
+    `**Criteria**: ${manifest.criteria.map(c => typeof c === 'string' ? c : c.label).join(', ')}`,
     `**Date**: ${new Date().toISOString()}`,
     '',
     '## Score Matrix',