@lythos/skill-arena 0.15.6 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -16,20 +16,20 @@
16
16
  ```bash
17
17
  bun add -d @lythos/skill-arena
18
18
  # or use directly
19
- bunx @lythos/skill-arena@0.15.6 <command>
19
+ bunx @lythos/skill-arena@0.16.0 <command>
20
20
  ```
21
21
 
22
22
  ## Quick Start
23
23
 
24
24
  ```bash
25
25
  # single — test one deck (most common)
26
- bunx @lythos/skill-arena@0.15.6 single \
26
+ bunx @lythos/skill-arena@0.16.0 single \
27
27
  --deck ./examples/decks/scout.toml \
28
28
  --brief "Generate auth flow diagram" \
29
29
  --out ./output
30
30
 
31
31
  # single with explicit player
32
- bunx @lythos/skill-arena@0.15.6 single \
32
+ bunx @lythos/skill-arena@0.16.0 single \
33
33
  --deck ./examples/decks/scout.toml \
34
34
  --brief "Generate auth flow diagram" \
35
35
  --player kimi \
@@ -37,10 +37,10 @@ bunx @lythos/skill-arena@0.15.6 single \
37
37
 
38
38
  # cross-deck vs — compare two decks (agent-orchestrated)
39
39
  # Create arena.toml declaring sides with different decks, then:
40
- bunx @lythos/skill-arena@0.15.6 vs --config ./arena.toml
40
+ bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml
41
41
 
42
42
  # cross-player vs — compare kimi vs codex (CLI only)
43
- bunx @lythos/skill-arena@0.15.6 vs --config ./arena.toml --player kimi
43
+ bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml --player kimi
44
44
  ```
45
45
 
46
46
  **What happens**: Agent creates isolated `/tmp` workdir per side, `deck link` skills, spawns parallel subagents, collects artifacts, judge scores outputs. Parent deck restored after.
@@ -50,14 +50,14 @@ bunx @lythos/skill-arena@0.15.6 vs --config ./arena.toml --player kimi
50
50
  ### `single` — one deck, one task
51
51
 
52
52
  ```bash
53
- bunx @lythos/skill-arena@0.15.6 single \
53
+ bunx @lythos/skill-arena@0.16.0 single \
54
54
  --deck ./deck.toml \
55
55
  --brief "Produce a .docx report with radar chart" \
56
56
  --timeout 600000 \
57
57
  --out ./output
58
58
 
59
59
  # with explicit player
60
- bunx @lythos/skill-arena@0.15.6 single \
60
+ bunx @lythos/skill-arena@0.16.0 single \
61
61
  --deck ./deck.toml \
62
62
  --brief "Produce a .docx report with radar chart" \
63
63
  --player kimi \
@@ -67,14 +67,14 @@ bunx @lythos/skill-arena@0.15.6 single \
67
67
  ### `vs` — multi-deck comparison
68
68
 
69
69
  ```bash
70
- bunx @lythos/skill-arena@0.15.6 vs --config ./arena.toml
71
- bunx @lythos/skill-arena@0.15.6 vs --config ./arena.toml --dry-run
70
+ bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml
71
+ bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml --dry-run
72
72
  ```
73
73
 
74
74
  ### `prepare-workdir` — isolate + link skills (agent-orchestrated)
75
75
 
76
76
  ```bash
77
- bunx @lythos/skill-arena@0.15.6 prepare-workdir \
77
+ bunx @lythos/skill-arena@0.16.0 prepare-workdir \
78
78
  --deck ./skill-deck.toml \
79
79
  --out /tmp/arena-side-a \
80
80
  --brief "task description"
@@ -85,7 +85,7 @@ Creates `/tmp`-isolated workdir with deck copied, AGENTS.md written, and `deck l
85
85
  ### `archive` — collect agent outputs (agent-orchestrated)
86
86
 
87
87
  ```bash
88
- bunx @lythos/skill-arena@0.15.6 archive \
88
+ bunx @lythos/skill-arena@0.16.0 archive \
89
89
  --from /tmp/arena-side-a \
90
90
  --to ./playground/output \
91
91
  --sides side-a
@@ -96,7 +96,7 @@ Copies agent artifacts from workdir(s) to output, skipping internal files (`.cla
96
96
  ### `viz` — render results (WIP — HTML report generation pending)
97
97
 
98
98
  ```bash
99
- bunx @lythos/skill-arena@0.15.6 viz runs/arena-<id>/
99
+ bunx @lythos/skill-arena@0.16.0 viz runs/arena-<id>/
100
100
  ```
101
101
 
102
102
  ## Parameters
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lythos/skill-arena",
3
- "version": "0.15.6",
3
+ "version": "0.16.0",
4
4
  "description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
5
5
  "keywords": [
6
6
  "ai-agent",
@@ -42,15 +42,15 @@
42
42
  "bun": ">=1.0.0"
43
43
  },
44
44
  "dependencies": {
45
- "@lythos/cold-pool": "^0.15.6",
46
- "@lythos/infra": "^0.15.6",
47
- "@lythos/test-utils": "^0.15.6",
45
+ "@lythos/cold-pool": "^0.16.0",
46
+ "@lythos/infra": "^0.16.0",
47
+ "@lythos/test-utils": "^0.16.0",
48
48
  "zod": "^3.24.0",
49
49
  "zod-to-json-schema": "^3.25.2"
50
50
  },
51
51
  "optionalDependencies": {
52
- "@lythos/agent-adapter-claude-sdk": "^0.15.6",
53
- "@lythos/agent-adapter-deepseek-serve": "^0.15.6",
54
- "@lythos/agent-adapter-codex": "^0.15.6"
52
+ "@lythos/agent-adapter-claude-sdk": "^0.16.0",
53
+ "@lythos/agent-adapter-deepseek-serve": "^0.16.0",
54
+ "@lythos/agent-adapter-codex": "^0.16.0"
55
55
  }
56
56
  }
@@ -0,0 +1,118 @@
1
+ import { describe, it, expect } from 'bun:test'
2
+ import { main, defaultArenaCliIO, type ArenaCliIO } from './cli'
3
+
4
+ // ─── Helpers ───────────────────────────────────────────────────────────────
5
+
6
+ async function catchExitAsync(fn: () => Promise<void>): Promise<number | undefined> {
7
+ let exitCode: number | undefined
8
+ try { await fn() } catch (e: any) {
9
+ if (!String(e).includes('EXIT:')) throw e
10
+ const m = String(e).match(/EXIT:(\d+)/)
11
+ if (m) exitCode = parseInt(m[1], 10)
12
+ }
13
+ return exitCode
14
+ }
15
+
16
+ function mockIO(): { io: Required<ArenaCliIO>; logs: string[]; errors: string[]; warns: string[] } {
17
+ const logs: string[] = []
18
+ const errors: string[] = []
19
+ const warns: string[] = []
20
+ const io: Required<ArenaCliIO> = {
21
+ log: (msg: string) => logs.push(msg),
22
+ error: (msg: string) => errors.push(msg),
23
+ warn: (msg: string) => warns.push(msg),
24
+ exit: (code: number) => { throw new Error(`EXIT:${code}`) },
25
+ }
26
+ return { io, logs, errors, warns }
27
+ }
28
+
29
+ // ─── Tests ─────────────────────────────────────────────────────────────────
30
+
31
+ describe('main', () => {
32
+ it('shows help on --help', async () => {
33
+ const { io, logs, errors } = mockIO()
34
+ const exitCode = await catchExitAsync(async () => {
35
+ await main(['--help'], io)
36
+ })
37
+ expect(exitCode).toBe(0)
38
+ expect(logs.length).toBeGreaterThan(0)
39
+ expect(logs[0]).toContain('lythoskill-arena')
40
+ expect(errors.length).toBe(0)
41
+ })
42
+
43
+ it('shows help on -h', async () => {
44
+ const { io, logs, errors } = mockIO()
45
+ const exitCode = await catchExitAsync(async () => {
46
+ await main(['-h'], io)
47
+ })
48
+ expect(exitCode).toBe(0)
49
+ expect(logs.length).toBeGreaterThan(0)
50
+ expect(logs[0]).toContain('lythoskill-arena')
51
+ expect(errors.length).toBe(0)
52
+ })
53
+
54
+ it('shows help on no args', async () => {
55
+ const { io, logs, errors } = mockIO()
56
+ const exitCode = await catchExitAsync(async () => {
57
+ await main([], io)
58
+ })
59
+ expect(exitCode).toBe(0)
60
+ expect(logs.length).toBeGreaterThan(0)
61
+ expect(errors.length).toBe(0)
62
+ })
63
+
64
+ it('errors on unknown command', async () => {
65
+ const { io, logs, errors } = mockIO()
66
+ const exitCode = await catchExitAsync(async () => {
67
+ await main(['bogus'], io)
68
+ })
69
+ expect(exitCode).toBe(1)
70
+ expect(errors.length).toBeGreaterThan(0)
71
+ expect(errors[0]).toContain('Unknown command')
72
+ expect(errors[0]).toContain('bogus')
73
+ })
74
+ })
75
+
76
+ describe('singleRun', () => {
77
+ it('errors when --deck is missing', async () => {
78
+ const { io, logs, errors } = mockIO()
79
+ const exitCode = await catchExitAsync(async () => {
80
+ await main(['single', '--brief', 'test task'], io)
81
+ })
82
+ expect(exitCode).toBe(1)
83
+ expect(errors.length).toBeGreaterThan(0)
84
+ expect(errors[0]).toContain('--deck')
85
+ expect(errors[0]).toContain('required')
86
+ })
87
+
88
+ it('errors when both --task and --brief are missing', async () => {
89
+ const { io, logs, errors } = mockIO()
90
+ const exitCode = await catchExitAsync(async () => {
91
+ await main(['single', '--deck', './some-deck.toml'], io)
92
+ })
93
+ expect(exitCode).toBe(1)
94
+ expect(errors.length).toBeGreaterThan(0)
95
+ expect(errors[0]).toContain('--task')
96
+ expect(errors[0]).toContain('--brief')
97
+ })
98
+
99
+ it('errors when --brief is empty string', async () => {
100
+ const { io, logs, errors } = mockIO()
101
+ const exitCode = await catchExitAsync(async () => {
102
+ await main(['single', '--deck', './some-deck.toml', '--brief', ''], io)
103
+ })
104
+ expect(exitCode).toBe(1)
105
+ expect(errors.length).toBeGreaterThan(0)
106
+ expect(errors[0]).toContain('--task')
107
+ expect(errors[0]).toContain('--brief')
108
+ })
109
+ })
110
+
111
+ describe('defaultArenaCliIO', () => {
112
+ it('has all required functions', () => {
113
+ expect(typeof defaultArenaCliIO.log).toBe('function')
114
+ expect(typeof defaultArenaCliIO.error).toBe('function')
115
+ expect(typeof defaultArenaCliIO.warn).toBe('function')
116
+ expect(typeof defaultArenaCliIO.exit).toBe('function')
117
+ })
118
+ })
package/src/cli.ts CHANGED
@@ -8,6 +8,22 @@ import { parseArenaToml, buildExecutionPlan } from './arena-toml'
8
8
  import { buildArchiveSidePlan, buildCopyPlan, buildPreparePlan, parseDeckSkills } from './preflight'
9
9
  import { checkSkillExistence, formatSkillWarnings, resolveColdPoolDir } from './preflight'
10
10
 
11
+ // ─── ArenaCliIO interface (injected for testability) ───────────────────────
12
+
13
+ export interface ArenaCliIO {
14
+ log?: (msg: string) => void
15
+ error?: (msg: string) => void
16
+ warn?: (msg: string) => void
17
+ exit?: (code: number) => never
18
+ }
19
+
20
+ export const defaultArenaCliIO: Required<ArenaCliIO> = {
21
+ log: console.log,
22
+ error: console.error,
23
+ warn: console.warn,
24
+ exit: (code) => { process.exit(code) },
25
+ }
26
+
11
27
  // ─── fetchWithProxy (infra dependency, no package boundary) ─────────────────
12
28
 
13
29
  async function fetchWithProxy(url: string, init?: RequestInit): Promise<Response> {
@@ -82,9 +98,10 @@ function validateLinkResult(exitCode: number | null, stderr: string): { ok: bool
82
98
  }
83
99
 
84
100
  // ═══════════════════════════════════════════════════════════════════════════
85
- export async function main(args: string[] = process.argv.slice(2)) {
101
+ export async function main(args: string[] = process.argv.slice(2), io: ArenaCliIO = defaultArenaCliIO) {
102
+ const { log, exit } = { ...defaultArenaCliIO, ...io }
86
103
  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
87
- console.log(`lythoskill-arena — skill evaluation CLI
104
+ log(`lythoskill-arena — skill evaluation CLI
88
105
 
89
106
  Usage:
90
107
  lythoskill-arena single|vs|viz <options>
@@ -103,28 +120,30 @@ Examples:
103
120
  lythoskill-arena prepare-workdir --deck ./decks/scout.toml --out /tmp/arena-20260517-side-a
104
121
  lythoskill-arena archive --from /tmp/arena-20260517 --to playground/arena-20260517 --sides side-a,side-b
105
122
  `)
106
- process.exit(0)
123
+ exit(0)
107
124
  }
108
- return cli(args)
125
+ return cli(args, io)
109
126
  }
110
127
 
111
- function cli(args: string[]) {
128
+ function cli(args: string[], io: ArenaCliIO) {
112
129
  const cmd = args[0]
113
130
  const rest = args.slice(1)
114
131
 
115
- if (cmd === 'vs' || cmd === 'compare') return vsRun(rest)
116
- if (cmd === 'single' || cmd === 'run') return singleRun(rest)
117
- if (cmd === 'viz') return vizRun(rest)
118
- if (cmd === 'prepare-workdir') return prepareWorkdir(rest)
119
- if (cmd === 'archive') return archiveRun(rest)
132
+ if (cmd === 'vs' || cmd === 'compare') return vsRun(rest, io)
133
+ if (cmd === 'single' || cmd === 'run') return singleRun(rest, io)
134
+ if (cmd === 'viz') return vizRun(rest, io)
135
+ if (cmd === 'prepare-workdir') return prepareWorkdir(rest, io)
136
+ if (cmd === 'archive') return archiveRun(rest, io)
120
137
 
121
- console.error(`Unknown command: ${cmd}`)
122
- process.exit(1)
138
+ const { error, exit } = { ...defaultArenaCliIO, ...io }
139
+ error(`Unknown command: ${cmd}`)
140
+ exit(1)
123
141
  }
124
142
 
125
143
  // ── single: single-player deck test (exec shortcut) ──────────────────────
126
144
 
127
- async function singleRun(args: string[]) {
145
+ async function singleRun(args: string[], io: ArenaCliIO) {
146
+ const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
128
147
  const opts: Record<string, string | undefined> = {}
129
148
  for (let i = 0; i < args.length; i++) {
130
149
  if (args[i] === '--task' || args[i] === '-t') opts.task = args[++i]
@@ -136,7 +155,7 @@ async function singleRun(args: string[]) {
136
155
  }
137
156
 
138
157
  if (!opts.deck) {
139
- console.error(`❌ --deck <path|url> is required.
158
+ error(`❌ --deck <path|url> is required.
140
159
  --deck accepts local paths and http/https URLs (auto-fetched).
141
160
 
142
161
  Example (no local file needed — URL is auto-fetched):
@@ -146,10 +165,10 @@ async function singleRun(args: string[]) {
146
165
 
147
166
  Or with a local deck file you already have:
148
167
  lythoskill-arena single --deck ./examples/decks/scout.toml --brief "your task"`)
149
- process.exit(1)
168
+ exit(1)
150
169
  }
151
170
  if (!opts.task && (!opts.brief || !opts.brief.trim())) {
152
- console.error(`❌ --task <path> or --brief "<text>" is required.
171
+ error(`❌ --task <path> or --brief "<text>" is required.
153
172
  --task reads a .agent.md scenario file; --brief takes inline text.
154
173
 
155
174
  Example (no local file needed — URL is auto-fetched):
@@ -159,24 +178,24 @@ async function singleRun(args: string[]) {
159
178
 
160
179
  Or with a local deck file:
161
180
  lythoskill-arena single --deck ./examples/decks/scout.toml --brief "your task"`)
162
- process.exit(1)
181
+ exit(1)
163
182
  }
164
183
 
165
184
  let resolvedTaskPath: string | undefined
166
185
  if (opts.task) {
167
186
  resolvedTaskPath = resolve(opts.task)
168
187
  if (!existsSync(resolvedTaskPath)) {
169
- console.error(`❌ Task file not found: ${resolvedTaskPath}
188
+ error(`❌ Task file not found: ${resolvedTaskPath}
170
189
  Use --brief for inline tasks, or point --task to an existing .agent.md file.
171
190
  Format: name + description + Given/When/Then sections.
172
191
 
173
192
  Example (URL): lythoskill-arena single --brief "your task" --deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml
174
193
  Or (local): lythoskill-arena single --brief "your task" --deck ./examples/decks/scout.toml`)
175
- process.exit(1)
194
+ exit(1)
176
195
  }
177
196
  const raw = readFileSync(resolvedTaskPath, 'utf-8')
178
197
  if (!raw.startsWith('---')) {
179
- console.error(`❌ Invalid .agent.md: missing frontmatter (must start with "---")
198
+ error(`❌ Invalid .agent.md: missing frontmatter (must start with "---")
180
199
  Correct format:
181
200
  ---
182
201
  name: my-scenario
@@ -189,13 +208,13 @@ async function singleRun(args: string[]) {
189
208
  ...
190
209
  ## Then
191
210
  ...`)
192
- process.exit(1)
211
+ exit(1)
193
212
  }
194
213
  if (!raw.includes('## When')) {
195
- console.error(`❌ Invalid .agent.md: missing "## When" section.
214
+ error(`❌ Invalid .agent.md: missing "## When" section.
196
215
  The ## When section defines what the agent should do.
197
216
  See template: playground/arena-one-shot/TASK-arena.agent.md`)
198
- process.exit(1)
217
+ exit(1)
199
218
  }
200
219
  }
201
220
 
@@ -213,7 +232,7 @@ async function singleRun(args: string[]) {
213
232
  }
214
233
  const { mirrorUrls, isLikelyGitHubBlock } = await import('../../lythoskill-cold-pool/src/mirror.js')
215
234
  const dest = resolve(process.cwd(), 'arena-deck.toml')
216
- console.log(`📥 Fetching arena deck: ${url}`)
235
+ log(`📥 Fetching arena deck: ${url}`)
217
236
  let res: Response | undefined
218
237
  let allFailed = true
219
238
 
@@ -222,7 +241,7 @@ async function singleRun(args: string[]) {
222
241
  if (!res?.ok) {
223
242
  for (const mirrorUrl of mirrorUrls(url)) {
224
243
  try {
225
- console.log(` ↳ trying mirror: ${mirrorUrl}`)
244
+ log(` ↳ trying mirror: ${mirrorUrl}`)
226
245
  const r = await fetchWithProxy(mirrorUrl, { signal: AbortSignal.timeout(30_000) })
227
246
  if (r.ok) { res = r; allFailed = false; break }
228
247
  } catch {}
@@ -231,21 +250,21 @@ async function singleRun(args: string[]) {
231
250
 
232
251
  if (!res?.ok) {
233
252
  const errorDetail = res ? `HTTP ${res.status}` : 'unreachable'
234
- console.error(`❌ Cannot reach ${url} (${errorDetail})`)
235
- if (allFailed) console.error(' Set LYTHOS_GH_MIRROR to use a custom mirror.')
236
- console.error(' Or download manually and reference the local file.')
237
- process.exit(1)
253
+ error(`❌ Cannot reach ${url} (${errorDetail})`)
254
+ if (allFailed) error(' Set LYTHOS_GH_MIRROR to use a custom mirror.')
255
+ error(' Or download manually and reference the local file.')
256
+ exit(1)
238
257
  }
239
258
 
240
259
  deckWrite(dest, await res.text())
241
- console.log(` → saved to ${dest}`)
260
+ log(` → saved to ${dest}`)
242
261
  deckPath = dest
243
262
  } else {
244
263
  deckPath = resolve(opts.deck)
245
- if (!deckExists(deckPath)) { console.error(`❌ Deck file not found: ${deckPath}
264
+ if (!deckExists(deckPath)) { error(`❌ Deck file not found: ${deckPath}
246
265
  Make sure the path is correct, or use a URL:
247
266
  --deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml
248
- (URLs are auto-fetched — no local file needed)`); process.exit(1) }
267
+ (URLs are auto-fetched — no local file needed)`); exit(1) }
249
268
  }
250
269
 
251
270
  const { useAgent } = await import('@lythos/test-utils/agents')
@@ -263,9 +282,9 @@ async function singleRun(args: string[]) {
263
282
  // Markdown is for LLM agents; task text is read/stored as a raw string.
264
283
  const taskText = resolvedTaskPath ? readFileSync(resolvedTaskPath, 'utf-8') : opts.brief!
265
284
 
266
- console.log(`🤖 agent-run: ${player} × ${deckPath}`)
267
- if (opts.task) console.log(`📋 task: ${resolve(opts.task!)}`)
268
- else console.log(`📋 brief: ${opts.brief!.slice(0, 60)}...`)
285
+ log(`🤖 agent-run: ${player} × ${deckPath}`)
286
+ if (opts.task) log(`📋 task: ${resolve(opts.task!)}`)
287
+ else log(`📋 brief: ${opts.brief!.slice(0, 60)}...`)
269
288
 
270
289
  // Setup workdir
271
290
  const agentWorkdir = join(tmpdir(), `arena-single-${Date.now()}`)
@@ -298,11 +317,11 @@ async function singleRun(args: string[]) {
298
317
  const linkStderr = await new Response(linkProc.stderr).text()
299
318
  const linkResult = validateLinkResult(linkProc.exitCode, linkStderr)
300
319
  if (!linkResult.ok) {
301
- console.error(`❌ ${linkResult.error}`)
302
- process.exit(1)
320
+ error(`❌ ${linkResult.error}`)
321
+ exit(1)
303
322
  }
304
323
  } else {
305
- console.log('ℹ️ No skills declared in deck — skipping link')
324
+ log('ℹ️ No skills declared in deck — skipping link')
306
325
  }
307
326
 
308
327
  const { existsSync: es } = await import('node:fs')
@@ -313,10 +332,10 @@ async function singleRun(args: string[]) {
313
332
  const skills = parseDeckSkills(deckParsed)
314
333
  const checks = checkSkillExistence(skills, coldPoolDir, es)
315
334
  for (const warning of formatSkillWarnings(checks)) {
316
- console.warn(`⚠️ ${warning}`)
335
+ warn(`⚠️ ${warning}`)
317
336
  }
318
337
  } catch (e) {
319
- console.warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
338
+ warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
320
339
  }
321
340
 
322
341
  // Template injection: brief is the {task} variable, template carries fixed contract
@@ -345,23 +364,24 @@ async function singleRun(args: string[]) {
345
364
  const plan = buildCopyPlan(agentWorkdir, outDir, entries, skipSet)
346
365
  for (const { src, dest, name } of plan) {
347
366
  try { cpSync(src, dest, { recursive: true }) } catch (e) {
348
- console.warn(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
367
+ warn(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
349
368
  }
350
369
  }
351
370
  } catch (e) {
352
- console.warn(`⚠️ Failed to copy agent output: ${e instanceof Error ? e.message : e}`)
371
+ warn(`⚠️ Failed to copy agent output: ${e instanceof Error ? e.message : e}`)
353
372
  }
354
373
  }
355
374
 
356
375
  // Summary (no judge — single mode is execution-only)
357
- console.log(`\n✅ Agent run complete → ${outDir}`)
358
- console.log(` deck: ${deckPath}`)
359
- console.log(` player: ${player}`)
376
+ log(`\n✅ Agent run complete → ${outDir}`)
377
+ log(` deck: ${deckPath}`)
378
+ log(` player: ${player}`)
360
379
  }
361
380
 
362
381
  // ── vs: arena.toml-driven comparison ──────────────────────────────────────
363
382
 
364
- async function vsRun(args: string[]) {
383
+ async function vsRun(args: string[], io: ArenaCliIO) {
384
+ const { log, error, exit } = { ...defaultArenaCliIO, ...io }
365
385
  // Native TOML parser is simpler than adding smol-toml dependency
366
386
  const opts: Record<string, string | undefined> = {}
367
387
  for (let i = 0; i < args.length; i++) {
@@ -372,14 +392,14 @@ async function vsRun(args: string[]) {
372
392
  }
373
393
 
374
394
  if (!opts.config) {
375
- console.error('❌ arena.toml path required: lythoskill-arena vs --config arena.toml')
376
- process.exit(1)
395
+ error('❌ arena.toml path required: lythoskill-arena vs --config arena.toml')
396
+ exit(1)
377
397
  }
378
398
 
379
399
  const configPath = resolve(opts.config)
380
400
  if (!existsSync(configPath)) {
381
- console.error(`❌ Config file not found: ${configPath}`)
382
- process.exit(1)
401
+ error(`❌ Config file not found: ${configPath}`)
402
+ exit(1)
383
403
  }
384
404
 
385
405
  const toml = parseArenaToml(readFileSync(configPath, 'utf-8'))
@@ -395,11 +415,11 @@ async function vsRun(args: string[]) {
395
415
  const isDryRun = opts.dryRun === 'true'
396
416
 
397
417
  if (isDryRun) {
398
- console.log(`🔍 Scanning arena.toml: ${configPath}`)
418
+ log(`🔍 Scanning arena.toml: ${configPath}`)
399
419
  } else {
400
- console.log(`🏟 Arena VS: ${configPath}`)
401
- console.log(` sides: ${toml.side.length}`)
402
- console.log(` runs per side: ${toml.arena.runs_per_side}`)
420
+ log(`🏟 Arena VS: ${configPath}`)
421
+ log(` sides: ${toml.side.length}`)
422
+ log(` runs per side: ${toml.arena.runs_per_side}`)
403
423
  }
404
424
 
405
425
  const { runArenaFromToml } = await import('./runner')
@@ -408,38 +428,40 @@ async function vsRun(args: string[]) {
408
428
  taskPath,
409
429
  outDir: opts.out ? resolve(opts.out) : undefined,
410
430
  dryRun: isDryRun,
411
- log: console.log,
431
+ log: log,
412
432
  configDir: resolve(configPath, '..'),
413
433
  })
414
434
 
415
435
  if ('plan' in result) {
416
- if (!isDryRun) console.log('📋 Execution plan (dry-run):')
417
- for (const line of formatPlanOutput(result.plan)) console.log(line)
436
+ if (!isDryRun) log('📋 Execution plan (dry-run):')
437
+ for (const line of formatPlanOutput(result.plan)) log(line)
418
438
  } else if ('manifest' in result) {
419
439
  const r = result
420
- console.log(`\n📊 Arena complete: ${r.manifest.id}`)
421
- console.log(` report: ${r.artifactsDir}/report.md`)
422
- console.log(` participants: ${r.manifest.participants.map(p => p.name).join(', ')}`)
440
+ log(`\n📊 Arena complete: ${r.manifest.id}`)
441
+ log(` report: ${r.artifactsDir}/report.md`)
442
+ log(` participants: ${r.manifest.participants.map(p => p.name).join(', ')}`)
423
443
  }
424
444
  }
425
445
 
426
446
  // ── viz: generate HTML report from arena.json ─────────────────────────────
427
447
 
428
- async function vizRun(args: string[]) {
448
+ async function vizRun(args: string[], io: ArenaCliIO) {
449
+ const { error, exit, log } = { ...defaultArenaCliIO, ...io }
429
450
  const runsDir = args.find(a => !a.startsWith('-'))
430
- if (!runsDir) { console.error('❌ runs/<arena-id> path required: lythoskill-arena viz runs/arena-20260504'); process.exit(1) }
451
+ if (!runsDir) { error('❌ runs/<arena-id> path required: lythoskill-arena viz runs/arena-20260504'); exit(1) }
431
452
 
432
453
  const arenaJsonPath = resolve(runsDir, 'arena.json')
433
- if (!existsSync(arenaJsonPath)) { console.error(`❌ arena.json not found in: ${runsDir}`); process.exit(1) }
454
+ if (!existsSync(arenaJsonPath)) { error(`❌ arena.json not found in: ${runsDir}`); exit(1) }
434
455
 
435
- console.log(`📈 Arena HTML report not yet implemented. See report.md in ${runsDir}/`)
456
+ log(`📈 Arena HTML report not yet implemented. See report.md in ${runsDir}/`)
436
457
  }
437
458
 
438
459
  // ═══════════════════════════════════════════════════════════════════════════
439
460
  // ── prepare-workdir: reusable workdir setup (used by both CLI and agent) ──
440
461
  // Intent: create an isolated arena workdir with deck linked and ready to run
441
462
 
442
- async function prepareWorkdir(args: string[]) {
463
+ async function prepareWorkdir(args: string[], io: ArenaCliIO) {
464
+ const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
443
465
  const opts: Record<string, string | undefined> = {}
444
466
  let dryRun = false
445
467
  for (let i = 0; i < args.length; i++) {
@@ -450,15 +472,15 @@ async function prepareWorkdir(args: string[]) {
450
472
  }
451
473
 
452
474
  if (!opts.deck) {
453
- console.error(`❌ --deck <path> is required.
475
+ error(`❌ --deck <path> is required.
454
476
  lythoskill-arena prepare-workdir --deck ./skill-deck.toml --out /tmp/arena-side-a`)
455
- process.exit(1)
477
+ exit(1)
456
478
  }
457
479
 
458
480
  const deckPath = resolve(opts.deck)
459
481
  if (!existsSync(deckPath)) {
460
- console.error(`❌ Deck file not found: ${deckPath}`)
461
- process.exit(1)
482
+ error(`❌ Deck file not found: ${deckPath}`)
483
+ exit(1)
462
484
  }
463
485
 
464
486
  const workDir = opts.out
@@ -475,16 +497,16 @@ async function prepareWorkdir(args: string[]) {
475
497
  brief: opts.brief,
476
498
  })
477
499
 
478
- console.log('📋 Prepare plan:')
479
- console.log(` deck: ${plan.deckPath}`)
480
- console.log(` workdir: ${plan.workDir}`)
481
- console.log(` skills: ${plan.skills.length} declared (${plan.skills.map(s => s.name).join(', ') || 'none'})`)
482
- console.log(` link: ${plan.hasSkills ? 'Bun.spawn deck link' : 'skip (no skills)'}`)
483
- console.log(` AGENTS.md: write (${plan.agentsMd.split('\n').length} lines)`)
484
- if (opts.brief) console.log(` brief: ${opts.brief!.slice(0, 60)}...`)
500
+ log('📋 Prepare plan:')
501
+ log(` deck: ${plan.deckPath}`)
502
+ log(` workdir: ${plan.workDir}`)
503
+ log(` skills: ${plan.skills.length} declared (${plan.skills.map(s => s.name).join(', ') || 'none'})`)
504
+ log(` link: ${plan.hasSkills ? 'Bun.spawn deck link' : 'skip (no skills)'}`)
505
+ log(` AGENTS.md: write (${plan.agentsMd.split('\n').length} lines)`)
506
+ if (opts.brief) log(` brief: ${opts.brief!.slice(0, 60)}...`)
485
507
 
486
508
  if (dryRun) {
487
- console.log(`\n🏁 Dry-run complete (no files created). Remove --dry-run to execute.`)
509
+ log(`\n🏁 Dry-run complete (no files created). Remove --dry-run to execute.`)
488
510
  return
489
511
  }
490
512
 
@@ -506,11 +528,11 @@ async function prepareWorkdir(args: string[]) {
506
528
  const linkStderr = await new Response(linkProc.stderr).text()
507
529
  const linkResult = validateLinkResult(linkProc.exitCode, linkStderr)
508
530
  if (!linkResult.ok) {
509
- console.error(`❌ ${linkResult.error}`)
510
- process.exit(1)
531
+ error(`❌ ${linkResult.error}`)
532
+ exit(1)
511
533
  }
512
534
  } else {
513
- console.log('ℹ️ No skills declared in deck — skipping link')
535
+ log('ℹ️ No skills declared in deck — skipping link')
514
536
  }
515
537
 
516
538
  // Skill existence check
@@ -519,22 +541,23 @@ async function prepareWorkdir(args: string[]) {
519
541
  const coldPoolDir = resolveColdPoolDir(Bun.TOML.parse(deckContent)?.deck?.cold_pool, homedir(), coldPoolDefault)
520
542
  const checks = checkSkillExistence(plan.skills, coldPoolDir, existsSync)
521
543
  for (const warning of formatSkillWarnings(checks)) {
522
- console.warn(`⚠️ ${warning}`)
544
+ warn(`⚠️ ${warning}`)
523
545
  }
524
546
  } catch (e) {
525
- console.warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
547
+ warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
526
548
  }
527
549
 
528
- console.log(`✅ Workdir ready → ${workDir}`)
529
- console.log(` deck: ${deckPath}`)
530
- if (opts.brief) console.log(` brief: ${opts.brief!.slice(0, 60)}...`)
550
+ log(`✅ Workdir ready → ${workDir}`)
551
+ log(` deck: ${deckPath}`)
552
+ if (opts.brief) log(` brief: ${opts.brief!.slice(0, 60)}...`)
531
553
  }
532
554
 
533
555
  // ═══════════════════════════════════════════════════════════════════════════
534
556
  // ── archive: copy agent outputs from workdir(s) to outDir ─────────────────
535
557
  // Intent: same copy behavior as CLI singleRun, reusable for agent-orchestrated
536
558
 
537
- async function archiveRun(args: string[]) {
559
+ async function archiveRun(args: string[], io: ArenaCliIO) {
560
+ const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
538
561
  const opts: Record<string, string | undefined> = {}
539
562
  let dryRun = false
540
563
  for (let i = 0; i < args.length; i++) {
@@ -546,9 +569,9 @@ async function archiveRun(args: string[]) {
546
569
  }
547
570
 
548
571
  if (!opts.from || !opts.to) {
549
- console.error(`❌ --from <workdir> and --to <outdir> are required.
572
+ error(`❌ --from <workdir> and --to <outdir> are required.
550
573
  lythoskill-arena archive --from /tmp/arena-20260517 --to playground/arena-20260517 --sides side-a,side-b --report ./report.md`)
551
- process.exit(1)
574
+ exit(1)
552
575
  }
553
576
 
554
577
  const fromDir = resolve(opts.from)
@@ -558,18 +581,18 @@ async function archiveRun(args: string[]) {
558
581
  const plan = buildArchiveSidePlan(fromDir, sides, existsSync)
559
582
 
560
583
  // ── Plan output (always shown, also serves as dry-run) ──────────────────
561
- console.log('📋 Archive plan:')
584
+ log('📋 Archive plan:')
562
585
  for (const pe of plan) {
563
586
  if (!pe.found) {
564
- console.log(` ⚠️ ${pe.side}: not found (${pe.sourceDir}) — will skip`)
587
+ log(` ⚠️ ${pe.side}: not found (${pe.sourceDir}) — will skip`)
565
588
  } else if (pe.sourceDir === fromDir && pe.side !== '.') {
566
- console.log(` ${pe.side}: ${pe.sourceDir} (fallback → root) → ${join(outDir, pe.side)}`)
589
+ log(` ${pe.side}: ${pe.sourceDir} (fallback → root) → ${join(outDir, pe.side)}`)
567
590
  } else {
568
- console.log(` ${pe.side}: ${pe.sourceDir} → ${join(outDir, pe.side)}`)
591
+ log(` ${pe.side}: ${pe.sourceDir} → ${join(outDir, pe.side)}`)
569
592
  }
570
593
  }
571
594
  if (dryRun) {
572
- console.log(`\n🏁 Dry-run complete (no files copied). Remove --dry-run to execute.`)
595
+ log(`\n🏁 Dry-run complete (no files copied). Remove --dry-run to execute.`)
573
596
  return
574
597
  }
575
598
 
@@ -579,7 +602,7 @@ async function archiveRun(args: string[]) {
579
602
  if (opts.report && existsSync(resolve(opts.report))) {
580
603
  const { cpSync: cpR } = await import('node:fs')
581
604
  cpR(resolve(opts.report), join(outDir, 'report.md'))
582
- console.log(`📄 report.md → ${outDir}/report.md`)
605
+ log(`📄 report.md → ${outDir}/report.md`)
583
606
  }
584
607
 
585
608
  const { cpSync, readdirSync } = await import('node:fs')
@@ -587,7 +610,7 @@ async function archiveRun(args: string[]) {
587
610
 
588
611
  for (const planEntry of plan) {
589
612
  if (!planEntry.found) {
590
- console.warn(`⚠️ Side workdir not found: ${planEntry.sourceDir}`)
613
+ warn(`⚠️ Side workdir not found: ${planEntry.sourceDir}`)
591
614
  continue
592
615
  }
593
616
 
@@ -601,14 +624,14 @@ async function archiveRun(args: string[]) {
601
624
  const dest = join(sideOutDir, entry.name)
602
625
  try {
603
626
  cpSync(src, dest, { recursive: entry.isDirectory() })
604
- console.log(` ${planEntry.side}/${entry.name} → ${dest}`)
627
+ log(` ${planEntry.side}/${entry.name} → ${dest}`)
605
628
  } catch (e) {
606
- console.warn(`⚠️ Failed to copy ${planEntry.side}/${entry.name}: ${e instanceof Error ? e.message : e}`)
629
+ warn(`⚠️ Failed to copy ${planEntry.side}/${entry.name}: ${e instanceof Error ? e.message : e}`)
607
630
  }
608
631
  }
609
632
  }
610
633
 
611
- console.log(`✅ Archive complete → ${outDir}`)
634
+ log(`✅ Archive complete → ${outDir}`)
612
635
  }
613
636
 
614
637
  // ── Entry point ────────────────────────────────────────────────────────────
@@ -1,5 +1,5 @@
1
1
  import { describe, test, expect } from 'bun:test'
2
- import { buildArenaPrompt, formatPlanOutput } from './runner'
2
+ import { buildArenaPrompt, formatPlanOutput, runArenaFromToml, type ArenaIO } from './runner'
3
3
  import { buildExecutionPlan, parseArenaToml } from './arena-toml'
4
4
 
5
5
  describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
@@ -89,3 +89,262 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
89
89
  expect(buildArenaPrompt(opts)).toBe(buildArenaPrompt(opts))
90
90
  })
91
91
  })
92
+
93
+ describe('runArenaFromToml — dry-run mode (no fs/spawn/agent calls)', () => {
94
+ test('returns plan without executing cells', async () => {
95
+ const toml = parseArenaToml(`
96
+ [arena]
97
+ task = "Write a hello world function"
98
+ criteria = ["completeness", "correctness"]
99
+ runs_per_side = 1
100
+
101
+ [[side]]
102
+ name = "claude"
103
+ player = "claude"
104
+ deck = "/tmp/deck.toml"
105
+
106
+ [[side]]
107
+ name = "kimi"
108
+ player = "kimi"
109
+ deck = "/tmp/deck.toml"
110
+ `)
111
+
112
+ const logs: string[] = []
113
+ const result = await runArenaFromToml({
114
+ toml,
115
+ taskPath: '/tmp/task.md',
116
+ dryRun: true,
117
+ log: (msg) => logs.push(msg),
118
+ })
119
+
120
+ expect('plan' in result).toBe(true)
121
+ const plan = (result as { plan: ReturnType<typeof buildExecutionPlan> }).plan
122
+ expect(plan.total_runs).toBe(2)
123
+ expect(plan.cells.length).toBe(2)
124
+ expect(logs.length).toBeGreaterThan(0)
125
+ expect(logs.some(l => l.includes('Dry-run'))).toBe(true)
126
+ })
127
+
128
+ test('dry-run does not call io.mkdir or io.agentSpawn', async () => {
129
+ const toml = parseArenaToml(`
130
+ [arena]
131
+ task = "Write a hello world function"
132
+ criteria = ["completeness"]
133
+ runs_per_side = 1
134
+
135
+ [[side]]
136
+ name = "claude"
137
+ player = "claude"
138
+ deck = "/tmp/deck.toml"
139
+
140
+ [[side]]
141
+ name = "kimi"
142
+ player = "kimi"
143
+ deck = "/tmp/deck.toml"
144
+ `)
145
+
146
+ let mkdirCalled = false
147
+ let agentSpawnCalled = false
148
+
149
+ const mockIO: ArenaIO = {
150
+ mkdir: () => { mkdirCalled = true },
151
+ agentSpawn: async () => { agentSpawnCalled = true; return { stdout: '', stderr: '', durationMs: 0 } },
152
+ }
153
+
154
+ await runArenaFromToml({
155
+ toml,
156
+ taskPath: '/tmp/task.md',
157
+ dryRun: true,
158
+ io: mockIO,
159
+ })
160
+
161
+ expect(mkdirCalled).toBe(false)
162
+ expect(agentSpawnCalled).toBe(false)
163
+ })
164
+ })
165
+
166
+ describe('runArenaFromToml — single cell execution with mock agentSpawn', () => {
167
+ test('executes one cell and writes outputs via io', async () => {
168
+ const toml = parseArenaToml(`
169
+ [arena]
170
+ task = "Write a hello world function"
171
+ criteria = ["completeness"]
172
+ runs_per_side = 1
173
+
174
+ [[side]]
175
+ name = "claude"
176
+ player = "claude"
177
+ deck = "/tmp/deck.toml"
178
+
179
+ [[side]]
180
+ name = "kimi"
181
+ player = "kimi"
182
+ deck = "/tmp/deck.toml"
183
+ `)
184
+
185
+ const files = new Map<string, string>()
186
+ const dirs = new Set<string>()
187
+
188
+ const mockIO: ArenaIO = {
189
+ log: () => {},
190
+ mkdir: (path: string) => { dirs.add(path) },
191
+ writeFile: (path: string, data: string) => { files.set(path, data) },
192
+ readFile: (path: string) => {
193
+ if (path === '/tmp/deck.toml') return 'skills = ["test"]'
194
+ throw new Error(`Unexpected read: ${path}`)
195
+ },
196
+ readdir: () => ['decision-log.jsonl'],
197
+ cp: () => {},
198
+ spawn: async () => ({ exitCode: 0, stderr: '' }),
199
+ agentSpawn: async () => ({
200
+ stdout: 'I wrote hello world',
201
+ stderr: '',
202
+ durationMs: 1234,
203
+ }),
204
+ exists: (path: string) => path === '/tmp/deck.toml',
205
+ chdir: () => {},
206
+ }
207
+
208
+ const result = await runArenaFromToml({
209
+ toml,
210
+ taskPath: '/tmp/task.md',
211
+ outDir: '/tmp/arena-out',
212
+ io: mockIO,
213
+ })
214
+
215
+ expect('manifest' in result).toBe(true)
216
+ const r = result as { manifest: { id: string; status: string }; artifactsDir: string }
217
+ expect(r.manifest.status).toBe('completed')
218
+ expect(r.artifactsDir).toBe('/tmp/arena-out')
219
+
220
+ // arena.json written twice (initial + final)
221
+ expect(files.has('/tmp/arena-out/arena.json')).toBe(true)
222
+ // agent stdout persisted
223
+ expect(files.has('/tmp/arena-out/runs/claude/run-1/agent-stdout.txt')).toBe(true)
224
+ expect(files.get('/tmp/arena-out/runs/claude/run-1/agent-stdout.txt')).toContain('I wrote hello world')
225
+ // judge-verdict.json written
226
+ expect(files.has('/tmp/arena-out/runs/claude/run-1/judge-verdict.json')).toBe(true)
227
+ // report.md written
228
+ expect(files.has('/tmp/arena-out/report.md')).toBe(true)
229
+ // directories created
230
+ expect(dirs.has('/tmp/arena-out')).toBe(true)
231
+ expect(dirs.has('/tmp/arena-out/runs/claude/run-1')).toBe(true)
232
+ expect(dirs.has('/tmp/arena-out/work/claude')).toBe(true)
233
+ })
234
+ })
235
+
236
+ describe('runArenaFromToml — error recovery path', () => {
237
+ test('cell exception produces ERROR verdict and continues', async () => {
238
+ const toml = parseArenaToml(`
239
+ [arena]
240
+ task = "Write a hello world function"
241
+ criteria = ["completeness"]
242
+ runs_per_side = 1
243
+
244
+ [[side]]
245
+ name = "claude"
246
+ player = "claude"
247
+ deck = "/tmp/deck.toml"
248
+
249
+ [[side]]
250
+ name = "kimi"
251
+ player = "kimi"
252
+ deck = "/tmp/deck.toml"
253
+ `)
254
+
255
+ const files = new Map<string, string>()
256
+
257
+ let spawnCount = 0
258
+ const mockIO: ArenaIO = {
259
+ log: () => {},
260
+ mkdir: () => {},
261
+ writeFile: (path: string, data: string) => { files.set(path, data) },
262
+ readFile: (path: string) => {
263
+ if (path === '/tmp/deck.toml') return 'skills = ["test"]'
264
+ throw new Error(`Unexpected read: ${path}`)
265
+ },
266
+ readdir: () => [],
267
+ cp: () => {},
268
+ spawn: async () => ({ exitCode: 0, stderr: '' }),
269
+ agentSpawn: async () => {
270
+ spawnCount++
271
+ if (spawnCount === 1) throw new Error('Agent spawn failed')
272
+ return { stdout: 'ok', stderr: '', durationMs: 100 }
273
+ },
274
+ exists: (path: string) => path === '/tmp/deck.toml',
275
+ chdir: () => {},
276
+ }
277
+
278
+ const result = await runArenaFromToml({
279
+ toml,
280
+ taskPath: '/tmp/task.md',
281
+ outDir: '/tmp/arena-out',
282
+ io: mockIO,
283
+ })
284
+
285
+ expect('manifest' in result).toBe(true)
286
+ const r = result as { manifest: { status: string }; stats: { sideName: string; errorRate: number }[] }
287
+ expect(r.manifest.status).toBe('completed')
288
+ expect(r.stats.length).toBe(2)
289
+ const claudeStats = r.stats.find(s => s.sideName === 'claude')
290
+ expect(claudeStats!.errorRate).toBe(1)
291
+
292
+ const verdictPath = '/tmp/arena-out/runs/claude/run-1/judge-verdict.json'
293
+ expect(files.has(verdictPath)).toBe(true)
294
+ const verdict = JSON.parse(files.get(verdictPath)!)
295
+ expect(verdict.verdict).toBe('ERROR')
296
+ expect(verdict.reason).toContain('Agent spawn failed')
297
+ })
298
+
299
+ test('spawn non-zero exit logs warning but continues', async () => {
300
+ const toml = parseArenaToml(`
301
+ [arena]
302
+ task = "Write a hello world function"
303
+ criteria = ["completeness"]
304
+ runs_per_side = 1
305
+
306
+ [[side]]
307
+ name = "claude"
308
+ player = "claude"
309
+ deck = "/tmp/deck.toml"
310
+
311
+ [[side]]
312
+ name = "kimi"
313
+ player = "kimi"
314
+ deck = "/tmp/deck.toml"
315
+ `)
316
+
317
+ const logs: string[] = []
318
+ const files = new Map<string, string>()
319
+
320
+ const mockIO: ArenaIO = {
321
+ log: (msg: string) => logs.push(msg),
322
+ mkdir: () => {},
323
+ writeFile: (path: string, data: string) => { files.set(path, data) },
324
+ readFile: (path: string) => {
325
+ if (path === '/tmp/deck.toml') return 'skills = ["test"]'
326
+ throw new Error(`Unexpected read: ${path}`)
327
+ },
328
+ readdir: () => [],
329
+ cp: () => {},
330
+ spawn: async () => ({ exitCode: 1, stderr: 'link failed' }),
331
+ agentSpawn: async () => ({
332
+ stdout: 'done',
333
+ stderr: '',
334
+ durationMs: 100,
335
+ }),
336
+ exists: (path: string) => path === '/tmp/deck.toml',
337
+ chdir: () => {},
338
+ }
339
+
340
+ const result = await runArenaFromToml({
341
+ toml,
342
+ taskPath: '/tmp/task.md',
343
+ outDir: '/tmp/arena-out',
344
+ io: mockIO,
345
+ })
346
+
347
+ expect('manifest' in result).toBe(true)
348
+ expect(logs.some(l => l.includes('deck link') && l.includes('exit 1'))).toBe(true)
349
+ })
350
+ })
package/src/runner.ts CHANGED
@@ -17,6 +17,42 @@ import { aggregateAllStats } from './stats'
17
17
  import type { SideStats } from './stats'
18
18
  import { buildCopyPlan } from './preflight'
19
19
 
20
+ // ── ArenaIO interface (Intent/Plan/Execute fractal pattern) ───────────────
21
+
22
+ export interface ArenaIO {
23
+ log?: (msg: string) => void
24
+ mkdir?: (path: string, opts?: { recursive?: boolean }) => void
25
+ writeFile?: (path: string, data: string) => void
26
+ readFile?: (path: string) => string
27
+ readdir?: (path: string) => string[]
28
+ cp?: (src: string, dest: string, opts?: { recursive?: boolean }) => void
29
+ spawn?: (cmd: string[], opts: { cwd: string; env?: Record<string, string> }) => Promise<{ exitCode: number | null; stderr: string }>
30
+ agentSpawn?: (opts: { player: string; cwd: string; brief: string; timeoutMs: number }) => Promise<{ stdout: string; stderr: string; durationMs: number }>
31
+ exists?: (path: string) => boolean
32
+ chdir?: (cwd: string) => void
33
+ }
34
+
35
+ export const defaultArenaIO: ArenaIO = {
36
+ log: (msg: string) => { console.log(msg) },
37
+ mkdir: (path: string, opts?: { recursive?: boolean }) => mkdirSync(path, opts),
38
+ writeFile: (path: string, data: string) => writeFileSync(path, data, 'utf-8'),
39
+ readFile: (path: string) => readFileSync(path, 'utf-8'),
40
+ readdir: (path: string) => readdirSync(path),
41
+ cp: (src: string, dest: string, opts?: { recursive?: boolean }) => cpSync(src, dest, opts),
42
+ spawn: async (cmd: string[], opts: { cwd: string; env?: Record<string, string> }) => {
43
+ const proc = Bun.spawn(cmd, { cwd: opts.cwd, env: opts.env })
44
+ await proc.exited
45
+ const stderr = await new Response(proc.stderr).text()
46
+ return { exitCode: proc.exitCode, stderr }
47
+ },
48
+ agentSpawn: async (opts: { player: string; cwd: string; brief: string; timeoutMs: number }) => {
49
+ const agent = useAgent(opts.player)
50
+ return agent.spawn({ cwd: opts.cwd, brief: opts.brief, timeoutMs: opts.timeoutMs })
51
+ },
52
+ exists: (path: string) => existsSync(path),
53
+ chdir: (cwd: string) => { process.chdir(cwd) },
54
+ }
55
+
20
56
  // ── Helpers ───────────────────────────────────────────────────────────────
21
57
 
22
58
  function stamp(): string {
@@ -33,18 +69,18 @@ export interface ArenaResult {
33
69
 
34
70
  // ── Task + judge text resolution (no parsing — natural language) ──────────
35
71
 
36
- function resolveTaskText(toml: ArenaToml, configDir?: string): string {
72
+ function resolveTaskText(toml: ArenaToml, io: Required<Pick<ArenaIO, 'exists' | 'readFile'>>, configDir?: string): string {
37
73
  const p = toml.arena.task
38
74
  const candidate = configDir ? resolve(configDir, p) : resolve(p)
39
- if (existsSync(candidate)) return readFileSync(candidate, 'utf-8')
75
+ if (io.exists(candidate)) return io.readFile(candidate)
40
76
  return p
41
77
  }
42
78
 
43
- function resolveJudgeText(toml: ArenaToml, configDir?: string): string | null {
79
+ function resolveJudgeText(toml: ArenaToml, io: Required<Pick<ArenaIO, 'exists' | 'readFile'>>, configDir?: string): string | null {
44
80
  if (toml.arena.judge) {
45
81
  const p = toml.arena.judge
46
82
  const candidate = configDir ? resolve(configDir, p) : resolve(p)
47
- if (existsSync(candidate)) return readFileSync(candidate, 'utf-8')
83
+ if (io.exists(candidate)) return io.readFile(candidate)
48
84
  return p
49
85
  }
50
86
  if (toml.arena.criteria && toml.arena.criteria.length > 0) {
@@ -116,8 +152,14 @@ export async function runArenaFromToml(opts: {
116
152
  dryRun?: boolean
117
153
  log?: (msg: string) => void
118
154
  configDir?: string
155
+ io?: ArenaIO
119
156
  }): Promise<ArenaResult | { plan: ReturnType<typeof buildExecutionPlan> }> {
120
157
  const { toml, taskPath, outDir, dryRun, log, configDir } = opts
158
+ const io: ArenaIO = {
159
+ ...defaultArenaIO,
160
+ ...opts.io,
161
+ log: log ?? opts.io?.log ?? defaultArenaIO.log,
162
+ }
121
163
 
122
164
  const resolvePath = (p: string) => {
123
165
  if (p.startsWith('/')) return p
@@ -125,7 +167,19 @@ export async function runArenaFromToml(opts: {
125
167
  return resolve(p)
126
168
  }
127
169
 
128
- const taskText = resolveTaskText(toml, configDir)
170
+ const ioWithDefaults = {
171
+ exists: io.exists!,
172
+ readFile: io.readFile!,
173
+ writeFile: io.writeFile!,
174
+ mkdir: io.mkdir!,
175
+ readdir: io.readdir!,
176
+ cp: io.cp!,
177
+ spawn: io.spawn!,
178
+ agentSpawn: io.agentSpawn!,
179
+ chdir: io.chdir!,
180
+ }
181
+
182
+ const taskText = resolveTaskText(toml, ioWithDefaults, configDir)
129
183
  const resolvedToml: ArenaToml = {
130
184
  ...toml,
131
185
  side: toml.side.map(s => ({ ...s, deck: resolvePath(s.deck) })),
@@ -134,7 +188,7 @@ export async function runArenaFromToml(opts: {
134
188
  const plan = buildExecutionPlan(resolvedToml)
135
189
 
136
190
  if (dryRun) {
137
- for (const line of formatPlanOutput(plan)) log?.(line)
191
+ for (const line of formatPlanOutput(plan)) io.log?.(line)
138
192
  return { plan }
139
193
  }
140
194
 
@@ -158,29 +212,29 @@ export async function runArenaFromToml(opts: {
158
212
  status: 'running',
159
213
  })
160
214
 
161
- mkdirSync(artifactsDir, { recursive: true })
162
- writeFileSync(join(artifactsDir, 'arena.json'), JSON.stringify(manifest, null, 2) + '\n')
215
+ ioWithDefaults.mkdir(artifactsDir, { recursive: true })
216
+ ioWithDefaults.writeFile(join(artifactsDir, 'arena.json'), JSON.stringify(manifest, null, 2) + '\n')
163
217
 
164
- const judgeText = resolveJudgeText(resolvedToml, configDir)
218
+ const judgeText = resolveJudgeText(resolvedToml, ioWithDefaults, configDir)
165
219
  const judgeInput: JudgeInput | undefined = judgeText
166
220
  ? { criteria: judgeText, task_context: taskText.slice(0, 500) }
167
221
  : undefined
168
222
 
169
- // ── Per-cell: agent.spawn directly, no AgentScenario/parseAgentMd ────
223
+ // ── Per-cell: agent.spawn via IO injection ─────────────────────────────
170
224
  const verdictsBySide = new Map<string, JudgeVerdict[]>()
171
225
 
172
226
  for (const cell of plan.cells) {
173
227
  const cellDir = join(artifactsDir, 'runs', cell.side, `run-${cell.run}`)
174
- mkdirSync(cellDir, { recursive: true })
228
+ ioWithDefaults.mkdir(cellDir, { recursive: true })
175
229
 
176
230
  const workDir = join(artifactsDir, 'work', cell.side)
177
- mkdirSync(workDir, { recursive: true })
231
+ ioWithDefaults.mkdir(workDir, { recursive: true })
178
232
  const originalCwd = process.cwd()
179
233
 
180
234
  try {
181
235
  // Setup: deck + AGENTS.md + link
182
- writeFileSync(join(workDir, 'skill-deck.toml'), readFileSync(cell.deck, 'utf-8'))
183
- writeFileSync(join(workDir, 'AGENTS.md'), [
236
+ ioWithDefaults.writeFile(join(workDir, 'skill-deck.toml'), ioWithDefaults.readFile(cell.deck))
237
+ ioWithDefaults.writeFile(join(workDir, 'AGENTS.md'), [
184
238
  '# Arena Test Environment',
185
239
  `**Side**: ${cell.side}`, `**Player**: ${cell.player}`, `**Run**: ${cell.run}`,
186
240
  '## How This Works',
@@ -188,55 +242,54 @@ export async function runArenaFromToml(opts: {
188
242
  '- Complete the task using available skills. Output to this directory.',
189
243
  '- MANDATORY: write decision-log.jsonl (see prompt for schema).',
190
244
  ].join('\n'))
191
- const linkProc = Bun.spawn(
245
+ const linkProc = await ioWithDefaults.spawn(
192
246
  ['bunx', '@lythos/skill-deck', 'link'],
193
247
  { cwd: workDir, env: { ...process.env, HOME: process.env.HOME! } },
194
248
  )
195
- await linkProc.exited
196
- log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
249
+ io.log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
197
250
 
198
- process.chdir(workDir)
251
+ ioWithDefaults.chdir(workDir)
199
252
 
200
- // Direct agent.spawn (no parseAgentMd, no AgentScenario)
201
- const agent = useAgent(resolvePlayer(cell.player))
253
+ // Agent spawn via IO injection
202
254
  const fullPrompt = buildArenaPrompt({
203
255
  brief: taskText,
204
256
  cwd: workDir,
205
257
  deckPath: cell.deck,
206
258
  outputDir: workDir,
207
259
  })
208
- const agentResult = await agent.spawn({
260
+ const agentResult = await ioWithDefaults.agentSpawn({
261
+ player: resolvePlayer(cell.player),
209
262
  cwd: workDir,
210
263
  brief: fullPrompt,
211
264
  timeoutMs: 300000,
212
265
  })
213
266
 
214
- process.chdir(originalCwd)
267
+ ioWithDefaults.chdir(originalCwd)
215
268
 
216
269
  // Persist agent output
217
270
  const sanitizer = createSanitizer({ projectRoot: process.cwd(), homeDir: homedir(), workDir })
218
- writeFileSync(join(cellDir, 'agent-stdout.txt'), sanitizer.sanitize(agentResult.stdout), 'utf-8')
219
- if (agentResult.stderr) writeFileSync(join(cellDir, 'agent-stderr.txt'), sanitizer.sanitize(agentResult.stderr), 'utf-8')
271
+ ioWithDefaults.writeFile(join(cellDir, 'agent-stdout.txt'), sanitizer.sanitize(agentResult.stdout))
272
+ if (agentResult.stderr) ioWithDefaults.writeFile(join(cellDir, 'agent-stderr.txt'), sanitizer.sanitize(agentResult.stderr))
220
273
 
221
274
  // Copy artifacts
222
275
  const skipSet = new Set(['.claude', 'skill-deck.toml', 'skill-deck.lock', 'AGENTS.md'])
223
276
  try {
224
- const entries = readdirSync(workDir)
277
+ const entries = ioWithDefaults.readdir(workDir)
225
278
  const copyPlan = buildCopyPlan(workDir, cellDir, entries, skipSet)
226
279
  for (const { src, dest, name } of copyPlan) {
227
- try { cpSync(src, dest, { recursive: true }) } catch (e) {
228
- log?.(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
280
+ try { ioWithDefaults.cp(src, dest, { recursive: true }) } catch (e) {
281
+ io.log?.(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
229
282
  }
230
283
  }
231
284
  } catch (e) {
232
- log?.(`⚠️ Failed to read agent workdir for copy: ${e instanceof Error ? e.message : e}`)
285
+ io.log?.(`⚠️ Failed to read agent workdir for copy: ${e instanceof Error ? e.message : e}`)
233
286
  }
234
287
 
235
288
  // Evidence
236
289
  const checkpoints = readCheckpoints(workDir)
237
290
  let artifactFiles: string[] = []
238
291
  try {
239
- for (const e of readdirSync(workDir)) {
292
+ for (const e of ioWithDefaults.readdir(workDir)) {
240
293
  if (!e.startsWith('.') && !skipSet.has(e) && e !== 'agent-stdout.txt' && e !== 'agent-stderr.txt' && e !== 'judge-verdict.json' && e !== '_checkpoints') {
241
294
  artifactFiles.push(e)
242
295
  }
@@ -259,7 +312,7 @@ export async function runArenaFromToml(opts: {
259
312
  v = { verdict: 'ERROR' as const, reason: 'No judge criteria provided', criteria: [] }
260
313
  }
261
314
 
262
- writeFileSync(join(cellDir, 'judge-verdict.json'), JSON.stringify({
315
+ ioWithDefaults.writeFile(join(cellDir, 'judge-verdict.json'), JSON.stringify({
263
316
  ...v,
264
317
  agent_stdout: agentResult.stdout.slice(0, 5000),
265
318
  agent_stderr: agentResult.stderr.slice(0, 1000),
@@ -274,7 +327,7 @@ export async function runArenaFromToml(opts: {
274
327
  reason: `Runner exception: ${e instanceof Error ? e.message : String(e)}`,
275
328
  criteria: [],
276
329
  }
277
- writeFileSync(join(cellDir, 'judge-verdict.json'), JSON.stringify(errVerdict, null, 2) + '\n')
330
+ ioWithDefaults.writeFile(join(cellDir, 'judge-verdict.json'), JSON.stringify(errVerdict, null, 2) + '\n')
278
331
  if (!verdictsBySide.has(cell.side)) verdictsBySide.set(cell.side, [])
279
332
  verdictsBySide.get(cell.side)!.push(errVerdict)
280
333
  }
@@ -288,10 +341,10 @@ export async function runArenaFromToml(opts: {
288
341
  }
289
342
  const judge = useAgent(resolved[0]?.platform ?? 'claude')
290
343
  const report = await runComparativeJudge({ manifest, verdicts: flatVerdicts, judge, workdir: artifactsDir })
291
- writeReport(artifactsDir, manifest, report, stats)
344
+ writeReport(artifactsDir, manifest, report, stats, ioWithDefaults)
292
345
 
293
346
  const finalManifest = ArenaManifest.parse({ ...manifest, status: 'completed' })
294
- writeFileSync(join(artifactsDir, 'arena.json'), JSON.stringify(finalManifest, null, 2) + '\n')
347
+ ioWithDefaults.writeFile(join(artifactsDir, 'arena.json'), JSON.stringify(finalManifest, null, 2) + '\n')
295
348
 
296
349
  return { manifest: finalManifest, report, stats, artifactsDir }
297
350
  }
@@ -319,7 +372,7 @@ export async function runArena(opts: {
319
372
 
320
373
  // ── Report ────────────────────────────────────────────────────────────────
321
374
 
322
- function writeReport(dir: string, manifest: ArenaManifestType, report: any, stats: SideStats[]): void {
375
+ function writeReport(dir: string, manifest: ArenaManifestType, report: any, stats: SideStats[], io: Required<Pick<ArenaIO, 'writeFile'>>): void {
323
376
  const lines: string[] = [
324
377
  `# Arena Report: ${manifest.id}`, '',
325
378
  `**Task**: ${manifest.task}`,
@@ -331,7 +384,7 @@ function writeReport(dir: string, manifest: ArenaManifestType, report: any, stat
331
384
  '## Key Findings', '', ...(report.key_findings ?? []).map((f: string) => `- ${f}`), '',
332
385
  '## Recommendations', '', ...(report.recommendations ?? []).map((r: any) => `- **${r.audience}**: ${r.recommendation}`),
333
386
  ]
334
- writeFileSync(join(dir, 'report.md'), lines.join('\n') + '\n')
387
+ io.writeFile(join(dir, 'report.md'), lines.join('\n') + '\n')
335
388
  }
336
389
 
337
390
  function renderStatsTable(stats: SideStats[]): string {