@lythos/skill-arena 0.15.7 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/package.json +7 -7
- package/src/cli.test.ts +118 -0
- package/src/cli.ts +121 -98
- package/src/runner.test.ts +260 -1
- package/src/runner.ts +88 -35
package/README.md
CHANGED
|
@@ -16,20 +16,20 @@
|
|
|
16
16
|
```bash
|
|
17
17
|
bun add -d @lythos/skill-arena
|
|
18
18
|
# or use directly
|
|
19
|
-
bunx @lythos/skill-arena@0.
|
|
19
|
+
bunx @lythos/skill-arena@0.16.0 <command>
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
## Quick Start
|
|
23
23
|
|
|
24
24
|
```bash
|
|
25
25
|
# single — test one deck (most common)
|
|
26
|
-
bunx @lythos/skill-arena@0.
|
|
26
|
+
bunx @lythos/skill-arena@0.16.0 single \
|
|
27
27
|
--deck ./examples/decks/scout.toml \
|
|
28
28
|
--brief "Generate auth flow diagram" \
|
|
29
29
|
--out ./output
|
|
30
30
|
|
|
31
31
|
# single with explicit player
|
|
32
|
-
bunx @lythos/skill-arena@0.
|
|
32
|
+
bunx @lythos/skill-arena@0.16.0 single \
|
|
33
33
|
--deck ./examples/decks/scout.toml \
|
|
34
34
|
--brief "Generate auth flow diagram" \
|
|
35
35
|
--player kimi \
|
|
@@ -37,10 +37,10 @@ bunx @lythos/skill-arena@0.15.7 single \
|
|
|
37
37
|
|
|
38
38
|
# cross-deck vs — compare two decks (agent-orchestrated)
|
|
39
39
|
# Create arena.toml declaring sides with different decks, then:
|
|
40
|
-
bunx @lythos/skill-arena@0.
|
|
40
|
+
bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml
|
|
41
41
|
|
|
42
42
|
# cross-player vs — compare kimi vs codex (CLI only)
|
|
43
|
-
bunx @lythos/skill-arena@0.
|
|
43
|
+
bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml --player kimi
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
**What happens**: Agent creates isolated `/tmp` workdir per side, `deck link` skills, spawns parallel subagents, collects artifacts, judge scores outputs. Parent deck restored after.
|
|
@@ -50,14 +50,14 @@ bunx @lythos/skill-arena@0.15.7 vs --config ./arena.toml --player kimi
|
|
|
50
50
|
### `single` — one deck, one task
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
bunx @lythos/skill-arena@0.
|
|
53
|
+
bunx @lythos/skill-arena@0.16.0 single \
|
|
54
54
|
--deck ./deck.toml \
|
|
55
55
|
--brief "Produce a .docx report with radar chart" \
|
|
56
56
|
--timeout 600000 \
|
|
57
57
|
--out ./output
|
|
58
58
|
|
|
59
59
|
# with explicit player
|
|
60
|
-
bunx @lythos/skill-arena@0.
|
|
60
|
+
bunx @lythos/skill-arena@0.16.0 single \
|
|
61
61
|
--deck ./deck.toml \
|
|
62
62
|
--brief "Produce a .docx report with radar chart" \
|
|
63
63
|
--player kimi \
|
|
@@ -67,14 +67,14 @@ bunx @lythos/skill-arena@0.15.7 single \
|
|
|
67
67
|
### `vs` — multi-deck comparison
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
bunx @lythos/skill-arena@0.
|
|
71
|
-
bunx @lythos/skill-arena@0.
|
|
70
|
+
bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml
|
|
71
|
+
bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml --dry-run
|
|
72
72
|
```
|
|
73
73
|
|
|
74
74
|
### `prepare-workdir` — isolate + link skills (agent-orchestrated)
|
|
75
75
|
|
|
76
76
|
```bash
|
|
77
|
-
bunx @lythos/skill-arena@0.
|
|
77
|
+
bunx @lythos/skill-arena@0.16.0 prepare-workdir \
|
|
78
78
|
--deck ./skill-deck.toml \
|
|
79
79
|
--out /tmp/arena-side-a \
|
|
80
80
|
--brief "task description"
|
|
@@ -85,7 +85,7 @@ Creates `/tmp`-isolated workdir with deck copied, AGENTS.md written, and `deck l
|
|
|
85
85
|
### `archive` — collect agent outputs (agent-orchestrated)
|
|
86
86
|
|
|
87
87
|
```bash
|
|
88
|
-
bunx @lythos/skill-arena@0.
|
|
88
|
+
bunx @lythos/skill-arena@0.16.0 archive \
|
|
89
89
|
--from /tmp/arena-side-a \
|
|
90
90
|
--to ./playground/output \
|
|
91
91
|
--sides side-a
|
|
@@ -96,7 +96,7 @@ Copies agent artifacts from workdir(s) to output, skipping internal files (`.cla
|
|
|
96
96
|
### `viz` — render results (WIP — HTML report generation pending)
|
|
97
97
|
|
|
98
98
|
```bash
|
|
99
|
-
bunx @lythos/skill-arena@0.
|
|
99
|
+
bunx @lythos/skill-arena@0.16.0 viz runs/arena-<id>/
|
|
100
100
|
```
|
|
101
101
|
|
|
102
102
|
## Parameters
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lythos/skill-arena",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.16.0",
|
|
4
4
|
"description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-agent",
|
|
@@ -42,15 +42,15 @@
|
|
|
42
42
|
"bun": ">=1.0.0"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@lythos/cold-pool": "^0.
|
|
46
|
-
"@lythos/infra": "^0.
|
|
47
|
-
"@lythos/test-utils": "^0.
|
|
45
|
+
"@lythos/cold-pool": "^0.16.0",
|
|
46
|
+
"@lythos/infra": "^0.16.0",
|
|
47
|
+
"@lythos/test-utils": "^0.16.0",
|
|
48
48
|
"zod": "^3.24.0",
|
|
49
49
|
"zod-to-json-schema": "^3.25.2"
|
|
50
50
|
},
|
|
51
51
|
"optionalDependencies": {
|
|
52
|
-
"@lythos/agent-adapter-claude-sdk": "^0.
|
|
53
|
-
"@lythos/agent-adapter-deepseek-serve": "^0.
|
|
54
|
-
"@lythos/agent-adapter-codex": "^0.
|
|
52
|
+
"@lythos/agent-adapter-claude-sdk": "^0.16.0",
|
|
53
|
+
"@lythos/agent-adapter-deepseek-serve": "^0.16.0",
|
|
54
|
+
"@lythos/agent-adapter-codex": "^0.16.0"
|
|
55
55
|
}
|
|
56
56
|
}
|
package/src/cli.test.ts
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
|
+
import { main, defaultArenaCliIO, type ArenaCliIO } from './cli'
|
|
3
|
+
|
|
4
|
+
// ─── Helpers ───────────────────────────────────────────────────────────────
|
|
5
|
+
|
|
6
|
+
async function catchExitAsync(fn: () => Promise<void>): Promise<number | undefined> {
|
|
7
|
+
let exitCode: number | undefined
|
|
8
|
+
try { await fn() } catch (e: any) {
|
|
9
|
+
if (!String(e).includes('EXIT:')) throw e
|
|
10
|
+
const m = String(e).match(/EXIT:(\d+)/)
|
|
11
|
+
if (m) exitCode = parseInt(m[1], 10)
|
|
12
|
+
}
|
|
13
|
+
return exitCode
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function mockIO(): { io: Required<ArenaCliIO>; logs: string[]; errors: string[]; warns: string[] } {
|
|
17
|
+
const logs: string[] = []
|
|
18
|
+
const errors: string[] = []
|
|
19
|
+
const warns: string[] = []
|
|
20
|
+
const io: Required<ArenaCliIO> = {
|
|
21
|
+
log: (msg: string) => logs.push(msg),
|
|
22
|
+
error: (msg: string) => errors.push(msg),
|
|
23
|
+
warn: (msg: string) => warns.push(msg),
|
|
24
|
+
exit: (code: number) => { throw new Error(`EXIT:${code}`) },
|
|
25
|
+
}
|
|
26
|
+
return { io, logs, errors, warns }
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// ─── Tests ─────────────────────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
describe('main', () => {
|
|
32
|
+
it('shows help on --help', async () => {
|
|
33
|
+
const { io, logs, errors } = mockIO()
|
|
34
|
+
const exitCode = await catchExitAsync(async () => {
|
|
35
|
+
await main(['--help'], io)
|
|
36
|
+
})
|
|
37
|
+
expect(exitCode).toBe(0)
|
|
38
|
+
expect(logs.length).toBeGreaterThan(0)
|
|
39
|
+
expect(logs[0]).toContain('lythoskill-arena')
|
|
40
|
+
expect(errors.length).toBe(0)
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
it('shows help on -h', async () => {
|
|
44
|
+
const { io, logs, errors } = mockIO()
|
|
45
|
+
const exitCode = await catchExitAsync(async () => {
|
|
46
|
+
await main(['-h'], io)
|
|
47
|
+
})
|
|
48
|
+
expect(exitCode).toBe(0)
|
|
49
|
+
expect(logs.length).toBeGreaterThan(0)
|
|
50
|
+
expect(logs[0]).toContain('lythoskill-arena')
|
|
51
|
+
expect(errors.length).toBe(0)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it('shows help on no args', async () => {
|
|
55
|
+
const { io, logs, errors } = mockIO()
|
|
56
|
+
const exitCode = await catchExitAsync(async () => {
|
|
57
|
+
await main([], io)
|
|
58
|
+
})
|
|
59
|
+
expect(exitCode).toBe(0)
|
|
60
|
+
expect(logs.length).toBeGreaterThan(0)
|
|
61
|
+
expect(errors.length).toBe(0)
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
it('errors on unknown command', async () => {
|
|
65
|
+
const { io, logs, errors } = mockIO()
|
|
66
|
+
const exitCode = await catchExitAsync(async () => {
|
|
67
|
+
await main(['bogus'], io)
|
|
68
|
+
})
|
|
69
|
+
expect(exitCode).toBe(1)
|
|
70
|
+
expect(errors.length).toBeGreaterThan(0)
|
|
71
|
+
expect(errors[0]).toContain('Unknown command')
|
|
72
|
+
expect(errors[0]).toContain('bogus')
|
|
73
|
+
})
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
describe('singleRun', () => {
|
|
77
|
+
it('errors when --deck is missing', async () => {
|
|
78
|
+
const { io, logs, errors } = mockIO()
|
|
79
|
+
const exitCode = await catchExitAsync(async () => {
|
|
80
|
+
await main(['single', '--brief', 'test task'], io)
|
|
81
|
+
})
|
|
82
|
+
expect(exitCode).toBe(1)
|
|
83
|
+
expect(errors.length).toBeGreaterThan(0)
|
|
84
|
+
expect(errors[0]).toContain('--deck')
|
|
85
|
+
expect(errors[0]).toContain('required')
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it('errors when both --task and --brief are missing', async () => {
|
|
89
|
+
const { io, logs, errors } = mockIO()
|
|
90
|
+
const exitCode = await catchExitAsync(async () => {
|
|
91
|
+
await main(['single', '--deck', './some-deck.toml'], io)
|
|
92
|
+
})
|
|
93
|
+
expect(exitCode).toBe(1)
|
|
94
|
+
expect(errors.length).toBeGreaterThan(0)
|
|
95
|
+
expect(errors[0]).toContain('--task')
|
|
96
|
+
expect(errors[0]).toContain('--brief')
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
it('errors when --brief is empty string', async () => {
|
|
100
|
+
const { io, logs, errors } = mockIO()
|
|
101
|
+
const exitCode = await catchExitAsync(async () => {
|
|
102
|
+
await main(['single', '--deck', './some-deck.toml', '--brief', ''], io)
|
|
103
|
+
})
|
|
104
|
+
expect(exitCode).toBe(1)
|
|
105
|
+
expect(errors.length).toBeGreaterThan(0)
|
|
106
|
+
expect(errors[0]).toContain('--task')
|
|
107
|
+
expect(errors[0]).toContain('--brief')
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
describe('defaultArenaCliIO', () => {
|
|
112
|
+
it('has all required functions', () => {
|
|
113
|
+
expect(typeof defaultArenaCliIO.log).toBe('function')
|
|
114
|
+
expect(typeof defaultArenaCliIO.error).toBe('function')
|
|
115
|
+
expect(typeof defaultArenaCliIO.warn).toBe('function')
|
|
116
|
+
expect(typeof defaultArenaCliIO.exit).toBe('function')
|
|
117
|
+
})
|
|
118
|
+
})
|
package/src/cli.ts
CHANGED
|
@@ -8,6 +8,22 @@ import { parseArenaToml, buildExecutionPlan } from './arena-toml'
|
|
|
8
8
|
import { buildArchiveSidePlan, buildCopyPlan, buildPreparePlan, parseDeckSkills } from './preflight'
|
|
9
9
|
import { checkSkillExistence, formatSkillWarnings, resolveColdPoolDir } from './preflight'
|
|
10
10
|
|
|
11
|
+
// ─── ArenaCliIO interface (injected for testability) ───────────────────────
|
|
12
|
+
|
|
13
|
+
export interface ArenaCliIO {
|
|
14
|
+
log?: (msg: string) => void
|
|
15
|
+
error?: (msg: string) => void
|
|
16
|
+
warn?: (msg: string) => void
|
|
17
|
+
exit?: (code: number) => never
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export const defaultArenaCliIO: Required<ArenaCliIO> = {
|
|
21
|
+
log: console.log,
|
|
22
|
+
error: console.error,
|
|
23
|
+
warn: console.warn,
|
|
24
|
+
exit: (code) => { process.exit(code) },
|
|
25
|
+
}
|
|
26
|
+
|
|
11
27
|
// ─── fetchWithProxy (infra dependency, no package boundary) ─────────────────
|
|
12
28
|
|
|
13
29
|
async function fetchWithProxy(url: string, init?: RequestInit): Promise<Response> {
|
|
@@ -82,9 +98,10 @@ function validateLinkResult(exitCode: number | null, stderr: string): { ok: bool
|
|
|
82
98
|
}
|
|
83
99
|
|
|
84
100
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
85
|
-
export async function main(args: string[] = process.argv.slice(2)) {
|
|
101
|
+
export async function main(args: string[] = process.argv.slice(2), io: ArenaCliIO = defaultArenaCliIO) {
|
|
102
|
+
const { log, exit } = { ...defaultArenaCliIO, ...io }
|
|
86
103
|
if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
|
|
87
|
-
|
|
104
|
+
log(`lythoskill-arena — skill evaluation CLI
|
|
88
105
|
|
|
89
106
|
Usage:
|
|
90
107
|
lythoskill-arena single|vs|viz <options>
|
|
@@ -103,28 +120,30 @@ Examples:
|
|
|
103
120
|
lythoskill-arena prepare-workdir --deck ./decks/scout.toml --out /tmp/arena-20260517-side-a
|
|
104
121
|
lythoskill-arena archive --from /tmp/arena-20260517 --to playground/arena-20260517 --sides side-a,side-b
|
|
105
122
|
`)
|
|
106
|
-
|
|
123
|
+
exit(0)
|
|
107
124
|
}
|
|
108
|
-
return cli(args)
|
|
125
|
+
return cli(args, io)
|
|
109
126
|
}
|
|
110
127
|
|
|
111
|
-
function cli(args: string[]) {
|
|
128
|
+
function cli(args: string[], io: ArenaCliIO) {
|
|
112
129
|
const cmd = args[0]
|
|
113
130
|
const rest = args.slice(1)
|
|
114
131
|
|
|
115
|
-
if (cmd === 'vs' || cmd === 'compare') return vsRun(rest)
|
|
116
|
-
if (cmd === 'single' || cmd === 'run') return singleRun(rest)
|
|
117
|
-
if (cmd === 'viz') return vizRun(rest)
|
|
118
|
-
if (cmd === 'prepare-workdir') return prepareWorkdir(rest)
|
|
119
|
-
if (cmd === 'archive') return archiveRun(rest)
|
|
132
|
+
if (cmd === 'vs' || cmd === 'compare') return vsRun(rest, io)
|
|
133
|
+
if (cmd === 'single' || cmd === 'run') return singleRun(rest, io)
|
|
134
|
+
if (cmd === 'viz') return vizRun(rest, io)
|
|
135
|
+
if (cmd === 'prepare-workdir') return prepareWorkdir(rest, io)
|
|
136
|
+
if (cmd === 'archive') return archiveRun(rest, io)
|
|
120
137
|
|
|
121
|
-
|
|
122
|
-
|
|
138
|
+
const { error, exit } = { ...defaultArenaCliIO, ...io }
|
|
139
|
+
error(`Unknown command: ${cmd}`)
|
|
140
|
+
exit(1)
|
|
123
141
|
}
|
|
124
142
|
|
|
125
143
|
// ── single: single-player deck test (exec shortcut) ──────────────────────
|
|
126
144
|
|
|
127
|
-
async function singleRun(args: string[]) {
|
|
145
|
+
async function singleRun(args: string[], io: ArenaCliIO) {
|
|
146
|
+
const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
|
|
128
147
|
const opts: Record<string, string | undefined> = {}
|
|
129
148
|
for (let i = 0; i < args.length; i++) {
|
|
130
149
|
if (args[i] === '--task' || args[i] === '-t') opts.task = args[++i]
|
|
@@ -136,7 +155,7 @@ async function singleRun(args: string[]) {
|
|
|
136
155
|
}
|
|
137
156
|
|
|
138
157
|
if (!opts.deck) {
|
|
139
|
-
|
|
158
|
+
error(`❌ --deck <path|url> is required.
|
|
140
159
|
--deck accepts local paths and http/https URLs (auto-fetched).
|
|
141
160
|
|
|
142
161
|
Example (no local file needed — URL is auto-fetched):
|
|
@@ -146,10 +165,10 @@ async function singleRun(args: string[]) {
|
|
|
146
165
|
|
|
147
166
|
Or with a local deck file you already have:
|
|
148
167
|
lythoskill-arena single --deck ./examples/decks/scout.toml --brief "your task"`)
|
|
149
|
-
|
|
168
|
+
exit(1)
|
|
150
169
|
}
|
|
151
170
|
if (!opts.task && (!opts.brief || !opts.brief.trim())) {
|
|
152
|
-
|
|
171
|
+
error(`❌ --task <path> or --brief "<text>" is required.
|
|
153
172
|
--task reads a .agent.md scenario file; --brief takes inline text.
|
|
154
173
|
|
|
155
174
|
Example (no local file needed — URL is auto-fetched):
|
|
@@ -159,24 +178,24 @@ async function singleRun(args: string[]) {
|
|
|
159
178
|
|
|
160
179
|
Or with a local deck file:
|
|
161
180
|
lythoskill-arena single --deck ./examples/decks/scout.toml --brief "your task"`)
|
|
162
|
-
|
|
181
|
+
exit(1)
|
|
163
182
|
}
|
|
164
183
|
|
|
165
184
|
let resolvedTaskPath: string | undefined
|
|
166
185
|
if (opts.task) {
|
|
167
186
|
resolvedTaskPath = resolve(opts.task)
|
|
168
187
|
if (!existsSync(resolvedTaskPath)) {
|
|
169
|
-
|
|
188
|
+
error(`❌ Task file not found: ${resolvedTaskPath}
|
|
170
189
|
Use --brief for inline tasks, or point --task to an existing .agent.md file.
|
|
171
190
|
Format: name + description + Given/When/Then sections.
|
|
172
191
|
|
|
173
192
|
Example (URL): lythoskill-arena single --brief "your task" --deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml
|
|
174
193
|
Or (local): lythoskill-arena single --brief "your task" --deck ./examples/decks/scout.toml`)
|
|
175
|
-
|
|
194
|
+
exit(1)
|
|
176
195
|
}
|
|
177
196
|
const raw = readFileSync(resolvedTaskPath, 'utf-8')
|
|
178
197
|
if (!raw.startsWith('---')) {
|
|
179
|
-
|
|
198
|
+
error(`❌ Invalid .agent.md: missing frontmatter (must start with "---")
|
|
180
199
|
Correct format:
|
|
181
200
|
---
|
|
182
201
|
name: my-scenario
|
|
@@ -189,13 +208,13 @@ async function singleRun(args: string[]) {
|
|
|
189
208
|
...
|
|
190
209
|
## Then
|
|
191
210
|
...`)
|
|
192
|
-
|
|
211
|
+
exit(1)
|
|
193
212
|
}
|
|
194
213
|
if (!raw.includes('## When')) {
|
|
195
|
-
|
|
214
|
+
error(`❌ Invalid .agent.md: missing "## When" section.
|
|
196
215
|
The ## When section defines what the agent should do.
|
|
197
216
|
See template: playground/arena-one-shot/TASK-arena.agent.md`)
|
|
198
|
-
|
|
217
|
+
exit(1)
|
|
199
218
|
}
|
|
200
219
|
}
|
|
201
220
|
|
|
@@ -213,7 +232,7 @@ async function singleRun(args: string[]) {
|
|
|
213
232
|
}
|
|
214
233
|
const { mirrorUrls, isLikelyGitHubBlock } = await import('../../lythoskill-cold-pool/src/mirror.js')
|
|
215
234
|
const dest = resolve(process.cwd(), 'arena-deck.toml')
|
|
216
|
-
|
|
235
|
+
log(`📥 Fetching arena deck: ${url}`)
|
|
217
236
|
let res: Response | undefined
|
|
218
237
|
let allFailed = true
|
|
219
238
|
|
|
@@ -222,7 +241,7 @@ async function singleRun(args: string[]) {
|
|
|
222
241
|
if (!res?.ok) {
|
|
223
242
|
for (const mirrorUrl of mirrorUrls(url)) {
|
|
224
243
|
try {
|
|
225
|
-
|
|
244
|
+
log(` ↳ trying mirror: ${mirrorUrl}`)
|
|
226
245
|
const r = await fetchWithProxy(mirrorUrl, { signal: AbortSignal.timeout(30_000) })
|
|
227
246
|
if (r.ok) { res = r; allFailed = false; break }
|
|
228
247
|
} catch {}
|
|
@@ -231,21 +250,21 @@ async function singleRun(args: string[]) {
|
|
|
231
250
|
|
|
232
251
|
if (!res?.ok) {
|
|
233
252
|
const errorDetail = res ? `HTTP ${res.status}` : 'unreachable'
|
|
234
|
-
|
|
235
|
-
if (allFailed)
|
|
236
|
-
|
|
237
|
-
|
|
253
|
+
error(`❌ Cannot reach ${url} (${errorDetail})`)
|
|
254
|
+
if (allFailed) error(' Set LYTHOS_GH_MIRROR to use a custom mirror.')
|
|
255
|
+
error(' Or download manually and reference the local file.')
|
|
256
|
+
exit(1)
|
|
238
257
|
}
|
|
239
258
|
|
|
240
259
|
deckWrite(dest, await res.text())
|
|
241
|
-
|
|
260
|
+
log(` → saved to ${dest}`)
|
|
242
261
|
deckPath = dest
|
|
243
262
|
} else {
|
|
244
263
|
deckPath = resolve(opts.deck)
|
|
245
|
-
if (!deckExists(deckPath)) {
|
|
264
|
+
if (!deckExists(deckPath)) { error(`❌ Deck file not found: ${deckPath}
|
|
246
265
|
Make sure the path is correct, or use a URL:
|
|
247
266
|
--deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml
|
|
248
|
-
(URLs are auto-fetched — no local file needed)`);
|
|
267
|
+
(URLs are auto-fetched — no local file needed)`); exit(1) }
|
|
249
268
|
}
|
|
250
269
|
|
|
251
270
|
const { useAgent } = await import('@lythos/test-utils/agents')
|
|
@@ -263,9 +282,9 @@ async function singleRun(args: string[]) {
|
|
|
263
282
|
// Markdown is for LLM agents; task text is read/stored as a raw string.
|
|
264
283
|
const taskText = resolvedTaskPath ? readFileSync(resolvedTaskPath, 'utf-8') : opts.brief!
|
|
265
284
|
|
|
266
|
-
|
|
267
|
-
if (opts.task)
|
|
268
|
-
else
|
|
285
|
+
log(`🤖 agent-run: ${player} × ${deckPath}`)
|
|
286
|
+
if (opts.task) log(`📋 task: ${resolve(opts.task!)}`)
|
|
287
|
+
else log(`📋 brief: ${opts.brief!.slice(0, 60)}...`)
|
|
269
288
|
|
|
270
289
|
// Setup workdir
|
|
271
290
|
const agentWorkdir = join(tmpdir(), `arena-single-${Date.now()}`)
|
|
@@ -298,11 +317,11 @@ async function singleRun(args: string[]) {
|
|
|
298
317
|
const linkStderr = await new Response(linkProc.stderr).text()
|
|
299
318
|
const linkResult = validateLinkResult(linkProc.exitCode, linkStderr)
|
|
300
319
|
if (!linkResult.ok) {
|
|
301
|
-
|
|
302
|
-
|
|
320
|
+
error(`❌ ${linkResult.error}`)
|
|
321
|
+
exit(1)
|
|
303
322
|
}
|
|
304
323
|
} else {
|
|
305
|
-
|
|
324
|
+
log('ℹ️ No skills declared in deck — skipping link')
|
|
306
325
|
}
|
|
307
326
|
|
|
308
327
|
const { existsSync: es } = await import('node:fs')
|
|
@@ -313,10 +332,10 @@ async function singleRun(args: string[]) {
|
|
|
313
332
|
const skills = parseDeckSkills(deckParsed)
|
|
314
333
|
const checks = checkSkillExistence(skills, coldPoolDir, es)
|
|
315
334
|
for (const warning of formatSkillWarnings(checks)) {
|
|
316
|
-
|
|
335
|
+
warn(`⚠️ ${warning}`)
|
|
317
336
|
}
|
|
318
337
|
} catch (e) {
|
|
319
|
-
|
|
338
|
+
warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
|
|
320
339
|
}
|
|
321
340
|
|
|
322
341
|
// Template injection: brief is the {task} variable, template carries fixed contract
|
|
@@ -345,23 +364,24 @@ async function singleRun(args: string[]) {
|
|
|
345
364
|
const plan = buildCopyPlan(agentWorkdir, outDir, entries, skipSet)
|
|
346
365
|
for (const { src, dest, name } of plan) {
|
|
347
366
|
try { cpSync(src, dest, { recursive: true }) } catch (e) {
|
|
348
|
-
|
|
367
|
+
warn(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
|
|
349
368
|
}
|
|
350
369
|
}
|
|
351
370
|
} catch (e) {
|
|
352
|
-
|
|
371
|
+
warn(`⚠️ Failed to copy agent output: ${e instanceof Error ? e.message : e}`)
|
|
353
372
|
}
|
|
354
373
|
}
|
|
355
374
|
|
|
356
375
|
// Summary (no judge — single mode is execution-only)
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
376
|
+
log(`\n✅ Agent run complete → ${outDir}`)
|
|
377
|
+
log(` deck: ${deckPath}`)
|
|
378
|
+
log(` player: ${player}`)
|
|
360
379
|
}
|
|
361
380
|
|
|
362
381
|
// ── vs: arena.toml-driven comparison ──────────────────────────────────────
|
|
363
382
|
|
|
364
|
-
async function vsRun(args: string[]) {
|
|
383
|
+
async function vsRun(args: string[], io: ArenaCliIO) {
|
|
384
|
+
const { log, error, exit } = { ...defaultArenaCliIO, ...io }
|
|
365
385
|
// Native TOML parser is simpler than adding smol-toml dependency
|
|
366
386
|
const opts: Record<string, string | undefined> = {}
|
|
367
387
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -372,14 +392,14 @@ async function vsRun(args: string[]) {
|
|
|
372
392
|
}
|
|
373
393
|
|
|
374
394
|
if (!opts.config) {
|
|
375
|
-
|
|
376
|
-
|
|
395
|
+
error('❌ arena.toml path required: lythoskill-arena vs --config arena.toml')
|
|
396
|
+
exit(1)
|
|
377
397
|
}
|
|
378
398
|
|
|
379
399
|
const configPath = resolve(opts.config)
|
|
380
400
|
if (!existsSync(configPath)) {
|
|
381
|
-
|
|
382
|
-
|
|
401
|
+
error(`❌ Config file not found: ${configPath}`)
|
|
402
|
+
exit(1)
|
|
383
403
|
}
|
|
384
404
|
|
|
385
405
|
const toml = parseArenaToml(readFileSync(configPath, 'utf-8'))
|
|
@@ -395,11 +415,11 @@ async function vsRun(args: string[]) {
|
|
|
395
415
|
const isDryRun = opts.dryRun === 'true'
|
|
396
416
|
|
|
397
417
|
if (isDryRun) {
|
|
398
|
-
|
|
418
|
+
log(`🔍 Scanning arena.toml: ${configPath}`)
|
|
399
419
|
} else {
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
420
|
+
log(`🏟 Arena VS: ${configPath}`)
|
|
421
|
+
log(` sides: ${toml.side.length}`)
|
|
422
|
+
log(` runs per side: ${toml.arena.runs_per_side}`)
|
|
403
423
|
}
|
|
404
424
|
|
|
405
425
|
const { runArenaFromToml } = await import('./runner')
|
|
@@ -408,38 +428,40 @@ async function vsRun(args: string[]) {
|
|
|
408
428
|
taskPath,
|
|
409
429
|
outDir: opts.out ? resolve(opts.out) : undefined,
|
|
410
430
|
dryRun: isDryRun,
|
|
411
|
-
log:
|
|
431
|
+
log: log,
|
|
412
432
|
configDir: resolve(configPath, '..'),
|
|
413
433
|
})
|
|
414
434
|
|
|
415
435
|
if ('plan' in result) {
|
|
416
|
-
if (!isDryRun)
|
|
417
|
-
for (const line of formatPlanOutput(result.plan))
|
|
436
|
+
if (!isDryRun) log('📋 Execution plan (dry-run):')
|
|
437
|
+
for (const line of formatPlanOutput(result.plan)) log(line)
|
|
418
438
|
} else if ('manifest' in result) {
|
|
419
439
|
const r = result
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
440
|
+
log(`\n📊 Arena complete: ${r.manifest.id}`)
|
|
441
|
+
log(` report: ${r.artifactsDir}/report.md`)
|
|
442
|
+
log(` participants: ${r.manifest.participants.map(p => p.name).join(', ')}`)
|
|
423
443
|
}
|
|
424
444
|
}
|
|
425
445
|
|
|
426
446
|
// ── viz: generate HTML report from arena.json ─────────────────────────────
|
|
427
447
|
|
|
428
|
-
async function vizRun(args: string[]) {
|
|
448
|
+
async function vizRun(args: string[], io: ArenaCliIO) {
|
|
449
|
+
const { error, exit, log } = { ...defaultArenaCliIO, ...io }
|
|
429
450
|
const runsDir = args.find(a => !a.startsWith('-'))
|
|
430
|
-
if (!runsDir) {
|
|
451
|
+
if (!runsDir) { error('❌ runs/<arena-id> path required: lythoskill-arena viz runs/arena-20260504'); exit(1) }
|
|
431
452
|
|
|
432
453
|
const arenaJsonPath = resolve(runsDir, 'arena.json')
|
|
433
|
-
if (!existsSync(arenaJsonPath)) {
|
|
454
|
+
if (!existsSync(arenaJsonPath)) { error(`❌ arena.json not found in: ${runsDir}`); exit(1) }
|
|
434
455
|
|
|
435
|
-
|
|
456
|
+
log(`📈 Arena HTML report not yet implemented. See report.md in ${runsDir}/`)
|
|
436
457
|
}
|
|
437
458
|
|
|
438
459
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
439
460
|
// ── prepare-workdir: reusable workdir setup (used by both CLI and agent) ──
|
|
440
461
|
// Intent: create an isolated arena workdir with deck linked and ready to run
|
|
441
462
|
|
|
442
|
-
async function prepareWorkdir(args: string[]) {
|
|
463
|
+
async function prepareWorkdir(args: string[], io: ArenaCliIO) {
|
|
464
|
+
const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
|
|
443
465
|
const opts: Record<string, string | undefined> = {}
|
|
444
466
|
let dryRun = false
|
|
445
467
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -450,15 +472,15 @@ async function prepareWorkdir(args: string[]) {
|
|
|
450
472
|
}
|
|
451
473
|
|
|
452
474
|
if (!opts.deck) {
|
|
453
|
-
|
|
475
|
+
error(`❌ --deck <path> is required.
|
|
454
476
|
lythoskill-arena prepare-workdir --deck ./skill-deck.toml --out /tmp/arena-side-a`)
|
|
455
|
-
|
|
477
|
+
exit(1)
|
|
456
478
|
}
|
|
457
479
|
|
|
458
480
|
const deckPath = resolve(opts.deck)
|
|
459
481
|
if (!existsSync(deckPath)) {
|
|
460
|
-
|
|
461
|
-
|
|
482
|
+
error(`❌ Deck file not found: ${deckPath}`)
|
|
483
|
+
exit(1)
|
|
462
484
|
}
|
|
463
485
|
|
|
464
486
|
const workDir = opts.out
|
|
@@ -475,16 +497,16 @@ async function prepareWorkdir(args: string[]) {
|
|
|
475
497
|
brief: opts.brief,
|
|
476
498
|
})
|
|
477
499
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
if (opts.brief)
|
|
500
|
+
log('📋 Prepare plan:')
|
|
501
|
+
log(` deck: ${plan.deckPath}`)
|
|
502
|
+
log(` workdir: ${plan.workDir}`)
|
|
503
|
+
log(` skills: ${plan.skills.length} declared (${plan.skills.map(s => s.name).join(', ') || 'none'})`)
|
|
504
|
+
log(` link: ${plan.hasSkills ? 'Bun.spawn deck link' : 'skip (no skills)'}`)
|
|
505
|
+
log(` AGENTS.md: write (${plan.agentsMd.split('\n').length} lines)`)
|
|
506
|
+
if (opts.brief) log(` brief: ${opts.brief!.slice(0, 60)}...`)
|
|
485
507
|
|
|
486
508
|
if (dryRun) {
|
|
487
|
-
|
|
509
|
+
log(`\n🏁 Dry-run complete (no files created). Remove --dry-run to execute.`)
|
|
488
510
|
return
|
|
489
511
|
}
|
|
490
512
|
|
|
@@ -506,11 +528,11 @@ async function prepareWorkdir(args: string[]) {
|
|
|
506
528
|
const linkStderr = await new Response(linkProc.stderr).text()
|
|
507
529
|
const linkResult = validateLinkResult(linkProc.exitCode, linkStderr)
|
|
508
530
|
if (!linkResult.ok) {
|
|
509
|
-
|
|
510
|
-
|
|
531
|
+
error(`❌ ${linkResult.error}`)
|
|
532
|
+
exit(1)
|
|
511
533
|
}
|
|
512
534
|
} else {
|
|
513
|
-
|
|
535
|
+
log('ℹ️ No skills declared in deck — skipping link')
|
|
514
536
|
}
|
|
515
537
|
|
|
516
538
|
// Skill existence check
|
|
@@ -519,22 +541,23 @@ async function prepareWorkdir(args: string[]) {
|
|
|
519
541
|
const coldPoolDir = resolveColdPoolDir(Bun.TOML.parse(deckContent)?.deck?.cold_pool, homedir(), coldPoolDefault)
|
|
520
542
|
const checks = checkSkillExistence(plan.skills, coldPoolDir, existsSync)
|
|
521
543
|
for (const warning of formatSkillWarnings(checks)) {
|
|
522
|
-
|
|
544
|
+
warn(`⚠️ ${warning}`)
|
|
523
545
|
}
|
|
524
546
|
} catch (e) {
|
|
525
|
-
|
|
547
|
+
warn('⚠️ Could not check skill existence:', e instanceof Error ? e.message : e)
|
|
526
548
|
}
|
|
527
549
|
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
if (opts.brief)
|
|
550
|
+
log(`✅ Workdir ready → ${workDir}`)
|
|
551
|
+
log(` deck: ${deckPath}`)
|
|
552
|
+
if (opts.brief) log(` brief: ${opts.brief!.slice(0, 60)}...`)
|
|
531
553
|
}
|
|
532
554
|
|
|
533
555
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
534
556
|
// ── archive: copy agent outputs from workdir(s) to outDir ─────────────────
|
|
535
557
|
// Intent: same copy behavior as CLI singleRun, reusable for agent-orchestrated
|
|
536
558
|
|
|
537
|
-
async function archiveRun(args: string[]) {
|
|
559
|
+
async function archiveRun(args: string[], io: ArenaCliIO) {
|
|
560
|
+
const { log, error, warn, exit } = { ...defaultArenaCliIO, ...io }
|
|
538
561
|
const opts: Record<string, string | undefined> = {}
|
|
539
562
|
let dryRun = false
|
|
540
563
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -546,9 +569,9 @@ async function archiveRun(args: string[]) {
|
|
|
546
569
|
}
|
|
547
570
|
|
|
548
571
|
if (!opts.from || !opts.to) {
|
|
549
|
-
|
|
572
|
+
error(`❌ --from <workdir> and --to <outdir> are required.
|
|
550
573
|
lythoskill-arena archive --from /tmp/arena-20260517 --to playground/arena-20260517 --sides side-a,side-b --report ./report.md`)
|
|
551
|
-
|
|
574
|
+
exit(1)
|
|
552
575
|
}
|
|
553
576
|
|
|
554
577
|
const fromDir = resolve(opts.from)
|
|
@@ -558,18 +581,18 @@ async function archiveRun(args: string[]) {
|
|
|
558
581
|
const plan = buildArchiveSidePlan(fromDir, sides, existsSync)
|
|
559
582
|
|
|
560
583
|
// ── Plan output (always shown, also serves as dry-run) ──────────────────
|
|
561
|
-
|
|
584
|
+
log('📋 Archive plan:')
|
|
562
585
|
for (const pe of plan) {
|
|
563
586
|
if (!pe.found) {
|
|
564
|
-
|
|
587
|
+
log(` ⚠️ ${pe.side}: not found (${pe.sourceDir}) — will skip`)
|
|
565
588
|
} else if (pe.sourceDir === fromDir && pe.side !== '.') {
|
|
566
|
-
|
|
589
|
+
log(` ${pe.side}: ${pe.sourceDir} (fallback → root) → ${join(outDir, pe.side)}`)
|
|
567
590
|
} else {
|
|
568
|
-
|
|
591
|
+
log(` ${pe.side}: ${pe.sourceDir} → ${join(outDir, pe.side)}`)
|
|
569
592
|
}
|
|
570
593
|
}
|
|
571
594
|
if (dryRun) {
|
|
572
|
-
|
|
595
|
+
log(`\n🏁 Dry-run complete (no files copied). Remove --dry-run to execute.`)
|
|
573
596
|
return
|
|
574
597
|
}
|
|
575
598
|
|
|
@@ -579,7 +602,7 @@ async function archiveRun(args: string[]) {
|
|
|
579
602
|
if (opts.report && existsSync(resolve(opts.report))) {
|
|
580
603
|
const { cpSync: cpR } = await import('node:fs')
|
|
581
604
|
cpR(resolve(opts.report), join(outDir, 'report.md'))
|
|
582
|
-
|
|
605
|
+
log(`📄 report.md → ${outDir}/report.md`)
|
|
583
606
|
}
|
|
584
607
|
|
|
585
608
|
const { cpSync, readdirSync } = await import('node:fs')
|
|
@@ -587,7 +610,7 @@ async function archiveRun(args: string[]) {
|
|
|
587
610
|
|
|
588
611
|
for (const planEntry of plan) {
|
|
589
612
|
if (!planEntry.found) {
|
|
590
|
-
|
|
613
|
+
warn(`⚠️ Side workdir not found: ${planEntry.sourceDir}`)
|
|
591
614
|
continue
|
|
592
615
|
}
|
|
593
616
|
|
|
@@ -601,14 +624,14 @@ async function archiveRun(args: string[]) {
|
|
|
601
624
|
const dest = join(sideOutDir, entry.name)
|
|
602
625
|
try {
|
|
603
626
|
cpSync(src, dest, { recursive: entry.isDirectory() })
|
|
604
|
-
|
|
627
|
+
log(` ${planEntry.side}/${entry.name} → ${dest}`)
|
|
605
628
|
} catch (e) {
|
|
606
|
-
|
|
629
|
+
warn(`⚠️ Failed to copy ${planEntry.side}/${entry.name}: ${e instanceof Error ? e.message : e}`)
|
|
607
630
|
}
|
|
608
631
|
}
|
|
609
632
|
}
|
|
610
633
|
|
|
611
|
-
|
|
634
|
+
log(`✅ Archive complete → ${outDir}`)
|
|
612
635
|
}
|
|
613
636
|
|
|
614
637
|
// ── Entry point ────────────────────────────────────────────────────────────
|
package/src/runner.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, test, expect } from 'bun:test'
|
|
2
|
-
import { buildArenaPrompt, formatPlanOutput } from './runner'
|
|
2
|
+
import { buildArenaPrompt, formatPlanOutput, runArenaFromToml, type ArenaIO } from './runner'
|
|
3
3
|
import { buildExecutionPlan, parseArenaToml } from './arena-toml'
|
|
4
4
|
|
|
5
5
|
describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
@@ -89,3 +89,262 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
89
89
|
expect(buildArenaPrompt(opts)).toBe(buildArenaPrompt(opts))
|
|
90
90
|
})
|
|
91
91
|
})
|
|
92
|
+
|
|
93
|
+
describe('runArenaFromToml — dry-run mode (no fs/spawn/agent calls)', () => {
|
|
94
|
+
test('returns plan without executing cells', async () => {
|
|
95
|
+
const toml = parseArenaToml(`
|
|
96
|
+
[arena]
|
|
97
|
+
task = "Write a hello world function"
|
|
98
|
+
criteria = ["completeness", "correctness"]
|
|
99
|
+
runs_per_side = 1
|
|
100
|
+
|
|
101
|
+
[[side]]
|
|
102
|
+
name = "claude"
|
|
103
|
+
player = "claude"
|
|
104
|
+
deck = "/tmp/deck.toml"
|
|
105
|
+
|
|
106
|
+
[[side]]
|
|
107
|
+
name = "kimi"
|
|
108
|
+
player = "kimi"
|
|
109
|
+
deck = "/tmp/deck.toml"
|
|
110
|
+
`)
|
|
111
|
+
|
|
112
|
+
const logs: string[] = []
|
|
113
|
+
const result = await runArenaFromToml({
|
|
114
|
+
toml,
|
|
115
|
+
taskPath: '/tmp/task.md',
|
|
116
|
+
dryRun: true,
|
|
117
|
+
log: (msg) => logs.push(msg),
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
expect('plan' in result).toBe(true)
|
|
121
|
+
const plan = (result as { plan: ReturnType<typeof buildExecutionPlan> }).plan
|
|
122
|
+
expect(plan.total_runs).toBe(2)
|
|
123
|
+
expect(plan.cells.length).toBe(2)
|
|
124
|
+
expect(logs.length).toBeGreaterThan(0)
|
|
125
|
+
expect(logs.some(l => l.includes('Dry-run'))).toBe(true)
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
test('dry-run does not call io.mkdir or io.agentSpawn', async () => {
|
|
129
|
+
const toml = parseArenaToml(`
|
|
130
|
+
[arena]
|
|
131
|
+
task = "Write a hello world function"
|
|
132
|
+
criteria = ["completeness"]
|
|
133
|
+
runs_per_side = 1
|
|
134
|
+
|
|
135
|
+
[[side]]
|
|
136
|
+
name = "claude"
|
|
137
|
+
player = "claude"
|
|
138
|
+
deck = "/tmp/deck.toml"
|
|
139
|
+
|
|
140
|
+
[[side]]
|
|
141
|
+
name = "kimi"
|
|
142
|
+
player = "kimi"
|
|
143
|
+
deck = "/tmp/deck.toml"
|
|
144
|
+
`)
|
|
145
|
+
|
|
146
|
+
let mkdirCalled = false
|
|
147
|
+
let agentSpawnCalled = false
|
|
148
|
+
|
|
149
|
+
const mockIO: ArenaIO = {
|
|
150
|
+
mkdir: () => { mkdirCalled = true },
|
|
151
|
+
agentSpawn: async () => { agentSpawnCalled = true; return { stdout: '', stderr: '', durationMs: 0 } },
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
await runArenaFromToml({
|
|
155
|
+
toml,
|
|
156
|
+
taskPath: '/tmp/task.md',
|
|
157
|
+
dryRun: true,
|
|
158
|
+
io: mockIO,
|
|
159
|
+
})
|
|
160
|
+
|
|
161
|
+
expect(mkdirCalled).toBe(false)
|
|
162
|
+
expect(agentSpawnCalled).toBe(false)
|
|
163
|
+
})
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
describe('runArenaFromToml — single cell execution with mock agentSpawn', () => {
|
|
167
|
+
test('executes one cell and writes outputs via io', async () => {
|
|
168
|
+
const toml = parseArenaToml(`
|
|
169
|
+
[arena]
|
|
170
|
+
task = "Write a hello world function"
|
|
171
|
+
criteria = ["completeness"]
|
|
172
|
+
runs_per_side = 1
|
|
173
|
+
|
|
174
|
+
[[side]]
|
|
175
|
+
name = "claude"
|
|
176
|
+
player = "claude"
|
|
177
|
+
deck = "/tmp/deck.toml"
|
|
178
|
+
|
|
179
|
+
[[side]]
|
|
180
|
+
name = "kimi"
|
|
181
|
+
player = "kimi"
|
|
182
|
+
deck = "/tmp/deck.toml"
|
|
183
|
+
`)
|
|
184
|
+
|
|
185
|
+
const files = new Map<string, string>()
|
|
186
|
+
const dirs = new Set<string>()
|
|
187
|
+
|
|
188
|
+
const mockIO: ArenaIO = {
|
|
189
|
+
log: () => {},
|
|
190
|
+
mkdir: (path: string) => { dirs.add(path) },
|
|
191
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
192
|
+
readFile: (path: string) => {
|
|
193
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
194
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
195
|
+
},
|
|
196
|
+
readdir: () => ['decision-log.jsonl'],
|
|
197
|
+
cp: () => {},
|
|
198
|
+
spawn: async () => ({ exitCode: 0, stderr: '' }),
|
|
199
|
+
agentSpawn: async () => ({
|
|
200
|
+
stdout: 'I wrote hello world',
|
|
201
|
+
stderr: '',
|
|
202
|
+
durationMs: 1234,
|
|
203
|
+
}),
|
|
204
|
+
exists: (path: string) => path === '/tmp/deck.toml',
|
|
205
|
+
chdir: () => {},
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const result = await runArenaFromToml({
|
|
209
|
+
toml,
|
|
210
|
+
taskPath: '/tmp/task.md',
|
|
211
|
+
outDir: '/tmp/arena-out',
|
|
212
|
+
io: mockIO,
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
expect('manifest' in result).toBe(true)
|
|
216
|
+
const r = result as { manifest: { id: string; status: string }; artifactsDir: string }
|
|
217
|
+
expect(r.manifest.status).toBe('completed')
|
|
218
|
+
expect(r.artifactsDir).toBe('/tmp/arena-out')
|
|
219
|
+
|
|
220
|
+
// arena.json written twice (initial + final)
|
|
221
|
+
expect(files.has('/tmp/arena-out/arena.json')).toBe(true)
|
|
222
|
+
// agent stdout persisted
|
|
223
|
+
expect(files.has('/tmp/arena-out/runs/claude/run-1/agent-stdout.txt')).toBe(true)
|
|
224
|
+
expect(files.get('/tmp/arena-out/runs/claude/run-1/agent-stdout.txt')).toContain('I wrote hello world')
|
|
225
|
+
// judge-verdict.json written
|
|
226
|
+
expect(files.has('/tmp/arena-out/runs/claude/run-1/judge-verdict.json')).toBe(true)
|
|
227
|
+
// report.md written
|
|
228
|
+
expect(files.has('/tmp/arena-out/report.md')).toBe(true)
|
|
229
|
+
// directories created
|
|
230
|
+
expect(dirs.has('/tmp/arena-out')).toBe(true)
|
|
231
|
+
expect(dirs.has('/tmp/arena-out/runs/claude/run-1')).toBe(true)
|
|
232
|
+
expect(dirs.has('/tmp/arena-out/work/claude')).toBe(true)
|
|
233
|
+
})
|
|
234
|
+
})
|
|
235
|
+
|
|
236
|
+
describe('runArenaFromToml — error recovery path', () => {
|
|
237
|
+
test('cell exception produces ERROR verdict and continues', async () => {
|
|
238
|
+
const toml = parseArenaToml(`
|
|
239
|
+
[arena]
|
|
240
|
+
task = "Write a hello world function"
|
|
241
|
+
criteria = ["completeness"]
|
|
242
|
+
runs_per_side = 1
|
|
243
|
+
|
|
244
|
+
[[side]]
|
|
245
|
+
name = "claude"
|
|
246
|
+
player = "claude"
|
|
247
|
+
deck = "/tmp/deck.toml"
|
|
248
|
+
|
|
249
|
+
[[side]]
|
|
250
|
+
name = "kimi"
|
|
251
|
+
player = "kimi"
|
|
252
|
+
deck = "/tmp/deck.toml"
|
|
253
|
+
`)
|
|
254
|
+
|
|
255
|
+
const files = new Map<string, string>()
|
|
256
|
+
|
|
257
|
+
let spawnCount = 0
|
|
258
|
+
const mockIO: ArenaIO = {
|
|
259
|
+
log: () => {},
|
|
260
|
+
mkdir: () => {},
|
|
261
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
262
|
+
readFile: (path: string) => {
|
|
263
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
264
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
265
|
+
},
|
|
266
|
+
readdir: () => [],
|
|
267
|
+
cp: () => {},
|
|
268
|
+
spawn: async () => ({ exitCode: 0, stderr: '' }),
|
|
269
|
+
agentSpawn: async () => {
|
|
270
|
+
spawnCount++
|
|
271
|
+
if (spawnCount === 1) throw new Error('Agent spawn failed')
|
|
272
|
+
return { stdout: 'ok', stderr: '', durationMs: 100 }
|
|
273
|
+
},
|
|
274
|
+
exists: (path: string) => path === '/tmp/deck.toml',
|
|
275
|
+
chdir: () => {},
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
const result = await runArenaFromToml({
|
|
279
|
+
toml,
|
|
280
|
+
taskPath: '/tmp/task.md',
|
|
281
|
+
outDir: '/tmp/arena-out',
|
|
282
|
+
io: mockIO,
|
|
283
|
+
})
|
|
284
|
+
|
|
285
|
+
expect('manifest' in result).toBe(true)
|
|
286
|
+
const r = result as { manifest: { status: string }; stats: { sideName: string; errorRate: number }[] }
|
|
287
|
+
expect(r.manifest.status).toBe('completed')
|
|
288
|
+
expect(r.stats.length).toBe(2)
|
|
289
|
+
const claudeStats = r.stats.find(s => s.sideName === 'claude')
|
|
290
|
+
expect(claudeStats!.errorRate).toBe(1)
|
|
291
|
+
|
|
292
|
+
const verdictPath = '/tmp/arena-out/runs/claude/run-1/judge-verdict.json'
|
|
293
|
+
expect(files.has(verdictPath)).toBe(true)
|
|
294
|
+
const verdict = JSON.parse(files.get(verdictPath)!)
|
|
295
|
+
expect(verdict.verdict).toBe('ERROR')
|
|
296
|
+
expect(verdict.reason).toContain('Agent spawn failed')
|
|
297
|
+
})
|
|
298
|
+
|
|
299
|
+
test('spawn non-zero exit logs warning but continues', async () => {
|
|
300
|
+
const toml = parseArenaToml(`
|
|
301
|
+
[arena]
|
|
302
|
+
task = "Write a hello world function"
|
|
303
|
+
criteria = ["completeness"]
|
|
304
|
+
runs_per_side = 1
|
|
305
|
+
|
|
306
|
+
[[side]]
|
|
307
|
+
name = "claude"
|
|
308
|
+
player = "claude"
|
|
309
|
+
deck = "/tmp/deck.toml"
|
|
310
|
+
|
|
311
|
+
[[side]]
|
|
312
|
+
name = "kimi"
|
|
313
|
+
player = "kimi"
|
|
314
|
+
deck = "/tmp/deck.toml"
|
|
315
|
+
`)
|
|
316
|
+
|
|
317
|
+
const logs: string[] = []
|
|
318
|
+
const files = new Map<string, string>()
|
|
319
|
+
|
|
320
|
+
const mockIO: ArenaIO = {
|
|
321
|
+
log: (msg: string) => logs.push(msg),
|
|
322
|
+
mkdir: () => {},
|
|
323
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
324
|
+
readFile: (path: string) => {
|
|
325
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
326
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
327
|
+
},
|
|
328
|
+
readdir: () => [],
|
|
329
|
+
cp: () => {},
|
|
330
|
+
spawn: async () => ({ exitCode: 1, stderr: 'link failed' }),
|
|
331
|
+
agentSpawn: async () => ({
|
|
332
|
+
stdout: 'done',
|
|
333
|
+
stderr: '',
|
|
334
|
+
durationMs: 100,
|
|
335
|
+
}),
|
|
336
|
+
exists: (path: string) => path === '/tmp/deck.toml',
|
|
337
|
+
chdir: () => {},
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const result = await runArenaFromToml({
|
|
341
|
+
toml,
|
|
342
|
+
taskPath: '/tmp/task.md',
|
|
343
|
+
outDir: '/tmp/arena-out',
|
|
344
|
+
io: mockIO,
|
|
345
|
+
})
|
|
346
|
+
|
|
347
|
+
expect('manifest' in result).toBe(true)
|
|
348
|
+
expect(logs.some(l => l.includes('deck link') && l.includes('exit 1'))).toBe(true)
|
|
349
|
+
})
|
|
350
|
+
})
|
package/src/runner.ts
CHANGED
|
@@ -17,6 +17,42 @@ import { aggregateAllStats } from './stats'
|
|
|
17
17
|
import type { SideStats } from './stats'
|
|
18
18
|
import { buildCopyPlan } from './preflight'
|
|
19
19
|
|
|
20
|
+
// ── ArenaIO interface (Intent/Plan/Execute fractal pattern) ───────────────
|
|
21
|
+
|
|
22
|
+
export interface ArenaIO {
|
|
23
|
+
log?: (msg: string) => void
|
|
24
|
+
mkdir?: (path: string, opts?: { recursive?: boolean }) => void
|
|
25
|
+
writeFile?: (path: string, data: string) => void
|
|
26
|
+
readFile?: (path: string) => string
|
|
27
|
+
readdir?: (path: string) => string[]
|
|
28
|
+
cp?: (src: string, dest: string, opts?: { recursive?: boolean }) => void
|
|
29
|
+
spawn?: (cmd: string[], opts: { cwd: string; env?: Record<string, string> }) => Promise<{ exitCode: number | null; stderr: string }>
|
|
30
|
+
agentSpawn?: (opts: { player: string; cwd: string; brief: string; timeoutMs: number }) => Promise<{ stdout: string; stderr: string; durationMs: number }>
|
|
31
|
+
exists?: (path: string) => boolean
|
|
32
|
+
chdir?: (cwd: string) => void
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export const defaultArenaIO: ArenaIO = {
|
|
36
|
+
log: (msg: string) => { console.log(msg) },
|
|
37
|
+
mkdir: (path: string, opts?: { recursive?: boolean }) => mkdirSync(path, opts),
|
|
38
|
+
writeFile: (path: string, data: string) => writeFileSync(path, data, 'utf-8'),
|
|
39
|
+
readFile: (path: string) => readFileSync(path, 'utf-8'),
|
|
40
|
+
readdir: (path: string) => readdirSync(path),
|
|
41
|
+
cp: (src: string, dest: string, opts?: { recursive?: boolean }) => cpSync(src, dest, opts),
|
|
42
|
+
spawn: async (cmd: string[], opts: { cwd: string; env?: Record<string, string> }) => {
|
|
43
|
+
const proc = Bun.spawn(cmd, { cwd: opts.cwd, env: opts.env })
|
|
44
|
+
await proc.exited
|
|
45
|
+
const stderr = await new Response(proc.stderr).text()
|
|
46
|
+
return { exitCode: proc.exitCode, stderr }
|
|
47
|
+
},
|
|
48
|
+
agentSpawn: async (opts: { player: string; cwd: string; brief: string; timeoutMs: number }) => {
|
|
49
|
+
const agent = useAgent(opts.player)
|
|
50
|
+
return agent.spawn({ cwd: opts.cwd, brief: opts.brief, timeoutMs: opts.timeoutMs })
|
|
51
|
+
},
|
|
52
|
+
exists: (path: string) => existsSync(path),
|
|
53
|
+
chdir: (cwd: string) => { process.chdir(cwd) },
|
|
54
|
+
}
|
|
55
|
+
|
|
20
56
|
// ── Helpers ───────────────────────────────────────────────────────────────
|
|
21
57
|
|
|
22
58
|
function stamp(): string {
|
|
@@ -33,18 +69,18 @@ export interface ArenaResult {
|
|
|
33
69
|
|
|
34
70
|
// ── Task + judge text resolution (no parsing — natural language) ──────────
|
|
35
71
|
|
|
36
|
-
function resolveTaskText(toml: ArenaToml, configDir?: string): string {
|
|
72
|
+
function resolveTaskText(toml: ArenaToml, io: Required<Pick<ArenaIO, 'exists' | 'readFile'>>, configDir?: string): string {
|
|
37
73
|
const p = toml.arena.task
|
|
38
74
|
const candidate = configDir ? resolve(configDir, p) : resolve(p)
|
|
39
|
-
if (
|
|
75
|
+
if (io.exists(candidate)) return io.readFile(candidate)
|
|
40
76
|
return p
|
|
41
77
|
}
|
|
42
78
|
|
|
43
|
-
function resolveJudgeText(toml: ArenaToml, configDir?: string): string | null {
|
|
79
|
+
function resolveJudgeText(toml: ArenaToml, io: Required<Pick<ArenaIO, 'exists' | 'readFile'>>, configDir?: string): string | null {
|
|
44
80
|
if (toml.arena.judge) {
|
|
45
81
|
const p = toml.arena.judge
|
|
46
82
|
const candidate = configDir ? resolve(configDir, p) : resolve(p)
|
|
47
|
-
if (
|
|
83
|
+
if (io.exists(candidate)) return io.readFile(candidate)
|
|
48
84
|
return p
|
|
49
85
|
}
|
|
50
86
|
if (toml.arena.criteria && toml.arena.criteria.length > 0) {
|
|
@@ -116,8 +152,14 @@ export async function runArenaFromToml(opts: {
|
|
|
116
152
|
dryRun?: boolean
|
|
117
153
|
log?: (msg: string) => void
|
|
118
154
|
configDir?: string
|
|
155
|
+
io?: ArenaIO
|
|
119
156
|
}): Promise<ArenaResult | { plan: ReturnType<typeof buildExecutionPlan> }> {
|
|
120
157
|
const { toml, taskPath, outDir, dryRun, log, configDir } = opts
|
|
158
|
+
const io: ArenaIO = {
|
|
159
|
+
...defaultArenaIO,
|
|
160
|
+
...opts.io,
|
|
161
|
+
log: log ?? opts.io?.log ?? defaultArenaIO.log,
|
|
162
|
+
}
|
|
121
163
|
|
|
122
164
|
const resolvePath = (p: string) => {
|
|
123
165
|
if (p.startsWith('/')) return p
|
|
@@ -125,7 +167,19 @@ export async function runArenaFromToml(opts: {
|
|
|
125
167
|
return resolve(p)
|
|
126
168
|
}
|
|
127
169
|
|
|
128
|
-
const
|
|
170
|
+
const ioWithDefaults = {
|
|
171
|
+
exists: io.exists!,
|
|
172
|
+
readFile: io.readFile!,
|
|
173
|
+
writeFile: io.writeFile!,
|
|
174
|
+
mkdir: io.mkdir!,
|
|
175
|
+
readdir: io.readdir!,
|
|
176
|
+
cp: io.cp!,
|
|
177
|
+
spawn: io.spawn!,
|
|
178
|
+
agentSpawn: io.agentSpawn!,
|
|
179
|
+
chdir: io.chdir!,
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
const taskText = resolveTaskText(toml, ioWithDefaults, configDir)
|
|
129
183
|
const resolvedToml: ArenaToml = {
|
|
130
184
|
...toml,
|
|
131
185
|
side: toml.side.map(s => ({ ...s, deck: resolvePath(s.deck) })),
|
|
@@ -134,7 +188,7 @@ export async function runArenaFromToml(opts: {
|
|
|
134
188
|
const plan = buildExecutionPlan(resolvedToml)
|
|
135
189
|
|
|
136
190
|
if (dryRun) {
|
|
137
|
-
for (const line of formatPlanOutput(plan)) log?.(line)
|
|
191
|
+
for (const line of formatPlanOutput(plan)) io.log?.(line)
|
|
138
192
|
return { plan }
|
|
139
193
|
}
|
|
140
194
|
|
|
@@ -158,29 +212,29 @@ export async function runArenaFromToml(opts: {
|
|
|
158
212
|
status: 'running',
|
|
159
213
|
})
|
|
160
214
|
|
|
161
|
-
|
|
162
|
-
|
|
215
|
+
ioWithDefaults.mkdir(artifactsDir, { recursive: true })
|
|
216
|
+
ioWithDefaults.writeFile(join(artifactsDir, 'arena.json'), JSON.stringify(manifest, null, 2) + '\n')
|
|
163
217
|
|
|
164
|
-
const judgeText = resolveJudgeText(resolvedToml, configDir)
|
|
218
|
+
const judgeText = resolveJudgeText(resolvedToml, ioWithDefaults, configDir)
|
|
165
219
|
const judgeInput: JudgeInput | undefined = judgeText
|
|
166
220
|
? { criteria: judgeText, task_context: taskText.slice(0, 500) }
|
|
167
221
|
: undefined
|
|
168
222
|
|
|
169
|
-
// ── Per-cell: agent.spawn
|
|
223
|
+
// ── Per-cell: agent.spawn via IO injection ─────────────────────────────
|
|
170
224
|
const verdictsBySide = new Map<string, JudgeVerdict[]>()
|
|
171
225
|
|
|
172
226
|
for (const cell of plan.cells) {
|
|
173
227
|
const cellDir = join(artifactsDir, 'runs', cell.side, `run-${cell.run}`)
|
|
174
|
-
|
|
228
|
+
ioWithDefaults.mkdir(cellDir, { recursive: true })
|
|
175
229
|
|
|
176
230
|
const workDir = join(artifactsDir, 'work', cell.side)
|
|
177
|
-
|
|
231
|
+
ioWithDefaults.mkdir(workDir, { recursive: true })
|
|
178
232
|
const originalCwd = process.cwd()
|
|
179
233
|
|
|
180
234
|
try {
|
|
181
235
|
// Setup: deck + AGENTS.md + link
|
|
182
|
-
|
|
183
|
-
|
|
236
|
+
ioWithDefaults.writeFile(join(workDir, 'skill-deck.toml'), ioWithDefaults.readFile(cell.deck))
|
|
237
|
+
ioWithDefaults.writeFile(join(workDir, 'AGENTS.md'), [
|
|
184
238
|
'# Arena Test Environment',
|
|
185
239
|
`**Side**: ${cell.side}`, `**Player**: ${cell.player}`, `**Run**: ${cell.run}`,
|
|
186
240
|
'## How This Works',
|
|
@@ -188,55 +242,54 @@ export async function runArenaFromToml(opts: {
|
|
|
188
242
|
'- Complete the task using available skills. Output to this directory.',
|
|
189
243
|
'- MANDATORY: write decision-log.jsonl (see prompt for schema).',
|
|
190
244
|
].join('\n'))
|
|
191
|
-
const linkProc =
|
|
245
|
+
const linkProc = await ioWithDefaults.spawn(
|
|
192
246
|
['bunx', '@lythos/skill-deck', 'link'],
|
|
193
247
|
{ cwd: workDir, env: { ...process.env, HOME: process.env.HOME! } },
|
|
194
248
|
)
|
|
195
|
-
|
|
196
|
-
log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
|
|
249
|
+
io.log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
|
|
197
250
|
|
|
198
|
-
|
|
251
|
+
ioWithDefaults.chdir(workDir)
|
|
199
252
|
|
|
200
|
-
//
|
|
201
|
-
const agent = useAgent(resolvePlayer(cell.player))
|
|
253
|
+
// Agent spawn via IO injection
|
|
202
254
|
const fullPrompt = buildArenaPrompt({
|
|
203
255
|
brief: taskText,
|
|
204
256
|
cwd: workDir,
|
|
205
257
|
deckPath: cell.deck,
|
|
206
258
|
outputDir: workDir,
|
|
207
259
|
})
|
|
208
|
-
const agentResult = await
|
|
260
|
+
const agentResult = await ioWithDefaults.agentSpawn({
|
|
261
|
+
player: resolvePlayer(cell.player),
|
|
209
262
|
cwd: workDir,
|
|
210
263
|
brief: fullPrompt,
|
|
211
264
|
timeoutMs: 300000,
|
|
212
265
|
})
|
|
213
266
|
|
|
214
|
-
|
|
267
|
+
ioWithDefaults.chdir(originalCwd)
|
|
215
268
|
|
|
216
269
|
// Persist agent output
|
|
217
270
|
const sanitizer = createSanitizer({ projectRoot: process.cwd(), homeDir: homedir(), workDir })
|
|
218
|
-
|
|
219
|
-
if (agentResult.stderr)
|
|
271
|
+
ioWithDefaults.writeFile(join(cellDir, 'agent-stdout.txt'), sanitizer.sanitize(agentResult.stdout))
|
|
272
|
+
if (agentResult.stderr) ioWithDefaults.writeFile(join(cellDir, 'agent-stderr.txt'), sanitizer.sanitize(agentResult.stderr))
|
|
220
273
|
|
|
221
274
|
// Copy artifacts
|
|
222
275
|
const skipSet = new Set(['.claude', 'skill-deck.toml', 'skill-deck.lock', 'AGENTS.md'])
|
|
223
276
|
try {
|
|
224
|
-
const entries =
|
|
277
|
+
const entries = ioWithDefaults.readdir(workDir)
|
|
225
278
|
const copyPlan = buildCopyPlan(workDir, cellDir, entries, skipSet)
|
|
226
279
|
for (const { src, dest, name } of copyPlan) {
|
|
227
|
-
try {
|
|
228
|
-
log?.(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
|
|
280
|
+
try { ioWithDefaults.cp(src, dest, { recursive: true }) } catch (e) {
|
|
281
|
+
io.log?.(`⚠️ Failed to copy agent output: ${name} — ${e instanceof Error ? e.message : e}`)
|
|
229
282
|
}
|
|
230
283
|
}
|
|
231
284
|
} catch (e) {
|
|
232
|
-
log?.(`⚠️ Failed to read agent workdir for copy: ${e instanceof Error ? e.message : e}`)
|
|
285
|
+
io.log?.(`⚠️ Failed to read agent workdir for copy: ${e instanceof Error ? e.message : e}`)
|
|
233
286
|
}
|
|
234
287
|
|
|
235
288
|
// Evidence
|
|
236
289
|
const checkpoints = readCheckpoints(workDir)
|
|
237
290
|
let artifactFiles: string[] = []
|
|
238
291
|
try {
|
|
239
|
-
for (const e of
|
|
292
|
+
for (const e of ioWithDefaults.readdir(workDir)) {
|
|
240
293
|
if (!e.startsWith('.') && !skipSet.has(e) && e !== 'agent-stdout.txt' && e !== 'agent-stderr.txt' && e !== 'judge-verdict.json' && e !== '_checkpoints') {
|
|
241
294
|
artifactFiles.push(e)
|
|
242
295
|
}
|
|
@@ -259,7 +312,7 @@ export async function runArenaFromToml(opts: {
|
|
|
259
312
|
v = { verdict: 'ERROR' as const, reason: 'No judge criteria provided', criteria: [] }
|
|
260
313
|
}
|
|
261
314
|
|
|
262
|
-
|
|
315
|
+
ioWithDefaults.writeFile(join(cellDir, 'judge-verdict.json'), JSON.stringify({
|
|
263
316
|
...v,
|
|
264
317
|
agent_stdout: agentResult.stdout.slice(0, 5000),
|
|
265
318
|
agent_stderr: agentResult.stderr.slice(0, 1000),
|
|
@@ -274,7 +327,7 @@ export async function runArenaFromToml(opts: {
|
|
|
274
327
|
reason: `Runner exception: ${e instanceof Error ? e.message : String(e)}`,
|
|
275
328
|
criteria: [],
|
|
276
329
|
}
|
|
277
|
-
|
|
330
|
+
ioWithDefaults.writeFile(join(cellDir, 'judge-verdict.json'), JSON.stringify(errVerdict, null, 2) + '\n')
|
|
278
331
|
if (!verdictsBySide.has(cell.side)) verdictsBySide.set(cell.side, [])
|
|
279
332
|
verdictsBySide.get(cell.side)!.push(errVerdict)
|
|
280
333
|
}
|
|
@@ -288,10 +341,10 @@ export async function runArenaFromToml(opts: {
|
|
|
288
341
|
}
|
|
289
342
|
const judge = useAgent(resolved[0]?.platform ?? 'claude')
|
|
290
343
|
const report = await runComparativeJudge({ manifest, verdicts: flatVerdicts, judge, workdir: artifactsDir })
|
|
291
|
-
writeReport(artifactsDir, manifest, report, stats)
|
|
344
|
+
writeReport(artifactsDir, manifest, report, stats, ioWithDefaults)
|
|
292
345
|
|
|
293
346
|
const finalManifest = ArenaManifest.parse({ ...manifest, status: 'completed' })
|
|
294
|
-
|
|
347
|
+
ioWithDefaults.writeFile(join(artifactsDir, 'arena.json'), JSON.stringify(finalManifest, null, 2) + '\n')
|
|
295
348
|
|
|
296
349
|
return { manifest: finalManifest, report, stats, artifactsDir }
|
|
297
350
|
}
|
|
@@ -319,7 +372,7 @@ export async function runArena(opts: {
|
|
|
319
372
|
|
|
320
373
|
// ── Report ────────────────────────────────────────────────────────────────
|
|
321
374
|
|
|
322
|
-
function writeReport(dir: string, manifest: ArenaManifestType, report: any, stats: SideStats[]): void {
|
|
375
|
+
function writeReport(dir: string, manifest: ArenaManifestType, report: any, stats: SideStats[], io: Required<Pick<ArenaIO, 'writeFile'>>): void {
|
|
323
376
|
const lines: string[] = [
|
|
324
377
|
`# Arena Report: ${manifest.id}`, '',
|
|
325
378
|
`**Task**: ${manifest.task}`,
|
|
@@ -331,7 +384,7 @@ function writeReport(dir: string, manifest: ArenaManifestType, report: any, stat
|
|
|
331
384
|
'## Key Findings', '', ...(report.key_findings ?? []).map((f: string) => `- ${f}`), '',
|
|
332
385
|
'## Recommendations', '', ...(report.recommendations ?? []).map((r: any) => `- **${r.audience}**: ${r.recommendation}`),
|
|
333
386
|
]
|
|
334
|
-
|
|
387
|
+
io.writeFile(join(dir, 'report.md'), lines.join('\n') + '\n')
|
|
335
388
|
}
|
|
336
389
|
|
|
337
390
|
function renderStatsTable(stats: SideStats[]): string {
|