@lythos/skill-arena 0.16.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/package.json +7 -7
- package/src/arena-toml.test.ts +19 -19
- package/src/comparative-judge.test.ts +31 -31
- package/src/player.test.ts +11 -11
- package/src/preflight.test.ts +56 -56
- package/src/runner.test.ts +14 -14
- package/src/stats.test.ts +10 -10
package/README.md
CHANGED
|
@@ -16,20 +16,20 @@
|
|
|
16
16
|
```bash
|
|
17
17
|
bun add -d @lythos/skill-arena
|
|
18
18
|
# or use directly
|
|
19
|
-
bunx @lythos/skill-arena@0.16.
|
|
19
|
+
bunx @lythos/skill-arena@0.16.1 <command>
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
## Quick Start
|
|
23
23
|
|
|
24
24
|
```bash
|
|
25
25
|
# single — test one deck (most common)
|
|
26
|
-
bunx @lythos/skill-arena@0.16.
|
|
26
|
+
bunx @lythos/skill-arena@0.16.1 single \
|
|
27
27
|
--deck ./examples/decks/scout.toml \
|
|
28
28
|
--brief "Generate auth flow diagram" \
|
|
29
29
|
--out ./output
|
|
30
30
|
|
|
31
31
|
# single with explicit player
|
|
32
|
-
bunx @lythos/skill-arena@0.16.
|
|
32
|
+
bunx @lythos/skill-arena@0.16.1 single \
|
|
33
33
|
--deck ./examples/decks/scout.toml \
|
|
34
34
|
--brief "Generate auth flow diagram" \
|
|
35
35
|
--player kimi \
|
|
@@ -37,10 +37,10 @@ bunx @lythos/skill-arena@0.16.0 single \
|
|
|
37
37
|
|
|
38
38
|
# cross-deck vs — compare two decks (agent-orchestrated)
|
|
39
39
|
# Create arena.toml declaring sides with different decks, then:
|
|
40
|
-
bunx @lythos/skill-arena@0.16.
|
|
40
|
+
bunx @lythos/skill-arena@0.16.1 vs --config ./arena.toml
|
|
41
41
|
|
|
42
42
|
# cross-player vs — compare kimi vs codex (CLI only)
|
|
43
|
-
bunx @lythos/skill-arena@0.16.
|
|
43
|
+
bunx @lythos/skill-arena@0.16.1 vs --config ./arena.toml --player kimi
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
**What happens**: Agent creates isolated `/tmp` workdir per side, `deck link` skills, spawns parallel subagents, collects artifacts, judge scores outputs. Parent deck restored after.
|
|
@@ -50,14 +50,14 @@ bunx @lythos/skill-arena@0.16.0 vs --config ./arena.toml --player kimi
|
|
|
50
50
|
### `single` — one deck, one task
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
bunx @lythos/skill-arena@0.16.
|
|
53
|
+
bunx @lythos/skill-arena@0.16.1 single \
|
|
54
54
|
--deck ./deck.toml \
|
|
55
55
|
--brief "Produce a .docx report with radar chart" \
|
|
56
56
|
--timeout 600000 \
|
|
57
57
|
--out ./output
|
|
58
58
|
|
|
59
59
|
# with explicit player
|
|
60
|
-
bunx @lythos/skill-arena@0.16.
|
|
60
|
+
bunx @lythos/skill-arena@0.16.1 single \
|
|
61
61
|
--deck ./deck.toml \
|
|
62
62
|
--brief "Produce a .docx report with radar chart" \
|
|
63
63
|
--player kimi \
|
|
@@ -67,14 +67,14 @@ bunx @lythos/skill-arena@0.16.0 single \
|
|
|
67
67
|
### `vs` — multi-deck comparison
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
bunx @lythos/skill-arena@0.16.
|
|
71
|
-
bunx @lythos/skill-arena@0.16.
|
|
70
|
+
bunx @lythos/skill-arena@0.16.1 vs --config ./arena.toml
|
|
71
|
+
bunx @lythos/skill-arena@0.16.1 vs --config ./arena.toml --dry-run
|
|
72
72
|
```
|
|
73
73
|
|
|
74
74
|
### `prepare-workdir` — isolate + link skills (agent-orchestrated)
|
|
75
75
|
|
|
76
76
|
```bash
|
|
77
|
-
bunx @lythos/skill-arena@0.16.
|
|
77
|
+
bunx @lythos/skill-arena@0.16.1 prepare-workdir \
|
|
78
78
|
--deck ./skill-deck.toml \
|
|
79
79
|
--out /tmp/arena-side-a \
|
|
80
80
|
--brief "task description"
|
|
@@ -85,7 +85,7 @@ Creates `/tmp`-isolated workdir with deck copied, AGENTS.md written, and `deck l
|
|
|
85
85
|
### `archive` — collect agent outputs (agent-orchestrated)
|
|
86
86
|
|
|
87
87
|
```bash
|
|
88
|
-
bunx @lythos/skill-arena@0.16.
|
|
88
|
+
bunx @lythos/skill-arena@0.16.1 archive \
|
|
89
89
|
--from /tmp/arena-side-a \
|
|
90
90
|
--to ./playground/output \
|
|
91
91
|
--sides side-a
|
|
@@ -96,7 +96,7 @@ Copies agent artifacts from workdir(s) to output, skipping internal files (`.cla
|
|
|
96
96
|
### `viz` — render results (WIP — HTML report generation pending)
|
|
97
97
|
|
|
98
98
|
```bash
|
|
99
|
-
bunx @lythos/skill-arena@0.16.
|
|
99
|
+
bunx @lythos/skill-arena@0.16.1 viz runs/arena-<id>/
|
|
100
100
|
```
|
|
101
101
|
|
|
102
102
|
## Parameters
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lythos/skill-arena",
|
|
3
|
-
"version": "0.16.
|
|
3
|
+
"version": "0.16.1",
|
|
4
4
|
"description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-agent",
|
|
@@ -42,15 +42,15 @@
|
|
|
42
42
|
"bun": ">=1.0.0"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@lythos/cold-pool": "^0.16.
|
|
46
|
-
"@lythos/infra": "^0.16.
|
|
47
|
-
"@lythos/test-utils": "^0.16.
|
|
45
|
+
"@lythos/cold-pool": "^0.16.1",
|
|
46
|
+
"@lythos/infra": "^0.16.1",
|
|
47
|
+
"@lythos/test-utils": "^0.16.1",
|
|
48
48
|
"zod": "^3.24.0",
|
|
49
49
|
"zod-to-json-schema": "^3.25.2"
|
|
50
50
|
},
|
|
51
51
|
"optionalDependencies": {
|
|
52
|
-
"@lythos/agent-adapter-claude-sdk": "^0.16.
|
|
53
|
-
"@lythos/agent-adapter-deepseek-serve": "^0.16.
|
|
54
|
-
"@lythos/agent-adapter-codex": "^0.16.
|
|
52
|
+
"@lythos/agent-adapter-claude-sdk": "^0.16.1",
|
|
53
|
+
"@lythos/agent-adapter-deepseek-serve": "^0.16.1",
|
|
54
|
+
"@lythos/agent-adapter-codex": "^0.16.1"
|
|
55
55
|
}
|
|
56
56
|
}
|
package/src/arena-toml.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe,
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
2
|
import { parseArenaToml, buildExecutionPlan, ArenaToml } from './arena-toml'
|
|
3
3
|
import { formatPlanOutput } from './runner'
|
|
4
4
|
|
|
@@ -63,7 +63,7 @@ working_dir = "/workspace"
|
|
|
63
63
|
`
|
|
64
64
|
|
|
65
65
|
describe('parseArenaToml', () => {
|
|
66
|
-
|
|
66
|
+
it('parses minimal two-side arena with criteria', () => {
|
|
67
67
|
const result = parseArenaToml(minimalToml)
|
|
68
68
|
expect(result.arena.task).toBe('Test task')
|
|
69
69
|
expect(result.arena.criteria).toEqual(['a', 'b'])
|
|
@@ -75,14 +75,14 @@ describe('parseArenaToml', () => {
|
|
|
75
75
|
expect(result.side[0].control).toBe(false)
|
|
76
76
|
})
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
it('parses arena with judge field (preferred over criteria)', () => {
|
|
79
79
|
const result = parseArenaToml(judgeToml)
|
|
80
80
|
expect(result.arena.judge).toContain('Evaluate completeness')
|
|
81
81
|
expect(result.arena.criteria).toBeUndefined()
|
|
82
82
|
expect(result.side).toHaveLength(2)
|
|
83
83
|
})
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
it('parses full arena with runs_per_side and control', () => {
|
|
86
86
|
const result = parseArenaToml(fullToml)
|
|
87
87
|
expect(result.arena.runs_per_side).toBe(3)
|
|
88
88
|
expect(result.side).toHaveLength(3)
|
|
@@ -90,7 +90,7 @@ describe('parseArenaToml', () => {
|
|
|
90
90
|
expect(result.side[2].control).toBe(true)
|
|
91
91
|
})
|
|
92
92
|
|
|
93
|
-
|
|
93
|
+
it('parses side env block', () => {
|
|
94
94
|
const result = parseArenaToml(fullToml)
|
|
95
95
|
const env = result.side[2].env
|
|
96
96
|
expect(env.container).toBe('node:20-alpine')
|
|
@@ -99,47 +99,47 @@ describe('parseArenaToml', () => {
|
|
|
99
99
|
expect(env.env_vars).toEqual({})
|
|
100
100
|
})
|
|
101
101
|
|
|
102
|
-
|
|
102
|
+
it('rejects fewer than 2 sides', () => {
|
|
103
103
|
const bad = `[arena]\ntask = "x"\ncriteria = ["a"]\n\n[[side]]\nname = "only"\nplayer = "c"\ndeck = "x.toml"`
|
|
104
104
|
expect(() => parseArenaToml(bad)).toThrow()
|
|
105
105
|
})
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
it('rejects neither judge nor criteria provided', () => {
|
|
108
108
|
const bad = `[arena]\ntask = "x"\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
109
109
|
expect(() => parseArenaToml(bad)).toThrow()
|
|
110
110
|
})
|
|
111
111
|
|
|
112
|
-
|
|
112
|
+
it('accepts judge without criteria (either is sufficient)', () => {
|
|
113
113
|
const toml = `[arena]\ntask = "x"\njudge = "Evaluate this."\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
114
114
|
expect(() => parseArenaToml(toml)).not.toThrow()
|
|
115
115
|
})
|
|
116
116
|
|
|
117
|
-
|
|
117
|
+
it('rejects empty criteria and no judge', () => {
|
|
118
118
|
const bad = `[arena]\ntask = "x"\ncriteria = []\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
119
119
|
expect(() => parseArenaToml(bad)).toThrow()
|
|
120
120
|
})
|
|
121
121
|
|
|
122
|
-
|
|
122
|
+
it('rejects non-object input', () => {
|
|
123
123
|
expect(() => ArenaToml.parse('not valid')).toThrow()
|
|
124
124
|
})
|
|
125
125
|
|
|
126
|
-
|
|
126
|
+
it('rejects missing arena section', () => {
|
|
127
127
|
expect(() => parseArenaToml('[[side]]\nname = "a"')).toThrow()
|
|
128
128
|
})
|
|
129
129
|
|
|
130
|
-
|
|
130
|
+
it('rejects runs_per_side = 0', () => {
|
|
131
131
|
const bad = `[arena]\ntask = "x"\ncriteria = ["a"]\nruns_per_side = 0\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
132
132
|
expect(() => parseArenaToml(bad)).toThrow()
|
|
133
133
|
})
|
|
134
134
|
|
|
135
|
-
|
|
135
|
+
it('parses integer and boolean values correctly', () => {
|
|
136
136
|
const toml = `[arena]\ntask = "x"\ncriteria = ["a"]\nruns_per_side = 2\nmax_participants = 5\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
137
137
|
const result = parseArenaToml(toml)
|
|
138
138
|
expect(result.arena.runs_per_side).toBe(2)
|
|
139
139
|
expect(result.arena.max_participants).toBe(5)
|
|
140
140
|
})
|
|
141
141
|
|
|
142
|
-
|
|
142
|
+
it('comments are stripped', () => {
|
|
143
143
|
const toml = `[arena]\n# this is a comment\ntask = "x"\ncriteria = ["a"]\n\n[[side]]\nname = "a"\nplayer = "c"\ndeck = "a.toml"\n\n[[side]]\nname = "b"\nplayer = "c"\ndeck = "b.toml"`
|
|
144
144
|
const result = parseArenaToml(toml)
|
|
145
145
|
expect(result.arena.task).toBe('x')
|
|
@@ -147,7 +147,7 @@ describe('parseArenaToml', () => {
|
|
|
147
147
|
})
|
|
148
148
|
|
|
149
149
|
describe('buildExecutionPlan', () => {
|
|
150
|
-
|
|
150
|
+
it('generates plan: 2 sides × 1 run = 2 cells', () => {
|
|
151
151
|
const toml = parseArenaToml(minimalToml)
|
|
152
152
|
const plan = buildExecutionPlan(toml)
|
|
153
153
|
expect(plan.task).toBe('Test task')
|
|
@@ -156,20 +156,20 @@ describe('buildExecutionPlan', () => {
|
|
|
156
156
|
expect(plan.total_runs).toBe(2)
|
|
157
157
|
})
|
|
158
158
|
|
|
159
|
-
|
|
159
|
+
it('generates plan with judge field populated', () => {
|
|
160
160
|
const toml = parseArenaToml(judgeToml)
|
|
161
161
|
const plan = buildExecutionPlan(toml)
|
|
162
162
|
expect(plan.judge).toContain('Evaluate completeness')
|
|
163
163
|
})
|
|
164
164
|
|
|
165
|
-
|
|
165
|
+
it('generates plan: 3 sides × 3 runs = 9 cells', () => {
|
|
166
166
|
const toml = parseArenaToml(fullToml)
|
|
167
167
|
const plan = buildExecutionPlan(toml)
|
|
168
168
|
expect(plan.cells).toHaveLength(9)
|
|
169
169
|
expect(plan.total_runs).toBe(9)
|
|
170
170
|
})
|
|
171
171
|
|
|
172
|
-
|
|
172
|
+
it('control flag preserved in plan cells', () => {
|
|
173
173
|
const toml = parseArenaToml(fullToml)
|
|
174
174
|
const plan = buildExecutionPlan(toml)
|
|
175
175
|
const baselineCells = plan.cells.filter(c => c.side === 'baseline')
|
|
@@ -177,7 +177,7 @@ describe('buildExecutionPlan', () => {
|
|
|
177
177
|
expect(baselineCells.every(c => c.control)).toBe(true)
|
|
178
178
|
})
|
|
179
179
|
|
|
180
|
-
|
|
180
|
+
it('dry-run: plan is pure data, no side effects', () => {
|
|
181
181
|
const toml = parseArenaToml(fullToml)
|
|
182
182
|
const plan = buildExecutionPlan(toml)
|
|
183
183
|
expect(plan.total_runs).toBeGreaterThan(0)
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { describe,
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
2
|
import { computePareto, buildComparativePrompt, toScoreMatrix, normalizeComparativeOutput } from './comparative-judge'
|
|
3
3
|
import { ArenaManifest, CriterionDef, ComparativeReport } from '@lythos/test-utils/schema'
|
|
4
4
|
import type { ArenaManifest as ArenaManifestType } from '@lythos/test-utils/schema'
|
|
5
5
|
|
|
6
6
|
describe('computePareto', () => {
|
|
7
|
-
|
|
7
|
+
it('single participant is always non-dominated', () => {
|
|
8
8
|
const result = computePareto([
|
|
9
9
|
{ participant_id: 'run-01', scores: { a: 5, b: 3 } },
|
|
10
10
|
])
|
|
@@ -13,7 +13,7 @@ describe('computePareto', () => {
|
|
|
13
13
|
expect(result[0].dominated_by).toEqual([])
|
|
14
14
|
})
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
it('clear dominance: run-01 dominates run-02 on all criteria', () => {
|
|
17
17
|
const result = computePareto([
|
|
18
18
|
{ participant_id: 'run-01', scores: { coverage: 5, relevance: 5 } },
|
|
19
19
|
{ participant_id: 'run-02', scores: { coverage: 3, relevance: 2 } },
|
|
@@ -23,7 +23,7 @@ describe('computePareto', () => {
|
|
|
23
23
|
expect(result[1].dominated_by).toEqual(['run-01'])
|
|
24
24
|
})
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
it('equal scores: no one dominates', () => {
|
|
27
27
|
const result = computePareto([
|
|
28
28
|
{ participant_id: 'run-01', scores: { a: 4, b: 4 } },
|
|
29
29
|
{ participant_id: 'run-02', scores: { a: 4, b: 4 } },
|
|
@@ -32,7 +32,7 @@ describe('computePareto', () => {
|
|
|
32
32
|
expect(result[1].dominated).toBe(false)
|
|
33
33
|
})
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
it('cross dominance: each wins on different criteria', () => {
|
|
36
36
|
const result = computePareto([
|
|
37
37
|
{ participant_id: 'run-01', scores: { speed: 5, accuracy: 2 } },
|
|
38
38
|
{ participant_id: 'run-02', scores: { speed: 2, accuracy: 5 } },
|
|
@@ -42,7 +42,7 @@ describe('computePareto', () => {
|
|
|
42
42
|
expect(result[1].dominated).toBe(false)
|
|
43
43
|
})
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
it('multi-participant: transitive dominance chain', () => {
|
|
46
46
|
const result = computePareto([
|
|
47
47
|
{ participant_id: 'best', scores: { a: 5, b: 5, c: 5 } },
|
|
48
48
|
{ participant_id: 'mid', scores: { a: 4, b: 4, c: 4 } },
|
|
@@ -56,7 +56,7 @@ describe('computePareto', () => {
|
|
|
56
56
|
expect(result[2].dominated_by.sort()).toEqual(['best', 'mid'].sort())
|
|
57
57
|
})
|
|
58
58
|
|
|
59
|
-
|
|
59
|
+
it('Pareto frontier from playground BDD-research: run-01 dominates run-02', () => {
|
|
60
60
|
// From playground/arena-bdd-research/report.md:
|
|
61
61
|
// Run-01: coverage=5, relevance=5, actionability=5, depth=5
|
|
62
62
|
// Run-02: coverage=3, relevance=2, actionability=2, depth=1
|
|
@@ -69,7 +69,7 @@ describe('computePareto', () => {
|
|
|
69
69
|
expect(result[1].dominated_by).toEqual(['run-01'])
|
|
70
70
|
})
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
it('empty scores object', () => {
|
|
73
73
|
const result = computePareto([
|
|
74
74
|
{ participant_id: 'a', scores: {} },
|
|
75
75
|
{ participant_id: 'b', scores: {} },
|
|
@@ -79,7 +79,7 @@ describe('computePareto', () => {
|
|
|
79
79
|
expect(result[1].dominated).toBe(false)
|
|
80
80
|
})
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
it('partial criteria overlap', () => {
|
|
83
83
|
const result = computePareto([
|
|
84
84
|
{ participant_id: 'run-01', scores: { a: 5, b: 3 } },
|
|
85
85
|
{ participant_id: 'run-02', scores: { a: 3, c: 5 } },
|
|
@@ -103,7 +103,7 @@ const manifestFixture: ArenaManifest = {
|
|
|
103
103
|
}
|
|
104
104
|
|
|
105
105
|
describe('buildComparativePrompt', () => {
|
|
106
|
-
|
|
106
|
+
it('includes task description', () => {
|
|
107
107
|
const prompt = buildComparativePrompt({
|
|
108
108
|
manifest: manifestFixture,
|
|
109
109
|
verdicts: [],
|
|
@@ -111,7 +111,7 @@ describe('buildComparativePrompt', () => {
|
|
|
111
111
|
expect(prompt).toContain('Write a function that adds two numbers')
|
|
112
112
|
})
|
|
113
113
|
|
|
114
|
-
|
|
114
|
+
it('includes all participants', () => {
|
|
115
115
|
const prompt = buildComparativePrompt({
|
|
116
116
|
manifest: manifestFixture,
|
|
117
117
|
verdicts: [],
|
|
@@ -122,7 +122,7 @@ describe('buildComparativePrompt', () => {
|
|
|
122
122
|
expect(prompt).toContain('Full test discipline')
|
|
123
123
|
})
|
|
124
124
|
|
|
125
|
-
|
|
125
|
+
it('includes criteria list', () => {
|
|
126
126
|
const prompt = buildComparativePrompt({
|
|
127
127
|
manifest: manifestFixture,
|
|
128
128
|
verdicts: [],
|
|
@@ -131,7 +131,7 @@ describe('buildComparativePrompt', () => {
|
|
|
131
131
|
expect(prompt).toContain('efficiency')
|
|
132
132
|
})
|
|
133
133
|
|
|
134
|
-
|
|
134
|
+
it('includes Zod schema in output spec', () => {
|
|
135
135
|
const prompt = buildComparativePrompt({
|
|
136
136
|
manifest: manifestFixture,
|
|
137
137
|
verdicts: [],
|
|
@@ -145,7 +145,7 @@ describe('buildComparativePrompt', () => {
|
|
|
145
145
|
// ── toScoreMatrix (pure Zod validation wrapper) ──────────────────────
|
|
146
146
|
|
|
147
147
|
describe('toScoreMatrix', () => {
|
|
148
|
-
|
|
148
|
+
it('passes through valid score cells', () => {
|
|
149
149
|
const result = toScoreMatrix(manifestFixture, [
|
|
150
150
|
{ participant_id: 'bare', criterion: 'correctness', weight: 0.5, score: 4, rationale: 'works' },
|
|
151
151
|
{ participant_id: 'bare', criterion: 'efficiency', weight: 0.5, score: 3, rationale: 'ok' },
|
|
@@ -166,7 +166,7 @@ const sampleScoreMatrix = [
|
|
|
166
166
|
]
|
|
167
167
|
|
|
168
168
|
describe('normalizeComparativeOutput', () => {
|
|
169
|
-
|
|
169
|
+
it('passes through already-correct format', () => {
|
|
170
170
|
const input = {
|
|
171
171
|
score_matrix: sampleScoreMatrix,
|
|
172
172
|
key_findings: ['TDD produced cleaner code'],
|
|
@@ -177,7 +177,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
177
177
|
expect((result.score_matrix as any[])[0].participant_id).toBe('bare')
|
|
178
178
|
})
|
|
179
179
|
|
|
180
|
-
|
|
180
|
+
it('maps participantId to participant_id', () => {
|
|
181
181
|
const input: Record<string, unknown> = {
|
|
182
182
|
score_matrix: [
|
|
183
183
|
{ participantId: 'bare', criterion: 'accuracy', weight: 0.5, score: 4, rationale: 'good' },
|
|
@@ -189,7 +189,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
189
189
|
expect((result.score_matrix as any[])[0].participant_id).toBe('bare')
|
|
190
190
|
})
|
|
191
191
|
|
|
192
|
-
|
|
192
|
+
it('maps side to participant_id', () => {
|
|
193
193
|
const input: Record<string, unknown> = {
|
|
194
194
|
score_matrix: [
|
|
195
195
|
{ side: 'tdd', criterion: 'quality', weight: 0.5, score: 5, rationale: 'excellent' },
|
|
@@ -201,7 +201,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
201
201
|
expect((result.score_matrix as any[])[0].participant_id).toBe('tdd')
|
|
202
202
|
})
|
|
203
203
|
|
|
204
|
-
|
|
204
|
+
it('coerces string score to number', () => {
|
|
205
205
|
const input: Record<string, unknown> = {
|
|
206
206
|
score_matrix: [
|
|
207
207
|
{ participant_id: 'bare', criterion: 'a', weight: 0.5, score: '4', rationale: 'ok' },
|
|
@@ -213,7 +213,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
213
213
|
expect((result.score_matrix as any[])[0].score).toBe(4)
|
|
214
214
|
})
|
|
215
215
|
|
|
216
|
-
|
|
216
|
+
it('normalizes weight >1 as percentage', () => {
|
|
217
217
|
const input: Record<string, unknown> = {
|
|
218
218
|
score_matrix: [
|
|
219
219
|
{ participant_id: 'bare', criterion: 'a', weight: 50, score: 4, rationale: 'ok' },
|
|
@@ -225,7 +225,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
225
225
|
expect((result.score_matrix as any[])[0].weight).toBe(0.5)
|
|
226
226
|
})
|
|
227
227
|
|
|
228
|
-
|
|
228
|
+
it('defaults weight to 0.25 when undefined', () => {
|
|
229
229
|
const input: Record<string, unknown> = {
|
|
230
230
|
score_matrix: [
|
|
231
231
|
{ participant_id: 'bare', criterion: 'a', score: 4, rationale: 'ok' },
|
|
@@ -237,7 +237,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
237
237
|
expect((result.score_matrix as any[])[0].weight).toBe(0.25)
|
|
238
238
|
})
|
|
239
239
|
|
|
240
|
-
|
|
240
|
+
it('maps reason to rationale', () => {
|
|
241
241
|
const input: Record<string, unknown> = {
|
|
242
242
|
score_matrix: [
|
|
243
243
|
{ participant_id: 'bare', criterion: 'a', weight: 0.5, score: 4, reason: 'looks fine' },
|
|
@@ -249,7 +249,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
249
249
|
expect((result.score_matrix as any[])[0].rationale).toBe('looks fine')
|
|
250
250
|
})
|
|
251
251
|
|
|
252
|
-
|
|
252
|
+
it('maps explanation to rationale', () => {
|
|
253
253
|
const input: Record<string, unknown> = {
|
|
254
254
|
score_matrix: [
|
|
255
255
|
{ participant_id: 'bare', criterion: 'a', weight: 0.5, score: 4, explanation: 'works' },
|
|
@@ -261,7 +261,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
261
261
|
expect((result.score_matrix as any[])[0].rationale).toBe('works')
|
|
262
262
|
})
|
|
263
263
|
|
|
264
|
-
|
|
264
|
+
it('normalizes recommendations with role fallback', () => {
|
|
265
265
|
const input: Record<string, unknown> = {
|
|
266
266
|
score_matrix: [],
|
|
267
267
|
key_findings: [],
|
|
@@ -275,7 +275,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
275
275
|
expect(recs[0].recommendation).toBe('Use TDD')
|
|
276
276
|
})
|
|
277
277
|
|
|
278
|
-
|
|
278
|
+
it('normalizes recommendations with advice fallback', () => {
|
|
279
279
|
const input: Record<string, unknown> = {
|
|
280
280
|
score_matrix: [],
|
|
281
281
|
key_findings: [],
|
|
@@ -287,7 +287,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
287
287
|
expect((result.recommendations as any[])[0].recommendation).toBe('Consider refactoring')
|
|
288
288
|
})
|
|
289
289
|
|
|
290
|
-
|
|
290
|
+
it('handles empty key_findings', () => {
|
|
291
291
|
const input: Record<string, unknown> = {
|
|
292
292
|
score_matrix: [],
|
|
293
293
|
}
|
|
@@ -295,7 +295,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
295
295
|
expect(result.key_findings).toEqual([])
|
|
296
296
|
})
|
|
297
297
|
|
|
298
|
-
|
|
298
|
+
it('converts pivot-table format: { participant: { criterion: score } }', () => {
|
|
299
299
|
const input: Record<string, unknown> = {
|
|
300
300
|
bare: { correctness: 4, correctness_rationale: 'works', efficiency: 3, efficiency_rationale: 'ok' },
|
|
301
301
|
tdd: { correctness: 5, correctness_rationale: 'tests', efficiency: 4, efficiency_rationale: 'clean' },
|
|
@@ -310,7 +310,7 @@ describe('normalizeComparativeOutput', () => {
|
|
|
310
310
|
expect(bareCorrectness.weight).toBe(0.25)
|
|
311
311
|
})
|
|
312
312
|
|
|
313
|
-
|
|
313
|
+
it('clamps score to 1-5 range', () => {
|
|
314
314
|
const input: Record<string, unknown> = {
|
|
315
315
|
score_matrix: [
|
|
316
316
|
{ participant_id: 'bare', criterion: 'a', weight: 0.5, score: 0, rationale: 'terrible' },
|
|
@@ -367,7 +367,7 @@ const manifestWithRubrics: ArenaManifestType = {
|
|
|
367
367
|
}
|
|
368
368
|
|
|
369
369
|
describe('buildComparativePrompt with structured criteria', () => {
|
|
370
|
-
|
|
370
|
+
it('injects rubric anchors into prompt', () => {
|
|
371
371
|
const prompt = buildComparativePrompt({ manifest: manifestWithRubrics, verdicts: [] })
|
|
372
372
|
expect(prompt).toContain('信息准确性')
|
|
373
373
|
expect(prompt).toContain('Evaluator: ISTJ测试员')
|
|
@@ -377,7 +377,7 @@ describe('buildComparativePrompt with structured criteria', () => {
|
|
|
377
377
|
expect(prompt).toContain('Evaluator: INTJ架构师')
|
|
378
378
|
})
|
|
379
379
|
|
|
380
|
-
|
|
380
|
+
it('falls back to bare format for string criteria', () => {
|
|
381
381
|
const manifest: ArenaManifestType = {
|
|
382
382
|
id: 'test', created_at: '2026-01-01T00:00:00Z', task: 'test', mode: 'decks',
|
|
383
383
|
participants: [{ id: 'a', name: 'A', deck: 'd1' }, { id: 'b', name: 'B', deck: 'd2' }],
|
|
@@ -393,7 +393,7 @@ describe('buildComparativePrompt with structured criteria', () => {
|
|
|
393
393
|
// Simulate a realistic LLM judge output — the kind of JSON an actual Claude
|
|
394
394
|
// comparative judge call would produce. Verify our normalization handles it.
|
|
395
395
|
describe('full pipeline: mock LLM output → schema validation', () => {
|
|
396
|
-
|
|
396
|
+
it('clean score_matrix passes through ComparativeReport.parse', () => {
|
|
397
397
|
const cleanOutput = {
|
|
398
398
|
score_matrix: [
|
|
399
399
|
{ participant_id: 'bare', criterion: 'accuracy', weight: 0.4, score: 3, rationale: 'Correct on Bun version but missed pnpm migration detail' },
|
|
@@ -419,7 +419,7 @@ describe('full pipeline: mock LLM output → schema validation', () => {
|
|
|
419
419
|
expect(deepAccuracy!.score).toBe(5)
|
|
420
420
|
})
|
|
421
421
|
|
|
422
|
-
|
|
422
|
+
it('messy LLM output with field name variants gets normalized', () => {
|
|
423
423
|
// Simulates a messy Claude output — participantId instead of participant_id,
|
|
424
424
|
// reason instead of rationale, string score
|
|
425
425
|
const messyLLMOutput = {
|
package/src/player.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe,
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
2
|
import { resolvePlayer, resolveSides, groupBySide, totalRuns } from './player'
|
|
3
3
|
import { parseArenaToml } from './arena-toml'
|
|
4
4
|
|
|
@@ -20,29 +20,29 @@ deck = "./decks/rich.toml"
|
|
|
20
20
|
`)
|
|
21
21
|
|
|
22
22
|
describe('resolvePlayer', () => {
|
|
23
|
-
|
|
23
|
+
it('maps claude-code → claude', () => {
|
|
24
24
|
expect(resolvePlayer('claude-code')).toBe('claude-sdk')
|
|
25
25
|
})
|
|
26
26
|
|
|
27
|
-
|
|
27
|
+
it('maps Claude → claude (case insensitive)', () => {
|
|
28
28
|
expect(resolvePlayer('Claude')).toBe('claude-sdk')
|
|
29
29
|
})
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
it('maps kimi → kimi', () => {
|
|
32
32
|
expect(resolvePlayer('kimi')).toBe('kimi')
|
|
33
33
|
})
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
it('passes through unknown player names', () => {
|
|
36
36
|
expect(resolvePlayer('expert-architect')).toBe('expert-architect')
|
|
37
37
|
})
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
it('trims whitespace', () => {
|
|
40
40
|
expect(resolvePlayer(' claude-code ')).toBe('claude-sdk')
|
|
41
41
|
})
|
|
42
42
|
})
|
|
43
43
|
|
|
44
44
|
describe('resolveSides', () => {
|
|
45
|
-
|
|
45
|
+
it('resolves all sides in arena.toml', () => {
|
|
46
46
|
const sides = resolveSides(toml)
|
|
47
47
|
expect(sides).toHaveLength(2)
|
|
48
48
|
expect(sides[0].platform).toBe('claude-sdk')
|
|
@@ -50,7 +50,7 @@ describe('resolveSides', () => {
|
|
|
50
50
|
expect(sides[0].playerName).toBe('claude-code')
|
|
51
51
|
})
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
it('preserves side config', () => {
|
|
54
54
|
const sides = resolveSides(toml)
|
|
55
55
|
expect(sides[0].side.name).toBe('minimal')
|
|
56
56
|
expect(sides[0].side.deck).toBe('./decks/minimal.toml')
|
|
@@ -58,7 +58,7 @@ describe('resolveSides', () => {
|
|
|
58
58
|
})
|
|
59
59
|
|
|
60
60
|
describe('groupBySide', () => {
|
|
61
|
-
|
|
61
|
+
it('groups by side name with run count', () => {
|
|
62
62
|
const groups = groupBySide(toml)
|
|
63
63
|
expect(groups).toHaveLength(2)
|
|
64
64
|
expect(groups[0].runs).toBe(3) // runs_per_side
|
|
@@ -66,7 +66,7 @@ describe('groupBySide', () => {
|
|
|
66
66
|
expect(groups[0].platform).toBe('claude-sdk')
|
|
67
67
|
})
|
|
68
68
|
|
|
69
|
-
|
|
69
|
+
it('control flag preserved', () => {
|
|
70
70
|
const controlToml = parseArenaToml(`
|
|
71
71
|
[arena]
|
|
72
72
|
task = "x"
|
|
@@ -89,7 +89,7 @@ control = true
|
|
|
89
89
|
})
|
|
90
90
|
|
|
91
91
|
describe('totalRuns', () => {
|
|
92
|
-
|
|
92
|
+
it('calculates sides × runs_per_side', () => {
|
|
93
93
|
expect(totalRuns(toml)).toBe(6) // 2 sides × 3 runs
|
|
94
94
|
})
|
|
95
95
|
})
|
package/src/preflight.test.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* formatSkillWarnings — warning string generation
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import { describe,
|
|
13
|
+
import { describe, it, expect } from 'bun:test'
|
|
14
14
|
import {
|
|
15
15
|
parseDeckSkills,
|
|
16
16
|
checkSkillExistence,
|
|
@@ -26,15 +26,15 @@ import {
|
|
|
26
26
|
|
|
27
27
|
describe('parseDeckSkills', () => {
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
it('empty deck → empty array', () => {
|
|
30
30
|
expect(parseDeckSkills({})).toEqual([])
|
|
31
31
|
})
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
it('deck with no skill sections → empty array', () => {
|
|
34
34
|
expect(parseDeckSkills({ deck: { max_cards: 10 } })).toEqual([])
|
|
35
35
|
})
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
it('inline-table format: single tool skill with path', () => {
|
|
38
38
|
const parsed = {
|
|
39
39
|
tool: {
|
|
40
40
|
skills: {
|
|
@@ -47,7 +47,7 @@ describe('parseDeckSkills', () => {
|
|
|
47
47
|
])
|
|
48
48
|
})
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
it('inline-table format: multiple skills', () => {
|
|
51
51
|
const parsed = {
|
|
52
52
|
tool: {
|
|
53
53
|
skills: {
|
|
@@ -62,7 +62,7 @@ describe('parseDeckSkills', () => {
|
|
|
62
62
|
])
|
|
63
63
|
})
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
it('array format: skills = ["a", "b"]', () => {
|
|
66
66
|
const parsed = {
|
|
67
67
|
tool: {
|
|
68
68
|
skills: ['web-search', 'docx']
|
|
@@ -74,7 +74,7 @@ describe('parseDeckSkills', () => {
|
|
|
74
74
|
])
|
|
75
75
|
})
|
|
76
76
|
|
|
77
|
-
|
|
77
|
+
it('innate section parsed separately', () => {
|
|
78
78
|
const parsed = {
|
|
79
79
|
innate: {
|
|
80
80
|
skills: {
|
|
@@ -93,7 +93,7 @@ describe('parseDeckSkills', () => {
|
|
|
93
93
|
])
|
|
94
94
|
})
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
it('transient section parsed', () => {
|
|
97
97
|
const parsed = {
|
|
98
98
|
transient: {
|
|
99
99
|
skills: {
|
|
@@ -106,7 +106,7 @@ describe('parseDeckSkills', () => {
|
|
|
106
106
|
])
|
|
107
107
|
})
|
|
108
108
|
|
|
109
|
-
|
|
109
|
+
it('object entry without path → path=null', () => {
|
|
110
110
|
const parsed = {
|
|
111
111
|
tool: {
|
|
112
112
|
skills: {
|
|
@@ -119,7 +119,7 @@ describe('parseDeckSkills', () => {
|
|
|
119
119
|
])
|
|
120
120
|
})
|
|
121
121
|
|
|
122
|
-
|
|
122
|
+
it('object entry with non-string path → path=null', () => {
|
|
123
123
|
const parsed = {
|
|
124
124
|
tool: {
|
|
125
125
|
skills: {
|
|
@@ -132,7 +132,7 @@ describe('parseDeckSkills', () => {
|
|
|
132
132
|
])
|
|
133
133
|
})
|
|
134
134
|
|
|
135
|
-
|
|
135
|
+
it('array entry that is not a string → skipped', () => {
|
|
136
136
|
const parsed = {
|
|
137
137
|
tool: { skills: ['valid', 123, null, 'also-valid'] }
|
|
138
138
|
}
|
|
@@ -142,7 +142,7 @@ describe('parseDeckSkills', () => {
|
|
|
142
142
|
])
|
|
143
143
|
})
|
|
144
144
|
|
|
145
|
-
|
|
145
|
+
it('all three sections populated → ordered innate, tool, transient', () => {
|
|
146
146
|
const parsed = {
|
|
147
147
|
innate: { skills: { a: { path: '/a' } } },
|
|
148
148
|
tool: { skills: { b: { path: '/b' } } },
|
|
@@ -162,12 +162,12 @@ describe('parseDeckSkills', () => {
|
|
|
162
162
|
|
|
163
163
|
describe('checkSkillExistence', () => {
|
|
164
164
|
|
|
165
|
-
|
|
165
|
+
it('empty skills → empty array', () => {
|
|
166
166
|
const exists = (_: string) => true
|
|
167
167
|
expect(checkSkillExistence([], '/cold', exists)).toEqual([])
|
|
168
168
|
})
|
|
169
169
|
|
|
170
|
-
|
|
170
|
+
it('skill with explicit path → resolves <coldPool>/<path>/SKILL.md', () => {
|
|
171
171
|
const exists = (p: string) => p === '/cold/github.com/owner/repo/skills/my-skill/SKILL.md'
|
|
172
172
|
const skills = [{ name: 'my-skill', path: 'github.com/owner/repo/skills/my-skill', section: 'tool' }]
|
|
173
173
|
const result = checkSkillExistence(skills, '/cold', exists)
|
|
@@ -176,7 +176,7 @@ describe('checkSkillExistence', () => {
|
|
|
176
176
|
])
|
|
177
177
|
})
|
|
178
178
|
|
|
179
|
-
|
|
179
|
+
it('skill without path (array format) → resolves <coldPool>/<name>/SKILL.md', () => {
|
|
180
180
|
const exists = (p: string) => p === '/cold/web-search/SKILL.md'
|
|
181
181
|
const skills = [{ name: 'web-search', path: null, section: 'tool' }]
|
|
182
182
|
const result = checkSkillExistence(skills, '/cold', exists)
|
|
@@ -185,7 +185,7 @@ describe('checkSkillExistence', () => {
|
|
|
185
185
|
])
|
|
186
186
|
})
|
|
187
187
|
|
|
188
|
-
|
|
188
|
+
it('HTTP path → uses name as fallback for path resolution', () => {
|
|
189
189
|
const exists = (p: string) => p === '/cold/my-skill/SKILL.md'
|
|
190
190
|
const skills = [{ name: 'my-skill', path: 'https://example.com/deck.toml', section: 'tool' }]
|
|
191
191
|
const result = checkSkillExistence(skills, '/cold', exists)
|
|
@@ -194,7 +194,7 @@ describe('checkSkillExistence', () => {
|
|
|
194
194
|
])
|
|
195
195
|
})
|
|
196
196
|
|
|
197
|
-
|
|
197
|
+
it('all found → all found=true', () => {
|
|
198
198
|
const exists = (_: string) => true
|
|
199
199
|
const skills = [
|
|
200
200
|
{ name: 'a', path: '/a', section: 'tool' },
|
|
@@ -206,7 +206,7 @@ describe('checkSkillExistence', () => {
|
|
|
206
206
|
])
|
|
207
207
|
})
|
|
208
208
|
|
|
209
|
-
|
|
209
|
+
it('some missing → mixed found/not-found', () => {
|
|
210
210
|
const exists = (p: string) => p.includes('a')
|
|
211
211
|
const skills = [
|
|
212
212
|
{ name: 'a', path: '/a', section: 'tool' },
|
|
@@ -218,7 +218,7 @@ describe('checkSkillExistence', () => {
|
|
|
218
218
|
])
|
|
219
219
|
})
|
|
220
220
|
|
|
221
|
-
|
|
221
|
+
it('different coldPoolDir → different expectedPath prefix', () => {
|
|
222
222
|
const exists = (_: string) => true
|
|
223
223
|
const skills = [{ name: 'x', path: 'github.com/x', section: 'tool' }]
|
|
224
224
|
const a = checkSkillExistence(skills, '/home/user/.agents/skill-repos', exists)
|
|
@@ -234,35 +234,35 @@ describe('checkSkillExistence', () => {
|
|
|
234
234
|
|
|
235
235
|
describe('validateLinkResult', () => {
|
|
236
236
|
|
|
237
|
-
|
|
237
|
+
it('exitCode 0 → ok', () => {
|
|
238
238
|
expect(validateLinkResult(0, '')).toEqual({ ok: true })
|
|
239
239
|
})
|
|
240
240
|
|
|
241
|
-
|
|
241
|
+
it('exitCode 0 with stderr → still ok (stderr is not always errors)', () => {
|
|
242
242
|
expect(validateLinkResult(0, 'some warning output')).toEqual({ ok: true })
|
|
243
243
|
})
|
|
244
244
|
|
|
245
|
-
|
|
245
|
+
it('exitCode 1 → not ok, error contains snippet', () => {
|
|
246
246
|
const result = validateLinkResult(1, 'something went wrong')
|
|
247
247
|
expect(result.ok).toBe(false)
|
|
248
248
|
expect(result.error).toContain('exit 1')
|
|
249
249
|
expect(result.error).toContain('something went wrong')
|
|
250
250
|
})
|
|
251
251
|
|
|
252
|
-
|
|
252
|
+
it('exitCode null → not ok (null !== 0)', () => {
|
|
253
253
|
const result = validateLinkResult(null, 'process killed')
|
|
254
254
|
expect(result.ok).toBe(false)
|
|
255
255
|
expect(result.error).toContain('exit null')
|
|
256
256
|
})
|
|
257
257
|
|
|
258
|
-
|
|
258
|
+
it('stderr truncated to 300 chars in error message', () => {
|
|
259
259
|
const longStderr = 'x'.repeat(500)
|
|
260
260
|
const result = validateLinkResult(1, longStderr)
|
|
261
261
|
expect(result.ok).toBe(false)
|
|
262
262
|
expect(result.error!.length).toBeLessThan(350) // "Deck link failed (exit 1): " + 300 chars
|
|
263
263
|
})
|
|
264
264
|
|
|
265
|
-
|
|
265
|
+
it('exitCode 0, empty stderr → ok with no error field', () => {
|
|
266
266
|
const result = validateLinkResult(0, '')
|
|
267
267
|
expect(result.ok).toBe(true)
|
|
268
268
|
expect(result.error).toBeUndefined()
|
|
@@ -275,16 +275,16 @@ describe('validateLinkResult', () => {
|
|
|
275
275
|
|
|
276
276
|
describe('buildCopyPlan', () => {
|
|
277
277
|
|
|
278
|
-
|
|
278
|
+
it('empty entries → empty plan', () => {
|
|
279
279
|
expect(buildCopyPlan('/work', '/out', [], new Set())).toEqual([])
|
|
280
280
|
})
|
|
281
281
|
|
|
282
|
-
|
|
282
|
+
it('all skipped → empty plan', () => {
|
|
283
283
|
const skip = new Set(['.claude', 'skill-deck.toml'])
|
|
284
284
|
expect(buildCopyPlan('/work', '/out', ['.claude', 'skill-deck.toml'], skip)).toEqual([])
|
|
285
285
|
})
|
|
286
286
|
|
|
287
|
-
|
|
287
|
+
it('normal entries → mapped to outDir', () => {
|
|
288
288
|
const skip = new Set<string>()
|
|
289
289
|
expect(buildCopyPlan('/work', '/out', ['output.md', 'report.docx'], skip)).toEqual([
|
|
290
290
|
{ src: '/work/output.md', dest: '/out/output.md', name: 'output.md' },
|
|
@@ -292,7 +292,7 @@ describe('buildCopyPlan', () => {
|
|
|
292
292
|
])
|
|
293
293
|
})
|
|
294
294
|
|
|
295
|
-
|
|
295
|
+
it('mixed skip and non-skip → only non-skipped', () => {
|
|
296
296
|
const skip = new Set(['.claude', 'skill-deck.toml', 'skill-deck.lock'])
|
|
297
297
|
const entries = ['.claude', 'output.md', 'skill-deck.toml', 'report.docx', 'skill-deck.lock']
|
|
298
298
|
expect(buildCopyPlan('/work', '/out', entries, skip)).toEqual([
|
|
@@ -301,13 +301,13 @@ describe('buildCopyPlan', () => {
|
|
|
301
301
|
])
|
|
302
302
|
})
|
|
303
303
|
|
|
304
|
-
|
|
304
|
+
it('preserves entry order', () => {
|
|
305
305
|
const skip = new Set<string>()
|
|
306
306
|
const entries = ['c', 'a', 'b']
|
|
307
307
|
expect(buildCopyPlan('/w', '/o', entries, skip).map(e => e.name)).toEqual(['c', 'a', 'b'])
|
|
308
308
|
})
|
|
309
309
|
|
|
310
|
-
|
|
310
|
+
it('nested paths work (agent-produced subdirectories)', () => {
|
|
311
311
|
const skip = new Set<string>()
|
|
312
312
|
expect(buildCopyPlan('/work', '/out', ['subdir/output.pdf'], skip)).toEqual([
|
|
313
313
|
{ src: '/work/subdir/output.pdf', dest: '/out/subdir/output.pdf', name: 'subdir/output.pdf' },
|
|
@@ -321,30 +321,30 @@ describe('buildCopyPlan', () => {
|
|
|
321
321
|
|
|
322
322
|
describe('resolveColdPoolDir', () => {
|
|
323
323
|
|
|
324
|
-
|
|
324
|
+
it('explicit absolute path → returned as-is', () => {
|
|
325
325
|
expect(resolveColdPoolDir('/opt/cold', '/home/user', '/fallback')).toBe('/opt/cold')
|
|
326
326
|
})
|
|
327
327
|
|
|
328
|
-
|
|
328
|
+
it('explicit relative path → returned as-is', () => {
|
|
329
329
|
expect(resolveColdPoolDir('my-cold-pool', '/home/user', '/fallback')).toBe('my-cold-pool')
|
|
330
330
|
})
|
|
331
331
|
|
|
332
|
-
|
|
332
|
+
it('tilde path → expanded with homeDir', () => {
|
|
333
333
|
expect(resolveColdPoolDir('~/.agents/skill-repos', '/home/user', '/fallback'))
|
|
334
334
|
.toBe('/home/user/.agents/skill-repos')
|
|
335
335
|
})
|
|
336
336
|
|
|
337
|
-
|
|
337
|
+
it('tilde at start only → expanded; tilde elsewhere not expanded', () => {
|
|
338
338
|
expect(resolveColdPoolDir('path/with~/tilde', '/home/user', '/fallback'))
|
|
339
339
|
.toBe('path/with~/tilde')
|
|
340
340
|
})
|
|
341
341
|
|
|
342
|
-
|
|
342
|
+
it('undefined → uses fallback', () => {
|
|
343
343
|
expect(resolveColdPoolDir(undefined, '/home/user', '/default/cold'))
|
|
344
344
|
.toBe('/default/cold')
|
|
345
345
|
})
|
|
346
346
|
|
|
347
|
-
|
|
347
|
+
it('empty string → uses fallback (|| operator)', () => {
|
|
348
348
|
expect(resolveColdPoolDir('', '/home/user', '/default/cold'))
|
|
349
349
|
.toBe('/default/cold')
|
|
350
350
|
})
|
|
@@ -356,7 +356,7 @@ describe('resolveColdPoolDir', () => {
|
|
|
356
356
|
|
|
357
357
|
describe('formatSkillWarnings', () => {
|
|
358
358
|
|
|
359
|
-
|
|
359
|
+
it('all found → empty array', () => {
|
|
360
360
|
const checks = [
|
|
361
361
|
{ name: 'a', expectedPath: '/p/a/SKILL.md', found: true, section: 'tool' },
|
|
362
362
|
{ name: 'b', expectedPath: '/p/b/SKILL.md', found: true, section: 'tool' },
|
|
@@ -364,7 +364,7 @@ describe('formatSkillWarnings', () => {
|
|
|
364
364
|
expect(formatSkillWarnings(checks)).toEqual([])
|
|
365
365
|
})
|
|
366
366
|
|
|
367
|
-
|
|
367
|
+
it('some missing → one warning per missing skill', () => {
|
|
368
368
|
const checks = [
|
|
369
369
|
{ name: 'pdf', expectedPath: '/cold/pdf/SKILL.md', found: false, section: 'tool' },
|
|
370
370
|
{ name: 'docx', expectedPath: '/cold/docx/SKILL.md', found: true, section: 'tool' },
|
|
@@ -374,7 +374,7 @@ describe('formatSkillWarnings', () => {
|
|
|
374
374
|
])
|
|
375
375
|
})
|
|
376
376
|
|
|
377
|
-
|
|
377
|
+
it('all missing → warning for each', () => {
|
|
378
378
|
const checks = [
|
|
379
379
|
{ name: 'a', expectedPath: '/p/a/SKILL.md', found: false, section: 'innate' },
|
|
380
380
|
{ name: 'b', expectedPath: '/p/b/SKILL.md', found: false, section: 'tool' },
|
|
@@ -382,11 +382,11 @@ describe('formatSkillWarnings', () => {
|
|
|
382
382
|
expect(formatSkillWarnings(checks)).toHaveLength(2)
|
|
383
383
|
})
|
|
384
384
|
|
|
385
|
-
|
|
385
|
+
it('empty array → empty array', () => {
|
|
386
386
|
expect(formatSkillWarnings([])).toEqual([])
|
|
387
387
|
})
|
|
388
388
|
|
|
389
|
-
|
|
389
|
+
it('section name appears in warning string', () => {
|
|
390
390
|
const checks = [
|
|
391
391
|
{ name: 'x', expectedPath: '/p/x', found: false, section: 'transient' },
|
|
392
392
|
]
|
|
@@ -411,14 +411,14 @@ const TMP = '/tmp/arena-test'
|
|
|
411
411
|
|
|
412
412
|
describe('buildArchiveSidePlan', () => {
|
|
413
413
|
|
|
414
|
-
|
|
414
|
+
it('default: sides=["."] maps to fromDir', () => {
|
|
415
415
|
const plan = buildArchiveSidePlan(TMP, ['.'], _p => true)
|
|
416
416
|
expect(plan).toEqual([
|
|
417
417
|
{ side: '.', sourceDir: TMP, found: true },
|
|
418
418
|
])
|
|
419
419
|
})
|
|
420
420
|
|
|
421
|
-
|
|
421
|
+
it('single side, subdirectory exists → source = fromDir/side', () => {
|
|
422
422
|
const exists = (p: string) => p === pathJoin(TMP, 'side-a')
|
|
423
423
|
const plan = buildArchiveSidePlan(TMP, ['side-a'], exists)
|
|
424
424
|
expect(plan).toEqual([
|
|
@@ -426,14 +426,14 @@ describe('buildArchiveSidePlan', () => {
|
|
|
426
426
|
])
|
|
427
427
|
})
|
|
428
428
|
|
|
429
|
-
|
|
429
|
+
it('single side, subdirectory MISSING → fallback to fromDir root', () => {
|
|
430
430
|
const plan = buildArchiveSidePlan(TMP, ['side-a'], _p => false)
|
|
431
431
|
expect(plan).toEqual([
|
|
432
432
|
{ side: 'side-a', sourceDir: TMP, found: true },
|
|
433
433
|
])
|
|
434
434
|
})
|
|
435
435
|
|
|
436
|
-
|
|
436
|
+
it('multi side, all subdirectories exist', () => {
|
|
437
437
|
const exists = (p: string) =>
|
|
438
438
|
p === pathJoin(TMP, 'side-a') || p === pathJoin(TMP, 'side-b')
|
|
439
439
|
const plan = buildArchiveSidePlan(TMP, ['side-a', 'side-b'], exists)
|
|
@@ -443,7 +443,7 @@ describe('buildArchiveSidePlan', () => {
|
|
|
443
443
|
])
|
|
444
444
|
})
|
|
445
445
|
|
|
446
|
-
|
|
446
|
+
it('multi side, one missing → found=false (caller handles warn+skip)', () => {
|
|
447
447
|
const exists = (p: string) => p === pathJoin(TMP, 'side-a')
|
|
448
448
|
const plan = buildArchiveSidePlan(TMP, ['side-a', 'side-b'], exists)
|
|
449
449
|
expect(plan).toEqual([
|
|
@@ -452,19 +452,19 @@ describe('buildArchiveSidePlan', () => {
|
|
|
452
452
|
])
|
|
453
453
|
})
|
|
454
454
|
|
|
455
|
-
|
|
455
|
+
it('"." side does NOT trigger fallback when missing (found=false)', () => {
|
|
456
456
|
const plan = buildArchiveSidePlan(TMP, ['.'], _p => false)
|
|
457
457
|
expect(plan).toEqual([
|
|
458
458
|
{ side: '.', sourceDir: TMP, found: false },
|
|
459
459
|
])
|
|
460
460
|
})
|
|
461
461
|
|
|
462
|
-
|
|
462
|
+
it('empty sides array → empty plan', () => {
|
|
463
463
|
const plan = buildArchiveSidePlan(TMP, [], _p => true)
|
|
464
464
|
expect(plan).toEqual([])
|
|
465
465
|
})
|
|
466
466
|
|
|
467
|
-
|
|
467
|
+
it('three sides, middle missing', () => {
|
|
468
468
|
const exists = (p: string) =>
|
|
469
469
|
p === pathJoin(TMP, 'side-a') || p === pathJoin(TMP, 'side-c')
|
|
470
470
|
const plan = buildArchiveSidePlan(TMP, ['side-a', 'side-b', 'side-c'], exists)
|
|
@@ -509,7 +509,7 @@ path = "github.com/anthropics/skills/skills/pdf"
|
|
|
509
509
|
|
|
510
510
|
describe('buildPreparePlan', () => {
|
|
511
511
|
|
|
512
|
-
|
|
512
|
+
it('single skill deck → plan with 1 skill, hasSkills=true', () => {
|
|
513
513
|
const plan = buildPreparePlan({
|
|
514
514
|
deckPath: '/tmp/test-deck.toml',
|
|
515
515
|
deckContent: DECK_ONE_SKILL,
|
|
@@ -524,7 +524,7 @@ describe('buildPreparePlan', () => {
|
|
|
524
524
|
expect(plan.deckPath).toBe('/tmp/test-deck.toml')
|
|
525
525
|
})
|
|
526
526
|
|
|
527
|
-
|
|
527
|
+
it('empty deck → skills=[], hasSkills=false', () => {
|
|
528
528
|
const plan = buildPreparePlan({
|
|
529
529
|
deckPath: '/tmp/empty.toml',
|
|
530
530
|
deckContent: DECK_EMPTY,
|
|
@@ -535,7 +535,7 @@ describe('buildPreparePlan', () => {
|
|
|
535
535
|
expect(plan.hasSkills).toBe(false)
|
|
536
536
|
})
|
|
537
537
|
|
|
538
|
-
|
|
538
|
+
it('two skills (innate + tool) → both parsed with correct sections', () => {
|
|
539
539
|
const plan = buildPreparePlan({
|
|
540
540
|
deckPath: '/tmp/two.toml',
|
|
541
541
|
deckContent: DECK_TWO_SKILLS,
|
|
@@ -548,7 +548,7 @@ describe('buildPreparePlan', () => {
|
|
|
548
548
|
expect(plan.hasSkills).toBe(true)
|
|
549
549
|
})
|
|
550
550
|
|
|
551
|
-
|
|
551
|
+
it('AGENTS.md contains mandatory sections', () => {
|
|
552
552
|
const plan = buildPreparePlan({
|
|
553
553
|
deckPath: '/tmp/d.toml',
|
|
554
554
|
deckContent: DECK_ONE_SKILL,
|
|
@@ -561,7 +561,7 @@ describe('buildPreparePlan', () => {
|
|
|
561
561
|
expect(plan.agentsMd).toContain('skill-deck.toml')
|
|
562
562
|
})
|
|
563
563
|
|
|
564
|
-
|
|
564
|
+
it('invalid TOML → skills=[], hasSkills=false (no crash)', () => {
|
|
565
565
|
const plan = buildPreparePlan({
|
|
566
566
|
deckPath: '/tmp/bad.toml',
|
|
567
567
|
deckContent: 'this is not toml {{{',
|
|
@@ -572,7 +572,7 @@ describe('buildPreparePlan', () => {
|
|
|
572
572
|
expect(plan.hasSkills).toBe(false)
|
|
573
573
|
})
|
|
574
574
|
|
|
575
|
-
|
|
575
|
+
it('deckContent is preserved in plan', () => {
|
|
576
576
|
const plan = buildPreparePlan({
|
|
577
577
|
deckPath: '/tmp/d.toml',
|
|
578
578
|
deckContent: DECK_ONE_SKILL,
|
package/src/runner.test.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { describe,
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
2
|
import { buildArenaPrompt, formatPlanOutput, runArenaFromToml, type ArenaIO } from './runner'
|
|
3
3
|
import { buildExecutionPlan, parseArenaToml } from './arena-toml'
|
|
4
4
|
|
|
5
5
|
describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
6
|
-
|
|
6
|
+
it('includes CWD, Deck, and output directory', () => {
|
|
7
7
|
const prompt = buildArenaPrompt({
|
|
8
8
|
brief: 'Write a hello world function',
|
|
9
9
|
cwd: '/tmp/arena-cell',
|
|
@@ -14,7 +14,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
14
14
|
expect(prompt).toContain('Produce output to: /tmp/arena-cell/')
|
|
15
15
|
})
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
it('includes decision-log.jsonl mandatory instructions', () => {
|
|
18
18
|
const prompt = buildArenaPrompt({
|
|
19
19
|
brief: 'test',
|
|
20
20
|
cwd: '/tmp/arena-cell',
|
|
@@ -27,7 +27,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
27
27
|
expect(prompt).toContain('reason (why)')
|
|
28
28
|
})
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
it('includes robustness and tools instructions', () => {
|
|
31
31
|
const prompt = buildArenaPrompt({
|
|
32
32
|
brief: 'test',
|
|
33
33
|
cwd: '/tmp/arena-cell',
|
|
@@ -38,7 +38,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
38
38
|
expect(prompt).toContain('.claude/skills/')
|
|
39
39
|
})
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
it('task brief appears at the end', () => {
|
|
42
42
|
const prompt = buildArenaPrompt({
|
|
43
43
|
brief: 'Generate a dark-mode CSS theme',
|
|
44
44
|
cwd: '/tmp/arena-cell',
|
|
@@ -54,7 +54,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
54
54
|
expect(briefIdx).toBeGreaterThan(toolsIdx)
|
|
55
55
|
})
|
|
56
56
|
|
|
57
|
-
|
|
57
|
+
it('outputDir overrides default output path', () => {
|
|
58
58
|
const prompt = buildArenaPrompt({
|
|
59
59
|
brief: 'test',
|
|
60
60
|
cwd: '/tmp/arena-cell',
|
|
@@ -64,7 +64,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
64
64
|
expect(prompt).toContain('Produce output to: /custom/output/')
|
|
65
65
|
})
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
it('preflightReport included when provided', () => {
|
|
68
68
|
const prompt = buildArenaPrompt({
|
|
69
69
|
brief: 'test',
|
|
70
70
|
cwd: '/tmp/arena-cell',
|
|
@@ -75,7 +75,7 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
75
75
|
expect(prompt).toContain('✅ 3 skills linked, 0 missing')
|
|
76
76
|
})
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
it('no preflight section when report not provided', () => {
|
|
79
79
|
const prompt = buildArenaPrompt({
|
|
80
80
|
brief: 'test',
|
|
81
81
|
cwd: '/tmp/arena-cell',
|
|
@@ -84,14 +84,14 @@ describe('buildArenaPrompt — plan-mode (pure, no IO)', () => {
|
|
|
84
84
|
expect(prompt).not.toContain('Preflight:')
|
|
85
85
|
})
|
|
86
86
|
|
|
87
|
-
|
|
87
|
+
it('prompt is deterministic — same inputs = same output', () => {
|
|
88
88
|
const opts = { brief: 'test', cwd: '/tmp/a', deckPath: '/tmp/d.toml' }
|
|
89
89
|
expect(buildArenaPrompt(opts)).toBe(buildArenaPrompt(opts))
|
|
90
90
|
})
|
|
91
91
|
})
|
|
92
92
|
|
|
93
93
|
describe('runArenaFromToml — dry-run mode (no fs/spawn/agent calls)', () => {
|
|
94
|
-
|
|
94
|
+
it('returns plan without executing cells', async () => {
|
|
95
95
|
const toml = parseArenaToml(`
|
|
96
96
|
[arena]
|
|
97
97
|
task = "Write a hello world function"
|
|
@@ -125,7 +125,7 @@ deck = "/tmp/deck.toml"
|
|
|
125
125
|
expect(logs.some(l => l.includes('Dry-run'))).toBe(true)
|
|
126
126
|
})
|
|
127
127
|
|
|
128
|
-
|
|
128
|
+
it('dry-run does not call io.mkdir or io.agentSpawn', async () => {
|
|
129
129
|
const toml = parseArenaToml(`
|
|
130
130
|
[arena]
|
|
131
131
|
task = "Write a hello world function"
|
|
@@ -164,7 +164,7 @@ deck = "/tmp/deck.toml"
|
|
|
164
164
|
})
|
|
165
165
|
|
|
166
166
|
describe('runArenaFromToml — single cell execution with mock agentSpawn', () => {
|
|
167
|
-
|
|
167
|
+
it('executes one cell and writes outputs via io', async () => {
|
|
168
168
|
const toml = parseArenaToml(`
|
|
169
169
|
[arena]
|
|
170
170
|
task = "Write a hello world function"
|
|
@@ -234,7 +234,7 @@ deck = "/tmp/deck.toml"
|
|
|
234
234
|
})
|
|
235
235
|
|
|
236
236
|
describe('runArenaFromToml — error recovery path', () => {
|
|
237
|
-
|
|
237
|
+
it('cell exception produces ERROR verdict and continues', async () => {
|
|
238
238
|
const toml = parseArenaToml(`
|
|
239
239
|
[arena]
|
|
240
240
|
task = "Write a hello world function"
|
|
@@ -296,7 +296,7 @@ deck = "/tmp/deck.toml"
|
|
|
296
296
|
expect(verdict.reason).toContain('Agent spawn failed')
|
|
297
297
|
})
|
|
298
298
|
|
|
299
|
-
|
|
299
|
+
it('spawn non-zero exit logs warning but continues', async () => {
|
|
300
300
|
const toml = parseArenaToml(`
|
|
301
301
|
[arena]
|
|
302
302
|
task = "Write a hello world function"
|
package/src/stats.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { describe,
|
|
1
|
+
import { describe, it, expect } from 'bun:test'
|
|
2
2
|
import { aggregateSideStats, aggregateAllStats } from './stats'
|
|
3
3
|
import type { JudgeVerdict } from '@lythos/test-utils/schema'
|
|
4
4
|
|
|
@@ -14,7 +14,7 @@ function makeVerdict(overrides?: Partial<JudgeVerdict>): JudgeVerdict {
|
|
|
14
14
|
// ── aggregateSideStats ─────────────────────────────────────────────────────
|
|
15
15
|
|
|
16
16
|
describe('aggregateSideStats', () => {
|
|
17
|
-
|
|
17
|
+
it('single run: passRate=1, no variance', () => {
|
|
18
18
|
const stats = aggregateSideStats('test', [makeVerdict()])
|
|
19
19
|
expect(stats.sideName).toBe('test')
|
|
20
20
|
expect(stats.runs).toBe(1)
|
|
@@ -23,7 +23,7 @@ describe('aggregateSideStats', () => {
|
|
|
23
23
|
expect(stats.errorRate).toBe(0)
|
|
24
24
|
})
|
|
25
25
|
|
|
26
|
-
|
|
26
|
+
it('3 runs: 2 PASS, 1 FAIL', () => {
|
|
27
27
|
const verdicts = [
|
|
28
28
|
makeVerdict(),
|
|
29
29
|
makeVerdict(),
|
|
@@ -34,7 +34,7 @@ describe('aggregateSideStats', () => {
|
|
|
34
34
|
expect(stats.failRate).toBeCloseTo(1 / 3)
|
|
35
35
|
})
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
it('confidence: mean across runs', () => {
|
|
38
38
|
const verdicts = [
|
|
39
39
|
makeVerdict({ confidence: 90 }),
|
|
40
40
|
makeVerdict({ confidence: 80 }),
|
|
@@ -45,13 +45,13 @@ describe('aggregateSideStats', () => {
|
|
|
45
45
|
expect(stats.confidenceVariance).toBeCloseTo(100) // (100+0+100)/2 = 100
|
|
46
46
|
})
|
|
47
47
|
|
|
48
|
-
|
|
48
|
+
it('confidence: null when no verdict has it', () => {
|
|
49
49
|
const stats = aggregateSideStats('test', [makeVerdict(), makeVerdict()])
|
|
50
50
|
expect(stats.meanConfidence).toBeNull()
|
|
51
51
|
expect(stats.confidenceVariance).toBeNull()
|
|
52
52
|
})
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
it('per-criterion pass rate', () => {
|
|
55
55
|
const verdicts = [
|
|
56
56
|
makeVerdict({ criteria: [{ name: 'accuracy', passed: true }] }),
|
|
57
57
|
makeVerdict({ criteria: [{ name: 'accuracy', passed: false }] }),
|
|
@@ -63,7 +63,7 @@ describe('aggregateSideStats', () => {
|
|
|
63
63
|
expect(stats.criteria[0].mean).toBeCloseTo(2 / 3)
|
|
64
64
|
})
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
it('per-criterion scores: mean and variance', () => {
|
|
67
67
|
const verdicts = [
|
|
68
68
|
makeVerdict({ scores: { coverage: 5, relevance: 4 } }),
|
|
69
69
|
makeVerdict({ scores: { coverage: 3, relevance: 4 } }),
|
|
@@ -75,14 +75,14 @@ describe('aggregateSideStats', () => {
|
|
|
75
75
|
expect(stats.scoreByCriterion.relevance.variance).toBe(0) // all 4s
|
|
76
76
|
})
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
it('zero runs: all zeros', () => {
|
|
79
79
|
const stats = aggregateSideStats('empty', [])
|
|
80
80
|
expect(stats.runs).toBe(0)
|
|
81
81
|
expect(stats.passRate).toBe(0)
|
|
82
82
|
expect(stats.meanConfidence).toBeNull()
|
|
83
83
|
})
|
|
84
84
|
|
|
85
|
-
|
|
85
|
+
it('handles ERROR verdicts correctly', () => {
|
|
86
86
|
const verdicts = [
|
|
87
87
|
makeVerdict(),
|
|
88
88
|
makeVerdict({ verdict: 'ERROR', reason: 'parse failed' }),
|
|
@@ -96,7 +96,7 @@ describe('aggregateSideStats', () => {
|
|
|
96
96
|
// ── aggregateAllStats ──────────────────────────────────────────────────────
|
|
97
97
|
|
|
98
98
|
describe('aggregateAllStats', () => {
|
|
99
|
-
|
|
99
|
+
it('aggregates multiple sides', () => {
|
|
100
100
|
const map = new Map<string, JudgeVerdict[]>()
|
|
101
101
|
map.set('side-a', [makeVerdict(), makeVerdict()])
|
|
102
102
|
map.set('side-b', [makeVerdict({ verdict: 'FAIL', reason: 'nope' })])
|