@lythos/skill-arena 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -12
- package/package.json +8 -8
- package/src/cli.test.ts +75 -0
- package/src/cli.ts +37 -12
- package/src/preflight.test.ts +216 -0
- package/src/preflight.ts +157 -19
- package/src/runner.test.ts +178 -0
- package/src/runner.ts +52 -12
package/README.md
CHANGED
|
@@ -16,20 +16,20 @@
|
|
|
16
16
|
```bash
|
|
17
17
|
bun add -d @lythos/skill-arena
|
|
18
18
|
# or use directly
|
|
19
|
-
bunx @lythos/skill-arena@0.19.
|
|
19
|
+
bunx @lythos/skill-arena@0.19.2 <command>
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
## Quick Start
|
|
23
23
|
|
|
24
24
|
```bash
|
|
25
25
|
# single — test one deck (most common)
|
|
26
|
-
bunx @lythos/skill-arena@0.19.
|
|
26
|
+
bunx @lythos/skill-arena@0.19.2 single \
|
|
27
27
|
--deck ./examples/decks/scout.toml \
|
|
28
28
|
--brief "Generate auth flow diagram" \
|
|
29
29
|
--out ./output
|
|
30
30
|
|
|
31
31
|
# single with explicit player
|
|
32
|
-
bunx @lythos/skill-arena@0.19.
|
|
32
|
+
bunx @lythos/skill-arena@0.19.2 single \
|
|
33
33
|
--deck ./examples/decks/scout.toml \
|
|
34
34
|
--brief "Generate auth flow diagram" \
|
|
35
35
|
--player kimi \
|
|
@@ -37,10 +37,10 @@ bunx @lythos/skill-arena@0.19.0 single \
|
|
|
37
37
|
|
|
38
38
|
# cross-deck vs — compare two decks (agent-orchestrated)
|
|
39
39
|
# Create arena.toml declaring sides with different decks, then:
|
|
40
|
-
bunx @lythos/skill-arena@0.19.
|
|
40
|
+
bunx @lythos/skill-arena@0.19.2 vs --config ./arena.toml
|
|
41
41
|
|
|
42
42
|
# cross-player vs — compare kimi vs codex (CLI only)
|
|
43
|
-
bunx @lythos/skill-arena@0.19.
|
|
43
|
+
bunx @lythos/skill-arena@0.19.2 vs --config ./arena.toml --player kimi
|
|
44
44
|
```
|
|
45
45
|
|
|
46
46
|
**What happens**: Agent creates isolated `/tmp` workdir per side, `deck link` skills, spawns parallel subagents, collects artifacts, judge scores outputs. Parent deck restored after.
|
|
@@ -50,14 +50,14 @@ bunx @lythos/skill-arena@0.19.0 vs --config ./arena.toml --player kimi
|
|
|
50
50
|
### `single` — one deck, one task
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
bunx @lythos/skill-arena@0.19.
|
|
53
|
+
bunx @lythos/skill-arena@0.19.2 single \
|
|
54
54
|
--deck ./deck.toml \
|
|
55
55
|
--brief "Produce a .docx report with radar chart" \
|
|
56
56
|
--timeout 600000 \
|
|
57
57
|
--out ./output
|
|
58
58
|
|
|
59
59
|
# with explicit player
|
|
60
|
-
bunx @lythos/skill-arena@0.19.
|
|
60
|
+
bunx @lythos/skill-arena@0.19.2 single \
|
|
61
61
|
--deck ./deck.toml \
|
|
62
62
|
--brief "Produce a .docx report with radar chart" \
|
|
63
63
|
--player kimi \
|
|
@@ -67,14 +67,14 @@ bunx @lythos/skill-arena@0.19.0 single \
|
|
|
67
67
|
### `vs` — multi-deck comparison
|
|
68
68
|
|
|
69
69
|
```bash
|
|
70
|
-
bunx @lythos/skill-arena@0.19.
|
|
71
|
-
bunx @lythos/skill-arena@0.19.
|
|
70
|
+
bunx @lythos/skill-arena@0.19.2 vs --config ./arena.toml
|
|
71
|
+
bunx @lythos/skill-arena@0.19.2 vs --config ./arena.toml --dry-run
|
|
72
72
|
```
|
|
73
73
|
|
|
74
74
|
### `prepare-workdir` — isolate + link skills (agent-orchestrated)
|
|
75
75
|
|
|
76
76
|
```bash
|
|
77
|
-
bunx @lythos/skill-arena@0.19.
|
|
77
|
+
bunx @lythos/skill-arena@0.19.2 prepare-workdir \
|
|
78
78
|
--deck ./skill-deck.toml \
|
|
79
79
|
--out /tmp/arena-side-a \
|
|
80
80
|
--brief "task description"
|
|
@@ -85,7 +85,7 @@ Creates `/tmp`-isolated workdir with deck copied, AGENTS.md written, and `deck l
|
|
|
85
85
|
### `archive` — collect agent outputs (agent-orchestrated)
|
|
86
86
|
|
|
87
87
|
```bash
|
|
88
|
-
bunx @lythos/skill-arena@0.19.
|
|
88
|
+
bunx @lythos/skill-arena@0.19.2 archive \
|
|
89
89
|
--from /tmp/arena-side-a \
|
|
90
90
|
--to ./playground/output \
|
|
91
91
|
--sides side-a
|
|
@@ -96,7 +96,7 @@ Copies agent artifacts from workdir(s) to output, skipping internal files (`.cla
|
|
|
96
96
|
### `viz` — render results (WIP — HTML report generation pending)
|
|
97
97
|
|
|
98
98
|
```bash
|
|
99
|
-
bunx @lythos/skill-arena@0.19.
|
|
99
|
+
bunx @lythos/skill-arena@0.19.2 viz runs/arena-<id>/
|
|
100
100
|
```
|
|
101
101
|
|
|
102
102
|
## Parameters
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lythos/skill-arena",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.2",
|
|
4
4
|
"description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-agent",
|
|
@@ -42,16 +42,16 @@
|
|
|
42
42
|
"bun": ">=1.0.0"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@lythos/cold-pool": "^0.19.
|
|
46
|
-
"@lythos/infra": "^0.19.
|
|
47
|
-
"@lythos/test-utils": "^0.19.
|
|
45
|
+
"@lythos/cold-pool": "^0.19.2",
|
|
46
|
+
"@lythos/infra": "^0.19.2",
|
|
47
|
+
"@lythos/test-utils": "^0.19.2",
|
|
48
48
|
"zod": "^3.24.0",
|
|
49
49
|
"zod-to-json-schema": "^3.25.2"
|
|
50
50
|
},
|
|
51
51
|
"optionalDependencies": {
|
|
52
|
-
"@lythos/agent-adapter-claude-sdk": "^0.19.
|
|
53
|
-
"@lythos/agent-adapter-deepseek-serve": "^0.19.
|
|
54
|
-
"@lythos/agent-adapter-deepseek-harness": "^0.19.
|
|
55
|
-
"@lythos/agent-adapter-codex": "^0.19.
|
|
52
|
+
"@lythos/agent-adapter-claude-sdk": "^0.19.2",
|
|
53
|
+
"@lythos/agent-adapter-deepseek-serve": "^0.19.2",
|
|
54
|
+
"@lythos/agent-adapter-deepseek-harness": "^0.19.2",
|
|
55
|
+
"@lythos/agent-adapter-codex": "^0.19.2"
|
|
56
56
|
}
|
|
57
57
|
}
|
package/src/cli.test.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'bun:test'
|
|
2
|
+
import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs'
|
|
3
|
+
import { tmpdir } from 'node:os'
|
|
4
|
+
import { join } from 'node:path'
|
|
2
5
|
import { main, defaultArenaCliIO, type ArenaCliIO } from './cli'
|
|
3
6
|
|
|
4
7
|
// ─── Helpers ───────────────────────────────────────────────────────────────
|
|
@@ -120,3 +123,75 @@ describe('defaultArenaCliIO', () => {
|
|
|
120
123
|
expect(typeof defaultArenaCliIO.exit).toBe('function')
|
|
121
124
|
})
|
|
122
125
|
})
|
|
126
|
+
|
|
127
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
128
|
+
// archive — collect step merges per-cell decision logs
|
|
129
|
+
// (TASK-20260909010121918)
|
|
130
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
131
|
+
|
|
132
|
+
describe('archive — per-cell decision-log merge', () => {
|
|
133
|
+
|
|
134
|
+
function scaffold() {
|
|
135
|
+
const root = mkdtempSync(join(tmpdir(), 'arena-archive-'))
|
|
136
|
+
const from = join(root, 'work')
|
|
137
|
+
const to = join(root, 'archived')
|
|
138
|
+
mkdirSync(join(from, 'side-a'), { recursive: true })
|
|
139
|
+
mkdirSync(join(from, 'side-b'), { recursive: true })
|
|
140
|
+
return { root, from, to }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
it('merges side-a per-cell logs into one decision-log.jsonl', async () => {
|
|
144
|
+
const { root, from, to } = scaffold()
|
|
145
|
+
try {
|
|
146
|
+
writeFileSync(join(from, 'side-a', 'decision-log-s1a.jsonl'), '{"t":0,"phase":"a","decision":"cell 1a","reason":"r"}\n')
|
|
147
|
+
writeFileSync(join(from, 'side-a', 'decision-log-s3a.jsonl'), '{"t":0,"phase":"a","decision":"cell 3a","reason":"r"}\n')
|
|
148
|
+
writeFileSync(join(from, 'side-a', 'report.md'), '# side-a report\n')
|
|
149
|
+
|
|
150
|
+
const { io, logs } = mockIO()
|
|
151
|
+
await main(['archive', '--from', from, '--to', to, '--sides', 'side-a'], io)
|
|
152
|
+
|
|
153
|
+
const merged = readFileSync(join(to, 'side-a', 'decision-log.jsonl'), 'utf-8')
|
|
154
|
+
expect(merged).toContain('cell 1a')
|
|
155
|
+
expect(merged).toContain('cell 3a')
|
|
156
|
+
expect(merged.trim().split('\n')).toHaveLength(2)
|
|
157
|
+
// Per-cell files stay as provenance — the merged line carries no cell attribution.
|
|
158
|
+
expect(existsSync(join(to, 'side-a', 'decision-log-s1a.jsonl'))).toBe(true)
|
|
159
|
+
expect(existsSync(join(to, 'side-a', 'decision-log-s3a.jsonl'))).toBe(true)
|
|
160
|
+
expect(logs.some(l => l.includes('merged 2 per-cell log(s)'))).toBe(true)
|
|
161
|
+
} finally {
|
|
162
|
+
rmSync(root, { recursive: true, force: true })
|
|
163
|
+
}
|
|
164
|
+
})
|
|
165
|
+
|
|
166
|
+
it('a single legacy decision-log.jsonl is already merged — not rewritten', async () => {
|
|
167
|
+
const { root, from, to } = scaffold()
|
|
168
|
+
try {
|
|
169
|
+
const legacy = '{"t":0,"phase":"a","decision":"only cell","reason":"r"}\n'
|
|
170
|
+
writeFileSync(join(from, 'side-b', 'decision-log.jsonl'), legacy)
|
|
171
|
+
|
|
172
|
+
const { io } = mockIO()
|
|
173
|
+
await main(['archive', '--from', from, '--to', to, '--sides', 'side-b'], io)
|
|
174
|
+
|
|
175
|
+
expect(readFileSync(join(to, 'side-b', 'decision-log.jsonl'), 'utf-8')).toBe(legacy)
|
|
176
|
+
} finally {
|
|
177
|
+
rmSync(root, { recursive: true, force: true })
|
|
178
|
+
}
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
it('mixed workdir: legacy + per-cell logs all survive the merge', async () => {
|
|
182
|
+
const { root, from, to } = scaffold()
|
|
183
|
+
try {
|
|
184
|
+
writeFileSync(join(from, 'side-a', 'decision-log.jsonl'), '{"t":0,"phase":"a","decision":"legacy entry","reason":"r"}\n')
|
|
185
|
+
writeFileSync(join(from, 'side-a', 'decision-log-s1a.jsonl'), '{"t":0,"phase":"a","decision":"cell 1a","reason":"r"}\n')
|
|
186
|
+
|
|
187
|
+
const { io } = mockIO()
|
|
188
|
+
await main(['archive', '--from', from, '--to', to, '--sides', 'side-a'], io)
|
|
189
|
+
|
|
190
|
+
const merged = readFileSync(join(to, 'side-a', 'decision-log.jsonl'), 'utf-8')
|
|
191
|
+
expect(merged).toContain('legacy entry')
|
|
192
|
+
expect(merged).toContain('cell 1a')
|
|
193
|
+
} finally {
|
|
194
|
+
rmSync(root, { recursive: true, force: true })
|
|
195
|
+
}
|
|
196
|
+
})
|
|
197
|
+
})
|
package/src/cli.ts
CHANGED
|
@@ -5,7 +5,7 @@ import { homedir, tmpdir } from 'node:os'
|
|
|
5
5
|
import { ZodError } from 'zod'
|
|
6
6
|
import { formatPlanOutput, type ArenaResult, buildArenaPrompt } from './runner'
|
|
7
7
|
import { parseArenaToml, buildExecutionPlan } from './arena-toml'
|
|
8
|
-
import { buildArchiveSidePlan, buildCopyPlan, buildPreparePlan, parseDeckSkills } from './preflight'
|
|
8
|
+
import { buildArchiveSidePlan, buildCopyPlan, buildPreparePlan, parseDeckCombos, parseDeckSkills, buildAgentsMd, isDecisionLogName, mergeDecisionLogs, LEGACY_DECISION_LOG } from './preflight'
|
|
9
9
|
import { checkSkillExistence, formatSkillWarnings, resolveColdPoolDir } from './preflight'
|
|
10
10
|
import { resolveSingleMode } from './host'
|
|
11
11
|
|
|
@@ -118,6 +118,7 @@ Modes (single):
|
|
|
118
118
|
|
|
119
119
|
Examples:
|
|
120
120
|
lythoskill-arena single --brief "find and research" --deck ./decks/scout.toml
|
|
121
|
+
lythoskill-arena single --brief "audit this repo" --deck ./decks/qa-sweep.toml --out /tmp/arena-audit
|
|
121
122
|
lythoskill-arena single --brief "find and research" --deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml
|
|
122
123
|
lythoskill-arena vs --config arena.toml --dry-run
|
|
123
124
|
lythoskill-arena vs --config arena.toml
|
|
@@ -339,17 +340,16 @@ async function singleRun(args: string[], io: ArenaCliIO) {
|
|
|
339
340
|
// Setup workdir
|
|
340
341
|
const agentWorkdir = join(tmpdir(), `arena-single-${Date.now()}`)
|
|
341
342
|
mkdirSync(agentWorkdir, { recursive: true })
|
|
342
|
-
|
|
343
|
-
writeFileSync(join(agentWorkdir, '
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
const deckRaw = readFileSync(join(agentWorkdir, 'skill-deck.toml'), 'utf-8')
|
|
343
|
+
const singleDeckRaw = readFileSync(deckPath, 'utf-8')
|
|
344
|
+
writeFileSync(join(agentWorkdir, 'skill-deck.toml'), singleDeckRaw)
|
|
345
|
+
let singleDeckParsed: Record<string, any> = {}
|
|
346
|
+
try { singleDeckParsed = Bun.TOML.parse(singleDeckRaw) as Record<string, any> } catch {}
|
|
347
|
+
writeFileSync(join(agentWorkdir, 'AGENTS.md'), buildAgentsMd({
|
|
348
|
+
mode: 'single',
|
|
349
|
+
combos: parseDeckCombos(singleDeckParsed),
|
|
350
|
+
}))
|
|
351
|
+
|
|
352
|
+
const deckRaw = singleDeckRaw
|
|
353
353
|
let deckParsed: Record<string, any> = {}
|
|
354
354
|
try { deckParsed = Bun.TOML.parse(deckRaw) as Record<string, any> } catch {}
|
|
355
355
|
const hasSkills = parseDeckSkills(deckParsed).length > 0
|
|
@@ -679,6 +679,31 @@ async function archiveRun(args: string[], io: ArenaCliIO) {
|
|
|
679
679
|
warn(`⚠️ Failed to copy ${planEntry.side}/${entry.name}: ${e instanceof Error ? e.message : e}`)
|
|
680
680
|
}
|
|
681
681
|
}
|
|
682
|
+
|
|
683
|
+
// ── Collect: merge per-cell decision logs into one decision-log.jsonl ──
|
|
684
|
+
// Cells sharing a side workdir each wrote `decision-log-<cell-id>.jsonl`
|
|
685
|
+
// (TASK-20260909010121918). Reconstructing the single-file view here is
|
|
686
|
+
// what makes the merged log possible to read; the per-cell files were
|
|
687
|
+
// just copied and stay as provenance. A lone legacy `decision-log.jsonl`
|
|
688
|
+
// is already the merged view — nothing to do.
|
|
689
|
+
const logNames = entries
|
|
690
|
+
.filter(e => e.isFile() && isDecisionLogName(e.name))
|
|
691
|
+
.map(e => e.name)
|
|
692
|
+
.sort()
|
|
693
|
+
// A lone `decision-log.jsonl` IS the merged view — do not rewrite it.
|
|
694
|
+
const needsMerge = logNames.length > 1 || (logNames.length === 1 && logNames[0] !== LEGACY_DECISION_LOG)
|
|
695
|
+
if (needsMerge) {
|
|
696
|
+
try {
|
|
697
|
+
const sources = logNames.map(name => ({
|
|
698
|
+
cell: name,
|
|
699
|
+
content: readFileSync(join(sideOutDir, name), 'utf-8'),
|
|
700
|
+
}))
|
|
701
|
+
writeFileSync(join(sideOutDir, LEGACY_DECISION_LOG), mergeDecisionLogs(sources))
|
|
702
|
+
log(` ${planEntry.side}/${LEGACY_DECISION_LOG} ← merged ${logNames.length} per-cell log(s)`)
|
|
703
|
+
} catch (e) {
|
|
704
|
+
warn(`⚠️ Failed to merge decision logs for ${planEntry.side}: ${e instanceof Error ? e.message : e}`)
|
|
705
|
+
}
|
|
706
|
+
}
|
|
682
707
|
}
|
|
683
708
|
|
|
684
709
|
log(`✅ Archive complete → ${outDir}`)
|
package/src/preflight.test.ts
CHANGED
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { describe, it, expect } from 'bun:test'
|
|
14
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
|
15
|
+
import { tmpdir } from 'node:os'
|
|
16
|
+
import { join } from 'node:path'
|
|
14
17
|
import {
|
|
15
18
|
parseDeckSkills,
|
|
16
19
|
checkSkillExistence,
|
|
@@ -18,6 +21,11 @@ import {
|
|
|
18
21
|
buildCopyPlan,
|
|
19
22
|
resolveColdPoolDir,
|
|
20
23
|
formatSkillWarnings,
|
|
24
|
+
decisionLogName,
|
|
25
|
+
isDecisionLogName,
|
|
26
|
+
mergeDecisionLogs,
|
|
27
|
+
LEGACY_DECISION_LOG,
|
|
28
|
+
type DecisionLogSource,
|
|
21
29
|
} from './preflight'
|
|
22
30
|
|
|
23
31
|
// ═══════════════════════════════════════════════════════════════════════════
|
|
@@ -582,3 +590,211 @@ describe('buildPreparePlan', () => {
|
|
|
582
590
|
expect(plan.deckContent).toBe(DECK_ONE_SKILL)
|
|
583
591
|
})
|
|
584
592
|
})
|
|
593
|
+
|
|
594
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
595
|
+
// parseDeckCombos + combo surfacing in AGENTS.md (F1/F2, TASK-20260909152255103)
|
|
596
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
597
|
+
|
|
598
|
+
import { parseDeckCombos, buildAgentsMd } from './preflight'
|
|
599
|
+
|
|
600
|
+
const DECK_WITH_COMBOS = `
|
|
601
|
+
[deck]
|
|
602
|
+
max_cards = 10
|
|
603
|
+
|
|
604
|
+
[tool.skills.pdf]
|
|
605
|
+
path = "github.com/anthropics/skills/skills/pdf"
|
|
606
|
+
|
|
607
|
+
[combo.promo]
|
|
608
|
+
prompt = "Pipeline: Position → Draft → Review."
|
|
609
|
+
|
|
610
|
+
[combo.research]
|
|
611
|
+
prompt = "Pipeline: Discover → Verify → Analyze."
|
|
612
|
+
`
|
|
613
|
+
|
|
614
|
+
describe('parseDeckCombos', () => {
|
|
615
|
+
|
|
616
|
+
it('extracts combo names in declaration order', () => {
|
|
617
|
+
const parsed = Bun.TOML.parse(DECK_WITH_COMBOS) as Record<string, any>
|
|
618
|
+
expect(parseDeckCombos(parsed)).toEqual(['promo', 'research'])
|
|
619
|
+
})
|
|
620
|
+
|
|
621
|
+
it('no combo section → empty array', () => {
|
|
622
|
+
const parsed = Bun.TOML.parse(DECK_ONE_SKILL) as Record<string, any>
|
|
623
|
+
expect(parseDeckCombos(parsed)).toEqual([])
|
|
624
|
+
})
|
|
625
|
+
|
|
626
|
+
it('legacy bare [combo] prompt field → empty array (default combo, no named section)', () => {
|
|
627
|
+
const parsed = Bun.TOML.parse('[combo]\nprompt = "do stuff"') as Record<string, any>
|
|
628
|
+
expect(parseDeckCombos(parsed)).toEqual([])
|
|
629
|
+
})
|
|
630
|
+
})
|
|
631
|
+
|
|
632
|
+
describe('buildAgentsMd combo surfacing (F1)', () => {
|
|
633
|
+
|
|
634
|
+
it('deck with combos → AGENTS.md lists them under "Read These First"', () => {
|
|
635
|
+
const parsed = Bun.TOML.parse(DECK_WITH_COMBOS) as Record<string, any>
|
|
636
|
+
const md = buildAgentsMd({ mode: 'single', combos: parseDeckCombos(parsed) })
|
|
637
|
+
expect(md).toContain('## Combos — Read These First')
|
|
638
|
+
expect(md).toContain('- `promo`')
|
|
639
|
+
expect(md).toContain('- `research`')
|
|
640
|
+
expect(md.indexOf('Combos — Read These First')).toBeLessThan(md.indexOf('## Setup Order'))
|
|
641
|
+
})
|
|
642
|
+
|
|
643
|
+
it('combo consumer declared: agent reads/executes, CLI only parses (F2)', () => {
|
|
644
|
+
const md = buildAgentsMd({ mode: 'single', combos: ['promo'] })
|
|
645
|
+
expect(md).toContain('executed by YOU (the agent)')
|
|
646
|
+
expect(md).toContain('the CLI only parses them; nothing runs them')
|
|
647
|
+
})
|
|
648
|
+
|
|
649
|
+
it('deck without combos → no combos section', () => {
|
|
650
|
+
const md = buildAgentsMd({ mode: 'single' })
|
|
651
|
+
expect(md).not.toContain('Combos')
|
|
652
|
+
})
|
|
653
|
+
})
|
|
654
|
+
|
|
655
|
+
describe('buildAgentsMd vocabulary gloss (F3)', () => {
|
|
656
|
+
|
|
657
|
+
it('glosses cold pool / working set / innate / tool / max_cards', () => {
|
|
658
|
+
const md = buildAgentsMd({ mode: 'single' })
|
|
659
|
+
expect(md).toContain('**cold pool**')
|
|
660
|
+
expect(md).toContain('**working set**')
|
|
661
|
+
expect(md).toContain('**innate**')
|
|
662
|
+
expect(md).toContain('**tool**')
|
|
663
|
+
expect(md).toContain('**max_cards**')
|
|
664
|
+
})
|
|
665
|
+
})
|
|
666
|
+
|
|
667
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
668
|
+
// decision logs — per-cell naming (TASK-20260909010121918)
|
|
669
|
+
//
|
|
670
|
+
// Defect: five cells per side shared one workdir and one mandated filename
|
|
671
|
+
// (`decision-log.jsonl`). Filename is a constant → shared path → last writer
|
|
672
|
+
// wins; the 2026-09-09 side-a log lost S3a/S4a/S5a entirely.
|
|
673
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
674
|
+
|
|
675
|
+
describe('decisionLogName', () => {
|
|
676
|
+
|
|
677
|
+
it('per-cell id → per-cell filename', () => {
|
|
678
|
+
expect(decisionLogName('s1a')).toBe('decision-log-s1a.jsonl')
|
|
679
|
+
expect(decisionLogName('claude-run-2')).toBe('decision-log-claude-run-2.jsonl')
|
|
680
|
+
})
|
|
681
|
+
|
|
682
|
+
it('two distinct cell ids never share a path', () => {
|
|
683
|
+
expect(decisionLogName('s1a')).not.toBe(decisionLogName('s3a'))
|
|
684
|
+
})
|
|
685
|
+
|
|
686
|
+
it('no id → legacy single-cell name', () => {
|
|
687
|
+
expect(decisionLogName()).toBe(LEGACY_DECISION_LOG)
|
|
688
|
+
expect(decisionLogName('')).toBe(LEGACY_DECISION_LOG)
|
|
689
|
+
})
|
|
690
|
+
|
|
691
|
+
it('id cannot escape the workdir (no separators, no traversal, no dotfile)', () => {
|
|
692
|
+
expect(decisionLogName('../evil')).toBe('decision-log-evil.jsonl')
|
|
693
|
+
expect(decisionLogName('a/b')).toBe('decision-log-a-b.jsonl')
|
|
694
|
+
expect(decisionLogName('claude run 1')).toBe('decision-log-claude-run-1.jsonl')
|
|
695
|
+
for (const hostile of ['../evil', 'a/b', '..', './x', 'a\\b', '..\0']) {
|
|
696
|
+
const name = decisionLogName(hostile)
|
|
697
|
+
expect(name).not.toContain('/')
|
|
698
|
+
expect(name).not.toContain('\\')
|
|
699
|
+
expect(name.startsWith('.')).toBe(false)
|
|
700
|
+
expect(isDecisionLogName(name)).toBe(true)
|
|
701
|
+
}
|
|
702
|
+
})
|
|
703
|
+
|
|
704
|
+
it('separator-only id degrades to the legacy name, never to a bare prefix', () => {
|
|
705
|
+
expect(decisionLogName('///')).toBe(LEGACY_DECISION_LOG)
|
|
706
|
+
})
|
|
707
|
+
})
|
|
708
|
+
|
|
709
|
+
describe('isDecisionLogName', () => {
|
|
710
|
+
|
|
711
|
+
it('accepts the legacy name and per-cell names', () => {
|
|
712
|
+
expect(isDecisionLogName('decision-log.jsonl')).toBe(true)
|
|
713
|
+
expect(isDecisionLogName('decision-log-s1a.jsonl')).toBe(true)
|
|
714
|
+
})
|
|
715
|
+
|
|
716
|
+
it('rejects unrelated files', () => {
|
|
717
|
+
expect(isDecisionLogName('decision-log.jsonl.bak')).toBe(false)
|
|
718
|
+
expect(isDecisionLogName('decision-log.txt')).toBe(false)
|
|
719
|
+
expect(isDecisionLogName('my-decision-log.jsonl')).toBe(false)
|
|
720
|
+
expect(isDecisionLogName('judge-verdict.json')).toBe(false)
|
|
721
|
+
})
|
|
722
|
+
})
|
|
723
|
+
|
|
724
|
+
describe('mergeDecisionLogs', () => {
|
|
725
|
+
|
|
726
|
+
const s1a: DecisionLogSource = {
|
|
727
|
+
cell: 's1a',
|
|
728
|
+
content: '{"t":0,"phase":"setup","decision":"picked palette","reason":"theme"}\n',
|
|
729
|
+
}
|
|
730
|
+
const s3a: DecisionLogSource = {
|
|
731
|
+
cell: 's3a',
|
|
732
|
+
content: '{"t":0,"phase":"setup","decision":"resolved id","reason":"card"}\n',
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
it('keeps every line from every cell — nothing is dropped', () => {
|
|
736
|
+
const merged = mergeDecisionLogs([s1a, s3a])
|
|
737
|
+
expect(merged).toContain('picked palette')
|
|
738
|
+
expect(merged).toContain('resolved id')
|
|
739
|
+
expect(merged.trim().split('\n')).toHaveLength(2)
|
|
740
|
+
})
|
|
741
|
+
|
|
742
|
+
it('order is by cell id, not input order — same inputs, same bytes', () => {
|
|
743
|
+
const a = mergeDecisionLogs([s1a, s3a])
|
|
744
|
+
const b = mergeDecisionLogs([s3a, s1a])
|
|
745
|
+
expect(a).toBe(b)
|
|
746
|
+
expect(a.indexOf('picked palette')).toBeLessThan(a.indexOf('resolved id'))
|
|
747
|
+
})
|
|
748
|
+
|
|
749
|
+
it('lines are kept verbatim — no re-encoding, no injected field', () => {
|
|
750
|
+
const odd: DecisionLogSource = { cell: 'x', content: 'not json at all\n{"t":1,"phase":"p","decision":"d","reason":"r"}' }
|
|
751
|
+
const merged = mergeDecisionLogs([odd])
|
|
752
|
+
expect(merged).toContain('not json at all')
|
|
753
|
+
expect(JSON.parse(merged.trim().split('\n')[1])).toEqual({ t: 1, phase: 'p', decision: 'd', reason: 'r' })
|
|
754
|
+
})
|
|
755
|
+
|
|
756
|
+
it('blank lines are not entries; each line keeps exactly one newline', () => {
|
|
757
|
+
const gappy: DecisionLogSource = { cell: 'g', content: '\n{"t":1,"phase":"p","decision":"d","reason":"r"}\n\n' }
|
|
758
|
+
expect(mergeDecisionLogs([gappy])).toBe('{"t":1,"phase":"p","decision":"d","reason":"r"}\n')
|
|
759
|
+
})
|
|
760
|
+
|
|
761
|
+
it('a lone legacy file merges to itself (idempotent)', () => {
|
|
762
|
+
const content = '{"t":0,"phase":"setup","decision":"d","reason":"r"}\n'
|
|
763
|
+
expect(mergeDecisionLogs([{ cell: LEGACY_DECISION_LOG, content }])).toBe(content)
|
|
764
|
+
})
|
|
765
|
+
|
|
766
|
+
it('no sources → empty string (callers can skip writing)', () => {
|
|
767
|
+
expect(mergeDecisionLogs([])).toBe('')
|
|
768
|
+
})
|
|
769
|
+
})
|
|
770
|
+
|
|
771
|
+
describe('decision-log — 2-writer concurrency (real fs)', () => {
|
|
772
|
+
|
|
773
|
+
it('two cells writing concurrently in ONE workdir both survive the merge', async () => {
|
|
774
|
+
const dir = mkdtempSync(join(tmpdir(), 'arena-decision-log-'))
|
|
775
|
+
try {
|
|
776
|
+
// Two cells, one shared workdir, one writer each, interleaved by an await.
|
|
777
|
+
// Before the fix both paths were the same constant, so this write pair
|
|
778
|
+
// clobbered and the merge below would lose one cell's only entry.
|
|
779
|
+
const write = async (cellId: string, line: string) => {
|
|
780
|
+
await Bun.sleep(0)
|
|
781
|
+
writeFileSync(join(dir, decisionLogName(cellId)), line + '\n', 'utf-8')
|
|
782
|
+
}
|
|
783
|
+
await Promise.all([
|
|
784
|
+
write('s1a', '{"t":0,"phase":"setup","decision":"cel a","reason":"ra"}'),
|
|
785
|
+
write('s3a', '{"t":0,"phase":"setup","decision":"cell b","reason":"rb"}'),
|
|
786
|
+
])
|
|
787
|
+
|
|
788
|
+
const merged = mergeDecisionLogs([
|
|
789
|
+
{ cell: 's1a', content: await Bun.file(join(dir, decisionLogName('s1a'))).text() },
|
|
790
|
+
{ cell: 's3a', content: await Bun.file(join(dir, decisionLogName('s3a'))).text() },
|
|
791
|
+
])
|
|
792
|
+
|
|
793
|
+
expect(merged).toContain('cel a')
|
|
794
|
+
expect(merged).toContain('cell b')
|
|
795
|
+
expect(merged.trim().split('\n')).toHaveLength(2)
|
|
796
|
+
} finally {
|
|
797
|
+
rmSync(dir, { recursive: true, force: true })
|
|
798
|
+
}
|
|
799
|
+
})
|
|
800
|
+
})
|
package/src/preflight.ts
CHANGED
|
@@ -237,6 +237,159 @@ export function buildArchiveSidePlan(
|
|
|
237
237
|
return plan
|
|
238
238
|
}
|
|
239
239
|
|
|
240
|
+
// ── parseDeckCombos ──────────────────────────────────────────────────────
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* Parse `[combo.<name>] prompt = "..."` sections from a parsed deck TOML.
|
|
244
|
+
*
|
|
245
|
+
* Combos are orchestration playbooks for the AGENT — the CLI never executes
|
|
246
|
+
* them (deck consumption contract, packages/lythoskill-deck/skill/SKILL.md).
|
|
247
|
+
* Returns combo names in declaration order so generated orientation docs
|
|
248
|
+
* (AGENTS.md) can point the agent at them before anything else.
|
|
249
|
+
*
|
|
250
|
+
* Pure: Record → string[]. No IO.
|
|
251
|
+
*/
|
|
252
|
+
export function parseDeckCombos(
|
|
253
|
+
deckParsed: Record<string, any>
|
|
254
|
+
): string[] {
|
|
255
|
+
const combo = deckParsed?.combo
|
|
256
|
+
if (!combo || typeof combo !== 'object') return []
|
|
257
|
+
return Object.keys(combo).filter(k => k !== 'prompt')
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// ── Decision logs: per-cell naming + collect-time merge ──────────────────
|
|
261
|
+
//
|
|
262
|
+
// Why per-cell names (TASK-20260909010121918): the 2026-09-09 run had five cells
|
|
263
|
+
// per side sharing one workdir, each mandated to write `decision-log.jsonl`.
|
|
264
|
+
// Filename is a constant, so the cells shared a path — last writer won, and
|
|
265
|
+
// side-a kept only S1a's 9 entries. A per-cell filename removes the shared path
|
|
266
|
+
// entirely: no write contention, no ordering assumption, no atomicity needed.
|
|
267
|
+
// Rejected alternative (append-only read-then-append in one op) stays racy on
|
|
268
|
+
// any multi-step append and silently depends on every agent obeying it.
|
|
269
|
+
|
|
270
|
+
/** Legacy name — a workdir with exactly one decision-writing cell. */
|
|
271
|
+
export const LEGACY_DECISION_LOG = 'decision-log.jsonl'
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Cell id → its own decision-log filename.
|
|
275
|
+
*
|
|
276
|
+
* The id is slugged to `[A-Za-z0-9_-]`: no `/` or `\` (an id cannot write
|
|
277
|
+
* outside the workdir or into a subdirectory) and no `.` (the result can never
|
|
278
|
+
* be `.`/`..` or a hidden file). Empty/absent id → the legacy single-cell name.
|
|
279
|
+
*
|
|
280
|
+
* Pure: string → string. No IO.
|
|
281
|
+
*/
|
|
282
|
+
export function decisionLogName(cellId?: string): string {
|
|
283
|
+
if (!cellId) return LEGACY_DECISION_LOG
|
|
284
|
+
const slug = cellId.replace(/[^A-Za-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '')
|
|
285
|
+
return slug ? `decision-log-${slug}.jsonl` : LEGACY_DECISION_LOG
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** True for `decision-log.jsonl` and `decision-log-<cell-id>.jsonl` alike. */
|
|
289
|
+
export function isDecisionLogName(name: string): boolean {
|
|
290
|
+
return /^decision-log(-[^/]+)?\.jsonl$/.test(name)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** One cell's log, as read from disk. `cell` is the id used for merge order. */
|
|
294
|
+
export interface DecisionLogSource {
|
|
295
|
+
cell: string
|
|
296
|
+
content: string
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Merge per-cell decision logs into one JSONL stream.
|
|
301
|
+
*
|
|
302
|
+
* Lossless by construction: every non-empty line of every source is kept
|
|
303
|
+
* verbatim (no dedupe, no field injection, no JSON re-parse), sources ordered
|
|
304
|
+
* by cell id so the same inputs always produce the same bytes. The per-cell
|
|
305
|
+
* files are NOT consumed — they stay as provenance, since a merged line carries
|
|
306
|
+
* no cell attribution and `t` is cell-relative, not a wall clock.
|
|
307
|
+
*
|
|
308
|
+
* Pure: DecisionLogSource[] → string. No IO.
|
|
309
|
+
*/
|
|
310
|
+
export function mergeDecisionLogs(sources: DecisionLogSource[]): string {
|
|
311
|
+
const ordered = [...sources].sort((a, b) => (a.cell < b.cell ? -1 : a.cell > b.cell ? 1 : 0))
|
|
312
|
+
const lines: string[] = []
|
|
313
|
+
for (const source of ordered) {
|
|
314
|
+
for (const line of source.content.split('\n')) {
|
|
315
|
+
const trimmed = line.trim()
|
|
316
|
+
if (trimmed) lines.push(trimmed)
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return lines.length > 0 ? lines.join('\n') + '\n' : ''
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ── buildAgentsMd ────────────────────────────────────────────────────────
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Build the unified arena workdir AGENTS.md — the agent's orientation artifact.
|
|
326
|
+
*
|
|
327
|
+
* Single source of truth for the template; all three write sites use it:
|
|
328
|
+
* - prepare-workdir (buildPreparePlan)
|
|
329
|
+
* - single (cli.ts agent-run)
|
|
330
|
+
* - A/B runner cells (runner.ts)
|
|
331
|
+
*
|
|
332
|
+
* F1 (TASK-20260909152255103): combo sections are surfaced FIRST when present —
|
|
333
|
+
* they are the best orientation artifact in the deck. F3: first-timer
|
|
334
|
+
* vocabulary glossed here (the one tool-level place; the full glossary lives
|
|
335
|
+
* in the deck skill's references/glossary.md).
|
|
336
|
+
*
|
|
337
|
+
* Pure: strings → string. No IO.
|
|
338
|
+
*/
|
|
339
|
+
export function buildAgentsMd(params: {
|
|
340
|
+
mode: string
|
|
341
|
+
headerLines?: string[]
|
|
342
|
+
combos?: string[]
|
|
343
|
+
}): string {
|
|
344
|
+
const lines: string[] = [
|
|
345
|
+
'# Arena Test Environment',
|
|
346
|
+
`**Mode**: ${params.mode}`,
|
|
347
|
+
...(params.headerLines ?? []),
|
|
348
|
+
'',
|
|
349
|
+
]
|
|
350
|
+
|
|
351
|
+
if (params.combos && params.combos.length > 0) {
|
|
352
|
+
lines.push(
|
|
353
|
+
'## Combos — Read These First',
|
|
354
|
+
'This deck has combo prompts in `skill-deck.toml`. They are YOUR orchestration',
|
|
355
|
+
'playbook: each `[combo.<name>] prompt` is read and executed by YOU (the agent) —',
|
|
356
|
+
'the CLI only parses them; nothing runs them. Open `skill-deck.toml` and follow',
|
|
357
|
+
'each combo in order:',
|
|
358
|
+
...params.combos.map(name => `- \`${name}\``),
|
|
359
|
+
'',
|
|
360
|
+
)
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
lines.push(
|
|
364
|
+
'## Setup Order (why this sequence)',
|
|
365
|
+
'1. `skill-deck.toml` copied here → declares which skills you can use',
|
|
366
|
+
'2. `deck link` runs → cold pool skills become visible in the working set',
|
|
367
|
+
'3. Skill existence checked → warns if any declared skill is missing from cold pool',
|
|
368
|
+
'4. `AGENTS.md` written last → confirms setup succeeded before agent starts',
|
|
369
|
+
'If setup fails mid-sequence, the workdir is incomplete and nothing runs.',
|
|
370
|
+
'',
|
|
371
|
+
'## Vocabulary (first time here?)',
|
|
372
|
+
'- **cold pool** — where skill sources live on disk. You never read it directly.',
|
|
373
|
+
'- **working set** — `.claude/skills/`: the symlinks you actually see. `deck link` keeps it matching the deck.',
|
|
374
|
+
'- **innate** — always-on skills (governance, meta). **tool** — skills you load on demand.',
|
|
375
|
+
'- **max_cards** — hard budget in the toml; `deck link` refuses to exceed it.',
|
|
376
|
+
'',
|
|
377
|
+
'## How This Works',
|
|
378
|
+
'- Write ALL output files to this directory (CWD).',
|
|
379
|
+
'- Use available skills — check the working set directory (e.g. `ls .claude/skills/`).',
|
|
380
|
+
'',
|
|
381
|
+
'## Output Contract',
|
|
382
|
+
'- MANDATORY: `decision-log.jsonl` — one JSON line per decision:',
|
|
383
|
+
' `{"t":<seconds>,"phase":"setup|content|design|output","decision":"...","reason":"..."}`',
|
|
384
|
+
'- MANDATORY when other cells share this directory: name yours',
|
|
385
|
+
' `decision-log-<cell-id>.jsonl` (`decision-log-s1a.jsonl`) instead. Two cells on one',
|
|
386
|
+
' filename means last-writer-wins — one cell\'s whole trail is erased. Per-cell names',
|
|
387
|
+
' never collide, and the run merges them into one `decision-log.jsonl` when it collects.',
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
return lines.join('\n')
|
|
391
|
+
}
|
|
392
|
+
|
|
240
393
|
// ── buildPreparePlan ─────────────────────────────────────────────────────
|
|
241
394
|
|
|
242
395
|
/**
|
|
@@ -270,25 +423,10 @@ export function buildPreparePlan(params: {
|
|
|
270
423
|
const skills = parseDeckSkills(deckParsed)
|
|
271
424
|
const hasSkills = skills.length > 0
|
|
272
425
|
|
|
273
|
-
const agentsMd =
|
|
274
|
-
'
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
'## Setup Order (why this sequence)',
|
|
278
|
-
'1. `skill-deck.toml` copied here → declares which skills you can use',
|
|
279
|
-
'2. `deck link` runs → cold pool skills become visible in the working set',
|
|
280
|
-
'3. Skill existence checked → warns if any declared skill is missing from cold pool',
|
|
281
|
-
'4. `AGENTS.md` written last → confirms setup succeeded before agent starts',
|
|
282
|
-
'If setup fails mid-sequence, the workdir is incomplete and nothing runs.',
|
|
283
|
-
'',
|
|
284
|
-
'## How This Works',
|
|
285
|
-
'- Write ALL output files to this directory (CWD).',
|
|
286
|
-
'- Use available skills — check the working set directory (e.g. `ls .claude/skills/`).',
|
|
287
|
-
'',
|
|
288
|
-
'## Output Contract',
|
|
289
|
-
'- MANDATORY: `decision-log.jsonl` — one JSON line per decision:',
|
|
290
|
-
' `{"t":<seconds>,"phase":"setup|content|design|output","decision":"...","reason":"..."}`',
|
|
291
|
-
].join('\n')
|
|
426
|
+
const agentsMd = buildAgentsMd({
|
|
427
|
+
mode: 'agent-orchestrated cell',
|
|
428
|
+
combos: parseDeckCombos(deckParsed),
|
|
429
|
+
})
|
|
292
430
|
|
|
293
431
|
return { deckPath: params.deckPath, deckContent: params.deckContent, workDir: params.workDir, skills, hasSkills, agentsMd }
|
|
294
432
|
}
|
package/src/runner.test.ts
CHANGED
|
@@ -348,3 +348,181 @@ deck = "/tmp/deck.toml"
|
|
|
348
348
|
expect(logs.some(l => l.includes('deck link') && l.includes('exit 1'))).toBe(true)
|
|
349
349
|
})
|
|
350
350
|
})
|
|
351
|
+
|
|
352
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
353
|
+
// Per-cell decision logs (TASK-20260909010121918)
|
|
354
|
+
//
|
|
355
|
+
// Cells of one side share a workdir (`work/<side>/`). Before the fix, every
|
|
356
|
+
// cell was told to write the same `decision-log.jsonl` there, so a run with
|
|
357
|
+
// runs_per_side > 1 kept only the last cell's trail. Cell id → per-cell file
|
|
358
|
+
// name; the run merges them at collect time.
|
|
359
|
+
// ═══════════════════════════════════════════════════════════════════════════
|
|
360
|
+
|
|
361
|
+
describe('buildArenaPrompt — per-cell decision-log name', () => {
|
|
362
|
+
|
|
363
|
+
it('cellId → mandates the per-cell filename, not the shared one', () => {
|
|
364
|
+
const prompt = buildArenaPrompt({
|
|
365
|
+
brief: 'test',
|
|
366
|
+
cwd: '/tmp/arena-cell',
|
|
367
|
+
deckPath: '/tmp/test-deck.toml',
|
|
368
|
+
cellId: 'claude-run-1',
|
|
369
|
+
})
|
|
370
|
+
expect(prompt).toContain('MANDATORY — write decision-log-claude-run-1.jsonl')
|
|
371
|
+
expect(prompt).toContain('shared with other cells')
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
it('no cellId → legacy single-cell filename (arena single keeps its contract)', () => {
|
|
375
|
+
const prompt = buildArenaPrompt({
|
|
376
|
+
brief: 'test',
|
|
377
|
+
cwd: '/tmp/arena-cell',
|
|
378
|
+
deckPath: '/tmp/test-deck.toml',
|
|
379
|
+
})
|
|
380
|
+
expect(prompt).toContain('MANDATORY — write decision-log.jsonl')
|
|
381
|
+
expect(prompt).not.toContain('shared with other cells')
|
|
382
|
+
})
|
|
383
|
+
|
|
384
|
+
it('two cells of one side are told two different filenames', () => {
|
|
385
|
+
const base = { brief: 't', cwd: '/tmp/c', deckPath: '/tmp/d.toml' }
|
|
386
|
+
const a = buildArenaPrompt({ ...base, cellId: 'side-a-run-1' })
|
|
387
|
+
const b = buildArenaPrompt({ ...base, cellId: 'side-a-run-2' })
|
|
388
|
+
expect(a).toContain('decision-log-side-a-run-1.jsonl')
|
|
389
|
+
expect(b).toContain('decision-log-side-a-run-2.jsonl')
|
|
390
|
+
})
|
|
391
|
+
})
|
|
392
|
+
|
|
393
|
+
describe('runArenaFromToml — collect merges per-cell decision logs', () => {
|
|
394
|
+
|
|
395
|
+
const TOML_TWO_SIDES_TWO_RUNS = `
|
|
396
|
+
[arena]
|
|
397
|
+
task = "Write a hello world function"
|
|
398
|
+
criteria = ["completeness"]
|
|
399
|
+
runs_per_side = 2
|
|
400
|
+
|
|
401
|
+
[[side]]
|
|
402
|
+
name = "claude"
|
|
403
|
+
player = "claude"
|
|
404
|
+
deck = "/tmp/deck.toml"
|
|
405
|
+
|
|
406
|
+
[[side]]
|
|
407
|
+
name = "kimi"
|
|
408
|
+
player = "kimi"
|
|
409
|
+
deck = "/tmp/deck.toml"
|
|
410
|
+
`
|
|
411
|
+
|
|
412
|
+
it('merges every cell log into <artifactsDir>/decision-log.jsonl', async () => {
|
|
413
|
+
const files = new Map<string, string>()
|
|
414
|
+
const cellLogs = new Map<string, string>([
|
|
415
|
+
['/tmp/arena-out/work/claude/decision-log-claude-run-1.jsonl', '{"t":0,"phase":"setup","decision":"claude cell 1","reason":"r"}\n'],
|
|
416
|
+
['/tmp/arena-out/work/claude/decision-log-claude-run-2.jsonl', '{"t":0,"phase":"setup","decision":"claude cell 2","reason":"r"}\n'],
|
|
417
|
+
['/tmp/arena-out/work/kimi/decision-log-kimi-run-1.jsonl', '{"t":0,"phase":"setup","decision":"kimi cell 1","reason":"r"}\n'],
|
|
418
|
+
['/tmp/arena-out/work/kimi/decision-log-kimi-run-2.jsonl', '{"t":0,"phase":"setup","decision":"kimi cell 2","reason":"r"}\n'],
|
|
419
|
+
])
|
|
420
|
+
|
|
421
|
+
const mockIO: ArenaIO = {
|
|
422
|
+
log: () => {},
|
|
423
|
+
mkdir: () => {},
|
|
424
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
425
|
+
readFile: (path: string) => {
|
|
426
|
+
if (cellLogs.has(path)) return cellLogs.get(path)!
|
|
427
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
428
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
429
|
+
},
|
|
430
|
+
readdir: () => [],
|
|
431
|
+
cp: () => {},
|
|
432
|
+
spawn: async () => ({ exitCode: 0, stderr: '' }),
|
|
433
|
+
agentSpawn: async () => ({ stdout: 'ok', stderr: '', durationMs: 10 }),
|
|
434
|
+
exists: (path: string) => path === '/tmp/deck.toml' || cellLogs.has(path),
|
|
435
|
+
chdir: () => {},
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
await runArenaFromToml({
|
|
439
|
+
toml: parseArenaToml(TOML_TWO_SIDES_TWO_RUNS),
|
|
440
|
+
taskPath: '/tmp/task.md',
|
|
441
|
+
outDir: '/tmp/arena-out',
|
|
442
|
+
io: mockIO,
|
|
443
|
+
})
|
|
444
|
+
|
|
445
|
+
const merged = files.get('/tmp/arena-out/decision-log.jsonl')
|
|
446
|
+
expect(merged).toBeDefined()
|
|
447
|
+
// Every cell survives — the failure mode was losing all but one.
|
|
448
|
+
expect(merged).toContain('claude cell 1')
|
|
449
|
+
expect(merged).toContain('claude cell 2')
|
|
450
|
+
expect(merged).toContain('kimi cell 1')
|
|
451
|
+
expect(merged).toContain('kimi cell 2')
|
|
452
|
+
expect(merged!.trim().split('\n')).toHaveLength(4)
|
|
453
|
+
// Deterministic: ordered by cell id.
|
|
454
|
+
expect(merged).toBe(
|
|
455
|
+
'{"t":0,"phase":"setup","decision":"claude cell 1","reason":"r"}\n' +
|
|
456
|
+
'{"t":0,"phase":"setup","decision":"claude cell 2","reason":"r"}\n' +
|
|
457
|
+
'{"t":0,"phase":"setup","decision":"kimi cell 1","reason":"r"}\n' +
|
|
458
|
+
'{"t":0,"phase":"setup","decision":"kimi cell 2","reason":"r"}\n'
|
|
459
|
+
)
|
|
460
|
+
})
|
|
461
|
+
|
|
462
|
+
it('the prompt each cell actually receives names its own log file', async () => {
|
|
463
|
+
const prompts: string[] = []
|
|
464
|
+
const files = new Map<string, string>()
|
|
465
|
+
|
|
466
|
+
const mockIO: ArenaIO = {
|
|
467
|
+
log: () => {},
|
|
468
|
+
mkdir: () => {},
|
|
469
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
470
|
+
readFile: (path: string) => {
|
|
471
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
472
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
473
|
+
},
|
|
474
|
+
readdir: () => [],
|
|
475
|
+
cp: () => {},
|
|
476
|
+
spawn: async () => ({ exitCode: 0, stderr: '' }),
|
|
477
|
+
agentSpawn: async (opts: { brief: string }) => {
|
|
478
|
+
prompts.push(opts.brief)
|
|
479
|
+
return { stdout: '', stderr: '', durationMs: 1 }
|
|
480
|
+
},
|
|
481
|
+
exists: (path: string) => path === '/tmp/deck.toml',
|
|
482
|
+
chdir: () => {},
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
await runArenaFromToml({
|
|
486
|
+
toml: parseArenaToml(TOML_TWO_SIDES_TWO_RUNS),
|
|
487
|
+
taskPath: '/tmp/task.md',
|
|
488
|
+
outDir: '/tmp/arena-out',
|
|
489
|
+
io: mockIO,
|
|
490
|
+
})
|
|
491
|
+
|
|
492
|
+
// Two cells per side share ONE workdir — the prompts must not name one file.
|
|
493
|
+
expect(prompts).toHaveLength(4)
|
|
494
|
+
expect(prompts[0]).toContain('MANDATORY — write decision-log-claude-run-1.jsonl')
|
|
495
|
+
expect(prompts[1]).toContain('MANDATORY — write decision-log-claude-run-2.jsonl')
|
|
496
|
+
expect(prompts[2]).toContain('MANDATORY — write decision-log-kimi-run-1.jsonl')
|
|
497
|
+
expect(prompts[3]).toContain('MANDATORY — write decision-log-kimi-run-2.jsonl')
|
|
498
|
+
expect(new Set(prompts.map(p => p.match(/write (\S+\.jsonl)/)?.[1])).size).toBe(4)
|
|
499
|
+
})
|
|
500
|
+
|
|
501
|
+
it('no cell produced a log → no merged file invented', async () => {
|
|
502
|
+
const files = new Map<string, string>()
|
|
503
|
+
const mockIO: ArenaIO = {
|
|
504
|
+
log: () => {},
|
|
505
|
+
mkdir: () => {},
|
|
506
|
+
writeFile: (path: string, data: string) => { files.set(path, data) },
|
|
507
|
+
readFile: (path: string) => {
|
|
508
|
+
if (path === '/tmp/deck.toml') return 'skills = ["test"]'
|
|
509
|
+
throw new Error(`Unexpected read: ${path}`)
|
|
510
|
+
},
|
|
511
|
+
readdir: () => [],
|
|
512
|
+
cp: () => {},
|
|
513
|
+
spawn: async () => ({ exitCode: 0, stderr: '' }),
|
|
514
|
+
agentSpawn: async () => ({ stdout: '', stderr: '', durationMs: 1 }),
|
|
515
|
+
exists: (path: string) => path === '/tmp/deck.toml',
|
|
516
|
+
chdir: () => {},
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
await runArenaFromToml({
|
|
520
|
+
toml: parseArenaToml(TOML_TWO_SIDES_TWO_RUNS),
|
|
521
|
+
taskPath: '/tmp/task.md',
|
|
522
|
+
outDir: '/tmp/arena-out',
|
|
523
|
+
io: mockIO,
|
|
524
|
+
})
|
|
525
|
+
|
|
526
|
+
expect(files.has('/tmp/arena-out/decision-log.jsonl')).toBe(false)
|
|
527
|
+
})
|
|
528
|
+
})
|
package/src/runner.ts
CHANGED
|
@@ -12,11 +12,11 @@ try { await import('@lythos/agent-adapter-deepseek-serve') } catch { /* package
|
|
|
12
12
|
try { await import('@lythos/agent-adapter-deepseek-harness') } catch { /* package not installed */ }
|
|
13
13
|
try { await import('@lythos/agent-adapter-codex') } catch { /* package not installed */ }
|
|
14
14
|
import { runComparativeJudge } from './comparative-judge'
|
|
15
|
-
import { parseArenaToml, buildExecutionPlan, type ArenaToml, type ExecutionPlan } from './arena-toml'
|
|
15
|
+
import { parseArenaToml, buildExecutionPlan, type ArenaToml, type ExecutionPlan, type ExecutionCell } from './arena-toml'
|
|
16
16
|
import { resolvePlayer, resolveSides } from './player'
|
|
17
17
|
import { aggregateAllStats } from './stats'
|
|
18
18
|
import type { SideStats } from './stats'
|
|
19
|
-
import { buildCopyPlan } from './preflight'
|
|
19
|
+
import { buildCopyPlan, buildAgentsMd, parseDeckCombos, decisionLogName, mergeDecisionLogs, type DecisionLogSource } from './preflight'
|
|
20
20
|
|
|
21
21
|
// ── ArenaIO interface (Intent/Plan/Execute fractal pattern) ───────────────
|
|
22
22
|
|
|
@@ -99,8 +99,11 @@ export function buildArenaPrompt(opts: {
|
|
|
99
99
|
deckPath: string
|
|
100
100
|
outputDir?: string
|
|
101
101
|
preflightReport?: string
|
|
102
|
+
/** Cell id — when set, the mandated log is `decision-log-<id>.jsonl` (per-cell). */
|
|
103
|
+
cellId?: string
|
|
102
104
|
}): string {
|
|
103
105
|
const out = opts.outputDir ?? opts.cwd
|
|
106
|
+
const logName = decisionLogName(opts.cellId)
|
|
104
107
|
const lines = [
|
|
105
108
|
'You are running an arena evaluation cell.',
|
|
106
109
|
'',
|
|
@@ -108,7 +111,11 @@ export function buildArenaPrompt(opts: {
|
|
|
108
111
|
`Deck: ${opts.deckPath}`,
|
|
109
112
|
`Produce output to: ${out}/`,
|
|
110
113
|
'',
|
|
111
|
-
|
|
114
|
+
`MANDATORY — write ${logName} to the output directory.`,
|
|
115
|
+
...(opts.cellId
|
|
116
|
+
? ['That per-cell name is yours alone: the workdir is shared with other cells,',
|
|
117
|
+
'and a shared filename means the last writer erases everyone else\'s trail.']
|
|
118
|
+
: []),
|
|
112
119
|
'Each line is one JSON object with: t (seconds elapsed),',
|
|
113
120
|
'phase (setup/content/design/output), decision (what you chose),',
|
|
114
121
|
'reason (why). This is your decision trail — the only way the',
|
|
@@ -132,6 +139,18 @@ export function buildArenaPrompt(opts: {
|
|
|
132
139
|
return lines.join('\n')
|
|
133
140
|
}
|
|
134
141
|
|
|
142
|
+
// ── Cell identity ─────────────────────────────────────────────────────────
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Deterministic cell id for a planned cell.
|
|
146
|
+
*
|
|
147
|
+
* Cells of one side share a workdir (`work/<side>/`), so the id is what keeps
|
|
148
|
+
* their decision logs — and any other per-cell artifact — from sharing a path.
|
|
149
|
+
*/
|
|
150
|
+
export function cellIdOf(cell: ExecutionCell): string {
|
|
151
|
+
return `${cell.side}-run-${cell.run}`
|
|
152
|
+
}
|
|
153
|
+
|
|
135
154
|
// ── Plan formatting ───────────────────────────────────────────────────────
|
|
136
155
|
|
|
137
156
|
export function formatPlanOutput(plan: ExecutionPlan): string[] {
|
|
@@ -234,15 +253,15 @@ export async function runArenaFromToml(opts: {
|
|
|
234
253
|
|
|
235
254
|
try {
|
|
236
255
|
// Setup: deck + AGENTS.md + link
|
|
237
|
-
|
|
238
|
-
ioWithDefaults.writeFile(join(workDir, '
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
'
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
256
|
+
const cellDeckRaw = ioWithDefaults.readFile(cell.deck)
|
|
257
|
+
ioWithDefaults.writeFile(join(workDir, 'skill-deck.toml'), cellDeckRaw)
|
|
258
|
+
let cellDeckParsed: Record<string, any> = {}
|
|
259
|
+
try { cellDeckParsed = Bun.TOML.parse(cellDeckRaw) as Record<string, any> } catch {}
|
|
260
|
+
ioWithDefaults.writeFile(join(workDir, 'AGENTS.md'), buildAgentsMd({
|
|
261
|
+
mode: 'A/B cell',
|
|
262
|
+
headerLines: [`**Side**: ${cell.side}`, `**Player**: ${cell.player}`, `**Run**: ${cell.run}`],
|
|
263
|
+
combos: parseDeckCombos(cellDeckParsed),
|
|
264
|
+
}))
|
|
246
265
|
const linkProc = await ioWithDefaults.spawn(
|
|
247
266
|
['bunx', '@lythos/skill-deck', 'link'],
|
|
248
267
|
{ cwd: workDir, env: { ...process.env, HOME: process.env.HOME! } },
|
|
@@ -257,6 +276,7 @@ export async function runArenaFromToml(opts: {
|
|
|
257
276
|
cwd: workDir,
|
|
258
277
|
deckPath: cell.deck,
|
|
259
278
|
outputDir: workDir,
|
|
279
|
+
cellId: cellIdOf(cell),
|
|
260
280
|
})
|
|
261
281
|
const agentResult = await ioWithDefaults.agentSpawn({
|
|
262
282
|
player: resolvePlayer(cell.player),
|
|
@@ -334,6 +354,26 @@ export async function runArenaFromToml(opts: {
|
|
|
334
354
|
}
|
|
335
355
|
}
|
|
336
356
|
|
|
357
|
+
// ── Collect: merge per-cell decision logs into one decision-log.jsonl ───
|
|
358
|
+
// Cells sharing a side workdir each wrote their own file (see cellIdOf);
|
|
359
|
+
// the merged file is what a reader/judge opens, the per-cell files stay as
|
|
360
|
+
// provenance. Written only when at least one cell produced a log.
|
|
361
|
+
const logSources: DecisionLogSource[] = []
|
|
362
|
+
for (const cell of plan.cells) {
|
|
363
|
+
const logPath = join(artifactsDir, 'work', cell.side, decisionLogName(cellIdOf(cell)))
|
|
364
|
+
try {
|
|
365
|
+
if (ioWithDefaults.exists(logPath)) {
|
|
366
|
+
logSources.push({ cell: cellIdOf(cell), content: ioWithDefaults.readFile(logPath) })
|
|
367
|
+
}
|
|
368
|
+
} catch (e) {
|
|
369
|
+
io.log?.(`⚠️ Failed to read decision log for ${cellIdOf(cell)}: ${e instanceof Error ? e.message : e}`)
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
if (logSources.length > 0) {
|
|
373
|
+
ioWithDefaults.writeFile(join(artifactsDir, 'decision-log.jsonl'), mergeDecisionLogs(logSources))
|
|
374
|
+
io.log?.(`[arena] merged ${logSources.length} per-cell decision log(s) → ${artifactsDir}/decision-log.jsonl`)
|
|
375
|
+
}
|
|
376
|
+
|
|
337
377
|
// Aggregate + comparative
|
|
338
378
|
const stats = aggregateAllStats(verdictsBySide)
|
|
339
379
|
const flatVerdicts: { participantId: string; verdict: unknown }[] = []
|