@lythos/skill-arena 0.9.51 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/package.json +5 -4
- package/src/cli.ts +28 -10
package/README.md
CHANGED
|
@@ -49,20 +49,20 @@ Note: Claude `-p` mode has known issues with web tools in Bun.spawn (deferred to
|
|
|
49
49
|
```bash
|
|
50
50
|
bun add -d @lythos/skill-arena
|
|
51
51
|
# or use directly
|
|
52
|
-
bunx @lythos/skill-arena@0.
|
|
52
|
+
bunx @lythos/skill-arena@0.11.0 <command>
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
## Quick Start
|
|
56
56
|
|
|
57
57
|
```bash
|
|
58
58
|
# Single: test a deck with one agent
|
|
59
|
-
bunx @lythos/skill-arena@0.
|
|
59
|
+
bunx @lythos/skill-arena@0.11.0 single \
|
|
60
60
|
--deck https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml \
|
|
61
61
|
--brief "Generate auth flow diagram"
|
|
62
62
|
|
|
63
63
|
# Vs: compare multiple decks side by side
|
|
64
64
|
curl -fsSL https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/arena/research-compare/arena.toml > arena.toml
|
|
65
|
-
bunx @lythos/skill-arena@0.
|
|
65
|
+
bunx @lythos/skill-arena@0.11.0 vs --config ./arena.toml
|
|
66
66
|
```
|
|
67
67
|
|
|
68
68
|
## Commands
|
|
@@ -71,23 +71,23 @@ bunx @lythos/skill-arena@0.9.51 vs --config ./arena.toml
|
|
|
71
71
|
|
|
72
72
|
```bash
|
|
73
73
|
# Print execution plan without running
|
|
74
|
-
bunx @lythos/skill-arena@0.
|
|
74
|
+
bunx @lythos/skill-arena@0.11.0 vs --config arena.toml --dry-run
|
|
75
75
|
|
|
76
76
|
# Execute with per-side runs_per_side and statistical aggregation
|
|
77
|
-
bunx @lythos/skill-arena@0.
|
|
77
|
+
bunx @lythos/skill-arena@0.11.0 vs --config arena.toml
|
|
78
78
|
```
|
|
79
79
|
|
|
80
80
|
### Scaffold mode (legacy, manual execution)
|
|
81
81
|
|
|
82
82
|
```
|
|
83
|
-
bunx @lythos/skill-arena@0.
|
|
83
|
+
bunx @lythos/skill-arena@0.11.0 scaffold --task "Generate auth flow diagram" \
|
|
84
84
|
--decks https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/scout.toml,https://raw.githubusercontent.com/lythos-labs/lythoskill/main/examples/decks/documents.toml
|
|
85
85
|
```
|
|
86
86
|
|
|
87
87
|
### Viz
|
|
88
88
|
|
|
89
89
|
```bash
|
|
90
|
-
bunx @lythos/skill-arena@0.
|
|
90
|
+
bunx @lythos/skill-arena@0.11.0 viz runs/arena-<id>/
|
|
91
91
|
```
|
|
92
92
|
|
|
93
93
|
## Skill Documentation
|
|
@@ -101,7 +101,7 @@ The agent-visible **Skill** layer documentation is here:
|
|
|
101
101
|
Part of the [lythoskill](https://github.com/lythos-labs/lythoskill) ecosystem — the thin-skill pattern separates heavy logic (this npm package) from lightweight agent instructions (SKILL.md).
|
|
102
102
|
|
|
103
103
|
```
|
|
104
|
-
Starter (this package) → npm publish → bunx @lythos/skill-arena@0.
|
|
104
|
+
Starter (this package) → npm publish → bunx @lythos/skill-arena@0.11.0 ...
|
|
105
105
|
Skill (packages/<name>/skill/) → build → SKILL.md + thin scripts
|
|
106
106
|
Output (skills/<name>/) → git commit → agent-visible skill
|
|
107
107
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lythos/skill-arena",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"ai-agent",
|
|
@@ -42,12 +42,13 @@
|
|
|
42
42
|
"bun": ">=1.0.0"
|
|
43
43
|
},
|
|
44
44
|
"dependencies": {
|
|
45
|
-
"@lythos/cold-pool": "
|
|
46
|
-
"@lythos/
|
|
45
|
+
"@lythos/cold-pool": "workspace:*",
|
|
46
|
+
"@lythos/infra": "workspace:*",
|
|
47
|
+
"@lythos/test-utils": "workspace:*",
|
|
47
48
|
"zod": "^3.24.0",
|
|
48
49
|
"zod-to-json-schema": "^3.25.2"
|
|
49
50
|
},
|
|
50
51
|
"optionalDependencies": {
|
|
51
|
-
"@lythos/agent-adapter-claude-sdk": "
|
|
52
|
+
"@lythos/agent-adapter-claude-sdk": "workspace:*"
|
|
52
53
|
}
|
|
53
54
|
}
|
package/src/cli.ts
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
existsSync, mkdirSync, writeFileSync, readFileSync,
|
|
10
10
|
} from 'node:fs'
|
|
11
11
|
import { join, resolve, basename } from 'node:path'
|
|
12
|
+
import { fetchWithProxy } from '@lythos/infra'
|
|
12
13
|
import {
|
|
13
14
|
parseDeckSkills,
|
|
14
15
|
checkSkillExistence,
|
|
@@ -171,20 +172,37 @@ async function singleRun(args: string[]) {
|
|
|
171
172
|
if (u.hostname === 'github.com' && u.pathname.includes('/blob/')) {
|
|
172
173
|
url = `https://raw.githubusercontent.com${u.pathname.replace('/blob/', '/')}`
|
|
173
174
|
}
|
|
174
|
-
} catch
|
|
175
|
+
} catch (e: any) {
|
|
176
|
+
if (e.code !== 'ERR_INVALID_URL') console.debug(`deck URL parse skipped (not a URL): ${url}`)
|
|
177
|
+
}
|
|
178
|
+
const { mirrorUrls, isLikelyGitHubBlock } = await import('../../lythoskill-cold-pool/src/mirror.js')
|
|
175
179
|
const dest = resolve(process.cwd(), 'arena-deck.toml')
|
|
176
180
|
console.log(`📥 Fetching arena deck: ${url}`)
|
|
177
|
-
let res: Response
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
181
|
+
let res: Response | undefined
|
|
182
|
+
let allFailed = true
|
|
183
|
+
|
|
184
|
+
// Try direct first
|
|
185
|
+
try { res = await fetchWithProxy(url, { signal: AbortSignal.timeout(30_000) }); if (res.ok) allFailed = false } catch {}
|
|
186
|
+
|
|
187
|
+
// Auto-fallback: try mirrors when direct fails
|
|
188
|
+
if (!res?.ok) {
|
|
189
|
+
for (const mirrorUrl of mirrorUrls(url)) {
|
|
190
|
+
try {
|
|
191
|
+
console.log(` ↳ trying mirror: ${mirrorUrl}`)
|
|
192
|
+
const r = await fetchWithProxy(mirrorUrl, { signal: AbortSignal.timeout(30_000) })
|
|
193
|
+
if (r.ok) { res = r; allFailed = false; break }
|
|
194
|
+
} catch {}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
if (!res?.ok) {
|
|
199
|
+
const errorDetail = res ? `HTTP ${res.status}` : 'unreachable'
|
|
200
|
+
console.error(`❌ Cannot reach ${url} (${errorDetail})`)
|
|
201
|
+
if (allFailed) console.error(' All mirrors exhausted. Set LYTHOSKILL_GH_MIRROR to use a custom mirror.')
|
|
202
|
+
console.error(' Or download manually and reference the local file.')
|
|
183
203
|
process.exit(1)
|
|
184
204
|
}
|
|
185
|
-
|
|
186
|
-
Try a GitHub proxy mirror:
|
|
187
|
-
${url.replace('https://raw.githubusercontent.com/', 'https://ghfast.top/https://raw.githubusercontent.com/')}`); process.exit(1) }
|
|
205
|
+
|
|
188
206
|
deckWrite(dest, await res.text())
|
|
189
207
|
console.log(` → saved to ${dest}`)
|
|
190
208
|
deckPath = dest
|