@sverka/arena 0.1.28 → 0.1.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.d.mts +8 -0
- package/dist/bin.d.mts.map +1 -1
- package/dist/bin.mjs +40 -7
- package/dist/bin.mjs.map +1 -1
- package/package.json +4 -4
package/dist/bin.d.mts
CHANGED
|
@@ -7,6 +7,14 @@ interface Io {
|
|
|
7
7
|
out(s: string): void;
|
|
8
8
|
err(s: string): void;
|
|
9
9
|
}
|
|
10
|
+
/** Filter config tasks by --task ids; duplicate ids collapse to one. */
|
|
11
|
+
export declare function filterTasks<T extends {
|
|
12
|
+
id: string;
|
|
13
|
+
}>(tasks: T[], ids: string[]): {
|
|
14
|
+
tasks: T[];
|
|
15
|
+
} | {
|
|
16
|
+
missing: string[];
|
|
17
|
+
};
|
|
10
18
|
/** Entry point — exported for tests; bin calls it with process.argv. */
|
|
11
19
|
export declare function main(argv: readonly string[], io?: Io): Promise<number>;
|
|
12
20
|
//#endregion
|
package/dist/bin.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bin.d.mts","names":[],"sources":["../src/bin.ts"],"mappings":";;;;;UAiBU;EACR,IAAI;EACJ,IAAI;;;
|
|
1
|
+
{"version":3,"file":"bin.d.mts","names":[],"sources":["../src/bin.ts"],"mappings":";;;;;UAiBU;EACR,IAAI;EACJ,IAAI;;;wBA6HU,YAAY;EAAY;GACtC,OAAO,KACP;EACG,OAAO;;EAAU;;;wBA6FA,KACpB,yBACA,KAAI,KACH"}
|
package/dist/bin.mjs
CHANGED
|
@@ -12,7 +12,7 @@ import process from "node:process";
|
|
|
12
12
|
const USAGE = `usage: sverka-arena <command> [options]
|
|
13
13
|
|
|
14
14
|
commands:
|
|
15
|
-
run [--config <path>] [--out <dir>] [--format json|text]
|
|
15
|
+
run [--config <path>] [--out <dir>] [--format json|text] [--task <id>]...
|
|
16
16
|
report <results.json> [--format json|text|html] [--out <file.html>]
|
|
17
17
|
doctor [--config <path>] [--format json|text]
|
|
18
18
|
`;
|
|
@@ -31,6 +31,7 @@ function parseArgs(argv) {
|
|
|
31
31
|
let out;
|
|
32
32
|
let format = "text";
|
|
33
33
|
let command;
|
|
34
|
+
const tasks = [];
|
|
34
35
|
for (let i = 0; i < argv.length; i++) {
|
|
35
36
|
const arg = argv[i];
|
|
36
37
|
switch (arg) {
|
|
@@ -46,6 +47,10 @@ function parseArgs(argv) {
|
|
|
46
47
|
format = parseFormat(flagValue(argv, i));
|
|
47
48
|
i += 1;
|
|
48
49
|
break;
|
|
50
|
+
case "--task":
|
|
51
|
+
tasks.push(flagValue(argv, i));
|
|
52
|
+
i += 1;
|
|
53
|
+
break;
|
|
49
54
|
default:
|
|
50
55
|
if (arg.startsWith("--")) throw new ArenaError(`unknown option '${arg}'`, "CONFIG_INVALID");
|
|
51
56
|
if (command === void 0) command = arg;
|
|
@@ -57,7 +62,8 @@ function parseArgs(argv) {
|
|
|
57
62
|
positional,
|
|
58
63
|
config,
|
|
59
64
|
out,
|
|
60
|
-
format
|
|
65
|
+
format,
|
|
66
|
+
tasks
|
|
61
67
|
};
|
|
62
68
|
}
|
|
63
69
|
function which(bin) {
|
|
@@ -90,6 +96,21 @@ async function cmdReport(args, io) {
|
|
|
90
96
|
} else io.out(renderReport(result) + "\n");
|
|
91
97
|
return 0;
|
|
92
98
|
}
|
|
99
|
+
/** Filter config tasks by --task ids; duplicate ids collapse to one. */
|
|
100
|
+
function filterTasks(tasks, ids) {
|
|
101
|
+
const seen = /* @__PURE__ */ new Set();
|
|
102
|
+
const uniq = tasks.filter((t) => {
|
|
103
|
+
if (seen.has(t.id)) return false;
|
|
104
|
+
seen.add(t.id);
|
|
105
|
+
return true;
|
|
106
|
+
});
|
|
107
|
+
if (ids.length === 0) return { tasks: uniq };
|
|
108
|
+
const known = new Set(uniq.map((t) => t.id));
|
|
109
|
+
const missing = ids.filter((id) => !known.has(id));
|
|
110
|
+
if (missing.length > 0) return { missing };
|
|
111
|
+
const wanted = new Set(ids);
|
|
112
|
+
return { tasks: uniq.filter((t) => wanted.has(t.id)) };
|
|
113
|
+
}
|
|
93
114
|
async function cmdRun(args, io) {
|
|
94
115
|
if (args.format === "html") {
|
|
95
116
|
io.err("run: --format html is only supported by 'report'\n");
|
|
@@ -97,16 +118,23 @@ async function cmdRun(args, io) {
|
|
|
97
118
|
}
|
|
98
119
|
const config = await loadArenaConfig(args.config);
|
|
99
120
|
if (args.out !== void 0) config.outputDir = resolve(args.out);
|
|
121
|
+
const filtered = filterTasks(config.tasks, args.tasks);
|
|
122
|
+
if ("missing" in filtered) {
|
|
123
|
+
io.err(`run: unknown --task '${filtered.missing.join("', '")}' (config defines: ${[...new Set(config.tasks.map((t) => t.id))].join(", ")})\n`);
|
|
124
|
+
return 2;
|
|
125
|
+
}
|
|
126
|
+
config.tasks = filtered.tasks;
|
|
100
127
|
const result = await runArena(config);
|
|
101
128
|
if (args.format === "json") io.out(JSON.stringify(result, null, 2) + "\n");
|
|
102
129
|
else io.out(renderReport(result) + "\n");
|
|
103
130
|
return 0;
|
|
104
131
|
}
|
|
105
132
|
function doctorChecks(config) {
|
|
133
|
+
const agentOnPath = which(config.agent.id);
|
|
106
134
|
const checks = [{
|
|
107
135
|
name: `agent binary: ${config.agent.id}`,
|
|
108
|
-
ok:
|
|
109
|
-
detail:
|
|
136
|
+
ok: agentOnPath,
|
|
137
|
+
detail: agentOnPath ? "on PATH" : "not found on PATH"
|
|
110
138
|
}];
|
|
111
139
|
const models = config.judge ? [...config.models, config.judge.model] : config.models;
|
|
112
140
|
for (const m of models) {
|
|
@@ -150,8 +178,13 @@ async function main(argv, io = defaultIo) {
|
|
|
150
178
|
try {
|
|
151
179
|
switch (args.command) {
|
|
152
180
|
case "run": return await cmdRun(args, io);
|
|
153
|
-
case "report":
|
|
154
|
-
case "doctor":
|
|
181
|
+
case "report":
|
|
182
|
+
case "doctor":
|
|
183
|
+
if (args.tasks.length > 0) {
|
|
184
|
+
io.err(`${args.command}: --task is only supported by 'run'\n`);
|
|
185
|
+
return 2;
|
|
186
|
+
}
|
|
187
|
+
return args.command === "report" ? await cmdReport(args, io) : await cmdDoctor(args, io);
|
|
155
188
|
default:
|
|
156
189
|
io.err(`unknown command '${args.command}'\n`);
|
|
157
190
|
io.err(USAGE);
|
|
@@ -171,6 +204,6 @@ if (import.meta.url === `file://${process.argv[1]}`) {
|
|
|
171
204
|
process.exit(code);
|
|
172
205
|
}
|
|
173
206
|
//#endregion
|
|
174
|
-
export { main };
|
|
207
|
+
export { filterTasks, main };
|
|
175
208
|
|
|
176
209
|
//# sourceMappingURL=bin.mjs.map
|
package/dist/bin.mjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * sverka-arena — CLI for the agent arena: run the benchmark matrix, render\n * saved results, check the environment. Spec: specs/49-arena-cli.\n */\n\nimport process from \"node:process\";\nimport { readFile } from \"node:fs/promises\";\nimport { existsSync } from \"node:fs\";\nimport { delimiter, join, resolve } from \"node:path\";\n\nimport type { ArenaResult } from \"./types.js\";\nimport { ArenaError, loadArenaConfig } from \"./config.js\";\nimport { renderReport } from \"./report.js\";\nimport { writeAggregateReport } from \"./aggregate-report.js\";\nimport { runArena } from \"./runner.js\";\n\ninterface Io {\n out(s: string): void;\n err(s: string): void;\n}\n\ntype Format = \"text\" | \"json\" | \"html\";\n\ninterface ParsedArgs {\n command: string | undefined;\n positional: string[];\n config: string;\n out: string | undefined;\n format: Format;\n}\n\nconst USAGE = `usage: sverka-arena <command> [options]\n\ncommands:\n run [--config <path>] [--out <dir>] [--format json|text]\n report <results.json> [--format json|text|html] [--out <file.html>]\n doctor [--config <path>] [--format json|text]\n`;\n\nfunction flagValue(argv: readonly string[], i: number): string {\n const val = argv[i + 1];\n if (val === undefined || val.startsWith(\"--\")) {\n throw new ArenaError(`${argv[i]} requires a value`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseFormat(val: string): Format {\n if (val !== \"json\" && val !== \"text\" && val !== \"html\") {\n throw new ArenaError(`invalid --format '${val}'`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseArgs(argv: readonly string[]): ParsedArgs {\n const positional: string[] = [];\n let config = \"arena.config.ts\";\n let out: string | undefined;\n let format: \"text\" | \"json\" | \"html\" = \"text\";\n let command: string | undefined;\n for (let i = 0; i < argv.length; i++) {\n const arg = argv[i]!;\n switch (arg) {\n case \"--config\":\n config = flagValue(argv, i);\n i += 1;\n break;\n case \"--out\":\n out = flagValue(argv, i);\n i += 1;\n break;\n case \"--format\":\n format = parseFormat(flagValue(argv, i));\n i += 1;\n break;\n default:\n if (arg.startsWith(\"--\")) {\n throw new ArenaError(`unknown option '${arg}'`, \"CONFIG_INVALID\");\n }\n if (command === undefined) command = arg;\n else positional.push(arg);\n }\n }\n return { command, positional, config, out, format };\n}\n\nfunction which(bin: string): boolean {\n const exts =\n process.platform === \"win32\"\n ? [\"\", ...(process.env.PATHEXT ?? \".EXE;.CMD;.BAT\").split(\";\")]\n : [\"\"];\n for (const dir of (process.env.PATH ?? \"\").split(delimiter)) {\n if (exts.some((ext) => existsSync(join(dir, bin + ext)))) return true;\n }\n return false;\n}\n\nasync function cmdReport(args: ParsedArgs, io: Io): Promise<number> {\n const file = args.positional[0];\n if (file === undefined) {\n io.err(\"report: missing <results.json> path\");\n return 2;\n }\n let result: ArenaResult;\n try {\n result = JSON.parse(await readFile(file, \"utf8\")) as ArenaResult;\n } catch (err) {\n io.err(\n `report: cannot read '${file}': ${err instanceof Error ? err.message : String(err)}`,\n );\n return 2;\n }\n if (\n !Array.isArray(result.aggregates) ||\n !Array.isArray(result.analysis) ||\n !Array.isArray(result.results) ||\n result.config === undefined ||\n !Array.isArray(result.config.models)\n ) {\n io.err(`report: '${file}' is not a sverka-arena results file\\n`);\n return 2;\n }\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else if (args.format === \"html\") {\n const out = args.out ?? \"arena-report.html\";\n writeAggregateReport(result, resolve(out), {\n command: `sverka-arena report ${file} --format html`,\n });\n io.out(`report written to ${out}\\n`);\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\nasync function cmdRun(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"run: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const config = await loadArenaConfig(args.config);\n if (args.out !== undefined) config.outputDir = resolve(args.out);\n const result = await runArena(config);\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\ninterface DoctorCheck {\n name: string;\n ok: boolean;\n detail: string;\n}\n\nfunction doctorChecks(\n config: Awaited<ReturnType<typeof loadArenaConfig>>,\n): DoctorCheck[] {\n const checks: DoctorCheck[] = [\n {\n name: `agent binary: ${config.agent.id}`,\n ok: which(config.agent.id),\n detail: which(config.agent.id) ? \"on PATH\" : \"not found on PATH\",\n },\n ];\n const models = config.judge\n ? [...config.models, config.judge.model]\n : config.models;\n for (const m of models) {\n if (m.envVar === undefined) continue;\n checks.push({\n name: `env: ${m.envVar} (model ${m.id})`,\n ok: process.env[m.envVar] !== undefined,\n detail: process.env[m.envVar] !== undefined ? \"set\" : \"not set\",\n });\n }\n return checks;\n}\n\nasync function cmdDoctor(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"doctor: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const checks = doctorChecks(await loadArenaConfig(args.config));\n\n if (args.format === \"json\") {\n io.out(JSON.stringify({ checks }, null, 2) + \"\\n\");\n } else {\n for (const chk of checks) {\n io.out(`${chk.ok ? \"ok\" : \"FAIL\"} ${chk.name} — ${chk.detail}\\n`);\n }\n }\n return checks.every((chk) => chk.ok) ? 0 : 1;\n}\n\nconst defaultIo: Io = {\n out: (s) => process.stdout.write(s),\n err: (s) => process.stderr.write(s),\n};\n\n/** Entry point — exported for tests; bin calls it with process.argv. */\nexport async function main(\n argv: readonly string[],\n io: Io = defaultIo,\n): Promise<number> {\n let args: ParsedArgs;\n try {\n args = parseArgs(argv);\n } catch (err) {\n io.err((err instanceof Error ? err.message : String(err)) + \"\\n\");\n io.err(USAGE);\n return 2;\n }\n\n if (args.command === undefined) {\n io.err(USAGE);\n return 2;\n }\n\n try {\n switch (args.command) {\n case \"run\":\n return await cmdRun(args, io);\n case \"report\":\n return await cmdReport(args, io);\n case \"doctor\":\n return await cmdDoctor(args, io);\n default:\n io.err(`unknown command '${args.command}'\\n`);\n io.err(USAGE);\n return 2;\n }\n } catch (err) {\n if (err instanceof ArenaError) {\n io.err(`error: ${err.message}\\n`);\n return 2;\n }\n io.err(`error: ${err instanceof Error ? err.message : String(err)}\\n`);\n return 3;\n }\n}\n\n// Run when invoked directly (not imported by tests).\nif (import.meta.url === `file://${process.argv[1]}`) {\n const code = await main(process.argv.slice(2));\n process.exit(code);\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAM,QAAQ;;;;;;;AAQd,SAAS,UAAU,MAAyB,GAAmB;CAC7D,MAAM,MAAM,KAAK,IAAI;CACrB,IAAI,QAAQ,KAAA,KAAa,IAAI,WAAW,IAAI,GAC1C,MAAM,IAAI,WAAW,GAAG,KAAK,GAAG,oBAAoB,gBAAgB;CAEtE,OAAO;AACT;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,QAAQ,UAAU,QAAQ,UAAU,QAAQ,QAC9C,MAAM,IAAI,WAAW,qBAAqB,IAAI,IAAI,gBAAgB;CAEpE,OAAO;AACT;AAEA,SAAS,UAAU,MAAqC;CACtD,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI;CACJ,IAAI,SAAmC;CACvC,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,QAAQ,KAAR;GACE,KAAK;IACH,SAAS,UAAU,MAAM,CAAC;IAC1B,KAAK;IACL;GACF,KAAK;IACH,MAAM,UAAU,MAAM,CAAC;IACvB,KAAK;IACL;GACF,KAAK;IACH,SAAS,YAAY,UAAU,MAAM,CAAC,CAAC;IACvC,KAAK;IACL;GACF;IACE,IAAI,IAAI,WAAW,IAAI,GACrB,MAAM,IAAI,WAAW,mBAAmB,IAAI,IAAI,gBAAgB;IAElE,IAAI,YAAY,KAAA,GAAW,UAAU;SAChC,WAAW,KAAK,GAAG;EAC5B;CACF;CACA,OAAO;EAAE;EAAS;EAAY;EAAQ;EAAK;CAAO;AACpD;AAEA,SAAS,MAAM,KAAsB;CACnC,MAAM,OACJ,QAAQ,aAAa,UACjB,CAAC,IAAI,IAAI,QAAQ,IAAI,WAAW,iBAAA,CAAkB,MAAM,GAAG,CAAC,IAC5D,CAAC,EAAE;CACT,KAAK,MAAM,QAAQ,QAAQ,IAAI,QAAQ,GAAA,CAAI,MAAM,SAAS,GACxD,IAAI,KAAK,MAAM,QAAQ,WAAW,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO;CAEnE,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,MAAM,OAAO,KAAK,WAAW;CAC7B,IAAI,SAAS,KAAA,GAAW;EACtB,GAAG,IAAI,qCAAqC;EAC5C,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;CAClD,SAAS,KAAK;EACZ,GAAG,IACD,wBAAwB,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GACnF;EACA,OAAO;CACT;CACA,IACE,CAAC,MAAM,QAAQ,OAAO,UAAU,KAChC,CAAC,MAAM,QAAQ,OAAO,QAAQ,KAC9B,CAAC,MAAM,QAAQ,OAAO,OAAO,KAC7B,OAAO,WAAW,KAAA,KAClB,CAAC,MAAM,QAAQ,OAAO,OAAO,MAAM,GACnC;EACA,GAAG,IAAI,YAAY,KAAK,uCAAuC;EAC/D,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MACxC,IAAI,KAAK,WAAW,QAAQ;EACjC,MAAM,MAAM,KAAK,OAAO;EACxB,qBAAqB,QAAQ,QAAQ,GAAG,GAAG,EACzC,SAAS,uBAAuB,KAAK,gBACvC,CAAC;EACD,GAAG,IAAI,qBAAqB,IAAI,GAAG;CACrC,OACE,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAEA,eAAe,OAAO,MAAkB,IAAyB;CAC/D,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,oDAAoD;EAC3D,OAAO;CACT;CACA,MAAM,SAAS,MAAM,gBAAgB,KAAK,MAAM;CAChD,IAAI,KAAK,QAAQ,KAAA,GAAW,OAAO,YAAY,QAAQ,KAAK,GAAG;CAC/D,MAAM,SAAS,MAAM,SAAS,MAAM;CACpC,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MAE7C,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAQA,SAAS,aACP,QACe;CACf,MAAM,SAAwB,CAC5B;EACE,MAAM,iBAAiB,OAAO,MAAM;EACpC,IAAI,MAAM,OAAO,MAAM,EAAE;EACzB,QAAQ,MAAM,OAAO,MAAM,EAAE,IAAI,YAAY;CAC/C,CACF;CACA,MAAM,SAAS,OAAO,QAClB,CAAC,GAAG,OAAO,QAAQ,OAAO,MAAM,KAAK,IACrC,OAAO;CACX,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,EAAE,WAAW,KAAA,GAAW;EAC5B,OAAO,KAAK;GACV,MAAM,QAAQ,EAAE,OAAO,UAAU,EAAE,GAAG;GACtC,IAAI,QAAQ,IAAI,EAAE,YAAY,KAAA;GAC9B,QAAQ,QAAQ,IAAI,EAAE,YAAY,KAAA,IAAY,QAAQ;EACxD,CAAC;CACH;CACA,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,uDAAuD;EAC9D,OAAO;CACT;CACA,MAAM,SAAS,aAAa,MAAM,gBAAgB,KAAK,MAAM,CAAC;CAE9D,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI;MAEjD,KAAK,MAAM,OAAO,QAChB,GAAG,IAAI,GAAG,IAAI,KAAK,OAAO,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,OAAO,GAAG;CAGrE,OAAO,OAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,IAAI;AAC7C;AAEA,MAAM,YAAgB;CACpB,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;CAClC,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;AACpC;;AAGA,eAAsB,KACpB,MACA,KAAS,WACQ;CACjB,IAAI;CACJ,IAAI;EACF,OAAO,UAAU,IAAI;CACvB,SAAS,KAAK;EACZ,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAAK,IAAI;EAChE,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI,KAAK,YAAY,KAAA,GAAW;EAC9B,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI;EACF,QAAQ,KAAK,SAAb;GACE,KAAK,OACH,OAAO,MAAM,OAAO,MAAM,EAAE;GAC9B,KAAK,UACH,OAAO,MAAM,UAAU,MAAM,EAAE;GACjC,KAAK,UACH,OAAO,MAAM,UAAU,MAAM,EAAE;GACjC;IACE,GAAG,IAAI,oBAAoB,KAAK,QAAQ,IAAI;IAC5C,GAAG,IAAI,KAAK;IACZ,OAAO;EACX;CACF,SAAS,KAAK;EACZ,IAAI,eAAe,YAAY;GAC7B,GAAG,IAAI,UAAU,IAAI,QAAQ,GAAG;GAChC,OAAO;EACT;EACA,GAAG,IAAI,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,GAAG;EACrE,OAAO;CACT;AACF;AAGA,IAAI,YAAY,QAAQ,UAAU,QAAQ,KAAK,MAAM;CACnD,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7C,QAAQ,KAAK,IAAI;AACnB"}
|
|
1
|
+
{"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * sverka-arena — CLI for the agent arena: run the benchmark matrix, render\n * saved results, check the environment. Spec: specs/49-arena-cli.\n */\n\nimport process from \"node:process\";\nimport { readFile } from \"node:fs/promises\";\nimport { existsSync } from \"node:fs\";\nimport { delimiter, join, resolve } from \"node:path\";\n\nimport type { ArenaResult } from \"./types.js\";\nimport { ArenaError, loadArenaConfig } from \"./config.js\";\nimport { renderReport } from \"./report.js\";\nimport { writeAggregateReport } from \"./aggregate-report.js\";\nimport { runArena } from \"./runner.js\";\n\ninterface Io {\n out(s: string): void;\n err(s: string): void;\n}\n\ntype Format = \"text\" | \"json\" | \"html\";\n\ninterface ParsedArgs {\n command: string | undefined;\n positional: string[];\n config: string;\n out: string | undefined;\n format: Format;\n tasks: string[];\n}\n\nconst USAGE = `usage: sverka-arena <command> [options]\n\ncommands:\n run [--config <path>] [--out <dir>] [--format json|text] [--task <id>]...\n report <results.json> [--format json|text|html] [--out <file.html>]\n doctor [--config <path>] [--format json|text]\n`;\n\nfunction flagValue(argv: readonly string[], i: number): string {\n const val = argv[i + 1];\n if (val === undefined || val.startsWith(\"--\")) {\n throw new ArenaError(`${argv[i]} requires a value`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseFormat(val: string): Format {\n if (val !== \"json\" && val !== \"text\" && val !== \"html\") {\n throw new ArenaError(`invalid --format '${val}'`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseArgs(argv: readonly string[]): ParsedArgs {\n const positional: string[] = [];\n let config = \"arena.config.ts\";\n let out: string | undefined;\n let format: \"text\" | \"json\" | \"html\" = \"text\";\n let command: string | undefined;\n const tasks: string[] = [];\n for (let i = 0; i < argv.length; i++) {\n const arg = argv[i]!;\n switch (arg) {\n case \"--config\":\n config = flagValue(argv, i);\n i += 1;\n break;\n case \"--out\":\n out = flagValue(argv, i);\n i += 1;\n break;\n case \"--format\":\n format = parseFormat(flagValue(argv, i));\n i += 1;\n break;\n case \"--task\":\n tasks.push(flagValue(argv, i));\n i += 1;\n break;\n default:\n if (arg.startsWith(\"--\")) {\n throw new ArenaError(`unknown option '${arg}'`, \"CONFIG_INVALID\");\n }\n if (command === undefined) command = arg;\n else positional.push(arg);\n }\n }\n return { command, positional, config, out, format, tasks };\n}\n\nfunction which(bin: string): boolean {\n const exts =\n process.platform === \"win32\"\n ? [\"\", ...(process.env.PATHEXT ?? \".EXE;.CMD;.BAT\").split(\";\")]\n : [\"\"];\n for (const dir of (process.env.PATH ?? \"\").split(delimiter)) {\n if (exts.some((ext) => existsSync(join(dir, bin + ext)))) return true;\n }\n return false;\n}\n\nasync function cmdReport(args: ParsedArgs, io: Io): Promise<number> {\n const file = args.positional[0];\n if (file === undefined) {\n io.err(\"report: missing <results.json> path\");\n return 2;\n }\n let result: ArenaResult;\n try {\n result = JSON.parse(await readFile(file, \"utf8\")) as ArenaResult;\n } catch (err) {\n io.err(\n `report: cannot read '${file}': ${err instanceof Error ? err.message : String(err)}`,\n );\n return 2;\n }\n if (\n !Array.isArray(result.aggregates) ||\n !Array.isArray(result.analysis) ||\n !Array.isArray(result.results) ||\n result.config === undefined ||\n !Array.isArray(result.config.models)\n ) {\n io.err(`report: '${file}' is not a sverka-arena results file\\n`);\n return 2;\n }\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else if (args.format === \"html\") {\n const out = args.out ?? \"arena-report.html\";\n writeAggregateReport(result, resolve(out), {\n command: `sverka-arena report ${file} --format html`,\n });\n io.out(`report written to ${out}\\n`);\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\n/** Filter config tasks by --task ids; duplicate ids collapse to one. */\nexport function filterTasks<T extends { id: string }>(\n tasks: T[],\n ids: string[],\n): { tasks: T[] } | { missing: string[] } {\n const seen = new Set<string>();\n const uniq = tasks.filter((t) => {\n if (seen.has(t.id)) return false;\n seen.add(t.id);\n return true;\n });\n if (ids.length === 0) return { tasks: uniq };\n const known = new Set(uniq.map((t) => t.id));\n const missing = ids.filter((id) => !known.has(id));\n if (missing.length > 0) return { missing };\n const wanted = new Set(ids);\n return { tasks: uniq.filter((t) => wanted.has(t.id)) };\n}\n\nasync function cmdRun(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"run: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const config = await loadArenaConfig(args.config);\n if (args.out !== undefined) config.outputDir = resolve(args.out);\n const filtered = filterTasks(config.tasks, args.tasks);\n if (\"missing\" in filtered) {\n io.err(\n `run: unknown --task '${filtered.missing.join(\"', '\")}' (config defines: ${[...new Set(config.tasks.map((t) => t.id))].join(\", \")})\\n`,\n );\n return 2;\n }\n config.tasks = filtered.tasks;\n const result = await runArena(config);\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\ninterface DoctorCheck {\n name: string;\n ok: boolean;\n detail: string;\n}\n\nfunction doctorChecks(\n config: Awaited<ReturnType<typeof loadArenaConfig>>,\n): DoctorCheck[] {\n const agentOnPath = which(config.agent.id);\n const checks: DoctorCheck[] = [\n {\n name: `agent binary: ${config.agent.id}`,\n ok: agentOnPath,\n detail: agentOnPath ? \"on PATH\" : \"not found on PATH\",\n },\n ];\n const models = config.judge\n ? [...config.models, config.judge.model]\n : config.models;\n for (const m of models) {\n if (m.envVar === undefined) continue;\n checks.push({\n name: `env: ${m.envVar} (model ${m.id})`,\n ok: process.env[m.envVar] !== undefined,\n detail: process.env[m.envVar] !== undefined ? \"set\" : \"not set\",\n });\n }\n return checks;\n}\n\nasync function cmdDoctor(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"doctor: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const checks = doctorChecks(await loadArenaConfig(args.config));\n\n if (args.format === \"json\") {\n io.out(JSON.stringify({ checks }, null, 2) + \"\\n\");\n } else {\n for (const chk of checks) {\n io.out(`${chk.ok ? \"ok\" : \"FAIL\"} ${chk.name} — ${chk.detail}\\n`);\n }\n }\n return checks.every((chk) => chk.ok) ? 0 : 1;\n}\n\nconst defaultIo: Io = {\n out: (s) => process.stdout.write(s),\n err: (s) => process.stderr.write(s),\n};\n\n/** Entry point — exported for tests; bin calls it with process.argv. */\nexport async function main(\n argv: readonly string[],\n io: Io = defaultIo,\n): Promise<number> {\n let args: ParsedArgs;\n try {\n args = parseArgs(argv);\n } catch (err) {\n io.err((err instanceof Error ? err.message : String(err)) + \"\\n\");\n io.err(USAGE);\n return 2;\n }\n\n if (args.command === undefined) {\n io.err(USAGE);\n return 2;\n }\n\n try {\n switch (args.command) {\n case \"run\":\n return await cmdRun(args, io);\n case \"report\":\n case \"doctor\":\n if (args.tasks.length > 0) {\n io.err(`${args.command}: --task is only supported by 'run'\\n`);\n return 2;\n }\n return args.command === \"report\"\n ? await cmdReport(args, io)\n : await cmdDoctor(args, io);\n default:\n io.err(`unknown command '${args.command}'\\n`);\n io.err(USAGE);\n return 2;\n }\n } catch (err) {\n if (err instanceof ArenaError) {\n io.err(`error: ${err.message}\\n`);\n return 2;\n }\n io.err(`error: ${err instanceof Error ? err.message : String(err)}\\n`);\n return 3;\n }\n}\n\n// Run when invoked directly (not imported by tests).\nif (import.meta.url === `file://${process.argv[1]}`) {\n const code = await main(process.argv.slice(2));\n process.exit(code);\n}\n"],"mappings":";;;;;;;;;;;AAiCA,MAAM,QAAQ;;;;;;;AAQd,SAAS,UAAU,MAAyB,GAAmB;CAC7D,MAAM,MAAM,KAAK,IAAI;CACrB,IAAI,QAAQ,KAAA,KAAa,IAAI,WAAW,IAAI,GAC1C,MAAM,IAAI,WAAW,GAAG,KAAK,GAAG,oBAAoB,gBAAgB;CAEtE,OAAO;AACT;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,QAAQ,UAAU,QAAQ,UAAU,QAAQ,QAC9C,MAAM,IAAI,WAAW,qBAAqB,IAAI,IAAI,gBAAgB;CAEpE,OAAO;AACT;AAEA,SAAS,UAAU,MAAqC;CACtD,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI;CACJ,IAAI,SAAmC;CACvC,IAAI;CACJ,MAAM,QAAkB,CAAC;CACzB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,QAAQ,KAAR;GACE,KAAK;IACH,SAAS,UAAU,MAAM,CAAC;IAC1B,KAAK;IACL;GACF,KAAK;IACH,MAAM,UAAU,MAAM,CAAC;IACvB,KAAK;IACL;GACF,KAAK;IACH,SAAS,YAAY,UAAU,MAAM,CAAC,CAAC;IACvC,KAAK;IACL;GACF,KAAK;IACH,MAAM,KAAK,UAAU,MAAM,CAAC,CAAC;IAC7B,KAAK;IACL;GACF;IACE,IAAI,IAAI,WAAW,IAAI,GACrB,MAAM,IAAI,WAAW,mBAAmB,IAAI,IAAI,gBAAgB;IAElE,IAAI,YAAY,KAAA,GAAW,UAAU;SAChC,WAAW,KAAK,GAAG;EAC5B;CACF;CACA,OAAO;EAAE;EAAS;EAAY;EAAQ;EAAK;EAAQ;CAAM;AAC3D;AAEA,SAAS,MAAM,KAAsB;CACnC,MAAM,OACJ,QAAQ,aAAa,UACjB,CAAC,IAAI,IAAI,QAAQ,IAAI,WAAW,iBAAA,CAAkB,MAAM,GAAG,CAAC,IAC5D,CAAC,EAAE;CACT,KAAK,MAAM,QAAQ,QAAQ,IAAI,QAAQ,GAAA,CAAI,MAAM,SAAS,GACxD,IAAI,KAAK,MAAM,QAAQ,WAAW,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO;CAEnE,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,MAAM,OAAO,KAAK,WAAW;CAC7B,IAAI,SAAS,KAAA,GAAW;EACtB,GAAG,IAAI,qCAAqC;EAC5C,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;CAClD,SAAS,KAAK;EACZ,GAAG,IACD,wBAAwB,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GACnF;EACA,OAAO;CACT;CACA,IACE,CAAC,MAAM,QAAQ,OAAO,UAAU,KAChC,CAAC,MAAM,QAAQ,OAAO,QAAQ,KAC9B,CAAC,MAAM,QAAQ,OAAO,OAAO,KAC7B,OAAO,WAAW,KAAA,KAClB,CAAC,MAAM,QAAQ,OAAO,OAAO,MAAM,GACnC;EACA,GAAG,IAAI,YAAY,KAAK,uCAAuC;EAC/D,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MACxC,IAAI,KAAK,WAAW,QAAQ;EACjC,MAAM,MAAM,KAAK,OAAO;EACxB,qBAAqB,QAAQ,QAAQ,GAAG,GAAG,EACzC,SAAS,uBAAuB,KAAK,gBACvC,CAAC;EACD,GAAG,IAAI,qBAAqB,IAAI,GAAG;CACrC,OACE,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;;AAGA,SAAgB,YACd,OACA,KACwC;CACxC,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,OAAO,MAAM,QAAQ,MAAM;EAC/B,IAAI,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO;EAC3B,KAAK,IAAI,EAAE,EAAE;EACb,OAAO;CACT,CAAC;CACD,IAAI,IAAI,WAAW,GAAG,OAAO,EAAE,OAAO,KAAK;CAC3C,MAAM,QAAQ,IAAI,IAAI,KAAK,KAAK,MAAM,EAAE,EAAE,CAAC;CAC3C,MAAM,UAAU,IAAI,QAAQ,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;CACjD,IAAI,QAAQ,SAAS,GAAG,OAAO,EAAE,QAAQ;CACzC,MAAM,SAAS,IAAI,IAAI,GAAG;CAC1B,OAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE;AACvD;AAEA,eAAe,OAAO,MAAkB,IAAyB;CAC/D,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,oDAAoD;EAC3D,OAAO;CACT;CACA,MAAM,SAAS,MAAM,gBAAgB,KAAK,MAAM;CAChD,IAAI,KAAK,QAAQ,KAAA,GAAW,OAAO,YAAY,QAAQ,KAAK,GAAG;CAC/D,MAAM,WAAW,YAAY,OAAO,OAAO,KAAK,KAAK;CACrD,IAAI,aAAa,UAAU;EACzB,GAAG,IACD,wBAAwB,SAAS,QAAQ,KAAK,MAAM,EAAE,qBAAqB,CAAC,GAAG,IAAI,IAAI,OAAO,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,IACpI;EACA,OAAO;CACT;CACA,OAAO,QAAQ,SAAS;CACxB,MAAM,SAAS,MAAM,SAAS,MAAM;CACpC,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MAE7C,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAQA,SAAS,aACP,QACe;CACf,MAAM,cAAc,MAAM,OAAO,MAAM,EAAE;CACzC,MAAM,SAAwB,CAC5B;EACE,MAAM,iBAAiB,OAAO,MAAM;EACpC,IAAI;EACJ,QAAQ,cAAc,YAAY;CACpC,CACF;CACA,MAAM,SAAS,OAAO,QAClB,CAAC,GAAG,OAAO,QAAQ,OAAO,MAAM,KAAK,IACrC,OAAO;CACX,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,EAAE,WAAW,KAAA,GAAW;EAC5B,OAAO,KAAK;GACV,MAAM,QAAQ,EAAE,OAAO,UAAU,EAAE,GAAG;GACtC,IAAI,QAAQ,IAAI,EAAE,YAAY,KAAA;GAC9B,QAAQ,QAAQ,IAAI,EAAE,YAAY,KAAA,IAAY,QAAQ;EACxD,CAAC;CACH;CACA,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,uDAAuD;EAC9D,OAAO;CACT;CACA,MAAM,SAAS,aAAa,MAAM,gBAAgB,KAAK,MAAM,CAAC;CAE9D,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI;MAEjD,KAAK,MAAM,OAAO,QAChB,GAAG,IAAI,GAAG,IAAI,KAAK,OAAO,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,OAAO,GAAG;CAGrE,OAAO,OAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,IAAI;AAC7C;AAEA,MAAM,YAAgB;CACpB,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;CAClC,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;AACpC;;AAGA,eAAsB,KACpB,MACA,KAAS,WACQ;CACjB,IAAI;CACJ,IAAI;EACF,OAAO,UAAU,IAAI;CACvB,SAAS,KAAK;EACZ,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAAK,IAAI;EAChE,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI,KAAK,YAAY,KAAA,GAAW;EAC9B,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI;EACF,QAAQ,KAAK,SAAb;GACE,KAAK,OACH,OAAO,MAAM,OAAO,MAAM,EAAE;GAC9B,KAAK;GACL,KAAK;IACH,IAAI,KAAK,MAAM,SAAS,GAAG;KACzB,GAAG,IAAI,GAAG,KAAK,QAAQ,sCAAsC;KAC7D,OAAO;IACT;IACA,OAAO,KAAK,YAAY,WACpB,MAAM,UAAU,MAAM,EAAE,IACxB,MAAM,UAAU,MAAM,EAAE;GAC9B;IACE,GAAG,IAAI,oBAAoB,KAAK,QAAQ,IAAI;IAC5C,GAAG,IAAI,KAAK;IACZ,OAAO;EACX;CACF,SAAS,KAAK;EACZ,IAAI,eAAe,YAAY;GAC7B,GAAG,IAAI,UAAU,IAAI,QAAQ,GAAG;GAChC,OAAO;EACT;EACA,GAAG,IAAI,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,GAAG;EACrE,OAAO;CACT;AACF;AAGA,IAAI,YAAY,QAAQ,UAAU,QAAQ,KAAK,MAAM;CACnD,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7C,QAAQ,KAAK,IAAI;AACnB"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sverka/arena",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.30",
|
|
4
4
|
"description": "Sverka agent arena — run agents against verification workflows",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
"dependencies": {
|
|
37
37
|
"@agentclientprotocol/sdk": "^1.5.0",
|
|
38
38
|
"zod": "^4.6.5",
|
|
39
|
-
"@sverka/runtime": "^0.1.
|
|
40
|
-
"@sverka/reporter": "^0.1.
|
|
41
|
-
"@sverka/workflow": "^0.1.
|
|
39
|
+
"@sverka/runtime": "^0.1.30",
|
|
40
|
+
"@sverka/reporter": "^0.1.30",
|
|
41
|
+
"@sverka/workflow": "^0.1.30"
|
|
42
42
|
},
|
|
43
43
|
"devDependencies": {
|
|
44
44
|
"vitest": "^5.0.0",
|