@sverka/arena 0.1.29 → 0.1.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/dist/bin.d.mts +8 -0
  2. package/dist/bin.d.mts.map +1 -1
  3. package/dist/bin.mjs +37 -5
  4. package/dist/bin.mjs.map +1 -1
  5. package/package.json +4 -4
package/dist/bin.d.mts CHANGED
@@ -7,6 +7,14 @@ interface Io {
7
7
  out(s: string): void;
8
8
  err(s: string): void;
9
9
  }
10
+ /** Filter config tasks by --task ids; duplicate ids collapse to one. */
11
+ export declare function filterTasks<T extends {
12
+ id: string;
13
+ }>(tasks: T[], ids: string[]): {
14
+ tasks: T[];
15
+ } | {
16
+ missing: string[];
17
+ };
10
18
  /** Entry point — exported for tests; bin calls it with process.argv. */
11
19
  export declare function main(argv: readonly string[], io?: Io): Promise<number>;
12
20
  //#endregion
package/dist/bin.d.mts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bin.d.mts","names":[],"sources":["../src/bin.ts"],"mappings":";;;;;UAiBU;EACR,IAAI;EACJ,IAAI;;;wBA4LgB,KACpB,yBACA,KAAI,KACH"}
1
+ {"version":3,"file":"bin.d.mts","names":[],"sources":["../src/bin.ts"],"mappings":";;;;;UAiBU;EACR,IAAI;EACJ,IAAI;;;wBA6HU,YAAY;EAAY;GACtC,OAAO,KACP;EACG,OAAO;;EAAU;;;wBA6FA,KACpB,yBACA,KAAI,KACH"}
package/dist/bin.mjs CHANGED
@@ -12,7 +12,7 @@ import process from "node:process";
12
12
  const USAGE = `usage: sverka-arena <command> [options]
13
13
 
14
14
  commands:
15
- run [--config <path>] [--out <dir>] [--format json|text]
15
+ run [--config <path>] [--out <dir>] [--format json|text] [--task <id>]...
16
16
  report <results.json> [--format json|text|html] [--out <file.html>]
17
17
  doctor [--config <path>] [--format json|text]
18
18
  `;
@@ -31,6 +31,7 @@ function parseArgs(argv) {
31
31
  let out;
32
32
  let format = "text";
33
33
  let command;
34
+ const tasks = [];
34
35
  for (let i = 0; i < argv.length; i++) {
35
36
  const arg = argv[i];
36
37
  switch (arg) {
@@ -46,6 +47,10 @@ function parseArgs(argv) {
46
47
  format = parseFormat(flagValue(argv, i));
47
48
  i += 1;
48
49
  break;
50
+ case "--task":
51
+ tasks.push(flagValue(argv, i));
52
+ i += 1;
53
+ break;
49
54
  default:
50
55
  if (arg.startsWith("--")) throw new ArenaError(`unknown option '${arg}'`, "CONFIG_INVALID");
51
56
  if (command === void 0) command = arg;
@@ -57,7 +62,8 @@ function parseArgs(argv) {
57
62
  positional,
58
63
  config,
59
64
  out,
60
- format
65
+ format,
66
+ tasks
61
67
  };
62
68
  }
63
69
  function which(bin) {
@@ -90,6 +96,21 @@ async function cmdReport(args, io) {
90
96
  } else io.out(renderReport(result) + "\n");
91
97
  return 0;
92
98
  }
99
+ /** Filter config tasks by --task ids; duplicate ids collapse to one. */
100
+ function filterTasks(tasks, ids) {
101
+ const seen = /* @__PURE__ */ new Set();
102
+ const uniq = tasks.filter((t) => {
103
+ if (seen.has(t.id)) return false;
104
+ seen.add(t.id);
105
+ return true;
106
+ });
107
+ if (ids.length === 0) return { tasks: uniq };
108
+ const known = new Set(uniq.map((t) => t.id));
109
+ const missing = ids.filter((id) => !known.has(id));
110
+ if (missing.length > 0) return { missing };
111
+ const wanted = new Set(ids);
112
+ return { tasks: uniq.filter((t) => wanted.has(t.id)) };
113
+ }
93
114
  async function cmdRun(args, io) {
94
115
  if (args.format === "html") {
95
116
  io.err("run: --format html is only supported by 'report'\n");
@@ -97,6 +118,12 @@ async function cmdRun(args, io) {
97
118
  }
98
119
  const config = await loadArenaConfig(args.config);
99
120
  if (args.out !== void 0) config.outputDir = resolve(args.out);
121
+ const filtered = filterTasks(config.tasks, args.tasks);
122
+ if ("missing" in filtered) {
123
+ io.err(`run: unknown --task '${filtered.missing.join("', '")}' (config defines: ${[...new Set(config.tasks.map((t) => t.id))].join(", ")})\n`);
124
+ return 2;
125
+ }
126
+ config.tasks = filtered.tasks;
100
127
  const result = await runArena(config);
101
128
  if (args.format === "json") io.out(JSON.stringify(result, null, 2) + "\n");
102
129
  else io.out(renderReport(result) + "\n");
@@ -151,8 +178,13 @@ async function main(argv, io = defaultIo) {
151
178
  try {
152
179
  switch (args.command) {
153
180
  case "run": return await cmdRun(args, io);
154
- case "report": return await cmdReport(args, io);
155
- case "doctor": return await cmdDoctor(args, io);
181
+ case "report":
182
+ case "doctor":
183
+ if (args.tasks.length > 0) {
184
+ io.err(`${args.command}: --task is only supported by 'run'\n`);
185
+ return 2;
186
+ }
187
+ return args.command === "report" ? await cmdReport(args, io) : await cmdDoctor(args, io);
156
188
  default:
157
189
  io.err(`unknown command '${args.command}'\n`);
158
190
  io.err(USAGE);
@@ -172,6 +204,6 @@ if (import.meta.url === `file://${process.argv[1]}`) {
172
204
  process.exit(code);
173
205
  }
174
206
  //#endregion
175
- export { main };
207
+ export { filterTasks, main };
176
208
 
177
209
  //# sourceMappingURL=bin.mjs.map
package/dist/bin.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * sverka-arena — CLI for the agent arena: run the benchmark matrix, render\n * saved results, check the environment. Spec: specs/49-arena-cli.\n */\n\nimport process from \"node:process\";\nimport { readFile } from \"node:fs/promises\";\nimport { existsSync } from \"node:fs\";\nimport { delimiter, join, resolve } from \"node:path\";\n\nimport type { ArenaResult } from \"./types.js\";\nimport { ArenaError, loadArenaConfig } from \"./config.js\";\nimport { renderReport } from \"./report.js\";\nimport { writeAggregateReport } from \"./aggregate-report.js\";\nimport { runArena } from \"./runner.js\";\n\ninterface Io {\n out(s: string): void;\n err(s: string): void;\n}\n\ntype Format = \"text\" | \"json\" | \"html\";\n\ninterface ParsedArgs {\n command: string | undefined;\n positional: string[];\n config: string;\n out: string | undefined;\n format: Format;\n}\n\nconst USAGE = `usage: sverka-arena <command> [options]\n\ncommands:\n run [--config <path>] [--out <dir>] [--format json|text]\n report <results.json> [--format json|text|html] [--out <file.html>]\n doctor [--config <path>] [--format json|text]\n`;\n\nfunction flagValue(argv: readonly string[], i: number): string {\n const val = argv[i + 1];\n if (val === undefined || val.startsWith(\"--\")) {\n throw new ArenaError(`${argv[i]} requires a value`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseFormat(val: string): Format {\n if (val !== \"json\" && val !== \"text\" && val !== \"html\") {\n throw new ArenaError(`invalid --format '${val}'`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseArgs(argv: readonly string[]): ParsedArgs {\n const positional: string[] = [];\n let config = \"arena.config.ts\";\n let out: string | undefined;\n let format: \"text\" | \"json\" | \"html\" = \"text\";\n let command: string | undefined;\n for (let i = 0; i < argv.length; i++) {\n const arg = argv[i]!;\n switch (arg) {\n case \"--config\":\n config = flagValue(argv, i);\n i += 1;\n break;\n case \"--out\":\n out = flagValue(argv, i);\n i += 1;\n break;\n case \"--format\":\n format = parseFormat(flagValue(argv, i));\n i += 1;\n break;\n default:\n if (arg.startsWith(\"--\")) {\n throw new ArenaError(`unknown option '${arg}'`, \"CONFIG_INVALID\");\n }\n if (command === undefined) command = arg;\n else positional.push(arg);\n }\n }\n return { command, positional, config, out, format };\n}\n\nfunction which(bin: string): boolean {\n const exts =\n process.platform === \"win32\"\n ? [\"\", ...(process.env.PATHEXT ?? \".EXE;.CMD;.BAT\").split(\";\")]\n : [\"\"];\n for (const dir of (process.env.PATH ?? \"\").split(delimiter)) {\n if (exts.some((ext) => existsSync(join(dir, bin + ext)))) return true;\n }\n return false;\n}\n\nasync function cmdReport(args: ParsedArgs, io: Io): Promise<number> {\n const file = args.positional[0];\n if (file === undefined) {\n io.err(\"report: missing <results.json> path\");\n return 2;\n }\n let result: ArenaResult;\n try {\n result = JSON.parse(await readFile(file, \"utf8\")) as ArenaResult;\n } catch (err) {\n io.err(\n `report: cannot read '${file}': ${err instanceof Error ? err.message : String(err)}`,\n );\n return 2;\n }\n if (\n !Array.isArray(result.aggregates) ||\n !Array.isArray(result.analysis) ||\n !Array.isArray(result.results) ||\n result.config === undefined ||\n !Array.isArray(result.config.models)\n ) {\n io.err(`report: '${file}' is not a sverka-arena results file\\n`);\n return 2;\n }\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else if (args.format === \"html\") {\n const out = args.out ?? \"arena-report.html\";\n writeAggregateReport(result, resolve(out), {\n command: `sverka-arena report ${file} --format html`,\n });\n io.out(`report written to ${out}\\n`);\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\nasync function cmdRun(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"run: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const config = await loadArenaConfig(args.config);\n if (args.out !== undefined) config.outputDir = resolve(args.out);\n const result = await runArena(config);\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\ninterface DoctorCheck {\n name: string;\n ok: boolean;\n detail: string;\n}\n\nfunction doctorChecks(\n config: Awaited<ReturnType<typeof loadArenaConfig>>,\n): DoctorCheck[] {\n const agentOnPath = which(config.agent.id);\n const checks: DoctorCheck[] = [\n {\n name: `agent binary: ${config.agent.id}`,\n ok: agentOnPath,\n detail: agentOnPath ? \"on PATH\" : \"not found on PATH\",\n },\n ];\n const models = config.judge\n ? [...config.models, config.judge.model]\n : config.models;\n for (const m of models) {\n if (m.envVar === undefined) continue;\n checks.push({\n name: `env: ${m.envVar} (model ${m.id})`,\n ok: process.env[m.envVar] !== undefined,\n detail: process.env[m.envVar] !== undefined ? \"set\" : \"not set\",\n });\n }\n return checks;\n}\n\nasync function cmdDoctor(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"doctor: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const checks = doctorChecks(await loadArenaConfig(args.config));\n\n if (args.format === \"json\") {\n io.out(JSON.stringify({ checks }, null, 2) + \"\\n\");\n } else {\n for (const chk of checks) {\n io.out(`${chk.ok ? \"ok\" : \"FAIL\"} ${chk.name} — ${chk.detail}\\n`);\n }\n }\n return checks.every((chk) => chk.ok) ? 0 : 1;\n}\n\nconst defaultIo: Io = {\n out: (s) => process.stdout.write(s),\n err: (s) => process.stderr.write(s),\n};\n\n/** Entry point — exported for tests; bin calls it with process.argv. */\nexport async function main(\n argv: readonly string[],\n io: Io = defaultIo,\n): Promise<number> {\n let args: ParsedArgs;\n try {\n args = parseArgs(argv);\n } catch (err) {\n io.err((err instanceof Error ? err.message : String(err)) + \"\\n\");\n io.err(USAGE);\n return 2;\n }\n\n if (args.command === undefined) {\n io.err(USAGE);\n return 2;\n }\n\n try {\n switch (args.command) {\n case \"run\":\n return await cmdRun(args, io);\n case \"report\":\n return await cmdReport(args, io);\n case \"doctor\":\n return await cmdDoctor(args, io);\n default:\n io.err(`unknown command '${args.command}'\\n`);\n io.err(USAGE);\n return 2;\n }\n } catch (err) {\n if (err instanceof ArenaError) {\n io.err(`error: ${err.message}\\n`);\n return 2;\n }\n io.err(`error: ${err instanceof Error ? err.message : String(err)}\\n`);\n return 3;\n }\n}\n\n// Run when invoked directly (not imported by tests).\nif (import.meta.url === `file://${process.argv[1]}`) {\n const code = await main(process.argv.slice(2));\n process.exit(code);\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAM,QAAQ;;;;;;;AAQd,SAAS,UAAU,MAAyB,GAAmB;CAC7D,MAAM,MAAM,KAAK,IAAI;CACrB,IAAI,QAAQ,KAAA,KAAa,IAAI,WAAW,IAAI,GAC1C,MAAM,IAAI,WAAW,GAAG,KAAK,GAAG,oBAAoB,gBAAgB;CAEtE,OAAO;AACT;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,QAAQ,UAAU,QAAQ,UAAU,QAAQ,QAC9C,MAAM,IAAI,WAAW,qBAAqB,IAAI,IAAI,gBAAgB;CAEpE,OAAO;AACT;AAEA,SAAS,UAAU,MAAqC;CACtD,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI;CACJ,IAAI,SAAmC;CACvC,IAAI;CACJ,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,QAAQ,KAAR;GACE,KAAK;IACH,SAAS,UAAU,MAAM,CAAC;IAC1B,KAAK;IACL;GACF,KAAK;IACH,MAAM,UAAU,MAAM,CAAC;IACvB,KAAK;IACL;GACF,KAAK;IACH,SAAS,YAAY,UAAU,MAAM,CAAC,CAAC;IACvC,KAAK;IACL;GACF;IACE,IAAI,IAAI,WAAW,IAAI,GACrB,MAAM,IAAI,WAAW,mBAAmB,IAAI,IAAI,gBAAgB;IAElE,IAAI,YAAY,KAAA,GAAW,UAAU;SAChC,WAAW,KAAK,GAAG;EAC5B;CACF;CACA,OAAO;EAAE;EAAS;EAAY;EAAQ;EAAK;CAAO;AACpD;AAEA,SAAS,MAAM,KAAsB;CACnC,MAAM,OACJ,QAAQ,aAAa,UACjB,CAAC,IAAI,IAAI,QAAQ,IAAI,WAAW,iBAAA,CAAkB,MAAM,GAAG,CAAC,IAC5D,CAAC,EAAE;CACT,KAAK,MAAM,QAAQ,QAAQ,IAAI,QAAQ,GAAA,CAAI,MAAM,SAAS,GACxD,IAAI,KAAK,MAAM,QAAQ,WAAW,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO;CAEnE,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,MAAM,OAAO,KAAK,WAAW;CAC7B,IAAI,SAAS,KAAA,GAAW;EACtB,GAAG,IAAI,qCAAqC;EAC5C,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;CAClD,SAAS,KAAK;EACZ,GAAG,IACD,wBAAwB,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GACnF;EACA,OAAO;CACT;CACA,IACE,CAAC,MAAM,QAAQ,OAAO,UAAU,KAChC,CAAC,MAAM,QAAQ,OAAO,QAAQ,KAC9B,CAAC,MAAM,QAAQ,OAAO,OAAO,KAC7B,OAAO,WAAW,KAAA,KAClB,CAAC,MAAM,QAAQ,OAAO,OAAO,MAAM,GACnC;EACA,GAAG,IAAI,YAAY,KAAK,uCAAuC;EAC/D,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MACxC,IAAI,KAAK,WAAW,QAAQ;EACjC,MAAM,MAAM,KAAK,OAAO;EACxB,qBAAqB,QAAQ,QAAQ,GAAG,GAAG,EACzC,SAAS,uBAAuB,KAAK,gBACvC,CAAC;EACD,GAAG,IAAI,qBAAqB,IAAI,GAAG;CACrC,OACE,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAEA,eAAe,OAAO,MAAkB,IAAyB;CAC/D,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,oDAAoD;EAC3D,OAAO;CACT;CACA,MAAM,SAAS,MAAM,gBAAgB,KAAK,MAAM;CAChD,IAAI,KAAK,QAAQ,KAAA,GAAW,OAAO,YAAY,QAAQ,KAAK,GAAG;CAC/D,MAAM,SAAS,MAAM,SAAS,MAAM;CACpC,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MAE7C,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAQA,SAAS,aACP,QACe;CACf,MAAM,cAAc,MAAM,OAAO,MAAM,EAAE;CACzC,MAAM,SAAwB,CAC5B;EACE,MAAM,iBAAiB,OAAO,MAAM;EACpC,IAAI;EACJ,QAAQ,cAAc,YAAY;CACpC,CACF;CACA,MAAM,SAAS,OAAO,QAClB,CAAC,GAAG,OAAO,QAAQ,OAAO,MAAM,KAAK,IACrC,OAAO;CACX,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,EAAE,WAAW,KAAA,GAAW;EAC5B,OAAO,KAAK;GACV,MAAM,QAAQ,EAAE,OAAO,UAAU,EAAE,GAAG;GACtC,IAAI,QAAQ,IAAI,EAAE,YAAY,KAAA;GAC9B,QAAQ,QAAQ,IAAI,EAAE,YAAY,KAAA,IAAY,QAAQ;EACxD,CAAC;CACH;CACA,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,uDAAuD;EAC9D,OAAO;CACT;CACA,MAAM,SAAS,aAAa,MAAM,gBAAgB,KAAK,MAAM,CAAC;CAE9D,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI;MAEjD,KAAK,MAAM,OAAO,QAChB,GAAG,IAAI,GAAG,IAAI,KAAK,OAAO,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,OAAO,GAAG;CAGrE,OAAO,OAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,IAAI;AAC7C;AAEA,MAAM,YAAgB;CACpB,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;CAClC,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;AACpC;;AAGA,eAAsB,KACpB,MACA,KAAS,WACQ;CACjB,IAAI;CACJ,IAAI;EACF,OAAO,UAAU,IAAI;CACvB,SAAS,KAAK;EACZ,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAAK,IAAI;EAChE,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI,KAAK,YAAY,KAAA,GAAW;EAC9B,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI;EACF,QAAQ,KAAK,SAAb;GACE,KAAK,OACH,OAAO,MAAM,OAAO,MAAM,EAAE;GAC9B,KAAK,UACH,OAAO,MAAM,UAAU,MAAM,EAAE;GACjC,KAAK,UACH,OAAO,MAAM,UAAU,MAAM,EAAE;GACjC;IACE,GAAG,IAAI,oBAAoB,KAAK,QAAQ,IAAI;IAC5C,GAAG,IAAI,KAAK;IACZ,OAAO;EACX;CACF,SAAS,KAAK;EACZ,IAAI,eAAe,YAAY;GAC7B,GAAG,IAAI,UAAU,IAAI,QAAQ,GAAG;GAChC,OAAO;EACT;EACA,GAAG,IAAI,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,GAAG;EACrE,OAAO;CACT;AACF;AAGA,IAAI,YAAY,QAAQ,UAAU,QAAQ,KAAK,MAAM;CACnD,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7C,QAAQ,KAAK,IAAI;AACnB"}
1
+ {"version":3,"file":"bin.mjs","names":[],"sources":["../src/bin.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * sverka-arena — CLI for the agent arena: run the benchmark matrix, render\n * saved results, check the environment. Spec: specs/49-arena-cli.\n */\n\nimport process from \"node:process\";\nimport { readFile } from \"node:fs/promises\";\nimport { existsSync } from \"node:fs\";\nimport { delimiter, join, resolve } from \"node:path\";\n\nimport type { ArenaResult } from \"./types.js\";\nimport { ArenaError, loadArenaConfig } from \"./config.js\";\nimport { renderReport } from \"./report.js\";\nimport { writeAggregateReport } from \"./aggregate-report.js\";\nimport { runArena } from \"./runner.js\";\n\ninterface Io {\n out(s: string): void;\n err(s: string): void;\n}\n\ntype Format = \"text\" | \"json\" | \"html\";\n\ninterface ParsedArgs {\n command: string | undefined;\n positional: string[];\n config: string;\n out: string | undefined;\n format: Format;\n tasks: string[];\n}\n\nconst USAGE = `usage: sverka-arena <command> [options]\n\ncommands:\n run [--config <path>] [--out <dir>] [--format json|text] [--task <id>]...\n report <results.json> [--format json|text|html] [--out <file.html>]\n doctor [--config <path>] [--format json|text]\n`;\n\nfunction flagValue(argv: readonly string[], i: number): string {\n const val = argv[i + 1];\n if (val === undefined || val.startsWith(\"--\")) {\n throw new ArenaError(`${argv[i]} requires a value`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseFormat(val: string): Format {\n if (val !== \"json\" && val !== \"text\" && val !== \"html\") {\n throw new ArenaError(`invalid --format '${val}'`, \"CONFIG_INVALID\");\n }\n return val;\n}\n\nfunction parseArgs(argv: readonly string[]): ParsedArgs {\n const positional: string[] = [];\n let config = \"arena.config.ts\";\n let out: string | undefined;\n let format: \"text\" | \"json\" | \"html\" = \"text\";\n let command: string | undefined;\n const tasks: string[] = [];\n for (let i = 0; i < argv.length; i++) {\n const arg = argv[i]!;\n switch (arg) {\n case \"--config\":\n config = flagValue(argv, i);\n i += 1;\n break;\n case \"--out\":\n out = flagValue(argv, i);\n i += 1;\n break;\n case \"--format\":\n format = parseFormat(flagValue(argv, i));\n i += 1;\n break;\n case \"--task\":\n tasks.push(flagValue(argv, i));\n i += 1;\n break;\n default:\n if (arg.startsWith(\"--\")) {\n throw new ArenaError(`unknown option '${arg}'`, \"CONFIG_INVALID\");\n }\n if (command === undefined) command = arg;\n else positional.push(arg);\n }\n }\n return { command, positional, config, out, format, tasks };\n}\n\nfunction which(bin: string): boolean {\n const exts =\n process.platform === \"win32\"\n ? [\"\", ...(process.env.PATHEXT ?? \".EXE;.CMD;.BAT\").split(\";\")]\n : [\"\"];\n for (const dir of (process.env.PATH ?? \"\").split(delimiter)) {\n if (exts.some((ext) => existsSync(join(dir, bin + ext)))) return true;\n }\n return false;\n}\n\nasync function cmdReport(args: ParsedArgs, io: Io): Promise<number> {\n const file = args.positional[0];\n if (file === undefined) {\n io.err(\"report: missing <results.json> path\");\n return 2;\n }\n let result: ArenaResult;\n try {\n result = JSON.parse(await readFile(file, \"utf8\")) as ArenaResult;\n } catch (err) {\n io.err(\n `report: cannot read '${file}': ${err instanceof Error ? err.message : String(err)}`,\n );\n return 2;\n }\n if (\n !Array.isArray(result.aggregates) ||\n !Array.isArray(result.analysis) ||\n !Array.isArray(result.results) ||\n result.config === undefined ||\n !Array.isArray(result.config.models)\n ) {\n io.err(`report: '${file}' is not a sverka-arena results file\\n`);\n return 2;\n }\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else if (args.format === \"html\") {\n const out = args.out ?? \"arena-report.html\";\n writeAggregateReport(result, resolve(out), {\n command: `sverka-arena report ${file} --format html`,\n });\n io.out(`report written to ${out}\\n`);\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\n/** Filter config tasks by --task ids; duplicate ids collapse to one. */\nexport function filterTasks<T extends { id: string }>(\n tasks: T[],\n ids: string[],\n): { tasks: T[] } | { missing: string[] } {\n const seen = new Set<string>();\n const uniq = tasks.filter((t) => {\n if (seen.has(t.id)) return false;\n seen.add(t.id);\n return true;\n });\n if (ids.length === 0) return { tasks: uniq };\n const known = new Set(uniq.map((t) => t.id));\n const missing = ids.filter((id) => !known.has(id));\n if (missing.length > 0) return { missing };\n const wanted = new Set(ids);\n return { tasks: uniq.filter((t) => wanted.has(t.id)) };\n}\n\nasync function cmdRun(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"run: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const config = await loadArenaConfig(args.config);\n if (args.out !== undefined) config.outputDir = resolve(args.out);\n const filtered = filterTasks(config.tasks, args.tasks);\n if (\"missing\" in filtered) {\n io.err(\n `run: unknown --task '${filtered.missing.join(\"', '\")}' (config defines: ${[...new Set(config.tasks.map((t) => t.id))].join(\", \")})\\n`,\n );\n return 2;\n }\n config.tasks = filtered.tasks;\n const result = await runArena(config);\n if (args.format === \"json\") {\n io.out(JSON.stringify(result, null, 2) + \"\\n\");\n } else {\n io.out(renderReport(result) + \"\\n\");\n }\n return 0;\n}\n\ninterface DoctorCheck {\n name: string;\n ok: boolean;\n detail: string;\n}\n\nfunction doctorChecks(\n config: Awaited<ReturnType<typeof loadArenaConfig>>,\n): DoctorCheck[] {\n const agentOnPath = which(config.agent.id);\n const checks: DoctorCheck[] = [\n {\n name: `agent binary: ${config.agent.id}`,\n ok: agentOnPath,\n detail: agentOnPath ? \"on PATH\" : \"not found on PATH\",\n },\n ];\n const models = config.judge\n ? [...config.models, config.judge.model]\n : config.models;\n for (const m of models) {\n if (m.envVar === undefined) continue;\n checks.push({\n name: `env: ${m.envVar} (model ${m.id})`,\n ok: process.env[m.envVar] !== undefined,\n detail: process.env[m.envVar] !== undefined ? \"set\" : \"not set\",\n });\n }\n return checks;\n}\n\nasync function cmdDoctor(args: ParsedArgs, io: Io): Promise<number> {\n if (args.format === \"html\") {\n io.err(\"doctor: --format html is only supported by 'report'\\n\");\n return 2;\n }\n const checks = doctorChecks(await loadArenaConfig(args.config));\n\n if (args.format === \"json\") {\n io.out(JSON.stringify({ checks }, null, 2) + \"\\n\");\n } else {\n for (const chk of checks) {\n io.out(`${chk.ok ? \"ok\" : \"FAIL\"} ${chk.name} — ${chk.detail}\\n`);\n }\n }\n return checks.every((chk) => chk.ok) ? 0 : 1;\n}\n\nconst defaultIo: Io = {\n out: (s) => process.stdout.write(s),\n err: (s) => process.stderr.write(s),\n};\n\n/** Entry point — exported for tests; bin calls it with process.argv. */\nexport async function main(\n argv: readonly string[],\n io: Io = defaultIo,\n): Promise<number> {\n let args: ParsedArgs;\n try {\n args = parseArgs(argv);\n } catch (err) {\n io.err((err instanceof Error ? err.message : String(err)) + \"\\n\");\n io.err(USAGE);\n return 2;\n }\n\n if (args.command === undefined) {\n io.err(USAGE);\n return 2;\n }\n\n try {\n switch (args.command) {\n case \"run\":\n return await cmdRun(args, io);\n case \"report\":\n case \"doctor\":\n if (args.tasks.length > 0) {\n io.err(`${args.command}: --task is only supported by 'run'\\n`);\n return 2;\n }\n return args.command === \"report\"\n ? await cmdReport(args, io)\n : await cmdDoctor(args, io);\n default:\n io.err(`unknown command '${args.command}'\\n`);\n io.err(USAGE);\n return 2;\n }\n } catch (err) {\n if (err instanceof ArenaError) {\n io.err(`error: ${err.message}\\n`);\n return 2;\n }\n io.err(`error: ${err instanceof Error ? err.message : String(err)}\\n`);\n return 3;\n }\n}\n\n// Run when invoked directly (not imported by tests).\nif (import.meta.url === `file://${process.argv[1]}`) {\n const code = await main(process.argv.slice(2));\n process.exit(code);\n}\n"],"mappings":";;;;;;;;;;;AAiCA,MAAM,QAAQ;;;;;;;AAQd,SAAS,UAAU,MAAyB,GAAmB;CAC7D,MAAM,MAAM,KAAK,IAAI;CACrB,IAAI,QAAQ,KAAA,KAAa,IAAI,WAAW,IAAI,GAC1C,MAAM,IAAI,WAAW,GAAG,KAAK,GAAG,oBAAoB,gBAAgB;CAEtE,OAAO;AACT;AAEA,SAAS,YAAY,KAAqB;CACxC,IAAI,QAAQ,UAAU,QAAQ,UAAU,QAAQ,QAC9C,MAAM,IAAI,WAAW,qBAAqB,IAAI,IAAI,gBAAgB;CAEpE,OAAO;AACT;AAEA,SAAS,UAAU,MAAqC;CACtD,MAAM,aAAuB,CAAC;CAC9B,IAAI,SAAS;CACb,IAAI;CACJ,IAAI,SAAmC;CACvC,IAAI;CACJ,MAAM,QAAkB,CAAC;CACzB,KAAK,IAAI,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;EACpC,MAAM,MAAM,KAAK;EACjB,QAAQ,KAAR;GACE,KAAK;IACH,SAAS,UAAU,MAAM,CAAC;IAC1B,KAAK;IACL;GACF,KAAK;IACH,MAAM,UAAU,MAAM,CAAC;IACvB,KAAK;IACL;GACF,KAAK;IACH,SAAS,YAAY,UAAU,MAAM,CAAC,CAAC;IACvC,KAAK;IACL;GACF,KAAK;IACH,MAAM,KAAK,UAAU,MAAM,CAAC,CAAC;IAC7B,KAAK;IACL;GACF;IACE,IAAI,IAAI,WAAW,IAAI,GACrB,MAAM,IAAI,WAAW,mBAAmB,IAAI,IAAI,gBAAgB;IAElE,IAAI,YAAY,KAAA,GAAW,UAAU;SAChC,WAAW,KAAK,GAAG;EAC5B;CACF;CACA,OAAO;EAAE;EAAS;EAAY;EAAQ;EAAK;EAAQ;CAAM;AAC3D;AAEA,SAAS,MAAM,KAAsB;CACnC,MAAM,OACJ,QAAQ,aAAa,UACjB,CAAC,IAAI,IAAI,QAAQ,IAAI,WAAW,iBAAA,CAAkB,MAAM,GAAG,CAAC,IAC5D,CAAC,EAAE;CACT,KAAK,MAAM,QAAQ,QAAQ,IAAI,QAAQ,GAAA,CAAI,MAAM,SAAS,GACxD,IAAI,KAAK,MAAM,QAAQ,WAAW,KAAK,KAAK,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO;CAEnE,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,MAAM,OAAO,KAAK,WAAW;CAC7B,IAAI,SAAS,KAAA,GAAW;EACtB,GAAG,IAAI,qCAAqC;EAC5C,OAAO;CACT;CACA,IAAI;CACJ,IAAI;EACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;CAClD,SAAS,KAAK;EACZ,GAAG,IACD,wBAAwB,KAAK,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,GACnF;EACA,OAAO;CACT;CACA,IACE,CAAC,MAAM,QAAQ,OAAO,UAAU,KAChC,CAAC,MAAM,QAAQ,OAAO,QAAQ,KAC9B,CAAC,MAAM,QAAQ,OAAO,OAAO,KAC7B,OAAO,WAAW,KAAA,KAClB,CAAC,MAAM,QAAQ,OAAO,OAAO,MAAM,GACnC;EACA,GAAG,IAAI,YAAY,KAAK,uCAAuC;EAC/D,OAAO;CACT;CACA,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MACxC,IAAI,KAAK,WAAW,QAAQ;EACjC,MAAM,MAAM,KAAK,OAAO;EACxB,qBAAqB,QAAQ,QAAQ,GAAG,GAAG,EACzC,SAAS,uBAAuB,KAAK,gBACvC,CAAC;EACD,GAAG,IAAI,qBAAqB,IAAI,GAAG;CACrC,OACE,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;;AAGA,SAAgB,YACd,OACA,KACwC;CACxC,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,OAAO,MAAM,QAAQ,MAAM;EAC/B,IAAI,KAAK,IAAI,EAAE,EAAE,GAAG,OAAO;EAC3B,KAAK,IAAI,EAAE,EAAE;EACb,OAAO;CACT,CAAC;CACD,IAAI,IAAI,WAAW,GAAG,OAAO,EAAE,OAAO,KAAK;CAC3C,MAAM,QAAQ,IAAI,IAAI,KAAK,KAAK,MAAM,EAAE,EAAE,CAAC;CAC3C,MAAM,UAAU,IAAI,QAAQ,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;CACjD,IAAI,QAAQ,SAAS,GAAG,OAAO,EAAE,QAAQ;CACzC,MAAM,SAAS,IAAI,IAAI,GAAG;CAC1B,OAAO,EAAE,OAAO,KAAK,QAAQ,MAAM,OAAO,IAAI,EAAE,EAAE,CAAC,EAAE;AACvD;AAEA,eAAe,OAAO,MAAkB,IAAyB;CAC/D,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,oDAAoD;EAC3D,OAAO;CACT;CACA,MAAM,SAAS,MAAM,gBAAgB,KAAK,MAAM;CAChD,IAAI,KAAK,QAAQ,KAAA,GAAW,OAAO,YAAY,QAAQ,KAAK,GAAG;CAC/D,MAAM,WAAW,YAAY,OAAO,OAAO,KAAK,KAAK;CACrD,IAAI,aAAa,UAAU;EACzB,GAAG,IACD,wBAAwB,SAAS,QAAQ,KAAK,MAAM,EAAE,qBAAqB,CAAC,GAAG,IAAI,IAAI,OAAO,MAAM,KAAK,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,IACpI;EACA,OAAO;CACT;CACA,OAAO,QAAQ,SAAS;CACxB,MAAM,SAAS,MAAM,SAAS,MAAM;CACpC,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;MAE7C,GAAG,IAAI,aAAa,MAAM,IAAI,IAAI;CAEpC,OAAO;AACT;AAQA,SAAS,aACP,QACe;CACf,MAAM,cAAc,MAAM,OAAO,MAAM,EAAE;CACzC,MAAM,SAAwB,CAC5B;EACE,MAAM,iBAAiB,OAAO,MAAM;EACpC,IAAI;EACJ,QAAQ,cAAc,YAAY;CACpC,CACF;CACA,MAAM,SAAS,OAAO,QAClB,CAAC,GAAG,OAAO,QAAQ,OAAO,MAAM,KAAK,IACrC,OAAO;CACX,KAAK,MAAM,KAAK,QAAQ;EACtB,IAAI,EAAE,WAAW,KAAA,GAAW;EAC5B,OAAO,KAAK;GACV,MAAM,QAAQ,EAAE,OAAO,UAAU,EAAE,GAAG;GACtC,IAAI,QAAQ,IAAI,EAAE,YAAY,KAAA;GAC9B,QAAQ,QAAQ,IAAI,EAAE,YAAY,KAAA,IAAY,QAAQ;EACxD,CAAC;CACH;CACA,OAAO;AACT;AAEA,eAAe,UAAU,MAAkB,IAAyB;CAClE,IAAI,KAAK,WAAW,QAAQ;EAC1B,GAAG,IAAI,uDAAuD;EAC9D,OAAO;CACT;CACA,MAAM,SAAS,aAAa,MAAM,gBAAgB,KAAK,MAAM,CAAC;CAE9D,IAAI,KAAK,WAAW,QAClB,GAAG,IAAI,KAAK,UAAU,EAAE,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI;MAEjD,KAAK,MAAM,OAAO,QAChB,GAAG,IAAI,GAAG,IAAI,KAAK,OAAO,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,OAAO,GAAG;CAGrE,OAAO,OAAO,OAAO,QAAQ,IAAI,EAAE,IAAI,IAAI;AAC7C;AAEA,MAAM,YAAgB;CACpB,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;CAClC,MAAM,MAAM,QAAQ,OAAO,MAAM,CAAC;AACpC;;AAGA,eAAsB,KACpB,MACA,KAAS,WACQ;CACjB,IAAI;CACJ,IAAI;EACF,OAAO,UAAU,IAAI;CACvB,SAAS,KAAK;EACZ,GAAG,KAAK,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,KAAK,IAAI;EAChE,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI,KAAK,YAAY,KAAA,GAAW;EAC9B,GAAG,IAAI,KAAK;EACZ,OAAO;CACT;CAEA,IAAI;EACF,QAAQ,KAAK,SAAb;GACE,KAAK,OACH,OAAO,MAAM,OAAO,MAAM,EAAE;GAC9B,KAAK;GACL,KAAK;IACH,IAAI,KAAK,MAAM,SAAS,GAAG;KACzB,GAAG,IAAI,GAAG,KAAK,QAAQ,sCAAsC;KAC7D,OAAO;IACT;IACA,OAAO,KAAK,YAAY,WACpB,MAAM,UAAU,MAAM,EAAE,IACxB,MAAM,UAAU,MAAM,EAAE;GAC9B;IACE,GAAG,IAAI,oBAAoB,KAAK,QAAQ,IAAI;IAC5C,GAAG,IAAI,KAAK;IACZ,OAAO;EACX;CACF,SAAS,KAAK;EACZ,IAAI,eAAe,YAAY;GAC7B,GAAG,IAAI,UAAU,IAAI,QAAQ,GAAG;GAChC,OAAO;EACT;EACA,GAAG,IAAI,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,GAAG;EACrE,OAAO;CACT;AACF;AAGA,IAAI,YAAY,QAAQ,UAAU,QAAQ,KAAK,MAAM;CACnD,MAAM,OAAO,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,CAAC;CAC7C,QAAQ,KAAK,IAAI;AACnB"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sverka/arena",
3
- "version": "0.1.29",
3
+ "version": "0.1.30",
4
4
  "description": "Sverka agent arena — run agents against verification workflows",
5
5
  "license": "MIT",
6
6
  "sideEffects": false,
@@ -36,9 +36,9 @@
36
36
  "dependencies": {
37
37
  "@agentclientprotocol/sdk": "^1.5.0",
38
38
  "zod": "^4.6.5",
39
- "@sverka/runtime": "^0.1.29",
40
- "@sverka/reporter": "^0.1.29",
41
- "@sverka/workflow": "^0.1.29"
39
+ "@sverka/runtime": "^0.1.30",
40
+ "@sverka/reporter": "^0.1.30",
41
+ "@sverka/workflow": "^0.1.30"
42
42
  },
43
43
  "devDependencies": {
44
44
  "vitest": "^5.0.0",