better-mastra 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/dist/evals/cli.js +75 -61
  2. package/dist/evals/cli.js.map +1 -1
  3. package/dist/tools/cli.js +29 -13
  4. package/dist/tools/cli.js.map +1 -1
  5. package/package.json +4 -1
  6. package/src/evals/cli.ts +85 -61
  7. package/src/tools/cli.ts +38 -21
package/dist/evals/cli.js CHANGED
@@ -1,12 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  // better-mastra-evals: real turns through the app's production turn, scored
3
3
  // by hard checks and a judge, against the app's `evals.config.ts`.
4
- //
5
- // better-mastra-evals run [--suite smoke] [--case <id-prefix>]... [--tag <tag>]
6
- // [--repeats 3] [--concurrency 3] [--model <id>]...
7
- // [--no-judge] [--compare <baseline.json>] [--verbose]
8
- // better-mastra-evals inspect <report.json> [--case <id-prefix>]...
9
- // better-mastra-evals compare <baseline.json> <candidate.json>
4
+ // `better-mastra-evals --help` lists the commands; `run` is the default.
10
5
  //
11
6
  // Several --model values run the suite on each and print a model comparison.
12
7
  // Run it with Bun (`bunx --bun better-mastra-evals`) or Node 22.18+ so the
@@ -14,27 +9,26 @@
14
9
  import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
15
10
  import { resolve } from 'node:path';
16
11
  import { pathToFileURL } from 'node:url';
17
- import { parseArgs } from 'node:util';
12
+ import { defineCli, defineCommand } from 'hanzio/cli';
13
+ import { z } from 'zod';
18
14
  import { runEvalSuite, selectCases } from './config.js';
19
15
  import { compareReports, formatModelComparison, formatReport } from './report.js';
20
- const { positionals, values } = parseArgs({
21
- args: process.argv.slice(2),
22
- allowPositionals: true,
23
- options: {
24
- config: { type: 'string', default: 'evals.config.ts' },
25
- suite: { type: 'string', default: 'smoke' },
26
- case: { type: 'string', multiple: true },
27
- tag: { type: 'string' },
28
- repeats: { type: 'string', default: '3' },
29
- concurrency: { type: 'string', default: '3' },
30
- model: { type: 'string', multiple: true },
31
- 'no-judge': { type: 'boolean', default: false },
32
- compare: { type: 'string' },
33
- verbose: { type: 'boolean', default: false }
34
- }
16
+ const runInput = z.object({
17
+ config: z.string().default('evals.config.ts').describe('Eval config module'),
18
+ suite: z.string().default('smoke').describe('Suite to run'),
19
+ case: z.array(z.string()).describe('Only cases whose id starts with this'),
20
+ tag: z.string().optional().describe('Only cases with this tag'),
21
+ repeats: z.number().default(3).describe('Runs per case'),
22
+ concurrency: z.number().default(3).describe('Cases in flight'),
23
+ model: z
24
+ .array(z.string())
25
+ .describe('Model to run on; several print a comparison'),
26
+ judge: z.boolean().default(true).describe('Score runs with the config judge'),
27
+ compare: z.string().optional().describe('Baseline report to compare with'),
28
+ verbose: z.boolean().describe('Keep app console logs')
35
29
  });
36
- async function loadConfig() {
37
- const file = resolve(values.config);
30
+ async function loadConfig(path) {
31
+ const file = resolve(path);
38
32
  const module = await import(pathToFileURL(file).href);
39
33
  if (!module.default)
40
34
  throw new Error(`${file} has no default export`);
@@ -54,43 +48,43 @@ async function save(config, report) {
54
48
  return file;
55
49
  }
56
50
  /** Apps log to the console; while cases run that buries the progress lines. */
57
- function silenceLogs() {
58
- if (values.verbose)
51
+ function silenceLogs(verbose) {
52
+ if (verbose)
59
53
  return () => { };
60
54
  const { log, info, debug } = console;
61
55
  console.log = console.info = console.debug = () => { };
62
56
  return () => Object.assign(console, { log, info, debug });
63
57
  }
64
- async function run() {
65
- const config = await loadConfig();
58
+ async function run(values) {
59
+ const config = await loadConfig(values.config);
66
60
  try {
67
- await runWith(config);
61
+ return await runWith(config, values);
68
62
  }
69
63
  finally {
70
64
  await config.teardown?.();
71
65
  }
72
66
  }
73
- async function runWith(config) {
67
+ async function runWith(config, values) {
74
68
  await config.setup?.();
75
69
  const all = typeof config.cases === 'function' ? await config.cases() : config.cases;
76
- const prefixes = values.case ?? [];
70
+ const prefixes = values.case;
77
71
  const cases = selectCases(all, {
78
72
  suite: values.suite,
79
73
  prefixes,
80
74
  ...(values.tag ? { tag: values.tag } : {})
81
75
  });
82
- const models = values.model?.length ? values.model : [config.defaultModel];
76
+ const models = values.model.length ? values.model : [config.defaultModel];
83
77
  const reports = [];
84
78
  for (const modelId of models) {
85
- console.error(`Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && !values['no-judge'] ? ', judged' : ''}`);
86
- const restore = silenceLogs();
79
+ console.error(`Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && values.judge ? ', judged' : ''}`);
80
+ const restore = silenceLogs(values.verbose);
87
81
  const report = await runEvalSuite(config, {
88
82
  cases,
89
83
  suite: prefixes.length ? 'selected' : values.suite,
90
84
  modelId,
91
- repeats: Number(values.repeats),
92
- concurrency: Number(values.concurrency),
93
- judge: !values['no-judge'],
85
+ repeats: values.repeats,
86
+ concurrency: values.concurrency,
87
+ judge: values.judge,
94
88
  onRun: (testCase, result) => console.error(`${result.pass ? 'pass' : 'FAIL'} ${testCase.id}`)
95
89
  }).finally(restore);
96
90
  console.log(formatReport(report));
@@ -108,8 +102,7 @@ async function runWith(config) {
108
102
  console.log(`\n${compareReports(await readReport(values.compare), first)}`);
109
103
  }
110
104
  const failed = reports.some((report) => report.results.some((result) => result.runs.some((r) => !r.pass)));
111
- if (failed)
112
- process.exitCode = 1;
105
+ return failed ? 1 : 0;
113
106
  }
114
107
  function inspect(report, prefixes) {
115
108
  for (const result of report.results) {
@@ -135,28 +128,49 @@ function inspect(report, prefixes) {
135
128
  });
136
129
  }
137
130
  }
138
- const command = positionals[0] ?? 'run';
139
- switch (command) {
140
- case 'run':
141
- await run();
142
- break;
143
- case 'inspect': {
144
- const file = positionals[1];
145
- if (!file)
146
- throw new Error('inspect needs a report file');
147
- inspect(await readReport(file), values.case ?? []);
148
- break;
149
- }
150
- case 'compare': {
151
- const [, baseline, candidate] = positionals;
152
- if (!baseline || !candidate)
153
- throw new Error('compare needs two report files');
154
- console.log(compareReports(await readReport(baseline), await readReport(candidate)));
155
- break;
131
+ const cli = defineCli({
132
+ name: 'better-mastra-evals',
133
+ description: "Real turns through the app's production turn, scored by checks and a judge",
134
+ commands: {
135
+ run: defineCommand({
136
+ description: 'Run a suite and save its report',
137
+ input: runInput,
138
+ run: async (input, ctx) => {
139
+ ctx.exitCode = await run(input);
140
+ }
141
+ }),
142
+ inspect: defineCommand({
143
+ description: "Print a report's failed checks and transcripts",
144
+ input: z.object({
145
+ report: z.string().meta({ positional: true }).describe('Report file'),
146
+ case: z
147
+ .array(z.string())
148
+ .describe('Only cases whose id starts with this')
149
+ }),
150
+ run: async (input) => inspect(await readReport(input.report), input.case)
151
+ }),
152
+ compare: defineCommand({
153
+ description: 'Compare a candidate report with a baseline',
154
+ input: z.object({
155
+ baseline: z
156
+ .string()
157
+ .meta({ positional: true })
158
+ .describe('Baseline report file'),
159
+ candidate: z
160
+ .string()
161
+ .meta({ positional: true })
162
+ .describe('Candidate report file')
163
+ }),
164
+ run: async (input) => compareReports(await readReport(input.baseline), await readReport(input.candidate))
165
+ })
156
166
  }
157
- default:
158
- throw new Error(`Unknown command ${command}: run, inspect or compare`);
159
- }
167
+ });
168
+ // `run` is the default command: no command, or options first, runs it.
169
+ const argv = process.argv.slice(2);
170
+ const [first] = argv;
171
+ if (first === undefined ||
172
+ (first.startsWith('-') && first !== '-h' && first !== '--help'))
173
+ argv.unshift('run');
160
174
  // App pools (database, Redis) keep the event loop alive after the last run.
161
- process.exit();
175
+ process.exit(await cli.run(argv));
162
176
  //# sourceMappingURL=cli.js.map
package/dist/evals/cli.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":";AACA,4EAA4E;AAC5E,mEAAmE;AACnE,EAAE;AACF,kFAAkF;AAClF,0EAA0E;AAC1E,6EAA6E;AAC7E,sEAAsE;AACtE,iEAAiE;AACjE,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,2BAA2B;AAE3B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAmB,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EACN,cAAc,EAEd,qBAAqB,EACrB,YAAY,EACZ,MAAM,aAAa,CAAA;AAEpB,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IACzC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3B,gBAAgB,EAAE,IAAI;IACtB,OAAO,EAAE;QACR,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,EAAE;QACtD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE;QAC3C,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QACxC,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACvB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;QACzC,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;QAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;QACzC,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;QAC/C,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC3B,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;KAC5C;CACD,CAAC,CAAA;AAEF,KAAK,UAAU,UAAU;IACxB,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAA;IACrD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,wBAAwB,CAAC,CAAA;IACrE,OAAO,MAAM,CAAC,OAAO,CAAA;AACtB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;AAChD,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,MAAkB,EAAE,MAAkB;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAA;IACjD,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAA;IACzC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAAE,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC3E,MAAM,IAAI,GAAG,OAAO,CACnB,GAAG,EACH,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CACtG,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAChE,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,+EAA+E;AAC/E,SAAS,WAAW;IACnB,IAAI,MAAM,CAAC,OAAO;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IACnC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;IACpC,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,GAAG,GAAG,EAAE,GAAE,CAAC,CAAA;IACrD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;AAC1D,CAAC;AAED,KAAK,UAAU,GAAG;IACjB,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAA;IACjC,IAAI,CAAC;QACJ,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;IACtB,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAA;IAC1B,CAAC;AACF,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,MAAkB;IACxC,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAA;IACtB,MAAM,GAAG,GACR,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAA;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAA;IAClC,MAAM,KAAK,GAAiC,WAAW,CAAC,GAAG,EAAE;QAC5D,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1C,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IAC1E,MAAM,OAAO,GAAiB,EAAE,CAAA;IAChC,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CACZ,WAAW,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,OAAO,OAAO,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CACxH,CAAA;QACD,MAAM,OAAO,GAAG,WAAW,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE;YACzC,KAAK;YACL,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK;YAClD,OAAO;YACP,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC;YAC/B,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC;YACvC,KAAK,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;YAC1B,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAC3B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;SACjE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACnB,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAA;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACrB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAA;IACnE,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAA;IAChC,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;IAC7D,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CACtC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACjE,CAAA;IACD,IAAI,MAAM;QAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;AACjC,CAAC;AAED,SAAS,OAAO,CAAC,MAAkB,EAAE,QAA2B;IAC/D,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACrC,IACC,QAAQ,CAAC,MAAM;YACf,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAE5D,SAAQ;QACT,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,GAAG,CACV,SAAS,MAAM,CAAC,MAAM,KAAK,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CACpE,CAAA;YACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CACV,OAAO,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/D,CAAA;YACF,CAAC;YACD,IAAI,GAAG,CAAC,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACvE,IAAI,GAAG,CAAC,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YACpE,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;gBACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnC,OAAO,CAAC,GAAG,CACV,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACjI,CAAA;gBACF,CAAC;gBACD,OAAO,CAAC,GAAG,CACV,gBAAgB,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,SAAS,EAAE,CACvI,CAAA;YACF,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;AACF,CAAC;AAED,MAAM,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,KAAK,CAAA;AACvC,QAAQ,OAAO,EAAE,CAAC;IACjB,KAAK,KAAK;QACT,MAAM,GAAG,EAAE,CAAA;QACX,MAAK;IACN,KAAK,SAAS,CAAC,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAA;QAC3B,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;QACzD,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;QAClD,MAAK;IACN,CAAC;IACD,KAAK,SAAS,CAAC,CAAC,CAAC;QAChB,MAAM,CAAC,EAAE,QAAQ,EAAE,SAAS,CAAC,GAAG,WAAW,CAAA;QAC3C,IAAI,CAAC,QAAQ,IAAI,CAAC,SAAS;YAC1B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAA;QAClD,OAAO,CAAC,GAAG,CACV,cAAc,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAE,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC,CACvE,CAAA;QACD,MAAK;IACN,CAAC;IACD;QACC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,2BAA2B,CAAC,CAAA;AACxE,CAAC;AAED,4EAA4E;AAC5E,OAAO,CAAC,IAAI,EAAE,CAAA","sourcesContent":["#!/usr/bin/env node\n// better-mastra-evals: real turns through the app's production turn, scored\n// by hard checks and a judge, against the app's `evals.config.ts`.\n//\n// better-mastra-evals run [--suite smoke] [--case <id-prefix>]... [--tag <tag>]\n// [--repeats 3] [--concurrency 3] [--model <id>]...\n// [--no-judge] [--compare <baseline.json>] [--verbose]\n// better-mastra-evals inspect <report.json> [--case <id-prefix>]...\n// better-mastra-evals compare <baseline.json> <candidate.json>\n//\n// Several --model values run the suite on each and print a model comparison.\n// Run it with Bun (`bunx --bun better-mastra-evals`) or Node 22.18+ so the\n// TypeScript config loads.\n\nimport { mkdir, readFile, stat, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\nimport { parseArgs } from 'node:util'\nimport type { EvalCase } from './case.js'\nimport { type EvalConfig, runEvalSuite, selectCases } from './config.js'\nimport {\n\tcompareReports,\n\ttype EvalReport,\n\tformatModelComparison,\n\tformatReport\n} from './report.js'\n\nconst { positionals, values } = parseArgs({\n\targs: process.argv.slice(2),\n\tallowPositionals: true,\n\toptions: {\n\t\tconfig: { type: 'string', default: 'evals.config.ts' },\n\t\tsuite: { type: 'string', default: 'smoke' },\n\t\tcase: { type: 'string', multiple: true },\n\t\ttag: { type: 'string' },\n\t\trepeats: { type: 'string', default: '3' },\n\t\tconcurrency: { type: 'string', default: '3' },\n\t\tmodel: { type: 'string', multiple: true },\n\t\t'no-judge': { type: 'boolean', default: false },\n\t\tcompare: { type: 'string' },\n\t\tverbose: { type: 'boolean', default: false }\n\t}\n})\n\nasync function loadConfig(): Promise<EvalConfig> {\n\tconst file = resolve(values.config)\n\tconst module = await import(pathToFileURL(file).href)\n\tif (!module.default) throw new Error(`${file} has no default export`)\n\treturn module.default\n}\n\nasync function readReport(file: string): Promise<EvalReport> {\n\treturn JSON.parse(await readFile(file, 'utf8'))\n}\n\nasync function save(config: EvalConfig, report: EvalReport) {\n\tconst dir = resolve(config.outputDir ?? '.evals')\n\tawait mkdir(dir, { recursive: true })\n\tconst ignore = resolve(dir, '.gitignore')\n\tif (!(await stat(ignore).catch(() => null))) await writeFile(ignore, '*\\n')\n\tconst file = resolve(\n\t\tdir,\n\t\t`${report.startedAt.replaceAll(':', '-')}-${report.suite}-${report.modelId.replaceAll('/', '_')}.json`\n\t)\n\tawait writeFile(file, `${JSON.stringify(report, null, '\\t')}\\n`)\n\treturn file\n}\n\n/** Apps log to the console; while cases run that buries the progress lines. */\nfunction silenceLogs() {\n\tif (values.verbose) return () => {}\n\tconst { log, info, debug } = console\n\tconsole.log = console.info = console.debug = () => {}\n\treturn () => Object.assign(console, { log, info, debug })\n}\n\nasync function run() {\n\tconst config = await loadConfig()\n\ttry {\n\t\tawait runWith(config)\n\t} finally {\n\t\tawait config.teardown?.()\n\t}\n}\n\nasync function runWith(config: EvalConfig) {\n\tawait config.setup?.()\n\tconst all =\n\t\ttypeof config.cases === 'function' ? await config.cases() : config.cases\n\tconst prefixes = values.case ?? []\n\tconst cases: readonly EvalCase<unknown>[] = selectCases(all, {\n\t\tsuite: values.suite,\n\t\tprefixes,\n\t\t...(values.tag ? { tag: values.tag } : {})\n\t})\n\tconst models = values.model?.length ? values.model : [config.defaultModel]\n\tconst reports: EvalReport[] = []\n\tfor (const modelId of models) {\n\t\tconsole.error(\n\t\t\t`Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && !values['no-judge'] ? ', judged' : ''}`\n\t\t)\n\t\tconst restore = silenceLogs()\n\t\tconst report = await runEvalSuite(config, {\n\t\t\tcases,\n\t\t\tsuite: prefixes.length ? 'selected' : values.suite,\n\t\t\tmodelId,\n\t\t\trepeats: Number(values.repeats),\n\t\t\tconcurrency: Number(values.concurrency),\n\t\t\tjudge: !values['no-judge'],\n\t\t\tonRun: (testCase, result) =>\n\t\t\t\tconsole.error(`${result.pass ? 'pass' : 'FAIL'} ${testCase.id}`)\n\t\t}).finally(restore)\n\t\tconsole.log(formatReport(report))\n\t\tconsole.log(`\\n${await save(config, report)}\\n`)\n\t\treports.push(report)\n\t}\n\tif (reports.length > 1) console.log(formatModelComparison(reports))\n\tconst [first, ...rest] = reports\n\tfor (const report of rest) {\n\t\tif (first) console.log(`\\n${compareReports(first, report)}`)\n\t}\n\tif (values.compare && first) {\n\t\tconsole.log(`\\n${compareReports(await readReport(values.compare), first)}`)\n\t}\n\tconst failed = reports.some((report) =>\n\t\treport.results.some((result) => result.runs.some((r) => !r.pass))\n\t)\n\tif (failed) process.exitCode = 1\n}\n\nfunction inspect(report: EvalReport, prefixes: readonly string[]) {\n\tfor (const result of report.results) {\n\t\tif (\n\t\t\tprefixes.length &&\n\t\t\t!prefixes.some((prefix) => result.caseId.startsWith(prefix))\n\t\t)\n\t\t\tcontinue\n\t\tresult.runs.forEach((run, index) => {\n\t\t\tconsole.log(\n\t\t\t\t`\\n=== ${result.caseId} #${index + 1} ${run.pass ? 'pass' : 'FAIL'}`\n\t\t\t)\n\t\t\tfor (const check of run.checks.filter((check) => !check.pass)) {\n\t\t\t\tconsole.log(\n\t\t\t\t\t` ✗ ${check.check}${check.detail ? ` (${check.detail})` : ''}`\n\t\t\t\t)\n\t\t\t}\n\t\t\tif (run.verdict) console.log(` judge: ${JSON.stringify(run.verdict)}`)\n\t\t\tif (run.judgeError) console.log(` judge failed: ${run.judgeError}`)\n\t\t\tfor (const turn of run.trace.turns) {\n\t\t\t\tconsole.log(`\\n User: ${turn.userText}`)\n\t\t\t\tfor (const call of turn.toolCalls) {\n\t\t\t\t\tconsole.log(\n\t\t\t\t\t\t` ${call.name} [${call.state}] ${JSON.stringify(call.input).slice(0, 200)}${call.errorText ? ` ERROR ${call.errorText}` : ''}`\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\tconsole.log(\n\t\t\t\t\t` Assistant (${turn.firstTextMs === null ? '-' : `${(turn.firstTextMs / 1000).toFixed(1)}s`}, ${turn.steps} steps): ${turn.replyText}`\n\t\t\t\t)\n\t\t\t}\n\t\t})\n\t}\n}\n\nconst command = positionals[0] ?? 'run'\nswitch (command) {\n\tcase 'run':\n\t\tawait run()\n\t\tbreak\n\tcase 'inspect': {\n\t\tconst file = positionals[1]\n\t\tif (!file) throw new Error('inspect needs a report file')\n\t\tinspect(await readReport(file), values.case ?? [])\n\t\tbreak\n\t}\n\tcase 'compare': {\n\t\tconst [, baseline, candidate] = positionals\n\t\tif (!baseline || !candidate)\n\t\t\tthrow new Error('compare needs two report files')\n\t\tconsole.log(\n\t\t\tcompareReports(await readReport(baseline), await readReport(candidate))\n\t\t)\n\t\tbreak\n\t}\n\tdefault:\n\t\tthrow new Error(`Unknown command ${command}: run, inspect or compare`)\n}\n\n// App pools (database, Redis) keep the event loop alive after the last run.\nprocess.exit()\n"]}
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":";AACA,4EAA4E;AAC5E,mEAAmE;AACnE,yEAAyE;AACzE,EAAE;AACF,6EAA6E;AAC7E,2EAA2E;AAC3E,2BAA2B;AAE3B,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AACrD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAEvB,OAAO,EAAmB,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACxE,OAAO,EACN,cAAc,EAEd,qBAAqB,EACrB,YAAY,EACZ,MAAM,aAAa,CAAA;AAEpB,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,QAAQ,CAAC,oBAAoB,CAAC;IAC5E,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC;IAC3D,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,sCAAsC,CAAC;IAC1E,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0BAA0B,CAAC;IAC/D,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;IACxD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC9D,KAAK,EAAE,CAAC;SACN,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;SACjB,QAAQ,CAAC,6CAA6C,CAAC;IACzD,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,kCAAkC,CAAC;IAC7E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC1E,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;CACtD,CAAC,CAAA;AAIF,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAA;IACrD,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,wBAAwB,CAAC,CAAA;IACrE,OAAO,MAAM,CAAC,OAAO,CAAA;AACtB,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACrC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;AAChD,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,MAAkB,EAAE,MAAkB;IACzD,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAA;IACjD,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,CAAC,CAAA;IACzC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAAE,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC3E,MAAM,IAAI,GAAG,OAAO,CACnB,GAAG,EACH,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,CACtG,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;IAChE,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,+EAA+E;AAC/E,SAAS,WAAW,CAAC,OAAgB;IACpC,IAAI,OAAO;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAA;IAC5B,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;IACpC,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,KAAK,GAAG,GAAG,EAAE,GAAE,CAAC,CAAA;IACrD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAA;AAC1D,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,MAAgB;IAClC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9C,IAAI,CAAC;QACJ,OAAO,MAAM,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACrC,CAAC;YAAS,CAAC;QACV,MAAM,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAA;IAC1B,CAAC;AACF,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,MAAkB,EAAE,MAAgB;IAC1D,MAAM,MAAM,CAAC,KAAK,EAAE,EAAE,CAAA;IACtB,MAAM,GAAG,GACR,OAAO,MAAM,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAA;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAA;IAC5B,MAAM,KAAK,GAAiC,WAAW,CAAC,GAAG,EAAE;QAC5D,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1C,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAA;IACzE,MAAM,OAAO,GAAiB,EAAE,CAAA;IAChC,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QAC9B,OAAO,CAAC,KAAK,CACZ,WAAW,KAAK,CAAC,MAAM,WAAW,MAAM,CAAC,OAAO,OAAO,OAAO,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CACjH,CAAA;QACD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC3C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE;YACzC,KAAK;YACL,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK;YAClD,OAAO;YACP,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,CAC3B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;SACjE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACnB,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAA;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAChD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IACrB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAA;IACnE,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,OAAO,CAAA;IAChC,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAA;IAC7D,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,IAAI,KAAK,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,cAAc,CAAC,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,EAAE,CAAC,CAAA;IAC5E,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CACtC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CACjE,CAAA;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED,SAAS,OAAO,CAAC,MAAkB,EAAE,QAA2B;IAC/D,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACrC,IACC,QAAQ,CAAC,MAAM;YACf,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAE5D,SAAQ;QACT,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAClC,OAAO,CAAC,GAAG,CACV,SAAS,MAAM,CAAC,MAAM,KAAK,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,CACpE,CAAA;YACD,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CACV,OAAO,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/D,CAAA;YACF,CAAC;YACD,IAAI,GAAG,CAAC,OAAO;gBAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACvE,IAAI,GAAG,CAAC,UAAU;gBAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;YACpE,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAA;gBACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACnC,OAAO,CAAC,GAAG,CACV,OAAO,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACjI,CAAA;gBACF,CAAC;gBACD,OAAO,CAAC,GAAG,CACV,gBAAgB,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,SAAS,EAAE,CACvI,CAAA;YACF,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC;AACF,CAAC;AAED,MAAM,GAAG,GAAG,SAAS,CAAC;IACrB,IAAI,EAAE,qBAAqB;IAC3B,WAAW,EACV,4EAA4E;IAC7E,QAAQ,EAAE;QACT,GAAG,EAAE,aAAa,CAAC;YAClB,WAAW,EAAE,iCAAiC;YAC9C,KAAK,EAAE,QAAQ;YACf,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBACzB,GAAG,CAAC,QAAQ,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,CAAA;YAChC,CAAC;SACD,CAAC;QACF,OAAO,EAAE,aAAa,CAAC;YACtB,WAAW,EAAE,gDAAgD;YAC7D,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBACf,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;gBACrE,IAAI,EAAE,CAAC;qBACL,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;qBACjB,QAAQ,CAAC,sCAAsC,CAAC;aAClD,CAAC;YACF,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;SACzE,CAAC;QACF,OAAO,EAAE,aAAa,CAAC;YACtB,WAAW,EAAE,4CAA4C;YACzD,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBACf,QAAQ,EAAE,CAAC;qBACT,MAAM,EAAE;qBACR,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;qBAC1B,QAAQ,CAAC,sBAAsB,CAAC;gBAClC,SAAS,EAAE,CAAC;qBACV,MAAM,EAAE;qBACR,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;qBAC1B,QAAQ,CAAC,uBAAuB,CAAC;aACnC,CAAC;YACF,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CACpB,cAAc,CACb,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,EAChC,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CACjC;SACF,CAAC;KACF;CACD,CAAC,CAAA;AAEF,uEAAuE;AACvE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAClC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;AACpB,IACC,KAAK,KAAK,SAAS;IACnB,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,QAAQ,CAAC;IAE/D,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAEpB,4EAA4E;AAC5E,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA","sourcesContent":["#!/usr/bin/env node\n// better-mastra-evals: real turns through the app's production turn, scored\n// by hard checks and a judge, against the app's `evals.config.ts`.\n// `better-mastra-evals --help` lists the commands; `run` is the default.\n//\n// Several --model values run the suite on each and print a model comparison.\n// Run it with Bun (`bunx --bun better-mastra-evals`) or Node 22.18+ so the\n// TypeScript config loads.\n\nimport { mkdir, readFile, stat, writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\nimport { defineCli, defineCommand } from 'hanzio/cli'\nimport { z } from 'zod'\nimport type { EvalCase } from './case.js'\nimport { type EvalConfig, runEvalSuite, selectCases } from './config.js'\nimport {\n\tcompareReports,\n\ttype EvalReport,\n\tformatModelComparison,\n\tformatReport\n} from './report.js'\n\nconst runInput = z.object({\n\tconfig: z.string().default('evals.config.ts').describe('Eval config module'),\n\tsuite: z.string().default('smoke').describe('Suite to run'),\n\tcase: z.array(z.string()).describe('Only cases whose id starts with this'),\n\ttag: z.string().optional().describe('Only cases with this tag'),\n\trepeats: z.number().default(3).describe('Runs per case'),\n\tconcurrency: z.number().default(3).describe('Cases in flight'),\n\tmodel: z\n\t\t.array(z.string())\n\t\t.describe('Model to run on; several print a comparison'),\n\tjudge: z.boolean().default(true).describe('Score runs with the config judge'),\n\tcompare: z.string().optional().describe('Baseline report to compare with'),\n\tverbose: z.boolean().describe('Keep app console logs')\n})\n\ntype RunInput = z.output<typeof runInput>\n\nasync function loadConfig(path: string): Promise<EvalConfig> {\n\tconst file = resolve(path)\n\tconst module = await import(pathToFileURL(file).href)\n\tif (!module.default) throw new Error(`${file} has no default export`)\n\treturn module.default\n}\n\nasync function readReport(file: string): Promise<EvalReport> {\n\treturn JSON.parse(await readFile(file, 'utf8'))\n}\n\nasync function save(config: EvalConfig, report: EvalReport) {\n\tconst dir = resolve(config.outputDir ?? '.evals')\n\tawait mkdir(dir, { recursive: true })\n\tconst ignore = resolve(dir, '.gitignore')\n\tif (!(await stat(ignore).catch(() => null))) await writeFile(ignore, '*\\n')\n\tconst file = resolve(\n\t\tdir,\n\t\t`${report.startedAt.replaceAll(':', '-')}-${report.suite}-${report.modelId.replaceAll('/', '_')}.json`\n\t)\n\tawait writeFile(file, `${JSON.stringify(report, null, '\\t')}\\n`)\n\treturn file\n}\n\n/** Apps log to the console; while cases run that buries the progress lines. */\nfunction silenceLogs(verbose: boolean) {\n\tif (verbose) return () => {}\n\tconst { log, info, debug } = console\n\tconsole.log = console.info = console.debug = () => {}\n\treturn () => Object.assign(console, { log, info, debug })\n}\n\nasync function run(values: RunInput) {\n\tconst config = await loadConfig(values.config)\n\ttry {\n\t\treturn await runWith(config, values)\n\t} finally {\n\t\tawait config.teardown?.()\n\t}\n}\n\nasync function runWith(config: EvalConfig, values: RunInput) {\n\tawait config.setup?.()\n\tconst all =\n\t\ttypeof config.cases === 'function' ? await config.cases() : config.cases\n\tconst prefixes = values.case\n\tconst cases: readonly EvalCase<unknown>[] = selectCases(all, {\n\t\tsuite: values.suite,\n\t\tprefixes,\n\t\t...(values.tag ? { tag: values.tag } : {})\n\t})\n\tconst models = values.model.length ? values.model : [config.defaultModel]\n\tconst reports: EvalReport[] = []\n\tfor (const modelId of models) {\n\t\tconsole.error(\n\t\t\t`Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && values.judge ? ', judged' : ''}`\n\t\t)\n\t\tconst restore = silenceLogs(values.verbose)\n\t\tconst report = await runEvalSuite(config, {\n\t\t\tcases,\n\t\t\tsuite: prefixes.length ? 'selected' : values.suite,\n\t\t\tmodelId,\n\t\t\trepeats: values.repeats,\n\t\t\tconcurrency: values.concurrency,\n\t\t\tjudge: values.judge,\n\t\t\tonRun: (testCase, result) =>\n\t\t\t\tconsole.error(`${result.pass ? 'pass' : 'FAIL'} ${testCase.id}`)\n\t\t}).finally(restore)\n\t\tconsole.log(formatReport(report))\n\t\tconsole.log(`\\n${await save(config, report)}\\n`)\n\t\treports.push(report)\n\t}\n\tif (reports.length > 1) console.log(formatModelComparison(reports))\n\tconst [first, ...rest] = reports\n\tfor (const report of rest) {\n\t\tif (first) console.log(`\\n${compareReports(first, report)}`)\n\t}\n\tif (values.compare && first) {\n\t\tconsole.log(`\\n${compareReports(await readReport(values.compare), first)}`)\n\t}\n\tconst failed = reports.some((report) =>\n\t\treport.results.some((result) => result.runs.some((r) => !r.pass))\n\t)\n\treturn failed ? 1 : 0\n}\n\nfunction inspect(report: EvalReport, prefixes: readonly string[]) {\n\tfor (const result of report.results) {\n\t\tif (\n\t\t\tprefixes.length &&\n\t\t\t!prefixes.some((prefix) => result.caseId.startsWith(prefix))\n\t\t)\n\t\t\tcontinue\n\t\tresult.runs.forEach((run, index) => {\n\t\t\tconsole.log(\n\t\t\t\t`\\n=== ${result.caseId} #${index + 1} ${run.pass ? 'pass' : 'FAIL'}`\n\t\t\t)\n\t\t\tfor (const check of run.checks.filter((check) => !check.pass)) {\n\t\t\t\tconsole.log(\n\t\t\t\t\t` ✗ ${check.check}${check.detail ? ` (${check.detail})` : ''}`\n\t\t\t\t)\n\t\t\t}\n\t\t\tif (run.verdict) console.log(` judge: ${JSON.stringify(run.verdict)}`)\n\t\t\tif (run.judgeError) console.log(` judge failed: ${run.judgeError}`)\n\t\t\tfor (const turn of run.trace.turns) {\n\t\t\t\tconsole.log(`\\n User: ${turn.userText}`)\n\t\t\t\tfor (const call of turn.toolCalls) {\n\t\t\t\t\tconsole.log(\n\t\t\t\t\t\t` ${call.name} [${call.state}] ${JSON.stringify(call.input).slice(0, 200)}${call.errorText ? ` ERROR ${call.errorText}` : ''}`\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\tconsole.log(\n\t\t\t\t\t` Assistant (${turn.firstTextMs === null ? '-' : `${(turn.firstTextMs / 1000).toFixed(1)}s`}, ${turn.steps} steps): ${turn.replyText}`\n\t\t\t\t)\n\t\t\t}\n\t\t})\n\t}\n}\n\nconst cli = defineCli({\n\tname: 'better-mastra-evals',\n\tdescription:\n\t\t\"Real turns through the app's production turn, scored by checks and a judge\",\n\tcommands: {\n\t\trun: defineCommand({\n\t\t\tdescription: 'Run a suite and save its report',\n\t\t\tinput: runInput,\n\t\t\trun: async (input, ctx) => {\n\t\t\t\tctx.exitCode = await run(input)\n\t\t\t}\n\t\t}),\n\t\tinspect: defineCommand({\n\t\t\tdescription: \"Print a report's failed checks and transcripts\",\n\t\t\tinput: z.object({\n\t\t\t\treport: z.string().meta({ positional: true }).describe('Report file'),\n\t\t\t\tcase: z\n\t\t\t\t\t.array(z.string())\n\t\t\t\t\t.describe('Only cases whose id starts with this')\n\t\t\t}),\n\t\t\trun: async (input) => inspect(await readReport(input.report), input.case)\n\t\t}),\n\t\tcompare: defineCommand({\n\t\t\tdescription: 'Compare a candidate report with a baseline',\n\t\t\tinput: z.object({\n\t\t\t\tbaseline: z\n\t\t\t\t\t.string()\n\t\t\t\t\t.meta({ positional: true })\n\t\t\t\t\t.describe('Baseline report file'),\n\t\t\t\tcandidate: z\n\t\t\t\t\t.string()\n\t\t\t\t\t.meta({ positional: true })\n\t\t\t\t\t.describe('Candidate report file')\n\t\t\t}),\n\t\t\trun: async (input) =>\n\t\t\t\tcompareReports(\n\t\t\t\t\tawait readReport(input.baseline),\n\t\t\t\t\tawait readReport(input.candidate)\n\t\t\t\t)\n\t\t})\n\t}\n})\n\n// `run` is the default command: no command, or options first, runs it.\nconst argv = process.argv.slice(2)\nconst [first] = argv\nif (\n\tfirst === undefined ||\n\t(first.startsWith('-') && first !== '-h' && first !== '--help')\n)\n\targv.unshift('run')\n\n// App pools (database, Redis) keep the event loop alive after the last run.\nprocess.exit(await cli.run(argv))\n"]}
package/dist/tools/cli.js CHANGED
@@ -7,18 +7,34 @@
7
7
  import { writeFile } from 'node:fs/promises';
8
8
  import { resolve } from 'node:path';
9
9
  import { pathToFileURL } from 'node:url';
10
+ import { CliError, defineCli, defineCommand } from 'hanzio/cli';
11
+ import { z } from 'zod';
10
12
  import { buildToolManifest, renderToolManifestSource } from './manifest.js';
11
- const [modulePath, outPath, exportName = 'tools'] = process.argv.slice(2);
12
- if (!modulePath || !outPath) {
13
- console.error('Usage: better-mastra-tool-manifest <module> <out> [export]');
14
- process.exit(1);
15
- }
16
- const loaded = (await import(pathToFileURL(resolve(modulePath)).href));
17
- const definitions = loaded[exportName];
18
- if (!Array.isArray(definitions)) {
19
- console.error(`${modulePath} has no array export named ${exportName}.`);
20
- process.exit(1);
21
- }
22
- await writeFile(resolve(outPath), renderToolManifestSource(buildToolManifest(definitions)));
23
- console.log(`Wrote ${definitions.length} tools to ${outPath}.`);
13
+ const cli = defineCli({
14
+ name: 'better-mastra-tool-manifest',
15
+ commands: defineCommand({
16
+ description: 'Write the client tool manifest module for a tools export',
17
+ input: z.object({
18
+ module: z
19
+ .string()
20
+ .meta({ positional: true })
21
+ .describe('Module that exports the tools'),
22
+ out: z.string().meta({ positional: true }).describe('File to write'),
23
+ export: z
24
+ .string()
25
+ .default('tools')
26
+ .meta({ positional: true })
27
+ .describe('Export holding the defineTool array')
28
+ }),
29
+ run: async (input) => {
30
+ const loaded = (await import(pathToFileURL(resolve(input.module)).href));
31
+ const definitions = loaded[input.export];
32
+ if (!Array.isArray(definitions))
33
+ throw new CliError(`${input.module} has no array export named ${input.export}.`);
34
+ await writeFile(resolve(input.out), renderToolManifestSource(buildToolManifest(definitions)));
35
+ return `Wrote ${definitions.length} tools to ${input.out}.`;
36
+ }
37
+ })
38
+ });
39
+ process.exitCode = await cli.run();
24
40
  //# sourceMappingURL=cli.js.map
package/dist/tools/cli.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/tools/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAGxC,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,UAAU,GAAG,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAEzE,IAAI,CAAC,UAAU,IAAI,CAAC,OAAO,EAAE,CAAC;IAC7B,OAAO,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAC3B,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CACvC,CAA4B,CAAA;AAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,CAAA;AAEtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;IACjC,OAAO,CAAC,KAAK,CAAC,GAAG,UAAU,8BAA8B,UAAU,GAAG,CAAC,CAAA;IACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AAChB,CAAC;AAED,MAAM,SAAS,CACd,OAAO,CAAC,OAAO,CAAC,EAChB,wBAAwB,CAAC,iBAAiB,CAAC,WAA4B,CAAC,CAAC,CACzE,CAAA;AACD,OAAO,CAAC,GAAG,CAAC,SAAS,WAAW,CAAC,MAAM,aAAa,OAAO,GAAG,CAAC,CAAA","sourcesContent":["#!/usr/bin/env node\n/**\n * `better-mastra-tool-manifest <module> <out> [export]`: imports the module,\n * reads its export (default `tools`, an array of `defineTool` results) and\n * writes the client manifest module to `out`.\n */\n\nimport { writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\n\nimport type { DefinedTool } from './defineTool.js'\nimport { buildToolManifest, renderToolManifestSource } from './manifest.js'\n\nconst [modulePath, outPath, exportName = 'tools'] = process.argv.slice(2)\n\nif (!modulePath || !outPath) {\n\tconsole.error('Usage: better-mastra-tool-manifest <module> <out> [export]')\n\tprocess.exit(1)\n}\n\nconst loaded = (await import(\n\tpathToFileURL(resolve(modulePath)).href\n)) as Record<string, unknown>\nconst definitions = loaded[exportName]\n\nif (!Array.isArray(definitions)) {\n\tconsole.error(`${modulePath} has no array export named ${exportName}.`)\n\tprocess.exit(1)\n}\n\nawait writeFile(\n\tresolve(outPath),\n\trenderToolManifestSource(buildToolManifest(definitions as DefinedTool[]))\n)\nconsole.log(`Wrote ${definitions.length} tools to ${outPath}.`)\n"]}
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/tools/cli.ts"],"names":[],"mappings":";AACA;;;;GAIG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AAGvB,OAAO,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,GAAG,GAAG,SAAS,CAAC;IACrB,IAAI,EAAE,6BAA6B;IACnC,QAAQ,EAAE,aAAa,CAAC;QACvB,WAAW,EAAE,0DAA0D;QACvE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACf,MAAM,EAAE,CAAC;iBACP,MAAM,EAAE;iBACR,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;iBAC1B,QAAQ,CAAC,+BAA+B,CAAC;YAC3C,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;YACpE,MAAM,EAAE,CAAC;iBACP,MAAM,EAAE;iBACR,OAAO,CAAC,OAAO,CAAC;iBAChB,IAAI,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;iBAC1B,QAAQ,CAAC,qCAAqC,CAAC;SACjD,CAAC;QACF,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAC3B,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CACzC,CAA4B,CAAA;YAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;YACxC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;gBAC9B,MAAM,IAAI,QAAQ,CACjB,GAAG,KAAK,CAAC,MAAM,8BAA8B,KAAK,CAAC,MAAM,GAAG,CAC5D,CAAA;YACF,MAAM,SAAS,CACd,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAClB,wBAAwB,CACvB,iBAAiB,CAAC,WAA4B,CAAC,CAC/C,CACD,CAAA;YACD,OAAO,SAAS,WAAW,CAAC,MAAM,aAAa,KAAK,CAAC,GAAG,GAAG,CAAA;QAC5D,CAAC;KACD,CAAC;CACF,CAAC,CAAA;AAEF,OAAO,CAAC,QAAQ,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAA","sourcesContent":["#!/usr/bin/env node\n/**\n * `better-mastra-tool-manifest <module> <out> [export]`: imports the module,\n * reads its export (default `tools`, an array of `defineTool` results) and\n * writes the client manifest module to `out`.\n */\n\nimport { writeFile } from 'node:fs/promises'\nimport { resolve } from 'node:path'\nimport { pathToFileURL } from 'node:url'\nimport { CliError, defineCli, defineCommand } from 'hanzio/cli'\nimport { z } from 'zod'\n\nimport type { DefinedTool } from './defineTool.js'\nimport { buildToolManifest, renderToolManifestSource } from './manifest.js'\n\nconst cli = defineCli({\n\tname: 'better-mastra-tool-manifest',\n\tcommands: defineCommand({\n\t\tdescription: 'Write the client tool manifest module for a tools export',\n\t\tinput: z.object({\n\t\t\tmodule: z\n\t\t\t\t.string()\n\t\t\t\t.meta({ positional: true })\n\t\t\t\t.describe('Module that exports the tools'),\n\t\t\tout: z.string().meta({ positional: true }).describe('File to write'),\n\t\t\texport: z\n\t\t\t\t.string()\n\t\t\t\t.default('tools')\n\t\t\t\t.meta({ positional: true })\n\t\t\t\t.describe('Export holding the defineTool array')\n\t\t}),\n\t\trun: async (input) => {\n\t\t\tconst loaded = (await import(\n\t\t\t\tpathToFileURL(resolve(input.module)).href\n\t\t\t)) as Record<string, unknown>\n\t\t\tconst definitions = loaded[input.export]\n\t\t\tif (!Array.isArray(definitions))\n\t\t\t\tthrow new CliError(\n\t\t\t\t\t`${input.module} has no array export named ${input.export}.`\n\t\t\t\t)\n\t\t\tawait writeFile(\n\t\t\t\tresolve(input.out),\n\t\t\t\trenderToolManifestSource(\n\t\t\t\t\tbuildToolManifest(definitions as DefinedTool[])\n\t\t\t\t)\n\t\t\t)\n\t\t\treturn `Wrote ${definitions.length} tools to ${input.out}.`\n\t\t}\n\t})\n})\n\nprocess.exitCode = await cli.run()\n"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "better-mastra",
3
- "version": "0.1.0",
3
+ "version": "0.1.1",
4
4
  "description": "A small layer on Mastra and the AI SDK: model registry, chat turns, resumable streams and chat clients for React and Vue.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -107,6 +107,9 @@
107
107
  "publint": "publint",
108
108
  "verify:package": "node scripts/verify-package.mjs"
109
109
  },
110
+ "dependencies": {
111
+ "hanzio": "^3.1.0"
112
+ },
110
113
  "peerDependencies": {
111
114
  "@ai-sdk/anthropic": "^4.0.0",
112
115
  "@ai-sdk/google-vertex": "^5.0.0",
package/src/evals/cli.ts CHANGED
@@ -1,12 +1,7 @@
1
1
  #!/usr/bin/env node
2
2
  // better-mastra-evals: real turns through the app's production turn, scored
3
3
  // by hard checks and a judge, against the app's `evals.config.ts`.
4
- //
5
- // better-mastra-evals run [--suite smoke] [--case <id-prefix>]... [--tag <tag>]
6
- // [--repeats 3] [--concurrency 3] [--model <id>]...
7
- // [--no-judge] [--compare <baseline.json>] [--verbose]
8
- // better-mastra-evals inspect <report.json> [--case <id-prefix>]...
9
- // better-mastra-evals compare <baseline.json> <candidate.json>
4
+ // `better-mastra-evals --help` lists the commands; `run` is the default.
10
5
  //
11
6
  // Several --model values run the suite on each and print a model comparison.
12
7
  // Run it with Bun (`bunx --bun better-mastra-evals`) or Node 22.18+ so the
@@ -15,7 +10,8 @@
15
10
  import { mkdir, readFile, stat, writeFile } from 'node:fs/promises'
16
11
  import { resolve } from 'node:path'
17
12
  import { pathToFileURL } from 'node:url'
18
- import { parseArgs } from 'node:util'
13
+ import { defineCli, defineCommand } from 'hanzio/cli'
14
+ import { z } from 'zod'
19
15
  import type { EvalCase } from './case.js'
20
16
  import { type EvalConfig, runEvalSuite, selectCases } from './config.js'
21
17
  import {
@@ -25,25 +21,25 @@ import {
25
21
  formatReport
26
22
  } from './report.js'
27
23
 
28
- const { positionals, values } = parseArgs({
29
- args: process.argv.slice(2),
30
- allowPositionals: true,
31
- options: {
32
- config: { type: 'string', default: 'evals.config.ts' },
33
- suite: { type: 'string', default: 'smoke' },
34
- case: { type: 'string', multiple: true },
35
- tag: { type: 'string' },
36
- repeats: { type: 'string', default: '3' },
37
- concurrency: { type: 'string', default: '3' },
38
- model: { type: 'string', multiple: true },
39
- 'no-judge': { type: 'boolean', default: false },
40
- compare: { type: 'string' },
41
- verbose: { type: 'boolean', default: false }
42
- }
24
+ const runInput = z.object({
25
+ config: z.string().default('evals.config.ts').describe('Eval config module'),
26
+ suite: z.string().default('smoke').describe('Suite to run'),
27
+ case: z.array(z.string()).describe('Only cases whose id starts with this'),
28
+ tag: z.string().optional().describe('Only cases with this tag'),
29
+ repeats: z.number().default(3).describe('Runs per case'),
30
+ concurrency: z.number().default(3).describe('Cases in flight'),
31
+ model: z
32
+ .array(z.string())
33
+ .describe('Model to run on; several print a comparison'),
34
+ judge: z.boolean().default(true).describe('Score runs with the config judge'),
35
+ compare: z.string().optional().describe('Baseline report to compare with'),
36
+ verbose: z.boolean().describe('Keep app console logs')
43
37
  })
44
38
 
45
- async function loadConfig(): Promise<EvalConfig> {
46
- const file = resolve(values.config)
39
+ type RunInput = z.output<typeof runInput>
40
+
41
+ async function loadConfig(path: string): Promise<EvalConfig> {
42
+ const file = resolve(path)
47
43
  const module = await import(pathToFileURL(file).href)
48
44
  if (!module.default) throw new Error(`${file} has no default export`)
49
45
  return module.default
@@ -67,46 +63,46 @@ async function save(config: EvalConfig, report: EvalReport) {
67
63
  }
68
64
 
69
65
  /** Apps log to the console; while cases run that buries the progress lines. */
70
- function silenceLogs() {
71
- if (values.verbose) return () => {}
66
+ function silenceLogs(verbose: boolean) {
67
+ if (verbose) return () => {}
72
68
  const { log, info, debug } = console
73
69
  console.log = console.info = console.debug = () => {}
74
70
  return () => Object.assign(console, { log, info, debug })
75
71
  }
76
72
 
77
- async function run() {
78
- const config = await loadConfig()
73
+ async function run(values: RunInput) {
74
+ const config = await loadConfig(values.config)
79
75
  try {
80
- await runWith(config)
76
+ return await runWith(config, values)
81
77
  } finally {
82
78
  await config.teardown?.()
83
79
  }
84
80
  }
85
81
 
86
- async function runWith(config: EvalConfig) {
82
+ async function runWith(config: EvalConfig, values: RunInput) {
87
83
  await config.setup?.()
88
84
  const all =
89
85
  typeof config.cases === 'function' ? await config.cases() : config.cases
90
- const prefixes = values.case ?? []
86
+ const prefixes = values.case
91
87
  const cases: readonly EvalCase<unknown>[] = selectCases(all, {
92
88
  suite: values.suite,
93
89
  prefixes,
94
90
  ...(values.tag ? { tag: values.tag } : {})
95
91
  })
96
- const models = values.model?.length ? values.model : [config.defaultModel]
92
+ const models = values.model.length ? values.model : [config.defaultModel]
97
93
  const reports: EvalReport[] = []
98
94
  for (const modelId of models) {
99
95
  console.error(
100
- `Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && !values['no-judge'] ? ', judged' : ''}`
96
+ `Running ${cases.length} cases x${values.repeats} on ${modelId}${config.judge && values.judge ? ', judged' : ''}`
101
97
  )
102
- const restore = silenceLogs()
98
+ const restore = silenceLogs(values.verbose)
103
99
  const report = await runEvalSuite(config, {
104
100
  cases,
105
101
  suite: prefixes.length ? 'selected' : values.suite,
106
102
  modelId,
107
- repeats: Number(values.repeats),
108
- concurrency: Number(values.concurrency),
109
- judge: !values['no-judge'],
103
+ repeats: values.repeats,
104
+ concurrency: values.concurrency,
105
+ judge: values.judge,
110
106
  onRun: (testCase, result) =>
111
107
  console.error(`${result.pass ? 'pass' : 'FAIL'} ${testCase.id}`)
112
108
  }).finally(restore)
@@ -125,7 +121,7 @@ async function runWith(config: EvalConfig) {
125
121
  const failed = reports.some((report) =>
126
122
  report.results.some((result) => result.runs.some((r) => !r.pass))
127
123
  )
128
- if (failed) process.exitCode = 1
124
+ return failed ? 1 : 0
129
125
  }
130
126
 
131
127
  function inspect(report: EvalReport, prefixes: readonly string[]) {
@@ -161,29 +157,57 @@ function inspect(report: EvalReport, prefixes: readonly string[]) {
161
157
  }
162
158
  }
163
159
 
164
- const command = positionals[0] ?? 'run'
165
- switch (command) {
166
- case 'run':
167
- await run()
168
- break
169
- case 'inspect': {
170
- const file = positionals[1]
171
- if (!file) throw new Error('inspect needs a report file')
172
- inspect(await readReport(file), values.case ?? [])
173
- break
174
- }
175
- case 'compare': {
176
- const [, baseline, candidate] = positionals
177
- if (!baseline || !candidate)
178
- throw new Error('compare needs two report files')
179
- console.log(
180
- compareReports(await readReport(baseline), await readReport(candidate))
181
- )
182
- break
160
+ const cli = defineCli({
161
+ name: 'better-mastra-evals',
162
+ description:
163
+ "Real turns through the app's production turn, scored by checks and a judge",
164
+ commands: {
165
+ run: defineCommand({
166
+ description: 'Run a suite and save its report',
167
+ input: runInput,
168
+ run: async (input, ctx) => {
169
+ ctx.exitCode = await run(input)
170
+ }
171
+ }),
172
+ inspect: defineCommand({
173
+ description: "Print a report's failed checks and transcripts",
174
+ input: z.object({
175
+ report: z.string().meta({ positional: true }).describe('Report file'),
176
+ case: z
177
+ .array(z.string())
178
+ .describe('Only cases whose id starts with this')
179
+ }),
180
+ run: async (input) => inspect(await readReport(input.report), input.case)
181
+ }),
182
+ compare: defineCommand({
183
+ description: 'Compare a candidate report with a baseline',
184
+ input: z.object({
185
+ baseline: z
186
+ .string()
187
+ .meta({ positional: true })
188
+ .describe('Baseline report file'),
189
+ candidate: z
190
+ .string()
191
+ .meta({ positional: true })
192
+ .describe('Candidate report file')
193
+ }),
194
+ run: async (input) =>
195
+ compareReports(
196
+ await readReport(input.baseline),
197
+ await readReport(input.candidate)
198
+ )
199
+ })
183
200
  }
184
- default:
185
- throw new Error(`Unknown command ${command}: run, inspect or compare`)
186
- }
201
+ })
202
+
203
+ // `run` is the default command: no command, or options first, runs it.
204
+ const argv = process.argv.slice(2)
205
+ const [first] = argv
206
+ if (
207
+ first === undefined ||
208
+ (first.startsWith('-') && first !== '-h' && first !== '--help')
209
+ )
210
+ argv.unshift('run')
187
211
 
188
212
  // App pools (database, Redis) keep the event loop alive after the last run.
189
- process.exit()
213
+ process.exit(await cli.run(argv))
package/src/tools/cli.ts CHANGED
@@ -8,29 +8,46 @@
8
8
  import { writeFile } from 'node:fs/promises'
9
9
  import { resolve } from 'node:path'
10
10
  import { pathToFileURL } from 'node:url'
11
+ import { CliError, defineCli, defineCommand } from 'hanzio/cli'
12
+ import { z } from 'zod'
11
13
 
12
14
  import type { DefinedTool } from './defineTool.js'
13
15
  import { buildToolManifest, renderToolManifestSource } from './manifest.js'
14
16
 
15
- const [modulePath, outPath, exportName = 'tools'] = process.argv.slice(2)
17
+ const cli = defineCli({
18
+ name: 'better-mastra-tool-manifest',
19
+ commands: defineCommand({
20
+ description: 'Write the client tool manifest module for a tools export',
21
+ input: z.object({
22
+ module: z
23
+ .string()
24
+ .meta({ positional: true })
25
+ .describe('Module that exports the tools'),
26
+ out: z.string().meta({ positional: true }).describe('File to write'),
27
+ export: z
28
+ .string()
29
+ .default('tools')
30
+ .meta({ positional: true })
31
+ .describe('Export holding the defineTool array')
32
+ }),
33
+ run: async (input) => {
34
+ const loaded = (await import(
35
+ pathToFileURL(resolve(input.module)).href
36
+ )) as Record<string, unknown>
37
+ const definitions = loaded[input.export]
38
+ if (!Array.isArray(definitions))
39
+ throw new CliError(
40
+ `${input.module} has no array export named ${input.export}.`
41
+ )
42
+ await writeFile(
43
+ resolve(input.out),
44
+ renderToolManifestSource(
45
+ buildToolManifest(definitions as DefinedTool[])
46
+ )
47
+ )
48
+ return `Wrote ${definitions.length} tools to ${input.out}.`
49
+ }
50
+ })
51
+ })
16
52
 
17
- if (!modulePath || !outPath) {
18
- console.error('Usage: better-mastra-tool-manifest <module> <out> [export]')
19
- process.exit(1)
20
- }
21
-
22
- const loaded = (await import(
23
- pathToFileURL(resolve(modulePath)).href
24
- )) as Record<string, unknown>
25
- const definitions = loaded[exportName]
26
-
27
- if (!Array.isArray(definitions)) {
28
- console.error(`${modulePath} has no array export named ${exportName}.`)
29
- process.exit(1)
30
- }
31
-
32
- await writeFile(
33
- resolve(outPath),
34
- renderToolManifestSource(buildToolManifest(definitions as DefinedTool[]))
35
- )
36
- console.log(`Wrote ${definitions.length} tools to ${outPath}.`)
53
+ process.exitCode = await cli.run()