@metrio-ai/cli 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +29 -0
- package/README.md +38 -0
- package/dist/auth/credentials.js +81 -0
- package/dist/auth/login.js +143 -0
- package/dist/auth/oauth.js +44 -0
- package/dist/commands/auth.js +74 -0
- package/dist/commands/chat.js +235 -0
- package/dist/commands/prompt-create-version.js +211 -0
- package/dist/commands/prompt.js +284 -0
- package/dist/commands/shared.js +328 -0
- package/dist/commands/tools.js +114 -0
- package/dist/commands/workflow.js +666 -0
- package/dist/console/client.js +108 -0
- package/dist/errors.js +27 -0
- package/dist/index.js +105 -0
- package/package.json +41 -0
|
@@ -0,0 +1,666 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.TRACE_MODES = void 0;
|
|
37
|
+
exports.shapeWorkflowOutput = shapeWorkflowOutput;
|
|
38
|
+
exports.runWorkflowCommand = runWorkflowCommand;
|
|
39
|
+
exports.parseWorkflowCases = parseWorkflowCases;
|
|
40
|
+
exports.mergeCaseJson = mergeCaseJson;
|
|
41
|
+
exports.runWorkflowBatch = runWorkflowBatch;
|
|
42
|
+
const client_1 = require("@metrio-ai/client");
|
|
43
|
+
const client_2 = require("@metrio-ai/client");
|
|
44
|
+
const shared_1 = require("./shared");
|
|
45
|
+
const path = __importStar(require("path"));
|
|
46
|
+
const WORKFLOW_HELP = `metrio workflow - run MetrioAI workflows
|
|
47
|
+
|
|
48
|
+
Usage:
|
|
49
|
+
metrio workflow run [options] Run one workflow completion
|
|
50
|
+
metrio workflow run-batch [options] Run several cases in one process
|
|
51
|
+
|
|
52
|
+
Run "metrio workflow run --help" or "metrio workflow run-batch --help" for the
|
|
53
|
+
full option list of either command.
|
|
54
|
+
`;
|
|
55
|
+
const WORKFLOW_RUN_HELP = `metrio workflow run - run a MetrioAI workflow completion
|
|
56
|
+
|
|
57
|
+
Usage:
|
|
58
|
+
metrio workflow run --workflow-id <slug> (--message <text> | --messages-file <path>) [options]
|
|
59
|
+
|
|
60
|
+
Required:
|
|
61
|
+
-w, --workflow-id <slug> Workflow slug (e.g. groceriesfriends-order)
|
|
62
|
+
One of:
|
|
63
|
+
--message <text> A single user text message
|
|
64
|
+
--messages-file <path> Path to a messages JSON file (same format as chat)
|
|
65
|
+
|
|
66
|
+
Options:
|
|
67
|
+
--version <n> Pin a workflow version (positive integer; default: active version)
|
|
68
|
+
--channel <json> Channel JSON, e.g. '{"User":{"name":"...","customer_id":"..."}}'
|
|
69
|
+
--variables <json> Variables JSON, surfaced as $variables
|
|
70
|
+
--timezone <tz> IANA timezone, e.g. "Asia/Taipei"
|
|
71
|
+
--request-tag <tag> Request-level tag recorded on the run, e.g. "test"
|
|
72
|
+
(repeatable; max 5; each 1-32 chars, lowercase
|
|
73
|
+
alphanumeric start, may contain "-" and "_")
|
|
74
|
+
--trace <mode> Node trace detail; requires --json (default: summary)
|
|
75
|
+
path node_id, type, went_to, elapsed_ms
|
|
76
|
+
(+ error when a node failed); no state
|
|
77
|
+
summary same fields, adds "state" (the default)
|
|
78
|
+
full adds per-node input/output (debugging)
|
|
79
|
+
-e, --env <file> Env file to load (default: .env in the current dir)
|
|
80
|
+
--api-key <key> API key (overrides env file / environment)
|
|
81
|
+
--base-url <url> Base API URL (overrides env file / environment)
|
|
82
|
+
--json Print the full response JSON (default: reply only)
|
|
83
|
+
-h, --help Show this help
|
|
84
|
+
|
|
85
|
+
Environment (read from the env file or the shell; flags take precedence):
|
|
86
|
+
METRIOAI_API_KEY / METRIO_API_KEY / METRIO_PROJECT_API_KEY API key
|
|
87
|
+
METRIOAI_BASE_URL / METRIO_BASE_URL Base URL
|
|
88
|
+
|
|
89
|
+
By default a human-readable reply is printed to stdout (text content, or the
|
|
90
|
+
pretty-printed structContent for struct replies), with a one-line run summary
|
|
91
|
+
(llmCalls, elapsedTime, taskId) on stderr. Use --json for the full response.
|
|
92
|
+
With --json the output is flattened for assertions: "path" (the node id
|
|
93
|
+
sequence), "terminalNodeId" and "tags" are always present — including on
|
|
94
|
+
silent runs — so one call is enough to verify a route.
|
|
95
|
+
`;
|
|
96
|
+
/** Accepted `--trace` values, in help-text order. */
|
|
97
|
+
exports.TRACE_MODES = ['path', 'summary', 'full'];
|
|
98
|
+
/**
|
|
99
|
+
* Flattens a workflow response into the shape assertions read.
|
|
100
|
+
*
|
|
101
|
+
* Adds `path` (the node id sequence) and guarantees `terminalNodeId`, deriving
|
|
102
|
+
* it from the trace when talking to a backend that predates the field. Every
|
|
103
|
+
* field the API returned is preserved, so this is always a superset of the raw
|
|
104
|
+
* body — modes only decide how much trace detail comes along.
|
|
105
|
+
*/
|
|
106
|
+
function shapeWorkflowOutput(res, mode) {
|
|
107
|
+
const nodeTrace = res.nodeTrace ?? [];
|
|
108
|
+
const shaped = {
|
|
109
|
+
...res,
|
|
110
|
+
path: nodeTrace.map((t) => t.node_id),
|
|
111
|
+
};
|
|
112
|
+
if (shaped.terminalNodeId === undefined) {
|
|
113
|
+
shaped.terminalNodeId = nodeTrace.length
|
|
114
|
+
? nodeTrace[nodeTrace.length - 1].node_id
|
|
115
|
+
: null;
|
|
116
|
+
}
|
|
117
|
+
// The documented --json contract promises `tags` (and, outside path mode,
|
|
118
|
+
// `state`) are always present, so default them when an older backend that
|
|
119
|
+
// predates the fields omits them — assertions must not be backend-version-
|
|
120
|
+
// dependent.
|
|
121
|
+
if (shaped.tags === undefined) {
|
|
122
|
+
shaped.tags = [];
|
|
123
|
+
}
|
|
124
|
+
if (shaped.state === undefined) {
|
|
125
|
+
shaped.state = {};
|
|
126
|
+
}
|
|
127
|
+
// `state` is the workflow's own run summary; path mode is deliberately the
|
|
128
|
+
// minimal routing view, so it does not carry it.
|
|
129
|
+
if (mode === 'path') {
|
|
130
|
+
delete shaped.state;
|
|
131
|
+
}
|
|
132
|
+
// The API only sends input/output for trace:"full", but strip defensively so
|
|
133
|
+
// the mode alone determines the output size.
|
|
134
|
+
if (mode !== 'full') {
|
|
135
|
+
shaped.nodeTrace = nodeTrace.map(({ input, output, ...rest }) => rest);
|
|
136
|
+
}
|
|
137
|
+
return shaped;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Parses the `--channel` flag. The runtime surfaces it as `$channel` and reads
|
|
141
|
+
* named keys off it, so a non-object (an array, a bare string) would silently
|
|
142
|
+
* produce an empty `$channel` — reject it at the CLI instead.
|
|
143
|
+
*/
|
|
144
|
+
function parseChannelFlag(raw) {
|
|
145
|
+
if (raw === undefined)
|
|
146
|
+
return undefined;
|
|
147
|
+
const parsed = (0, shared_1.parseJsonFlag)(raw, '--channel');
|
|
148
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
149
|
+
throw new Error('--channel must be a JSON object');
|
|
150
|
+
}
|
|
151
|
+
return parsed;
|
|
152
|
+
}
|
|
153
|
+
/** Parses argv for `workflow run`. Throws on malformed input. */
|
|
154
|
+
function parseWorkflowArgs(argv) {
|
|
155
|
+
const options = { json: false, help: false, tags: [] };
|
|
156
|
+
const next = (i, flag) => {
|
|
157
|
+
const value = argv[i + 1];
|
|
158
|
+
if (value === undefined) {
|
|
159
|
+
throw new Error(`Missing value for ${flag}`);
|
|
160
|
+
}
|
|
161
|
+
return value;
|
|
162
|
+
};
|
|
163
|
+
for (let i = 0; i < argv.length; i++) {
|
|
164
|
+
let arg = argv[i];
|
|
165
|
+
let inline;
|
|
166
|
+
// Accept both `--flag value` and `--flag=value`.
|
|
167
|
+
const eq = arg.indexOf('=');
|
|
168
|
+
if (arg.startsWith('--') && eq > 2) {
|
|
169
|
+
inline = arg.slice(eq + 1);
|
|
170
|
+
arg = arg.slice(0, eq);
|
|
171
|
+
}
|
|
172
|
+
const take = (flag) => {
|
|
173
|
+
if (inline !== undefined)
|
|
174
|
+
return inline;
|
|
175
|
+
return next(i++, flag);
|
|
176
|
+
};
|
|
177
|
+
switch (arg) {
|
|
178
|
+
case '-w':
|
|
179
|
+
case '--workflow-id':
|
|
180
|
+
options.workflowId = take(arg);
|
|
181
|
+
break;
|
|
182
|
+
case '--message':
|
|
183
|
+
options.message = take(arg);
|
|
184
|
+
break;
|
|
185
|
+
case '--messages-file':
|
|
186
|
+
options.messagesFile = take(arg);
|
|
187
|
+
break;
|
|
188
|
+
case '--version': {
|
|
189
|
+
const raw = take(arg);
|
|
190
|
+
const v = Number(raw);
|
|
191
|
+
if (!Number.isInteger(v) || v <= 0) {
|
|
192
|
+
throw new Error(`--version must be a positive integer, got "${raw}"`);
|
|
193
|
+
}
|
|
194
|
+
options.version = v;
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
197
|
+
case '--channel':
|
|
198
|
+
options.channel = take(arg);
|
|
199
|
+
break;
|
|
200
|
+
case '--variables':
|
|
201
|
+
options.variables = take(arg);
|
|
202
|
+
break;
|
|
203
|
+
case '--timezone':
|
|
204
|
+
options.timezone = take(arg);
|
|
205
|
+
break;
|
|
206
|
+
case '--request-tag': {
|
|
207
|
+
const tag = take(arg).trim();
|
|
208
|
+
if (!tag) {
|
|
209
|
+
throw new Error('--request-tag cannot be empty');
|
|
210
|
+
}
|
|
211
|
+
options.tags.push(tag);
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
case '--trace': {
|
|
215
|
+
const raw = take(arg);
|
|
216
|
+
if (!exports.TRACE_MODES.includes(raw)) {
|
|
217
|
+
throw new Error(`--trace must be one of ${exports.TRACE_MODES.join('|')}, got "${raw}"`);
|
|
218
|
+
}
|
|
219
|
+
options.trace = raw;
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
case '-e':
|
|
223
|
+
case '--env':
|
|
224
|
+
options.envFile = take(arg);
|
|
225
|
+
break;
|
|
226
|
+
case '--api-key':
|
|
227
|
+
options.apiKey = take(arg);
|
|
228
|
+
break;
|
|
229
|
+
case '--base-url':
|
|
230
|
+
options.baseUrl = take(arg);
|
|
231
|
+
break;
|
|
232
|
+
case '--json':
|
|
233
|
+
if (inline !== undefined) {
|
|
234
|
+
throw new Error('--json does not take a value');
|
|
235
|
+
}
|
|
236
|
+
options.json = true;
|
|
237
|
+
break;
|
|
238
|
+
case '-h':
|
|
239
|
+
case '--help':
|
|
240
|
+
if (inline !== undefined) {
|
|
241
|
+
throw new Error('--help does not take a value');
|
|
242
|
+
}
|
|
243
|
+
options.help = true;
|
|
244
|
+
break;
|
|
245
|
+
default:
|
|
246
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return options;
|
|
250
|
+
}
|
|
251
|
+
/** The `workflow run` subcommand: run a workflow completion. */
|
|
252
|
+
async function runWorkflowCommand(argv) {
|
|
253
|
+
const action = argv[0];
|
|
254
|
+
if (action === undefined || action === '-h' || action === '--help') {
|
|
255
|
+
process.stdout.write(WORKFLOW_HELP);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (action === 'run-batch') {
|
|
259
|
+
await runWorkflowBatch(argv.slice(1));
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
if (action !== 'run') {
|
|
263
|
+
throw new Error(`Unknown workflow command: "${action}". Try "workflow run" or "workflow run-batch".`);
|
|
264
|
+
}
|
|
265
|
+
const options = parseWorkflowArgs(argv.slice(1));
|
|
266
|
+
if (options.help) {
|
|
267
|
+
process.stdout.write(WORKFLOW_RUN_HELP);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
if (!options.workflowId || options.workflowId.trim() === '') {
|
|
271
|
+
throw new Error('Missing required option: --workflow-id');
|
|
272
|
+
}
|
|
273
|
+
if (options.message === undefined && options.messagesFile === undefined) {
|
|
274
|
+
throw new Error('Provide either --message <text> or --messages-file <path>');
|
|
275
|
+
}
|
|
276
|
+
if (options.message !== undefined && options.messagesFile !== undefined) {
|
|
277
|
+
throw new Error('Use only one of --message or --messages-file');
|
|
278
|
+
}
|
|
279
|
+
if (options.trace !== undefined && !options.json) {
|
|
280
|
+
throw new Error('--trace requires --json (the human-readable output prints the reply only)');
|
|
281
|
+
}
|
|
282
|
+
(0, shared_1.loadEnv)(options.envFile);
|
|
283
|
+
// Build messages: either a single user text message or a full messages file.
|
|
284
|
+
let messages;
|
|
285
|
+
if (options.messagesFile !== undefined) {
|
|
286
|
+
const baseDir = path.dirname(path.resolve(options.messagesFile));
|
|
287
|
+
messages = (0, shared_1.buildMessages)((0, shared_1.readMessagesFile)(options.messagesFile), baseDir);
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
if (options.message.trim() === '') {
|
|
291
|
+
throw new Error('--message cannot be empty');
|
|
292
|
+
}
|
|
293
|
+
messages = [{ role: 'user', content: { type: 'text', text: options.message } }];
|
|
294
|
+
}
|
|
295
|
+
if (messages.length === 0) {
|
|
296
|
+
throw new Error('Messages cannot be empty');
|
|
297
|
+
}
|
|
298
|
+
const channel = parseChannelFlag(options.channel);
|
|
299
|
+
const variables = options.variables !== undefined ? (0, shared_1.parseJsonFlag)(options.variables, '--variables') : undefined;
|
|
300
|
+
const client = new client_1.MetrioAI({
|
|
301
|
+
apiKey: (0, shared_1.resolveApiKey)(options.apiKey),
|
|
302
|
+
baseUrl: (0, shared_1.resolveBaseUrl)(options.baseUrl),
|
|
303
|
+
});
|
|
304
|
+
const req = {
|
|
305
|
+
workflowId: options.workflowId,
|
|
306
|
+
messages,
|
|
307
|
+
version: options.version,
|
|
308
|
+
channel,
|
|
309
|
+
variables,
|
|
310
|
+
timezone: options.timezone,
|
|
311
|
+
tags: options.tags.length ? options.tags : undefined,
|
|
312
|
+
// Only "full" needs the heavier server-side trace; path and summary are
|
|
313
|
+
// both served by the default routing trace.
|
|
314
|
+
trace: options.trace === 'full' ? 'full' : undefined,
|
|
315
|
+
};
|
|
316
|
+
const res = await client.workflowCompletion(req);
|
|
317
|
+
if (options.json) {
|
|
318
|
+
const shaped = shapeWorkflowOutput(res, options.trace ?? 'summary');
|
|
319
|
+
process.stdout.write(`${JSON.stringify(shaped, null, 2)}\n`);
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
const reply = res.reply;
|
|
323
|
+
if (!reply) {
|
|
324
|
+
process.stdout.write(`${res.response ?? ''}\n`);
|
|
325
|
+
}
|
|
326
|
+
else if (reply.responseType === 'struct') {
|
|
327
|
+
process.stdout.write(`${JSON.stringify(reply.structContent, null, 2)}\n`);
|
|
328
|
+
}
|
|
329
|
+
else if (reply.responseType === 'text') {
|
|
330
|
+
process.stdout.write(`${reply.content ?? ''}\n`);
|
|
331
|
+
}
|
|
332
|
+
else {
|
|
333
|
+
process.stdout.write(`${reply.reason ?? '(no response)'}\n`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
// Run summary goes to stderr so stdout stays clean for piping. A disabled
|
|
337
|
+
// workflow returns 200 with skipped=true, so surface that instead of letting
|
|
338
|
+
// it look like a normal successful run.
|
|
339
|
+
const summary = [
|
|
340
|
+
`version=${res.version}`,
|
|
341
|
+
`llmCalls=${res.llmCalls}`,
|
|
342
|
+
`elapsedTime=${res.elapsedTime}ms`,
|
|
343
|
+
`taskId=${res.taskId}`,
|
|
344
|
+
];
|
|
345
|
+
if (res.terminalNodeId)
|
|
346
|
+
summary.push(`terminal=${res.terminalNodeId}`);
|
|
347
|
+
if (res.tags?.length)
|
|
348
|
+
summary.push(`tags=${res.tags.join(',')}`);
|
|
349
|
+
if (res.skipped)
|
|
350
|
+
summary.push('skipped=true');
|
|
351
|
+
process.stderr.write(`workflow run: ${summary.join(', ')}\n`);
|
|
352
|
+
}
|
|
353
|
+
const WORKFLOW_BATCH_HELP = `metrio workflow run-batch - run several workflow cases in one process
|
|
354
|
+
|
|
355
|
+
Usage:
|
|
356
|
+
metrio workflow run-batch -w <slug> --cases <file.jsonl> [options]
|
|
357
|
+
|
|
358
|
+
Required:
|
|
359
|
+
-w, --workflow-id <slug> Workflow slug (e.g. groceriesfriends-order)
|
|
360
|
+
--cases <file> JSONL cases file; "-" reads stdin
|
|
361
|
+
|
|
362
|
+
Options:
|
|
363
|
+
--concurrency <n> Cases to run at once (default: 4)
|
|
364
|
+
--version <n> Pin a workflow version for every case
|
|
365
|
+
(positive integer; a case may override)
|
|
366
|
+
--channel <json> Channel JSON applied to every case
|
|
367
|
+
--variables <json> Variables JSON applied to every case
|
|
368
|
+
--timezone <tz> IANA timezone, e.g. "Asia/Taipei"
|
|
369
|
+
--request-tag <tag> Request-level tag recorded on every run, e.g. "test"
|
|
370
|
+
(repeatable; max 5; each 1-32 chars, lowercase
|
|
371
|
+
alphanumeric start, may contain "-" and "_")
|
|
372
|
+
--trace <mode> Node trace detail per case (default: summary;
|
|
373
|
+
no --json needed, batch output is always JSON)
|
|
374
|
+
path node_id, type, went_to, elapsed_ms
|
|
375
|
+
(+ error when a node failed); no state
|
|
376
|
+
summary same fields, adds "state" (the default)
|
|
377
|
+
full adds per-node input/output (debugging)
|
|
378
|
+
-o, --out <file> Write result JSONL here (default: stdout)
|
|
379
|
+
-e, --env <file> Env file to load (default: .env in the current dir)
|
|
380
|
+
--api-key <key> API key (overrides env file / environment)
|
|
381
|
+
--base-url <url> Base API URL (overrides env file / environment)
|
|
382
|
+
-h, --help Show this help
|
|
383
|
+
|
|
384
|
+
Cases file format (one JSON object per line; blank and "//" lines are ignored):
|
|
385
|
+
{"id":"c1","message":"我要訂三鳥"}
|
|
386
|
+
{"id":"c2","messages":[{"role":"user","content":"…"}],"channel":{"User":{"name":"…"}},"variables":{"k":"v"},"version":3}
|
|
387
|
+
|
|
388
|
+
Each case takes either "message" (a single user text) or "messages" (the same
|
|
389
|
+
format as the chat command's messages file, including "file" attachments —
|
|
390
|
+
paths resolve relative to the cases file, or the current directory when reading
|
|
391
|
+
stdin). A case without an "id" is named by its line number, e.g. "case-3".
|
|
392
|
+
|
|
393
|
+
"channel" and "variables" merge key by key over the global flags with the case
|
|
394
|
+
winning; "version" replaces the global --version for that case.
|
|
395
|
+
|
|
396
|
+
Results are JSONL in input order, one line per case — a successful row carries
|
|
397
|
+
the same fields as "workflow run --json", so route assertions read the same:
|
|
398
|
+
{"id":"c1","ok":true,"path":["parse","reply"],"terminalNodeId":"reply","tags":[],"state":{…},"reply":{…}}
|
|
399
|
+
{"id":"c2","ok":false,"error":{"message":"…","status":404}}
|
|
400
|
+
|
|
401
|
+
Progress goes to stderr as each case finishes — "[2/3] c2 ok 843ms
|
|
402
|
+
terminal=reply" in completion order, then a "3 cases: 2 ok, 1 failed" tally —
|
|
403
|
+
so stdout carries nothing but the result rows. Those rows are written once the
|
|
404
|
+
whole batch has finished, not streamed per case, so watch stderr for live
|
|
405
|
+
progress.
|
|
406
|
+
|
|
407
|
+
A failing case does not stop the batch. The exit code is 1 if any case failed.
|
|
408
|
+
`;
|
|
409
|
+
/** True for a JSON object — not an array, not null, not a scalar. */
|
|
410
|
+
function isPlainObject(value) {
|
|
411
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
412
|
+
}
|
|
413
|
+
/**
|
|
414
|
+
* Parses a JSONL cases file into workflow cases, validating the fields this
|
|
415
|
+
* command sends. Line numbering and the `case-<line>` fallback id come from
|
|
416
|
+
* the shared JSONL reader, so every batch command names cases the same way.
|
|
417
|
+
*/
|
|
418
|
+
function parseWorkflowCases(raw) {
|
|
419
|
+
return (0, shared_1.parseJsonlObjects)(raw).map(({ line, value }) => {
|
|
420
|
+
const hasMessage = value.message !== undefined;
|
|
421
|
+
const hasMessages = value.messages !== undefined;
|
|
422
|
+
if (hasMessage && hasMessages) {
|
|
423
|
+
throw new Error(`Case on line ${line} must use only one of "message" or "messages"`);
|
|
424
|
+
}
|
|
425
|
+
let messages;
|
|
426
|
+
if (hasMessage) {
|
|
427
|
+
if (typeof value.message !== 'string' || value.message.trim() === '') {
|
|
428
|
+
throw new Error(`Case on line ${line}: "message" must be a non-empty string`);
|
|
429
|
+
}
|
|
430
|
+
messages = [{ role: 'user', content: value.message }];
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
if (!Array.isArray(value.messages) || value.messages.length === 0) {
|
|
434
|
+
throw new Error(`Case on line ${line} must have a "message" string or a non-empty "messages" array`);
|
|
435
|
+
}
|
|
436
|
+
messages = value.messages;
|
|
437
|
+
}
|
|
438
|
+
// A non-object channel would reach the runtime as an empty `$channel` and
|
|
439
|
+
// the case would run "ok" against the wrong user — reject it here.
|
|
440
|
+
if (value.channel !== undefined && !isPlainObject(value.channel)) {
|
|
441
|
+
throw new Error(`Case on line ${line}: "channel" must be a JSON object`);
|
|
442
|
+
}
|
|
443
|
+
if (value.version !== undefined &&
|
|
444
|
+
(!Number.isInteger(value.version) || value.version <= 0)) {
|
|
445
|
+
throw new Error(`Case on line ${line}: "version" must be a positive integer`);
|
|
446
|
+
}
|
|
447
|
+
return {
|
|
448
|
+
id: (0, shared_1.caseIdFor)(value, line),
|
|
449
|
+
messages,
|
|
450
|
+
channel: value.channel,
|
|
451
|
+
variables: value.variables,
|
|
452
|
+
version: value.version,
|
|
453
|
+
};
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Layers a case's own `channel`/`variables` over the batch-wide flag value.
|
|
458
|
+
* Two plain objects merge key by key with the case winning, so a batch can
|
|
459
|
+
* share a channel and vary one field; anything else (an array, a scalar) is
|
|
460
|
+
* replaced wholesale, because there is no meaningful per-key merge.
|
|
461
|
+
*/
|
|
462
|
+
function mergeCaseJson(global, caseValue) {
|
|
463
|
+
if (caseValue === undefined)
|
|
464
|
+
return global;
|
|
465
|
+
if (global === undefined)
|
|
466
|
+
return caseValue;
|
|
467
|
+
if (isPlainObject(global) && isPlainObject(caseValue))
|
|
468
|
+
return { ...global, ...caseValue };
|
|
469
|
+
return caseValue;
|
|
470
|
+
}
|
|
471
|
+
/** Parses argv for `workflow run-batch`. Throws on malformed input. */
|
|
472
|
+
function parseWorkflowBatchArgs(argv) {
|
|
473
|
+
const options = {
|
|
474
|
+
concurrency: 4,
|
|
475
|
+
tags: [],
|
|
476
|
+
trace: 'summary',
|
|
477
|
+
help: false,
|
|
478
|
+
};
|
|
479
|
+
const next = (i, flag) => {
|
|
480
|
+
const value = argv[i + 1];
|
|
481
|
+
if (value === undefined) {
|
|
482
|
+
throw new Error(`Missing value for ${flag}`);
|
|
483
|
+
}
|
|
484
|
+
return value;
|
|
485
|
+
};
|
|
486
|
+
for (let i = 0; i < argv.length; i++) {
|
|
487
|
+
let arg = argv[i];
|
|
488
|
+
let inline;
|
|
489
|
+
// Accept both `--flag value` and `--flag=value`.
|
|
490
|
+
const eq = arg.indexOf('=');
|
|
491
|
+
if (arg.startsWith('--') && eq > 2) {
|
|
492
|
+
inline = arg.slice(eq + 1);
|
|
493
|
+
arg = arg.slice(0, eq);
|
|
494
|
+
}
|
|
495
|
+
const take = (flag) => {
|
|
496
|
+
if (inline !== undefined)
|
|
497
|
+
return inline;
|
|
498
|
+
return next(i++, flag);
|
|
499
|
+
};
|
|
500
|
+
switch (arg) {
|
|
501
|
+
case '-w':
|
|
502
|
+
case '--workflow-id':
|
|
503
|
+
options.workflowId = take(arg);
|
|
504
|
+
break;
|
|
505
|
+
case '--cases':
|
|
506
|
+
options.casesFile = take(arg);
|
|
507
|
+
break;
|
|
508
|
+
case '--concurrency': {
|
|
509
|
+
const raw = take(arg);
|
|
510
|
+
const n = Number(raw);
|
|
511
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
512
|
+
throw new Error(`--concurrency must be a positive integer, got "${raw}"`);
|
|
513
|
+
}
|
|
514
|
+
options.concurrency = n;
|
|
515
|
+
break;
|
|
516
|
+
}
|
|
517
|
+
case '--version': {
|
|
518
|
+
const raw = take(arg);
|
|
519
|
+
const v = Number(raw);
|
|
520
|
+
if (!Number.isInteger(v) || v <= 0) {
|
|
521
|
+
throw new Error(`--version must be a positive integer, got "${raw}"`);
|
|
522
|
+
}
|
|
523
|
+
options.version = v;
|
|
524
|
+
break;
|
|
525
|
+
}
|
|
526
|
+
case '--channel':
|
|
527
|
+
options.channel = take(arg);
|
|
528
|
+
break;
|
|
529
|
+
case '--variables':
|
|
530
|
+
options.variables = take(arg);
|
|
531
|
+
break;
|
|
532
|
+
case '--timezone':
|
|
533
|
+
options.timezone = take(arg);
|
|
534
|
+
break;
|
|
535
|
+
case '--request-tag': {
|
|
536
|
+
const tag = take(arg).trim();
|
|
537
|
+
if (!tag) {
|
|
538
|
+
throw new Error('--request-tag cannot be empty');
|
|
539
|
+
}
|
|
540
|
+
options.tags.push(tag);
|
|
541
|
+
break;
|
|
542
|
+
}
|
|
543
|
+
case '--trace': {
|
|
544
|
+
const raw = take(arg);
|
|
545
|
+
if (!exports.TRACE_MODES.includes(raw)) {
|
|
546
|
+
throw new Error(`--trace must be one of ${exports.TRACE_MODES.join('|')}, got "${raw}"`);
|
|
547
|
+
}
|
|
548
|
+
options.trace = raw;
|
|
549
|
+
break;
|
|
550
|
+
}
|
|
551
|
+
case '-o':
|
|
552
|
+
case '--out':
|
|
553
|
+
options.out = take(arg);
|
|
554
|
+
break;
|
|
555
|
+
case '-e':
|
|
556
|
+
case '--env':
|
|
557
|
+
options.envFile = take(arg);
|
|
558
|
+
break;
|
|
559
|
+
case '--api-key':
|
|
560
|
+
options.apiKey = take(arg);
|
|
561
|
+
break;
|
|
562
|
+
case '--base-url':
|
|
563
|
+
options.baseUrl = take(arg);
|
|
564
|
+
break;
|
|
565
|
+
case '-h':
|
|
566
|
+
case '--help':
|
|
567
|
+
if (inline !== undefined) {
|
|
568
|
+
throw new Error('--help does not take a value');
|
|
569
|
+
}
|
|
570
|
+
options.help = true;
|
|
571
|
+
break;
|
|
572
|
+
default:
|
|
573
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
return options;
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* The `workflow run-batch` subcommand: run several cases against one workflow
|
|
580
|
+
* with bounded concurrency.
|
|
581
|
+
*
|
|
582
|
+
* Each case is an independent workflow completion; the wins over a shell loop
|
|
583
|
+
* are paying process start-up and env loading once, running cases in parallel,
|
|
584
|
+
* and getting one JSONL stream in input order to diff across runs.
|
|
585
|
+
*/
|
|
586
|
+
async function runWorkflowBatch(argv) {
|
|
587
|
+
const options = parseWorkflowBatchArgs(argv);
|
|
588
|
+
if (options.help) {
|
|
589
|
+
process.stdout.write(WORKFLOW_BATCH_HELP);
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
if (!options.workflowId || options.workflowId.trim() === '') {
|
|
593
|
+
throw new Error('Missing required option: --workflow-id');
|
|
594
|
+
}
|
|
595
|
+
if (!options.casesFile) {
|
|
596
|
+
throw new Error('Missing required option: --cases');
|
|
597
|
+
}
|
|
598
|
+
(0, shared_1.loadEnv)(options.envFile);
|
|
599
|
+
const globalChannel = parseChannelFlag(options.channel);
|
|
600
|
+
const globalVariables = options.variables !== undefined ? (0, shared_1.parseJsonFlag)(options.variables, '--variables') : undefined;
|
|
601
|
+
const { raw, baseDir } = await (0, shared_1.readCasesSource)(options.casesFile);
|
|
602
|
+
const cases = parseWorkflowCases(raw);
|
|
603
|
+
const client = new client_1.MetrioAI({
|
|
604
|
+
apiKey: (0, shared_1.resolveApiKey)(options.apiKey),
|
|
605
|
+
baseUrl: (0, shared_1.resolveBaseUrl)(options.baseUrl),
|
|
606
|
+
});
|
|
607
|
+
let completed = 0;
|
|
608
|
+
const results = await (0, shared_1.mapWithConcurrency)(cases, options.concurrency, async (c) => {
|
|
609
|
+
const row = await runOneWorkflowCase(client, c, baseDir, options, {
|
|
610
|
+
channel: globalChannel,
|
|
611
|
+
variables: globalVariables,
|
|
612
|
+
});
|
|
613
|
+
// Progress is reported as cases finish, so the counter is completion order
|
|
614
|
+
// even though the emitted rows stay in input order.
|
|
615
|
+
completed++;
|
|
616
|
+
process.stderr.write(`[${completed}/${cases.length}] ${c.id} ${describeRow(row)}\n`);
|
|
617
|
+
return row;
|
|
618
|
+
});
|
|
619
|
+
(0, shared_1.emitBatchResults)(results, options.out);
|
|
620
|
+
}
|
|
621
|
+
/** The one-line stderr progress description for a finished case. */
|
|
622
|
+
function describeRow(row) {
|
|
623
|
+
if (!row.ok) {
|
|
624
|
+
return `failed: ${row.error?.message}`;
|
|
625
|
+
}
|
|
626
|
+
const parts = [`ok ${row.elapsedTime}ms`];
|
|
627
|
+
if (row.terminalNodeId)
|
|
628
|
+
parts.push(`terminal=${row.terminalNodeId}`);
|
|
629
|
+
if (row.skipped)
|
|
630
|
+
parts.push('skipped=true');
|
|
631
|
+
return parts.join(' ');
|
|
632
|
+
}
|
|
633
|
+
/** Runs a single case, converting any failure into a result row. */
|
|
634
|
+
async function runOneWorkflowCase(client, c, baseDir, options, globals) {
|
|
635
|
+
try {
|
|
636
|
+
const req = {
|
|
637
|
+
workflowId: options.workflowId,
|
|
638
|
+
messages: (0, shared_1.buildMessages)(c.messages, baseDir),
|
|
639
|
+
version: c.version ?? options.version,
|
|
640
|
+
channel: mergeCaseJson(globals.channel, c.channel),
|
|
641
|
+
variables: mergeCaseJson(globals.variables, c.variables),
|
|
642
|
+
timezone: options.timezone,
|
|
643
|
+
tags: options.tags.length ? options.tags : undefined,
|
|
644
|
+
// Only "full" needs the heavier server-side trace; path and summary are
|
|
645
|
+
// both served by the default routing trace.
|
|
646
|
+
trace: options.trace === 'full' ? 'full' : undefined,
|
|
647
|
+
};
|
|
648
|
+
const res = await client.workflowCompletion(req);
|
|
649
|
+
const shaped = shapeWorkflowOutput(res, options.trace);
|
|
650
|
+
// Every row leads with the case id and ok; a response field of either name
|
|
651
|
+
// must not shadow them, since that is what assertions key off.
|
|
652
|
+
delete shaped.id;
|
|
653
|
+
delete shaped.ok;
|
|
654
|
+
return { id: c.id, ok: true, ...shaped };
|
|
655
|
+
}
|
|
656
|
+
catch (err) {
|
|
657
|
+
return {
|
|
658
|
+
id: c.id,
|
|
659
|
+
ok: false,
|
|
660
|
+
error: {
|
|
661
|
+
message: err instanceof Error ? err.message : String(err),
|
|
662
|
+
status: err instanceof client_2.MetrioApiError ? err.statusCode : undefined,
|
|
663
|
+
},
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
}
|