@outputai/cli 0.10.1-next.3d1f9bd.0 → 0.10.1-next.442c500.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/generated/api.d.ts +81 -26
- package/dist/api/generated/api.js +7 -4
- package/dist/assets/docker/docker-compose-dev.yml +2 -2
- package/dist/commands/workflow/monitor.d.ts +5 -20
- package/dist/commands/workflow/monitor.js +20 -182
- package/dist/commands/workflow/monitor.spec.js +82 -3
- package/dist/commands/workflow/result.js +2 -2
- package/dist/commands/workflow/result.spec.js +65 -1
- package/dist/commands/workflow/run.js +10 -3
- package/dist/commands/workflow/run.spec.js +42 -5
- package/dist/commands/workflow/start.d.ts +7 -1
- package/dist/commands/workflow/start.js +107 -12
- package/dist/commands/workflow/start.spec.js +282 -5
- package/dist/commands/workflow/status.spec.js +1 -1
- package/dist/commands/workflow/{test_eval.d.ts → test.d.ts} +0 -1
- package/dist/commands/workflow/{test_eval.js → test.js} +0 -1
- package/dist/commands/workflow/{test_eval.spec.js → test.spec.js} +4 -4
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/monitor_stream.d.ts +62 -0
- package/dist/services/monitor_stream.js +285 -0
- package/dist/services/monitor_stream.spec.d.ts +1 -0
- package/dist/services/monitor_stream.spec.js +285 -0
- package/dist/services/workflow_history.js +2 -2
- package/dist/templates/agent_instructions/CLAUDE.md.template +4 -2
- package/dist/templates/project/README.md.template +3 -1
- package/dist/templates/project/package.json.template +2 -2
- package/dist/utils/env_loader.js +6 -2
- package/dist/utils/env_loader.spec.js +61 -32
- package/dist/utils/error_handler.d.ts +10 -0
- package/dist/utils/error_handler.js +14 -0
- package/dist/utils/error_handler.spec.d.ts +1 -0
- package/dist/utils/error_handler.spec.js +62 -0
- package/dist/utils/format_workflow_result.d.ts +15 -3
- package/dist/utils/format_workflow_result.js +39 -6
- package/dist/utils/format_workflow_result.spec.js +39 -6
- package/dist/utils/monitor_flags.d.ts +35 -0
- package/dist/utils/monitor_flags.js +76 -0
- package/dist/utils/normalize_workflow_status.d.ts +4 -3
- package/dist/utils/normalize_workflow_status.js +12 -3
- package/dist/utils/normalize_workflow_status.spec.js +3 -0
- package/dist/utils/resolve_input.d.ts +9 -1
- package/dist/utils/resolve_input.js +8 -2
- package/dist/utils/resolve_input.spec.d.ts +1 -0
- package/dist/utils/resolve_input.spec.js +75 -0
- package/dist/views/dev/components/workflow_status.js +1 -1
- package/dist/views/dev/hooks/use_run_detail.js +4 -4
- package/dist/views/dev/hooks/use_run_detail.spec.js +1 -1
- package/dist/views/dev/panels/runs_panel.js +2 -2
- package/oclif.manifest.json +54 -9
- package/package.json +6 -8
- /package/dist/commands/workflow/{test_eval.spec.d.ts → test.spec.d.ts} +0 -0
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
+
import { Parser } from '@oclif/core';
|
|
4
|
+
import { CLIError } from '@oclif/core/errors';
|
|
3
5
|
vi.mock('#api/generated/api.js', () => ({
|
|
4
6
|
postWorkflowStart: vi.fn()
|
|
5
7
|
}));
|
|
6
8
|
vi.mock('#utils/resolve_input.js', () => ({
|
|
7
9
|
resolveInput: vi.fn()
|
|
8
10
|
}));
|
|
11
|
+
// Only the streaming loop is stubbed; monitorErrorOverrides and commandStreamIo
|
|
12
|
+
// stay real so the default-application and error-mapping branches are exercised
|
|
13
|
+
// against the values `workflow monitor` actually uses.
|
|
14
|
+
vi.mock('#services/monitor_stream.js', async (importOriginal) => ({
|
|
15
|
+
...await importOriginal(),
|
|
16
|
+
streamWorkflowUpdates: vi.fn()
|
|
17
|
+
}));
|
|
9
18
|
describe('workflow start command', () => {
|
|
10
19
|
beforeEach(async () => {
|
|
11
20
|
vi.clearAllMocks();
|
|
@@ -36,13 +45,72 @@ describe('workflow start command', () => {
|
|
|
36
45
|
expect(WorkflowStart.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
|
|
37
46
|
expect(WorkflowStart.flags.catalog.char).toBe('c');
|
|
38
47
|
});
|
|
48
|
+
it('enables the built-in --json flag', async () => {
|
|
49
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
50
|
+
expect(WorkflowStart.enableJsonFlag).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
it('exposes --monitor with no default so dependsOn stays enforceable', async () => {
|
|
53
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
54
|
+
expect(WorkflowStart.flags.monitor.char).toBe('m');
|
|
55
|
+
// A defaulted flag counts as present, so a default here would satisfy the
|
|
56
|
+
// dependsOn guards below and let --interval through without --monitor.
|
|
57
|
+
expect(WorkflowStart.flags.monitor.default).toBeUndefined();
|
|
58
|
+
});
|
|
59
|
+
it('leaves the --monitor/--json conflict to the runtime guard', async () => {
|
|
60
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
61
|
+
// An `exclusive: [ 'json' ]` relationship would fire first and print a bare
|
|
62
|
+
// "--json=true cannot also be provided", pre-empting the guard in run()
|
|
63
|
+
// that explains what to use instead — and it still wouldn't catch
|
|
64
|
+
// CONTENT_TYPE=json, which never reaches argv.
|
|
65
|
+
expect(WorkflowStart.flags.monitor.exclusive).toBeUndefined();
|
|
66
|
+
});
|
|
67
|
+
it('gates every monitor passthrough flag behind --monitor and leaves them undefaulted', async () => {
|
|
68
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
69
|
+
for (const name of ['interval', 'include-payloads', 'color']) {
|
|
70
|
+
expect(WorkflowStart.flags[name].dependsOn).toEqual(['monitor']);
|
|
71
|
+
// A default would count as present and trigger this flag's own dependsOn
|
|
72
|
+
// check, failing every invocation that omits --monitor.
|
|
73
|
+
expect(WorkflowStart.flags[name].default).toBeUndefined();
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
// The properties asserted above are only the inputs; what matters is what
|
|
77
|
+
// oclif does with them. Parser.parse needs no oclif Config, so the actual
|
|
78
|
+
// gating is cheap to pin — worth doing because the no-default design is
|
|
79
|
+
// subtle enough that someone will try to "fix" it by adding one.
|
|
80
|
+
describe('flag parsing', () => {
|
|
81
|
+
const parse = async (argv) => {
|
|
82
|
+
const WorkflowStart = (await import('./start.js')).default;
|
|
83
|
+
return Parser.parse(argv, { flags: WorkflowStart.flags, strict: false });
|
|
84
|
+
};
|
|
85
|
+
it('accepts a plain start with none of the monitor flags', async () => {
|
|
86
|
+
await expect(parse(['my_workflow'])).resolves.toBeDefined();
|
|
87
|
+
});
|
|
88
|
+
it('accepts the passthrough flags once --monitor is given', async () => {
|
|
89
|
+
const { flags } = await parse(['my_workflow', '--monitor', '--interval', '500', '--include-payloads']);
|
|
90
|
+
expect(flags).toMatchObject({ monitor: true, interval: 500, 'include-payloads': true });
|
|
91
|
+
});
|
|
92
|
+
it('rejects a passthrough flag without --monitor', async () => {
|
|
93
|
+
await expect(parse(['my_workflow', '--interval', '500'])).rejects.toThrow(/--monitor/);
|
|
94
|
+
});
|
|
95
|
+
it('gates --no-color behind --monitor as well', async () => {
|
|
96
|
+
// A consequence of gating --color, not an independent decision: --no-color
|
|
97
|
+
// is reflexive enough that this is worth stating outright rather than
|
|
98
|
+
// leaving as a surprise.
|
|
99
|
+
await expect(parse(['my_workflow', '--no-color'])).rejects.toThrow(/--monitor/);
|
|
100
|
+
await expect(parse(['my_workflow', '--monitor', '--no-color']))
|
|
101
|
+
.resolves.toMatchObject({ flags: { color: false } });
|
|
102
|
+
});
|
|
103
|
+
it('rejects an interval below the minimum', async () => {
|
|
104
|
+
await expect(parse(['my_workflow', '--monitor', '--interval', '0'])).rejects.toThrow();
|
|
105
|
+
});
|
|
106
|
+
});
|
|
39
107
|
});
|
|
40
108
|
describe('run()', () => {
|
|
41
|
-
const createCommand = async (flagOverrides = {}) => {
|
|
109
|
+
const createCommand = async (flagOverrides = {}, argv = ['my_workflow']) => {
|
|
42
110
|
const WorkflowStart = (await import('./start.js')).default;
|
|
43
111
|
const { postWorkflowStart } = await import('#api/generated/api.js');
|
|
44
112
|
const { resolveInput } = await import('#utils/resolve_input.js');
|
|
45
|
-
const cmd = new WorkflowStart(
|
|
113
|
+
const cmd = new WorkflowStart(argv, {});
|
|
46
114
|
cmd.log = vi.fn();
|
|
47
115
|
cmd.error = vi.fn(() => {
|
|
48
116
|
throw new Error('error called');
|
|
@@ -53,6 +121,7 @@ describe('workflow start command', () => {
|
|
|
53
121
|
});
|
|
54
122
|
return { cmd, postWorkflowStart: vi.mocked(postWorkflowStart), resolveInput: vi.mocked(resolveInput) };
|
|
55
123
|
};
|
|
124
|
+
const logged = (cmd) => cmd.log.mock.calls.map(([line]) => line);
|
|
56
125
|
it('threads the resolved catalog to resolveInput and postWorkflowStart', async () => {
|
|
57
126
|
const { cmd, postWorkflowStart, resolveInput } = await createCommand({ catalog: 'my-catalog' });
|
|
58
127
|
resolveInput.mockResolvedValue({ key: 'value' });
|
|
@@ -61,9 +130,15 @@ describe('workflow start command', () => {
|
|
|
61
130
|
status: 200,
|
|
62
131
|
headers: new Headers()
|
|
63
132
|
});
|
|
64
|
-
await cmd.run();
|
|
65
|
-
expect(resolveInput).toHaveBeenCalledWith(
|
|
133
|
+
const result = await cmd.run();
|
|
134
|
+
expect(resolveInput).toHaveBeenCalledWith(expect.objectContaining({
|
|
135
|
+
workflowName: 'my_workflow',
|
|
136
|
+
commandName: 'start',
|
|
137
|
+
catalog: 'my-catalog',
|
|
138
|
+
json: false
|
|
139
|
+
}));
|
|
66
140
|
expect(postWorkflowStart).toHaveBeenCalledWith(expect.objectContaining({ workflowName: 'my_workflow', catalog: 'my-catalog' }));
|
|
141
|
+
expect(result).toEqual({ workflowId: 'wf-123' });
|
|
67
142
|
});
|
|
68
143
|
it('passes undefined catalog through when none is set', async () => {
|
|
69
144
|
const { cmd, postWorkflowStart, resolveInput } = await createCommand();
|
|
@@ -74,7 +149,209 @@ describe('workflow start command', () => {
|
|
|
74
149
|
headers: new Headers()
|
|
75
150
|
});
|
|
76
151
|
await cmd.run();
|
|
77
|
-
expect(resolveInput).toHaveBeenCalledWith(
|
|
152
|
+
expect(resolveInput).toHaveBeenCalledWith(expect.objectContaining({
|
|
153
|
+
workflowName: 'my_workflow',
|
|
154
|
+
commandName: 'start',
|
|
155
|
+
catalog: undefined
|
|
156
|
+
}));
|
|
157
|
+
});
|
|
158
|
+
it('tells resolveInput to stay quiet when --json is set', async () => {
|
|
159
|
+
const { cmd, postWorkflowStart, resolveInput } = await createCommand({}, ['my_workflow', 'basic', '--json']);
|
|
160
|
+
resolveInput.mockResolvedValue({});
|
|
161
|
+
postWorkflowStart.mockResolvedValue({
|
|
162
|
+
data: { workflowId: 'wf-123' },
|
|
163
|
+
status: 200,
|
|
164
|
+
headers: new Headers()
|
|
165
|
+
});
|
|
166
|
+
await cmd.run();
|
|
167
|
+
expect(resolveInput).toHaveBeenCalledWith(expect.objectContaining({ json: true }));
|
|
168
|
+
});
|
|
169
|
+
describe('--monitor', () => {
|
|
170
|
+
const startResponse = (data) => ({
|
|
171
|
+
data, status: 200, headers: new Headers()
|
|
172
|
+
});
|
|
173
|
+
it('does not attach when the flag is absent, and keeps the follow-up hints', async () => {
|
|
174
|
+
const { cmd, postWorkflowStart } = await createCommand();
|
|
175
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
176
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
177
|
+
await cmd.run();
|
|
178
|
+
expect(streamWorkflowUpdates).not.toHaveBeenCalled();
|
|
179
|
+
const printed = logged(cmd).join('\n');
|
|
180
|
+
expect(printed).toContain('workflow status wf-123');
|
|
181
|
+
expect(printed).toContain('workflow result wf-123');
|
|
182
|
+
});
|
|
183
|
+
it('streams updates pinned to the run it just started', async () => {
|
|
184
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
185
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
186
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
187
|
+
await cmd.run();
|
|
188
|
+
expect(streamWorkflowUpdates).toHaveBeenCalledWith(
|
|
189
|
+
// runId is pinned rather than left undefined so a rapid re-start can't
|
|
190
|
+
// make the monitor resolve "latest run" to a different execution.
|
|
191
|
+
expect.objectContaining({ workflowId: 'wf-123', runId: 'run-1', json: false }), expect.objectContaining({ log: expect.any(Function), error: expect.any(Function) }));
|
|
192
|
+
});
|
|
193
|
+
it('still returns the start result after monitoring finishes', async () => {
|
|
194
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
195
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
196
|
+
await expect(cmd.run()).resolves.toEqual({ workflowId: 'wf-123', runId: 'run-1' });
|
|
197
|
+
});
|
|
198
|
+
it('drops the up-front status hint that duplicates what monitoring already does', async () => {
|
|
199
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
200
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
201
|
+
await cmd.run();
|
|
202
|
+
const printed = logged(cmd).join('\n');
|
|
203
|
+
expect(printed).toContain('Workflow ID: wf-123');
|
|
204
|
+
expect(printed).not.toContain('workflow status wf-123');
|
|
205
|
+
});
|
|
206
|
+
it('points at "workflow result" once monitoring finishes', async () => {
|
|
207
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
208
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
209
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
210
|
+
vi.mocked(streamWorkflowUpdates).mockResolvedValue('completed');
|
|
211
|
+
await cmd.run();
|
|
212
|
+
// The stream reports progress, never the return value, so the command
|
|
213
|
+
// that fetches it has to be named somewhere.
|
|
214
|
+
const printed = logged(cmd);
|
|
215
|
+
expect(printed.at(-1)).toContain('workflow result wf-123');
|
|
216
|
+
});
|
|
217
|
+
it('points at "workflow debug" instead when the workflow failed', async () => {
|
|
218
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
219
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
220
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
221
|
+
vi.mocked(streamWorkflowUpdates).mockResolvedValue('failed');
|
|
222
|
+
await cmd.run();
|
|
223
|
+
// A failed run has no result to fetch.
|
|
224
|
+
const printed = logged(cmd);
|
|
225
|
+
expect(printed.at(-1)).toContain('workflow debug wf-123');
|
|
226
|
+
expect(printed.at(-1)).not.toContain('workflow result');
|
|
227
|
+
});
|
|
228
|
+
it('adds no follow-up hint when the user detached mid-run', async () => {
|
|
229
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
230
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
231
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: 'run-1' }));
|
|
232
|
+
// Detaching returns no terminal status; the detach message carries its
|
|
233
|
+
// own hints, so a second one guessing at the outcome would be wrong.
|
|
234
|
+
vi.mocked(streamWorkflowUpdates).mockResolvedValue(undefined);
|
|
235
|
+
await cmd.run();
|
|
236
|
+
const printed = logged(cmd).join('\n');
|
|
237
|
+
expect(printed).not.toContain('workflow result wf-123');
|
|
238
|
+
expect(printed).not.toContain('workflow debug wf-123');
|
|
239
|
+
});
|
|
240
|
+
it('applies monitor defaults for the passthrough flags left unset', async () => {
|
|
241
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
242
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
243
|
+
const { MONITOR_DEFAULTS } = await import('#utils/monitor_flags.js');
|
|
244
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123' }));
|
|
245
|
+
await cmd.run();
|
|
246
|
+
// These carry no oclif default (that would defeat dependsOn), so run() must
|
|
247
|
+
// supply them — asserted against the shared source rather than re-stating
|
|
248
|
+
// the literals, which is exactly how the two commands would drift apart.
|
|
249
|
+
expect(streamWorkflowUpdates).toHaveBeenCalledWith(expect.objectContaining({
|
|
250
|
+
interval: MONITOR_DEFAULTS.interval,
|
|
251
|
+
color: MONITOR_DEFAULTS.color,
|
|
252
|
+
includePayloads: MONITOR_DEFAULTS.includePayloads
|
|
253
|
+
}), expect.anything());
|
|
254
|
+
});
|
|
255
|
+
it('forwards explicit passthrough flag values', async () => {
|
|
256
|
+
const { cmd, postWorkflowStart } = await createCommand({
|
|
257
|
+
monitor: true, interval: 500, color: false, 'include-payloads': true
|
|
258
|
+
});
|
|
259
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
260
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123' }));
|
|
261
|
+
await cmd.run();
|
|
262
|
+
expect(streamWorkflowUpdates).toHaveBeenCalledWith(expect.objectContaining({ interval: 500, color: false, includePayloads: true }), expect.anything());
|
|
263
|
+
});
|
|
264
|
+
it('leaves runId undefined when the API omits it, falling back to the latest run', async () => {
|
|
265
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
266
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
267
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123', runId: null }));
|
|
268
|
+
await cmd.run();
|
|
269
|
+
expect(streamWorkflowUpdates).toHaveBeenCalledWith(expect.objectContaining({ runId: undefined }), expect.anything());
|
|
270
|
+
});
|
|
271
|
+
it('errors instead of monitoring when the API returns no workflow ID', async () => {
|
|
272
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
273
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
274
|
+
postWorkflowStart.mockResolvedValue(startResponse({ runId: 'run-1' }));
|
|
275
|
+
await expect(cmd.run()).rejects.toThrow();
|
|
276
|
+
expect(streamWorkflowUpdates).not.toHaveBeenCalled();
|
|
277
|
+
const [message, options] = cmd.error.mock.calls.at(-1);
|
|
278
|
+
// The start succeeded — only monitoring is impossible — so this is the
|
|
279
|
+
// exit-3 case. Exit 1 here would tell a CI job retrying a failed workflow
|
|
280
|
+
// to re-submit one that is already running.
|
|
281
|
+
expect(message).toContain('started');
|
|
282
|
+
expect(message).toContain('cannot be monitored');
|
|
283
|
+
expect(options).toEqual(expect.objectContaining({ exit: 3 }));
|
|
284
|
+
// Claiming success and then failing on the next line contradicts itself,
|
|
285
|
+
// and the "unknown" placeholder id isn't something the user can act on.
|
|
286
|
+
const printed = logged(cmd).join('\n');
|
|
287
|
+
expect(printed).not.toContain('Workflow started successfully');
|
|
288
|
+
expect(printed).not.toContain('unknown');
|
|
289
|
+
});
|
|
290
|
+
it('refuses to monitor under --json instead of silently swallowing the stream', async () => {
|
|
291
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
292
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
293
|
+
// CONTENT_TYPE=json enables json mode without --json ever reaching argv,
|
|
294
|
+
// so oclif's `exclusive` check has nothing to reject — this guard catches it.
|
|
295
|
+
vi.spyOn(cmd, 'jsonEnabled').mockReturnValue(true);
|
|
296
|
+
await expect(cmd.run()).rejects.toThrow();
|
|
297
|
+
expect(cmd.error).toHaveBeenCalledWith(expect.stringContaining('Cannot combine --monitor with --json'), expect.objectContaining({ exit: 2 }));
|
|
298
|
+
expect(postWorkflowStart).not.toHaveBeenCalled();
|
|
299
|
+
expect(streamWorkflowUpdates).not.toHaveBeenCalled();
|
|
300
|
+
});
|
|
301
|
+
// Shaped like the API's real 404 body so the two 404 paths below differ the
|
|
302
|
+
// way they do in practice: `catch()`'s override replaces the body, while
|
|
303
|
+
// monitoring has no override and lets the body through.
|
|
304
|
+
const notFound = () => Object.assign(new Error('not found'), {
|
|
305
|
+
response: { status: 404, data: { error: 'WorkflowNotFoundError', message: 'Workflow "wf-123" not found' } }
|
|
306
|
+
});
|
|
307
|
+
it('blames the workflow name for a 404 raised before monitoring begins', async () => {
|
|
308
|
+
const { cmd } = await createCommand();
|
|
309
|
+
await expect(cmd.catch(notFound())).rejects.toThrow();
|
|
310
|
+
expect(cmd.error).toHaveBeenCalledWith('Workflow not found. Check the workflow name.', expect.objectContaining({ exit: 1 }));
|
|
311
|
+
});
|
|
312
|
+
it('reports a monitoring failure as a live workflow, not a failed start', async () => {
|
|
313
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
314
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
315
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123' }));
|
|
316
|
+
vi.mocked(streamWorkflowUpdates).mockRejectedValue(notFound());
|
|
317
|
+
await expect(cmd.run()).rejects.toThrow();
|
|
318
|
+
const [message, options] = cmd.error.mock.calls.at(-1);
|
|
319
|
+
// The workflow started fine, so a 404 here is about the run being polled.
|
|
320
|
+
// The server's own message surfaces; `workflow monitor`'s "Check the
|
|
321
|
+
// workflow ID" must not, since this id came back from postWorkflowStart —
|
|
322
|
+
// it would both misdirect the user and contradict the "still running" line.
|
|
323
|
+
expect(message).toContain('WorkflowNotFoundError');
|
|
324
|
+
expect(message).not.toContain('Check the workflow ID');
|
|
325
|
+
expect(message).toContain('wf-123 started, but monitoring stopped');
|
|
326
|
+
expect(message).toContain('workflow status wf-123');
|
|
327
|
+
// Exit 3, not 1: a caller retrying on a failed workflow must not
|
|
328
|
+
// re-submit one that is already running.
|
|
329
|
+
expect(options).toEqual(expect.objectContaining({ exit: 3 }));
|
|
330
|
+
});
|
|
331
|
+
it('still reports a live workflow when the stream raises its own CLIError', async () => {
|
|
332
|
+
const { cmd, postWorkflowStart } = await createCommand({ monitor: true });
|
|
333
|
+
const { streamWorkflowUpdates } = await import('#services/monitor_stream.js');
|
|
334
|
+
postWorkflowStart.mockResolvedValue(startResponse({ workflowId: 'wf-123' }));
|
|
335
|
+
// What `io.error` produces — e.g. the continue-as-new branch. run() must
|
|
336
|
+
// use handleApiError here, not handleCommandError: the latter rethrows a
|
|
337
|
+
// CLIError untouched, which would surface this as a bare exit 1 and lose
|
|
338
|
+
// the "still running" message entirely.
|
|
339
|
+
vi.mocked(streamWorkflowUpdates).mockRejectedValue(new CLIError('Workflow continued as a new run, but the new run ID could not be determined.'));
|
|
340
|
+
await expect(cmd.run()).rejects.toThrow();
|
|
341
|
+
const [message, options] = cmd.error.mock.calls.at(-1);
|
|
342
|
+
expect(message).toContain('wf-123 started, but monitoring stopped');
|
|
343
|
+
expect(message).toContain('The workflow is still running');
|
|
344
|
+
expect(options).toEqual(expect.objectContaining({ exit: 3 }));
|
|
345
|
+
});
|
|
346
|
+
it('rethrows oclif errors instead of flattening them to exit 1', async () => {
|
|
347
|
+
const { cmd } = await createCommand();
|
|
348
|
+
// `catch` re-raising a CLIError through handleApiError would discard both
|
|
349
|
+
// its exit code (2 for usage, 3 for a dropped stream) and oclif's own
|
|
350
|
+
// formatted flag-validation output.
|
|
351
|
+
const usageError = new CLIError('Cannot combine --monitor with --json.', { exit: 2 });
|
|
352
|
+
await expect(cmd.catch(usageError)).rejects.toBe(usageError);
|
|
353
|
+
expect(cmd.error).not.toHaveBeenCalled();
|
|
354
|
+
});
|
|
78
355
|
});
|
|
79
356
|
});
|
|
80
357
|
});
|
|
@@ -2,7 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
|
2
2
|
vi.mock('../../api/generated/api.js', () => ({
|
|
3
3
|
getWorkflowIdStatus: vi.fn(),
|
|
4
4
|
GetWorkflowIdStatus200Status: {
|
|
5
|
-
|
|
5
|
+
cancelled: 'cancelled',
|
|
6
6
|
completed: 'completed',
|
|
7
7
|
continued_as_new: 'continued_as_new',
|
|
8
8
|
failed: 'failed',
|
|
@@ -7,7 +7,6 @@ import { diagnoseMissingEvalWorkflow } from '#utils/eval_diagnostics.js';
|
|
|
7
7
|
import { handleApiError } from '#utils/error_handler.js';
|
|
8
8
|
import { getEvalWorkflowName, renderEvalOutput, computeExitCode, EvalOutputSchema } from '@outputai/evals';
|
|
9
9
|
export default class WorkflowTest extends Command {
|
|
10
|
-
static aliases = ['workflow:test'];
|
|
11
10
|
static description = 'Run evaluations against a workflow using its datasets';
|
|
12
11
|
static enableJsonFlag = true;
|
|
13
12
|
static examples = [
|
|
@@ -45,11 +45,11 @@ describe('workflow test command', () => {
|
|
|
45
45
|
});
|
|
46
46
|
describe('command definition', () => {
|
|
47
47
|
it('enables the built-in --json flag', async () => {
|
|
48
|
-
const WorkflowTest = (await import('./
|
|
48
|
+
const WorkflowTest = (await import('./test.js')).default;
|
|
49
49
|
expect(WorkflowTest.enableJsonFlag).toBe(true);
|
|
50
50
|
});
|
|
51
51
|
it('binds the catalog flag to OUTPUT_CATALOG_ID', async () => {
|
|
52
|
-
const WorkflowTest = (await import('./
|
|
52
|
+
const WorkflowTest = (await import('./test.js')).default;
|
|
53
53
|
expect(WorkflowTest.flags).toHaveProperty('catalog');
|
|
54
54
|
expect(WorkflowTest.flags.catalog.env).toBe('OUTPUT_CATALOG_ID');
|
|
55
55
|
expect(WorkflowTest.flags.catalog.char).toBe('c');
|
|
@@ -57,7 +57,7 @@ describe('workflow test command', () => {
|
|
|
57
57
|
});
|
|
58
58
|
describe('run()', () => {
|
|
59
59
|
const createCommand = async (jsonEnabled) => {
|
|
60
|
-
const WorkflowTest = (await import('./
|
|
60
|
+
const WorkflowTest = (await import('./test.js')).default;
|
|
61
61
|
const { postWorkflowRun } = await import('#api/generated/api.js');
|
|
62
62
|
const cmd = new WorkflowTest(['simple'], {});
|
|
63
63
|
cmd.log = vi.fn();
|
|
@@ -99,7 +99,7 @@ describe('workflow test command', () => {
|
|
|
99
99
|
expect(process.exitCode).toBe(1);
|
|
100
100
|
});
|
|
101
101
|
it('routes registration, dataset runs, and the eval run to the resolved catalog', async () => {
|
|
102
|
-
const WorkflowTest = (await import('./
|
|
102
|
+
const WorkflowTest = (await import('./test.js')).default;
|
|
103
103
|
const { postWorkflowRun } = await import('#api/generated/api.js');
|
|
104
104
|
const { fetchWorkflowCatalog } = await import('#api/workflow_catalog.js');
|
|
105
105
|
const cmd = new WorkflowTest(['my_workflow'], {});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type TerminalStatus } from '#utils/format_workflow_result.js';
|
|
2
|
+
export type MonitorStreamOptions = {
|
|
3
|
+
workflowId: string;
|
|
4
|
+
runId?: string;
|
|
5
|
+
includePayloads: boolean;
|
|
6
|
+
interval: number;
|
|
7
|
+
json: boolean;
|
|
8
|
+
color: boolean;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* The command surface the stream needs, kept as a parameter rather than a
|
|
12
|
+
* `Command` instance so `workflow start --monitor` can reuse the loop without
|
|
13
|
+
* constructing (or delegating to) a second oclif command — the repo has no
|
|
14
|
+
* `runCommand` precedent and `Command.run( argv, config )` would need a real
|
|
15
|
+
* oclif `Config` that unit tests don't have. `error` must be typed `never` so
|
|
16
|
+
* callers keep type narrowing after an error branch.
|
|
17
|
+
*/
|
|
18
|
+
export type MonitorStreamIo = {
|
|
19
|
+
log: (message: string) => void;
|
|
20
|
+
warn: (message: string) => void;
|
|
21
|
+
error: (message: string) => never;
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Adapts an oclif command to the above. Late-bound arrows rather than
|
|
25
|
+
* `command.log.bind( command )`: oclif (and the unit tests) replace these as own
|
|
26
|
+
* properties on the instance, so they must resolve at call time. Structurally
|
|
27
|
+
* typed so a test double satisfies it without a real oclif `Config`.
|
|
28
|
+
*/
|
|
29
|
+
export declare function commandStreamIo(command: {
|
|
30
|
+
log: (message: string) => void;
|
|
31
|
+
warn: (message: string) => unknown;
|
|
32
|
+
error: (message: string, options: {
|
|
33
|
+
exit: number;
|
|
34
|
+
}) => never;
|
|
35
|
+
}): MonitorStreamIo;
|
|
36
|
+
/**
|
|
37
|
+
* Polls a workflow run and emits status updates until it reaches a terminal
|
|
38
|
+
* state, following continue-as-new chains. Shared by `workflow monitor` and
|
|
39
|
+
* `workflow start --monitor` so both behave identically; see `MonitorStreamIo`
|
|
40
|
+
* for why output is injected rather than taken from a `Command`.
|
|
41
|
+
*
|
|
42
|
+
* Sets `process.exitCode = 1` on a terminal error status rather than throwing,
|
|
43
|
+
* so the caller's own output (e.g. `start`'s "Workflow started successfully")
|
|
44
|
+
* is still the command's primary result. Returns the terminal status it stopped
|
|
45
|
+
* on — `undefined` if it stopped because the user detached — so a caller can
|
|
46
|
+
* tailor its own follow-up (`workflow result` vs `workflow debug`).
|
|
47
|
+
*/
|
|
48
|
+
export declare function streamWorkflowUpdates(options: MonitorStreamOptions, io: MonitorStreamIo): Promise<TerminalStatus | undefined>;
|
|
49
|
+
/**
|
|
50
|
+
* Shared `catch` handling for both entry points. A 400 is the generic status for
|
|
51
|
+
* several distinct causes (invalid pageToken, a missing runId, an out-of-range
|
|
52
|
+
* longPollTimeoutMs) — only override it with the stale-cursor message when the
|
|
53
|
+
* server actually identifies that specific cause; otherwise let the real
|
|
54
|
+
* validation error surface instead of misdiagnosing an unrelated 400.
|
|
55
|
+
*
|
|
56
|
+
* Deliberately no 404 here: "check the workflow ID" only reads correctly where
|
|
57
|
+
* the user typed the id, so `workflow monitor` adds it and `start --monitor`
|
|
58
|
+
* doesn't — there the id came back from `postWorkflowStart`, and the server's own
|
|
59
|
+
* message is left to surface inside the "started, but monitoring stopped" wrapper
|
|
60
|
+
* instead of advising a fix that isn't the user's to make.
|
|
61
|
+
*/
|
|
62
|
+
export declare function monitorErrorOverrides(error: Error): Record<number, string>;
|