@outputai/cli 0.10.1-dev.b7b2fbe.0 → 0.10.1-next.09ed166.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/generated/api.d.ts +12 -0
- package/dist/assets/docker/docker-compose-dev.yml +1 -1
- package/dist/commands/workflow/history.js +3 -3
- package/dist/commands/workflow/history.spec.js +31 -2
- package/dist/commands/workflow/monitor.d.ts +49 -0
- package/dist/commands/workflow/monitor.js +230 -0
- package/dist/commands/workflow/monitor.spec.d.ts +1 -0
- package/dist/commands/workflow/monitor.spec.js +243 -0
- package/dist/generated/framework_version.json +1 -1
- package/dist/services/workflow_history/correlator.d.ts +2 -0
- package/dist/services/workflow_history/correlator.js +2 -2
- package/dist/services/workflow_history.d.ts +28 -0
- package/dist/services/workflow_history.js +95 -12
- package/dist/services/workflow_history.spec.js +183 -1
- package/dist/utils/color.d.ts +7 -0
- package/dist/utils/color.js +12 -0
- package/dist/utils/color.spec.d.ts +1 -0
- package/dist/utils/color.spec.js +43 -0
- package/dist/utils/format_workflow_result.d.ts +1 -0
- package/dist/utils/format_workflow_result.js +4 -0
- package/dist/utils/monitor_log.d.ts +20 -0
- package/dist/utils/monitor_log.js +48 -0
- package/dist/utils/monitor_log.spec.d.ts +1 -0
- package/dist/utils/monitor_log.spec.js +71 -0
- package/dist/utils/waterfall.d.ts +3 -1
- package/dist/utils/waterfall.js +8 -2
- package/dist/views/dev/hooks/use_run_detail.js +4 -7
- package/oclif.manifest.json +75 -1
- package/package.json +4 -4
|
@@ -393,6 +393,12 @@ export type GetWorkflowIdHistoryParams = {
|
|
|
393
393
|
* Include decoded input/output payloads in events
|
|
394
394
|
*/
|
|
395
395
|
includePayloads?: boolean;
|
|
396
|
+
/**
|
|
397
|
+
* When set, long-poll for a new event once caught up to the end of history instead of returning immediately, bounding the block by this many milliseconds. Clamped to the server's configured maximum — a caller can shorten the wait but never exceed it. Omit for an immediate response; on timeout returns the same page's cursor unchanged with an empty events array so the caller can retry. Lets a poller keep the block roughly aligned with its own tick interval.
|
|
398
|
+
|
|
399
|
+
* @minimum 1
|
|
400
|
+
*/
|
|
401
|
+
longPollTimeoutMs?: number;
|
|
396
402
|
};
|
|
397
403
|
/**
|
|
398
404
|
* Workflow metadata (null on subsequent pages)
|
|
@@ -431,6 +437,12 @@ export type GetWorkflowIdRunsRidHistoryParams = {
|
|
|
431
437
|
* Include decoded input/output payloads in events
|
|
432
438
|
*/
|
|
433
439
|
includePayloads?: boolean;
|
|
440
|
+
/**
|
|
441
|
+
* When set, long-poll for a new event once caught up to the end of history instead of returning immediately, bounding the block by this many milliseconds. Clamped to the server's configured maximum — a caller can shorten the wait but never exceed it. Omit for an immediate response; on timeout returns the same page's cursor unchanged with an empty events array so the caller can retry. Lets a poller keep the block roughly aligned with its own tick interval.
|
|
442
|
+
|
|
443
|
+
* @minimum 1
|
|
444
|
+
*/
|
|
445
|
+
longPollTimeoutMs?: number;
|
|
434
446
|
};
|
|
435
447
|
/**
|
|
436
448
|
* Workflow metadata (null on subsequent pages)
|
|
@@ -3,6 +3,7 @@ import { fetchWorkflowHistory } from '#services/workflow_history.js';
|
|
|
3
3
|
import buildSpanLabels from '#utils/span_labels.js';
|
|
4
4
|
import renderWaterfall, { formatDurationLabel } from '#utils/waterfall.js';
|
|
5
5
|
import { handleApiError } from '#utils/error_handler.js';
|
|
6
|
+
import { shouldColorize } from '#utils/color.js';
|
|
6
7
|
const DEFAULT_WIDTH = 80;
|
|
7
8
|
const OUTPUT_FORMAT = { JSON: 'json', TEXT: 'text' };
|
|
8
9
|
export default class WorkflowHistory extends Command {
|
|
@@ -56,7 +57,7 @@ export default class WorkflowHistory extends Command {
|
|
|
56
57
|
});
|
|
57
58
|
if (flags.raw) {
|
|
58
59
|
this.log(JSON.stringify({
|
|
59
|
-
workflow: result.
|
|
60
|
+
workflow: result.rawWorkflow,
|
|
60
61
|
runId: result.runId,
|
|
61
62
|
events: result.events
|
|
62
63
|
}, null, 2));
|
|
@@ -73,8 +74,7 @@ export default class WorkflowHistory extends Command {
|
|
|
73
74
|
}
|
|
74
75
|
const labels = buildSpanLabels(result.spans);
|
|
75
76
|
const width = flags.width ?? process.stdout.columns ?? DEFAULT_WIDTH;
|
|
76
|
-
const color = flags.color
|
|
77
|
-
(!!process.env.FORCE_COLOR || process.stdout.isTTY === true);
|
|
77
|
+
const color = shouldColorize(flags.color);
|
|
78
78
|
this.log(renderWaterfall(result.spans, result.totalDurationMs, {
|
|
79
79
|
width,
|
|
80
80
|
color,
|
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
1
2
|
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
-
// Isolate the command module from the API/service layer at import time.
|
|
3
|
-
|
|
3
|
+
// Isolate the command module from the API/service layer at import time. Must use the
|
|
4
|
+
// `#`-aliased specifier — history.ts imports via that alias, and a relative specifier here
|
|
5
|
+
// resolves to a different module id, so the mock silently never intercepts the real import.
|
|
6
|
+
vi.mock('#services/workflow_history.js', () => ({
|
|
4
7
|
fetchWorkflowHistory: vi.fn()
|
|
5
8
|
}));
|
|
6
9
|
describe('workflow history command', () => {
|
|
@@ -23,4 +26,30 @@ describe('workflow history command', () => {
|
|
|
23
26
|
expect(flags.format.default).toBe('text');
|
|
24
27
|
expect(flags.raw.default).toBe(false);
|
|
25
28
|
});
|
|
29
|
+
describe('run() --raw', () => {
|
|
30
|
+
it('prints the server\'s literal status, not the client-normalized one', async () => {
|
|
31
|
+
const WorkflowHistory = (await import('./history.js')).default;
|
|
32
|
+
const { fetchWorkflowHistory } = await import('#services/workflow_history.js');
|
|
33
|
+
// `workflow` carries the normalized status (what monitor/waterfall consume);
|
|
34
|
+
// `rawWorkflow` is the untouched server value — `--raw` must use the latter.
|
|
35
|
+
vi.mocked(fetchWorkflowHistory).mockResolvedValueOnce({
|
|
36
|
+
workflow: { workflowId: 'wf-1', runId: 'run-1', status: 'continued_as_new' },
|
|
37
|
+
rawWorkflow: { workflowId: 'wf-1', runId: 'run-1', status: 'continued' },
|
|
38
|
+
runId: 'run-1',
|
|
39
|
+
events: [],
|
|
40
|
+
spans: [],
|
|
41
|
+
totalDurationMs: 0,
|
|
42
|
+
continuedAsNewRunId: null
|
|
43
|
+
});
|
|
44
|
+
const cmd = new WorkflowHistory(['wf-1', '--raw'], {});
|
|
45
|
+
cmd.log = vi.fn();
|
|
46
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
47
|
+
args: { workflowId: 'wf-1' },
|
|
48
|
+
flags: { 'run-id': undefined, format: 'text', raw: true, 'include-payloads': false, width: undefined, color: false }
|
|
49
|
+
});
|
|
50
|
+
await cmd.run();
|
|
51
|
+
const printed = JSON.parse(cmd.log.mock.calls[0][0]);
|
|
52
|
+
expect(printed.workflow.status).toBe('continued');
|
|
53
|
+
});
|
|
54
|
+
});
|
|
26
55
|
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { Command } from '@oclif/core';
|
|
2
|
+
/**
|
|
3
|
+
* Unlike `run`/`status`/`result` (migrated to oclif's native `--json` in
|
|
4
|
+
* OUT-419, #281), this command deliberately keeps a custom `--format json`
|
|
5
|
+
* instead of `enableJsonFlag`. Native `--json` suppresses all `this.log()`
|
|
6
|
+
* calls and prints exactly one JSON object — the command's return value —
|
|
7
|
+
* after `run()` resolves. `monitor` has no single "return value": it emits a
|
|
8
|
+
* live stream of discrete events (span status changes, a continue-as-new
|
|
9
|
+
* notice, a final summary) while the workflow is still in progress, and
|
|
10
|
+
* `--format json` prints each as its own NDJSON line as it happens. That's
|
|
11
|
+
* the point — a caller (often another automated/agent process, not a human)
|
|
12
|
+
* can tail and parse the stream incrementally, which native `--json`'s
|
|
13
|
+
* "one object at the end" model can't do. See docs/guides/packages/cli.mdx
|
|
14
|
+
* ("output workflow monitor") for the same rationale written up for users.
|
|
15
|
+
*/
|
|
16
|
+
export default class WorkflowMonitor extends Command {
|
|
17
|
+
static description: string;
|
|
18
|
+
static examples: string[];
|
|
19
|
+
static args: {
|
|
20
|
+
workflowId: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
21
|
+
};
|
|
22
|
+
static flags: {
|
|
23
|
+
'run-id': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
24
|
+
format: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
25
|
+
'include-payloads': import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
26
|
+
interval: import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
27
|
+
color: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
28
|
+
};
|
|
29
|
+
run(): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Wraps a single poll: a failure on the very first tick propagates (there's
|
|
32
|
+
* nothing to fall back on), but a transient blip (see `isTransientPollError`)
|
|
33
|
+
* after we've already been monitoring successfully just returns `null` so the
|
|
34
|
+
* loop can retry — matching the dev TUI's `useStepGraph` behavior of keeping
|
|
35
|
+
* the last good state on a poll hiccup. A non-transient error (e.g. a stale
|
|
36
|
+
* resume cursor, or a bug in the parsing pipeline) rethrows immediately since
|
|
37
|
+
* retrying it cannot succeed. `MAX_CONSECUTIVE_ERRORS` bounds how long we'll
|
|
38
|
+
* retry transient failures before giving up.
|
|
39
|
+
*
|
|
40
|
+
* Fetch strategy is driven by `state.cursor`, not tick count: no cursor yet
|
|
41
|
+
* (the very first poll, or the first poll of a run chained via continue-as-new)
|
|
42
|
+
* uses `fetchWorkflowHistory` (fast, no long-poll) so that render isn't delayed;
|
|
43
|
+
* once a cursor exists, every poll resumes via `fetchWorkflowHistoryUpdates`
|
|
44
|
+
* instead of re-paging the whole history — see `plan_workflow_monitor_history.md`
|
|
45
|
+
* for why a full re-fetch every tick is expensive for long-running workflows.
|
|
46
|
+
*/
|
|
47
|
+
private poll;
|
|
48
|
+
catch(error: Error): Promise<void>;
|
|
49
|
+
}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
import { Args, Command, Flags } from '@oclif/core';
|
|
2
|
+
import { fetchWorkflowHistory, fetchWorkflowHistoryUpdates } from '#services/workflow_history.js';
|
|
3
|
+
import buildSpanLabels from '#utils/span_labels.js';
|
|
4
|
+
import { formatDurationLabel } from '#utils/waterfall.js';
|
|
5
|
+
import { diffSpanUpdates, formatContinuedAsNew, formatSpanUpdate } from '#utils/monitor_log.js';
|
|
6
|
+
import { ERROR_STATUSES, TERMINAL_STATUSES } from '#utils/format_workflow_result.js';
|
|
7
|
+
import { handleApiError } from '#utils/error_handler.js';
|
|
8
|
+
import { getErrorMessage } from '#utils/error_utils.js';
|
|
9
|
+
import { sleep } from '#utils/sleep.js';
|
|
10
|
+
import { shouldColorize } from '#utils/color.js';
|
|
11
|
+
import { HttpError } from '#api/http_client.js';
|
|
12
|
+
const DEFAULT_INTERVAL_MS = 2500;
|
|
13
|
+
const MAX_CONSECUTIVE_ERRORS = 5;
|
|
14
|
+
const OUTPUT_FORMAT = { JSON: 'json', TEXT: 'text' };
|
|
15
|
+
const SIGINT_EXIT_CODE = 130;
|
|
16
|
+
const TRANSIENT_ERROR_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN', 'ENOTFOUND']);
|
|
17
|
+
/**
|
|
18
|
+
* Distinguishes blips worth retrying from errors that will fail identically on
|
|
19
|
+
* every attempt: network hiccups, a client-side request timeout, and 5xx/408/429
|
|
20
|
+
* responses are transient. Everything else — a 4xx like a stale/invalid resume
|
|
21
|
+
* cursor (`InvalidPageTokenError`, surfaced as 400), or a bug in correlate()/
|
|
22
|
+
* buildResult() re-throwing the same exception — can't self-resolve by waiting,
|
|
23
|
+
* so it should surface immediately instead of burning the retry budget.
|
|
24
|
+
*/
|
|
25
|
+
function isTransientPollError(error) {
|
|
26
|
+
if (error instanceof HttpError) {
|
|
27
|
+
const status = error.response.status;
|
|
28
|
+
return status >= 500 || status === 408 || status === 429;
|
|
29
|
+
}
|
|
30
|
+
const err = error;
|
|
31
|
+
if (err.name === 'TimeoutError') {
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
return Boolean((err.code && TRANSIENT_ERROR_CODES.has(err.code)) ||
|
|
35
|
+
(err.cause?.code && TRANSIENT_ERROR_CODES.has(err.cause.code)));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Unlike `run`/`status`/`result` (migrated to oclif's native `--json` in
|
|
39
|
+
* OUT-419, #281), this command deliberately keeps a custom `--format json`
|
|
40
|
+
* instead of `enableJsonFlag`. Native `--json` suppresses all `this.log()`
|
|
41
|
+
* calls and prints exactly one JSON object — the command's return value —
|
|
42
|
+
* after `run()` resolves. `monitor` has no single "return value": it emits a
|
|
43
|
+
* live stream of discrete events (span status changes, a continue-as-new
|
|
44
|
+
* notice, a final summary) while the workflow is still in progress, and
|
|
45
|
+
* `--format json` prints each as its own NDJSON line as it happens. That's
|
|
46
|
+
* the point — a caller (often another automated/agent process, not a human)
|
|
47
|
+
* can tail and parse the stream incrementally, which native `--json`'s
|
|
48
|
+
* "one object at the end" model can't do. See docs/guides/packages/cli.mdx
|
|
49
|
+
* ("output workflow monitor") for the same rationale written up for users.
|
|
50
|
+
*/
|
|
51
|
+
export default class WorkflowMonitor extends Command {
|
|
52
|
+
static description = 'Attach to a workflow run and stream status updates until it ends';
|
|
53
|
+
static examples = [
|
|
54
|
+
'<%= config.bin %> <%= command.id %> wf-12345',
|
|
55
|
+
'<%= config.bin %> <%= command.id %> wf-12345 --run-id 2fe0b36b-...',
|
|
56
|
+
'<%= config.bin %> <%= command.id %> wf-12345 --format json'
|
|
57
|
+
];
|
|
58
|
+
static args = {
|
|
59
|
+
workflowId: Args.string({
|
|
60
|
+
description: 'The workflow ID to monitor',
|
|
61
|
+
required: true
|
|
62
|
+
})
|
|
63
|
+
};
|
|
64
|
+
static flags = {
|
|
65
|
+
'run-id': Flags.string({
|
|
66
|
+
char: 'r',
|
|
67
|
+
description: 'Monitor a specific run (defaults to the latest run; continue-as-new chains are followed regardless)'
|
|
68
|
+
}),
|
|
69
|
+
format: Flags.string({
|
|
70
|
+
char: 'f',
|
|
71
|
+
description: 'Output format',
|
|
72
|
+
options: [OUTPUT_FORMAT.TEXT, OUTPUT_FORMAT.JSON],
|
|
73
|
+
default: OUTPUT_FORMAT.TEXT
|
|
74
|
+
}),
|
|
75
|
+
'include-payloads': Flags.boolean({
|
|
76
|
+
description: 'Include decoded step input/output payloads',
|
|
77
|
+
default: false
|
|
78
|
+
}),
|
|
79
|
+
interval: Flags.integer({
|
|
80
|
+
description: 'Poll interval in milliseconds. Once a resumed poll is long-polling ' +
|
|
81
|
+
'server-side for new events, this also bounds how long that block may last (capped ' +
|
|
82
|
+
'at the server\'s configured max), so an idle workflow\'s update cadence is roughly ' +
|
|
83
|
+
'twice this value (the long-poll bound, then this sleep) rather than a much longer, ' +
|
|
84
|
+
'separate server default.',
|
|
85
|
+
default: DEFAULT_INTERVAL_MS,
|
|
86
|
+
min: 1
|
|
87
|
+
}),
|
|
88
|
+
color: Flags.boolean({
|
|
89
|
+
description: 'Colorize status output (use --no-color to disable)',
|
|
90
|
+
default: true,
|
|
91
|
+
allowNo: true
|
|
92
|
+
})
|
|
93
|
+
};
|
|
94
|
+
async run() {
|
|
95
|
+
const { args, flags } = await this.parse(WorkflowMonitor);
|
|
96
|
+
const color = shouldColorize(flags.color);
|
|
97
|
+
const json = flags.format === OUTPUT_FORMAT.JSON;
|
|
98
|
+
// Threaded via mutable properties (not `let` reassignment) so state
|
|
99
|
+
// persists across polls without local variable reassignment.
|
|
100
|
+
const state = {
|
|
101
|
+
runId: flags['run-id'],
|
|
102
|
+
consecutiveErrors: 0,
|
|
103
|
+
firstTick: true,
|
|
104
|
+
// Undefined until a resumable cursor is established (see `poll` and
|
|
105
|
+
// `fetchWorkflowHistoryUpdates`); reset on continue-as-new since a new run's
|
|
106
|
+
// cursor position is meaningless carried over from the old one.
|
|
107
|
+
cursor: undefined
|
|
108
|
+
};
|
|
109
|
+
const seen = new Map();
|
|
110
|
+
// Assigned once per span id and never overwritten: `buildSpanLabels` numbers
|
|
111
|
+
// same-named spans by how many are in the array *at call time*, so recomputing
|
|
112
|
+
// it fresh every poll could retroactively change a label already printed to
|
|
113
|
+
// the user (e.g. an unnumbered "Scrape Page" becoming "Scrape Page #1" once a
|
|
114
|
+
// second instance appears). Freezing on first sight keeps printed labels stable.
|
|
115
|
+
const labels = new Map();
|
|
116
|
+
// One emit point for both output formats: json mode wraps `fields` (plus
|
|
117
|
+
// the ambient workflow/run id) as a line of NDJSON, text mode prints `text`.
|
|
118
|
+
const emit = (fields, text) => {
|
|
119
|
+
this.log(json ?
|
|
120
|
+
JSON.stringify({ workflowId: args.workflowId, runId: state.runId, ...fields }) :
|
|
121
|
+
text);
|
|
122
|
+
};
|
|
123
|
+
emit({ monitoring: true }, `Monitoring ${args.workflowId}${state.runId ? ` (run ${state.runId})` : ''}... (Ctrl+C to detach)`);
|
|
124
|
+
const sigintHandler = () => {
|
|
125
|
+
emit({ detached: true }, '\nDetached (the workflow keeps running).');
|
|
126
|
+
process.exit(SIGINT_EXIT_CODE);
|
|
127
|
+
};
|
|
128
|
+
process.on('SIGINT', sigintHandler);
|
|
129
|
+
try {
|
|
130
|
+
while (true) {
|
|
131
|
+
const outcome = await this.poll(args.workflowId, flags, state);
|
|
132
|
+
if (outcome === null) {
|
|
133
|
+
state.consecutiveErrors += 1;
|
|
134
|
+
await sleep(flags.interval);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
state.consecutiveErrors = 0;
|
|
138
|
+
state.firstTick = false;
|
|
139
|
+
state.cursor = outcome.cursor;
|
|
140
|
+
const result = outcome.result;
|
|
141
|
+
state.runId = result.runId ?? state.runId;
|
|
142
|
+
for (const [id, label] of buildSpanLabels(result.spans)) {
|
|
143
|
+
if (!labels.has(id)) {
|
|
144
|
+
labels.set(id, label);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
for (const update of diffSpanUpdates(result.spans, labels, seen)) {
|
|
148
|
+
emit({ span: update.span }, formatSpanUpdate(update, color));
|
|
149
|
+
}
|
|
150
|
+
const status = result.workflow?.status;
|
|
151
|
+
if (status === 'continued_as_new') {
|
|
152
|
+
if (!result.continuedAsNewRunId) {
|
|
153
|
+
this.error('Workflow continued as a new run, but the new run ID could not be determined.', { exit: 1 });
|
|
154
|
+
}
|
|
155
|
+
emit({ continuedAsNewRunId: result.continuedAsNewRunId }, formatContinuedAsNew(result.continuedAsNewRunId));
|
|
156
|
+
state.runId = result.continuedAsNewRunId;
|
|
157
|
+
state.cursor = undefined;
|
|
158
|
+
seen.clear();
|
|
159
|
+
labels.clear();
|
|
160
|
+
await sleep(flags.interval);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (status && TERMINAL_STATUSES.has(status)) {
|
|
164
|
+
const summary = `${status === 'completed' ? '✓' : '✗'} workflow ${status} · ${formatDurationLabel(result.totalDurationMs)}`;
|
|
165
|
+
emit({ status }, summary);
|
|
166
|
+
if (ERROR_STATUSES.has(status)) {
|
|
167
|
+
process.exitCode = 1;
|
|
168
|
+
}
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
await sleep(flags.interval);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
finally {
|
|
175
|
+
process.removeListener('SIGINT', sigintHandler);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Wraps a single poll: a failure on the very first tick propagates (there's
|
|
180
|
+
* nothing to fall back on), but a transient blip (see `isTransientPollError`)
|
|
181
|
+
* after we've already been monitoring successfully just returns `null` so the
|
|
182
|
+
* loop can retry — matching the dev TUI's `useStepGraph` behavior of keeping
|
|
183
|
+
* the last good state on a poll hiccup. A non-transient error (e.g. a stale
|
|
184
|
+
* resume cursor, or a bug in the parsing pipeline) rethrows immediately since
|
|
185
|
+
* retrying it cannot succeed. `MAX_CONSECUTIVE_ERRORS` bounds how long we'll
|
|
186
|
+
* retry transient failures before giving up.
|
|
187
|
+
*
|
|
188
|
+
* Fetch strategy is driven by `state.cursor`, not tick count: no cursor yet
|
|
189
|
+
* (the very first poll, or the first poll of a run chained via continue-as-new)
|
|
190
|
+
* uses `fetchWorkflowHistory` (fast, no long-poll) so that render isn't delayed;
|
|
191
|
+
* once a cursor exists, every poll resumes via `fetchWorkflowHistoryUpdates`
|
|
192
|
+
* instead of re-paging the whole history — see `plan_workflow_monitor_history.md`
|
|
193
|
+
* for why a full re-fetch every tick is expensive for long-running workflows.
|
|
194
|
+
*/
|
|
195
|
+
async poll(workflowId, flags, state) {
|
|
196
|
+
try {
|
|
197
|
+
// Keeps the resumed long-poll's server-side block roughly aligned with `--interval`
|
|
198
|
+
// instead of always blocking for the server's full configured deadline regardless of
|
|
199
|
+
// it (see `plan_workflow_monitor_history.md`'s "Known tradeoff" note). Only takes
|
|
200
|
+
// effect once resuming (i.e. from the second tick onward); `fetchWorkflowHistory`
|
|
201
|
+
// ignores it on the initial full walk.
|
|
202
|
+
const options = { workflowId, runId: state.runId, includePayloads: flags['include-payloads'], longPollTimeoutMs: flags.interval };
|
|
203
|
+
if (!state.cursor) {
|
|
204
|
+
const result = await fetchWorkflowHistory(options);
|
|
205
|
+
return { result, cursor: result.cursor };
|
|
206
|
+
}
|
|
207
|
+
return await fetchWorkflowHistoryUpdates(options, state.cursor);
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
if (state.firstTick || !isTransientPollError(error) || state.consecutiveErrors + 1 >= MAX_CONSECUTIVE_ERRORS) {
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
this.warn(`Poll failed (${state.consecutiveErrors + 1}/${MAX_CONSECUTIVE_ERRORS}), retrying: ${getErrorMessage(error)}`);
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
async catch(error) {
|
|
218
|
+
// A 400 is the generic status for several distinct causes (invalid pageToken, a
|
|
219
|
+
// missing runId, an out-of-range longPollTimeoutMs) — only override it with the stale-cursor
|
|
220
|
+
// message when the server actually identifies that specific cause; otherwise let
|
|
221
|
+
// the real validation error surface instead of misdiagnosing an unrelated 400.
|
|
222
|
+
const response = error.response;
|
|
223
|
+
const isStaleCursor = response?.status === 400 && response.data?.error === 'InvalidPageTokenError';
|
|
224
|
+
const overrides = { 404: 'Workflow not found. Check the workflow ID.' };
|
|
225
|
+
if (isStaleCursor) {
|
|
226
|
+
overrides[400] = 'Resume cursor is no longer valid for this workflow; restart the monitor.';
|
|
227
|
+
}
|
|
228
|
+
return handleApiError(error, (...args) => this.error(...args), overrides);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
|
+
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
3
|
+
import { HttpError } from '#api/http_client.js';
|
|
4
|
+
vi.mock('#services/workflow_history.js', () => ({ fetchWorkflowHistory: vi.fn(), fetchWorkflowHistoryUpdates: vi.fn() }));
|
|
5
|
+
vi.mock('#utils/sleep.js', () => ({ sleep: vi.fn().mockResolvedValue(undefined) }));
|
|
6
|
+
const span = (id, status, overrides = {}) => ({
|
|
7
|
+
id,
|
|
8
|
+
name: `Step ${id}`,
|
|
9
|
+
technicalName: `wf#step${id}`,
|
|
10
|
+
description: null,
|
|
11
|
+
status,
|
|
12
|
+
kind: 'activity',
|
|
13
|
+
attempt: 1,
|
|
14
|
+
startedAt: null,
|
|
15
|
+
scheduledAt: null,
|
|
16
|
+
completedAt: null,
|
|
17
|
+
startOffsetMs: 0,
|
|
18
|
+
endOffsetMs: 0,
|
|
19
|
+
durationMs: 1000,
|
|
20
|
+
failureMessage: null,
|
|
21
|
+
...overrides
|
|
22
|
+
});
|
|
23
|
+
const history = (status, overrides = {}) => ({
|
|
24
|
+
workflow: { status },
|
|
25
|
+
runId: 'run-1',
|
|
26
|
+
events: [],
|
|
27
|
+
spans: [],
|
|
28
|
+
totalDurationMs: 0,
|
|
29
|
+
continuedAsNewRunId: null,
|
|
30
|
+
// Every result carries a cursor now (see workflow_history.ts) — `poll()` uses its
|
|
31
|
+
// presence to decide fetchWorkflowHistory vs. fetchWorkflowHistoryUpdates on the next tick.
|
|
32
|
+
cursor: { pageToken: 'token', lastEventId: 1, meta: null, runId: 'run-1', events: [] },
|
|
33
|
+
...overrides
|
|
34
|
+
});
|
|
35
|
+
// `fetchWorkflowHistoryUpdates` (used from the second poll onward) wraps the same result
|
|
36
|
+
// shape with a resume cursor; tests don't care about cursor contents, only that it's threaded.
|
|
37
|
+
const update = (status, overrides = {}) => ({
|
|
38
|
+
result: history(status, overrides),
|
|
39
|
+
cursor: { pageToken: 'token', lastEventId: 1, meta: null, runId: 'run-1', events: [] }
|
|
40
|
+
});
|
|
41
|
+
describe('workflow monitor command', () => {
|
|
42
|
+
beforeEach(() => {
|
|
43
|
+
vi.clearAllMocks();
|
|
44
|
+
process.exitCode = undefined;
|
|
45
|
+
});
|
|
46
|
+
describe('command definition', () => {
|
|
47
|
+
it('exports a valid OCLIF command with a required workflowId arg', async () => {
|
|
48
|
+
const WorkflowMonitor = (await import('./monitor.js')).default;
|
|
49
|
+
expect(WorkflowMonitor).toBeDefined();
|
|
50
|
+
expect(WorkflowMonitor.args).toHaveProperty('workflowId');
|
|
51
|
+
expect(WorkflowMonitor.args.workflowId.required).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
it('declares the expected flags and defaults', async () => {
|
|
54
|
+
const WorkflowMonitor = (await import('./monitor.js')).default;
|
|
55
|
+
const flags = WorkflowMonitor.flags;
|
|
56
|
+
expect(flags).toHaveProperty('run-id');
|
|
57
|
+
expect(flags).toHaveProperty('include-payloads');
|
|
58
|
+
expect(flags).toHaveProperty('interval');
|
|
59
|
+
expect(flags).toHaveProperty('color');
|
|
60
|
+
expect(flags.format.options).toEqual(['text', 'json']);
|
|
61
|
+
expect(flags.format.default).toBe('text');
|
|
62
|
+
expect(flags.interval.default).toBe(2500);
|
|
63
|
+
expect(flags.color.default).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
describe('run()', () => {
|
|
67
|
+
const createCommand = async (flagOverrides = {}) => {
|
|
68
|
+
const WorkflowMonitor = (await import('./monitor.js')).default;
|
|
69
|
+
const { fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await import('#services/workflow_history.js');
|
|
70
|
+
const cmd = new WorkflowMonitor(['wf-1'], {});
|
|
71
|
+
cmd.log = vi.fn();
|
|
72
|
+
cmd.warn = vi.fn();
|
|
73
|
+
cmd.error = vi.fn((message) => {
|
|
74
|
+
throw new Error(message);
|
|
75
|
+
});
|
|
76
|
+
cmd.parse = vi.fn().mockResolvedValue({
|
|
77
|
+
args: { workflowId: 'wf-1' },
|
|
78
|
+
flags: {
|
|
79
|
+
'run-id': undefined,
|
|
80
|
+
format: 'text',
|
|
81
|
+
'include-payloads': false,
|
|
82
|
+
interval: 1,
|
|
83
|
+
color: false,
|
|
84
|
+
...flagOverrides
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
return {
|
|
88
|
+
cmd,
|
|
89
|
+
fetchWorkflowHistory: vi.mocked(fetchWorkflowHistory),
|
|
90
|
+
fetchWorkflowHistoryUpdates: vi.mocked(fetchWorkflowHistoryUpdates)
|
|
91
|
+
};
|
|
92
|
+
};
|
|
93
|
+
it('prints span updates and exits cleanly when the workflow is already completed on the first poll', async () => {
|
|
94
|
+
const { cmd, fetchWorkflowHistory } = await createCommand();
|
|
95
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('completed', { spans: [span('1', 'completed')], totalDurationMs: 5000 }));
|
|
96
|
+
await cmd.run();
|
|
97
|
+
expect(fetchWorkflowHistory).toHaveBeenCalledTimes(1);
|
|
98
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('Step 1'));
|
|
99
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('workflow completed'));
|
|
100
|
+
expect(process.exitCode).toBeUndefined();
|
|
101
|
+
});
|
|
102
|
+
it('sets exit code 1 when the workflow ends in a failed status', async () => {
|
|
103
|
+
const { cmd, fetchWorkflowHistory } = await createCommand();
|
|
104
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('failed', {
|
|
105
|
+
spans: [span('1', 'failed', { failureMessage: 'boom' })], totalDurationMs: 1000
|
|
106
|
+
}));
|
|
107
|
+
await cmd.run();
|
|
108
|
+
expect(process.exitCode).toBe(1);
|
|
109
|
+
});
|
|
110
|
+
it('polls again while the workflow is running, then stops once it completes', async () => {
|
|
111
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
112
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running', { spans: [span('1', 'running')] }));
|
|
113
|
+
fetchWorkflowHistoryUpdates.mockResolvedValueOnce(update('completed', {
|
|
114
|
+
spans: [span('1', 'completed')], totalDurationMs: 2000
|
|
115
|
+
}));
|
|
116
|
+
await cmd.run();
|
|
117
|
+
expect(fetchWorkflowHistory).toHaveBeenCalledTimes(1);
|
|
118
|
+
expect(fetchWorkflowHistoryUpdates).toHaveBeenCalledTimes(1);
|
|
119
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('running'));
|
|
120
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('1s'));
|
|
121
|
+
expect(process.exitCode).toBeUndefined();
|
|
122
|
+
});
|
|
123
|
+
it('keeps a span label stable once printed, even when a same-named span appears on a later poll', async () => {
|
|
124
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
125
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running', { spans: [span('1', 'running', { name: 'Scrape Page' })] }));
|
|
126
|
+
// Span 2 shares span 1's name, only showing up on the second poll — buildSpanLabels
|
|
127
|
+
// would number both "#1"/"#2" if recomputed fresh, retroactively relabeling span 1's
|
|
128
|
+
// already-printed "running" line.
|
|
129
|
+
fetchWorkflowHistoryUpdates.mockResolvedValueOnce(update('completed', {
|
|
130
|
+
spans: [
|
|
131
|
+
span('1', 'completed', { name: 'Scrape Page' }),
|
|
132
|
+
span('2', 'completed', { name: 'Scrape Page' })
|
|
133
|
+
]
|
|
134
|
+
}));
|
|
135
|
+
await cmd.run();
|
|
136
|
+
const lines = cmd.log.mock.calls.map(([line]) => line);
|
|
137
|
+
expect(lines.some((line) => line.includes('Scrape Page running'))).toBe(true);
|
|
138
|
+
// Span 1's completion line keeps its original unnumbered label...
|
|
139
|
+
expect(lines.some((line) => line.includes('Scrape Page '))).toBe(true);
|
|
140
|
+
expect(lines.some((line) => line.includes('Scrape Page #1'))).toBe(false);
|
|
141
|
+
// ...while span 2, new this tick, is free to be numbered.
|
|
142
|
+
expect(lines.some((line) => line.includes('Scrape Page #2'))).toBe(true);
|
|
143
|
+
});
|
|
144
|
+
it('does not re-print a span whose status has not changed between polls', async () => {
|
|
145
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
146
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running', { spans: [span('1', 'running')] }));
|
|
147
|
+
fetchWorkflowHistoryUpdates.mockResolvedValueOnce(update('completed', {
|
|
148
|
+
spans: [span('1', 'running'), span('2', 'completed')], totalDurationMs: 1000
|
|
149
|
+
}));
|
|
150
|
+
await cmd.run();
|
|
151
|
+
const runningLines = cmd.log.mock.calls.filter(([line]) => line.includes('Step 1'));
|
|
152
|
+
expect(runningLines).toHaveLength(1); // span 1 only reported once, not again on the second poll
|
|
153
|
+
});
|
|
154
|
+
it('follows a continue-as-new chain by re-polling with the new run id', async () => {
|
|
155
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
156
|
+
fetchWorkflowHistory
|
|
157
|
+
.mockResolvedValueOnce(history('continued_as_new', { continuedAsNewRunId: 'run-2' }))
|
|
158
|
+
// The new run's cursor was reset, so its first poll takes the same fast,
|
|
159
|
+
// non-waiting path as the very first poll of the command — not a resumed one.
|
|
160
|
+
.mockResolvedValueOnce(history('completed', { runId: 'run-2', totalDurationMs: 500 }));
|
|
161
|
+
await cmd.run();
|
|
162
|
+
expect(fetchWorkflowHistory).toHaveBeenCalledTimes(2);
|
|
163
|
+
expect(fetchWorkflowHistory).toHaveBeenNthCalledWith(2, expect.objectContaining({ runId: 'run-2' }));
|
|
164
|
+
expect(fetchWorkflowHistoryUpdates).not.toHaveBeenCalled();
|
|
165
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('continued as new run run-2'));
|
|
166
|
+
expect(process.exitCode).toBeUndefined();
|
|
167
|
+
});
|
|
168
|
+
it('errors when the workflow continues as new but no new run id can be determined', async () => {
|
|
169
|
+
const { cmd, fetchWorkflowHistory } = await createCommand();
|
|
170
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('continued_as_new'));
|
|
171
|
+
await expect(cmd.run()).rejects.toThrow(/new run ID could not be determined/);
|
|
172
|
+
});
|
|
173
|
+
it('propagates a failure on the very first poll', async () => {
|
|
174
|
+
const { cmd, fetchWorkflowHistory } = await createCommand();
|
|
175
|
+
fetchWorkflowHistory.mockRejectedValue(new Error('network down'));
|
|
176
|
+
await expect(cmd.run()).rejects.toThrow('network down');
|
|
177
|
+
expect(fetchWorkflowHistory).toHaveBeenCalledTimes(1);
|
|
178
|
+
});
|
|
179
|
+
it('retries a transient network failure after the first successful poll instead of crashing', async () => {
|
|
180
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
181
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running'));
|
|
182
|
+
fetchWorkflowHistoryUpdates
|
|
183
|
+
.mockRejectedValueOnce(Object.assign(new Error('blip'), { code: 'ECONNRESET' }))
|
|
184
|
+
.mockResolvedValueOnce(update('completed', { totalDurationMs: 1000 }));
|
|
185
|
+
await cmd.run();
|
|
186
|
+
expect(fetchWorkflowHistory).toHaveBeenCalledTimes(1);
|
|
187
|
+
expect(fetchWorkflowHistoryUpdates).toHaveBeenCalledTimes(2);
|
|
188
|
+
expect(cmd.warn).toHaveBeenCalledWith(expect.stringContaining('(1/5)'));
|
|
189
|
+
expect(process.exitCode).toBeUndefined();
|
|
190
|
+
});
|
|
191
|
+
it('retries a transient 503 HttpError after the first successful poll', async () => {
|
|
192
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
193
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running'));
|
|
194
|
+
fetchWorkflowHistoryUpdates
|
|
195
|
+
.mockRejectedValueOnce(new HttpError('Service unavailable', { status: 503 }))
|
|
196
|
+
.mockResolvedValueOnce(update('completed', { totalDurationMs: 1000 }));
|
|
197
|
+
await cmd.run();
|
|
198
|
+
expect(fetchWorkflowHistoryUpdates).toHaveBeenCalledTimes(2);
|
|
199
|
+
expect(cmd.warn).toHaveBeenCalledWith(expect.stringContaining('(1/5)'));
|
|
200
|
+
expect(process.exitCode).toBeUndefined();
|
|
201
|
+
});
|
|
202
|
+
it('immediately rethrows a non-transient error (e.g. a stale resume cursor) without retrying', async () => {
|
|
203
|
+
const { cmd, fetchWorkflowHistory, fetchWorkflowHistoryUpdates } = await createCommand();
|
|
204
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('running'));
|
|
205
|
+
fetchWorkflowHistoryUpdates.mockRejectedValueOnce(new HttpError('Invalid page token', { status: 400 }));
|
|
206
|
+
await expect(cmd.run()).rejects.toThrow('Invalid page token');
|
|
207
|
+
expect(fetchWorkflowHistoryUpdates).toHaveBeenCalledTimes(1);
|
|
208
|
+
expect(cmd.warn).not.toHaveBeenCalled();
|
|
209
|
+
});
|
|
210
|
+
it('registers a SIGINT handler that detaches and exits 130 without affecting the workflow', async () => {
|
|
211
|
+
const { cmd, fetchWorkflowHistory } = await createCommand();
|
|
212
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('completed'));
|
|
213
|
+
const onSpy = vi.spyOn(process, 'on');
|
|
214
|
+
const exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined));
|
|
215
|
+
await cmd.run();
|
|
216
|
+
const sigintCall = onSpy.mock.calls.find(([event]) => event === 'SIGINT');
|
|
217
|
+
expect(sigintCall).toBeDefined();
|
|
218
|
+
const handler = sigintCall[1];
|
|
219
|
+
handler();
|
|
220
|
+
expect(exitSpy).toHaveBeenCalledWith(130);
|
|
221
|
+
expect(cmd.log).toHaveBeenCalledWith(expect.stringContaining('Detached'));
|
|
222
|
+
onSpy.mockRestore();
|
|
223
|
+
exitSpy.mockRestore();
|
|
224
|
+
});
|
|
225
|
+
it('emits NDJSON lines under --format json', async () => {
|
|
226
|
+
const { cmd, fetchWorkflowHistory } = await createCommand({ format: 'json' });
|
|
227
|
+
fetchWorkflowHistory.mockResolvedValueOnce(history('completed', { spans: [span('1', 'completed')], totalDurationMs: 1000 }));
|
|
228
|
+
await cmd.run();
|
|
229
|
+
const lines = cmd.log.mock.calls.map(([line]) => line);
|
|
230
|
+
expect(lines.every((line) => {
|
|
231
|
+
try {
|
|
232
|
+
JSON.parse(line);
|
|
233
|
+
return true;
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
return false;
|
|
237
|
+
}
|
|
238
|
+
})).toBe(true);
|
|
239
|
+
expect(lines.some((line) => JSON.parse(line).span?.id === '1')).toBe(true);
|
|
240
|
+
expect(lines.some((line) => JSON.parse(line).monitoring === true)).toBe(true);
|
|
241
|
+
});
|
|
242
|
+
});
|
|
243
|
+
});
|
|
@@ -19,6 +19,8 @@ export interface Span {
|
|
|
19
19
|
output?: unknown;
|
|
20
20
|
}
|
|
21
21
|
export type HistoryEvent = Record<string, unknown>;
|
|
22
|
+
export declare function eventTypeName(event: HistoryEvent): string;
|
|
23
|
+
export declare function eventAttributes(event?: HistoryEvent): Record<string, unknown> | undefined;
|
|
22
24
|
/**
|
|
23
25
|
* @param events - flat Temporal history events, in chronological order
|
|
24
26
|
* @param workflowStartTimeMs - epoch ms used as the timeline origin (0 offset)
|