@osolmaz/pi-workflows 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -5
- package/dist/builtins/catalog.d.ts +2 -0
- package/dist/builtins/catalog.js +22 -0
- package/dist/builtins/catalog.js.map +1 -0
- package/dist/builtins/monitor.workflow.d.ts +71 -0
- package/dist/builtins/monitor.workflow.js +250 -0
- package/dist/builtins/monitor.workflow.js.map +1 -0
- package/dist/controllers/conditions.d.ts +6 -0
- package/dist/controllers/conditions.js +68 -0
- package/dist/controllers/conditions.js.map +1 -0
- package/dist/controllers/definition.d.ts +6 -0
- package/dist/controllers/definition.js +45 -0
- package/dist/controllers/definition.js.map +1 -0
- package/dist/controllers/effects.d.ts +14 -0
- package/dist/controllers/effects.js +104 -0
- package/dist/controllers/effects.js.map +1 -0
- package/dist/controllers/errors.d.ts +12 -0
- package/dist/controllers/errors.js +25 -0
- package/dist/controllers/errors.js.map +1 -0
- package/dist/controllers/index.d.ts +13 -0
- package/dist/controllers/index.js +13 -0
- package/dist/controllers/index.js.map +1 -0
- package/dist/controllers/json.d.ts +5 -0
- package/dist/controllers/json.js +57 -0
- package/dist/controllers/json.js.map +1 -0
- package/dist/controllers/loader.d.ts +23 -0
- package/dist/controllers/loader.js +74 -0
- package/dist/controllers/loader.js.map +1 -0
- package/dist/controllers/manager.d.ts +58 -0
- package/dist/controllers/manager.js +399 -0
- package/dist/controllers/manager.js.map +1 -0
- package/dist/controllers/results.d.ts +5 -0
- package/dist/controllers/results.js +32 -0
- package/dist/controllers/results.js.map +1 -0
- package/dist/controllers/sqlite.d.ts +235 -0
- package/dist/controllers/sqlite.js +1087 -0
- package/dist/controllers/sqlite.js.map +1 -0
- package/dist/controllers/store.d.ts +112 -0
- package/dist/controllers/store.js +32 -0
- package/dist/controllers/store.js.map +1 -0
- package/dist/controllers/types.d.ts +159 -0
- package/dist/controllers/types.js +2 -0
- package/dist/controllers/types.js.map +1 -0
- package/dist/controllers/workflow-engine-scheduler.d.ts +25 -0
- package/dist/controllers/workflow-engine-scheduler.js +95 -0
- package/dist/controllers/workflow-engine-scheduler.js.map +1 -0
- package/dist/controllers/workflows.d.ts +27 -0
- package/dist/controllers/workflows.js +109 -0
- package/dist/controllers/workflows.js.map +1 -0
- package/dist/extension/controller-host.d.ts +47 -0
- package/dist/extension/controller-host.js +110 -0
- package/dist/extension/controller-host.js.map +1 -0
- package/dist/extension/executor.d.ts +3 -0
- package/dist/extension/executor.js +12 -2
- package/dist/extension/executor.js.map +1 -1
- package/dist/extension/index.d.ts +7 -0
- package/dist/extension/index.js +1118 -106
- package/dist/extension/index.js.map +1 -1
- package/dist/extension/recorder.d.ts +6 -0
- package/dist/extension/recorder.js +20 -8
- package/dist/extension/recorder.js.map +1 -1
- package/dist/extension/workflow-tool.d.ts +28 -0
- package/dist/extension/workflow-tool.js +33 -0
- package/dist/extension/workflow-tool.js.map +1 -0
- package/dist/host/processes.d.ts +24 -0
- package/dist/host/processes.js +114 -0
- package/dist/host/processes.js.map +1 -0
- package/dist/host/rpc-bridge.d.ts +9 -0
- package/dist/host/rpc-bridge.js +39 -0
- package/dist/host/rpc-bridge.js.map +1 -0
- package/dist/host/rpc-executor.d.ts +38 -0
- package/dist/host/rpc-executor.js +254 -0
- package/dist/host/rpc-executor.js.map +1 -0
- package/dist/host/runner.d.ts +50 -0
- package/dist/host/runner.js +379 -0
- package/dist/host/runner.js.map +1 -0
- package/dist/viewer/cli.d.ts +7 -3
- package/dist/viewer/cli.js +150 -19
- package/dist/viewer/cli.js.map +1 -1
- package/dist/workflows/catalog.d.ts +43 -0
- package/dist/workflows/catalog.js +79 -0
- package/dist/workflows/catalog.js.map +1 -0
- package/dist/workflows/engine.d.ts +37 -2
- package/dist/workflows/engine.js +297 -29
- package/dist/workflows/engine.js.map +1 -1
- package/dist/workflows/errors.d.ts +23 -0
- package/dist/workflows/errors.js +38 -0
- package/dist/workflows/errors.js.map +1 -1
- package/dist/workflows/graph.js +0 -5
- package/dist/workflows/graph.js.map +1 -1
- package/dist/workflows/index.d.ts +2 -2
- package/dist/workflows/index.js +1 -1
- package/dist/workflows/index.js.map +1 -1
- package/dist/workflows/loader.d.ts +19 -15
- package/dist/workflows/loader.js +63 -19
- package/dist/workflows/loader.js.map +1 -1
- package/dist/workflows/migrate-sources.d.ts +41 -0
- package/dist/workflows/migrate-sources.js +129 -0
- package/dist/workflows/migrate-sources.js.map +1 -0
- package/dist/workflows/schema.js +3 -2
- package/dist/workflows/schema.js.map +1 -1
- package/dist/workflows/store.d.ts +50 -6
- package/dist/workflows/store.js +448 -53
- package/dist/workflows/store.js.map +1 -1
- package/dist/workflows/types.d.ts +27 -3
- package/docs/CONTROLLERS.md +215 -0
- package/docs/development.md +14 -9
- package/docs/plans/2026-08-04-controller-runtime-plan.md +169 -0
- package/docs/plans/2026-08-05-always-on-workflows-plan.md +125 -0
- package/docs/plans/2026-08-10-agent-managed-monitor-workflows-plan.md +184 -0
- package/docs/plans/2026-08-12-coordinated-workflow-timeouts-plan.md +74 -0
- package/docs/plans/2026-08-13-built-in-workflow-catalog-plan.md +97 -0
- package/docs/run-bundles.md +85 -28
- package/docs/workflows.md +161 -17
- package/examples/controllers/pull-request.controller.ts +215 -0
- package/package.json +10 -2
- package/src/builtins/catalog.ts +22 -0
- package/src/builtins/monitor.workflow.ts +302 -0
- package/src/controllers/conditions.ts +110 -0
- package/src/controllers/definition.ts +65 -0
- package/src/controllers/effects.ts +123 -0
- package/src/controllers/errors.ts +27 -0
- package/src/controllers/index.ts +90 -0
- package/src/controllers/json.ts +62 -0
- package/src/controllers/loader.ts +104 -0
- package/src/controllers/manager.ts +533 -0
- package/src/controllers/results.ts +46 -0
- package/src/controllers/sqlite.ts +1545 -0
- package/src/controllers/store.ts +160 -0
- package/src/controllers/types.ts +183 -0
- package/src/controllers/workflow-engine-scheduler.ts +148 -0
- package/src/controllers/workflows.ts +152 -0
- package/src/extension/controller-host.ts +163 -0
- package/src/extension/executor.ts +13 -2
- package/src/extension/index.ts +1325 -130
- package/src/extension/recorder.ts +65 -36
- package/src/extension/workflow-tool.ts +59 -0
- package/src/host/processes.ts +119 -0
- package/src/host/rpc-bridge.ts +44 -0
- package/src/host/rpc-executor.ts +299 -0
- package/src/host/runner.ts +445 -0
- package/src/viewer/cli.ts +167 -21
- package/src/workflows/catalog.ts +135 -0
- package/src/workflows/engine.ts +404 -41
- package/src/workflows/errors.ts +45 -0
- package/src/workflows/graph.ts +0 -5
- package/src/workflows/index.ts +2 -0
- package/src/workflows/loader.ts +76 -22
- package/src/workflows/migrate-sources.ts +167 -0
- package/src/workflows/schema.ts +3 -2
- package/src/workflows/store.ts +557 -48
- package/src/workflows/types.ts +22 -3
package/src/workflows/engine.ts
CHANGED
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { isDeepStrictEqual } from "node:util";
|
|
3
|
-
import {
|
|
3
|
+
import { resolveArtifacts } from "./artifacts.js";
|
|
4
|
+
import {
|
|
5
|
+
CancelledError,
|
|
6
|
+
errorMessage,
|
|
7
|
+
isAbortLikeError,
|
|
8
|
+
isClaimLostError,
|
|
9
|
+
isRunParkedError,
|
|
10
|
+
RunParkedError,
|
|
11
|
+
TimeoutError,
|
|
12
|
+
WorkflowSourceChangedError,
|
|
13
|
+
} from "./errors.js";
|
|
4
14
|
import { resolveNext, resolveNextForOutcome, validateWorkflowDefinition } from "./graph.js";
|
|
5
15
|
import { extractJsonValue } from "./json.js";
|
|
6
16
|
import { runShellAction, shellResultFromError } from "./shell.js";
|
|
7
|
-
import { RUN_STATE_SCHEMA, WorkflowRunStore, createRunId } from "./store.js";
|
|
17
|
+
import { RUN_STATE_SCHEMA, WorkflowRunStore, createRunId, readRunBundle } from "./store.js";
|
|
8
18
|
import type {
|
|
9
19
|
AgentNodeDefinition,
|
|
10
20
|
AgentStepExecutor,
|
|
@@ -22,6 +32,7 @@ import type {
|
|
|
22
32
|
WorkflowNodeResult,
|
|
23
33
|
WorkflowRunResult,
|
|
24
34
|
WorkflowRunState,
|
|
35
|
+
WorkflowSource,
|
|
25
36
|
WorkflowStepRecord,
|
|
26
37
|
WorkflowTraceEventDraft,
|
|
27
38
|
} from "./types.js";
|
|
@@ -29,6 +40,7 @@ import type {
|
|
|
29
40
|
const DEFAULT_NODE_TIMEOUT_MS = 15 * 60_000;
|
|
30
41
|
const DEFAULT_MAX_STEPS = 100;
|
|
31
42
|
const TITLE_TIMEOUT_MS = 30_000;
|
|
43
|
+
const TIMEOUT_RESOLUTION_TIMEOUT_MS = 30_000;
|
|
32
44
|
// Covers the shell SIGTERM → SIGKILL escalation (1s) plus stdio close.
|
|
33
45
|
const ABORT_CLEANUP_GRACE_MS = 2_000;
|
|
34
46
|
|
|
@@ -70,6 +82,7 @@ export class WorkflowEngine {
|
|
|
70
82
|
private readonly onRunFinishing?: WorkflowEngineOptions["onRunFinishing"];
|
|
71
83
|
private activeAbort: AbortController | null = null;
|
|
72
84
|
private cancelled = false;
|
|
85
|
+
private parked = false;
|
|
73
86
|
private paused = false;
|
|
74
87
|
private wakePause: (() => void) | null = null;
|
|
75
88
|
|
|
@@ -96,6 +109,17 @@ export class WorkflowEngine {
|
|
|
96
109
|
this.wakePause?.();
|
|
97
110
|
}
|
|
98
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Stop without a terminal event so another runner can claim and resume
|
|
114
|
+
* the run. The active node aborts; its partial attempt is never recorded,
|
|
115
|
+
* so resume reruns that node from its last persisted boundary.
|
|
116
|
+
*/
|
|
117
|
+
park(): void {
|
|
118
|
+
this.parked = true;
|
|
119
|
+
this.activeAbort?.abort(new CancelledError());
|
|
120
|
+
this.wakePause?.();
|
|
121
|
+
}
|
|
122
|
+
|
|
99
123
|
/**
|
|
100
124
|
* Request a pause. The current step finishes normally; the engine then
|
|
101
125
|
* holds before dispatching the next node until `resume` (or `cancel`).
|
|
@@ -118,17 +142,26 @@ export class WorkflowEngine {
|
|
|
118
142
|
async run(
|
|
119
143
|
workflow: WorkflowDefinition,
|
|
120
144
|
input: unknown,
|
|
121
|
-
options: {
|
|
145
|
+
options: { workflowSource?: WorkflowSource; runId?: string } = {},
|
|
122
146
|
): Promise<WorkflowRunResult> {
|
|
123
147
|
validateWorkflowDefinition(workflow);
|
|
124
148
|
// Fail before any bundle exists so bad input cannot leave a partial run
|
|
125
149
|
// on disk or silently change shape when state.json round-trips.
|
|
126
150
|
const normalizedInput = input === undefined ? null : input;
|
|
127
151
|
assertJsonSerializable(normalizedInput, "Workflow run input");
|
|
152
|
+
if (options.runId !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(options.runId)) {
|
|
153
|
+
throw new Error(`Invalid workflow run id: ${JSON.stringify(options.runId)}`);
|
|
154
|
+
}
|
|
128
155
|
this.cancelled = false;
|
|
129
156
|
this.paused = false;
|
|
157
|
+
this.parked = false;
|
|
130
158
|
|
|
131
|
-
const state = await this.createRunState(
|
|
159
|
+
const state = await this.createRunState(
|
|
160
|
+
workflow,
|
|
161
|
+
normalizedInput,
|
|
162
|
+
options.workflowSource,
|
|
163
|
+
options.runId,
|
|
164
|
+
);
|
|
132
165
|
const runDir = await this.store.initializeRunBundle(workflow, state);
|
|
133
166
|
await this.persist(runDir, state, {
|
|
134
167
|
scope: "run",
|
|
@@ -147,13 +180,266 @@ export class WorkflowEngine {
|
|
|
147
180
|
try {
|
|
148
181
|
await this.executeGraph(workflow, state, runDir);
|
|
149
182
|
} catch (error) {
|
|
150
|
-
|
|
183
|
+
if (isRunParkedError(error) || this.parked) {
|
|
184
|
+
return { runDir, state };
|
|
185
|
+
}
|
|
186
|
+
await this.finishAfterError(runDir, state, error);
|
|
187
|
+
return { runDir, state };
|
|
188
|
+
}
|
|
189
|
+
return { runDir, state };
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Resume an interrupted run at the node it stopped on. The caller must
|
|
194
|
+
* hold the run's queue claim. Completed nodes replay from the recorded
|
|
195
|
+
* state; only the interrupted node and everything downstream rerun.
|
|
196
|
+
*/
|
|
197
|
+
async resumeRun(
|
|
198
|
+
workflow: WorkflowDefinition,
|
|
199
|
+
runId: string,
|
|
200
|
+
options: { workflowSource?: WorkflowSource; force?: boolean } = {},
|
|
201
|
+
): Promise<WorkflowRunResult> {
|
|
202
|
+
validateWorkflowDefinition(workflow);
|
|
203
|
+
// Reset before any await: a park or cancel landing during preparation
|
|
204
|
+
// must survive, or a host drain would hang while the run executes.
|
|
205
|
+
this.cancelled = false;
|
|
206
|
+
this.paused = false;
|
|
207
|
+
this.parked = false;
|
|
208
|
+
const bundle = await this.store.prepareRunResume(runId);
|
|
209
|
+
const { runDir } = bundle;
|
|
210
|
+
const state = bundle.state;
|
|
211
|
+
const sourceMismatch = workflowSourceMismatch(state, options.workflowSource);
|
|
212
|
+
if (sourceMismatch && options.force !== true) {
|
|
213
|
+
throw new WorkflowSourceChangedError(runId);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const point = this.resumePointFor(workflow, state, "wait");
|
|
217
|
+
// A resumed run starts unpaused; the operator can pause again. The
|
|
218
|
+
// interrupted node's stale in-flight markers go away before the resume
|
|
219
|
+
// event so the projection matches what the engine is about to do.
|
|
220
|
+
delete state.paused;
|
|
221
|
+
delete state.currentNode;
|
|
222
|
+
delete state.currentAttemptId;
|
|
223
|
+
delete state.currentNodeStartedAt;
|
|
224
|
+
delete state.statusDetail;
|
|
225
|
+
await this.persist(runDir, state, {
|
|
226
|
+
scope: "run",
|
|
227
|
+
type: "run_resumed",
|
|
228
|
+
payload: {
|
|
229
|
+
...(point.nodeId !== null ? { resumeAt: point.nodeId } : {}),
|
|
230
|
+
replayedSteps: state.steps.length,
|
|
231
|
+
...(sourceMismatch ? { workflowSourceMismatch: true, forced: true } : {}),
|
|
232
|
+
},
|
|
233
|
+
});
|
|
234
|
+
await this.onRunStarted?.(runDir, state);
|
|
235
|
+
|
|
236
|
+
if (point.nodeId === null) {
|
|
237
|
+
// The last recorded transition already finished the graph; the crash
|
|
238
|
+
// happened before the terminal event was written. A finished
|
|
239
|
+
// checkpoint restores its waiting gate rather than completing.
|
|
240
|
+
if (point.waitingOn !== undefined) {
|
|
241
|
+
await this.finishRun(runDir, state, "waiting", {
|
|
242
|
+
waitingOn: point.waitingOn,
|
|
243
|
+
finalOutput: point.lastOutput,
|
|
244
|
+
});
|
|
245
|
+
} else if (point.failedResult === undefined) {
|
|
246
|
+
await this.finishRun(runDir, state, "completed", { finalOutput: point.lastOutput });
|
|
247
|
+
} else {
|
|
248
|
+
const timedOut = point.failedResult.outcome === "timed_out";
|
|
249
|
+
await this.finishRun(runDir, state, timedOut ? "timed_out" : "failed", {
|
|
250
|
+
error: point.failedResult.error ?? `Workflow node failed: ${point.failedResult.nodeId}`,
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
return { runDir, state };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
try {
|
|
257
|
+
await this.executeGraph(
|
|
258
|
+
workflow,
|
|
259
|
+
state,
|
|
260
|
+
runDir,
|
|
261
|
+
point.nodeId,
|
|
262
|
+
state.steps.length,
|
|
263
|
+
point.lastOutput,
|
|
264
|
+
);
|
|
265
|
+
} catch (error) {
|
|
266
|
+
if (isRunParkedError(error) || this.parked) {
|
|
267
|
+
return { runDir, state };
|
|
268
|
+
}
|
|
269
|
+
await this.finishAfterError(runDir, state, error);
|
|
270
|
+
return { runDir, state };
|
|
271
|
+
}
|
|
272
|
+
return { runDir, state };
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Start a continuation run from a checkpointed parent. The new run gets a
|
|
277
|
+
* fresh bundle and trace, carries forward the parent's outputs, results,
|
|
278
|
+
* and step accounting, and continues routing after the checkpoint.
|
|
279
|
+
*/
|
|
280
|
+
async continueRun(
|
|
281
|
+
workflow: WorkflowDefinition,
|
|
282
|
+
parentRunId: string,
|
|
283
|
+
input: unknown,
|
|
284
|
+
options: { workflowSource?: WorkflowSource; runId?: string; force?: boolean } = {},
|
|
285
|
+
): Promise<WorkflowRunResult> {
|
|
286
|
+
validateWorkflowDefinition(workflow);
|
|
287
|
+
this.cancelled = false;
|
|
288
|
+
this.paused = false;
|
|
289
|
+
this.parked = false;
|
|
290
|
+
const parent = await readRunBundle(this.store.runDirFor(parentRunId));
|
|
291
|
+
if (parent === null) {
|
|
292
|
+
throw new Error(`Cannot continue from unreadable workflow run: ${parentRunId}`);
|
|
293
|
+
}
|
|
294
|
+
if (parent.state.status !== "waiting" || parent.state.waitingOn === undefined) {
|
|
295
|
+
throw new Error(
|
|
296
|
+
`Cannot continue workflow run ${parentRunId} with status ${parent.state.status}`,
|
|
297
|
+
);
|
|
298
|
+
}
|
|
299
|
+
const sourceMismatch = workflowSourceMismatch(parent.state, options.workflowSource);
|
|
300
|
+
if (sourceMismatch && options.force !== true) {
|
|
301
|
+
throw new WorkflowSourceChangedError(parentRunId);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
const normalizedInput = input === undefined ? null : input;
|
|
305
|
+
assertJsonSerializable(normalizedInput, "Workflow run input");
|
|
306
|
+
if (options.runId !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(options.runId)) {
|
|
307
|
+
throw new Error(`Invalid workflow run id: ${JSON.stringify(options.runId)}`);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
const state = await this.createRunState(
|
|
311
|
+
workflow,
|
|
312
|
+
normalizedInput,
|
|
313
|
+
options.workflowSource,
|
|
314
|
+
options.runId,
|
|
315
|
+
);
|
|
316
|
+
state.parentRunId = parentRunId;
|
|
317
|
+
// Artifact references point into the parent's bundle, so carried values
|
|
318
|
+
// are fully resolved here and re-externalized into the new bundle.
|
|
319
|
+
state.outputs = (await resolveArtifacts(
|
|
320
|
+
parent.state.outputs,
|
|
321
|
+
parent.runDir,
|
|
322
|
+
)) as WorkflowRunState["outputs"];
|
|
323
|
+
state.results = (await resolveArtifacts(
|
|
324
|
+
parent.state.results,
|
|
325
|
+
parent.runDir,
|
|
326
|
+
)) as WorkflowRunState["results"];
|
|
327
|
+
state.steps = (await resolveArtifacts(
|
|
328
|
+
parent.state.steps,
|
|
329
|
+
parent.runDir,
|
|
330
|
+
)) as WorkflowRunState["steps"];
|
|
331
|
+
state.carriedStepCount = state.steps.length;
|
|
332
|
+
|
|
333
|
+
const runDir = await this.store.initializeRunBundle(workflow, state);
|
|
334
|
+
await this.persist(runDir, state, {
|
|
335
|
+
scope: "run",
|
|
336
|
+
type: "run_started",
|
|
337
|
+
payload: {
|
|
338
|
+
workflowName: workflow.name,
|
|
339
|
+
...(state.runTitle ? { runTitle: state.runTitle } : {}),
|
|
340
|
+
input: state.input,
|
|
341
|
+
continuedFrom: parentRunId,
|
|
342
|
+
checkpoint: parent.state.waitingOn,
|
|
343
|
+
carriedSteps: state.steps.length,
|
|
344
|
+
},
|
|
345
|
+
});
|
|
346
|
+
await this.onRunStarted?.(runDir, state);
|
|
347
|
+
|
|
348
|
+
const point = this.resumePointFor(workflow, state, "continue");
|
|
349
|
+
if (point.nodeId === null) {
|
|
350
|
+
// The checkpoint was the final node; the answer completes the chain.
|
|
351
|
+
await this.finishRun(runDir, state, "completed", { finalOutput: point.lastOutput });
|
|
352
|
+
return { runDir, state };
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
await this.executeGraph(
|
|
356
|
+
workflow,
|
|
357
|
+
state,
|
|
358
|
+
runDir,
|
|
359
|
+
point.nodeId,
|
|
360
|
+
state.steps.length,
|
|
361
|
+
point.lastOutput,
|
|
362
|
+
);
|
|
363
|
+
} catch (error) {
|
|
364
|
+
if (isRunParkedError(error) || this.parked) {
|
|
365
|
+
return { runDir, state };
|
|
366
|
+
}
|
|
367
|
+
await this.finishAfterError(runDir, state, error);
|
|
368
|
+
return { runDir, state };
|
|
369
|
+
}
|
|
370
|
+
return { runDir, state };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Find where a resumed run continues. An in-flight node reruns; otherwise
|
|
375
|
+
* routing continues from the last recorded step. A null nodeId means the
|
|
376
|
+
* graph was already done when the crash hit.
|
|
377
|
+
*/
|
|
378
|
+
private resumePointFor(
|
|
379
|
+
workflow: WorkflowDefinition,
|
|
380
|
+
state: WorkflowRunState,
|
|
381
|
+
checkpointBehavior: "wait" | "continue",
|
|
382
|
+
): {
|
|
383
|
+
nodeId: string | null;
|
|
384
|
+
lastOutput?: unknown;
|
|
385
|
+
failedResult?: WorkflowNodeResult;
|
|
386
|
+
waitingOn?: string;
|
|
387
|
+
} {
|
|
388
|
+
if (state.currentNode !== undefined) {
|
|
389
|
+
if (workflow.nodes[state.currentNode] === undefined) {
|
|
390
|
+
throw new Error(`Resume node is missing from the workflow: ${state.currentNode}`);
|
|
391
|
+
}
|
|
392
|
+
return { nodeId: state.currentNode };
|
|
393
|
+
}
|
|
394
|
+
const lastStep = state.steps.at(-1);
|
|
395
|
+
if (lastStep === undefined) {
|
|
396
|
+
return { nodeId: workflow.startAt };
|
|
397
|
+
}
|
|
398
|
+
const result = state.results[lastStep.nodeId];
|
|
399
|
+
if (result === undefined) {
|
|
400
|
+
return { nodeId: lastStep.nodeId };
|
|
401
|
+
}
|
|
402
|
+
if (result.outcome === "ok") {
|
|
403
|
+
// A recorded checkpoint means the run should be waiting; a crash
|
|
404
|
+
// before the run_waiting persist restores the gate instead of
|
|
405
|
+
// routing past it. The gate applies to this run's own checkpoint
|
|
406
|
+
// only: a continuation's carried steps end with the parent's
|
|
407
|
+
// already-answered checkpoint, and routing must continue from it.
|
|
408
|
+
const isCarriedStep = state.steps.length <= (state.carriedStepCount ?? 0);
|
|
409
|
+
if (
|
|
410
|
+
checkpointBehavior === "wait" &&
|
|
411
|
+
!isCarriedStep &&
|
|
412
|
+
workflow.nodes[lastStep.nodeId]?.nodeType === "checkpoint"
|
|
413
|
+
) {
|
|
414
|
+
return { nodeId: null, waitingOn: lastStep.nodeId, lastOutput: result.output };
|
|
415
|
+
}
|
|
416
|
+
const next = resolveNext(workflow.edges, lastStep.nodeId, result.output, result);
|
|
417
|
+
return next === null
|
|
418
|
+
? { nodeId: null, lastOutput: result.output }
|
|
419
|
+
: { nodeId: next, lastOutput: result.output };
|
|
420
|
+
}
|
|
421
|
+
const next = resolveNextForOutcome(workflow.edges, lastStep.nodeId, result);
|
|
422
|
+
return next === null ? { nodeId: null, failedResult: result } : { nodeId: next };
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
private async finishAfterError(
|
|
426
|
+
runDir: string,
|
|
427
|
+
state: WorkflowRunState,
|
|
428
|
+
error: unknown,
|
|
429
|
+
): Promise<void> {
|
|
430
|
+
const cancelled = this.cancelled || isAbortLikeError(error);
|
|
431
|
+
try {
|
|
151
432
|
await this.finishRun(runDir, state, cancelled ? "cancelled" : "failed", {
|
|
152
433
|
error: errorMessage(error),
|
|
153
434
|
});
|
|
154
|
-
|
|
435
|
+
} catch (finishError) {
|
|
436
|
+
// A fenced-out runner must not touch the bundle, including terminal
|
|
437
|
+
// projections. Propagate the claim loss instead of the node error.
|
|
438
|
+
if (isClaimLostError(finishError)) {
|
|
439
|
+
throw finishError;
|
|
440
|
+
}
|
|
441
|
+
throw error;
|
|
155
442
|
}
|
|
156
|
-
return { runDir, state };
|
|
157
443
|
}
|
|
158
444
|
|
|
159
445
|
/**
|
|
@@ -185,16 +471,17 @@ export class WorkflowEngine {
|
|
|
185
471
|
private async createRunState(
|
|
186
472
|
workflow: WorkflowDefinition,
|
|
187
473
|
input: unknown,
|
|
188
|
-
|
|
474
|
+
workflowSource: WorkflowSource | undefined,
|
|
475
|
+
runId: string | undefined,
|
|
189
476
|
): Promise<WorkflowRunState> {
|
|
190
477
|
const now = new Date().toISOString();
|
|
191
478
|
return {
|
|
192
479
|
schema: RUN_STATE_SCHEMA,
|
|
193
480
|
traceSeq: 0,
|
|
194
|
-
runId: createRunId(workflow.name),
|
|
481
|
+
runId: runId ?? createRunId(workflow.name),
|
|
195
482
|
workflowName: workflow.name,
|
|
196
483
|
...(await this.resolveTitleBounded(workflow, input)),
|
|
197
|
-
...(
|
|
484
|
+
...(workflowSource !== undefined ? { workflowSource } : {}),
|
|
198
485
|
startedAt: now,
|
|
199
486
|
updatedAt: now,
|
|
200
487
|
status: "running",
|
|
@@ -209,11 +496,14 @@ export class WorkflowEngine {
|
|
|
209
496
|
workflow: WorkflowDefinition,
|
|
210
497
|
state: WorkflowRunState,
|
|
211
498
|
runDir: string,
|
|
499
|
+
startNodeId: string | null = workflow.startAt,
|
|
500
|
+
executedStepsBase = 0,
|
|
501
|
+
initialLastOutput?: unknown,
|
|
212
502
|
): Promise<void> {
|
|
213
503
|
const maxSteps = workflow.maxSteps ?? this.maxSteps;
|
|
214
|
-
let currentNodeId: string | null =
|
|
215
|
-
let executedSteps =
|
|
216
|
-
let lastOutput: unknown;
|
|
504
|
+
let currentNodeId: string | null = startNodeId;
|
|
505
|
+
let executedSteps = executedStepsBase;
|
|
506
|
+
let lastOutput: unknown = initialLastOutput;
|
|
217
507
|
|
|
218
508
|
while (currentNodeId !== null) {
|
|
219
509
|
await this.holdWhilePaused(state, runDir);
|
|
@@ -230,6 +520,11 @@ export class WorkflowEngine {
|
|
|
230
520
|
}
|
|
231
521
|
|
|
232
522
|
const attempt = await this.executeNode(workflow, state, runDir, currentNodeId, node);
|
|
523
|
+
if (this.parked) {
|
|
524
|
+
// Do not record the aborted attempt: the projection keeps the node
|
|
525
|
+
// as in-flight, and resume reruns it with a fresh attempt.
|
|
526
|
+
throw new RunParkedError();
|
|
527
|
+
}
|
|
233
528
|
this.recordAttempt(state, attempt);
|
|
234
529
|
// The terminal node event carries the output, receipt, and conversation
|
|
235
530
|
// linkage so the trace alone is sufficient to reconstruct the run.
|
|
@@ -279,6 +574,9 @@ export class WorkflowEngine {
|
|
|
279
574
|
* never interrupts a node mid-flight; it only delays the next dispatch.
|
|
280
575
|
*/
|
|
281
576
|
private async holdWhilePaused(state: WorkflowRunState, runDir: string): Promise<void> {
|
|
577
|
+
if (this.parked) {
|
|
578
|
+
throw new RunParkedError();
|
|
579
|
+
}
|
|
282
580
|
if (this.cancelled) {
|
|
283
581
|
throw new CancelledError();
|
|
284
582
|
}
|
|
@@ -287,12 +585,15 @@ export class WorkflowEngine {
|
|
|
287
585
|
}
|
|
288
586
|
state.paused = true;
|
|
289
587
|
await this.persist(runDir, state, { scope: "run", type: "run_paused", payload: {} });
|
|
290
|
-
while (this.paused && !this.cancelled) {
|
|
588
|
+
while (this.paused && !this.cancelled && !this.parked) {
|
|
291
589
|
await new Promise<void>((resolve) => {
|
|
292
590
|
this.wakePause = resolve;
|
|
293
591
|
});
|
|
294
592
|
}
|
|
295
593
|
this.wakePause = null;
|
|
594
|
+
if (this.parked) {
|
|
595
|
+
throw new RunParkedError();
|
|
596
|
+
}
|
|
296
597
|
delete state.paused;
|
|
297
598
|
if (this.cancelled) {
|
|
298
599
|
throw new CancelledError();
|
|
@@ -449,31 +750,43 @@ export class WorkflowEngine {
|
|
|
449
750
|
node: WorkflowNodeDefinition,
|
|
450
751
|
meta: NodeExecutionMeta,
|
|
451
752
|
): Promise<NodeExecution> {
|
|
452
|
-
const timeoutMs = node.timeoutMs ?? this.defaultNodeTimeoutMs;
|
|
453
753
|
const abort = new AbortController();
|
|
754
|
+
const context = this.createNodeContext(state, abort.signal);
|
|
755
|
+
let timer: NodeJS.Timeout | undefined;
|
|
756
|
+
let dispatchSettled: Promise<void> | undefined;
|
|
454
757
|
this.activeAbort = abort;
|
|
455
|
-
if (this.cancelled) {
|
|
456
|
-
throw new CancelledError();
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
const timer = setTimeout(() => {
|
|
460
|
-
abort.abort(new TimeoutError(timeoutMs));
|
|
461
|
-
}, timeoutMs);
|
|
462
|
-
const dispatched = this.dispatchNode(
|
|
463
|
-
workflow,
|
|
464
|
-
state,
|
|
465
|
-
runDir,
|
|
466
|
-
nodeId,
|
|
467
|
-
attemptId,
|
|
468
|
-
node,
|
|
469
|
-
abort.signal,
|
|
470
|
-
meta,
|
|
471
|
-
);
|
|
472
|
-
const dispatchSettled = dispatched.then(
|
|
473
|
-
() => undefined,
|
|
474
|
-
() => undefined,
|
|
475
|
-
);
|
|
476
758
|
try {
|
|
759
|
+
if (this.parked) {
|
|
760
|
+
// A park that landed during the node_started persist must not let the
|
|
761
|
+
// node dispatch: its discarded side effects would rerun on resume.
|
|
762
|
+
throw new RunParkedError();
|
|
763
|
+
}
|
|
764
|
+
if (this.cancelled) {
|
|
765
|
+
throw new CancelledError();
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
const timeoutMs = await this.resolveNodeTimeout(node, context, abort);
|
|
769
|
+
if (abort.signal.aborted) {
|
|
770
|
+
throw abortError(abort.signal);
|
|
771
|
+
}
|
|
772
|
+
timer = setTimeout(() => {
|
|
773
|
+
abort.abort(new TimeoutError(timeoutMs));
|
|
774
|
+
}, timeoutMs);
|
|
775
|
+
const dispatched = this.dispatchNode(
|
|
776
|
+
workflow,
|
|
777
|
+
state,
|
|
778
|
+
runDir,
|
|
779
|
+
nodeId,
|
|
780
|
+
attemptId,
|
|
781
|
+
node,
|
|
782
|
+
context,
|
|
783
|
+
abort.signal,
|
|
784
|
+
meta,
|
|
785
|
+
);
|
|
786
|
+
dispatchSettled = dispatched.then(
|
|
787
|
+
() => undefined,
|
|
788
|
+
() => undefined,
|
|
789
|
+
);
|
|
477
790
|
// Race the dispatch against the abort signal so timeouts and cancel
|
|
478
791
|
// take effect even for node callbacks that never observe the signal.
|
|
479
792
|
const execution = await Promise.race([dispatched, abortRejection(abort.signal)]);
|
|
@@ -485,7 +798,7 @@ export class WorkflowEngine {
|
|
|
485
798
|
assertJsonSerializable(execution.output, `Node ${nodeId} output`);
|
|
486
799
|
return execution;
|
|
487
800
|
} catch (error) {
|
|
488
|
-
if (node.nodeType === "action" && "exec" in node) {
|
|
801
|
+
if (node.nodeType === "action" && "exec" in node && dispatchSettled !== undefined) {
|
|
489
802
|
// Give the killed shell command a short grace period to close so its
|
|
490
803
|
// action receipt lands in `meta` before the failed attempt persists.
|
|
491
804
|
await Promise.race([
|
|
@@ -495,9 +808,37 @@ export class WorkflowEngine {
|
|
|
495
808
|
}
|
|
496
809
|
const reason: unknown = abort.signal.aborted ? abort.signal.reason : undefined;
|
|
497
810
|
throw reason instanceof TimeoutError || reason instanceof CancelledError ? reason : error;
|
|
811
|
+
} finally {
|
|
812
|
+
if (timer !== undefined) {
|
|
813
|
+
clearTimeout(timer);
|
|
814
|
+
}
|
|
815
|
+
if (this.activeAbort === abort) {
|
|
816
|
+
this.activeAbort = null;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
private async resolveNodeTimeout(
|
|
822
|
+
node: WorkflowNodeDefinition,
|
|
823
|
+
context: WorkflowNodeContext,
|
|
824
|
+
abort: AbortController,
|
|
825
|
+
): Promise<number> {
|
|
826
|
+
const configured = node.timeoutMs;
|
|
827
|
+
if (typeof configured !== "function") {
|
|
828
|
+
return assertValidTimeout(configured ?? this.defaultNodeTimeoutMs);
|
|
829
|
+
}
|
|
830
|
+
const timer = setTimeout(
|
|
831
|
+
() => abort.abort(new TimeoutError(TIMEOUT_RESOLUTION_TIMEOUT_MS)),
|
|
832
|
+
TIMEOUT_RESOLUTION_TIMEOUT_MS,
|
|
833
|
+
);
|
|
834
|
+
try {
|
|
835
|
+
const resolved = await Promise.race([
|
|
836
|
+
Promise.resolve(configured(context)),
|
|
837
|
+
abortRejection(abort.signal),
|
|
838
|
+
]);
|
|
839
|
+
return assertValidTimeout(resolved);
|
|
498
840
|
} finally {
|
|
499
841
|
clearTimeout(timer);
|
|
500
|
-
this.activeAbort = null;
|
|
501
842
|
}
|
|
502
843
|
}
|
|
503
844
|
|
|
@@ -508,10 +849,10 @@ export class WorkflowEngine {
|
|
|
508
849
|
nodeId: string,
|
|
509
850
|
attemptId: string,
|
|
510
851
|
node: WorkflowNodeDefinition,
|
|
852
|
+
context: WorkflowNodeContext,
|
|
511
853
|
signal: AbortSignal,
|
|
512
854
|
meta: NodeExecutionMeta,
|
|
513
855
|
): Promise<NodeExecution> {
|
|
514
|
-
const context = this.createNodeContext(state, signal);
|
|
515
856
|
switch (node.nodeType) {
|
|
516
857
|
case "agent":
|
|
517
858
|
return await this.runAgentNode(
|
|
@@ -730,7 +1071,13 @@ async function runShellActionNode(
|
|
|
730
1071
|
return { output, promptText: null, action: shellReceipt(result) };
|
|
731
1072
|
}
|
|
732
1073
|
|
|
733
|
-
|
|
1074
|
+
function assertValidTimeout(value: number): number {
|
|
1075
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
1076
|
+
throw new Error("Node timeoutMs must resolve to a finite positive number");
|
|
1077
|
+
}
|
|
1078
|
+
return value;
|
|
1079
|
+
}
|
|
1080
|
+
|
|
734
1081
|
/** The error carried by an aborted signal, normalized to an Error. */
|
|
735
1082
|
function abortError(signal: AbortSignal): Error {
|
|
736
1083
|
const reason: unknown = signal.reason ?? new CancelledError();
|
|
@@ -755,6 +1102,22 @@ function abortRejection(signal: AbortSignal): Promise<never> {
|
|
|
755
1102
|
* Failing here turns a bad callback return value into a normal node failure
|
|
756
1103
|
* instead of corrupting the run state.
|
|
757
1104
|
*/
|
|
1105
|
+
function workflowSourceMismatch(
|
|
1106
|
+
state: WorkflowRunState,
|
|
1107
|
+
source: WorkflowSource | undefined,
|
|
1108
|
+
): boolean {
|
|
1109
|
+
if (source === undefined) return false;
|
|
1110
|
+
if (state.workflowSource !== undefined) {
|
|
1111
|
+
return !isDeepStrictEqual(state.workflowSource, source);
|
|
1112
|
+
}
|
|
1113
|
+
// Bounded compatibility check for pre-catalog file runs. Startup normally
|
|
1114
|
+
// converts these records with migrateLegacyWorkflowSources first.
|
|
1115
|
+
return (
|
|
1116
|
+
state.workflowHash !== undefined &&
|
|
1117
|
+
(source.kind !== "file" || state.workflowHash !== source.hash)
|
|
1118
|
+
);
|
|
1119
|
+
}
|
|
1120
|
+
|
|
758
1121
|
function assertJsonSerializable(value: unknown, what: string): void {
|
|
759
1122
|
let encoded: string | undefined;
|
|
760
1123
|
try {
|
|
@@ -803,7 +1166,7 @@ export function appendStepContract(
|
|
|
803
1166
|
`Workflow step contract (workflow: ${workflowName}, step: ${nodeId}, attempt: ${attemptId})`,
|
|
804
1167
|
"",
|
|
805
1168
|
"Complete this step by calling the `workflow` tool exactly once with:",
|
|
806
|
-
`{"step": ${JSON.stringify(nodeId)}, "attempt": ${JSON.stringify(attemptId)}, "output": <your result>}`,
|
|
1169
|
+
`{"action": "submit", "step": ${JSON.stringify(nodeId)}, "attempt": ${JSON.stringify(attemptId)}, "output": <your result>}`,
|
|
807
1170
|
`Expected output: ${expectedOutput ?? "a JSON object with your result"}`,
|
|
808
1171
|
"The step is complete only after the workflow tool accepts the output.",
|
|
809
1172
|
"If the tool reports a validation error, correct the output and call it again.",
|
package/src/workflows/errors.ts
CHANGED
|
@@ -15,6 +15,51 @@ export class CancelledError extends Error {
|
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Thrown when a runner writes to a run bundle after losing its queue claim.
|
|
20
|
+
* The current claim holder owns the run from that point on, so the fenced
|
|
21
|
+
* writer must stop touching the bundle immediately.
|
|
22
|
+
*/
|
|
23
|
+
export class ClaimLostError extends Error {
|
|
24
|
+
readonly runId: string;
|
|
25
|
+
|
|
26
|
+
constructor(runId: string) {
|
|
27
|
+
super(`Workflow run claim lost: ${runId}`);
|
|
28
|
+
this.name = "ClaimLostError";
|
|
29
|
+
this.runId = runId;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Internal stop signal for close-to-park: the engine halts without writing
|
|
35
|
+
* a terminal event, leaving a resumable bundle for the next claim holder.
|
|
36
|
+
*/
|
|
37
|
+
export class RunParkedError extends Error {
|
|
38
|
+
constructor() {
|
|
39
|
+
super("Workflow run was parked");
|
|
40
|
+
this.name = "RunParkedError";
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function isRunParkedError(error: unknown): error is RunParkedError {
|
|
45
|
+
return error instanceof RunParkedError;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** The workflow source changed after the run started; resume needs force. */
|
|
49
|
+
export class WorkflowSourceChangedError extends Error {
|
|
50
|
+
readonly runId: string;
|
|
51
|
+
|
|
52
|
+
constructor(runId: string) {
|
|
53
|
+
super(`Workflow source changed since run ${runId} started; pass force to resume anyway`);
|
|
54
|
+
this.name = "WorkflowSourceChangedError";
|
|
55
|
+
this.runId = runId;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function isClaimLostError(error: unknown): error is ClaimLostError {
|
|
60
|
+
return error instanceof ClaimLostError;
|
|
61
|
+
}
|
|
62
|
+
|
|
18
63
|
export function isAbortLikeError(error: unknown): boolean {
|
|
19
64
|
return error instanceof CancelledError || (error instanceof Error && error.name === "AbortError");
|
|
20
65
|
}
|
package/src/workflows/graph.ts
CHANGED
|
@@ -58,11 +58,6 @@ function validateWorkflowEdge(
|
|
|
58
58
|
outgoingEdges: Set<string>,
|
|
59
59
|
): void {
|
|
60
60
|
assertKnownNode(workflow, edge.from, "Workflow edge references unknown from-node");
|
|
61
|
-
if (workflow.nodes[edge.from]?.nodeType === "checkpoint") {
|
|
62
|
-
// A checkpoint terminates the run as `waiting` and nothing resumes it,
|
|
63
|
-
// so an outgoing edge would make its targets silently unreachable.
|
|
64
|
-
throw new Error(`Workflow checkpoint node must not declare an outgoing edge: ${edge.from}`);
|
|
65
|
-
}
|
|
66
61
|
if (outgoingEdges.has(edge.from)) {
|
|
67
62
|
throw new Error(`Workflow node must not declare multiple outgoing edges: ${edge.from}`);
|
|
68
63
|
}
|
package/src/workflows/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ export {
|
|
|
21
21
|
discoverWorkflows,
|
|
22
22
|
loadWorkflowFile,
|
|
23
23
|
resolveWorkflowRef,
|
|
24
|
+
resolveWorkflowSource,
|
|
24
25
|
workflowFileStem,
|
|
25
26
|
workflowSearchDirs,
|
|
26
27
|
type DiscoveredWorkflow,
|
|
@@ -83,6 +84,7 @@ export type {
|
|
|
83
84
|
WorkflowRunResult,
|
|
84
85
|
WorkflowRunState,
|
|
85
86
|
WorkflowRunStatus,
|
|
87
|
+
WorkflowSource,
|
|
86
88
|
WorkflowSessionBinding,
|
|
87
89
|
WorkflowSessionEntryRecord,
|
|
88
90
|
WorkflowStepRecord,
|