bunqueue 2.8.46 → 2.8.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/workflow/admission.d.ts +45 -0
- package/dist/client/workflow/admission.js +50 -0
- package/dist/client/workflow/clock.d.ts +58 -0
- package/dist/client/workflow/clock.js +104 -0
- package/dist/client/workflow/compensator.d.ts +56 -3
- package/dist/client/workflow/compensator.js +406 -27
- package/dist/client/workflow/emitter.d.ts +1 -1
- package/dist/client/workflow/emitter.js +4 -3
- package/dist/client/workflow/engine.d.ts +17 -0
- package/dist/client/workflow/engine.js +25 -0
- package/dist/client/workflow/executor.d.ts +40 -5
- package/dist/client/workflow/executor.js +227 -83
- package/dist/client/workflow/identity.d.ts +46 -0
- package/dist/client/workflow/identity.js +93 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.js +147 -14
- package/dist/client/workflow/recovery.d.ts +10 -1
- package/dist/client/workflow/recovery.js +36 -9
- package/dist/client/workflow/rollbackControl.d.ts +36 -0
- package/dist/client/workflow/rollbackControl.js +51 -0
- package/dist/client/workflow/runner.d.ts +22 -2
- package/dist/client/workflow/runner.js +125 -27
- package/dist/client/workflow/store.d.ts +64 -4
- package/dist/client/workflow/store.js +123 -30
- package/dist/client/workflow/storeCodec.d.ts +7 -0
- package/dist/client/workflow/storeCodec.js +16 -0
- package/dist/client/workflow/storeSignals.d.ts +61 -0
- package/dist/client/workflow/storeSignals.js +118 -0
- package/dist/client/workflow/types.d.ts +147 -4
- package/dist/client/workflow/unwindPlan.d.ts +87 -0
- package/dist/client/workflow/unwindPlan.js +142 -0
- package/dist/client/workflow/waitFor.d.ts +52 -0
- package/dist/client/workflow/waitFor.js +137 -0
- package/dist/client/workflow/workflow.d.ts +71 -1
- package/dist/client/workflow/workflow.js +184 -14
- package/package.json +16 -6
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SignalCoordinator — sole owner of the `signals` column.
|
|
3
|
+
*
|
|
4
|
+
* Signal payloads cannot be persisted the way step state is. A worker holds one
|
|
5
|
+
* in-memory `Execution` for the whole duration of a node, so any full-row write from
|
|
6
|
+
* that snapshot silently reverts a signal that landed while the node was running —
|
|
7
|
+
* the approval disappears and the run parks forever
|
|
8
|
+
* (test/repro-workflow-signal-lost-update.test.ts).
|
|
9
|
+
*
|
|
10
|
+
* So every mutation of `signals` goes through here instead: read-modify-write inside
|
|
11
|
+
* a transaction, and the two state transitions that pair with it ('waiting' ->
|
|
12
|
+
* 'running' on resume, 'running' -> 'waiting' on park) expressed as conditional
|
|
13
|
+
* UPDATEs. Making those conditional is also what collapses duplicate or concurrent
|
|
14
|
+
* signals to exactly one resume, at the database rather than relying on the event
|
|
15
|
+
* loop never yielding at the right moment.
|
|
16
|
+
*/
|
|
17
|
+
import type { Database } from 'bun:sqlite';
|
|
18
|
+
import type { ParkOutcome, SignalOutcome } from './types';
|
|
19
|
+
/**
|
|
20
|
+
* Has this signal arrived? A KEY test, deliberately not a value test.
|
|
21
|
+
*
|
|
22
|
+
* `payload` is optional on `Engine.signal()`, so "the human said go" with nothing to
|
|
23
|
+
* carry is recorded as `signals[event] = undefined`. The codec is configured with
|
|
24
|
+
* `structuredClone: true` and round-trips that faithfully — the key is present, the
|
|
25
|
+
* value is `undefined` — so `signals[event] !== undefined` answers "did it arrive?"
|
|
26
|
+
* with NO for the most idiomatic call in the whole human-in-the-loop API.
|
|
27
|
+
*
|
|
28
|
+
* That disagreed with `record()`, which claims the resume without inspecting the
|
|
29
|
+
* payload: the run resumed, re-entered the waitFor, was told no signal had arrived,
|
|
30
|
+
* and parked again. With a timeout it then expired and compensated work the approver
|
|
31
|
+
* had just authorised (`test/repro-workflow-signal-no-payload.test.ts`).
|
|
32
|
+
*
|
|
33
|
+
* The predicate itself is `Object.hasOwn`, and an earlier revision of this comment
|
|
34
|
+
* claimed `in` was safe here because the object is always a fresh decode. That was
|
|
35
|
+
* wrong twice over: `in` walks the prototype chain regardless of where the object came
|
|
36
|
+
* from, and the EVENT NAME is user-supplied even when the object is not.
|
|
37
|
+
*/
|
|
38
|
+
export declare function hasSignal(signals: Record<string, unknown>, event: string): boolean;
|
|
39
|
+
export declare class SignalCoordinator {
|
|
40
|
+
private readonly db;
|
|
41
|
+
private readonly read;
|
|
42
|
+
private readonly write;
|
|
43
|
+
private readonly claimResume;
|
|
44
|
+
private readonly claimPark;
|
|
45
|
+
constructor(db: Database);
|
|
46
|
+
/**
|
|
47
|
+
* Record a signal payload and, if the run is parked at a `waitFor`, atomically
|
|
48
|
+
* claim the single resume for this caller.
|
|
49
|
+
*/
|
|
50
|
+
record(id: string, event: string, payload: unknown): SignalOutcome;
|
|
51
|
+
/**
|
|
52
|
+
* Park a running execution at a `waitFor`, unless the awaited signal has already
|
|
53
|
+
* been recorded.
|
|
54
|
+
*
|
|
55
|
+
* Closes the window where a signal lands after `runWaitFor` read its in-memory
|
|
56
|
+
* snapshot but before the run reaches state `waiting`: without the re-check the
|
|
57
|
+
* payload is stored, nobody claims the resume, and the run hangs.
|
|
58
|
+
*/
|
|
59
|
+
park(id: string, event: string): ParkOutcome;
|
|
60
|
+
private decode;
|
|
61
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SignalCoordinator — sole owner of the `signals` column.
|
|
3
|
+
*
|
|
4
|
+
* Signal payloads cannot be persisted the way step state is. A worker holds one
|
|
5
|
+
* in-memory `Execution` for the whole duration of a node, so any full-row write from
|
|
6
|
+
* that snapshot silently reverts a signal that landed while the node was running —
|
|
7
|
+
* the approval disappears and the run parks forever
|
|
8
|
+
* (test/repro-workflow-signal-lost-update.test.ts).
|
|
9
|
+
*
|
|
10
|
+
* So every mutation of `signals` goes through here instead: read-modify-write inside
|
|
11
|
+
* a transaction, and the two state transitions that pair with it ('waiting' ->
|
|
12
|
+
* 'running' on resume, 'running' -> 'waiting' on park) expressed as conditional
|
|
13
|
+
* UPDATEs. Making those conditional is also what collapses duplicate or concurrent
|
|
14
|
+
* signals to exactly one resume, at the database rather than relying on the event
|
|
15
|
+
* loop never yielding at the right moment.
|
|
16
|
+
*/
|
|
17
|
+
import { pack, unpack } from './storeCodec';
|
|
18
|
+
import { clock } from './clock';
|
|
19
|
+
/**
|
|
20
|
+
* Has this signal arrived? A KEY test, deliberately not a value test.
|
|
21
|
+
*
|
|
22
|
+
* `payload` is optional on `Engine.signal()`, so "the human said go" with nothing to
|
|
23
|
+
* carry is recorded as `signals[event] = undefined`. The codec is configured with
|
|
24
|
+
* `structuredClone: true` and round-trips that faithfully — the key is present, the
|
|
25
|
+
* value is `undefined` — so `signals[event] !== undefined` answers "did it arrive?"
|
|
26
|
+
* with NO for the most idiomatic call in the whole human-in-the-loop API.
|
|
27
|
+
*
|
|
28
|
+
* That disagreed with `record()`, which claims the resume without inspecting the
|
|
29
|
+
* payload: the run resumed, re-entered the waitFor, was told no signal had arrived,
|
|
30
|
+
* and parked again. With a timeout it then expired and compensated work the approver
|
|
31
|
+
* had just authorised (`test/repro-workflow-signal-no-payload.test.ts`).
|
|
32
|
+
*
|
|
33
|
+
* The predicate itself is `Object.hasOwn`, and an earlier revision of this comment
|
|
34
|
+
* claimed `in` was safe here because the object is always a fresh decode. That was
|
|
35
|
+
* wrong twice over: `in` walks the prototype chain regardless of where the object came
|
|
36
|
+
* from, and the EVENT NAME is user-supplied even when the object is not.
|
|
37
|
+
*/
|
|
38
|
+
export function hasSignal(signals, event) {
|
|
39
|
+
// `Object.hasOwn`, not `in`, and not `signals[event] !== undefined`. All three read
|
|
40
|
+
// like the same question and only one of them asks it.
|
|
41
|
+
//
|
|
42
|
+
// `signals` is a plain object used as a map, so `in` walks the prototype chain:
|
|
43
|
+
// `'toString' in {}` is true, and a gate named `toString`, `constructor`, `valueOf`
|
|
44
|
+
// or `hasOwnProperty` opened the instant it parked, with nobody having signalled
|
|
45
|
+
// anything. The value test that preceded it failed identically, because
|
|
46
|
+
// `signals['toString']` is the inherited function and not `undefined`.
|
|
47
|
+
//
|
|
48
|
+
// The names are not exotic in a real vocabulary, and an event name read from config
|
|
49
|
+
// or user input is attacker-influenced. The failure is silent and it opens the one
|
|
50
|
+
// control that exists to stop things
|
|
51
|
+
// (`test/repro-workflow-prototype-gate.test.ts`, found by the generated-input suite
|
|
52
|
+
// in `test/workflow-properties.test.ts`).
|
|
53
|
+
return Object.hasOwn(signals, event);
|
|
54
|
+
}
|
|
55
|
+
export class SignalCoordinator {
|
|
56
|
+
db;
|
|
57
|
+
read;
|
|
58
|
+
write;
|
|
59
|
+
claimResume;
|
|
60
|
+
claimPark;
|
|
61
|
+
constructor(db) {
|
|
62
|
+
this.db = db;
|
|
63
|
+
this.read = db.prepare(`SELECT workflow_name, state, current_node_index, signals FROM workflow_executions WHERE id = ?`);
|
|
64
|
+
this.write = db.prepare(`UPDATE workflow_executions SET signals = ?, updated_at = ? WHERE id = ?`);
|
|
65
|
+
this.claimResume = db.prepare(`UPDATE workflow_executions SET state = 'running', updated_at = ? WHERE id = ? AND state = 'waiting'`);
|
|
66
|
+
// 'waiting' is accepted as a source state so re-parking is idempotent: a clamped
|
|
67
|
+
// or partial timeout re-arm re-enters the same waitFor node while the row is
|
|
68
|
+
// still 'waiting', and must be allowed to park (and re-arm) again.
|
|
69
|
+
this.claimPark = db.prepare(`UPDATE workflow_executions SET state = 'waiting', updated_at = ? WHERE id = ? AND state IN ('running', 'waiting')`);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Record a signal payload and, if the run is parked at a `waitFor`, atomically
|
|
73
|
+
* claim the single resume for this caller.
|
|
74
|
+
*/
|
|
75
|
+
record(id, event, payload) {
|
|
76
|
+
const tx = this.db.transaction(() => {
|
|
77
|
+
const row = this.read.get(id);
|
|
78
|
+
if (!row)
|
|
79
|
+
return { found: false, resumed: false, workflowName: '', currentNodeIndex: 0 };
|
|
80
|
+
const signals = this.decode(row);
|
|
81
|
+
signals[event] = payload;
|
|
82
|
+
const now = clock().now();
|
|
83
|
+
this.write.run(pack(signals), now, id);
|
|
84
|
+
const claimed = this.claimResume.run(now, id);
|
|
85
|
+
return {
|
|
86
|
+
found: true,
|
|
87
|
+
resumed: claimed.changes === 1,
|
|
88
|
+
workflowName: row.workflow_name,
|
|
89
|
+
currentNodeIndex: row.current_node_index,
|
|
90
|
+
};
|
|
91
|
+
});
|
|
92
|
+
return tx();
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Park a running execution at a `waitFor`, unless the awaited signal has already
|
|
96
|
+
* been recorded.
|
|
97
|
+
*
|
|
98
|
+
* Closes the window where a signal lands after `runWaitFor` read its in-memory
|
|
99
|
+
* snapshot but before the run reaches state `waiting`: without the re-check the
|
|
100
|
+
* payload is stored, nobody claims the resume, and the run hangs.
|
|
101
|
+
*/
|
|
102
|
+
park(id, event) {
|
|
103
|
+
const tx = this.db.transaction(() => {
|
|
104
|
+
const row = this.read.get(id);
|
|
105
|
+
const signals = this.decode(row);
|
|
106
|
+
if (!row)
|
|
107
|
+
return { signalPresent: false, parked: false, signals };
|
|
108
|
+
if (hasSignal(signals, event))
|
|
109
|
+
return { signalPresent: true, parked: false, signals };
|
|
110
|
+
const claimed = this.claimPark.run(clock().now(), id);
|
|
111
|
+
return { signalPresent: false, parked: claimed.changes === 1, signals };
|
|
112
|
+
});
|
|
113
|
+
return tx();
|
|
114
|
+
}
|
|
115
|
+
decode(row) {
|
|
116
|
+
return unpack(row?.signals) ?? {};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -12,6 +12,18 @@ export interface StepContext<TInput = unknown, TSteps extends Record<string, unk
|
|
|
12
12
|
readonly signals: Readonly<Record<string, unknown>>;
|
|
13
13
|
/** Current execution ID */
|
|
14
14
|
readonly executionId: string;
|
|
15
|
+
/**
|
|
16
|
+
* Idempotency key for THIS execution of the step. Stable across automatic retries
|
|
17
|
+
* and across crash-resume; different for a different run. Pass it straight to the
|
|
18
|
+
* provider so a repeat lands on the same operation instead of a new one.
|
|
19
|
+
*/
|
|
20
|
+
readonly idempotencyKey?: string;
|
|
21
|
+
/**
|
|
22
|
+
* Compensation only: the key the FORWARD step used. When the forward outcome is
|
|
23
|
+
* in doubt, this is what lets a rollback ask the provider "did this actually
|
|
24
|
+
* happen?" instead of depending on an output that may never have been persisted.
|
|
25
|
+
*/
|
|
26
|
+
readonly forwardIdempotencyKey?: string;
|
|
15
27
|
}
|
|
16
28
|
/** Step handler function (type-erased for internal storage) */
|
|
17
29
|
export type StepHandler<TInput = unknown, TResult = unknown> = (ctx: StepContext<TInput, any>) => Promise<TResult> | TResult;
|
|
@@ -25,11 +37,41 @@ export type TypedCompensateHandler<TInput, TSteps extends Record<string, unknown
|
|
|
25
37
|
export interface SchemaLike {
|
|
26
38
|
parse(data: unknown): unknown;
|
|
27
39
|
}
|
|
28
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Options for a single step.
|
|
42
|
+
*
|
|
43
|
+
* `TSteps` is part of the signature for source compatibility and for symmetry with
|
|
44
|
+
* `TypedStepHandler`, even though `compensate` no longer narrows on it (see below).
|
|
45
|
+
* Removing the parameter would break every explicit `StepOptions<In, Steps>` in
|
|
46
|
+
* user code.
|
|
47
|
+
*/
|
|
29
48
|
export interface StepOptions<TInput = unknown, TSteps extends Record<string, unknown> = Record<string, unknown>> {
|
|
30
49
|
retry?: number;
|
|
31
50
|
timeout?: number;
|
|
32
|
-
|
|
51
|
+
/**
|
|
52
|
+
* A METHOD taking a permissively-typed context, and every part of that is load
|
|
53
|
+
* bearing. Three forms were measured against real handler shapes:
|
|
54
|
+
*
|
|
55
|
+
* shape union method<TSteps> method<any>
|
|
56
|
+
* compensate: async (ctx) => ... TS7006 ok ok
|
|
57
|
+
* annotated with steps it reads ok ok ok
|
|
58
|
+
* annotated with a step that does not exist ok TS2322 ok
|
|
59
|
+
* CompensateHandler<TInput> alias ok ok ok
|
|
60
|
+
*
|
|
61
|
+
* Not a union (`TypedCompensateHandler | CompensateHandler`): TypeScript cannot
|
|
62
|
+
* contextually type a parameter against a union of signatures, so the inline arrow
|
|
63
|
+
* every documented example uses was an implicit `any` and failed `noImplicitAny`.
|
|
64
|
+
*
|
|
65
|
+
* A method rather than a property so parameters stay bivariant under
|
|
66
|
+
* `strictFunctionTypes`, which is what lets an explicitly annotated handler through.
|
|
67
|
+
*
|
|
68
|
+
* `any` rather than `TSteps` for the step map, deliberately: with `TSteps` an
|
|
69
|
+
* annotation naming a step this workflow does not declare is rejected, and the
|
|
70
|
+
* published union accepted it. Keeping the looser map costs typed access to
|
|
71
|
+
* `ctx.steps` inside a rollback, which handlers already narrow with a cast in
|
|
72
|
+
* practice, and buys source compatibility with every handler written before.
|
|
73
|
+
*/
|
|
74
|
+
compensate?(ctx: StepContext<TInput, TSteps extends never ? never : any>): Promise<void> | void;
|
|
33
75
|
/** Validate step input before execution */
|
|
34
76
|
inputSchema?: SchemaLike;
|
|
35
77
|
/** Validate step output after execution */
|
|
@@ -111,9 +153,35 @@ export type WorkflowNode = {
|
|
|
111
153
|
} | {
|
|
112
154
|
type: 'map';
|
|
113
155
|
def: MapDefinition;
|
|
156
|
+
} | {
|
|
157
|
+
type: 'pivot';
|
|
114
158
|
};
|
|
115
159
|
/** Execution state */
|
|
116
|
-
export type ExecutionState = 'running' | 'waiting' | 'completed' | 'failed' | 'compensating'
|
|
160
|
+
export type ExecutionState = 'running' | 'waiting' | 'completed' | 'failed' | 'compensating'
|
|
161
|
+
/**
|
|
162
|
+
* A compensation failed definitively and the unwind stopped. Deliberately NOT
|
|
163
|
+
* terminal: halting and calling it done leaves the operator nothing to act on,
|
|
164
|
+
* and ploughing on would undo work whose dependencies are still standing. The run
|
|
165
|
+
* parks here until someone retries the failed handler or abandons the unwind.
|
|
166
|
+
*/
|
|
167
|
+
| 'compensation-stuck';
|
|
168
|
+
/**
|
|
169
|
+
* What the engine did AFTER a failure. Deliberately a separate axis from the
|
|
170
|
+
* failure reason: the rollback is not the cause of the failure, and collapsing the
|
|
171
|
+
* two makes it impossible to alert on the right thing — "the payment failed" and
|
|
172
|
+
* "the refund never went through" need different pagers.
|
|
173
|
+
*
|
|
174
|
+
* Absent means no unwind was attempted, which is the case for a run that has not
|
|
175
|
+
* failed. There is deliberately no `'not-started'` member: it was documented in three
|
|
176
|
+
* places and never once assigned, so anything branching on it was dead code.
|
|
177
|
+
*/
|
|
178
|
+
export type RollbackStatus =
|
|
179
|
+
/** Every eligible step was compensated successfully. */
|
|
180
|
+
'completed'
|
|
181
|
+
/** The unwind ran past the pivot cutoff or had nothing left to do. */
|
|
182
|
+
| 'not-applicable'
|
|
183
|
+
/** A compensation failed definitively; the remaining ones were not attempted. */
|
|
184
|
+
| 'stuck';
|
|
117
185
|
/** Step execution state */
|
|
118
186
|
export type StepState = 'pending' | 'running' | 'completed' | 'failed';
|
|
119
187
|
/** Record of a step's execution */
|
|
@@ -124,10 +192,45 @@ export interface StepRecord {
|
|
|
124
192
|
startedAt?: number;
|
|
125
193
|
completedAt?: number;
|
|
126
194
|
attempts?: number;
|
|
195
|
+
/**
|
|
196
|
+
* Did this step declare a `compensate` handler when it ran?
|
|
197
|
+
*
|
|
198
|
+
* Persisted because a later deploy can remove or rename the step, and then nothing
|
|
199
|
+
* else can tell "this step never owed a reversal" from "this step owed one and the
|
|
200
|
+
* handler is gone". Without it a renamed step that had not been reversed yet was
|
|
201
|
+
* silently dropped from the unwind and the run reported a clean rollback.
|
|
202
|
+
*/
|
|
203
|
+
compensatable?: boolean;
|
|
127
204
|
/** forEach iteration item — persisted so compensation can restore __item */
|
|
128
205
|
loopItem?: unknown;
|
|
129
206
|
/** forEach iteration index — persisted so compensation can restore __index */
|
|
130
207
|
loopIndex?: number;
|
|
208
|
+
/**
|
|
209
|
+
* Outcome of this step's rollback. Written exactly once per unwind, for every
|
|
210
|
+
* eligible step — including the ones the unwind never reached, which are recorded
|
|
211
|
+
* as 'compensation-skipped' rather than left blank. "Never zero, never two" is
|
|
212
|
+
* only checkable if success is recorded as loudly as failure.
|
|
213
|
+
*/
|
|
214
|
+
compensation?: CompensationOutcome;
|
|
215
|
+
/** Idempotency key of the FORWARD execution, persisted before the body runs. */
|
|
216
|
+
idempotencyKey?: string;
|
|
217
|
+
/**
|
|
218
|
+
* For a `sub:<name>` record: the child execution this step started. Rolling back a
|
|
219
|
+
* nested workflow means running the CHILD's own unwind, so the parent has to keep
|
|
220
|
+
* a handle on it — otherwise a child that succeeded before the parent failed is
|
|
221
|
+
* left standing, with every resource it created orphaned.
|
|
222
|
+
*/
|
|
223
|
+
childExecutionId?: string;
|
|
224
|
+
/** Occurrence of this step name within the run — loops reuse a single name. */
|
|
225
|
+
occurrence?: number;
|
|
226
|
+
}
|
|
227
|
+
/** Terminal outcome of one step's rollback. */
|
|
228
|
+
export type CompensationStatus = 'compensated' | 'compensation-failed' | 'compensation-skipped';
|
|
229
|
+
export interface CompensationOutcome {
|
|
230
|
+
status: CompensationStatus;
|
|
231
|
+
at: number;
|
|
232
|
+
/** Why it failed, or why it was skipped. */
|
|
233
|
+
error?: string;
|
|
131
234
|
}
|
|
132
235
|
/** Full execution state */
|
|
133
236
|
export interface Execution {
|
|
@@ -139,12 +242,34 @@ export interface Execution {
|
|
|
139
242
|
currentNodeIndex: number;
|
|
140
243
|
/** Flattened step list for branch resolution */
|
|
141
244
|
resolvedSteps?: string[];
|
|
245
|
+
/** What happened to the rollback. Independent of `failureReason`. */
|
|
246
|
+
rollbackStatus?: RollbackStatus;
|
|
247
|
+
/** Why the run failed. Independent of `rollbackStatus`. */
|
|
248
|
+
failureReason?: string;
|
|
249
|
+
/**
|
|
250
|
+
* Node index at which `.pivot()` committed, if it was reached.
|
|
251
|
+
*
|
|
252
|
+
* Once set, the saga is committed and backward recovery is OFF ENTIRELY — not just
|
|
253
|
+
* for the steps after it. That is what "point of no return" means: releasing the
|
|
254
|
+
* subdomain of a tenant who has already been sent a welcome email is precisely the
|
|
255
|
+
* outcome the pivot exists to prevent. After it, the only correct recovery is
|
|
256
|
+
* forward.
|
|
257
|
+
*/
|
|
258
|
+
committedAt?: number;
|
|
142
259
|
signals: Record<string, unknown>;
|
|
260
|
+
/**
|
|
261
|
+
* Set when this run was started BY a `subWorkflow` node, naming the parent that owns
|
|
262
|
+
* it. A child is a row like any other, so without this nothing distinguishes it from
|
|
263
|
+
* a top-level run and `recover()` drives it on its own, re-running its steps and
|
|
264
|
+
* re-arming its rollback behind the parent's back
|
|
265
|
+
* (`test/repro-model-child-recovered-alone.test.ts`).
|
|
266
|
+
*/
|
|
267
|
+
parentExecutionId?: string;
|
|
143
268
|
createdAt: number;
|
|
144
269
|
updatedAt: number;
|
|
145
270
|
}
|
|
146
271
|
/** All workflow event types */
|
|
147
|
-
export type WorkflowEventType = 'step:started' | 'step:completed' | 'step:failed' | 'step:retry' | 'workflow:started' | 'workflow:completed' | 'workflow:failed' | 'workflow:compensating' | 'workflow:waiting' | 'signal:received' | 'signal:timeout';
|
|
272
|
+
export type WorkflowEventType = 'step:started' | 'step:completed' | 'step:failed' | 'step:retry' | 'workflow:started' | 'workflow:completed' | 'workflow:failed' | 'workflow:compensating' | 'workflow:waiting' | 'signal:received' | 'signal:timeout' | 'compensation:started' | 'compensation:completed' | 'compensation:failed' | 'compensation:skipped';
|
|
148
273
|
/** Base event payload */
|
|
149
274
|
export interface WorkflowEvent {
|
|
150
275
|
type: WorkflowEventType;
|
|
@@ -206,6 +331,24 @@ export interface RecoverResult {
|
|
|
206
331
|
/** Total recovered */
|
|
207
332
|
total: number;
|
|
208
333
|
}
|
|
334
|
+
/** Result of WorkflowStore.recordSignal() */
|
|
335
|
+
export interface SignalOutcome {
|
|
336
|
+
/** Whether the execution row exists */
|
|
337
|
+
found: boolean;
|
|
338
|
+
/** True for the single caller that claimed the resume of a parked run */
|
|
339
|
+
resumed: boolean;
|
|
340
|
+
workflowName: string;
|
|
341
|
+
currentNodeIndex: number;
|
|
342
|
+
}
|
|
343
|
+
/** Result of WorkflowStore.parkForSignal() */
|
|
344
|
+
export interface ParkOutcome {
|
|
345
|
+
/** The awaited signal was already recorded — advance instead of parking */
|
|
346
|
+
signalPresent: boolean;
|
|
347
|
+
/** This caller transitioned the run to 'waiting' */
|
|
348
|
+
parked: boolean;
|
|
349
|
+
/** Signals as persisted, for refreshing a stale in-memory snapshot */
|
|
350
|
+
signals: Record<string, unknown>;
|
|
351
|
+
}
|
|
209
352
|
/** Options for cleanup */
|
|
210
353
|
export interface CleanupOptions {
|
|
211
354
|
maxAge: number;
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the unwind should DO with each eligible record, decided as a pure function.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS FILE EXISTS. Every rollback defect this engine shipped lived in the four
|
|
5
|
+
* lines that decide whether a record is skipped, halted on, or reversed: a loop
|
|
6
|
+
* mirror compensated twice, a step called `charge:extra` resolved to the loop `charge`,
|
|
7
|
+
* a renamed step dropped so the unwind reported a clean rollback over money that was
|
|
8
|
+
* never refunded. None of those are I/O bugs. They are decisions, and while they lived
|
|
9
|
+
* inside a 90-line async function that also wrote to SQLite and emitted events, the
|
|
10
|
+
* only way to test them was to stand up an Engine, a database and a real failure, and
|
|
11
|
+
* then infer the decision from its side effects.
|
|
12
|
+
*
|
|
13
|
+
* Here the decision is a value. It takes data and returns what to do, so it can be
|
|
14
|
+
* exercised with generated inputs in microseconds and read without tracing an await.
|
|
15
|
+
* The impure loop in `compensator.ts` becomes what it should be: a dispatcher that
|
|
16
|
+
* performs the actions this file names.
|
|
17
|
+
*
|
|
18
|
+
* The ORDER of the checks is load-bearing and each one earned its place by breaking:
|
|
19
|
+
*
|
|
20
|
+
* 1. vanished before settled — a record settled by an earlier attempt whose step
|
|
21
|
+
* no longer resolves must halt, and the settled check would have skipped it.
|
|
22
|
+
* 2. settled before halted — "never twice" outranks "stop after a failure": a
|
|
23
|
+
* record already carrying an outcome is never re-run, halted or not.
|
|
24
|
+
* 3. halted before owes — once something has failed definitively the rest
|
|
25
|
+
* are left WITHOUT an outcome on purpose, so a resume can still reach them.
|
|
26
|
+
*/
|
|
27
|
+
import type { Execution, StepRecord } from './types';
|
|
28
|
+
import type { Workflow } from './workflow';
|
|
29
|
+
/** What to do with one eligible record. */
|
|
30
|
+
export type UnwindAction =
|
|
31
|
+
/** Already has an outcome, or owes none. Move on, nothing to record. */
|
|
32
|
+
{
|
|
33
|
+
kind: 'skip';
|
|
34
|
+
reason: 'already-settled' | 'owes-no-outcome';
|
|
35
|
+
}
|
|
36
|
+
/** A previous failure stopped the chain. Leave the rest untouched. */
|
|
37
|
+
| {
|
|
38
|
+
kind: 'stop';
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* This record already FAILED and is not being retried. The chain is still blocked
|
|
42
|
+
* here, so the pass must stop rather than walk past it.
|
|
43
|
+
*/
|
|
44
|
+
| {
|
|
45
|
+
kind: 'halt-failed';
|
|
46
|
+
}
|
|
47
|
+
/** The step is gone from the workflow. Record why and stop the chain. */
|
|
48
|
+
| {
|
|
49
|
+
kind: 'halt-vanished';
|
|
50
|
+
error: string;
|
|
51
|
+
}
|
|
52
|
+
/** Roll back a nested workflow by running the child's own unwind. */
|
|
53
|
+
| {
|
|
54
|
+
kind: 'unwind-child';
|
|
55
|
+
}
|
|
56
|
+
/** Run this step's compensate handler, bounded by `timeoutMs`. */
|
|
57
|
+
| {
|
|
58
|
+
kind: 'compensate';
|
|
59
|
+
timeoutMs: number;
|
|
60
|
+
};
|
|
61
|
+
/** Bound applied to a reversal whose step declined to set one (`timeout: 0`). */
|
|
62
|
+
export declare const DEFAULT_COMPENSATE_TIMEOUT_MS = 30000;
|
|
63
|
+
/**
|
|
64
|
+
* Decide what to do with `record`, named `name`, in workflow `wf`.
|
|
65
|
+
*
|
|
66
|
+
* `halted` is whether an earlier record in this same pass already failed definitively.
|
|
67
|
+
*/
|
|
68
|
+
export declare function decideUnwindAction(wf: Workflow, name: string, record: StepRecord, halted: boolean,
|
|
69
|
+
/**
|
|
70
|
+
* An operator resume. A record that FAILED is retried instead of skipped.
|
|
71
|
+
*
|
|
72
|
+
* This replaced wiping the failed outcome before the pass. The wipe worked and cost
|
|
73
|
+
* more than it bought: the operator's `compensation-failed` record was destroyed on
|
|
74
|
+
* the way in, so a resume that then hit a failing store left a run with the
|
|
75
|
+
* diagnostic gone from disk and nothing saying which reversal had failed. Retrying
|
|
76
|
+
* from a flag leaves the record in place until a real outcome replaces it.
|
|
77
|
+
*/
|
|
78
|
+
retryFailed?: boolean): UnwindAction;
|
|
79
|
+
/**
|
|
80
|
+
* Does this step still owe an outcome when the run terminates?
|
|
81
|
+
*
|
|
82
|
+
* Used by `abandonCompensation` to discharge "exactly one outcome per eligible step,
|
|
83
|
+
* never zero" for everything the unwind never reached.
|
|
84
|
+
*/
|
|
85
|
+
export declare function owesOutcome(wf: Workflow, name: string, record?: StepRecord): boolean;
|
|
86
|
+
/** Is the pivot committed, making the whole run ineligible for rollback? */
|
|
87
|
+
export declare function isCommitted(exec: Execution): boolean;
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the unwind should DO with each eligible record, decided as a pure function.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS FILE EXISTS. Every rollback defect this engine shipped lived in the four
|
|
5
|
+
* lines that decide whether a record is skipped, halted on, or reversed: a loop
|
|
6
|
+
* mirror compensated twice, a step called `charge:extra` resolved to the loop `charge`,
|
|
7
|
+
* a renamed step dropped so the unwind reported a clean rollback over money that was
|
|
8
|
+
* never refunded. None of those are I/O bugs. They are decisions, and while they lived
|
|
9
|
+
* inside a 90-line async function that also wrote to SQLite and emitted events, the
|
|
10
|
+
* only way to test them was to stand up an Engine, a database and a real failure, and
|
|
11
|
+
* then infer the decision from its side effects.
|
|
12
|
+
*
|
|
13
|
+
* Here the decision is a value. It takes data and returns what to do, so it can be
|
|
14
|
+
* exercised with generated inputs in microseconds and read without tracing an await.
|
|
15
|
+
* The impure loop in `compensator.ts` becomes what it should be: a dispatcher that
|
|
16
|
+
* performs the actions this file names.
|
|
17
|
+
*
|
|
18
|
+
* The ORDER of the checks is load-bearing and each one earned its place by breaking:
|
|
19
|
+
*
|
|
20
|
+
* 1. vanished before settled — a record settled by an earlier attempt whose step
|
|
21
|
+
* no longer resolves must halt, and the settled check would have skipped it.
|
|
22
|
+
* 2. settled before halted — "never twice" outranks "stop after a failure": a
|
|
23
|
+
* record already carrying an outcome is never re-run, halted or not.
|
|
24
|
+
* 3. halted before owes — once something has failed definitively the rest
|
|
25
|
+
* are left WITHOUT an outcome on purpose, so a resume can still reach them.
|
|
26
|
+
*/
|
|
27
|
+
import { findStepDef } from './runner';
|
|
28
|
+
/** Bound applied to a reversal whose step declined to set one (`timeout: 0`). */
|
|
29
|
+
export const DEFAULT_COMPENSATE_TIMEOUT_MS = 30_000;
|
|
30
|
+
/**
|
|
31
|
+
* Decide what to do with `record`, named `name`, in workflow `wf`.
|
|
32
|
+
*
|
|
33
|
+
* `halted` is whether an earlier record in this same pass already failed definitively.
|
|
34
|
+
*/
|
|
35
|
+
export function decideUnwindAction(wf, name, record, halted,
|
|
36
|
+
/**
|
|
37
|
+
* An operator resume. A record that FAILED is retried instead of skipped.
|
|
38
|
+
*
|
|
39
|
+
* This replaced wiping the failed outcome before the pass. The wipe worked and cost
|
|
40
|
+
* more than it bought: the operator's `compensation-failed` record was destroyed on
|
|
41
|
+
* the way in, so a resume that then hit a failing store left a run with the
|
|
42
|
+
* diagnostic gone from disk and nothing saying which reversal had failed. Retrying
|
|
43
|
+
* from a flag leaves the record in place until a real outcome replaces it.
|
|
44
|
+
*/
|
|
45
|
+
retryFailed = false) {
|
|
46
|
+
const isChild = name.startsWith('sub:');
|
|
47
|
+
const def = isChild ? null : findStepDef(wf, name);
|
|
48
|
+
// 1. The step is gone from the workflow, and this record is owed a reversal that can
|
|
49
|
+
// no longer be produced. Checked FIRST, because the settled check below would skip
|
|
50
|
+
// it and the pass would reach its end and call itself clean over a step nobody
|
|
51
|
+
// reversed.
|
|
52
|
+
//
|
|
53
|
+
// Exactly two shapes are owed one. A reversal that already FAILED is still
|
|
54
|
+
// outstanding. And a record with no outcome that ran WITH a `compensate` handler
|
|
55
|
+
// was going to be reversed and now cannot be, which is the likelier rename case:
|
|
56
|
+
// the unwind had simply not reached it yet.
|
|
57
|
+
//
|
|
58
|
+
// A record settled `compensated` or `compensation-skipped` is emphatically not
|
|
59
|
+
// owed anything, and an earlier version of this check caught it too. The
|
|
60
|
+
// dispatcher then wrote `compensation-failed` OVER a reversal that had provably
|
|
61
|
+
// run and succeeded, emitted a `compensation:failed` for it, and halted there for
|
|
62
|
+
// good, so the record that actually failed was never retried. An operator acting
|
|
63
|
+
// on that would release the same stock twice, which is precisely the "never twice"
|
|
64
|
+
// this module exists to guarantee.
|
|
65
|
+
if (!isChild && !def) {
|
|
66
|
+
const failedBefore = record.compensation?.status === 'compensation-failed';
|
|
67
|
+
const owedAndUnreached = record.compensation === undefined && record.compensatable === true;
|
|
68
|
+
if (failedBefore || owedAndUnreached) {
|
|
69
|
+
return {
|
|
70
|
+
kind: 'halt-vanished',
|
|
71
|
+
error: `step "${name}" is no longer declared by workflow "${wf.name}", so its rollback cannot run`,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// 2. Never twice. An unwind interrupted by a crash resumes where it stopped. The one
|
|
76
|
+
// exception is an operator explicitly retrying a reversal that FAILED: a success
|
|
77
|
+
// or a skip is never re-run, whatever was asked.
|
|
78
|
+
if (record.compensation) {
|
|
79
|
+
if (record.compensation.status === 'compensation-failed') {
|
|
80
|
+
// An UNRESOLVED failure. Skipping it is how a run that had a refused refund came
|
|
81
|
+
// back from `recover()` after a crash mid-unwind, walked past the reversal that
|
|
82
|
+
// had failed, reached the end and wrote `rollbackStatus: 'completed'` over money
|
|
83
|
+
// nobody had returned. The chain stopped here before and it still stops here,
|
|
84
|
+
// unless an operator explicitly asked for the retry.
|
|
85
|
+
if (!retryFailed)
|
|
86
|
+
return { kind: 'halt-failed' };
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
// `compensated` or `compensation-skipped`: settled for good, never re-run.
|
|
90
|
+
return { kind: 'skip', reason: 'already-settled' };
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// 3. Stop at the first definitive failure. The rest are deliberately left WITHOUT an
|
|
94
|
+
// outcome: the run is parked, not finished, and pre-marking them skipped would
|
|
95
|
+
// make a later resume believe they were already dealt with.
|
|
96
|
+
if (halted)
|
|
97
|
+
return { kind: 'stop' };
|
|
98
|
+
// 4. A nested workflow is reversed by running the child's own unwind, so the parent
|
|
99
|
+
// needs no handler of its own.
|
|
100
|
+
if (isChild)
|
|
101
|
+
return { kind: 'unwind-child' };
|
|
102
|
+
// 5. No handler registered: not part of the unwind set, so no outcome is owed.
|
|
103
|
+
if (!def?.compensate)
|
|
104
|
+
return { kind: 'skip', reason: 'owes-no-outcome' };
|
|
105
|
+
// `timeout: 0` means "no bound" on the forward path, and that is the caller's choice
|
|
106
|
+
// to make. A reversal is not the same case: it holds the engine's in-flight claim,
|
|
107
|
+
// so one that never settles locks the run out of every operator exit for the life of
|
|
108
|
+
// the process. When the step declines to say what bounds it, the default applies.
|
|
109
|
+
return {
|
|
110
|
+
kind: 'compensate',
|
|
111
|
+
timeoutMs: def.timeout > 0 ? def.timeout : DEFAULT_COMPENSATE_TIMEOUT_MS,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Does this step still owe an outcome when the run terminates?
|
|
116
|
+
*
|
|
117
|
+
* Used by `abandonCompensation` to discharge "exactly one outcome per eligible step,
|
|
118
|
+
* never zero" for everything the unwind never reached.
|
|
119
|
+
*/
|
|
120
|
+
export function owesOutcome(wf, name, record) {
|
|
121
|
+
if (name.startsWith('sub:'))
|
|
122
|
+
return true;
|
|
123
|
+
const def = findStepDef(wf, name);
|
|
124
|
+
// A step that is still declared owes an outcome only if it declared a handler.
|
|
125
|
+
// `unwindSet` admits EVERY step with a definition, so this is the gate that keeps
|
|
126
|
+
// `abandonCompensation` from inventing outcomes for steps that were never part of the
|
|
127
|
+
// unwind.
|
|
128
|
+
if (def)
|
|
129
|
+
return def.compensate !== undefined;
|
|
130
|
+
// The definition is gone, and here the two gates used to disagree. `unwindSet` keeps
|
|
131
|
+
// such a record when it already carries an outcome or ran WITH a handler; this one
|
|
132
|
+
// answered from the definition alone and so returned false for exactly the record
|
|
133
|
+
// `unwindSet` had gone out of its way to admit. `abandonCompensation` walked past it,
|
|
134
|
+
// and a renamed step ended a TERMINAL run with no outcome at all, in the function that
|
|
135
|
+
// exists to discharge "exactly one outcome per eligible step, never zero"
|
|
136
|
+
// (`test/repro-workflow-abandon-vanished.test.ts`).
|
|
137
|
+
return record?.compensation !== undefined || record?.compensatable === true;
|
|
138
|
+
}
|
|
139
|
+
/** Is the pivot committed, making the whole run ineligible for rollback? */
|
|
140
|
+
export function isCommitted(exec) {
|
|
141
|
+
return exec.committedAt !== undefined;
|
|
142
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* waitFor node execution — parking a run for a human/external signal, and the
|
|
3
|
+
* timeout timers that bound the wait.
|
|
4
|
+
*
|
|
5
|
+
* Split out of the executor because parking is the one node type with a genuinely
|
|
6
|
+
* concurrent counterpart: `signal()` mutates the same row from outside the worker,
|
|
7
|
+
* so every transition here has to be expressed as a claim against the store rather
|
|
8
|
+
* than an in-memory state change. See storeSignals.ts for the ownership rules.
|
|
9
|
+
*/
|
|
10
|
+
import type { Queue } from '../queue/queue';
|
|
11
|
+
import type { Workflow } from './workflow';
|
|
12
|
+
import type { WorkflowStore } from './store';
|
|
13
|
+
import type { WorkflowEmitter } from './emitter';
|
|
14
|
+
import type { Execution, WorkflowNode } from './types';
|
|
15
|
+
import { type TimerHandle } from './clock';
|
|
16
|
+
/** Largest delay setTimeout accepts before wrapping (2**31-1 ms, ~24.8 days) */
|
|
17
|
+
export declare const MAX_TIMER_MS = 2147483647;
|
|
18
|
+
export interface TimerDeps {
|
|
19
|
+
queue: Queue;
|
|
20
|
+
timers: Map<string, TimerHandle>;
|
|
21
|
+
}
|
|
22
|
+
export interface WaitForDeps {
|
|
23
|
+
store: WorkflowStore;
|
|
24
|
+
emitter: WorkflowEmitter | null;
|
|
25
|
+
advance: (exec: Execution, nextIdx: number, wf: Workflow) => Promise<void>;
|
|
26
|
+
compensate: (exec: Execution, wf: Workflow) => Promise<void>;
|
|
27
|
+
scheduleTimeoutCheck: (execId: string, workflowName: string, nodeIdx: number, ms: number) => void;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Arm the timer that re-enters a parked node once its wait budget elapses.
|
|
31
|
+
*
|
|
32
|
+
* setTimeout takes a 32-bit signed delay; anything larger wraps to 1ms and fires
|
|
33
|
+
* immediately (`TimeoutOverflowWarning`). Clamping and letting the re-check job
|
|
34
|
+
* re-arm for whatever remains is what makes multi-week approval windows survive
|
|
35
|
+
* (test/repro-workflow-timeout-overflow.test.ts).
|
|
36
|
+
*/
|
|
37
|
+
export declare function scheduleTimeoutCheck(deps: TimerDeps, execId: string, workflowName: string, nodeIdx: number, ms: number): void;
|
|
38
|
+
/**
|
|
39
|
+
* Cancel every armed wait timer. Called on engine shutdown: `unref` alone lets a
|
|
40
|
+
* process exit, but a still-armed timer can also fire into a queue that is closing,
|
|
41
|
+
* and a caller that shuts one engine down while keeping the process alive has no
|
|
42
|
+
* other way to release them.
|
|
43
|
+
*/
|
|
44
|
+
export declare function clearTimers(timers: Map<string, TimerHandle>): void;
|
|
45
|
+
/**
|
|
46
|
+
* Execute a `waitFor` node: advance if the signal is already there, fail if the wait
|
|
47
|
+
* has expired, otherwise park the run and throw the sentinel so processStep
|
|
48
|
+
* short-circuits without treating the pause as an error.
|
|
49
|
+
*/
|
|
50
|
+
export declare function runWaitFor(deps: WaitForDeps, exec: Execution, node: Extract<WorkflowNode, {
|
|
51
|
+
type: 'waitFor';
|
|
52
|
+
}>, idx: number, wf: Workflow): Promise<void>;
|