bunqueue 2.8.46 → 2.8.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/workflow/admission.d.ts +45 -0
- package/dist/client/workflow/admission.js +50 -0
- package/dist/client/workflow/clock.d.ts +58 -0
- package/dist/client/workflow/clock.js +104 -0
- package/dist/client/workflow/compensator.d.ts +56 -3
- package/dist/client/workflow/compensator.js +406 -27
- package/dist/client/workflow/emitter.d.ts +1 -1
- package/dist/client/workflow/emitter.js +4 -3
- package/dist/client/workflow/engine.d.ts +17 -0
- package/dist/client/workflow/engine.js +25 -0
- package/dist/client/workflow/executor.d.ts +40 -5
- package/dist/client/workflow/executor.js +227 -83
- package/dist/client/workflow/identity.d.ts +46 -0
- package/dist/client/workflow/identity.js +93 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.js +147 -14
- package/dist/client/workflow/recovery.d.ts +10 -1
- package/dist/client/workflow/recovery.js +36 -9
- package/dist/client/workflow/rollbackControl.d.ts +36 -0
- package/dist/client/workflow/rollbackControl.js +51 -0
- package/dist/client/workflow/runner.d.ts +22 -2
- package/dist/client/workflow/runner.js +125 -27
- package/dist/client/workflow/store.d.ts +64 -4
- package/dist/client/workflow/store.js +123 -30
- package/dist/client/workflow/storeCodec.d.ts +7 -0
- package/dist/client/workflow/storeCodec.js +16 -0
- package/dist/client/workflow/storeSignals.d.ts +61 -0
- package/dist/client/workflow/storeSignals.js +118 -0
- package/dist/client/workflow/types.d.ts +147 -4
- package/dist/client/workflow/unwindPlan.d.ts +87 -0
- package/dist/client/workflow/unwindPlan.js +142 -0
- package/dist/client/workflow/waitFor.d.ts +52 -0
- package/dist/client/workflow/waitFor.js +137 -0
- package/dist/client/workflow/workflow.d.ts +71 -1
- package/dist/client/workflow/workflow.js +184 -14
- package/package.json +16 -6
|
@@ -1,13 +1,28 @@
|
|
|
1
|
+
import { assertNoIndexCollision, assertNoDuplicateWaitFor, unusableEventName } from './workflow';
|
|
1
2
|
import { executeStepWithRetry, executeParallelSteps, executeSubWorkflow, buildContext, } from './runner';
|
|
2
3
|
import { executeDoUntil, executeDoWhile, executeForEach, executeMap } from './loops';
|
|
3
4
|
import { WaitForSignalError, runCompensation } from './compensator';
|
|
5
|
+
import { abandonParkedCompensation, resumeCompensation, } from './rollbackControl';
|
|
4
6
|
import { recoverExecutions } from './recovery';
|
|
7
|
+
import { clearTimers, runWaitFor, scheduleTimeoutCheck } from './waitFor';
|
|
8
|
+
import { clock } from './clock';
|
|
9
|
+
import { describeError } from './identity';
|
|
10
|
+
import { claimKey, decideAdmission } from './admission';
|
|
5
11
|
export class WorkflowExecutor {
|
|
6
12
|
store;
|
|
7
13
|
queue;
|
|
8
14
|
emitter;
|
|
9
15
|
workflows = new Map();
|
|
10
16
|
timeoutTimers = new Map();
|
|
17
|
+
/**
|
|
18
|
+
* Release every armed waitFor timer. The engine owns the executor's lifetime, so
|
|
19
|
+
* `Engine.close()` must call this: an armed timer would otherwise fire into a
|
|
20
|
+
* closing queue, and a caller that keeps the process alive after closing one engine
|
|
21
|
+
* has no other handle on them.
|
|
22
|
+
*/
|
|
23
|
+
close() {
|
|
24
|
+
clearTimers(this.timeoutTimers);
|
|
25
|
+
}
|
|
11
26
|
updateFn;
|
|
12
27
|
constructor(store, queue, emitter = null) {
|
|
13
28
|
this.store = store;
|
|
@@ -23,16 +38,18 @@ export class WorkflowExecutor {
|
|
|
23
38
|
if (dupes.length > 0) {
|
|
24
39
|
throw new Error(`Duplicate step names in "${workflow.name}": ${dupes.join(', ')}`);
|
|
25
40
|
}
|
|
41
|
+
assertNoIndexCollision(workflow);
|
|
42
|
+
assertNoDuplicateWaitFor(workflow);
|
|
26
43
|
this.workflows.set(workflow.name, workflow);
|
|
27
44
|
}
|
|
28
|
-
async start(workflowName, input) {
|
|
45
|
+
async start(workflowName, input, parentExecutionId) {
|
|
29
46
|
const wf = this.workflows.get(workflowName);
|
|
30
47
|
if (!wf)
|
|
31
48
|
throw new Error(`Workflow "${workflowName}" not registered`);
|
|
32
49
|
if (wf.nodes.length === 0)
|
|
33
50
|
throw new Error(`Workflow "${workflowName}" has no steps`);
|
|
34
|
-
const now =
|
|
35
|
-
const id = `wf_${now}_${
|
|
51
|
+
const now = clock().now();
|
|
52
|
+
const id = `wf_${now}_${clock().random().toString(36).slice(2, 10)}`;
|
|
36
53
|
const exec = {
|
|
37
54
|
id,
|
|
38
55
|
workflowName,
|
|
@@ -41,6 +58,7 @@ export class WorkflowExecutor {
|
|
|
41
58
|
steps: {},
|
|
42
59
|
currentNodeIndex: 0,
|
|
43
60
|
signals: {},
|
|
61
|
+
...(parentExecutionId ? { parentExecutionId } : {}),
|
|
44
62
|
createdAt: now,
|
|
45
63
|
updatedAt: now,
|
|
46
64
|
};
|
|
@@ -49,10 +67,43 @@ export class WorkflowExecutor {
|
|
|
49
67
|
await this.enqueue(exec);
|
|
50
68
|
return { id, workflowName };
|
|
51
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* Nodes this process is currently executing, keyed `<execution>:<nodeIndex>`.
|
|
72
|
+
*
|
|
73
|
+
* The cursor guard below rejects a job for a node the run has already left, but not
|
|
74
|
+
* a SECOND job for the node it is on right now: both carry the same index. That is
|
|
75
|
+
* the reachable duplicate, because `recover()` re-enqueues the current node of every
|
|
76
|
+
* `running` execution and is documented as callable on a live engine. Without this
|
|
77
|
+
* claim the node runs twice and each copy advances the run independently, doubling
|
|
78
|
+
* every side effect after it while the run still ends `completed`.
|
|
79
|
+
*
|
|
80
|
+
* A claim, not a queue-level dedup: a deterministic `jobId` was tried and could
|
|
81
|
+
* swallow a LEGITIMATE later re-enqueue of the same node, wedging the run forever.
|
|
82
|
+
* This drops only a duplicate that overlaps in time.
|
|
83
|
+
*/
|
|
84
|
+
nodesInFlight = new Set();
|
|
52
85
|
async processStep(data) {
|
|
86
|
+
// The admission DECISION lives in `admission.ts` as a pure function; this method
|
|
87
|
+
// only carries it out. Delivery is at-least-once, so the same node job arrives
|
|
88
|
+
// twice routinely, and when one of the three guards was missing a duplicate re-ran
|
|
89
|
+
// the node and every node after it: two advance chains on one execution, doubled
|
|
90
|
+
// side effects, and a final `completed` that hid it. That took a long model
|
|
91
|
+
// campaign to find because the reasoning was buried in a method that also read
|
|
92
|
+
// SQLite and dispatched work.
|
|
53
93
|
const exec = this.store.get(data.executionId);
|
|
54
|
-
|
|
94
|
+
const admission = decideAdmission(exec, data.nodeIndex, this.nodesInFlight);
|
|
95
|
+
if (admission.kind === 'reject' || !exec)
|
|
55
96
|
return null;
|
|
97
|
+
const claim = claimKey(data.executionId, data.nodeIndex);
|
|
98
|
+
this.nodesInFlight.add(claim);
|
|
99
|
+
try {
|
|
100
|
+
return await this.runNode(data, exec);
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
this.nodesInFlight.delete(claim);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async runNode(data, exec) {
|
|
56
107
|
// If waiting, set back to running for timeout re-check
|
|
57
108
|
if (exec.state === 'waiting')
|
|
58
109
|
exec.state = 'running';
|
|
@@ -73,6 +124,15 @@ export class WorkflowExecutor {
|
|
|
73
124
|
if (err instanceof WaitForSignalError)
|
|
74
125
|
return null;
|
|
75
126
|
exec.state = 'failed';
|
|
127
|
+
// Why the run failed — kept distinct from what the rollback then did.
|
|
128
|
+
exec.failureReason = describeError(err);
|
|
129
|
+
// Deliberately UNGUARDED, unlike the writes on the throwing paths in `runner.ts` and
|
|
130
|
+
// `runSubWorkflow`. A throw here skips `compensate()` below, which sounds worse and
|
|
131
|
+
// is not: disk still says `running`, `listRecoverable()` covers `running`, so the
|
|
132
|
+
// next `recover()` re-drives this node and the unwind happens then. Swallowing it
|
|
133
|
+
// would instead leave a run that looks failed and was never rolled back, with
|
|
134
|
+
// nothing scheduled to notice. Guard this only alongside a durable signal that the
|
|
135
|
+
// rollback is still owed.
|
|
76
136
|
this.store.update(exec);
|
|
77
137
|
this.emitter?.emitWorkflow('workflow:failed', exec.id, exec.workflowName, 'failed');
|
|
78
138
|
await this.compensate(exec, wf);
|
|
@@ -81,32 +141,63 @@ export class WorkflowExecutor {
|
|
|
81
141
|
return null;
|
|
82
142
|
}
|
|
83
143
|
async signal(executionId, event, payload) {
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
144
|
+
// Same predicate as registration. A name that cannot be stored has to fail here
|
|
145
|
+
// too: a caller who signals it would otherwise get a clean return for a delivery
|
|
146
|
+
// that went nowhere.
|
|
147
|
+
const bad = unusableEventName(event);
|
|
148
|
+
if (bad)
|
|
149
|
+
throw new Error(`Cannot signal an event ${bad}`);
|
|
150
|
+
// A finished run cannot receive anything. Accepting it wrote the payload into the
|
|
151
|
+
// persisted row and emitted `signal:received`, so a dashboard reported an approval
|
|
152
|
+
// against a run that had already ended and a closed audit record was mutated after
|
|
153
|
+
// the fact, while the caller got a clean return for a delivery that did nothing.
|
|
154
|
+
// A signal racing a run to its end is real, and rejecting is how the caller finds
|
|
155
|
+
// out (`test/repro-workflow-operator-signal.test.ts`).
|
|
156
|
+
const current = this.store.get(executionId);
|
|
157
|
+
if (current && current.state !== 'running' && current.state !== 'waiting') {
|
|
158
|
+
throw new Error(`Execution "${executionId}" is "${current.state}" and cannot receive the signal "${event}"`);
|
|
159
|
+
}
|
|
87
160
|
const timer = this.timeoutTimers.get(executionId);
|
|
88
161
|
if (timer) {
|
|
89
|
-
clearTimeout(timer);
|
|
162
|
+
clock().clearTimeout(timer);
|
|
90
163
|
this.timeoutTimers.delete(executionId);
|
|
91
164
|
}
|
|
92
|
-
//
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
//
|
|
102
|
-
|
|
103
|
-
if (
|
|
104
|
-
|
|
165
|
+
// Record the payload and claim the resume in a single transaction that touches
|
|
166
|
+
// only the `signals` and `state` columns. Writing through the store — rather than
|
|
167
|
+
// mutating a snapshot and calling update() — is what keeps a concurrently
|
|
168
|
+
// executing step from overwriting the payload with its own stale `signals`
|
|
169
|
+
// (test/repro-workflow-signal-lost-update.test.ts).
|
|
170
|
+
//
|
|
171
|
+
// The claim is a conditional `state = 'waiting'` UPDATE, so duplicate or
|
|
172
|
+
// concurrent signals collapse to exactly one resume and every step after the
|
|
173
|
+
// waitFor runs exactly once. A signal that lands before the run parks records
|
|
174
|
+
// its payload and returns; runWaitFor then consumes it via parkForSignal().
|
|
175
|
+
const outcome = this.store.recordSignal(executionId, event, payload);
|
|
176
|
+
if (!outcome.found)
|
|
177
|
+
throw new Error(`Execution "${executionId}" not found`);
|
|
178
|
+
this.emitter?.emitSignal('signal:received', executionId, outcome.workflowName, event, payload);
|
|
179
|
+
if (!outcome.resumed)
|
|
105
180
|
return;
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
181
|
+
await this.queue.add('wf:step', {
|
|
182
|
+
executionId,
|
|
183
|
+
workflowName: outcome.workflowName,
|
|
184
|
+
nodeIndex: outcome.currentNodeIndex,
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
/** Retry the compensation that parked the run, then finish the unwind. */
|
|
188
|
+
async resumeCompensation(executionId) {
|
|
189
|
+
await resumeCompensation(this.rollbackDeps, executionId);
|
|
190
|
+
}
|
|
191
|
+
/** Give up on a parked unwind, recording the outstanding steps as skipped. */
|
|
192
|
+
abandonCompensation(executionId) {
|
|
193
|
+
abandonParkedCompensation(this.rollbackDeps, executionId);
|
|
194
|
+
}
|
|
195
|
+
get rollbackDeps() {
|
|
196
|
+
return {
|
|
197
|
+
store: this.store,
|
|
198
|
+
emitter: this.emitter,
|
|
199
|
+
workflows: this.workflows,
|
|
200
|
+
};
|
|
110
201
|
}
|
|
111
202
|
getExecution(id) {
|
|
112
203
|
return this.store.get(id);
|
|
@@ -124,19 +215,21 @@ export class WorkflowExecutor {
|
|
|
124
215
|
else if (node.type === 'subWorkflow')
|
|
125
216
|
await this.runSubWorkflow(exec, node, idx, wf);
|
|
126
217
|
else if (node.type === 'doUntil')
|
|
127
|
-
await this.
|
|
218
|
+
await this.runBody(exec, idx, wf, executeDoUntil, node.def);
|
|
128
219
|
else if (node.type === 'doWhile')
|
|
129
|
-
await this.
|
|
220
|
+
await this.runBody(exec, idx, wf, executeDoWhile, node.def);
|
|
130
221
|
else if (node.type === 'forEach')
|
|
131
|
-
await this.
|
|
222
|
+
await this.runBody(exec, idx, wf, executeForEach, node.def);
|
|
132
223
|
else if (node.type === 'map')
|
|
133
|
-
await this.
|
|
224
|
+
await this.runBody(exec, idx, wf, executeMap, node.def);
|
|
225
|
+
else if (node.type === 'pivot')
|
|
226
|
+
await this.runPivot(exec, idx, wf);
|
|
134
227
|
else
|
|
135
|
-
await this.
|
|
228
|
+
await runWaitFor(this.waitForDeps, exec, node, idx, wf);
|
|
136
229
|
}
|
|
137
230
|
async runStep(exec, def, idx, wf) {
|
|
138
231
|
const ctx = buildContext(exec);
|
|
139
|
-
await executeStepWithRetry(def, ctx, exec, this.emitter, this.updateFn);
|
|
232
|
+
await executeStepWithRetry(def, ctx, exec, { emitter: this.emitter, updateFn: this.updateFn });
|
|
140
233
|
await this.advance(exec, idx + 1, wf);
|
|
141
234
|
}
|
|
142
235
|
async runBranch(exec, node, idx, wf) {
|
|
@@ -144,7 +237,10 @@ export class WorkflowExecutor {
|
|
|
144
237
|
const pathSteps = node.def.paths.get(pathName);
|
|
145
238
|
if (pathSteps && pathSteps.length > 0) {
|
|
146
239
|
for (const step of pathSteps) {
|
|
147
|
-
await executeStepWithRetry(step, buildContext(exec), exec,
|
|
240
|
+
await executeStepWithRetry(step, buildContext(exec), exec, {
|
|
241
|
+
emitter: this.emitter,
|
|
242
|
+
updateFn: this.updateFn,
|
|
243
|
+
});
|
|
148
244
|
}
|
|
149
245
|
}
|
|
150
246
|
await this.advance(exec, idx + 1, wf);
|
|
@@ -155,57 +251,90 @@ export class WorkflowExecutor {
|
|
|
155
251
|
}
|
|
156
252
|
async runSubWorkflow(exec, node, idx, wf) {
|
|
157
253
|
const subInput = node.inputMapper(buildContext(exec));
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
254
|
+
const recordKey = `sub:${node.name}`;
|
|
255
|
+
try {
|
|
256
|
+
const { results, executionId } = await executeSubWorkflow(node.name, subInput,
|
|
257
|
+
// The child must record who owns it: without this, recovery treats it as a
|
|
258
|
+
// top-level run and drives it behind this parent's back.
|
|
259
|
+
async (name, input) => {
|
|
260
|
+
const handle = await this.start(name, input, exec.id);
|
|
261
|
+
// Claim it in the parent's record BEFORE waiting on it. The record used to be
|
|
262
|
+
// written only on completion, so a re-entry after a restart found nothing to
|
|
263
|
+
// resume and started a second child.
|
|
264
|
+
//
|
|
265
|
+
// Awaited rather than done on a side branch: a detached `.then()` on a start
|
|
266
|
+
// that REJECTS, which is what an unregistered child does, leaves an unhandled
|
|
267
|
+
// rejection even though the caller handles the same rejection properly.
|
|
268
|
+
exec.steps[recordKey] = { status: 'running', childExecutionId: handle.id };
|
|
269
|
+
this.store.update(exec);
|
|
270
|
+
return handle;
|
|
271
|
+
}, (id) => this.store.get(id), undefined, exec.steps[recordKey]?.childExecutionId);
|
|
272
|
+
exec.steps[recordKey] = {
|
|
273
|
+
status: 'completed',
|
|
274
|
+
result: results,
|
|
275
|
+
completedAt: clock().now(),
|
|
276
|
+
childExecutionId: executionId,
|
|
277
|
+
};
|
|
166
278
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
279
|
+
catch (error) {
|
|
280
|
+
// Settle the record before letting the failure through.
|
|
281
|
+
//
|
|
282
|
+
// Only the success write existed, so every interesting outcome — the child failed,
|
|
283
|
+
// parked in `compensation-stuck`, or timed out — left the record `running`.
|
|
284
|
+
// `unwindSet` drops anything that is neither `completed` nor `failed` one line
|
|
285
|
+
// before the `sub:` branch can admit it, so `unwindChild` was never called: a
|
|
286
|
+
// parent whose child was parked with stock still reserved reversed its own steps,
|
|
287
|
+
// reached the end of the pass and reported `rollbackStatus: 'completed'`
|
|
288
|
+
// (`test/repro-workflow-child-park-inherit.test.ts`).
|
|
289
|
+
//
|
|
290
|
+
// It is also the truthful record on its own terms. A `sub:` step left `running`
|
|
291
|
+
// after the parent has failed reads on a dashboard as a child still in flight.
|
|
292
|
+
const claimed = exec.steps[recordKey];
|
|
293
|
+
if (claimed) {
|
|
294
|
+
exec.steps[recordKey] = {
|
|
295
|
+
...claimed,
|
|
176
296
|
status: 'failed',
|
|
177
|
-
|
|
178
|
-
completedAt:
|
|
179
|
-
error: `Signal "${node.event}" timed out after ${node.timeout}ms`,
|
|
297
|
+
error: describeError(error),
|
|
298
|
+
completedAt: clock().now(),
|
|
180
299
|
};
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
//
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
300
|
+
// Guarded for the same reason as the failure write in `runner.ts`: `error` below
|
|
301
|
+
// carries the real cause, and letting a write failure propagate instead replaced
|
|
302
|
+
// it. The child's own diagnostic is what the rollback guide's field table points
|
|
303
|
+
// an operator at, `... timed out` or `... is parked mid-rollback; resolve it with
|
|
304
|
+
// resumeCompensation or abandonCompensation`, so destroying it takes away the
|
|
305
|
+
// pointer to the child.
|
|
306
|
+
//
|
|
307
|
+
// This one does not self-heal, which is what made it worth guarding rather than
|
|
308
|
+
// commenting: the next write succeeds, persists the wrong diagnostic, and the run
|
|
309
|
+
// goes terminal carrying it
|
|
310
|
+
// (`test/repro-workflow-step-error-masking.test.ts`).
|
|
311
|
+
try {
|
|
312
|
+
this.store.update(exec);
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
// Deliberately swallowed; the in-memory record still settles the `sub:` step, and
|
|
316
|
+
// `runNode` re-persists the whole execution one frame up.
|
|
317
|
+
}
|
|
189
318
|
}
|
|
190
|
-
|
|
191
|
-
const remaining = node.timeout - (Date.now() - waitingSince);
|
|
192
|
-
this.scheduleTimeoutCheck(exec.id, exec.workflowName, exec.currentNodeIndex, remaining);
|
|
319
|
+
throw error;
|
|
193
320
|
}
|
|
194
|
-
exec.state = 'waiting';
|
|
195
|
-
this.store.update(exec);
|
|
196
|
-
this.emitter?.emitWorkflow('workflow:waiting', exec.id, exec.workflowName, 'waiting');
|
|
197
|
-
throw new WaitForSignalError(node.event);
|
|
198
|
-
}
|
|
199
|
-
async runLoop(exec, node, idx, wf, loopFn) {
|
|
200
|
-
await loopFn(node.def, exec, this.emitter, this.updateFn);
|
|
201
321
|
await this.advance(exec, idx + 1, wf);
|
|
202
322
|
}
|
|
203
|
-
|
|
204
|
-
|
|
323
|
+
/**
|
|
324
|
+
* Commit the saga. From here the run can still fail, but it can no longer be
|
|
325
|
+
* rolled back: recovery past this point is forward-only.
|
|
326
|
+
*/
|
|
327
|
+
async runPivot(exec, idx, wf) {
|
|
328
|
+
exec.committedAt = idx;
|
|
329
|
+
this.store.update(exec);
|
|
205
330
|
await this.advance(exec, idx + 1, wf);
|
|
206
331
|
}
|
|
207
|
-
|
|
208
|
-
|
|
332
|
+
/**
|
|
333
|
+
* Run a node body that manages its own step records (loops, forEach, map), then
|
|
334
|
+
* move on. These differ only in which executor they delegate to.
|
|
335
|
+
*/
|
|
336
|
+
async runBody(exec, idx, wf, run, def) {
|
|
337
|
+
await run(def, exec, this.emitter, this.updateFn);
|
|
209
338
|
await this.advance(exec, idx + 1, wf);
|
|
210
339
|
}
|
|
211
340
|
async advance(exec, nextIdx, wf) {
|
|
@@ -226,20 +355,34 @@ export class WorkflowExecutor {
|
|
|
226
355
|
workflowName: exec.workflowName,
|
|
227
356
|
nodeIndex: exec.currentNodeIndex,
|
|
228
357
|
};
|
|
358
|
+
// Deliberately NO deterministic jobId here.
|
|
359
|
+
//
|
|
360
|
+
// `<execution>:<nodeIndex>` was tried, to let the queue's custom-id dedup collapse
|
|
361
|
+
// a duplicate enqueue. It buys nothing the cursor guard in processStep does not
|
|
362
|
+
// already provide (a duplicate job is ignored there), and it introduces a liveness
|
|
363
|
+
// risk in exchange: if the custom-id entry outlives the job it names, a legitimate
|
|
364
|
+
// re-enqueue of the same node is swallowed and the run wedges permanently. A
|
|
365
|
+
// generated model campaign produced exactly one unexplained `execution wedged in
|
|
366
|
+
// "running"` with it enabled and none without. A duplicate job that is ignored is
|
|
367
|
+
// strictly safer than a missing job that never arrives.
|
|
229
368
|
await this.queue.add('wf:step', jobData);
|
|
230
369
|
}
|
|
370
|
+
get waitForDeps() {
|
|
371
|
+
return {
|
|
372
|
+
store: this.store,
|
|
373
|
+
emitter: this.emitter,
|
|
374
|
+
advance: (e, next, w) => this.advance(e, next, w),
|
|
375
|
+
compensate: (e, w) => this.compensate(e, w),
|
|
376
|
+
scheduleTimeoutCheck: (id, name, i, ms) => {
|
|
377
|
+
this.scheduleTimeoutCheck(id, name, i, ms);
|
|
378
|
+
},
|
|
379
|
+
};
|
|
380
|
+
}
|
|
231
381
|
scheduleTimeoutCheck(execId, workflowName, nodeIdx, ms) {
|
|
232
|
-
|
|
233
|
-
this.timeoutTimers.delete(execId);
|
|
234
|
-
const jobData = { executionId: execId, workflowName, nodeIndex: nodeIdx };
|
|
235
|
-
this.queue.add('wf:step', jobData).catch(() => {
|
|
236
|
-
/* Queue may be closed */
|
|
237
|
-
});
|
|
238
|
-
}, ms);
|
|
239
|
-
this.timeoutTimers.set(execId, timer);
|
|
382
|
+
scheduleTimeoutCheck({ queue: this.queue, timers: this.timeoutTimers }, execId, workflowName, nodeIdx, ms);
|
|
240
383
|
}
|
|
241
384
|
async compensate(exec, wf) {
|
|
242
|
-
await runCompensation(exec, wf, this.store, this.emitter);
|
|
385
|
+
await runCompensation(exec, wf, this.store, this.emitter, this.workflows);
|
|
243
386
|
}
|
|
244
387
|
/** Recover orphaned executions after a crash/restart */
|
|
245
388
|
async recover() {
|
|
@@ -249,6 +392,7 @@ export class WorkflowExecutor {
|
|
|
249
392
|
workflows: this.workflows,
|
|
250
393
|
emitter: this.emitter,
|
|
251
394
|
timeoutTimers: this.timeoutTimers,
|
|
395
|
+
nodesInFlight: this.nodesInFlight,
|
|
252
396
|
scheduleTimeoutCheck: (id, name, idx, ms) => {
|
|
253
397
|
this.scheduleTimeoutCheck(id, name, idx, ms);
|
|
254
398
|
},
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Idempotency identity for saga steps.
|
|
3
|
+
*
|
|
4
|
+
* The key is a function of (run, step name, occurrence, direction) and deliberately
|
|
5
|
+
* NOT of the attempt number. That is the single most common way to get this wrong:
|
|
6
|
+
* derive it from the attempt and every automatic retry asks the provider for a
|
|
7
|
+
* brand-new charge instead of being deduplicated into the first one.
|
|
8
|
+
*
|
|
9
|
+
* The three retry semantics collapse into two key behaviours:
|
|
10
|
+
*
|
|
11
|
+
* automatic retry after a transient error -> SAME key (we do not know whether the
|
|
12
|
+
* crash-and-resume of the same run -> SAME key effect landed; we want dedup)
|
|
13
|
+
* a different run of the same logic -> different key (different run id)
|
|
14
|
+
*
|
|
15
|
+
* `occurrence` exists because loop bodies reuse one step name. It is taken from the
|
|
16
|
+
* loop's iteration index rather than a running counter precisely so that it stays
|
|
17
|
+
* stable when a resumed loop replays its earlier iterations: replaying iteration 2
|
|
18
|
+
* must present the key iteration 2 already used, or the provider bills twice.
|
|
19
|
+
*/
|
|
20
|
+
export type Direction = 'forward' | 'compensate';
|
|
21
|
+
export declare function idempotencyKey(runId: string, stepName: string, occurrence: number, direction: Direction): string;
|
|
22
|
+
/**
|
|
23
|
+
* A loop iteration record is `<step>:<digits>` and nothing else.
|
|
24
|
+
*
|
|
25
|
+
* Three separate defects came from treating "contains a colon" as the test: a step
|
|
26
|
+
* legitimately named `charge:extra` was resolved to a loop body called `charge`, was
|
|
27
|
+
* dropped from the unwind set, and had its compensate context rebound to another
|
|
28
|
+
* step's result. A colon is a legal character in a user-chosen step name; only the
|
|
29
|
+
* numeric suffix this engine appends is structural.
|
|
30
|
+
*/
|
|
31
|
+
export declare const LOOP_ITERATION_SUFFIX: RegExp;
|
|
32
|
+
/** Is `name` an iteration record of the loop body step `base`? */
|
|
33
|
+
export declare function isIterationOf(name: string, base: string): boolean;
|
|
34
|
+
/** `charge:2` -> `charge`; `charge:extra` -> `charge:extra` (not an iteration). */
|
|
35
|
+
export declare function loopBaseName(name: string): string;
|
|
36
|
+
/**
|
|
37
|
+
* A human-readable description of anything that was thrown.
|
|
38
|
+
*
|
|
39
|
+
* `String(err)` is what this replaced, and for the shape an HTTP client throws most
|
|
40
|
+
* often, a structured object, it yields `"[object Object]"`. That string was then
|
|
41
|
+
* persisted as the diagnostic on a run parked in `compensation-stuck`, which is the
|
|
42
|
+
* state that exists so an operator has something to act on. They were handed a
|
|
43
|
+
* sentence that says nothing about a refund that did not go through
|
|
44
|
+
* (`test/repro-workflow-operator-signal.test.ts`).
|
|
45
|
+
*/
|
|
46
|
+
export declare function describeError(err: unknown): string;
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Idempotency identity for saga steps.
|
|
3
|
+
*
|
|
4
|
+
* The key is a function of (run, step name, occurrence, direction) and deliberately
|
|
5
|
+
* NOT of the attempt number. That is the single most common way to get this wrong:
|
|
6
|
+
* derive it from the attempt and every automatic retry asks the provider for a
|
|
7
|
+
* brand-new charge instead of being deduplicated into the first one.
|
|
8
|
+
*
|
|
9
|
+
* The three retry semantics collapse into two key behaviours:
|
|
10
|
+
*
|
|
11
|
+
* automatic retry after a transient error -> SAME key (we do not know whether the
|
|
12
|
+
* crash-and-resume of the same run -> SAME key effect landed; we want dedup)
|
|
13
|
+
* a different run of the same logic -> different key (different run id)
|
|
14
|
+
*
|
|
15
|
+
* `occurrence` exists because loop bodies reuse one step name. It is taken from the
|
|
16
|
+
* loop's iteration index rather than a running counter precisely so that it stays
|
|
17
|
+
* stable when a resumed loop replays its earlier iterations: replaying iteration 2
|
|
18
|
+
* must present the key iteration 2 already used, or the provider bills twice.
|
|
19
|
+
*/
|
|
20
|
+
export function idempotencyKey(runId, stepName, occurrence, direction) {
|
|
21
|
+
return `${runId}:${stepName}#${occurrence}:${direction}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* A loop iteration record is `<step>:<digits>` and nothing else.
|
|
25
|
+
*
|
|
26
|
+
* Three separate defects came from treating "contains a colon" as the test: a step
|
|
27
|
+
* legitimately named `charge:extra` was resolved to a loop body called `charge`, was
|
|
28
|
+
* dropped from the unwind set, and had its compensate context rebound to another
|
|
29
|
+
* step's result. A colon is a legal character in a user-chosen step name; only the
|
|
30
|
+
* numeric suffix this engine appends is structural.
|
|
31
|
+
*/
|
|
32
|
+
export const LOOP_ITERATION_SUFFIX = /:\d+$/;
|
|
33
|
+
/**
|
|
34
|
+
* What is left of an iteration record once its base name is removed: `:` then digits
|
|
35
|
+
* and NOTHING else.
|
|
36
|
+
*
|
|
37
|
+
* Anchoring at both ends is the whole point. Testing the unanchored
|
|
38
|
+
* `LOOP_ITERATION_SUFFIX` against the remainder only checks that the name ENDS in a
|
|
39
|
+
* numeric segment, so a loop body whose own name contains a colon collides with a
|
|
40
|
+
* shorter one: `charge:extra:1` matched base `charge`, leaving remainder `:extra:1`,
|
|
41
|
+
* which ends in `:1`. `findStepDef` then handed every `charge:extra` iteration the
|
|
42
|
+
* `charge` handler — `charge`'s reversal ran twice per iteration, `charge:extra`'s
|
|
43
|
+
* never ran, and the run still reported `rollbackStatus: 'completed'`.
|
|
44
|
+
*/
|
|
45
|
+
const EXACT_ITERATION_REMAINDER = /^:\d+$/;
|
|
46
|
+
/** Is `name` an iteration record of the loop body step `base`? */
|
|
47
|
+
export function isIterationOf(name, base) {
|
|
48
|
+
return name.startsWith(`${base}:`) && EXACT_ITERATION_REMAINDER.test(name.slice(base.length));
|
|
49
|
+
}
|
|
50
|
+
/** `charge:2` -> `charge`; `charge:extra` -> `charge:extra` (not an iteration). */
|
|
51
|
+
export function loopBaseName(name) {
|
|
52
|
+
return LOOP_ITERATION_SUFFIX.test(name) ? name.replace(LOOP_ITERATION_SUFFIX, '') : name;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* A human-readable description of anything that was thrown.
|
|
56
|
+
*
|
|
57
|
+
* `String(err)` is what this replaced, and for the shape an HTTP client throws most
|
|
58
|
+
* often, a structured object, it yields `"[object Object]"`. That string was then
|
|
59
|
+
* persisted as the diagnostic on a run parked in `compensation-stuck`, which is the
|
|
60
|
+
* state that exists so an operator has something to act on. They were handed a
|
|
61
|
+
* sentence that says nothing about a refund that did not go through
|
|
62
|
+
* (`test/repro-workflow-operator-signal.test.ts`).
|
|
63
|
+
*/
|
|
64
|
+
export function describeError(err) {
|
|
65
|
+
// An AggregateError carries every failure of a `parallel()` group, and taking only
|
|
66
|
+
// `.message` reported the FIRST one: two steps failed, the persisted
|
|
67
|
+
// `failureReason` named one, and whoever read it went looking for a single cause
|
|
68
|
+
// that was not the only cause.
|
|
69
|
+
if (err instanceof AggregateError && Array.isArray(err.errors) && err.errors.length > 0) {
|
|
70
|
+
// Recursive, not `String(e)`: a user-thrown aggregate can carry plain objects, and
|
|
71
|
+
// `String({...})` is the `[object Object]` this function exists to kill.
|
|
72
|
+
const parts = err.errors.map((e) => describeError(e));
|
|
73
|
+
return parts.length === 1 ? parts[0] : `${parts.length} failures: ${parts.join('; ')}`;
|
|
74
|
+
}
|
|
75
|
+
if (err instanceof Error)
|
|
76
|
+
return err.message;
|
|
77
|
+
if (typeof err === 'object' && err !== null) {
|
|
78
|
+
try {
|
|
79
|
+
const json = JSON.stringify(err);
|
|
80
|
+
// A class instance with no enumerable fields serialises to `{}`, which is as
|
|
81
|
+
// useless as the string it replaced; fall back to the constructor name.
|
|
82
|
+
if (json && json !== '{}')
|
|
83
|
+
return json;
|
|
84
|
+
const name = err.constructor?.name;
|
|
85
|
+
return name && name !== 'Object' ? `${name} (no serialisable detail)` : String(err);
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
// Circular, or a throwing getter. The type is still more than nothing.
|
|
89
|
+
return `${err.constructor?.name ?? 'object'} (not serialisable)`;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return String(err);
|
|
93
|
+
}
|
|
@@ -20,4 +20,4 @@ import '../../require-bun';
|
|
|
20
20
|
export { Workflow } from './workflow';
|
|
21
21
|
export { Engine } from './engine';
|
|
22
22
|
export { WorkflowEmitter } from './emitter';
|
|
23
|
-
export type { StepContext, StepHandler, TypedStepHandler, CompensateHandler, TypedCompensateHandler, StepOptions, SchemaLike, Execution, ExecutionState, StepState, StepRecord, EngineOptions, RunHandle, ParallelDefinition, SubWorkflowInputMapper, LoopCondition, ForEachItemsExtractor, MapTransformFn, LoopDefinition, ForEachDefinition, MapDefinition, RecoverResult, CleanupOptions, WorkflowEventType, WorkflowEvent, StepEvent, WorkflowLifecycleEvent, SignalEvent, WorkflowEventListener, } from './types';
|
|
23
|
+
export type { StepContext, StepHandler, TypedStepHandler, CompensateHandler, TypedCompensateHandler, StepOptions, SchemaLike, Execution, ExecutionState, StepState, StepRecord, RollbackStatus, CompensationStatus, CompensationOutcome, BranchCondition, WorkflowNode, EngineOptions, RunHandle, ParallelDefinition, SubWorkflowInputMapper, LoopCondition, ForEachItemsExtractor, MapTransformFn, LoopDefinition, ForEachDefinition, MapDefinition, RecoverResult, CleanupOptions, WorkflowEventType, WorkflowEvent, StepEvent, WorkflowLifecycleEvent, SignalEvent, WorkflowEventListener, } from './types';
|