bunqueue 2.8.59 → 2.8.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/application/dlqManager.d.ts +1 -1
- package/dist/application/dlqManager.js +1 -1
- package/dist/application/lockManager.js +1 -1
- package/dist/application/queue-manager/ack.js +1 -1
- package/dist/application/queue-manager/delivery.js +1 -1
- package/dist/application/statsManager.d.ts +1 -1
- package/dist/application/statsManager.js +1 -1
- package/dist/client/events.js +2 -2
- package/dist/client/flow.js +0 -1
- package/dist/client/queue/operations/add/single.js +1 -1
- package/dist/client/sandboxed/runtime/state.js +2 -2
- package/dist/client/tcp/connection.d.ts +1 -1
- package/dist/client/tcp/connection.js +1 -1
- package/dist/client/tcp/runtime/state.js +2 -2
- package/dist/client/types/metrics.d.ts +1 -1
- package/dist/client/worker/runtime/control.js +0 -1
- package/dist/client/worker/runtime/lifecycle.js +0 -1
- package/dist/client/worker/runtime/state.js +3 -3
- package/dist/client/workflow/compensationChild.d.ts +2 -0
- package/dist/client/workflow/compensationChild.js +3 -1
- package/dist/client/workflow/compensationPass.d.ts +1 -0
- package/dist/client/workflow/compensationPass.js +18 -2
- package/dist/client/workflow/compensator.d.ts +2 -1
- package/dist/client/workflow/compensator.js +11 -3
- package/dist/client/workflow/engine.js +1 -1
- package/dist/client/workflow/executionFence.d.ts +15 -0
- package/dist/client/workflow/executionFence.js +29 -0
- package/dist/client/workflow/executor.d.ts +5 -11
- package/dist/client/workflow/executor.js +48 -47
- package/dist/client/workflow/executorLifecycle.d.ts +1 -0
- package/dist/client/workflow/executorLifecycle.js +41 -10
- package/dist/client/workflow/executorNodes.d.ts +1 -0
- package/dist/client/workflow/executorNodes.js +45 -16
- package/dist/client/workflow/executorQueue.d.ts +9 -0
- package/dist/client/workflow/executorQueue.js +15 -0
- package/dist/client/workflow/forEachRunner.d.ts +4 -0
- package/dist/client/workflow/forEachRunner.js +70 -0
- package/dist/client/workflow/loops.d.ts +4 -5
- package/dist/client/workflow/loops.js +27 -93
- package/dist/client/workflow/mapRunner.d.ts +1 -1
- package/dist/client/workflow/mapRunner.js +11 -2
- package/dist/client/workflow/recovery.d.ts +1 -0
- package/dist/client/workflow/recovery.js +24 -18
- package/dist/client/workflow/rollbackControl.d.ts +1 -0
- package/dist/client/workflow/rollbackControl.js +5 -2
- package/dist/client/workflow/runner.d.ts +3 -1
- package/dist/client/workflow/runner.js +26 -3
- package/dist/client/workflow/stepTypes.js +1 -1
- package/dist/client/workflow/subWorkflowRunner.d.ts +1 -0
- package/dist/client/workflow/subWorkflowRunner.js +7 -0
- package/dist/client/workflow/waitFor.d.ts +4 -0
- package/dist/client/workflow/waitFor.js +23 -11
- package/dist/client/workflow/workflowDecisions.d.ts +1 -1
- package/dist/client/workflow/workflowDecisions.js +3 -1
- package/dist/infrastructure/persistence/sqlite.d.ts +1 -1
- package/dist/infrastructure/server/handler-routes/jobs.js +0 -3
- package/dist/infrastructure/server/handlerRoutes.d.ts +2 -2
- package/dist/infrastructure/server/handlerRoutes.js +2 -2
- package/dist/infrastructure/server/httpEndpoints.js +2 -0
- package/package.json +12 -10
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Loop execution logic for the Workflow Engine.
|
|
3
3
|
*/
|
|
4
|
+
import { assertWorkflowActive, isWorkflowExecutionClosed } from './executionFence';
|
|
4
5
|
import { executeStepWithRetry, buildContext } from './runner';
|
|
5
|
-
import {
|
|
6
|
+
import { loopDecisionKey, resolveDecision } from './workflowDecisions';
|
|
6
7
|
export { executeMap } from './mapRunner';
|
|
8
|
+
export { executeForEach } from './forEachRunner';
|
|
7
9
|
/**
|
|
8
10
|
* Run one iteration of a loop body step, unless it already ran.
|
|
9
11
|
*
|
|
@@ -34,7 +36,10 @@ export { executeMap } from './mapRunner';
|
|
|
34
36
|
* the opposite, which was true only while a loop compensated its last iteration
|
|
35
37
|
* alone.
|
|
36
38
|
*/
|
|
37
|
-
async function runIteration(step, exec, iteration,
|
|
39
|
+
async function runIteration(step, exec, iteration, hooks) {
|
|
40
|
+
const { emitter, updateFn } = hooks;
|
|
41
|
+
const assertActive = hooks.assertActive ?? assertWorkflowActive;
|
|
42
|
+
assertActive();
|
|
38
43
|
const indexed = `${step.name}:${iteration}`;
|
|
39
44
|
const already = exec.steps[indexed];
|
|
40
45
|
if (already?.status === 'completed') {
|
|
@@ -47,9 +52,13 @@ async function runIteration(step, exec, iteration, emitter, updateFn) {
|
|
|
47
52
|
let thrown;
|
|
48
53
|
let threw = false;
|
|
49
54
|
try {
|
|
50
|
-
await executeStepWithRetry(step, buildContext(exec), exec, { emitter, updateFn }, iteration);
|
|
55
|
+
await executeStepWithRetry(step, buildContext(exec), exec, { emitter, updateFn, assertActive }, iteration);
|
|
56
|
+
assertActive();
|
|
51
57
|
}
|
|
52
58
|
catch (error) {
|
|
59
|
+
if (isWorkflowExecutionClosed(error))
|
|
60
|
+
throw error;
|
|
61
|
+
assertActive();
|
|
53
62
|
// Caught rather than left to a `finally`, because the mirror write below must not be
|
|
54
63
|
// able to REPLACE this error. An exception thrown from a `finally` supersedes the one
|
|
55
64
|
// in flight, so a store that refused the mirror write turned "provider timeout after
|
|
@@ -88,6 +97,8 @@ async function runIteration(step, exec, iteration, emitter, updateFn) {
|
|
|
88
97
|
updateFn(exec);
|
|
89
98
|
}
|
|
90
99
|
catch (writeError) {
|
|
100
|
+
if (isWorkflowExecutionClosed(writeError))
|
|
101
|
+
throw writeError;
|
|
91
102
|
if (!threw)
|
|
92
103
|
throw writeError;
|
|
93
104
|
}
|
|
@@ -97,29 +108,35 @@ async function runIteration(step, exec, iteration, emitter, updateFn) {
|
|
|
97
108
|
throw thrown;
|
|
98
109
|
}
|
|
99
110
|
/** Execute a doUntil loop: run steps, then check condition. Repeat until condition returns true. */
|
|
100
|
-
export async function executeDoUntil(def, exec, emitter, updateFn) {
|
|
111
|
+
export async function executeDoUntil(def, exec, emitter, updateFn, assertActive = assertWorkflowActive) {
|
|
112
|
+
assertActive();
|
|
101
113
|
let iteration = 0;
|
|
102
114
|
let shouldStop = false;
|
|
103
115
|
while (!shouldStop) {
|
|
116
|
+
assertActive();
|
|
104
117
|
if (iteration >= def.maxIterations) {
|
|
105
118
|
throw new Error(`doUntil exceeded maxIterations (${def.maxIterations})`);
|
|
106
119
|
}
|
|
107
120
|
for (const step of def.steps) {
|
|
108
|
-
await runIteration(step, exec, iteration, emitter, updateFn);
|
|
121
|
+
await runIteration(step, exec, iteration, { emitter, updateFn, assertActive });
|
|
122
|
+
assertActive();
|
|
109
123
|
}
|
|
110
124
|
iteration++;
|
|
111
125
|
const ctx = buildContext(exec);
|
|
112
|
-
shouldStop = await resolveDecision(exec, loopDecisionKey('doUntil', def.steps[0].name, iteration), () => def.condition(ctx, iteration), updateFn);
|
|
126
|
+
shouldStop = await resolveDecision(exec, loopDecisionKey('doUntil', def.steps[0].name, iteration), () => def.condition(ctx, iteration), updateFn, assertActive);
|
|
127
|
+
assertActive();
|
|
113
128
|
if (typeof shouldStop !== 'boolean') {
|
|
114
129
|
throw new Error('doUntil condition must return a boolean');
|
|
115
130
|
}
|
|
116
131
|
}
|
|
117
132
|
}
|
|
118
133
|
/** Execute a doWhile loop: check condition first, then run steps. Repeat while condition is true. */
|
|
119
|
-
export async function executeDoWhile(def, exec, emitter, updateFn) {
|
|
134
|
+
export async function executeDoWhile(def, exec, emitter, updateFn, assertActive = assertWorkflowActive) {
|
|
135
|
+
assertActive();
|
|
120
136
|
for (let iteration = 0;; iteration++) {
|
|
121
137
|
const ctx = buildContext(exec);
|
|
122
|
-
const shouldContinue = await resolveDecision(exec, loopDecisionKey('doWhile', def.steps[0].name, iteration), () => def.condition(ctx, iteration), updateFn);
|
|
138
|
+
const shouldContinue = await resolveDecision(exec, loopDecisionKey('doWhile', def.steps[0].name, iteration), () => def.condition(ctx, iteration), updateFn, assertActive);
|
|
139
|
+
assertActive();
|
|
123
140
|
if (typeof shouldContinue !== 'boolean') {
|
|
124
141
|
throw new Error('doWhile condition must return a boolean');
|
|
125
142
|
}
|
|
@@ -129,91 +146,8 @@ export async function executeDoWhile(def, exec, emitter, updateFn) {
|
|
|
129
146
|
throw new Error(`doWhile exceeded maxIterations (${def.maxIterations})`);
|
|
130
147
|
}
|
|
131
148
|
for (const step of def.steps) {
|
|
132
|
-
await runIteration(step, exec, iteration, emitter, updateFn);
|
|
133
|
-
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
/** Execute a forEach loop: iterate over items, executing the step for each */
|
|
137
|
-
export async function executeForEach(def, exec, emitter, updateFn) {
|
|
138
|
-
const ctx = buildContext(exec);
|
|
139
|
-
const items = await resolveDecision(exec, forEachItemsDecisionKey(def.step.name), () => structuredClone(def.items(ctx)), updateFn);
|
|
140
|
-
// Anything with a `length` used to be accepted, and JavaScript is generous about
|
|
141
|
-
// what has one. A number iterated ZERO times and the run reported `completed`, so a
|
|
142
|
-
// batch that processed nothing was indistinguishable from a batch with nothing to
|
|
143
|
-
// do. A string iterated its CHARACTERS, so an id list that arrived as `'u1,u2'`
|
|
144
|
-
// silently processed five "items" nobody passed. `null`/`undefined` were caught only
|
|
145
|
-
// by accident, because reading `.length` throws. The likely production shapes were
|
|
146
|
-
// exactly the ones that passed (`test/repro-workflow-foreach-non-array.test.ts`).
|
|
147
|
-
if (!Array.isArray(items)) {
|
|
148
|
-
throw new Error(`forEach items must be an array, got ${items === null ? 'null' : typeof items}`);
|
|
149
|
-
}
|
|
150
|
-
if (items.length > def.maxIterations) {
|
|
151
|
-
throw new Error(`forEach items (${items.length}) exceeds maxIterations (${def.maxIterations})`);
|
|
152
|
-
}
|
|
153
|
-
for (let i = 0; i < items.length; i++) {
|
|
154
|
-
const item = structuredClone(items[i]);
|
|
155
|
-
const indexedName = `${def.step.name}:${i}`;
|
|
156
|
-
const indexedStep = {
|
|
157
|
-
...def.step,
|
|
158
|
-
name: indexedName,
|
|
159
|
-
handler: (stepCtx) => {
|
|
160
|
-
const enrichedCtx = {
|
|
161
|
-
...stepCtx,
|
|
162
|
-
steps: { ...stepCtx.steps, __item: item, __index: i },
|
|
163
|
-
};
|
|
164
|
-
return def.step.handler(enrichedCtx);
|
|
165
|
-
},
|
|
166
|
-
};
|
|
167
|
-
// Memoised the same way as doUntil/doWhile: an item already provisioned before a
|
|
168
|
-
// crash must not be provisioned again when the node is re-entered. Restore the
|
|
169
|
-
// declared bare name too: it is the public result of the final iteration, while
|
|
170
|
-
// the indexed record remains the durable execution/compensation identity.
|
|
171
|
-
const completed = exec.steps[indexedName];
|
|
172
|
-
if (completed?.status === 'completed') {
|
|
173
|
-
exec.steps[def.step.name] = { ...completed };
|
|
174
|
-
updateFn(exec);
|
|
175
|
-
continue;
|
|
176
|
-
}
|
|
177
|
-
const stepCtx = buildContext(exec);
|
|
178
|
-
let thrown;
|
|
179
|
-
let threw = false;
|
|
180
|
-
try {
|
|
181
|
-
await executeStepWithRetry(indexedStep, stepCtx, exec, { emitter, updateFn }, i);
|
|
182
|
-
}
|
|
183
|
-
catch (error) {
|
|
184
|
-
// Same reason `runIteration` catches instead of using a `finally`: a write that
|
|
185
|
-
// fails below must not REPLACE the step's own error, which is what `failureReason`
|
|
186
|
-
// records and the only account of what went wrong.
|
|
187
|
-
thrown = error;
|
|
188
|
-
threw = true;
|
|
189
|
-
}
|
|
190
|
-
{
|
|
191
|
-
// Persist this iteration's __item/__index alongside the step record so saga
|
|
192
|
-
// compensation can restore the correct per-iteration context later.
|
|
193
|
-
//
|
|
194
|
-
// Recording on BOTH paths is load-bearing: the iteration that THROWS is itself
|
|
195
|
-
// eligible for compensation, and without its item recorded its compensate handler
|
|
196
|
-
// is handed an undefined `__item` and silently releases nothing — the reservation
|
|
197
|
-
// it had already taken at the warehouse leaks
|
|
198
|
-
// (test/workflow-saga-extreme.test.ts).
|
|
199
|
-
const record = exec.steps[indexedName];
|
|
200
|
-
if (record) {
|
|
201
|
-
record.loopItem = item;
|
|
202
|
-
record.loopIndex = i;
|
|
203
|
-
// Keep the documented aggregate view in sync on success and failure. The
|
|
204
|
-
// indexed record is still the authoritative unit of work; compensator.ts
|
|
205
|
-
// excludes this mirror whenever an indexed sibling exists.
|
|
206
|
-
exec.steps[def.step.name] = { ...record };
|
|
207
|
-
try {
|
|
208
|
-
updateFn(exec);
|
|
209
|
-
}
|
|
210
|
-
catch (writeError) {
|
|
211
|
-
if (!threw)
|
|
212
|
-
throw writeError;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
149
|
+
await runIteration(step, exec, iteration, { emitter, updateFn, assertActive });
|
|
150
|
+
assertActive();
|
|
215
151
|
}
|
|
216
|
-
if (threw)
|
|
217
|
-
throw thrown;
|
|
218
152
|
}
|
|
219
153
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import type { WorkflowEmitter } from './emitter';
|
|
2
2
|
import type { Execution, MapDefinition } from './types';
|
|
3
3
|
/** Execute a map node with the same durable lifecycle signals as a regular step. */
|
|
4
|
-
export declare function executeMap(def: MapDefinition, exec: Execution, emitter: WorkflowEmitter | null, updateFn: (exec: Execution) => void): Promise<void>;
|
|
4
|
+
export declare function executeMap(def: MapDefinition, exec: Execution, emitter: WorkflowEmitter | null, updateFn: (exec: Execution) => void, assertActive?: () => void): Promise<void>;
|
|
@@ -1,19 +1,26 @@
|
|
|
1
1
|
import { clock } from './clock';
|
|
2
|
+
import { assertWorkflowActive, isWorkflowExecutionClosed } from './executionFence';
|
|
2
3
|
import { describeError } from './identity';
|
|
3
4
|
import { buildContext } from './runner';
|
|
4
5
|
/** Execute a map node with the same durable lifecycle signals as a regular step. */
|
|
5
|
-
export async function executeMap(def, exec, emitter, updateFn) {
|
|
6
|
+
export async function executeMap(def, exec, emitter, updateFn, assertActive = assertWorkflowActive) {
|
|
7
|
+
assertActive();
|
|
6
8
|
if (exec.steps[def.name]?.status === 'completed')
|
|
7
9
|
return;
|
|
8
10
|
const startedAt = clock().now();
|
|
9
11
|
exec.steps[def.name] = { status: 'running', startedAt };
|
|
10
12
|
updateFn(exec);
|
|
11
13
|
emitter?.emitStep('step:started', exec.id, exec.workflowName, def.name);
|
|
14
|
+
assertActive();
|
|
12
15
|
let result;
|
|
13
16
|
try {
|
|
14
17
|
result = await def.transform(buildContext(exec));
|
|
18
|
+
assertActive();
|
|
15
19
|
}
|
|
16
20
|
catch (error) {
|
|
21
|
+
if (isWorkflowExecutionClosed(error))
|
|
22
|
+
throw error;
|
|
23
|
+
assertActive();
|
|
17
24
|
const failure = error instanceof Error ? error : new Error(describeError(error));
|
|
18
25
|
exec.steps[def.name] = {
|
|
19
26
|
status: 'failed',
|
|
@@ -24,7 +31,9 @@ export async function executeMap(def, exec, emitter, updateFn) {
|
|
|
24
31
|
try {
|
|
25
32
|
updateFn(exec);
|
|
26
33
|
}
|
|
27
|
-
catch {
|
|
34
|
+
catch (writeError) {
|
|
35
|
+
if (isWorkflowExecutionClosed(writeError))
|
|
36
|
+
throw writeError;
|
|
28
37
|
// Preserve the transform failure; generic workflow failure handling retries the write.
|
|
29
38
|
}
|
|
30
39
|
emitter?.emitStep('step:failed', exec.id, exec.workflowName, def.name, {
|
|
@@ -11,11 +11,14 @@ import { hasSignal } from './storeSignals';
|
|
|
11
11
|
import { clock } from './clock';
|
|
12
12
|
import { decideAdmission } from './admission';
|
|
13
13
|
import { bindExecutionDefinition } from './definitionGuard';
|
|
14
|
+
import { enqueueWorkflowStep } from './executorQueue';
|
|
14
15
|
export async function recoverExecutions(deps) {
|
|
15
16
|
const { store, workflows } = deps;
|
|
17
|
+
deps.assertActive();
|
|
16
18
|
const executions = store.listRecoverable();
|
|
17
19
|
const result = { running: 0, waiting: 0, compensating: 0, total: 0 };
|
|
18
20
|
for (const snapshot of executions) {
|
|
21
|
+
deps.assertActive();
|
|
19
22
|
const wf = workflows.get(snapshot.workflowName);
|
|
20
23
|
if (!wf)
|
|
21
24
|
continue;
|
|
@@ -36,12 +39,14 @@ export async function recoverExecutions(deps) {
|
|
|
36
39
|
// Only `already-in-flight` is reachable here: the state is `running` by the
|
|
37
40
|
// branch and the index is the execution's own cursor.
|
|
38
41
|
if (admission.kind === 'run') {
|
|
39
|
-
await
|
|
42
|
+
await enqueueWorkflowStep(deps.queue, exec, deps.assertActive);
|
|
43
|
+
deps.assertActive();
|
|
40
44
|
result.running++;
|
|
41
45
|
}
|
|
42
46
|
}
|
|
43
47
|
else if (exec.state === 'waiting') {
|
|
44
48
|
await recoverWaiting(exec, wf, deps);
|
|
49
|
+
deps.assertActive();
|
|
45
50
|
result.waiting++;
|
|
46
51
|
}
|
|
47
52
|
else if (exec.state === 'compensating' ||
|
|
@@ -51,8 +56,10 @@ export async function recoverExecutions(deps) {
|
|
|
51
56
|
// recoverable until runCompensation writes an explicit rollbackStatus.
|
|
52
57
|
// A lost claim means another driver owns this unwind, so nothing happened here
|
|
53
58
|
// and counting it as a recovery would overstate what recover() did.
|
|
54
|
-
if (await recoverCompensation(exec.id, wf, deps))
|
|
59
|
+
if (await recoverCompensation(exec.id, wf, deps)) {
|
|
60
|
+
deps.assertActive();
|
|
55
61
|
result.compensating++;
|
|
62
|
+
}
|
|
56
63
|
}
|
|
57
64
|
}
|
|
58
65
|
result.total = result.running + result.waiting + result.compensating;
|
|
@@ -60,6 +67,7 @@ export async function recoverExecutions(deps) {
|
|
|
60
67
|
}
|
|
61
68
|
async function recoverCompensation(executionId, wf, deps) {
|
|
62
69
|
while (true) {
|
|
70
|
+
deps.assertActive();
|
|
63
71
|
const exec = deps.store.get(executionId);
|
|
64
72
|
if (!exec ||
|
|
65
73
|
(exec.state !== 'compensating' &&
|
|
@@ -67,7 +75,10 @@ async function recoverCompensation(executionId, wf, deps) {
|
|
|
67
75
|
return false;
|
|
68
76
|
}
|
|
69
77
|
bindExecutionDefinition(exec, wf, (value) => deps.store.update(value));
|
|
70
|
-
const outcome = await runCompensation(exec, wf, deps.store, deps.emitter, deps.workflows
|
|
78
|
+
const outcome = await runCompensation(exec, wf, deps.store, deps.emitter, deps.workflows, {
|
|
79
|
+
assertActive: deps.assertActive,
|
|
80
|
+
});
|
|
81
|
+
deps.assertActive();
|
|
71
82
|
if (outcome === 'ran')
|
|
72
83
|
return true;
|
|
73
84
|
// A live unwind owned by this executor's store is not orphaned. Waiting here
|
|
@@ -80,33 +91,25 @@ async function recoverCompensation(executionId, wf, deps) {
|
|
|
80
91
|
// identity, so it waits for that owner, then re-reads the durable row: it may have
|
|
81
92
|
// finished, or this engine must resume the owed unwind.
|
|
82
93
|
await outcome.settled;
|
|
94
|
+
deps.assertActive();
|
|
83
95
|
}
|
|
84
96
|
}
|
|
85
|
-
async function enqueueExecution(exec, queue) {
|
|
86
|
-
const jobData = {
|
|
87
|
-
executionId: exec.id,
|
|
88
|
-
workflowName: exec.workflowName,
|
|
89
|
-
nodeIndex: exec.currentNodeIndex,
|
|
90
|
-
};
|
|
91
|
-
// No deterministic jobId, for the reason documented in WorkflowExecutor.enqueue:
|
|
92
|
-
// the duplicate this path can create is neutralised by the cursor guard in
|
|
93
|
-
// processStep, and dedup here risks swallowing a legitimate re-enqueue after a
|
|
94
|
-
// restart, which wedges the run for good.
|
|
95
|
-
await queue.add('wf:step', jobData);
|
|
96
|
-
}
|
|
97
97
|
async function recoverWaiting(exec, wf, deps) {
|
|
98
98
|
const node = wf.nodes[exec.currentNodeIndex];
|
|
99
99
|
if (node?.type !== 'waitFor') {
|
|
100
|
-
await
|
|
100
|
+
await enqueueWorkflowStep(deps.queue, exec, deps.assertActive);
|
|
101
|
+
deps.assertActive();
|
|
101
102
|
return;
|
|
102
103
|
}
|
|
103
104
|
// The signal may have been persisted before the crash, or accepted after this
|
|
104
105
|
// engine was recreated but before recover() reached the row. Key presence, not
|
|
105
106
|
// value: a payload-less signal records the key with an `undefined` value.
|
|
106
107
|
if (hasSignal(exec.signals, node.event)) {
|
|
108
|
+
deps.assertActive();
|
|
107
109
|
exec.state = 'running';
|
|
108
110
|
deps.store.update(exec);
|
|
109
|
-
await
|
|
111
|
+
await enqueueWorkflowStep(deps.queue, exec, deps.assertActive);
|
|
112
|
+
deps.assertActive();
|
|
110
113
|
return;
|
|
111
114
|
}
|
|
112
115
|
// Re-arm timeout if configured
|
|
@@ -119,11 +122,14 @@ async function recoverWaiting(exec, wf, deps) {
|
|
|
119
122
|
const elapsed = clock().now() - waitingSince;
|
|
120
123
|
const remaining = node.timeout - elapsed;
|
|
121
124
|
if (remaining <= 0) {
|
|
125
|
+
deps.assertActive();
|
|
122
126
|
exec.state = 'running';
|
|
123
127
|
deps.store.update(exec);
|
|
124
|
-
await
|
|
128
|
+
await enqueueWorkflowStep(deps.queue, exec, deps.assertActive);
|
|
129
|
+
deps.assertActive();
|
|
125
130
|
}
|
|
126
131
|
else {
|
|
132
|
+
deps.assertActive();
|
|
127
133
|
deps.scheduleTimeoutCheck(exec.id, exec.workflowName, exec.currentNodeIndex, remaining);
|
|
128
134
|
}
|
|
129
135
|
}
|
|
@@ -25,8 +25,9 @@ import { abandonCompensation, runCompensation } from './compensator';
|
|
|
25
25
|
* the parked run exactly as the operator found it.
|
|
26
26
|
*/
|
|
27
27
|
export async function resumeCompensation(deps, executionId) {
|
|
28
|
+
deps.assertActive();
|
|
28
29
|
const { exec, wf } = parked(deps, executionId);
|
|
29
|
-
const outcome = await runCompensation(exec, wf, deps.store, deps.emitter, deps.workflows, { retryFailed: true });
|
|
30
|
+
const outcome = await runCompensation(exec, wf, deps.store, deps.emitter, deps.workflows, { retryFailed: true, assertActive: deps.assertActive });
|
|
30
31
|
if (outcome !== 'ran') {
|
|
31
32
|
// Nothing was mutated: `runCompensation` returns this before the unwind begins.
|
|
32
33
|
throw new Error(`execution "${executionId}" is already being rolled back by another driver`);
|
|
@@ -34,10 +35,12 @@ export async function resumeCompensation(deps, executionId) {
|
|
|
34
35
|
}
|
|
35
36
|
/** Accept a partial rollback: the outstanding steps are recorded as skipped. */
|
|
36
37
|
export function abandonParkedCompensation(deps, executionId) {
|
|
38
|
+
deps.assertActive();
|
|
37
39
|
const { exec, wf } = parked(deps, executionId);
|
|
38
|
-
abandonCompensation(exec, wf, deps.store, deps.emitter);
|
|
40
|
+
abandonCompensation(exec, wf, deps.store, deps.emitter, deps.assertActive);
|
|
39
41
|
}
|
|
40
42
|
function parked(deps, executionId) {
|
|
43
|
+
deps.assertActive();
|
|
41
44
|
const exec = deps.store.get(executionId);
|
|
42
45
|
if (!exec)
|
|
43
46
|
throw new Error(`Execution "${executionId}" not found`);
|
|
@@ -10,11 +10,13 @@ export { executeSubWorkflow } from './subWorkflowRunner';
|
|
|
10
10
|
export interface StepHooks {
|
|
11
11
|
emitter: WorkflowEmitter | null;
|
|
12
12
|
updateFn: (exec: Execution) => void;
|
|
13
|
+
assertActive?: () => void;
|
|
13
14
|
}
|
|
15
|
+
export { isWorkflowExecutionClosed, WorkflowExecutionClosedError } from './executionFence';
|
|
14
16
|
/** Execute a step with retry logic and exponential backoff */
|
|
15
17
|
export declare function executeStepWithRetry(def: StepDefinition, ctx: StepContext, exec: Execution, hooks: StepHooks, occurrence?: number): Promise<void>;
|
|
16
18
|
/** Execute multiple steps in parallel via Promise.allSettled */
|
|
17
|
-
export declare function executeParallelSteps(steps: StepDefinition[], ctx: StepContext, exec: Execution,
|
|
19
|
+
export declare function executeParallelSteps(steps: StepDefinition[], ctx: StepContext, exec: Execution, hooks: StepHooks): Promise<void>;
|
|
18
20
|
/** Find a step definition by name across all node types */
|
|
19
21
|
export declare function findStepDef(wf: Workflow, name: string): StepDefinition | null;
|
|
20
22
|
/** Build a StepContext from the current execution state */
|
|
@@ -4,11 +4,15 @@
|
|
|
4
4
|
import { idempotencyKey, isIterationOf, describeError } from './identity';
|
|
5
5
|
import { clock } from './clock';
|
|
6
6
|
import { retryBackoffDelay, runWithTimeout } from './runnerTiming';
|
|
7
|
+
import { assertWorkflowActive, isWorkflowExecutionClosed } from './executionFence';
|
|
7
8
|
export { runWithTimeout } from './runnerTiming';
|
|
8
9
|
export { executeSubWorkflow } from './subWorkflowRunner';
|
|
10
|
+
export { isWorkflowExecutionClosed, WorkflowExecutionClosedError } from './executionFence';
|
|
9
11
|
/** Execute a step with retry logic and exponential backoff */
|
|
10
12
|
export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0) {
|
|
11
13
|
const { emitter, updateFn } = hooks;
|
|
14
|
+
const assertActive = hooks.assertActive ?? assertWorkflowActive;
|
|
15
|
+
assertActive();
|
|
12
16
|
const maxAttempts = def.retry;
|
|
13
17
|
const current = exec.steps[def.name];
|
|
14
18
|
const previous = current && (current.occurrence ?? 0) === occurrence ? current : undefined;
|
|
@@ -29,6 +33,7 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
29
33
|
let inputParsed = false;
|
|
30
34
|
let finalAttempt = attemptsUsed;
|
|
31
35
|
for (let attempt = attemptsUsed + 1; attempt <= maxAttempts; attempt++) {
|
|
36
|
+
assertActive();
|
|
32
37
|
finalAttempt = attempt;
|
|
33
38
|
exec.steps[def.name] = {
|
|
34
39
|
status: 'running',
|
|
@@ -42,6 +47,7 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
42
47
|
attempt,
|
|
43
48
|
maxAttempts,
|
|
44
49
|
});
|
|
50
|
+
assertActive();
|
|
45
51
|
try {
|
|
46
52
|
if (!inputParsed) {
|
|
47
53
|
inputParsed = true;
|
|
@@ -55,6 +61,7 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
55
61
|
catch (error) {
|
|
56
62
|
inputValidationError = new Error(`Input validation failed for "${def.name}": ${describeError(error)}`, { cause: error });
|
|
57
63
|
}
|
|
64
|
+
assertActive();
|
|
58
65
|
}
|
|
59
66
|
if (inputValidationError)
|
|
60
67
|
throw inputValidationError;
|
|
@@ -64,7 +71,9 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
64
71
|
input: validatedInput,
|
|
65
72
|
signal: controller.signal,
|
|
66
73
|
};
|
|
74
|
+
assertActive();
|
|
67
75
|
let result = await runWithTimeout(def.handler(handlerCtx), def.timeout, controller);
|
|
76
|
+
assertActive();
|
|
68
77
|
if (def.outputSchema) {
|
|
69
78
|
try {
|
|
70
79
|
const parsed = def.outputSchema.parse(result);
|
|
@@ -76,7 +85,9 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
76
85
|
cause: e,
|
|
77
86
|
});
|
|
78
87
|
}
|
|
88
|
+
assertActive();
|
|
79
89
|
}
|
|
90
|
+
assertActive();
|
|
80
91
|
exec.steps[def.name] = {
|
|
81
92
|
status: 'completed',
|
|
82
93
|
compensatable: def.compensate !== undefined,
|
|
@@ -96,6 +107,9 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
96
107
|
return;
|
|
97
108
|
}
|
|
98
109
|
catch (err) {
|
|
110
|
+
if (isWorkflowExecutionClosed(err))
|
|
111
|
+
throw err;
|
|
112
|
+
assertActive();
|
|
99
113
|
lastError = err instanceof Error ? err : new Error(describeError(err));
|
|
100
114
|
if (attempt < maxAttempts) {
|
|
101
115
|
emitter?.emitStep('step:retry', exec.id, exec.workflowName, def.name, {
|
|
@@ -103,12 +117,15 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
103
117
|
attempt,
|
|
104
118
|
maxAttempts,
|
|
105
119
|
});
|
|
120
|
+
assertActive();
|
|
106
121
|
await new Promise((r) => clock().setTimeout(() => r(), retryBackoffDelay(attempt)));
|
|
122
|
+
assertActive();
|
|
107
123
|
continue;
|
|
108
124
|
}
|
|
109
125
|
}
|
|
110
126
|
}
|
|
111
127
|
const finalError = lastError ?? new Error('Step failed');
|
|
128
|
+
assertActive();
|
|
112
129
|
exec.steps[def.name] = {
|
|
113
130
|
status: 'failed',
|
|
114
131
|
compensatable: def.compensate !== undefined,
|
|
@@ -135,7 +152,9 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
135
152
|
try {
|
|
136
153
|
updateFn(exec);
|
|
137
154
|
}
|
|
138
|
-
catch {
|
|
155
|
+
catch (error) {
|
|
156
|
+
if (isWorkflowExecutionClosed(error))
|
|
157
|
+
throw error;
|
|
139
158
|
// Deliberately swallowed: `finalError` is thrown below and carries the real cause.
|
|
140
159
|
}
|
|
141
160
|
emitter?.emitStep('step:failed', exec.id, exec.workflowName, def.name, {
|
|
@@ -146,8 +165,12 @@ export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0
|
|
|
146
165
|
throw finalError;
|
|
147
166
|
}
|
|
148
167
|
/** Execute multiple steps in parallel via Promise.allSettled */
|
|
149
|
-
export async function executeParallelSteps(steps, ctx, exec,
|
|
150
|
-
const
|
|
168
|
+
export async function executeParallelSteps(steps, ctx, exec, hooks) {
|
|
169
|
+
const { emitter, updateFn } = hooks;
|
|
170
|
+
const assertActive = hooks.assertActive ?? assertWorkflowActive;
|
|
171
|
+
assertActive();
|
|
172
|
+
const results = await Promise.allSettled(steps.map((def) => executeStepWithRetry(def, ctx, exec, { emitter, updateFn, assertActive })));
|
|
173
|
+
assertActive();
|
|
151
174
|
const failed = results.filter((r) => r.status === 'rejected');
|
|
152
175
|
if (failed.length > 0) {
|
|
153
176
|
const errors = failed.map((r) => r.reason instanceof Error ? r.reason : new Error(describeError(r.reason)));
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
/*
|
|
1
|
+
/* oxlint-disable typescript/no-explicit-any -- runtime definitions intentionally erase accumulated generics */
|
|
2
2
|
/** Workflow handler, definition, and graph types. */
|
|
3
3
|
export {};
|
|
@@ -3,6 +3,7 @@ export interface SubWorkflowRunOptions {
|
|
|
3
3
|
pollIntervalMs?: number;
|
|
4
4
|
maxWaitMs?: number;
|
|
5
5
|
existingChildId?: string;
|
|
6
|
+
assertActive?: () => void;
|
|
6
7
|
}
|
|
7
8
|
/** Start or resume a child execution and poll its durable terminal state. */
|
|
8
9
|
export declare function executeSubWorkflow(workflowName: string, input: unknown, startFn: (name: string, input: unknown) => Promise<{
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { clock } from './clock';
|
|
2
|
+
import { assertWorkflowActive } from './executionFence';
|
|
2
3
|
const MAX_TIMER_DELAY_MS = 2_147_483_647;
|
|
3
4
|
/** Start or resume a child execution and poll its durable terminal state. */
|
|
4
5
|
export async function executeSubWorkflow(workflowName, input, startFn, getFn, options = {}) {
|
|
5
6
|
const pollIntervalMs = options.pollIntervalMs ?? 100;
|
|
6
7
|
const maxWaitMs = options.maxWaitMs ?? 300_000;
|
|
8
|
+
const assertActive = options.assertActive ?? assertWorkflowActive;
|
|
9
|
+
assertActive();
|
|
7
10
|
if (!Number.isFinite(pollIntervalMs) || pollIntervalMs <= 0) {
|
|
8
11
|
throw new Error('Sub-workflow pollIntervalMs must be finite and greater than 0');
|
|
9
12
|
}
|
|
@@ -11,10 +14,13 @@ export async function executeSubWorkflow(workflowName, input, startFn, getFn, op
|
|
|
11
14
|
throw new Error('Sub-workflow maxWaitMs must be finite and greater than 0');
|
|
12
15
|
}
|
|
13
16
|
// Resume a durable child claimed by an earlier entry into this parent node.
|
|
17
|
+
assertActive();
|
|
14
18
|
const existing = options.existingChildId ? getFn(options.existingChildId) : null;
|
|
15
19
|
const handle = existing ? { id: existing.id } : await startFn(workflowName, input);
|
|
20
|
+
assertActive();
|
|
16
21
|
const startedAt = existing?.createdAt ?? getFn(handle.id)?.createdAt ?? clock().now();
|
|
17
22
|
for (;;) {
|
|
23
|
+
assertActive();
|
|
18
24
|
const subExec = getFn(handle.id);
|
|
19
25
|
if (subExec?.state === 'completed') {
|
|
20
26
|
const results = {};
|
|
@@ -36,5 +42,6 @@ export async function executeSubWorkflow(workflowName, input, startFn, getFn, op
|
|
|
36
42
|
throw new Error(`Sub-workflow "${workflowName}" (${handle.id}) timed out after ${maxWaitMs}ms`);
|
|
37
43
|
}
|
|
38
44
|
await new Promise((resolve) => clock().setTimeout(() => resolve(), Math.min(pollIntervalMs, remaining, MAX_TIMER_DELAY_MS)));
|
|
45
|
+
assertActive();
|
|
39
46
|
}
|
|
40
47
|
}
|
|
@@ -18,6 +18,8 @@ export declare const MAX_TIMER_MS = 2147483647;
|
|
|
18
18
|
export interface TimerDeps {
|
|
19
19
|
queue: Queue;
|
|
20
20
|
timers: Map<string, TimerHandle>;
|
|
21
|
+
assertActive: () => void;
|
|
22
|
+
isActive: () => boolean;
|
|
21
23
|
}
|
|
22
24
|
export interface WaitForDeps {
|
|
23
25
|
store: WorkflowStore;
|
|
@@ -25,6 +27,8 @@ export interface WaitForDeps {
|
|
|
25
27
|
advance: (exec: Execution, nextIdx: number, wf: Workflow) => Promise<void>;
|
|
26
28
|
compensate: (exec: Execution, wf: Workflow) => Promise<void>;
|
|
27
29
|
scheduleTimeoutCheck: (execId: string, workflowName: string, nodeIdx: number, ms: number) => void;
|
|
30
|
+
updateFn: (exec: Execution) => void;
|
|
31
|
+
assertActive: () => void;
|
|
28
32
|
}
|
|
29
33
|
/**
|
|
30
34
|
* Arm the timer that re-enters a parked node once its wait budget elapses.
|