bunqueue 2.8.49 → 2.8.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -9
- package/dist/application/backgroundTasks.js +21 -4
- package/dist/application/cleanupTasks.js +7 -3
- package/dist/application/contextFactory.d.ts +6 -3
- package/dist/application/contextFactory.js +8 -0
- package/dist/application/dependencyCompletions.d.ts +53 -0
- package/dist/application/dependencyCompletions.js +123 -0
- package/dist/application/dependencyProcessor.d.ts +5 -0
- package/dist/application/dependencyProcessor.js +25 -13
- package/dist/application/flowFailureRecovery.d.ts +20 -0
- package/dist/application/flowFailureRecovery.js +94 -0
- package/dist/application/flowParentBackpatch.d.ts +27 -0
- package/dist/application/flowParentBackpatch.js +120 -0
- package/dist/application/operations/ack.d.ts +6 -3
- package/dist/application/operations/ack.js +36 -10
- package/dist/application/operations/ackHelpers.d.ts +5 -4
- package/dist/application/operations/ackHelpers.js +4 -10
- package/dist/application/operations/customId.d.ts +3 -0
- package/dist/application/operations/customId.js +10 -0
- package/dist/application/operations/flowPush.d.ts +9 -0
- package/dist/application/operations/flowPush.js +112 -0
- package/dist/application/operations/flowTopologyValidation.d.ts +3 -0
- package/dist/application/operations/flowTopologyValidation.js +101 -0
- package/dist/application/operations/flowValidation.d.ts +3 -0
- package/dist/application/operations/flowValidation.js +166 -0
- package/dist/application/operations/jobManagement.d.ts +3 -0
- package/dist/application/operations/jobManagement.js +10 -4
- package/dist/application/operations/push.d.ts +6 -1
- package/dist/application/operations/push.js +2 -2
- package/dist/application/operations/pushInsert.d.ts +11 -2
- package/dist/application/operations/pushInsert.js +20 -8
- package/dist/application/operations/pushLocks.d.ts +1 -0
- package/dist/application/operations/pushLocks.js +5 -1
- package/dist/application/operations/queryOperations.js +7 -6
- package/dist/application/queueManager.d.ts +7 -0
- package/dist/application/queueManager.js +261 -67
- package/dist/application/types.d.ts +3 -1
- package/dist/cli/commandRouter.d.ts +1 -1
- package/dist/client/flow.d.ts +3 -6
- package/dist/client/flow.js +71 -210
- package/dist/client/flowAtomic.d.ts +9 -0
- package/dist/client/flowAtomic.js +24 -0
- package/dist/client/flowJobCoreMethods.d.ts +28 -0
- package/dist/client/flowJobCoreMethods.js +151 -0
- package/dist/client/flowJobDependencies.d.ts +7 -0
- package/dist/client/flowJobDependencies.js +64 -0
- package/dist/client/flowJobFactory.d.ts +11 -24
- package/dist/client/flowJobFactory.js +91 -360
- package/dist/client/flowJobMoveMethods.d.ts +16 -0
- package/dist/client/flowJobMoveMethods.js +112 -0
- package/dist/client/flowJobTypes.d.ts +29 -0
- package/dist/client/flowJobTypes.js +6 -0
- package/dist/client/flowLegacyPlan.d.ts +16 -0
- package/dist/client/flowLegacyPlan.js +118 -0
- package/dist/client/flowOptions.d.ts +10 -0
- package/dist/client/flowOptions.js +44 -0
- package/dist/client/flowPlan.d.ts +16 -0
- package/dist/client/flowPlan.js +101 -0
- package/dist/client/flowPush.js +4 -1
- package/dist/client/flowReader.d.ts +10 -0
- package/dist/client/flowReader.js +147 -0
- package/dist/client/flowTypes.d.ts +2 -2
- package/dist/client/jobHelpers.js +2 -0
- package/dist/client/workflow/clock.d.ts +3 -1
- package/dist/client/workflow/clock.js +18 -1
- package/dist/client/workflow/compensationChild.d.ts +30 -0
- package/dist/client/workflow/compensationChild.js +65 -0
- package/dist/client/workflow/compensationPass.d.ts +16 -0
- package/dist/client/workflow/compensationPass.js +96 -0
- package/dist/client/workflow/compensationSupport.d.ts +24 -0
- package/dist/client/workflow/compensationSupport.js +57 -0
- package/dist/client/workflow/compensator.d.ts +11 -50
- package/dist/client/workflow/compensator.js +22 -374
- package/dist/client/workflow/definitionGuard.d.ts +11 -0
- package/dist/client/workflow/definitionGuard.js +22 -0
- package/dist/client/workflow/engine.d.ts +2 -2
- package/dist/client/workflow/engine.js +8 -8
- package/dist/client/workflow/eventTypes.d.ts +44 -0
- package/dist/client/workflow/eventTypes.js +2 -0
- package/dist/client/workflow/executionTypes.d.ts +133 -0
- package/dist/client/workflow/executionTypes.js +2 -0
- package/dist/client/workflow/executor.d.ts +4 -17
- package/dist/client/workflow/executor.js +50 -208
- package/dist/client/workflow/executorLifecycle.d.ts +18 -0
- package/dist/client/workflow/executorLifecycle.js +69 -0
- package/dist/client/workflow/executorNodes.d.ts +17 -0
- package/dist/client/workflow/executorNodes.js +138 -0
- package/dist/client/workflow/identity.d.ts +2 -0
- package/dist/client/workflow/identity.js +8 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.d.ts +3 -5
- package/dist/client/workflow/loops.js +25 -23
- package/dist/client/workflow/mapRunner.d.ts +4 -0
- package/dist/client/workflow/mapRunner.js +45 -0
- package/dist/client/workflow/recovery.js +10 -3
- package/dist/client/workflow/runner.d.ts +2 -21
- package/dist/client/workflow/runner.js +39 -104
- package/dist/client/workflow/runnerTiming.d.ts +14 -0
- package/dist/client/workflow/runnerTiming.js +66 -0
- package/dist/client/workflow/stepTypes.d.ts +171 -0
- package/dist/client/workflow/stepTypes.js +3 -0
- package/dist/client/workflow/store.d.ts +17 -17
- package/dist/client/workflow/store.js +65 -100
- package/dist/client/workflow/storeExecutionCodec.d.ts +11 -0
- package/dist/client/workflow/storeExecutionCodec.js +34 -0
- package/dist/client/workflow/storeListing.d.ts +11 -0
- package/dist/client/workflow/storeListing.js +45 -0
- package/dist/client/workflow/storeMaintenance.d.ts +4 -0
- package/dist/client/workflow/storeMaintenance.js +40 -0
- package/dist/client/workflow/storeSignals.d.ts +9 -0
- package/dist/client/workflow/storeSignals.js +38 -2
- package/dist/client/workflow/subWorkflowRunner.d.ts +13 -0
- package/dist/client/workflow/subWorkflowRunner.js +40 -0
- package/dist/client/workflow/types.d.ts +4 -356
- package/dist/client/workflow/types.js +1 -3
- package/dist/client/workflow/waitFor.js +39 -26
- package/dist/client/workflow/workflow.d.ts +16 -59
- package/dist/client/workflow/workflow.js +53 -179
- package/dist/client/workflow/workflowDecisions.d.ts +11 -0
- package/dist/client/workflow/workflowDecisions.js +27 -0
- package/dist/client/workflow/workflowDefinition.d.ts +16 -0
- package/dist/client/workflow/workflowDefinition.js +123 -0
- package/dist/client/workflow/workflowIntrospection.d.ts +5 -0
- package/dist/client/workflow/workflowIntrospection.js +46 -0
- package/dist/client/workflow/workflowValidation.d.ts +44 -0
- package/dist/client/workflow/workflowValidation.js +143 -0
- package/dist/domain/types/command.d.ts +7 -1
- package/dist/domain/types/flow.d.ts +25 -0
- package/dist/domain/types/flow.js +1 -0
- package/dist/infrastructure/persistence/dependencyCompletionSchema.d.ts +6 -0
- package/dist/infrastructure/persistence/dependencyCompletionSchema.js +16 -0
- package/dist/infrastructure/persistence/dependencyCompletionStore.d.ts +38 -0
- package/dist/infrastructure/persistence/dependencyCompletionStore.js +105 -0
- package/dist/infrastructure/persistence/schema.d.ts +2 -5
- package/dist/infrastructure/persistence/schema.js +57 -3
- package/dist/infrastructure/persistence/sqlite.d.ts +35 -0
- package/dist/infrastructure/persistence/sqlite.js +238 -12
- package/dist/infrastructure/persistence/sqliteBatch.js +10 -4
- package/dist/infrastructure/persistence/sqliteSerializer.d.ts +5 -0
- package/dist/infrastructure/persistence/sqliteSerializer.js +28 -5
- package/dist/infrastructure/persistence/statements.d.ts +4 -0
- package/dist/infrastructure/persistence/statements.js +8 -2
- package/dist/infrastructure/server/handlerRoutes.js +3 -0
- package/dist/infrastructure/server/handlers/advanced.js +2 -2
- package/dist/infrastructure/server/handlers/flow.d.ts +7 -0
- package/dist/infrastructure/server/handlers/flow.js +11 -0
- package/dist/infrastructure/server/handlers/index.d.ts +1 -0
- package/dist/infrastructure/server/handlers/index.js +1 -0
- package/package.json +4 -2
|
@@ -1,30 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Public saga-compensation entry points.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* failure, and a step the unwind never reached is recorded as skipped rather
|
|
8
|
-
* than left blank. "Never zero, never two" is not checkable otherwise.
|
|
9
|
-
* - NEVER TWICE. A step that already carries an outcome is not re-run, so an
|
|
10
|
-
* unwind interrupted by a crash resumes where it stopped instead of replaying
|
|
11
|
-
* reversals from the top.
|
|
12
|
-
* - HALT, NOT PLOUGH ON. A compensation that fails definitively stops the chain:
|
|
13
|
-
* continuing past it would undo things whose dependencies are still standing.
|
|
14
|
-
* The remaining eligible steps are marked skipped so the gap is visible.
|
|
15
|
-
* - PIVOT. Past `.pivot()` the saga is committed; those steps are never eligible.
|
|
16
|
-
*
|
|
17
|
-
* `rollbackStatus` on the execution is a separate axis from the failure reason. The
|
|
18
|
-
* rollback is what the engine did *after* the failure, not why it failed, and an
|
|
19
|
-
* operator needs to alert on "the refund never went through" independently of "the
|
|
20
|
-
* payment failed".
|
|
4
|
+
* A pass is checkpointed per step, halts on an uncertain reversal, and is protected
|
|
5
|
+
* from overlapping in-process drivers. The execution mechanics live in
|
|
6
|
+
* compensationPass.ts; record selection lives in compensationSupport.ts.
|
|
21
7
|
*/
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
/** Sentinel
|
|
8
|
+
import { unwind } from './compensationPass';
|
|
9
|
+
import { AbandonedCompensationError } from './compensationChild';
|
|
10
|
+
import { settle, unwindSet } from './compensationSupport';
|
|
11
|
+
import { owesOutcome } from './unwindPlan';
|
|
12
|
+
export { AbandonedCompensationError, CommittedChildError } from './compensationChild';
|
|
13
|
+
/** Sentinel thrown when an execution deliberately parks for a signal. */
|
|
28
14
|
export class WaitForSignalError extends Error {
|
|
29
15
|
event;
|
|
30
16
|
constructor(event) {
|
|
@@ -33,204 +19,42 @@ export class WaitForSignalError extends Error {
|
|
|
33
19
|
}
|
|
34
20
|
}
|
|
35
21
|
/**
|
|
36
|
-
* Executions with an unwind in flight in
|
|
37
|
-
*
|
|
38
|
-
* The "never compensate twice" guard inside the loop reads `record.compensation`
|
|
39
|
-
* from the caller's own snapshot, which was loaded before any concurrent unwind
|
|
40
|
-
* settled anything. Two overlapping unwinds of the same execution therefore both see
|
|
41
|
-
* an unsettled record and both run the handler: a refund issued twice, stock released
|
|
42
|
-
* twice. `recover()` reaches this legitimately (it lists `compensating` runs) and can
|
|
43
|
-
* be called on a live engine that is already driving one.
|
|
22
|
+
* Executions with an unwind in flight in this process.
|
|
44
23
|
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
* Engine instances that somehow reach the same execution. An earlier version of this
|
|
48
|
-
* comment justified the scope by citing a single-instance guarantee in
|
|
49
|
-
* `docs/features/workflow-engine.md`; that statement is not in that file and the
|
|
50
|
-
* citation was wrong.
|
|
24
|
+
* This is not a distributed lock. It prevents two local recovery/operator paths from
|
|
25
|
+
* reading the same unsettled snapshot and issuing the same reversal twice.
|
|
51
26
|
*/
|
|
52
27
|
const inFlight = new Set();
|
|
53
28
|
/** Run compensation handlers in reverse start order for every eligible step. */
|
|
54
|
-
//
|
|
55
|
-
// child's own workflow, and `opts` carries the operator retry; folding them into one bag
|
|
56
|
-
// would hide which of the two a call site is actually asking for, across four call sites
|
|
57
|
-
// that each pass a different combination.
|
|
58
|
-
// biome-ignore lint/complexity/useMaxParams: see above
|
|
29
|
+
// biome-ignore lint/complexity/useMaxParams: registry and operator retry are independent inputs
|
|
59
30
|
export async function runCompensation(exec, wf, store, emitter, workflows, opts = {}) {
|
|
31
|
+
// `failed` + `stuck` is the durable terminal form written by an explicit
|
|
32
|
+
// abandon. No caller, including an ancestor retry, may reopen that decision.
|
|
33
|
+
if (exec.state === 'failed' && exec.rollbackStatus === 'stuck') {
|
|
34
|
+
throw new AbandonedCompensationError(exec.workflowName, exec.id);
|
|
35
|
+
}
|
|
60
36
|
const eligible = unwindSet(exec, wf);
|
|
61
37
|
if (eligible.length === 0) {
|
|
62
38
|
exec.rollbackStatus = 'not-applicable';
|
|
63
|
-
// The run must also become terminal here. On the `runNode` path the caller has
|
|
64
|
-
// already set `failed`, which hid this; `recoverExecutions` has no such caller, so
|
|
65
|
-
// a persisted `compensating` run whose steps no longer resolve (a deploy renamed a
|
|
66
|
-
// step) stayed `compensating` forever, was returned by `listRecoverable()` again,
|
|
67
|
-
// and was re-driven and re-counted at every single startup.
|
|
68
39
|
if (exec.state !== 'failed' && exec.state !== 'completed')
|
|
69
40
|
exec.state = 'failed';
|
|
70
|
-
// Unguarded: a throw here replaces the caller's node error, which is the masking this
|
|
71
|
-
// module fixed elsewhere, but nothing was eligible so there is no rollback to lose, and
|
|
72
|
-
// a `failureReason` from the original failure is already on disk. (On the `runNode`
|
|
73
|
-
// path its caller wrote it; the `recovery.ts` and `rollbackControl.ts` entries have no
|
|
74
|
-
// such caller, and reach here on a row that already carries one.)
|
|
75
|
-
//
|
|
76
|
-
// Unlike the other unguarded writes, this one does NOT self-heal: on the `runNode` path
|
|
77
|
-
// disk ends `failed`, which `listRecoverable()` does not cover, so `rollbackStatus`
|
|
78
|
-
// stays absent instead of `'not-applicable'` for good. That residue is the honest
|
|
79
|
-
// reading anyway, since absent means no unwind was attempted, and guarding the write
|
|
80
|
-
// would swallow a store failure with nothing left to report it.
|
|
81
41
|
store.update(exec);
|
|
82
42
|
return 'ran';
|
|
83
43
|
}
|
|
84
|
-
// Claim before the first await. Losing the claim means another unwind of this same
|
|
85
|
-
// execution is already running, and the caller MUST be told: a parent rolling back
|
|
86
|
-
// a sub-workflow cannot tell "the child finished" from "someone else is driving the
|
|
87
|
-
// child" if both look like a plain return, and it would settle the child's record
|
|
88
|
-
// as `compensated` while that other unwind was still free to fail.
|
|
89
44
|
if (inFlight.has(exec.id))
|
|
90
45
|
return 'claim-lost';
|
|
91
46
|
inFlight.add(exec.id);
|
|
92
47
|
try {
|
|
93
|
-
await unwind({ exec, wf, store, emitter, eligible, workflows, ...opts });
|
|
48
|
+
await unwind({ exec, wf, store, emitter, eligible, workflows, ...opts }, ({ exec, workflow, store, emitter, workflows, retryFailed }) => runCompensation(exec, workflow, store, emitter, workflows, { retryFailed }));
|
|
94
49
|
}
|
|
95
50
|
finally {
|
|
96
51
|
inFlight.delete(exec.id);
|
|
97
52
|
}
|
|
98
53
|
return 'ran';
|
|
99
54
|
}
|
|
100
|
-
async function unwind(pass) {
|
|
101
|
-
const { exec, wf, store, emitter, eligible, workflows, retryFailed } = pass;
|
|
102
|
-
const baseCtx = buildContext(exec);
|
|
103
|
-
let haltedAt = null;
|
|
104
|
-
/** A store write that failed, kept so the caller still learns why the pass stopped. */
|
|
105
|
-
let writeFailure;
|
|
106
|
-
exec.state = 'compensating';
|
|
107
|
-
try {
|
|
108
|
-
store.update(exec);
|
|
109
|
-
}
|
|
110
|
-
catch (err) {
|
|
111
|
-
// The write that happens on EVERY unwind, and it was the last one left unguarded.
|
|
112
|
-
//
|
|
113
|
-
// Unguarded, the throw escaped with the run left `compensating` in memory and, since
|
|
114
|
-
// the caller on the `runNode` path has already persisted `failed`, `failed` on disk
|
|
115
|
-
// with zero reversals run. `listRecoverable()` covers `running|waiting|compensating`,
|
|
116
|
-
// so recovery never revisited it, and both operator exits require
|
|
117
|
-
// `compensation-stuck`, so `resumeCompensation` and `abandonCompensation` both threw.
|
|
118
|
-
// No reversals, no signal, and no way back in
|
|
119
|
-
// (`test/repro-workflow-unwind-write-failure.test.ts`).
|
|
120
|
-
//
|
|
121
|
-
// Nothing has been undone at this point, so parking is the honest state as well as
|
|
122
|
-
// the actionable one: the unwind is owed and has not happened.
|
|
123
|
-
writeFailure = err;
|
|
124
|
-
haltedAt = '(the compensating transition)';
|
|
125
|
-
}
|
|
126
|
-
if (writeFailure === undefined) {
|
|
127
|
-
emitter?.emitWorkflow('workflow:compensating', exec.id, exec.workflowName, 'compensating');
|
|
128
|
-
}
|
|
129
|
-
for (const [name, record] of eligible) {
|
|
130
|
-
// A pass that could not record its own state transition decides nothing.
|
|
131
|
-
//
|
|
132
|
-
// Setting `haltedAt` above is not enough on its own: `decideUnwindAction` checks the
|
|
133
|
-
// vanished-step case FIRST, ahead of the halted check, deliberately, so a renamed step
|
|
134
|
-
// still had `compensation-failed` written and a `compensation:failed` emitted in a pass
|
|
135
|
-
// where no handler ran and the store had already refused everything. The in-memory
|
|
136
|
-
// outcomes then disagreed with a disk that had received nothing, and the event pointed
|
|
137
|
-
// an operator at the wrong cause
|
|
138
|
-
// (`test/repro-workflow-unwind-write-failure.test.ts`).
|
|
139
|
-
if (writeFailure !== undefined)
|
|
140
|
-
break;
|
|
141
|
-
// The DECISION lives in `unwindPlan.ts` as a pure function; this loop only carries
|
|
142
|
-
// it out. Every rollback defect this engine shipped was in these few lines of
|
|
143
|
-
// reasoning rather than in the I/O around them, and while the two were tangled the
|
|
144
|
-
// only way to test a decision was to stand up a database and infer it from side
|
|
145
|
-
// effects.
|
|
146
|
-
const action = decideUnwindAction(wf, name, record, haltedAt !== null, retryFailed);
|
|
147
|
-
if (action.kind === 'stop')
|
|
148
|
-
break;
|
|
149
|
-
if (action.kind === 'skip')
|
|
150
|
-
continue;
|
|
151
|
-
if (action.kind === 'halt-failed') {
|
|
152
|
-
// Its outcome is already recorded and still stands: nothing to write, everything
|
|
153
|
-
// to stop for.
|
|
154
|
-
haltedAt = name;
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (action.kind === 'halt-vanished') {
|
|
158
|
-
haltedAt = name;
|
|
159
|
-
settle(record, 'compensation-failed', action.error);
|
|
160
|
-
emitter?.emitStep('compensation:failed', exec.id, exec.workflowName, name, {
|
|
161
|
-
error: 'step no longer declared',
|
|
162
|
-
});
|
|
163
|
-
continue;
|
|
164
|
-
}
|
|
165
|
-
emitter?.emitStep('compensation:started', exec.id, exec.workflowName, name);
|
|
166
|
-
try {
|
|
167
|
-
if (action.kind === 'unwind-child') {
|
|
168
|
-
await unwindChild(record, store, emitter, workflows, { retryFailed });
|
|
169
|
-
}
|
|
170
|
-
else {
|
|
171
|
-
const def = findStepDef(wf, name);
|
|
172
|
-
await runWithTimeout(def?.compensate?.(compensationContext(exec, baseCtx, name, record)), action.timeoutMs);
|
|
173
|
-
}
|
|
174
|
-
settle(record, 'compensated');
|
|
175
|
-
emitter?.emitStep('compensation:completed', exec.id, exec.workflowName, name);
|
|
176
|
-
}
|
|
177
|
-
catch (err) {
|
|
178
|
-
const message = describeError(err);
|
|
179
|
-
settle(record, 'compensation-failed', message);
|
|
180
|
-
emitter?.emitStep('compensation:failed', exec.id, exec.workflowName, name, {
|
|
181
|
-
error: message,
|
|
182
|
-
});
|
|
183
|
-
haltedAt = name;
|
|
184
|
-
}
|
|
185
|
-
try {
|
|
186
|
-
store.update(exec);
|
|
187
|
-
}
|
|
188
|
-
catch (err) {
|
|
189
|
-
// The outcome just recorded did not reach disk.
|
|
190
|
-
//
|
|
191
|
-
// This write used to sit outside any catch, so a `SQLITE_BUSY` escaped the whole
|
|
192
|
-
// pass and left the run in `compensating`: `resumeCompensation` and
|
|
193
|
-
// `abandonCompensation` both require `compensation-stuck`, so the operator had no
|
|
194
|
-
// exit at all, while `listRecoverable()` DOES return `compensating` and the next
|
|
195
|
-
// `recover()` re-drove the pass and ran the unpersisted reversal a second time
|
|
196
|
-
// (`test/repro-workflow-unwind-write-failure.test.ts`).
|
|
197
|
-
//
|
|
198
|
-
// Stop here rather than carrying on. Every further reversal would have the same
|
|
199
|
-
// problem persisting its outcome, and an unrecorded reversal is one that runs
|
|
200
|
-
// again: "never twice" lost one handler at a time. The record itself is left
|
|
201
|
-
// exactly as it is — overwriting a reversal that provably succeeded with a write
|
|
202
|
-
// error would destroy the outcome this module works hardest to protect.
|
|
203
|
-
writeFailure = err;
|
|
204
|
-
haltedAt = name;
|
|
205
|
-
break;
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
if (haltedAt !== null) {
|
|
209
|
-
exec.state = 'compensation-stuck';
|
|
210
|
-
exec.rollbackStatus = 'stuck';
|
|
211
|
-
}
|
|
212
|
-
else {
|
|
213
|
-
exec.state = 'failed';
|
|
214
|
-
exec.rollbackStatus = 'completed';
|
|
215
|
-
}
|
|
216
|
-
try {
|
|
217
|
-
store.update(exec);
|
|
218
|
-
}
|
|
219
|
-
catch (err) {
|
|
220
|
-
// If the store is what broke, parking cannot be persisted either. The in-memory run
|
|
221
|
-
// is still left parked so a caller holding it sees an actionable state, and the
|
|
222
|
-
// ORIGINAL write error is the one that propagates: it is the cause, and this second
|
|
223
|
-
// failure is its consequence.
|
|
224
|
-
if (writeFailure === undefined)
|
|
225
|
-
throw err;
|
|
226
|
-
}
|
|
227
|
-
if (writeFailure !== undefined)
|
|
228
|
-
throw writeFailure;
|
|
229
|
-
}
|
|
230
55
|
/**
|
|
231
|
-
* Give up on a parked unwind
|
|
232
|
-
*
|
|
233
|
-
* outcome per eligible step" is finally discharged.
|
|
56
|
+
* Give up on a parked unwind and record an explicit skipped outcome for every
|
|
57
|
+
* remaining eligible step.
|
|
234
58
|
*/
|
|
235
59
|
export function abandonCompensation(exec, wf, store, emitter) {
|
|
236
60
|
for (const [name, record] of unwindSet(exec, wf)) {
|
|
@@ -246,179 +70,3 @@ export function abandonCompensation(exec, wf, store, emitter) {
|
|
|
246
70
|
exec.rollbackStatus = 'stuck';
|
|
247
71
|
store.update(exec);
|
|
248
72
|
}
|
|
249
|
-
/**
|
|
250
|
-
* Steps eligible for rollback, in the order they must be undone.
|
|
251
|
-
*
|
|
252
|
-
* `exec.steps` is written when a step STARTS, so reversing insertion order gives
|
|
253
|
-
* reverse start order — deterministic even when parallel steps finish out of
|
|
254
|
-
* sequence, which completion order is not.
|
|
255
|
-
*
|
|
256
|
-
* A `failed` step is included on purpose: it is the one most likely to need undoing,
|
|
257
|
-
* because a charge that reached the provider and then lost its response is recorded
|
|
258
|
-
* failed while the money has already moved.
|
|
259
|
-
*/
|
|
260
|
-
function unwindSet(exec, wf) {
|
|
261
|
-
// The saga committed at the pivot. Backward recovery is off for the WHOLE run,
|
|
262
|
-
// including the steps before it — unwinding them now would contradict work the
|
|
263
|
-
// outside world has already been told about.
|
|
264
|
-
if (exec.committedAt !== undefined)
|
|
265
|
-
return [];
|
|
266
|
-
return Object.entries(exec.steps)
|
|
267
|
-
.filter(([name, s]) => {
|
|
268
|
-
if (name.startsWith('__'))
|
|
269
|
-
return false; // engine bookkeeping, not a user step
|
|
270
|
-
if (s.status !== 'completed' && s.status !== 'failed')
|
|
271
|
-
return false;
|
|
272
|
-
// `doUntil`/`doWhile` write BOTH a per-iteration record (`turn:0`, `turn:1`, ...)
|
|
273
|
-
// and a bare `turn` mirroring the last iteration, so downstream steps and the
|
|
274
|
-
// loop condition can read it by name. Compensating both would undo the final
|
|
275
|
-
// iteration twice. The indexed records are the real history; the mirror is not.
|
|
276
|
-
//
|
|
277
|
-
// Gated on the name actually being a loop body. Testing only for a `${name}:0`
|
|
278
|
-
// sibling dropped a plain step called `foo` from the unwind whenever an
|
|
279
|
-
// unrelated step called `foo:0` existed: `foo` finished with NO outcome while
|
|
280
|
-
// the run still reported `rollbackStatus: 'completed'`, which is the one
|
|
281
|
-
// reading an operator alerting on rollback failure must be able to trust.
|
|
282
|
-
if (isLoopBody(wf, name) && exec.steps[`${name}:0`] !== undefined)
|
|
283
|
-
return false;
|
|
284
|
-
// A nested workflow is eligible through its child handle rather than a step
|
|
285
|
-
// definition: rolling it back means running the child's own unwind.
|
|
286
|
-
if (name.startsWith('sub:'))
|
|
287
|
-
return s.childExecutionId !== undefined;
|
|
288
|
-
if (findStepDef(wf, name) !== null)
|
|
289
|
-
return true;
|
|
290
|
-
// The definition is gone. Two kinds of record are still owed a reversal, and
|
|
291
|
-
// dropping either is how a parked run whose step a deploy renamed reported
|
|
292
|
-
// `rollbackStatus: 'completed'` over work nobody undid: nothing was left to halt
|
|
293
|
-
// on, so the unwind reached the end and called itself clean.
|
|
294
|
-
//
|
|
295
|
-
// One already carries an outcome, so it was part of the set when the earlier
|
|
296
|
-
// attempt ran. The other carries none but ran WITH a handler, which is the
|
|
297
|
-
// likelier case, because the unwind simply had not reached it yet
|
|
298
|
-
// (`test/repro-workflow-nested-and-settled.test.ts`).
|
|
299
|
-
return s.compensation !== undefined || s.compensatable === true;
|
|
300
|
-
})
|
|
301
|
-
.reverse();
|
|
302
|
-
}
|
|
303
|
-
/**
|
|
304
|
-
* Roll back a nested workflow by running the CHILD's own unwind.
|
|
305
|
-
*
|
|
306
|
-
* A child that succeeded before its parent failed used to be left untouched: every
|
|
307
|
-
* resource it created stayed live with nothing pointing at it. Its compensation is
|
|
308
|
-
* not a handler the parent can call — it is the child's whole rollback.
|
|
309
|
-
*
|
|
310
|
-
* If the child parks (`compensation-stuck`), the parent inherits it: this throws, so
|
|
311
|
-
* the parent's own unwind halts and parks too. A half-rolled-back child is not
|
|
312
|
-
* something the parent can paper over.
|
|
313
|
-
*/
|
|
314
|
-
async function unwindChild(record, store, emitter, workflows,
|
|
315
|
-
/**
|
|
316
|
-
* Forwarded from the parent's own pass.
|
|
317
|
-
*
|
|
318
|
-
* Without it `resumeCompensation(parentId)` on a nested saga was a silent no-op that
|
|
319
|
-
* resolved successfully: the retry stopped at the parent, the child halted on its own
|
|
320
|
-
* `compensation-failed`, `unwindChild` threw, and the parent re-parked. The operator
|
|
321
|
-
* got a clean return for an action that did nothing, which is the same anti-pattern
|
|
322
|
-
* the signal API was just fixed for, and the guide names this exact call as the way
|
|
323
|
-
* out of a parent that inherited its child's park.
|
|
324
|
-
*/
|
|
325
|
-
opts = {}) {
|
|
326
|
-
const childId = record.childExecutionId;
|
|
327
|
-
if (!childId)
|
|
328
|
-
throw new Error('sub-workflow record carries no child execution id');
|
|
329
|
-
if (!workflows)
|
|
330
|
-
throw new Error('workflow registry unavailable to unwind a sub-workflow');
|
|
331
|
-
const child = store.get(childId);
|
|
332
|
-
if (!child)
|
|
333
|
-
throw new Error(`child execution "${childId}" not found`);
|
|
334
|
-
const childWf = workflows.get(child.workflowName);
|
|
335
|
-
if (!childWf)
|
|
336
|
-
throw new Error(`child workflow "${child.workflowName}" is not registered`);
|
|
337
|
-
// A child that has NOT stopped is never rolled back. Running its reversals while it is
|
|
338
|
-
// still stepping forward puts two writers on one row: the child's own `advance()`
|
|
339
|
-
// overwrites the compensation from its stale snapshot, compensate handlers interleave
|
|
340
|
-
// with forward steps, and the child can go on to reach `completed` with its reversals
|
|
341
|
-
// already executed.
|
|
342
|
-
//
|
|
343
|
-
// The parent reaches this legitimately. `executeSubWorkflow` gives up after a hardcoded
|
|
344
|
-
// 300 second ceiling, which the guide documents as a supported case, and the parent
|
|
345
|
-
// then settles its `sub:` record `failed` while the child is very much alive. Refusing
|
|
346
|
-
// here rather than at the settle keeps the record truthful: the parent's node DID fail.
|
|
347
|
-
// The parent still does not claim a clean rollback, it parks for an operator, which is
|
|
348
|
-
// the correct outcome for "the child may or may not still be changing the world".
|
|
349
|
-
if (isLive(child.state)) {
|
|
350
|
-
throw new Error(`child "${child.workflowName}" (${childId}) is still ${child.state}; it cannot be rolled back until it stops`);
|
|
351
|
-
}
|
|
352
|
-
const outcome = await runCompensation(child, childWf, store, emitter, workflows, opts);
|
|
353
|
-
// Another driver holds this child's unwind. It may still fail, so recording the
|
|
354
|
-
// parent's `sub:` record as compensated here would claim a rollback whose result is
|
|
355
|
-
// not yet known, and nothing would ever correct it.
|
|
356
|
-
if (outcome === 'claim-lost') {
|
|
357
|
-
throw new Error(`child "${child.workflowName}" (${childId}) is being rolled back by another driver; outcome unknown`);
|
|
358
|
-
}
|
|
359
|
-
if (child.rollbackStatus === 'stuck') {
|
|
360
|
-
throw new Error(`child "${child.workflowName}" (${childId}) parked mid-rollback`);
|
|
361
|
-
}
|
|
362
|
-
// A child past its own .pivot() is committed: nothing of it was undone. Reporting
|
|
363
|
-
// the parent's `sub:` record as 'compensated' would claim a rollback that provably
|
|
364
|
-
// did not happen, and an operator reading the parent would see a clean unwind over
|
|
365
|
-
// a child that still holds every effect it created.
|
|
366
|
-
if (child.rollbackStatus === 'not-applicable' && child.committedAt !== undefined) {
|
|
367
|
-
throw new CommittedChildError(child.workflowName, childId);
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
/**
|
|
371
|
-
* A sub-workflow that passed its own pivot cannot be rolled back. Distinct from a
|
|
372
|
-
* handler failure so the parent's record carries the reason rather than a generic
|
|
373
|
-
* error, and so the parent parks (operator decision) instead of silently claiming
|
|
374
|
-
* success.
|
|
375
|
-
*/
|
|
376
|
-
export class CommittedChildError extends Error {
|
|
377
|
-
constructor(workflowName, executionId) {
|
|
378
|
-
super(`child "${workflowName}" (${executionId}) is committed past its pivot; nothing was rolled back`);
|
|
379
|
-
this.name = 'CommittedChildError';
|
|
380
|
-
}
|
|
381
|
-
}
|
|
382
|
-
/** Is `name` the body of a doUntil/doWhile loop, i.e. a step that writes a mirror record? */
|
|
383
|
-
function isLoopBody(wf, name) {
|
|
384
|
-
return wf.nodes.some((node) => (node.type === 'doUntil' || node.type === 'doWhile') &&
|
|
385
|
-
node.def.steps.some((s) => s.name === name));
|
|
386
|
-
}
|
|
387
|
-
function settle(record, status, error) {
|
|
388
|
-
record.compensation = { status, at: clock().now(), ...(error ? { error } : {}) };
|
|
389
|
-
}
|
|
390
|
-
/**
|
|
391
|
-
* Context handed to a compensate handler.
|
|
392
|
-
*
|
|
393
|
-
* Beyond the run's own data it carries the identity needed to reconcile: this
|
|
394
|
-
* rollback's key, and the key the forward step used. When the forward outcome is in
|
|
395
|
-
* doubt the handler can ask the provider what actually happened by key rather than
|
|
396
|
-
* depending on an output that may never have been persisted.
|
|
397
|
-
*/
|
|
398
|
-
function compensationContext(exec, baseCtx, name, record) {
|
|
399
|
-
const occurrence = record.occurrence ?? 0;
|
|
400
|
-
// Rebind the step's bare name to THIS record's own result.
|
|
401
|
-
//
|
|
402
|
-
// `buildContext` exposes results under bare names, and a loop's bare name mirrors
|
|
403
|
-
// its LAST iteration. Without this, every iteration's compensate handler reads
|
|
404
|
-
// `ctx.steps.charge` and sees the final charge: three charges produced three
|
|
405
|
-
// refunds of the third one. `base` is the name without the `:index` suffix, so a
|
|
406
|
-
// non-loop step rebinds to itself, which is a no-op.
|
|
407
|
-
// Only a numeric suffix is an iteration. Stripping at the last colon rebound
|
|
408
|
-
// `payment:charge` onto `payment`, feeding that handler a sibling step's result.
|
|
409
|
-
const base = loopBaseName(name);
|
|
410
|
-
const ctx = {
|
|
411
|
-
...baseCtx,
|
|
412
|
-
steps: { ...baseCtx.steps, [base]: record.result, [name]: record.result },
|
|
413
|
-
idempotencyKey: idempotencyKey(exec.id, name, occurrence, 'compensate'),
|
|
414
|
-
forwardIdempotencyKey: record.idempotencyKey,
|
|
415
|
-
};
|
|
416
|
-
// For forEach iterations, restore that iteration's __item/__index so the handler
|
|
417
|
-
// knows exactly which item it is rolling back.
|
|
418
|
-
if (record.loopIndex === undefined)
|
|
419
|
-
return ctx;
|
|
420
|
-
return {
|
|
421
|
-
...ctx,
|
|
422
|
-
steps: { ...ctx.steps, __item: record.loopItem, __index: record.loopIndex },
|
|
423
|
-
};
|
|
424
|
-
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** Runtime guard binding persisted executions to one sealed workflow graph. */
|
|
2
|
+
import type { Execution } from './types';
|
|
3
|
+
import type { Workflow } from './workflow';
|
|
4
|
+
export declare class WorkflowDefinitionMismatchError extends Error {
|
|
5
|
+
constructor(executionId: string, workflowName: string);
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Legacy rows have no hash; bind them once to the currently registered graph. New rows
|
|
9
|
+
* always arrive pre-bound by startExecution.
|
|
10
|
+
*/
|
|
11
|
+
export declare function bindExecutionDefinition(exec: Execution, workflow: Workflow, update: (exec: Execution) => void): void;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Runtime guard binding persisted executions to one sealed workflow graph. */
|
|
2
|
+
export class WorkflowDefinitionMismatchError extends Error {
|
|
3
|
+
constructor(executionId, workflowName) {
|
|
4
|
+
super(`Workflow definition mismatch for active execution "${executionId}" of "${workflowName}"`);
|
|
5
|
+
this.name = 'WorkflowDefinitionMismatchError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Legacy rows have no hash; bind them once to the currently registered graph. New rows
|
|
10
|
+
* always arrive pre-bound by startExecution.
|
|
11
|
+
*/
|
|
12
|
+
export function bindExecutionDefinition(exec, workflow, update) {
|
|
13
|
+
const registeredHash = workflow.seal();
|
|
14
|
+
if (exec.definitionHash === undefined) {
|
|
15
|
+
exec.definitionHash = registeredHash;
|
|
16
|
+
update(exec);
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
if (exec.definitionHash !== registeredHash) {
|
|
20
|
+
throw new WorkflowDefinitionMismatchError(exec.id, exec.workflowName);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Manages lifecycle of internal Queue, Worker, and Store.
|
|
4
4
|
*/
|
|
5
5
|
import type { Workflow } from './workflow';
|
|
6
|
-
import type { EngineOptions, RunHandle, Execution, ExecutionState, RecoverResult, WorkflowEventType, WorkflowEventListener } from './types';
|
|
6
|
+
import type { EngineOptions, RunHandle, Execution, ExecutionState, ExecutionListOptions, RecoverResult, WorkflowEventType, WorkflowEventListener } from './types';
|
|
7
7
|
export declare class Engine {
|
|
8
8
|
private readonly queue;
|
|
9
9
|
private readonly worker;
|
|
@@ -18,7 +18,7 @@ export declare class Engine {
|
|
|
18
18
|
/** Get execution state by ID */
|
|
19
19
|
getExecution(id: string): Execution | null;
|
|
20
20
|
/** List executions with optional filters */
|
|
21
|
-
listExecutions(workflowName?: string, state?: ExecutionState): Execution[];
|
|
21
|
+
listExecutions(workflowName?: string, state?: ExecutionState, options?: ExecutionListOptions): Execution[];
|
|
22
22
|
/**
|
|
23
23
|
* Retry the compensation that parked a `compensation-stuck` run and continue the
|
|
24
24
|
* unwind. Use once the cause of the failed reversal has been fixed.
|
|
@@ -29,7 +29,7 @@ export class Engine {
|
|
|
29
29
|
this.executor = new WorkflowExecutor(this.store, this.queue, this.emitter);
|
|
30
30
|
this.worker = new Worker(queueName, async (job) => {
|
|
31
31
|
const data = job.data;
|
|
32
|
-
return this.executor.processStep(data);
|
|
32
|
+
return await this.executor.processStep(data);
|
|
33
33
|
}, {
|
|
34
34
|
connection: opts.connection,
|
|
35
35
|
embedded: opts.embedded,
|
|
@@ -44,22 +44,22 @@ export class Engine {
|
|
|
44
44
|
}
|
|
45
45
|
/** Start a new workflow execution */
|
|
46
46
|
async start(workflowName, input) {
|
|
47
|
-
return this.executor.start(workflowName, input);
|
|
47
|
+
return await this.executor.start(workflowName, input);
|
|
48
48
|
}
|
|
49
49
|
/** Get execution state by ID */
|
|
50
50
|
getExecution(id) {
|
|
51
51
|
return this.executor.getExecution(id);
|
|
52
52
|
}
|
|
53
53
|
/** List executions with optional filters */
|
|
54
|
-
listExecutions(workflowName, state) {
|
|
55
|
-
return this.executor.listExecutions(workflowName, state);
|
|
54
|
+
listExecutions(workflowName, state, options) {
|
|
55
|
+
return this.executor.listExecutions(workflowName, state, options);
|
|
56
56
|
}
|
|
57
57
|
/**
|
|
58
58
|
* Retry the compensation that parked a `compensation-stuck` run and continue the
|
|
59
59
|
* unwind. Use once the cause of the failed reversal has been fixed.
|
|
60
60
|
*/
|
|
61
61
|
async resumeCompensation(executionId) {
|
|
62
|
-
return this.executor.resumeCompensation(executionId);
|
|
62
|
+
return await this.executor.resumeCompensation(executionId);
|
|
63
63
|
}
|
|
64
64
|
/**
|
|
65
65
|
* Abandon a parked unwind: the steps still un-compensated are recorded as skipped
|
|
@@ -73,11 +73,11 @@ export class Engine {
|
|
|
73
73
|
* blows up. Matching the sibling costs nothing while the API is experimental.
|
|
74
74
|
*/
|
|
75
75
|
async abandonCompensation(executionId) {
|
|
76
|
-
this.executor.abandonCompensation(executionId);
|
|
76
|
+
await Promise.resolve(this.executor.abandonCompensation(executionId));
|
|
77
77
|
}
|
|
78
78
|
/** Send a signal to a waiting execution */
|
|
79
79
|
async signal(executionId, event, payload) {
|
|
80
|
-
return this.executor.signal(executionId, event, payload);
|
|
80
|
+
return await this.executor.signal(executionId, event, payload);
|
|
81
81
|
}
|
|
82
82
|
/**
|
|
83
83
|
* Recover orphaned executions after a crash/restart.
|
|
@@ -86,7 +86,7 @@ export class Engine {
|
|
|
86
86
|
* - 'compensating' executions: compensation re-run (handlers must be idempotent)
|
|
87
87
|
*/
|
|
88
88
|
async recover() {
|
|
89
|
-
return this.executor.recover();
|
|
89
|
+
return await this.executor.recover();
|
|
90
90
|
}
|
|
91
91
|
// ============ Observability ============
|
|
92
92
|
/** Subscribe to a specific workflow event type */
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Workflow event and engine configuration types. */
|
|
2
|
+
import type { ConnectionOptions } from '../types';
|
|
3
|
+
import type { ExecutionState } from './executionTypes';
|
|
4
|
+
/** All workflow event types */
|
|
5
|
+
export type WorkflowEventType = 'step:started' | 'step:completed' | 'step:failed' | 'step:retry' | 'workflow:started' | 'workflow:completed' | 'workflow:failed' | 'workflow:compensating' | 'workflow:waiting' | 'signal:received' | 'signal:timeout' | 'compensation:started' | 'compensation:completed' | 'compensation:failed' | 'compensation:skipped';
|
|
6
|
+
/** Base event payload */
|
|
7
|
+
export interface WorkflowEvent {
|
|
8
|
+
type: WorkflowEventType;
|
|
9
|
+
executionId: string;
|
|
10
|
+
workflowName: string;
|
|
11
|
+
timestamp: number;
|
|
12
|
+
}
|
|
13
|
+
/** Step-level event payload */
|
|
14
|
+
export interface StepEvent extends WorkflowEvent {
|
|
15
|
+
stepName: string;
|
|
16
|
+
result?: unknown;
|
|
17
|
+
error?: string;
|
|
18
|
+
attempt?: number;
|
|
19
|
+
maxAttempts?: number;
|
|
20
|
+
}
|
|
21
|
+
/** Workflow lifecycle event payload */
|
|
22
|
+
export interface WorkflowLifecycleEvent extends WorkflowEvent {
|
|
23
|
+
state: ExecutionState;
|
|
24
|
+
input?: unknown;
|
|
25
|
+
}
|
|
26
|
+
/** Signal event payload */
|
|
27
|
+
export interface SignalEvent extends WorkflowEvent {
|
|
28
|
+
event: string;
|
|
29
|
+
payload?: unknown;
|
|
30
|
+
}
|
|
31
|
+
/** Event listener function */
|
|
32
|
+
export type WorkflowEventListener = (event: WorkflowEvent | StepEvent | WorkflowLifecycleEvent | SignalEvent) => void;
|
|
33
|
+
/** Engine configuration */
|
|
34
|
+
export interface EngineOptions {
|
|
35
|
+
embedded?: boolean;
|
|
36
|
+
dataPath?: string;
|
|
37
|
+
connection?: ConnectionOptions;
|
|
38
|
+
/** Internal queue name (default: __wf:steps) */
|
|
39
|
+
queueName?: string;
|
|
40
|
+
/** Worker concurrency (default: 5) */
|
|
41
|
+
concurrency?: number;
|
|
42
|
+
/** Global event listener for observability */
|
|
43
|
+
onEvent?: WorkflowEventListener;
|
|
44
|
+
}
|