bunqueue 2.8.46 → 2.8.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/workflow/admission.d.ts +45 -0
- package/dist/client/workflow/admission.js +50 -0
- package/dist/client/workflow/clock.d.ts +58 -0
- package/dist/client/workflow/clock.js +104 -0
- package/dist/client/workflow/compensator.d.ts +56 -3
- package/dist/client/workflow/compensator.js +406 -27
- package/dist/client/workflow/emitter.d.ts +1 -1
- package/dist/client/workflow/emitter.js +4 -3
- package/dist/client/workflow/engine.d.ts +17 -0
- package/dist/client/workflow/engine.js +25 -0
- package/dist/client/workflow/executor.d.ts +40 -5
- package/dist/client/workflow/executor.js +227 -83
- package/dist/client/workflow/identity.d.ts +46 -0
- package/dist/client/workflow/identity.js +93 -0
- package/dist/client/workflow/index.d.ts +1 -1
- package/dist/client/workflow/loops.js +147 -14
- package/dist/client/workflow/recovery.d.ts +10 -1
- package/dist/client/workflow/recovery.js +36 -9
- package/dist/client/workflow/rollbackControl.d.ts +36 -0
- package/dist/client/workflow/rollbackControl.js +51 -0
- package/dist/client/workflow/runner.d.ts +22 -2
- package/dist/client/workflow/runner.js +125 -27
- package/dist/client/workflow/store.d.ts +64 -4
- package/dist/client/workflow/store.js +123 -30
- package/dist/client/workflow/storeCodec.d.ts +7 -0
- package/dist/client/workflow/storeCodec.js +16 -0
- package/dist/client/workflow/storeSignals.d.ts +61 -0
- package/dist/client/workflow/storeSignals.js +118 -0
- package/dist/client/workflow/types.d.ts +147 -4
- package/dist/client/workflow/unwindPlan.d.ts +87 -0
- package/dist/client/workflow/unwindPlan.js +142 -0
- package/dist/client/workflow/waitFor.d.ts +52 -0
- package/dist/client/workflow/waitFor.js +137 -0
- package/dist/client/workflow/workflow.d.ts +71 -1
- package/dist/client/workflow/workflow.js +184 -14
- package/package.json +16 -6
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* WorkflowRunner - Step execution with retry, parallel execution, sub-workflow dispatch
|
|
3
3
|
*/
|
|
4
|
+
import { idempotencyKey, isIterationOf, describeError } from './identity';
|
|
5
|
+
import { clock } from './clock';
|
|
4
6
|
/** Exponential backoff with jitter */
|
|
5
7
|
function backoffDelay(attempt, baseMs = 500, maxMs = 30_000) {
|
|
6
8
|
const delay = Math.min(baseMs * 2 ** (attempt - 1), maxMs);
|
|
7
|
-
const jitter = delay * 0.5 *
|
|
9
|
+
const jitter = delay * 0.5 * clock().random();
|
|
8
10
|
return delay + jitter;
|
|
9
11
|
}
|
|
10
12
|
/** Run a promise with a timeout */
|
|
@@ -14,28 +16,39 @@ export function runWithTimeout(promise, timeoutMs) {
|
|
|
14
16
|
if (timeoutMs <= 0)
|
|
15
17
|
return promise;
|
|
16
18
|
return new Promise((resolve, reject) => {
|
|
17
|
-
const timer = setTimeout(() => {
|
|
19
|
+
const timer = clock().setTimeout(() => {
|
|
18
20
|
reject(new Error(`Step timed out after ${timeoutMs}ms`));
|
|
19
21
|
}, timeoutMs);
|
|
20
22
|
promise.then((v) => {
|
|
21
|
-
clearTimeout(timer);
|
|
23
|
+
clock().clearTimeout(timer);
|
|
22
24
|
resolve(v);
|
|
23
25
|
}, (e) => {
|
|
24
|
-
clearTimeout(timer);
|
|
25
|
-
|
|
26
|
+
clock().clearTimeout(timer);
|
|
27
|
+
// `describeError`, not `String`: this wrapper runs BEFORE the compensator's own
|
|
28
|
+
// catch, so converting here with `String` is what destroyed a structured throw
|
|
29
|
+
// into `[object Object]` no matter how carefully the catch handled it.
|
|
30
|
+
reject(e instanceof Error ? e : new Error(describeError(e)));
|
|
26
31
|
});
|
|
27
32
|
});
|
|
28
33
|
}
|
|
29
34
|
/** Execute a step with retry logic and exponential backoff */
|
|
30
|
-
export async function executeStepWithRetry(def, ctx, exec,
|
|
35
|
+
export async function executeStepWithRetry(def, ctx, exec, hooks, occurrence = 0) {
|
|
36
|
+
const { emitter, updateFn } = hooks;
|
|
31
37
|
const maxAttempts = def.retry;
|
|
32
38
|
let lastError;
|
|
39
|
+
// Derived once, outside the retry loop, and persisted with the START record: a
|
|
40
|
+
// rollback for a step whose outcome is unknown needs this key to reconcile, and by
|
|
41
|
+
// then the body may never have reached the point of writing anything.
|
|
42
|
+
const forwardKey = idempotencyKey(exec.id, def.name, occurrence, 'forward');
|
|
43
|
+
const stepCtx = { ...ctx, idempotencyKey: forwardKey };
|
|
33
44
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
34
45
|
const prev = exec.steps[def.name];
|
|
35
46
|
exec.steps[def.name] = {
|
|
36
47
|
status: 'running',
|
|
37
|
-
startedAt: prev?.startedAt ??
|
|
48
|
+
startedAt: prev?.startedAt ?? clock().now(),
|
|
38
49
|
attempts: attempt,
|
|
50
|
+
idempotencyKey: forwardKey,
|
|
51
|
+
occurrence,
|
|
39
52
|
};
|
|
40
53
|
updateFn(exec);
|
|
41
54
|
emitter?.emitStep('step:started', exec.id, exec.workflowName, def.name, {
|
|
@@ -43,29 +56,50 @@ export async function executeStepWithRetry(def, ctx, exec, emitter, updateFn) {
|
|
|
43
56
|
maxAttempts,
|
|
44
57
|
});
|
|
45
58
|
try {
|
|
59
|
+
let input = stepCtx.input;
|
|
46
60
|
if (def.inputSchema) {
|
|
47
61
|
try {
|
|
48
|
-
|
|
62
|
+
// The RETURN VALUE matters. `parse()` is the coercing entry point of every
|
|
63
|
+
// schema library the docs point at: `.default()` fills gaps, `.transform()`
|
|
64
|
+
// rewrites, `z.coerce.date()` builds a Date from a string. Calling it purely
|
|
65
|
+
// for its throw validated the shape and silently dropped every coercion, so
|
|
66
|
+
// a step declaring `.default('EUR')` ran with no currency at all
|
|
67
|
+
// (`test/repro-workflow-gate-and-schema.test.ts`). A validator that returns
|
|
68
|
+
// nothing is still supported: `undefined` means "I only assert", so the
|
|
69
|
+
// original value is kept rather than blanked.
|
|
70
|
+
const parsed = def.inputSchema.parse(stepCtx.input);
|
|
71
|
+
if (parsed !== undefined)
|
|
72
|
+
input = parsed;
|
|
49
73
|
}
|
|
50
74
|
catch (e) {
|
|
51
|
-
throw new Error(`Input validation failed for "${def.name}": ${
|
|
75
|
+
throw new Error(`Input validation failed for "${def.name}": ${describeError(e)}`, {
|
|
76
|
+
cause: e,
|
|
77
|
+
});
|
|
52
78
|
}
|
|
53
79
|
}
|
|
54
|
-
const
|
|
80
|
+
const handlerCtx = input === stepCtx.input ? stepCtx : { ...stepCtx, input };
|
|
81
|
+
let result = await runWithTimeout(def.handler(handlerCtx), def.timeout);
|
|
55
82
|
if (def.outputSchema) {
|
|
56
83
|
try {
|
|
57
|
-
def.outputSchema.parse(result);
|
|
84
|
+
const parsed = def.outputSchema.parse(result);
|
|
85
|
+
if (parsed !== undefined)
|
|
86
|
+
result = parsed;
|
|
58
87
|
}
|
|
59
88
|
catch (e) {
|
|
60
|
-
throw new Error(`Output validation failed for "${def.name}": ${
|
|
89
|
+
throw new Error(`Output validation failed for "${def.name}": ${describeError(e)}`, {
|
|
90
|
+
cause: e,
|
|
91
|
+
});
|
|
61
92
|
}
|
|
62
93
|
}
|
|
63
94
|
exec.steps[def.name] = {
|
|
64
95
|
status: 'completed',
|
|
96
|
+
compensatable: def.compensate !== undefined,
|
|
65
97
|
result,
|
|
66
98
|
startedAt: exec.steps[def.name].startedAt,
|
|
67
|
-
completedAt:
|
|
99
|
+
completedAt: clock().now(),
|
|
68
100
|
attempts: attempt,
|
|
101
|
+
idempotencyKey: forwardKey,
|
|
102
|
+
occurrence,
|
|
69
103
|
};
|
|
70
104
|
updateFn(exec);
|
|
71
105
|
emitter?.emitStep('step:completed', exec.id, exec.workflowName, def.name, {
|
|
@@ -76,14 +110,14 @@ export async function executeStepWithRetry(def, ctx, exec, emitter, updateFn) {
|
|
|
76
110
|
return;
|
|
77
111
|
}
|
|
78
112
|
catch (err) {
|
|
79
|
-
lastError = err instanceof Error ? err : new Error(
|
|
113
|
+
lastError = err instanceof Error ? err : new Error(describeError(err));
|
|
80
114
|
if (attempt < maxAttempts) {
|
|
81
115
|
emitter?.emitStep('step:retry', exec.id, exec.workflowName, def.name, {
|
|
82
116
|
error: lastError.message,
|
|
83
117
|
attempt,
|
|
84
118
|
maxAttempts,
|
|
85
119
|
});
|
|
86
|
-
await new Promise((r) => setTimeout(r, backoffDelay(attempt)));
|
|
120
|
+
await new Promise((r) => clock().setTimeout(() => r(), backoffDelay(attempt)));
|
|
87
121
|
continue;
|
|
88
122
|
}
|
|
89
123
|
}
|
|
@@ -91,12 +125,33 @@ export async function executeStepWithRetry(def, ctx, exec, emitter, updateFn) {
|
|
|
91
125
|
const finalError = lastError ?? new Error('Step failed');
|
|
92
126
|
exec.steps[def.name] = {
|
|
93
127
|
status: 'failed',
|
|
128
|
+
compensatable: def.compensate !== undefined,
|
|
94
129
|
error: String(finalError),
|
|
95
130
|
startedAt: exec.steps[def.name].startedAt,
|
|
96
|
-
completedAt:
|
|
131
|
+
completedAt: clock().now(),
|
|
97
132
|
attempts: maxAttempts,
|
|
133
|
+
idempotencyKey: forwardKey,
|
|
134
|
+
occurrence,
|
|
98
135
|
};
|
|
99
|
-
|
|
136
|
+
// The step's own error is the cause; a write that fails while RECORDING it is
|
|
137
|
+
// downstream, and letting it propagate instead replaced "warehouse rejected the
|
|
138
|
+
// reservation" with "SQLITE_BUSY". `executor.ts` writes whatever surfaces into
|
|
139
|
+
// `failureReason`, so the operator got the name of the database's problem and no trace
|
|
140
|
+
// of the real one (`test/repro-workflow-step-error-masking.test.ts`).
|
|
141
|
+
//
|
|
142
|
+
// The same masking was fixed in `loops.ts` one frame above this, at the call sites.
|
|
143
|
+
// That was not a fix of the behaviour: this writer is what every step goes through,
|
|
144
|
+
// loop or not, so the property only holds once it is enforced here.
|
|
145
|
+
//
|
|
146
|
+
// On the success path above there is no step error to protect, so that write stays
|
|
147
|
+
// unguarded and a store that cannot persist a completed step still fails loudly: a
|
|
148
|
+
// record that never reached disk is one whose work runs again.
|
|
149
|
+
try {
|
|
150
|
+
updateFn(exec);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
// Deliberately swallowed: `finalError` is thrown below and carries the real cause.
|
|
154
|
+
}
|
|
100
155
|
emitter?.emitStep('step:failed', exec.id, exec.workflowName, def.name, {
|
|
101
156
|
error: String(finalError),
|
|
102
157
|
attempt: maxAttempts,
|
|
@@ -106,19 +161,39 @@ export async function executeStepWithRetry(def, ctx, exec, emitter, updateFn) {
|
|
|
106
161
|
}
|
|
107
162
|
/** Execute multiple steps in parallel via Promise.allSettled */
|
|
108
163
|
export async function executeParallelSteps(steps, ctx, exec, emitter, updateFn) {
|
|
109
|
-
const results = await Promise.allSettled(steps.map((def) => executeStepWithRetry(def, ctx, exec, emitter, updateFn)));
|
|
164
|
+
const results = await Promise.allSettled(steps.map((def) => executeStepWithRetry(def, ctx, exec, { emitter, updateFn })));
|
|
110
165
|
const failed = results.filter((r) => r.status === 'rejected');
|
|
111
166
|
if (failed.length > 0) {
|
|
112
|
-
const errors = failed.map((r) => r.reason instanceof Error ? r.reason : new Error(
|
|
167
|
+
const errors = failed.map((r) => r.reason instanceof Error ? r.reason : new Error(describeError(r.reason)));
|
|
113
168
|
throw new AggregateError(errors, errors[0].message);
|
|
114
169
|
}
|
|
115
170
|
}
|
|
116
171
|
/** Execute a sub-workflow by starting it and polling for completion */
|
|
117
|
-
|
|
118
|
-
|
|
172
|
+
// 6 params, one over the limit. `existingChildId` is what makes re-entering this node resume
|
|
173
|
+
// the child a restart already started, instead of abandoning it and provisioning a second
|
|
174
|
+
// one, so it belongs in the signature where a caller cannot forget it rather than in an
|
|
175
|
+
// options bag where omitting it looks deliberate.
|
|
176
|
+
// biome-ignore lint/complexity/useMaxParams: see above
|
|
177
|
+
export async function executeSubWorkflow(workflowName, input, startFn, getFn, pollIntervalMs = 100,
|
|
178
|
+
/**
|
|
179
|
+
* A child this node already started, from an earlier entry into the same node.
|
|
180
|
+
*
|
|
181
|
+
* Without it the node started a BRAND NEW child every time it was re-entered, and
|
|
182
|
+
* re-entry is routine: a restart followed by `recover()` re-enqueues the parent's
|
|
183
|
+
* current node. Measured across one restart, the child ran twice and both rows were
|
|
184
|
+
* left `running` forever, since a child is excluded from recovery while its parent
|
|
185
|
+
* exists and `cleanup`/`archive` only reap terminal states. Duplicated work, not just
|
|
186
|
+
* a leaked row: a child that provisions a resource provisioned it twice
|
|
187
|
+
* (`test/repro-workflow-orphan-child.test.ts`).
|
|
188
|
+
*/
|
|
189
|
+
existingChildId) {
|
|
190
|
+
// Resume the existing child when it is still there. A row that has been cleaned away
|
|
191
|
+
// cannot be resumed, so that case starts fresh.
|
|
192
|
+
const existing = existingChildId ? getFn(existingChildId) : null;
|
|
193
|
+
const handle = existing ? { id: existing.id } : await startFn(workflowName, input);
|
|
119
194
|
const maxWait = 300_000;
|
|
120
|
-
const start =
|
|
121
|
-
while (
|
|
195
|
+
const start = clock().now();
|
|
196
|
+
while (clock().now() - start < maxWait) {
|
|
122
197
|
const subExec = getFn(handle.id);
|
|
123
198
|
if (subExec?.state === 'completed') {
|
|
124
199
|
const results = {};
|
|
@@ -126,12 +201,21 @@ export async function executeSubWorkflow(workflowName, input, startFn, getFn, po
|
|
|
126
201
|
if (record.status === 'completed')
|
|
127
202
|
results[name] = record.result;
|
|
128
203
|
}
|
|
129
|
-
return results;
|
|
204
|
+
return { results, executionId: handle.id };
|
|
130
205
|
}
|
|
131
206
|
if (subExec?.state === 'failed') {
|
|
132
207
|
throw new Error(`Sub-workflow "${workflowName}" (${handle.id}) failed`);
|
|
133
208
|
}
|
|
134
|
-
|
|
209
|
+
// A child that parks mid-rollback is terminal FOR THIS POLL: nothing it does next
|
|
210
|
+
// happens without an operator. Waiting for it was measured at the full 300 s, after
|
|
211
|
+
// which the parent reported a timeout, which is the wrong diagnostic for precisely
|
|
212
|
+
// the scenario this module exists to handle, and it held a worker slot for five
|
|
213
|
+
// minutes to say it. The parent parks too, with the real reason.
|
|
214
|
+
if (subExec?.state === 'compensation-stuck') {
|
|
215
|
+
throw new Error(`Sub-workflow "${workflowName}" (${handle.id}) is parked mid-rollback ` +
|
|
216
|
+
`(compensation-stuck); resolve it with resumeCompensation or abandonCompensation`);
|
|
217
|
+
}
|
|
218
|
+
await new Promise((r) => clock().setTimeout(() => r(), pollIntervalMs));
|
|
135
219
|
}
|
|
136
220
|
throw new Error(`Sub-workflow "${workflowName}" (${handle.id}) timed out`);
|
|
137
221
|
}
|
|
@@ -158,9 +242,23 @@ export function findStepDef(wf, name) {
|
|
|
158
242
|
return found;
|
|
159
243
|
}
|
|
160
244
|
if (node.type === 'forEach') {
|
|
161
|
-
if (node.def.step.name === name
|
|
245
|
+
if (node.def.step.name === name)
|
|
162
246
|
return node.def.step;
|
|
163
|
-
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
// Only now consider iteration records. Two passes, not one: a loop body named
|
|
250
|
+
// `charge` and a separate step named `charge:extra` both "start with charge:", so a
|
|
251
|
+
// single prefix pass resolved `charge:extra` to the LOOP's definition. Its own
|
|
252
|
+
// rollback never ran and the loop's ran twice. Exact names win outright, and an
|
|
253
|
+
// iteration suffix must be numeric — the only suffix this engine generates.
|
|
254
|
+
for (const node of wf.nodes) {
|
|
255
|
+
if (node.type === 'doUntil' || node.type === 'doWhile') {
|
|
256
|
+
const found = node.def.steps.find((s) => isIterationOf(name, s.name));
|
|
257
|
+
if (found)
|
|
258
|
+
return found;
|
|
259
|
+
}
|
|
260
|
+
if (node.type === 'forEach' && isIterationOf(name, node.def.step.name)) {
|
|
261
|
+
return node.def.step;
|
|
164
262
|
}
|
|
165
263
|
}
|
|
166
264
|
return null;
|
|
@@ -1,20 +1,80 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* WorkflowStore - SQLite persistence for workflow executions
|
|
3
3
|
*/
|
|
4
|
-
import type { Execution, ExecutionState } from './types';
|
|
4
|
+
import type { Execution, ExecutionState, ParkOutcome, SignalOutcome } from './types';
|
|
5
5
|
export declare class WorkflowStore {
|
|
6
6
|
private readonly db;
|
|
7
7
|
private readonly stmts;
|
|
8
|
+
/** Sole owner of the `signals` column — see storeSignals.ts */
|
|
9
|
+
private readonly signals;
|
|
8
10
|
constructor(dbPath?: string);
|
|
11
|
+
/**
|
|
12
|
+
* INSERT a brand-new execution. Not an upsert in spirit, despite the statement.
|
|
13
|
+
*
|
|
14
|
+
* A plain INSERT, not an upsert: the statement used to be `INSERT OR REPLACE`, so a
|
|
15
|
+
* duplicate execution id silently OVERWROTE a live run instead of failing. Ids carry
|
|
16
|
+
* a random component, which makes that vanishingly rare in production and reachable
|
|
17
|
+
* in a simulation, where a seeded generator draws from a far smaller space. A lost
|
|
18
|
+
* execution is the worst possible presentation of a collision; a constraint error is
|
|
19
|
+
* the best.
|
|
20
|
+
*
|
|
21
|
+
* This is the ONLY writer of `signals` outside SignalCoordinator, and it is safe
|
|
22
|
+
* only because its single caller passes a fresh execution whose signals are `{}`.
|
|
23
|
+
* Calling it on a live run would write a stale snapshot over signals delivered
|
|
24
|
+
* since it was read, which is exactly the lost-update SignalCoordinator exists to
|
|
25
|
+
* prevent. Use `update()` for anything that already exists.
|
|
26
|
+
*/
|
|
9
27
|
save(exec: Execution): void;
|
|
10
28
|
get(id: string): Execution | null;
|
|
29
|
+
/**
|
|
30
|
+
* Persist the step-level columns of an execution.
|
|
31
|
+
*
|
|
32
|
+
* Deliberately does NOT write `signals`. A worker holds one in-memory `Execution`
|
|
33
|
+
* for the whole duration of a node, so its `signals` snapshot goes stale as soon as
|
|
34
|
+
* `recordSignal()` writes to the row. Rewriting the column from that stale snapshot
|
|
35
|
+
* silently destroyed signals delivered mid-step and parked the run forever
|
|
36
|
+
* (`test/repro-workflow-signal-lost-update.test.ts`). Signal payloads are owned
|
|
37
|
+
* exclusively by `recordSignal()`/`parkForSignal()`, which read-modify-write the
|
|
38
|
+
* column inside a transaction.
|
|
39
|
+
*/
|
|
11
40
|
update(exec: Execution): void;
|
|
41
|
+
/**
|
|
42
|
+
* Record a signal payload and, if the run is parked at a `waitFor`, atomically
|
|
43
|
+
* claim the single resume for this caller.
|
|
44
|
+
*/
|
|
45
|
+
recordSignal(id: string, event: string, payload: unknown): SignalOutcome;
|
|
46
|
+
/**
|
|
47
|
+
* Park a running execution at a `waitFor`, unless the awaited signal has already
|
|
48
|
+
* been recorded (in which case the caller must advance instead).
|
|
49
|
+
*/
|
|
50
|
+
parkForSignal(id: string, event: string): ParkOutcome;
|
|
12
51
|
list(workflowName?: string, state?: ExecutionState): Execution[];
|
|
13
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Executions in a recoverable state that recovery may drive ON ITS OWN.
|
|
54
|
+
*
|
|
55
|
+
* A `subWorkflow` child is a row like any other, so the state filter alone offered
|
|
56
|
+
* it to `recover()` as if it were a top-level run. Driving it independently re-ran
|
|
57
|
+
* its steps and produced fresh records with no `compensation`, which defeats the
|
|
58
|
+
* "a step that already carries an outcome is not re-run" guard: the parent's later
|
|
59
|
+
* unwind dispatched the child's reversal a SECOND time, against a provider that had
|
|
60
|
+
* already been refunded (`test/repro-model-child-recovered-alone.test.ts`; found by
|
|
61
|
+
* the state-machine model, seed 1267197984).
|
|
62
|
+
*
|
|
63
|
+
* A child's lifecycle belongs to its parent, which unwinds it through `unwindChild`.
|
|
64
|
+
* One exception: if the parent row is gone, nothing owns the child any more, so it
|
|
65
|
+
* is returned rather than stranded forever in a non-terminal state.
|
|
66
|
+
*/
|
|
14
67
|
listRecoverable(): Execution[];
|
|
15
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Delete executions at least `maxAgeMs` old in terminal states.
|
|
70
|
+
*
|
|
71
|
+
* The cutoff is INCLUSIVE. A strict `<` makes `cleanup(0)`, the documented way to
|
|
72
|
+
* flush everything terminal right now, skip every row whose `updated_at` lands on
|
|
73
|
+
* the current millisecond, which is precisely where a run that just finished lands
|
|
74
|
+
* (`test/repro-workflow-archive-boundary.test.ts`).
|
|
75
|
+
*/
|
|
16
76
|
cleanup(maxAgeMs: number, states?: string[]): number;
|
|
17
|
-
/** Archive executions
|
|
77
|
+
/** Archive executions at least `maxAgeMs` old to the archive table. Cutoff inclusive, as in `cleanup`. */
|
|
18
78
|
archive(maxAgeMs: number, states?: string[]): number;
|
|
19
79
|
/** Get archived execution count */
|
|
20
80
|
getArchivedCount(): number;
|
|
@@ -2,17 +2,9 @@
|
|
|
2
2
|
* WorkflowStore - SQLite persistence for workflow executions
|
|
3
3
|
*/
|
|
4
4
|
import { Database } from 'bun:sqlite';
|
|
5
|
-
import {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
function pack(data) {
|
|
9
|
-
return packr.pack(data);
|
|
10
|
-
}
|
|
11
|
-
function unpack(buf) {
|
|
12
|
-
if (!buf)
|
|
13
|
-
return null;
|
|
14
|
-
return unpackr.unpack(buf);
|
|
15
|
-
}
|
|
5
|
+
import { pack, unpack } from './storeCodec';
|
|
6
|
+
import { SignalCoordinator } from './storeSignals';
|
|
7
|
+
import { clock } from './clock';
|
|
16
8
|
const CREATE_TABLE = `
|
|
17
9
|
CREATE TABLE IF NOT EXISTS workflow_executions (
|
|
18
10
|
id TEXT PRIMARY KEY,
|
|
@@ -42,26 +34,59 @@ CREATE TABLE IF NOT EXISTS workflow_executions_archive (
|
|
|
42
34
|
)`;
|
|
43
35
|
const CREATE_IDX_NAME = `CREATE INDEX IF NOT EXISTS idx_wf_name ON workflow_executions(workflow_name)`;
|
|
44
36
|
const CREATE_IDX_STATE = `CREATE INDEX IF NOT EXISTS idx_wf_state ON workflow_executions(state)`;
|
|
37
|
+
function packMeta(exec) {
|
|
38
|
+
const meta = {};
|
|
39
|
+
if (exec.rollbackStatus !== undefined)
|
|
40
|
+
meta.rollbackStatus = exec.rollbackStatus;
|
|
41
|
+
if (exec.failureReason !== undefined)
|
|
42
|
+
meta.failureReason = exec.failureReason;
|
|
43
|
+
if (exec.committedAt !== undefined)
|
|
44
|
+
meta.committedAt = exec.committedAt;
|
|
45
|
+
if (exec.parentExecutionId !== undefined)
|
|
46
|
+
meta.parentExecutionId = exec.parentExecutionId;
|
|
47
|
+
return Object.keys(meta).length > 0 ? pack(meta) : null;
|
|
48
|
+
}
|
|
45
49
|
export class WorkflowStore {
|
|
46
50
|
db;
|
|
47
51
|
stmts;
|
|
52
|
+
/** Sole owner of the `signals` column — see storeSignals.ts */
|
|
53
|
+
signals;
|
|
48
54
|
constructor(dbPath) {
|
|
49
55
|
this.db = new Database(dbPath ?? ':memory:', { create: true });
|
|
50
56
|
this.db.run('PRAGMA journal_mode = WAL');
|
|
57
|
+
// The Engine hands the SAME dataPath to this store and to its embedded
|
|
58
|
+
// Queue/Worker, so two connections share one file. Without a busy timeout a
|
|
59
|
+
// read-then-upgrade inside signal()/parkForSignal() can surface SQLITE_BUSY
|
|
60
|
+
// straight to the caller, with no retry, the moment the queue happens to be
|
|
61
|
+
// writing. Five seconds matches the server's own persistence layer.
|
|
62
|
+
this.db.run('PRAGMA busy_timeout = 5000');
|
|
51
63
|
this.db.run(CREATE_TABLE);
|
|
52
64
|
this.db.run(CREATE_ARCHIVE_TABLE);
|
|
53
65
|
this.db.run(CREATE_IDX_NAME);
|
|
54
66
|
this.db.run(CREATE_IDX_STATE);
|
|
67
|
+
// Rollback bookkeeping arrived after the original schema. One nullable blob
|
|
68
|
+
// rather than three columns keeps the migration to a single guarded statement
|
|
69
|
+
// per table; SQLite has no ADD COLUMN IF NOT EXISTS, so the throw IS the check.
|
|
70
|
+
for (const table of ['workflow_executions', 'workflow_executions_archive']) {
|
|
71
|
+
try {
|
|
72
|
+
this.db.run(`ALTER TABLE ${table} ADD COLUMN meta BLOB`);
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
/* already migrated */
|
|
76
|
+
}
|
|
77
|
+
}
|
|
55
78
|
this.stmts = {
|
|
56
79
|
upsert: this.db.prepare(`
|
|
57
|
-
INSERT
|
|
58
|
-
(id, workflow_name, state, input, steps, current_node_index, resolved_steps, signals, created_at, updated_at)
|
|
59
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
80
|
+
INSERT INTO workflow_executions
|
|
81
|
+
(id, workflow_name, state, input, steps, current_node_index, resolved_steps, signals, created_at, updated_at, meta)
|
|
82
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
60
83
|
`),
|
|
61
84
|
get: this.db.prepare(`SELECT * FROM workflow_executions WHERE id = ?`),
|
|
85
|
+
// NOTE: `signals` is deliberately absent. That column is owned exclusively by
|
|
86
|
+
// recordSignal()/parkForSignal() — see the comment on `update()`.
|
|
62
87
|
updateState: this.db.prepare(`
|
|
63
88
|
UPDATE workflow_executions
|
|
64
|
-
SET state = ?, steps = ?, current_node_index = ?, resolved_steps = ?,
|
|
89
|
+
SET state = ?, steps = ?, current_node_index = ?, resolved_steps = ?, updated_at = ?, meta = ?
|
|
65
90
|
WHERE id = ?
|
|
66
91
|
`),
|
|
67
92
|
list: this.db.prepare(`SELECT * FROM workflow_executions ORDER BY created_at DESC LIMIT 100`),
|
|
@@ -70,17 +95,62 @@ export class WorkflowStore {
|
|
|
70
95
|
listByBoth: this.db.prepare(`SELECT * FROM workflow_executions WHERE workflow_name = ? AND state = ? ORDER BY created_at DESC LIMIT 100`),
|
|
71
96
|
listRecoverable: this.db.prepare(`SELECT * FROM workflow_executions WHERE state IN ('running', 'waiting', 'compensating') ORDER BY updated_at ASC`),
|
|
72
97
|
};
|
|
98
|
+
this.signals = new SignalCoordinator(this.db);
|
|
73
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* INSERT a brand-new execution. Not an upsert in spirit, despite the statement.
|
|
102
|
+
*
|
|
103
|
+
* A plain INSERT, not an upsert: the statement used to be `INSERT OR REPLACE`, so a
|
|
104
|
+
* duplicate execution id silently OVERWROTE a live run instead of failing. Ids carry
|
|
105
|
+
* a random component, which makes that vanishingly rare in production and reachable
|
|
106
|
+
* in a simulation, where a seeded generator draws from a far smaller space. A lost
|
|
107
|
+
* execution is the worst possible presentation of a collision; a constraint error is
|
|
108
|
+
* the best.
|
|
109
|
+
*
|
|
110
|
+
* This is the ONLY writer of `signals` outside SignalCoordinator, and it is safe
|
|
111
|
+
* only because its single caller passes a fresh execution whose signals are `{}`.
|
|
112
|
+
* Calling it on a live run would write a stale snapshot over signals delivered
|
|
113
|
+
* since it was read, which is exactly the lost-update SignalCoordinator exists to
|
|
114
|
+
* prevent. Use `update()` for anything that already exists.
|
|
115
|
+
*/
|
|
74
116
|
save(exec) {
|
|
75
|
-
|
|
117
|
+
if (Object.keys(exec.signals).length > 0) {
|
|
118
|
+
throw new Error('WorkflowStore.save() is for new executions only; it would overwrite delivered signals. Use update().');
|
|
119
|
+
}
|
|
120
|
+
this.stmts.upsert.run(exec.id, exec.workflowName, exec.state, pack(exec.input), pack(exec.steps), exec.currentNodeIndex, exec.resolvedSteps ? pack(exec.resolvedSteps) : null, pack(exec.signals), exec.createdAt, exec.updatedAt, packMeta(exec));
|
|
76
121
|
}
|
|
77
122
|
get(id) {
|
|
78
123
|
const row = this.stmts.get.get(id);
|
|
79
124
|
return row ? this.rowToExecution(row) : null;
|
|
80
125
|
}
|
|
126
|
+
/**
|
|
127
|
+
* Persist the step-level columns of an execution.
|
|
128
|
+
*
|
|
129
|
+
* Deliberately does NOT write `signals`. A worker holds one in-memory `Execution`
|
|
130
|
+
* for the whole duration of a node, so its `signals` snapshot goes stale as soon as
|
|
131
|
+
* `recordSignal()` writes to the row. Rewriting the column from that stale snapshot
|
|
132
|
+
* silently destroyed signals delivered mid-step and parked the run forever
|
|
133
|
+
* (`test/repro-workflow-signal-lost-update.test.ts`). Signal payloads are owned
|
|
134
|
+
* exclusively by `recordSignal()`/`parkForSignal()`, which read-modify-write the
|
|
135
|
+
* column inside a transaction.
|
|
136
|
+
*/
|
|
81
137
|
update(exec) {
|
|
82
|
-
exec.updatedAt =
|
|
83
|
-
this.stmts.updateState.run(exec.state, pack(exec.steps), exec.currentNodeIndex, exec.resolvedSteps ? pack(exec.resolvedSteps) : null,
|
|
138
|
+
exec.updatedAt = clock().now();
|
|
139
|
+
this.stmts.updateState.run(exec.state, pack(exec.steps), exec.currentNodeIndex, exec.resolvedSteps ? pack(exec.resolvedSteps) : null, exec.updatedAt, packMeta(exec), exec.id);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Record a signal payload and, if the run is parked at a `waitFor`, atomically
|
|
143
|
+
* claim the single resume for this caller.
|
|
144
|
+
*/
|
|
145
|
+
recordSignal(id, event, payload) {
|
|
146
|
+
return this.signals.record(id, event, payload);
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Park a running execution at a `waitFor`, unless the awaited signal has already
|
|
150
|
+
* been recorded (in which case the caller must advance instead).
|
|
151
|
+
*/
|
|
152
|
+
parkForSignal(id, event) {
|
|
153
|
+
return this.signals.park(id, event);
|
|
84
154
|
}
|
|
85
155
|
list(workflowName, state) {
|
|
86
156
|
let rows;
|
|
@@ -98,38 +168,60 @@ export class WorkflowStore {
|
|
|
98
168
|
}
|
|
99
169
|
return rows.map((r) => this.rowToExecution(r));
|
|
100
170
|
}
|
|
101
|
-
/**
|
|
171
|
+
/**
|
|
172
|
+
* Executions in a recoverable state that recovery may drive ON ITS OWN.
|
|
173
|
+
*
|
|
174
|
+
* A `subWorkflow` child is a row like any other, so the state filter alone offered
|
|
175
|
+
* it to `recover()` as if it were a top-level run. Driving it independently re-ran
|
|
176
|
+
* its steps and produced fresh records with no `compensation`, which defeats the
|
|
177
|
+
* "a step that already carries an outcome is not re-run" guard: the parent's later
|
|
178
|
+
* unwind dispatched the child's reversal a SECOND time, against a provider that had
|
|
179
|
+
* already been refunded (`test/repro-model-child-recovered-alone.test.ts`; found by
|
|
180
|
+
* the state-machine model, seed 1267197984).
|
|
181
|
+
*
|
|
182
|
+
* A child's lifecycle belongs to its parent, which unwinds it through `unwindChild`.
|
|
183
|
+
* One exception: if the parent row is gone, nothing owns the child any more, so it
|
|
184
|
+
* is returned rather than stranded forever in a non-terminal state.
|
|
185
|
+
*/
|
|
102
186
|
listRecoverable() {
|
|
103
187
|
const rows = this.stmts.listRecoverable.all();
|
|
104
|
-
|
|
188
|
+
const all = rows.map((r) => this.rowToExecution(r));
|
|
189
|
+
return all.filter((e) => !e.parentExecutionId || this.get(e.parentExecutionId) === null);
|
|
105
190
|
}
|
|
106
|
-
/**
|
|
191
|
+
/**
|
|
192
|
+
* Delete executions at least `maxAgeMs` old in terminal states.
|
|
193
|
+
*
|
|
194
|
+
* The cutoff is INCLUSIVE. A strict `<` makes `cleanup(0)`, the documented way to
|
|
195
|
+
* flush everything terminal right now, skip every row whose `updated_at` lands on
|
|
196
|
+
* the current millisecond, which is precisely where a run that just finished lands
|
|
197
|
+
* (`test/repro-workflow-archive-boundary.test.ts`).
|
|
198
|
+
*/
|
|
107
199
|
cleanup(maxAgeMs, states = ['completed', 'failed']) {
|
|
108
|
-
const cutoff =
|
|
200
|
+
const cutoff = clock().now() - maxAgeMs;
|
|
109
201
|
const placeholders = states.map(() => '?').join(',');
|
|
110
|
-
const stmt = this.db.prepare(`DELETE FROM workflow_executions WHERE updated_at
|
|
202
|
+
const stmt = this.db.prepare(`DELETE FROM workflow_executions WHERE updated_at <= ? AND state IN (${placeholders})`);
|
|
111
203
|
const result = stmt.run(cutoff, ...states);
|
|
112
204
|
return result.changes;
|
|
113
205
|
}
|
|
114
|
-
/** Archive executions
|
|
206
|
+
/** Archive executions at least `maxAgeMs` old to the archive table. Cutoff inclusive, as in `cleanup`. */
|
|
115
207
|
archive(maxAgeMs, states = ['completed', 'failed']) {
|
|
116
|
-
const cutoff =
|
|
117
|
-
const now =
|
|
208
|
+
const cutoff = clock().now() - maxAgeMs;
|
|
209
|
+
const now = clock().now();
|
|
118
210
|
const placeholders = states.map(() => '?').join(',');
|
|
119
211
|
const rows = this.db
|
|
120
|
-
.prepare(`SELECT * FROM workflow_executions WHERE updated_at
|
|
212
|
+
.prepare(`SELECT * FROM workflow_executions WHERE updated_at <= ? AND state IN (${placeholders}) LIMIT 1000`)
|
|
121
213
|
.all(cutoff, ...states);
|
|
122
214
|
if (rows.length === 0)
|
|
123
215
|
return 0;
|
|
124
216
|
const insertArchive = this.db.prepare(`
|
|
125
217
|
INSERT OR REPLACE INTO workflow_executions_archive
|
|
126
|
-
(id, workflow_name, state, input, steps, current_node_index, resolved_steps, signals, created_at, updated_at, archived_at)
|
|
127
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
218
|
+
(id, workflow_name, state, input, steps, current_node_index, resolved_steps, signals, created_at, updated_at, archived_at, meta)
|
|
219
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
128
220
|
`);
|
|
129
221
|
const deleteOriginal = this.db.prepare(`DELETE FROM workflow_executions WHERE id = ?`);
|
|
130
222
|
const tx = this.db.transaction(() => {
|
|
131
223
|
for (const row of rows) {
|
|
132
|
-
insertArchive.run(row.id, row.workflow_name, row.state, row.input, row.steps, row.current_node_index, row.resolved_steps, row.signals, row.created_at, row.updated_at, now);
|
|
224
|
+
insertArchive.run(row.id, row.workflow_name, row.state, row.input, row.steps, row.current_node_index, row.resolved_steps, row.signals, row.created_at, row.updated_at, now, row.meta ?? null);
|
|
133
225
|
deleteOriginal.run(row.id);
|
|
134
226
|
}
|
|
135
227
|
});
|
|
@@ -160,6 +252,7 @@ export class WorkflowStore {
|
|
|
160
252
|
signals: unpack(row.signals) ?? {},
|
|
161
253
|
createdAt: row.created_at,
|
|
162
254
|
updatedAt: row.updated_at,
|
|
255
|
+
...(unpack(row.meta) ?? {}),
|
|
163
256
|
};
|
|
164
257
|
}
|
|
165
258
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* msgpack codec shared by the workflow store and its signal coordinator.
|
|
3
|
+
* Kept in its own module so both can encode blobs identically without either
|
|
4
|
+
* owning the packer.
|
|
5
|
+
*/
|
|
6
|
+
export declare function pack(data: unknown): Uint8Array;
|
|
7
|
+
export declare function unpack(buf: Uint8Array | null | undefined): unknown;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* msgpack codec shared by the workflow store and its signal coordinator.
|
|
3
|
+
* Kept in its own module so both can encode blobs identically without either
|
|
4
|
+
* owning the packer.
|
|
5
|
+
*/
|
|
6
|
+
import { Packr, Unpackr } from 'msgpackr';
|
|
7
|
+
const packr = new Packr({ structuredClone: true });
|
|
8
|
+
const unpackr = new Unpackr({ structuredClone: true });
|
|
9
|
+
export function pack(data) {
|
|
10
|
+
return packr.pack(data);
|
|
11
|
+
}
|
|
12
|
+
export function unpack(buf) {
|
|
13
|
+
if (!buf)
|
|
14
|
+
return null;
|
|
15
|
+
return unpackr.unpack(buf);
|
|
16
|
+
}
|