@marianmeres/workflow 1.0.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +207 -94
- package/API.md +277 -157
- package/README.md +182 -83
- package/dist/correlator.d.ts +31 -14
- package/dist/correlator.js +130 -73
- package/dist/definition.d.ts +5 -1
- package/dist/definition.js +54 -1
- package/dist/driver.d.ts +28 -23
- package/dist/driver.js +223 -59
- package/dist/migrations/1_1_0.d.ts +11 -0
- package/dist/migrations/1_1_0.js +61 -0
- package/dist/migrations/1_2_0.d.ts +16 -0
- package/dist/migrations/1_2_0.js +44 -0
- package/dist/migrations/index.js +5 -1
- package/dist/mod.d.ts +8 -5
- package/dist/mod.js +7 -4
- package/dist/persistence/history.d.ts +1 -1
- package/dist/persistence/history.js +3 -3
- package/dist/persistence/inbox.d.ts +9 -3
- package/dist/persistence/inbox.js +16 -7
- package/dist/persistence/instances.d.ts +46 -13
- package/dist/persistence/instances.js +78 -33
- package/dist/scheduler.d.ts +33 -9
- package/dist/scheduler.js +45 -21
- package/dist/types.d.ts +126 -13
- package/dist/types.js +28 -2
- package/dist/workflow.d.ts +76 -4
- package/dist/workflow.js +237 -35
- package/docs/_archive/analysis/00-overview-and-roadmap.md +233 -0
- package/docs/_archive/analysis/01-durability.md +404 -0
- package/docs/_archive/analysis/02-correlation.md +228 -0
- package/docs/_archive/analysis/03-definition-and-api.md +248 -0
- package/docs/_archive/analysis/04-docs-tests-ops.md +191 -0
- package/docs/_archive/analysis/PROGRESS.md +172 -0
- package/docs/_archive/analysis/README.md +38 -0
- package/package.json +9 -8
package/dist/driver.js
CHANGED
|
@@ -1,26 +1,39 @@
|
|
|
1
1
|
import { FSM } from "@marianmeres/fsm";
|
|
2
2
|
import { clog } from "./log.js";
|
|
3
3
|
import { appendHistory } from "./persistence/history.js";
|
|
4
|
-
import {
|
|
4
|
+
import { lockInboxRow, markProcessed } from "./persistence/inbox.js";
|
|
5
|
+
import { lockInstance, updateInstance } from "./persistence/instances.js";
|
|
5
6
|
import { withTransaction } from "./persistence/tx.js";
|
|
6
|
-
import { EXECUTION_STATE, HISTORY_EVENT, JOB_TYPE_ADVANCE, JOB_TYPE_EFFECT_PREFIX, } from "./types.js";
|
|
7
|
+
import { EXECUTION_STATE, HISTORY_EVENT, JOB_TYPE_ADVANCE, JOB_TYPE_EFFECT_PREFIX, PURE_ENTER_EVENT, SIGNAL_MATCHED_EVENT, } from "./types.js";
|
|
7
8
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
9
|
+
* Reads the tenant scope off a job payload, tolerating the legacy `project_id`
|
|
10
|
+
* key used before the 1.1.0 rename.
|
|
10
11
|
*
|
|
11
|
-
*
|
|
12
|
+
* Jobs already sitting in the steve queue when a deployment upgrades still
|
|
13
|
+
* carry the old key. Without this fallback their `tenant_id` would read as
|
|
14
|
+
* `undefined`, the history insert would violate its NOT NULL constraint, and
|
|
15
|
+
* the job would retry until it exhausted its attempts.
|
|
12
16
|
*
|
|
13
|
-
*
|
|
14
|
-
* meta: { kind: "pure" },
|
|
15
|
-
* on: {
|
|
16
|
-
* ENTER: [
|
|
17
|
-
* { target: "go_left", guard: (ctx) => ctx.x > 5 },
|
|
18
|
-
* { target: "go_right" },
|
|
19
|
-
* ],
|
|
20
|
-
* },
|
|
21
|
-
* }
|
|
17
|
+
* Safe to delete once no pre-1.1.0 jobs can remain in any deployed queue.
|
|
22
18
|
*/
|
|
23
|
-
|
|
19
|
+
function payloadTenantId(payload) {
|
|
20
|
+
return (payload.tenant_id ?? payload.project_id);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* What produced this advance. Jobs queued before the fence existed carry no
|
|
24
|
+
* `kind`, so it is inferred from the legacy payload shape: `timeout: true` was
|
|
25
|
+
* the scheduler, an `outcome` without it was an effect completion or a matched
|
|
26
|
+
* signal, and neither was the initial dispatch.
|
|
27
|
+
*
|
|
28
|
+
* Same transitional status as {@link payloadTenantId} — delete both together.
|
|
29
|
+
*/
|
|
30
|
+
function advanceKind(payload) {
|
|
31
|
+
if (payload.kind)
|
|
32
|
+
return payload.kind;
|
|
33
|
+
if (payload.timeout === true)
|
|
34
|
+
return "timeout";
|
|
35
|
+
return payload.outcome !== undefined ? "effect" : "start";
|
|
36
|
+
}
|
|
24
37
|
/**
|
|
25
38
|
* Maximum number of inline pure-state hops the driver will take within a
|
|
26
39
|
* single advance before bailing out (loop guard against pathological
|
|
@@ -33,6 +46,11 @@ const MAX_PURE_HOPS = 64;
|
|
|
33
46
|
* Wrapped in a single PG transaction:
|
|
34
47
|
* 1. Lock the instance row.
|
|
35
48
|
* 2. If in a terminal execution_state, no-op.
|
|
49
|
+
* 2b. If the payload's fence (`expected_seq`) does not match the locked row, or
|
|
50
|
+
* the row is not in the state this `kind` of advance expects, no-op — the job
|
|
51
|
+
* is a duplicate, or a zombie from a step the instance already left.
|
|
52
|
+
* 2c. `kind: "signal"` → lock the inbox row; it supplies the outcome data and is
|
|
53
|
+
* marked processed in this same transaction.
|
|
36
54
|
* 3. If `outcome` was supplied, apply it via fsm.transition(outcome, data).
|
|
37
55
|
* - Reject → mark failed, append history.
|
|
38
56
|
* 4. Loop on the current state's meta.kind:
|
|
@@ -42,12 +60,17 @@ const MAX_PURE_HOPS = 64;
|
|
|
42
60
|
* - suspending → mark waiting, set wake_at/correlation_token, return.
|
|
43
61
|
*/
|
|
44
62
|
export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
45
|
-
const { instance_id
|
|
63
|
+
const { instance_id } = payload;
|
|
64
|
+
const tenant_id = payloadTenantId(payload);
|
|
65
|
+
const kind = advanceKind(payload);
|
|
46
66
|
await withTransaction(pool, async (client) => {
|
|
47
67
|
const row = await lockInstance(client, instance_id);
|
|
48
68
|
if (!row) {
|
|
49
|
-
|
|
50
|
-
|
|
69
|
+
// Also the window where the creating transaction has not committed yet
|
|
70
|
+
// (steve's job insert is autocommit on its own connection, so the job
|
|
71
|
+
// can be claimed first). Throw rather than drop the only job the
|
|
72
|
+
// instance will ever get — steve retries.
|
|
73
|
+
throw new Error(`advance: instance ${instance_id} not found`);
|
|
51
74
|
}
|
|
52
75
|
if (row.execution_state === EXECUTION_STATE.COMPLETED ||
|
|
53
76
|
row.execution_state === EXECUTION_STATE.FAILED ||
|
|
@@ -55,6 +78,53 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
55
78
|
clog.debug?.(`advance: instance ${instance_id} already terminal (${row.execution_state}); no-op`);
|
|
56
79
|
return;
|
|
57
80
|
}
|
|
81
|
+
// The fence. Only meaningful against the locked row, hence its position.
|
|
82
|
+
// A mismatch means the instance has settled at least once since this job
|
|
83
|
+
// was issued: the job is a duplicate or a zombie. Debug-log it and drop
|
|
84
|
+
// it — with re-poking ticks these are routine, not events, so no history.
|
|
85
|
+
if (payload.expected_seq !== undefined && row.seq !== payload.expected_seq) {
|
|
86
|
+
clog.debug?.(`advance: stale (row.seq=${row.seq}, expected=${payload.expected_seq}); no-op`);
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
// Per-kind preconditions. Applied only to payloads that state their kind:
|
|
90
|
+
// on a legacy payload the kind is a guess, and guessing wrong must not
|
|
91
|
+
// swallow the job. `timeout` re-checks due-ness because the scheduler tick
|
|
92
|
+
// is read-only: the timer is still live until this advance clears it, and
|
|
93
|
+
// an advance that applied TIMEOUT to a not-yet-due row would cut the wait
|
|
94
|
+
// short.
|
|
95
|
+
if ((payload.kind === "start" &&
|
|
96
|
+
row.execution_state !== EXECUTION_STATE.PENDING) ||
|
|
97
|
+
(payload.kind === "effect" &&
|
|
98
|
+
row.execution_state !== EXECUTION_STATE.RUNNING) ||
|
|
99
|
+
(payload.kind === "signal" &&
|
|
100
|
+
row.execution_state !== EXECUTION_STATE.WAITING) ||
|
|
101
|
+
(payload.kind === "timeout" &&
|
|
102
|
+
(row.execution_state !== EXECUTION_STATE.WAITING || !row.wake_at ||
|
|
103
|
+
row.wake_at > new Date()))) {
|
|
104
|
+
clog.debug?.(`advance: ${payload.kind} precondition not met (${row.execution_state}); no-op`);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
// A signal advance carries the inbox row id, not its payload: the row is
|
|
108
|
+
// the source of truth, and marking it processed happens in this very
|
|
109
|
+
// transaction, so delivery and transition commit together or not at all.
|
|
110
|
+
let outcome = payload.outcome;
|
|
111
|
+
let outcome_data = payload.outcome_data;
|
|
112
|
+
let inbox = null;
|
|
113
|
+
if (payload.kind === "signal" && payload.inbox_id) {
|
|
114
|
+
inbox = await lockInboxRow(client, payload.inbox_id);
|
|
115
|
+
if (!inbox || inbox.processed_at) {
|
|
116
|
+
clog.debug?.(`advance: inbox row ${payload.inbox_id} already delivered; no-op`);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
outcome = SIGNAL_MATCHED_EVENT;
|
|
120
|
+
outcome_data = inbox.payload;
|
|
121
|
+
}
|
|
122
|
+
// A handler may hand back the token its side effect produced (an SMTP
|
|
123
|
+
// Message-ID, a provider session id). Written at the settle point below,
|
|
124
|
+
// so it is on the row the moment the instance becomes signallable.
|
|
125
|
+
const tokenPatch = payload.correlation_token !== undefined
|
|
126
|
+
? { correlation_token: payload.correlation_token }
|
|
127
|
+
: {};
|
|
58
128
|
let def;
|
|
59
129
|
try {
|
|
60
130
|
def = registry.requireDefinition(row.definition_id, row.definition_version);
|
|
@@ -64,11 +134,15 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
64
134
|
return;
|
|
65
135
|
}
|
|
66
136
|
// Construct the FSM at the saved cursor. fromSnapshot skips onEnter
|
|
67
|
-
// (a resume is not entry) and preserves previous.
|
|
137
|
+
// (a resume is not entry) and preserves previous. A handler's context
|
|
138
|
+
// patch goes in here, before the outcome is applied, so guards and
|
|
139
|
+
// actions on the outcome edge already see it.
|
|
68
140
|
const positioned = FSM.fromSnapshot(def.fsm, {
|
|
69
141
|
state: row.cursor,
|
|
70
142
|
previous: row.previous_cursor,
|
|
71
|
-
context:
|
|
143
|
+
context: payload.context_patch
|
|
144
|
+
? { ...row.context, ...payload.context_patch }
|
|
145
|
+
: row.context,
|
|
72
146
|
});
|
|
73
147
|
// If an outcome was supplied, apply it. This is the wake-up after an effect
|
|
74
148
|
// completion, a matched signal, or a timer expiry.
|
|
@@ -77,7 +151,7 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
77
151
|
const result = positioned.transition(outcome, outcome_data ?? undefined, false);
|
|
78
152
|
if (result === null) {
|
|
79
153
|
await appendHistory(client, {
|
|
80
|
-
|
|
154
|
+
tenant_id,
|
|
81
155
|
instance_id,
|
|
82
156
|
event_type: HISTORY_EVENT.TRANSITION_REJECTED,
|
|
83
157
|
from_node: before,
|
|
@@ -86,18 +160,38 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
86
160
|
await failInstance(client, row, `fsm rejected outcome "${outcome}" at state "${before}"`);
|
|
87
161
|
return;
|
|
88
162
|
}
|
|
89
|
-
//
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
//
|
|
93
|
-
|
|
163
|
+
// `outcome_data` stays the fsm payload and is never auto-merged: what
|
|
164
|
+
// lands in the context is either an fsm action hook the user wired, or
|
|
165
|
+
// what the handler explicitly returned in `HandlerResult.context`
|
|
166
|
+
// (merged above).
|
|
167
|
+
if (kind === "effect" && payload.handler) {
|
|
168
|
+
await appendHistory(client, {
|
|
169
|
+
tenant_id,
|
|
170
|
+
instance_id,
|
|
171
|
+
event_type: HISTORY_EVENT.EFFECT_COMPLETED,
|
|
172
|
+
from_node: before,
|
|
173
|
+
data: { handler: payload.handler, outcome },
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
if (inbox) {
|
|
177
|
+
await appendHistory(client, {
|
|
178
|
+
tenant_id,
|
|
179
|
+
instance_id,
|
|
180
|
+
event_type: HISTORY_EVENT.SIGNAL_RECEIVED,
|
|
181
|
+
from_node: before,
|
|
182
|
+
data: { inbox_id: inbox.id, source: inbox.source },
|
|
183
|
+
});
|
|
184
|
+
await markProcessed(client, inbox.id);
|
|
185
|
+
}
|
|
94
186
|
await appendHistory(client, {
|
|
95
|
-
|
|
187
|
+
tenant_id,
|
|
96
188
|
instance_id,
|
|
97
|
-
event_type:
|
|
189
|
+
event_type: kind === "timeout"
|
|
190
|
+
? HISTORY_EVENT.TIMEOUT
|
|
191
|
+
: HISTORY_EVENT.TRANSITION,
|
|
98
192
|
from_node: before,
|
|
99
193
|
to_node: positioned.state,
|
|
100
|
-
data: { outcome, outcome_data },
|
|
194
|
+
data: { outcome, outcome_data, ...tokenPatch },
|
|
101
195
|
});
|
|
102
196
|
}
|
|
103
197
|
// Drive forward through pure states, settling at terminal/effectful/suspending.
|
|
@@ -116,9 +210,9 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
116
210
|
execution_state: EXECUTION_STATE.COMPLETED,
|
|
117
211
|
wake_at: null,
|
|
118
212
|
correlation_token: null,
|
|
119
|
-
});
|
|
213
|
+
}, { bumpSeq: true });
|
|
120
214
|
await appendHistory(client, {
|
|
121
|
-
|
|
215
|
+
tenant_id,
|
|
122
216
|
instance_id,
|
|
123
217
|
event_type: HISTORY_EVENT.COMPLETED,
|
|
124
218
|
from_node: positioned.state,
|
|
@@ -127,20 +221,23 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
127
221
|
}
|
|
128
222
|
case "effectful": {
|
|
129
223
|
// Persist context (any guards/actions may have updated it) + flip to running.
|
|
130
|
-
await updateInstance(client, instance_id, {
|
|
224
|
+
const settled = await updateInstance(client, instance_id, {
|
|
131
225
|
cursor: positioned.state,
|
|
132
226
|
previous_cursor: positioned.previous,
|
|
133
227
|
context: positioned.context,
|
|
134
228
|
execution_state: EXECUTION_STATE.RUNNING,
|
|
135
229
|
wake_at: null,
|
|
136
|
-
|
|
230
|
+
...tokenPatch,
|
|
231
|
+
}, { bumpSeq: true });
|
|
137
232
|
await enqueuer.enqueueEffect(client, meta.handler, {
|
|
138
|
-
|
|
233
|
+
tenant_id,
|
|
139
234
|
instance_id,
|
|
140
235
|
handler: meta.handler,
|
|
236
|
+
seq: settled.seq,
|
|
237
|
+
cursor: settled.cursor,
|
|
141
238
|
});
|
|
142
239
|
await appendHistory(client, {
|
|
143
|
-
|
|
240
|
+
tenant_id,
|
|
144
241
|
instance_id,
|
|
145
242
|
event_type: HISTORY_EVENT.EFFECT_DISPATCHED,
|
|
146
243
|
from_node: positioned.state,
|
|
@@ -152,17 +249,26 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
152
249
|
const wake_at = meta.timeoutSec
|
|
153
250
|
? new Date(Date.now() + meta.timeoutSec * 1000)
|
|
154
251
|
: null;
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
await updateInstance(client, instance_id, {
|
|
252
|
+
// The token is whatever create() set or the handler returned — the
|
|
253
|
+
// driver never generates one.
|
|
254
|
+
const settled = await updateInstance(client, instance_id, {
|
|
158
255
|
cursor: positioned.state,
|
|
159
256
|
previous_cursor: positioned.previous,
|
|
160
257
|
context: positioned.context,
|
|
161
258
|
execution_state: EXECUTION_STATE.WAITING,
|
|
162
259
|
wake_at,
|
|
163
|
-
|
|
260
|
+
...tokenPatch,
|
|
261
|
+
}, { bumpSeq: true });
|
|
262
|
+
// Nothing can deliver a signal to a tokenless wait, and the
|
|
263
|
+
// validator cannot catch it — tokens exist only at runtime.
|
|
264
|
+
const on = def.fsm.states[positioned.state].on;
|
|
265
|
+
if (!settled.correlation_token &&
|
|
266
|
+
(SIGNAL_MATCHED_EVENT in on || "*" in on)) {
|
|
267
|
+
clog.warn?.(`advance: instance ${instance_id} waits at "${positioned.state}" ` +
|
|
268
|
+
`with no correlation_token — no signal can reach it`);
|
|
269
|
+
}
|
|
164
270
|
await appendHistory(client, {
|
|
165
|
-
|
|
271
|
+
tenant_id,
|
|
166
272
|
instance_id,
|
|
167
273
|
event_type: HISTORY_EVENT.TRANSITION,
|
|
168
274
|
from_node: positioned.state,
|
|
@@ -175,17 +281,20 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
175
281
|
const result = positioned.transition(PURE_ENTER_EVENT, undefined, false);
|
|
176
282
|
if (result === null) {
|
|
177
283
|
await appendHistory(client, {
|
|
178
|
-
|
|
284
|
+
tenant_id,
|
|
179
285
|
instance_id,
|
|
180
286
|
event_type: HISTORY_EVENT.TRANSITION_REJECTED,
|
|
181
287
|
from_node: before,
|
|
182
|
-
data: {
|
|
288
|
+
data: {
|
|
289
|
+
event: PURE_ENTER_EVENT,
|
|
290
|
+
reason: "no guarded transition matched",
|
|
291
|
+
},
|
|
183
292
|
});
|
|
184
293
|
await failInstance(client, row, `pure state "${before}" did not route on ENTER`);
|
|
185
294
|
return;
|
|
186
295
|
}
|
|
187
296
|
await appendHistory(client, {
|
|
188
|
-
|
|
297
|
+
tenant_id,
|
|
189
298
|
instance_id,
|
|
190
299
|
event_type: HISTORY_EVENT.TRANSITION,
|
|
191
300
|
from_node: before,
|
|
@@ -204,17 +313,43 @@ export async function runAdvance(pool, registry, enqueuer, payload) {
|
|
|
204
313
|
await failInstance(client, row, `exceeded ${MAX_PURE_HOPS} pure-state hops (likely cycle)`);
|
|
205
314
|
});
|
|
206
315
|
}
|
|
207
|
-
async function failInstance(client, row, reason) {
|
|
316
|
+
async function failInstance(client, row, reason, extra = {}) {
|
|
208
317
|
clog.error?.(`workflow instance ${row.id} failed: ${reason}`);
|
|
209
318
|
await updateInstance(client, row.id, {
|
|
210
319
|
execution_state: EXECUTION_STATE.FAILED,
|
|
211
|
-
});
|
|
320
|
+
}, { bumpSeq: true });
|
|
212
321
|
await appendHistory(client, {
|
|
213
|
-
|
|
322
|
+
tenant_id: row.tenant_id,
|
|
214
323
|
instance_id: row.id,
|
|
215
324
|
event_type: HISTORY_EVENT.FAILED,
|
|
216
325
|
from_node: row.cursor,
|
|
217
|
-
data: { reason },
|
|
326
|
+
data: { reason, ...extra },
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Called by the Workflow layer when steve gives up on an advance job — every
|
|
331
|
+
* attempt threw, or the job expired once too often to keep re-dispatching it.
|
|
332
|
+
* Without this the instance would sit in `pending`/`running` with nothing left
|
|
333
|
+
* that could ever move it.
|
|
334
|
+
*
|
|
335
|
+
* Fenced like every other write: an advance that lost the race to a later one
|
|
336
|
+
* must not fail the instance that later one moved on.
|
|
337
|
+
*/
|
|
338
|
+
export async function failAdvanceJob(pool, payload, reason, jobUid) {
|
|
339
|
+
await withTransaction(pool, async (client) => {
|
|
340
|
+
const row = await lockInstance(client, payload.instance_id);
|
|
341
|
+
if (!row)
|
|
342
|
+
return;
|
|
343
|
+
if (payload.expected_seq !== undefined && row.seq !== payload.expected_seq) {
|
|
344
|
+
clog.debug?.(`failAdvanceJob: stale (row.seq=${row.seq}, expected=${payload.expected_seq}); no-op`);
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
if (row.execution_state === EXECUTION_STATE.COMPLETED ||
|
|
348
|
+
row.execution_state === EXECUTION_STATE.FAILED ||
|
|
349
|
+
row.execution_state === EXECUTION_STATE.CANCELLED) {
|
|
350
|
+
return;
|
|
351
|
+
}
|
|
352
|
+
await failInstance(client, row, reason, { job_uid: jobUid ?? null });
|
|
218
353
|
});
|
|
219
354
|
}
|
|
220
355
|
/**
|
|
@@ -223,9 +358,12 @@ async function failInstance(client, row, reason) {
|
|
|
223
358
|
* 1. Looks up the user handler in the registry by `payload.handler`.
|
|
224
359
|
* 2. Loads the instance to grab the current context (no lock — handlers run
|
|
225
360
|
* outside the advance tx).
|
|
226
|
-
* 3.
|
|
227
|
-
*
|
|
228
|
-
*
|
|
361
|
+
* 3. Checks the fence: the handler runs only if the row is still at the `seq`
|
|
362
|
+
* that dispatched it. A row that moved on means this job is a duplicate or a
|
|
363
|
+
* zombie — skip, so the side effect does not fire twice.
|
|
364
|
+
* 4. Runs the handler.
|
|
365
|
+
* 5. On success: enqueues a `workflow.advance` job with the outcome + data.
|
|
366
|
+
* 6. On throw: re-throws so steve records the attempt as failed (steve retries
|
|
229
367
|
* per the job's `max_attempts`). When steve gives up, the on-failed hook in
|
|
230
368
|
* Workflow marks the instance failed.
|
|
231
369
|
*
|
|
@@ -234,14 +372,15 @@ async function failInstance(client, row, reason) {
|
|
|
234
372
|
* fails after the advance has been enqueued.
|
|
235
373
|
*/
|
|
236
374
|
export async function runEffect(pool, registry, enqueuer, payload, signal) {
|
|
237
|
-
const { instance_id,
|
|
375
|
+
const { instance_id, handler: handlerName } = payload;
|
|
376
|
+
const tenant_id = payloadTenantId(payload);
|
|
238
377
|
const handler = registry.requireHandler(handlerName);
|
|
239
378
|
const client = await pool.connect();
|
|
240
379
|
let row;
|
|
241
380
|
try {
|
|
242
|
-
const r = await client.query(`SELECT id,
|
|
381
|
+
const r = await client.query(`SELECT id, tenant_id, definition_id, definition_version, cursor,
|
|
243
382
|
previous_cursor, context, execution_state, wake_at,
|
|
244
|
-
correlation_token, created_at, updated_at
|
|
383
|
+
correlation_token, seq, created_at, updated_at
|
|
245
384
|
FROM __workflow_instances WHERE id = $1`, [instance_id]);
|
|
246
385
|
row = r.rows[0] ?? null;
|
|
247
386
|
}
|
|
@@ -251,29 +390,54 @@ export async function runEffect(pool, registry, enqueuer, payload, signal) {
|
|
|
251
390
|
if (!row) {
|
|
252
391
|
throw new Error(`effect: instance ${instance_id} not found`);
|
|
253
392
|
}
|
|
393
|
+
if (payload.seq !== undefined && row.seq !== payload.seq) {
|
|
394
|
+
if (row.seq < payload.seq) {
|
|
395
|
+
// The dispatching advance has not committed yet — steve's job insert
|
|
396
|
+
// is autocommit on its own connection, so the job can be claimed
|
|
397
|
+
// before the transaction that enqueued it lands. Throw to retry.
|
|
398
|
+
throw new Error(`effect: instance ${instance_id} not yet at seq ${payload.seq} (row.seq=${row.seq})`);
|
|
399
|
+
}
|
|
400
|
+
clog.debug?.(`effect: stale ${handlerName} (row.seq=${row.seq}, job seq=${payload.seq}); skipped`);
|
|
401
|
+
return { skipped: "stale" };
|
|
402
|
+
}
|
|
403
|
+
if (row.execution_state !== EXECUTION_STATE.RUNNING) {
|
|
404
|
+
clog.debug?.(`effect: ${handlerName} on a non-running instance (${row.execution_state}); skipped`);
|
|
405
|
+
return { skipped: "stale" };
|
|
406
|
+
}
|
|
254
407
|
const result = await handler({
|
|
255
408
|
instanceId: instance_id,
|
|
256
|
-
|
|
409
|
+
tenantId: tenant_id,
|
|
257
410
|
context: row.context,
|
|
258
411
|
signal,
|
|
259
412
|
});
|
|
260
413
|
await enqueuer.enqueueAdvance(pool, {
|
|
261
|
-
|
|
414
|
+
tenant_id,
|
|
262
415
|
instance_id,
|
|
416
|
+
kind: "effect",
|
|
417
|
+
expected_seq: payload.seq,
|
|
263
418
|
outcome: result.outcome,
|
|
264
419
|
outcome_data: result.data,
|
|
420
|
+
handler: handlerName,
|
|
421
|
+
correlation_token: result.correlationToken,
|
|
422
|
+
context_patch: result.context,
|
|
265
423
|
});
|
|
266
424
|
return { outcome: result.outcome, data: result.data };
|
|
267
425
|
}
|
|
268
426
|
/**
|
|
269
427
|
* Called by the Workflow layer when steve gives up on an effect job (all
|
|
270
|
-
* attempts exhausted or expired). Marks the workflow instance as failed
|
|
428
|
+
* attempts exhausted or expired). Marks the workflow instance as failed —
|
|
429
|
+
* unless the job is fenced out, i.e. it belongs to a step the instance has
|
|
430
|
+
* already left, in which case a dead job must not kill a healthy instance.
|
|
271
431
|
*/
|
|
272
432
|
export async function failEffectJob(pool, payload, reason) {
|
|
273
433
|
await withTransaction(pool, async (client) => {
|
|
274
434
|
const row = await lockInstance(client, payload.instance_id);
|
|
275
435
|
if (!row)
|
|
276
436
|
return;
|
|
437
|
+
if (payload.seq !== undefined && row.seq !== payload.seq) {
|
|
438
|
+
clog.debug?.(`failEffectJob: stale (row.seq=${row.seq}, job seq=${payload.seq}); no-op`);
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
277
441
|
if (row.execution_state === EXECUTION_STATE.COMPLETED ||
|
|
278
442
|
row.execution_state === EXECUTION_STATE.FAILED ||
|
|
279
443
|
row.execution_state === EXECUTION_STATE.CANCELLED) {
|
|
@@ -281,9 +445,9 @@ export async function failEffectJob(pool, payload, reason) {
|
|
|
281
445
|
}
|
|
282
446
|
await updateInstance(client, row.id, {
|
|
283
447
|
execution_state: EXECUTION_STATE.FAILED,
|
|
284
|
-
});
|
|
448
|
+
}, { bumpSeq: true });
|
|
285
449
|
await appendHistory(client, {
|
|
286
|
-
|
|
450
|
+
tenant_id: row.tenant_id,
|
|
287
451
|
instance_id: row.id,
|
|
288
452
|
event_type: HISTORY_EVENT.EFFECT_FAILED,
|
|
289
453
|
from_node: row.cursor,
|
|
@@ -295,4 +459,4 @@ export async function failEffectJob(pool, payload, reason) {
|
|
|
295
459
|
export function effectJobType(handlerName) {
|
|
296
460
|
return `${JOB_TYPE_EFFECT_PREFIX}${handlerName}`;
|
|
297
461
|
}
|
|
298
|
-
export { JOB_TYPE_ADVANCE };
|
|
462
|
+
export { JOB_TYPE_ADVANCE, PURE_ENTER_EVENT, SIGNAL_MATCHED_EVENT };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Forward migration for schema version 1.1.0 — renames `project_id` to
|
|
3
|
+
* `tenant_id` on all three framework tables, aligning with the ecosystem's
|
|
4
|
+
* tenant-scoping convention (`@marianmeres/cron`, `@marianmeres/steve`).
|
|
5
|
+
*
|
|
6
|
+
* The rename is in place: existing data is preserved. Idempotent — safe to run
|
|
7
|
+
* against a schema that already uses `tenant_id`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function up(ctx?: Record<string, unknown>): Promise<void>;
|
|
10
|
+
/** Reverse migration — renames `tenant_id` back to `project_id`. Non-destructive. */
|
|
11
|
+
export declare function down(ctx?: Record<string, unknown>): Promise<void>;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builds an idempotent `RENAME COLUMN` guarded on the source column existing
|
|
3
|
+
* and the target not existing — so it is a no-op on an already-migrated schema.
|
|
4
|
+
* Postgres has no `IF EXISTS` for `RENAME COLUMN`, hence the DO block.
|
|
5
|
+
*
|
|
6
|
+
* The guard is scoped to `current_schema()` because the `ALTER TABLE` it guards
|
|
7
|
+
* is unqualified and so resolves through `search_path`: without the predicate a
|
|
8
|
+
* same-named table in *another* schema decides whether this schema gets renamed.
|
|
9
|
+
*
|
|
10
|
+
* Index definitions follow a renamed column automatically, and none of this
|
|
11
|
+
* package's index *names* embed the column name, so nothing else to adjust.
|
|
12
|
+
*/
|
|
13
|
+
function renameColumn(table, from, to) {
|
|
14
|
+
const col = (c) => `EXISTS (SELECT 1 FROM information_schema.columns
|
|
15
|
+
WHERE table_schema = current_schema()
|
|
16
|
+
AND table_name = '${table}' AND column_name = '${c}')`;
|
|
17
|
+
return `
|
|
18
|
+
DO $$
|
|
19
|
+
BEGIN
|
|
20
|
+
IF ${col(from)} AND NOT ${col(to)} THEN
|
|
21
|
+
EXECUTE 'ALTER TABLE ${table} RENAME COLUMN ${from} TO ${to}';
|
|
22
|
+
END IF;
|
|
23
|
+
END $$;
|
|
24
|
+
`;
|
|
25
|
+
}
|
|
26
|
+
const TABLES = [
|
|
27
|
+
"__workflow_instances",
|
|
28
|
+
"__workflow_inbox",
|
|
29
|
+
"__workflow_history",
|
|
30
|
+
];
|
|
31
|
+
const SQL_UP = TABLES.map((t) => renameColumn(t, "project_id", "tenant_id")).join("\n");
|
|
32
|
+
const SQL_DOWN = TABLES.map((t) => renameColumn(t, "tenant_id", "project_id")).join("\n");
|
|
33
|
+
/**
|
|
34
|
+
* Forward migration for schema version 1.1.0 — renames `project_id` to
|
|
35
|
+
* `tenant_id` on all three framework tables, aligning with the ecosystem's
|
|
36
|
+
* tenant-scoping convention (`@marianmeres/cron`, `@marianmeres/steve`).
|
|
37
|
+
*
|
|
38
|
+
* The rename is in place: existing data is preserved. Idempotent — safe to run
|
|
39
|
+
* against a schema that already uses `tenant_id`.
|
|
40
|
+
*/
|
|
41
|
+
export async function up(ctx) {
|
|
42
|
+
const { pool } = ctx;
|
|
43
|
+
const client = await pool.connect();
|
|
44
|
+
try {
|
|
45
|
+
await client.query(SQL_UP);
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
client.release();
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/** Reverse migration — renames `tenant_id` back to `project_id`. Non-destructive. */
|
|
52
|
+
export async function down(ctx) {
|
|
53
|
+
const { pool } = ctx;
|
|
54
|
+
const client = await pool.connect();
|
|
55
|
+
try {
|
|
56
|
+
await client.query(SQL_DOWN);
|
|
57
|
+
}
|
|
58
|
+
finally {
|
|
59
|
+
client.release();
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Forward migration for schema version 1.2.0 — adds the `seq` fencing token to
|
|
3
|
+
* `__workflow_instances`, plus the partial index the scheduler's stale-`pending`
|
|
4
|
+
* scan runs on.
|
|
5
|
+
*
|
|
6
|
+
* Every settle-point write bumps `seq`, and every advance/effect job carries the
|
|
7
|
+
* `seq` it was issued against, so a duplicate or zombie job lands as a no-op
|
|
8
|
+
* instead of being applied to a row that has moved on.
|
|
9
|
+
*
|
|
10
|
+
* Additive and non-volatile, so on PG >= 11 the column is a metadata-only
|
|
11
|
+
* change: existing rows are not rewritten and start at `0`. The index is partial
|
|
12
|
+
* on a transient state, so it stays tiny.
|
|
13
|
+
*/
|
|
14
|
+
export declare function up(ctx?: Record<string, unknown>): Promise<void>;
|
|
15
|
+
/** Reverse migration — drops `seq`. Destructive only of the fence itself. */
|
|
16
|
+
export declare function down(ctx?: Record<string, unknown>): Promise<void>;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
const SQL_UP = `
|
|
2
|
+
ALTER TABLE __workflow_instances ADD COLUMN IF NOT EXISTS seq integer NOT NULL DEFAULT 0;
|
|
3
|
+
|
|
4
|
+
CREATE INDEX IF NOT EXISTS __workflow_instances_stale_pending_idx
|
|
5
|
+
ON __workflow_instances (tenant_id, updated_at) WHERE execution_state = 'pending';
|
|
6
|
+
`;
|
|
7
|
+
const SQL_DOWN = `
|
|
8
|
+
DROP INDEX IF EXISTS __workflow_instances_stale_pending_idx;
|
|
9
|
+
ALTER TABLE __workflow_instances DROP COLUMN IF EXISTS seq;
|
|
10
|
+
`;
|
|
11
|
+
/**
|
|
12
|
+
* Forward migration for schema version 1.2.0 — adds the `seq` fencing token to
|
|
13
|
+
* `__workflow_instances`, plus the partial index the scheduler's stale-`pending`
|
|
14
|
+
* scan runs on.
|
|
15
|
+
*
|
|
16
|
+
* Every settle-point write bumps `seq`, and every advance/effect job carries the
|
|
17
|
+
* `seq` it was issued against, so a duplicate or zombie job lands as a no-op
|
|
18
|
+
* instead of being applied to a row that has moved on.
|
|
19
|
+
*
|
|
20
|
+
* Additive and non-volatile, so on PG >= 11 the column is a metadata-only
|
|
21
|
+
* change: existing rows are not rewritten and start at `0`. The index is partial
|
|
22
|
+
* on a transient state, so it stays tiny.
|
|
23
|
+
*/
|
|
24
|
+
export async function up(ctx) {
|
|
25
|
+
const { pool } = ctx;
|
|
26
|
+
const client = await pool.connect();
|
|
27
|
+
try {
|
|
28
|
+
await client.query(SQL_UP);
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
client.release();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Reverse migration — drops `seq`. Destructive only of the fence itself. */
|
|
35
|
+
export async function down(ctx) {
|
|
36
|
+
const { pool } = ctx;
|
|
37
|
+
const client = await pool.connect();
|
|
38
|
+
try {
|
|
39
|
+
await client.query(SQL_DOWN);
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
client.release();
|
|
43
|
+
}
|
|
44
|
+
}
|
package/dist/migrations/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Migrate } from "@marianmeres/migrate";
|
|
2
|
-
import {
|
|
2
|
+
import { down as down_1_0_0, up as up_1_0_0 } from "./1_0_0.js";
|
|
3
|
+
import { down as down_1_1_0, up as up_1_1_0 } from "./1_1_0.js";
|
|
4
|
+
import { down as down_1_2_0, up as up_1_2_0 } from "./1_2_0.js";
|
|
3
5
|
const VERSION_TABLE = "__workflow_migrations";
|
|
4
6
|
/**
|
|
5
7
|
* Builds a {@link Migrate} instance pre-loaded with the package's schema
|
|
@@ -25,6 +27,8 @@ export function createMigrate(pool, options = {}) {
|
|
|
25
27
|
logger: options.logger,
|
|
26
28
|
}, { pool });
|
|
27
29
|
migrate.addVersion("1.0.0", up_1_0_0, down_1_0_0);
|
|
30
|
+
migrate.addVersion("1.1.0", up_1_1_0, down_1_1_0);
|
|
31
|
+
migrate.addVersion("1.2.0", up_1_2_0, down_1_2_0);
|
|
28
32
|
return migrate;
|
|
29
33
|
}
|
|
30
34
|
async function ensureVersionTable(client) {
|
package/dist/mod.d.ts
CHANGED
|
@@ -23,7 +23,10 @@
|
|
|
23
23
|
* const pool = new pg.Pool({ /* ... *\/ });
|
|
24
24
|
* await createMigrate(pool).up("latest");
|
|
25
25
|
*
|
|
26
|
-
*
|
|
26
|
+
* // `autoCleanup` runs steve's reaper, which is what turns a crashed worker
|
|
27
|
+
* // into an `expired` job the framework can re-dispatch. Without it such a
|
|
28
|
+
* // job stays `running` forever and its instance never moves again.
|
|
29
|
+
* const jobs = new Jobs({ db: pool, autoCleanup: true });
|
|
27
30
|
* const wf = new Workflow({
|
|
28
31
|
* db: pool,
|
|
29
32
|
* jobs,
|
|
@@ -36,11 +39,11 @@
|
|
|
36
39
|
* ```
|
|
37
40
|
*/
|
|
38
41
|
export { Workflow, type WorkflowOptions } from "./workflow.js";
|
|
39
|
-
export { WorkflowScheduler, type WorkflowSchedulerOptions, } from "./scheduler.js";
|
|
42
|
+
export { type SchedulerTickResult, WorkflowScheduler, type WorkflowSchedulerOptions, } from "./scheduler.js";
|
|
40
43
|
export { WorkflowInboxCorrelator, type WorkflowInboxCorrelatorOptions, } from "./correlator.js";
|
|
41
44
|
export { WorkflowRegistry } from "./registry.js";
|
|
42
|
-
export {
|
|
45
|
+
export { defKey, validateDefinition } from "./definition.js";
|
|
43
46
|
export { createMigrate } from "./migrations/index.js";
|
|
44
47
|
export { getHistory } from "./persistence/history.js";
|
|
45
|
-
export { effectJobType, JOB_TYPE_ADVANCE, PURE_ENTER_EVENT
|
|
46
|
-
export {
|
|
48
|
+
export { effectJobType, JOB_TYPE_ADVANCE, PURE_ENTER_EVENT } from "./driver.js";
|
|
49
|
+
export { type AdvanceJobPayload, type AdvanceKind, DEFAULT_TENANT_ID, type EffectJobPayload, EXECUTION_STATE, type ExecutionState, type Handler, type HandlerArgs, type HandlerResult, HISTORY_EVENT, type HistoryEventType, type HistoryRow, type InboxRow, JOB_TYPE_EFFECT_PREFIX, type Matcher, type MatcherArgs, type NodeMeta, type WorkflowContext, type WorkflowDefinition, type WorkflowFSMConfig, type WorkflowInstanceRow, type WorkflowSnapshot, type WorkflowStateConfig, } from "./types.js";
|