@marianmeres/workflow 1.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +207 -94
- package/API.md +277 -157
- package/README.md +182 -83
- package/dist/correlator.d.ts +31 -14
- package/dist/correlator.js +130 -73
- package/dist/definition.d.ts +5 -1
- package/dist/definition.js +54 -1
- package/dist/driver.d.ts +28 -23
- package/dist/driver.js +223 -59
- package/dist/migrations/1_1_0.d.ts +11 -0
- package/dist/migrations/1_1_0.js +61 -0
- package/dist/migrations/1_2_0.d.ts +16 -0
- package/dist/migrations/1_2_0.js +44 -0
- package/dist/migrations/index.js +5 -1
- package/dist/mod.d.ts +8 -5
- package/dist/mod.js +7 -4
- package/dist/persistence/history.d.ts +1 -1
- package/dist/persistence/history.js +3 -3
- package/dist/persistence/inbox.d.ts +9 -3
- package/dist/persistence/inbox.js +16 -7
- package/dist/persistence/instances.d.ts +46 -13
- package/dist/persistence/instances.js +78 -33
- package/dist/scheduler.d.ts +33 -9
- package/dist/scheduler.js +45 -21
- package/dist/types.d.ts +126 -13
- package/dist/types.js +28 -2
- package/dist/workflow.d.ts +76 -4
- package/dist/workflow.js +237 -35
- package/docs/_archive/analysis/00-overview-and-roadmap.md +233 -0
- package/docs/_archive/analysis/01-durability.md +404 -0
- package/docs/_archive/analysis/02-correlation.md +228 -0
- package/docs/_archive/analysis/03-definition-and-api.md +248 -0
- package/docs/_archive/analysis/04-docs-tests-ops.md +191 -0
- package/docs/_archive/analysis/PROGRESS.md +172 -0
- package/docs/_archive/analysis/README.md +38 -0
- package/package.json +8 -7
package/dist/workflow.js
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
|
-
import { JOB_STATUS
|
|
2
|
-
import { effectJobType, failEffectJob, JOB_TYPE_ADVANCE, runAdvance, runEffect, } from "./driver.js";
|
|
1
|
+
import { JOB_STATUS } from "@marianmeres/steve";
|
|
2
|
+
import { effectJobType, failAdvanceJob, failEffectJob, JOB_TYPE_ADVANCE, runAdvance, runEffect, } from "./driver.js";
|
|
3
3
|
import { clog } from "./log.js";
|
|
4
4
|
import { appendHistory } from "./persistence/history.js";
|
|
5
5
|
import { appendInbox } from "./persistence/inbox.js";
|
|
6
|
-
import { createInstance, findInstance, } from "./persistence/instances.js";
|
|
6
|
+
import { createInstance, findInstance, lockInstance, updateInstance, } from "./persistence/instances.js";
|
|
7
|
+
import { withTransaction } from "./persistence/tx.js";
|
|
7
8
|
import { WorkflowRegistry } from "./registry.js";
|
|
8
|
-
import {
|
|
9
|
+
import { DEFAULT_TENANT_ID, EXECUTION_STATE, HISTORY_EVENT, } from "./types.js";
|
|
10
|
+
/**
|
|
11
|
+
* Steve's `setHandler` is last-writer-wins, so a second `Workflow` on the same
|
|
12
|
+
* `Jobs` would silently route every `workflow.advance` through its own registry
|
|
13
|
+
* and fail the first one's instances with "Unknown workflow definition". The
|
|
14
|
+
* constructor refuses that; {@link Workflow.detach} releases the claim.
|
|
15
|
+
*/
|
|
16
|
+
const attachedWorkflows = new WeakMap();
|
|
9
17
|
/**
|
|
10
18
|
* The user-facing API for creating workflow instances and appending inbox
|
|
11
19
|
* signals.
|
|
@@ -22,14 +30,22 @@ import { DEFAULT_PROJECT_ID, HISTORY_EVENT, } from "./types.js";
|
|
|
22
30
|
export class Workflow {
|
|
23
31
|
db;
|
|
24
32
|
jobs;
|
|
25
|
-
|
|
33
|
+
tenantId;
|
|
26
34
|
registry;
|
|
27
35
|
#effectMaxAttempts;
|
|
28
36
|
#effectMaxAttemptDurationMs;
|
|
37
|
+
#advanceMaxAttempts;
|
|
38
|
+
#redispatchLimit;
|
|
39
|
+
#jobTypes = [];
|
|
40
|
+
#unsubscribers = [];
|
|
29
41
|
constructor(options) {
|
|
42
|
+
if (attachedWorkflows.has(options.jobs)) {
|
|
43
|
+
throw new Error("Workflow: a Workflow is already attached to this Jobs instance. " +
|
|
44
|
+
"Call detach() on it first, or pass a separate Jobs instance.");
|
|
45
|
+
}
|
|
30
46
|
this.db = options.db;
|
|
31
47
|
this.jobs = options.jobs;
|
|
32
|
-
this.
|
|
48
|
+
this.tenantId = options.tenantId ?? DEFAULT_TENANT_ID;
|
|
33
49
|
this.registry = new WorkflowRegistry({
|
|
34
50
|
definitions: options.definitions,
|
|
35
51
|
handlers: options.handlers,
|
|
@@ -37,7 +53,10 @@ export class Workflow {
|
|
|
37
53
|
});
|
|
38
54
|
this.#effectMaxAttempts = options.effectMaxAttempts ?? 3;
|
|
39
55
|
this.#effectMaxAttemptDurationMs = options.effectMaxAttemptDurationMs ?? 0;
|
|
56
|
+
this.#advanceMaxAttempts = options.advanceMaxAttempts ?? 10;
|
|
57
|
+
this.#redispatchLimit = options.redispatchLimit ?? 3;
|
|
40
58
|
// Register the advance handler.
|
|
59
|
+
this.#jobTypes.push(JOB_TYPE_ADVANCE);
|
|
41
60
|
this.jobs.setHandler(JOB_TYPE_ADVANCE, async (job, _signal) => {
|
|
42
61
|
await runAdvance(this.db, this.registry, this, job.payload);
|
|
43
62
|
return { ok: true };
|
|
@@ -45,29 +64,101 @@ export class Workflow {
|
|
|
45
64
|
// Register one job-type handler per registered effect handler.
|
|
46
65
|
const handlerNames = this.registry.handlerNames();
|
|
47
66
|
for (const name of handlerNames) {
|
|
67
|
+
this.#jobTypes.push(effectJobType(name));
|
|
48
68
|
this.jobs.setHandler(effectJobType(name), async (job, signal) => {
|
|
49
69
|
return await runEffect(this.db, this.registry, this, job.payload, signal);
|
|
50
70
|
});
|
|
51
71
|
}
|
|
52
|
-
// Watch
|
|
53
|
-
//
|
|
72
|
+
// Watch job outcomes. onDone fires on completed / failed / expired — we
|
|
73
|
+
// react only to non-success. `failed` means steve exhausted the attempts,
|
|
74
|
+
// `expired` means the worker died mid-run and steve will not retry it, so
|
|
75
|
+
// the two get different treatment: give up vs. re-dispatch.
|
|
54
76
|
for (const name of handlerNames) {
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
if (job.status === JOB_STATUS.
|
|
58
|
-
job.
|
|
59
|
-
|
|
77
|
+
const unsub = this.jobs.onDone(effectJobType(name), (job) => {
|
|
78
|
+
const payload = job.payload;
|
|
79
|
+
if (job.status === JOB_STATUS.EXPIRED) {
|
|
80
|
+
this.#redispatchEffect(job, payload).catch((e) => clog.error?.(`redispatchEffect: ${e}`));
|
|
81
|
+
}
|
|
82
|
+
else if (job.status === JOB_STATUS.FAILED) {
|
|
60
83
|
failEffectJob(this.db, payload, `steve: ${job.status}`).catch((e) => clog.error?.(`failEffectJob: ${e}`));
|
|
61
84
|
}
|
|
62
85
|
});
|
|
86
|
+
this.#unsubscribers.push(unsub);
|
|
87
|
+
}
|
|
88
|
+
const unsubAdvance = this.jobs.onDone(JOB_TYPE_ADVANCE, (job) => {
|
|
89
|
+
const payload = job.payload;
|
|
90
|
+
if (job.status === JOB_STATUS.EXPIRED) {
|
|
91
|
+
this.#redispatchAdvance(job, payload).catch((e) => clog.error?.(`redispatchAdvance: ${e}`));
|
|
92
|
+
}
|
|
93
|
+
else if (job.status === JOB_STATUS.FAILED) {
|
|
94
|
+
this.#failAdvance(job, payload).catch((e) => clog.error?.(`failAdvanceJob: ${e}`));
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
this.#unsubscribers.push(unsubAdvance);
|
|
98
|
+
attachedWorkflows.set(this.jobs, this);
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Removes everything the constructor registered on the `Jobs` instance — the
|
|
102
|
+
* `workflow.advance` and `workflow.effect.<name>` handlers plus the `onDone`
|
|
103
|
+
* subscriptions — and releases the one-Workflow-per-Jobs claim, so a
|
|
104
|
+
* replacement can be constructed on the same queue (tests, hot reload).
|
|
105
|
+
*
|
|
106
|
+
* Detaching does not stop the queue and does not touch running instances:
|
|
107
|
+
* jobs enqueued for this `Workflow` stay in the table and fall back to
|
|
108
|
+
* steve's noop handler until something re-attaches.
|
|
109
|
+
*
|
|
110
|
+
* A no-op when this instance is not the attached one (already detached, or
|
|
111
|
+
* superseded by a later `Workflow`).
|
|
112
|
+
*/
|
|
113
|
+
detach() {
|
|
114
|
+
if (attachedWorkflows.get(this.jobs) !== this)
|
|
115
|
+
return;
|
|
116
|
+
for (const type of this.#jobTypes)
|
|
117
|
+
this.jobs.setHandler(type, null);
|
|
118
|
+
for (const unsub of this.#unsubscribers)
|
|
119
|
+
unsub();
|
|
120
|
+
attachedWorkflows.delete(this.jobs);
|
|
121
|
+
}
|
|
122
|
+
/** `redispatch` value for the next dispatch, or `null` if the budget is spent. */
|
|
123
|
+
#nextRedispatch(current) {
|
|
124
|
+
const next = (current ?? 0) + 1;
|
|
125
|
+
return next < this.#redispatchLimit ? next : null;
|
|
126
|
+
}
|
|
127
|
+
async #redispatchEffect(job, payload) {
|
|
128
|
+
const next = this.#nextRedispatch(payload.redispatch);
|
|
129
|
+
if (next === null) {
|
|
130
|
+
await failEffectJob(this.db, payload, `steve: expired (redispatch limit ${this.#redispatchLimit} reached)`);
|
|
131
|
+
return;
|
|
63
132
|
}
|
|
133
|
+
clog.debug?.(`effect ${payload.handler} expired; re-dispatch #${next} (job ${job.uid})`);
|
|
134
|
+
await this.enqueueEffect(this.db, payload.handler, {
|
|
135
|
+
...payload,
|
|
136
|
+
redispatch: next,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
async #redispatchAdvance(job, payload) {
|
|
140
|
+
const next = this.#nextRedispatch(payload.redispatch);
|
|
141
|
+
if (next === null) {
|
|
142
|
+
await failAdvanceJob(this.db, payload, `advance job expired (redispatch limit ${this.#redispatchLimit} reached)`, job.uid);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
clog.debug?.(`advance expired; re-dispatch #${next} (job ${job.uid})`);
|
|
146
|
+
await this.enqueueAdvance(this.db, { ...payload, redispatch: next });
|
|
147
|
+
}
|
|
148
|
+
async #failAdvance(job, payload) {
|
|
149
|
+
const { attempts } = await this.jobs.find(job.uid, true);
|
|
150
|
+
const last = attempts?.at(-1)?.error_message;
|
|
151
|
+
await failAdvanceJob(this.db, payload, `advance job failed: ${last ?? `steve: ${job.status}`}`, job.uid);
|
|
64
152
|
}
|
|
65
153
|
/** Internal — used by the driver to enqueue an advance job. */
|
|
66
154
|
async enqueueAdvance(_client, payload) {
|
|
67
155
|
// Steve creates within its own connection/tx; passing the locked client
|
|
68
156
|
// would not gain us atomicity vs steve's bookkeeping. The advance is
|
|
69
|
-
//
|
|
70
|
-
|
|
157
|
+
// fenced (`expected_seq`), so an enqueue-but-no-commit race costs at most
|
|
158
|
+
// a duplicate job that no-ops.
|
|
159
|
+
await this.jobs.create(JOB_TYPE_ADVANCE, payload, {
|
|
160
|
+
max_attempts: this.#advanceMaxAttempts,
|
|
161
|
+
});
|
|
71
162
|
}
|
|
72
163
|
/** Internal — used by the driver to enqueue an effect job. */
|
|
73
164
|
async enqueueEffect(_client, handler, payload) {
|
|
@@ -79,37 +170,148 @@ export class Workflow {
|
|
|
79
170
|
/**
|
|
80
171
|
* Creates a new workflow instance and enqueues its first advance.
|
|
81
172
|
*
|
|
173
|
+
* @param input.context - shallow-merged over the definition's own
|
|
174
|
+
* `fsm.context` defaults (top-level keys replace, they do not deep-merge).
|
|
82
175
|
* @param input.correlationToken - optional, set up-front for signal-suspending
|
|
83
176
|
* nodes that need to be matchable from the moment the instance is born
|
|
84
177
|
* (e.g., outbound emails using UUID subaddressing).
|
|
85
178
|
*/
|
|
86
179
|
async create(input) {
|
|
87
180
|
const def = this.registry.requireDefinition(input.definitionId, input.definitionVersion);
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
181
|
+
// The driver always resumes via `FSM.fromSnapshot`, which takes the
|
|
182
|
+
// persisted context verbatim — so the definition's `context` defaults are
|
|
183
|
+
// only ever applied here. Same value/factory handling as fsm's own init.
|
|
184
|
+
const defaults = typeof def.fsm.context === "function"
|
|
185
|
+
? def.fsm.context()
|
|
186
|
+
: structuredClone(def.fsm.context ?? {});
|
|
187
|
+
// One transaction: a throwing enqueue rolls the instance back rather than
|
|
188
|
+
// leaving a row nothing will ever advance. Steve inserts on its own
|
|
189
|
+
// connection, so the job can be claimed before this commits — the advance
|
|
190
|
+
// throws on the not-yet-visible row and steve retries it.
|
|
191
|
+
return await withTransaction(this.db, async (client) => {
|
|
192
|
+
const row = await createInstance(client, {
|
|
193
|
+
tenant_id: this.tenantId,
|
|
194
|
+
definition_id: input.definitionId,
|
|
195
|
+
definition_version: input.definitionVersion,
|
|
196
|
+
cursor: def.fsm.initial,
|
|
197
|
+
context: { ...defaults, ...input.context },
|
|
198
|
+
correlation_token: input.correlationToken ?? null,
|
|
199
|
+
});
|
|
200
|
+
await appendHistory(client, {
|
|
201
|
+
tenant_id: this.tenantId,
|
|
202
|
+
instance_id: row.id,
|
|
203
|
+
event_type: HISTORY_EVENT.CREATED,
|
|
204
|
+
to_node: row.cursor,
|
|
205
|
+
data: {
|
|
206
|
+
definitionId: input.definitionId,
|
|
207
|
+
definitionVersion: input.definitionVersion,
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
await this.jobs.create(JOB_TYPE_ADVANCE, {
|
|
211
|
+
tenant_id: this.tenantId,
|
|
212
|
+
instance_id: row.id,
|
|
213
|
+
kind: "start",
|
|
214
|
+
expected_seq: row.seq,
|
|
215
|
+
}, { max_attempts: this.#advanceMaxAttempts });
|
|
216
|
+
return row;
|
|
95
217
|
});
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Operator escape hatch: aborts a live instance. The row goes `cancelled`
|
|
221
|
+
* (terminal), loses its `wake_at` and its correlation token — so no timer, no
|
|
222
|
+
* signal and no queued job can move it again: the `seq` bump makes every job
|
|
223
|
+
* already in flight for it stale, and a stale effect job skips its handler
|
|
224
|
+
* instead of running the side effect.
|
|
225
|
+
*
|
|
226
|
+
* Returns `false` when the instance does not exist, belongs to another tenant,
|
|
227
|
+
* or is already terminal.
|
|
228
|
+
*
|
|
229
|
+
* @param reason - free-form, recorded on the `cancelled` history row.
|
|
230
|
+
*/
|
|
231
|
+
async cancel(id, reason) {
|
|
232
|
+
return await withTransaction(this.db, async (client) => {
|
|
233
|
+
const row = await lockInstance(client, id);
|
|
234
|
+
if (!row || row.tenant_id !== this.tenantId)
|
|
235
|
+
return false;
|
|
236
|
+
if (row.execution_state === EXECUTION_STATE.COMPLETED ||
|
|
237
|
+
row.execution_state === EXECUTION_STATE.FAILED ||
|
|
238
|
+
row.execution_state === EXECUTION_STATE.CANCELLED) {
|
|
239
|
+
return false;
|
|
240
|
+
}
|
|
241
|
+
await updateInstance(client, row.id, {
|
|
242
|
+
execution_state: EXECUTION_STATE.CANCELLED,
|
|
243
|
+
wake_at: null,
|
|
244
|
+
correlation_token: null,
|
|
245
|
+
}, { bumpSeq: true });
|
|
246
|
+
await appendHistory(client, {
|
|
247
|
+
tenant_id: this.tenantId,
|
|
248
|
+
instance_id: row.id,
|
|
249
|
+
event_type: HISTORY_EVENT.CANCELLED,
|
|
250
|
+
from_node: row.cursor,
|
|
251
|
+
data: { reason: reason ?? null },
|
|
252
|
+
});
|
|
253
|
+
return true;
|
|
102
254
|
});
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Operator escape hatch: resumes a `failed` instance from its current cursor,
|
|
258
|
+
* rather than starting a new one and replaying every side effect from the top.
|
|
259
|
+
* The row goes back to `pending` and a fresh `start` advance re-runs whatever
|
|
260
|
+
* the cursor node calls for — typically re-dispatching its effect.
|
|
261
|
+
*
|
|
262
|
+
* Nothing is fixed by retrying alone: an instance failed by a rejected
|
|
263
|
+
* transition or a deterministically throwing handler will fail again unless
|
|
264
|
+
* the code changed.
|
|
265
|
+
*
|
|
266
|
+
* Returns `false` when the instance does not exist, belongs to another tenant,
|
|
267
|
+
* or is in a state this does not cover.
|
|
268
|
+
*
|
|
269
|
+
* @param opts.force - also allow a `running` instance, i.e. the operator
|
|
270
|
+
* asserting that its effect job is dead (a worker crashed with no reaper to
|
|
271
|
+
* expire the job). Safe by construction: the `seq` bump makes the zombie
|
|
272
|
+
* stale, so a late completion is dropped.
|
|
273
|
+
*/
|
|
274
|
+
async retry(id, opts = {}) {
|
|
275
|
+
const settled = await withTransaction(this.db, async (client) => {
|
|
276
|
+
const row = await lockInstance(client, id);
|
|
277
|
+
if (!row || row.tenant_id !== this.tenantId)
|
|
278
|
+
return null;
|
|
279
|
+
const allowed = row.execution_state === EXECUTION_STATE.FAILED ||
|
|
280
|
+
(opts.force === true &&
|
|
281
|
+
row.execution_state === EXECUTION_STATE.RUNNING);
|
|
282
|
+
if (!allowed)
|
|
283
|
+
return null;
|
|
284
|
+
const updated = await updateInstance(client, row.id, {
|
|
285
|
+
execution_state: EXECUTION_STATE.PENDING,
|
|
286
|
+
wake_at: null,
|
|
287
|
+
}, { bumpSeq: true });
|
|
288
|
+
await appendHistory(client, {
|
|
289
|
+
tenant_id: this.tenantId,
|
|
290
|
+
instance_id: row.id,
|
|
291
|
+
event_type: HISTORY_EVENT.RETRIED,
|
|
292
|
+
from_node: row.cursor,
|
|
293
|
+
data: { from_state: row.execution_state },
|
|
294
|
+
});
|
|
295
|
+
return updated;
|
|
106
296
|
});
|
|
107
|
-
|
|
297
|
+
if (!settled)
|
|
298
|
+
return false;
|
|
299
|
+
// After the commit, not inside it: an advance claimed before the commit
|
|
300
|
+
// landed would see the row still `failed` at the old `seq` and drop itself
|
|
301
|
+
// as stale. A crash in this gap leaves a `pending` row, which the
|
|
302
|
+
// scheduler's stale-pending scan re-pokes.
|
|
303
|
+
await this.enqueueAdvance(this.db, {
|
|
304
|
+
tenant_id: this.tenantId,
|
|
305
|
+
instance_id: settled.id,
|
|
306
|
+
kind: "start",
|
|
307
|
+
expected_seq: settled.seq,
|
|
308
|
+
});
|
|
309
|
+
return true;
|
|
108
310
|
}
|
|
109
|
-
/** Looks up an instance by id, scoped to this Workflow's
|
|
311
|
+
/** Looks up an instance by id, scoped to this Workflow's tenant. */
|
|
110
312
|
async find(id) {
|
|
111
313
|
const row = await findInstance(this.db, id);
|
|
112
|
-
if (row && row.
|
|
314
|
+
if (row && row.tenant_id !== this.tenantId)
|
|
113
315
|
return null;
|
|
114
316
|
return row;
|
|
115
317
|
}
|
|
@@ -120,7 +322,7 @@ export class Workflow {
|
|
|
120
322
|
*/
|
|
121
323
|
async appendInbox(input) {
|
|
122
324
|
return await appendInbox(this.db, {
|
|
123
|
-
|
|
325
|
+
tenant_id: this.tenantId,
|
|
124
326
|
source: input.source,
|
|
125
327
|
correlation_token: input.correlationToken,
|
|
126
328
|
payload: input.payload,
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
GENERATED ANALYSIS — @marianmeres/workflow
|
|
3
|
+
Produced 2026-08-26 by an inline single-agent review: full read of src/, tests/, docs, and the
|
|
4
|
+
resolved sources of @marianmeres/fsm 3.1.0, steve 3.0.0, cron 3.2.0 → self-verification pass
|
|
5
|
+
re-opening every cited line → synthesis. Claims verified against the codebase at commit 02a7635.
|
|
6
|
+
Planning artifact; no code was changed.
|
|
7
|
+
-->
|
|
8
|
+
|
|
9
|
+
# @marianmeres/workflow — Analysis Overview & Roadmap
|
|
10
|
+
|
|
11
|
+
> **Overall verdict: the architecture is right and the code is clean; nothing needs
|
|
12
|
+
> rethinking.** The three-layer model (definition / instance / driver), the two-column
|
|
13
|
+
> state (`cursor` + `execution_state`), pure-data definitions with string handler
|
|
14
|
+
> references, pin-and-freeze versioning, the `meta`-driven dispatch over fsm 3.1's
|
|
15
|
+
> `fromSnapshot`, the append-only history, and the migration ledger are all sound and
|
|
16
|
+
> should stay exactly as they are. The four dimension docs found no design-level problem.
|
|
17
|
+
|
|
18
|
+
> **The one systemic gap: the driver assumes exactly-once job delivery; steve delivers
|
|
19
|
+
> at-least-once and never retries a crashed job.** There is no fencing token, so a
|
|
20
|
+
> duplicate or stale job is not ignored — it is applied: a stale outcome is rejected by the
|
|
21
|
+
> FSM and marks a healthy instance `failed`, a replayed initial advance dispatches the same
|
|
22
|
+
> effect twice, and an `expired` effect job from a step the instance already left marks it
|
|
23
|
+
> `failed`. The scheduler's claim-then-enqueue and `create()`'s insert-then-enqueue leave
|
|
24
|
+
> rows `pending` with no job if the process dies in between; `workflow.advance` failures
|
|
25
|
+
> are never observed. For a package whose reason to exist is "survive restarts and sleep
|
|
26
|
+
> for days," this is the thing to fix first — and one mechanism (a per-instance `seq`
|
|
27
|
+
> carried as `expected_seq` on every job, with ticks reduced to idempotent pokes) closes
|
|
28
|
+
> all of it. Detail: [`01-durability.md`](./01-durability.md).
|
|
29
|
+
|
|
30
|
+
> **The bug you will most likely hit first in practice: signals that arrive early are
|
|
31
|
+
> thrown away, and a signal landing on a timer-only wait fails the instance.** The
|
|
32
|
+
> correlator only looks for `waiting` instances and consumes every inbox row it examines,
|
|
33
|
+
> so a supplier reply that arrives while `send_order` is still running — or during any
|
|
34
|
+
> delay node — is marked processed and lost; the instance then waits out its full timeout.
|
|
35
|
+
> Replies arrive whenever they like; the fix is to defer instead of drop, deliver only to
|
|
36
|
+
> nodes that accept `MATCHED`, and mark the inbox row processed inside the advance
|
|
37
|
+
> transaction. Detail: [`02-correlation.md`](./02-correlation.md).
|
|
38
|
+
|
|
39
|
+
> **The rest is edges and honesty.** The docs promise exactly the guarantees the driver
|
|
40
|
+
> lacks (a single transaction with the job insert; "steve retries" — it marks `expired`;
|
|
41
|
+
> and `autoCleanup`, the switch that makes any crash recovery happen, is never mentioned).
|
|
42
|
+
> There is no `cancel()` or `retry()`. The validator accepts definitions that must fail at
|
|
43
|
+
> runtime. `create()` ignores the definition's `context` defaults. Two `Workflow`s on one
|
|
44
|
+
> `Jobs` silently overwrite each other. Detail: [`03-definition-and-api.md`](./03-definition-and-api.md),
|
|
45
|
+
> [`04-docs-tests-ops.md`](./04-docs-tests-ops.md).
|
|
46
|
+
|
|
47
|
+
> **How to read `docs/analysis/`.** This file is the map. Read `01-durability.md` first —
|
|
48
|
+
> its #1 (the fence) is the foundation every other durability change and the correlator
|
|
49
|
+
> redesign build on — then `02-correlation.md`, then `03` and `04` in any order. Each doc
|
|
50
|
+
> has a ranked summary, findings with `file:line` evidence and a **Done when** line, and an
|
|
51
|
+
> "Open questions" section; [`PROGRESS.md`](./PROGRESS.md) is the tracker and the decisions
|
|
52
|
+
> log.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## Top recommendations across all dimensions (ranked)
|
|
57
|
+
|
|
58
|
+
Ranked by impact on "this package is about to run real workflows" × feasibility for a solo
|
|
59
|
+
maintainer. Effort: S = small, M = medium.
|
|
60
|
+
|
|
61
|
+
| Rank | Recommendation | Dimension (doc) | Value | Effort | Risk | Why now |
|
|
62
|
+
| ---- | ------------------------------------------------------------------------------------------------------------------- | ----------------------- | ----- | ------ | ---- | ------------------------------------------------------------------------------------------------------------------------------- |
|
|
63
|
+
| 1 | Fencing token `seq` on instances; `expected_seq` + `kind` on every advance; fenced `runEffect` / `failEffectJob` | durability (01 #1) | high | M | med | Every duplicate/stale job today corrupts an instance; every other durability fix needs this. Migration is additive, shimmed. |
|
|
64
|
+
| 2 | Correlator: defer early signals, deliver only to `MATCHED`-accepting waits, processed-in-advance, same-tick dedupe | correlation (02 #1, #2) | high | M | low | Realistic data loss in the primary use case (email replies). Needs #1 for the fenced poke. |
|
|
65
|
+
| 3 | Observe `workflow.advance` failures; bounded re-dispatch of `expired` jobs; `advanceMaxAttempts` | durability (01 #3) | high | S | low | Today a throwing guard or a 10 s DB blip strands an instance silently; a crashed worker fails it (or strands it forever). |
|
|
66
|
+
| 4 | Scheduler tick becomes a read-only poke; stale-`pending` re-poke; `create()` in a transaction | durability (01 #2) | high | S–M | low | Closes the two "pending with no job, forever" crash windows. Safe only after #1 and #2 (see sequencing). |
|
|
67
|
+
| 5 | Docs: correct the three durability over-claims, document `autoCleanup`, the fence/poke model, signal semantics | docs (04 #1) | high | S | low | The current text tells consumers not to add safety nets the code needs. Do last in the sprint so it describes shipped behavior. |
|
|
68
|
+
| 6 | `cancel()` / `retry()` admin API | durability (01 #4) | med | S | low | First thing an operator asks for; the constants and terminal checks already exist. Needs #1. |
|
|
69
|
+
| 7 | Validator: reject definitions that must fail at runtime (`pure` without `ENTER`, `timeoutSec` without `TIMEOUT`, …) | definition (03 #1) | med | S | low | Turns a deterministic runtime `failed` into a construction-time error. |
|
|
70
|
+
| 8 | `create()` seeds context from `def.fsm.context` | definition (03 #2) | med | S | low | Definitions declaring defaults are silently ignored today. |
|
|
71
|
+
| 9 | `HandlerResult.correlationToken` — set the token from the effect that produced it | correlation (02 #3) | med | S | low | `Message-ID` threading and provider session ids are impossible with a create-time-only token. Needs #1's payload shape. |
|
|
72
|
+
| 10 | Guard: one `Workflow` per `Jobs` (+ `detach()`) | definition (03 #3a) | med | S | low | Silent handler overwrite; a WeakMap and one throw. |
|
|
73
|
+
| 11 | Migration 1.1.0 guard: `table_schema = current_schema()` | ops (04 #2) | med | S | low | Wrong result in multi-schema databases; one line. |
|
|
74
|
+
| 12 | Driver tests: pure routing, hop guard, rejection paths, matcher rejection; shared `waitUntil` | tests (04 #3) | med | M | low | The untested branches are the ones the sprint touches. |
|
|
75
|
+
| 13 | Tenant-per-call runtime (one `Workflow` serves many tenants; `"*"` ticks) — **deferred 2026-08-26** | definition (03 #3b) | med | M | med | Guard only (T10) for now; revisit when several tenants must share one `Workflow`/`Jobs` in one process. |
|
|
76
|
+
| 14 | Opt-in `HandlerResult.context` patch — **approved 2026-08-26** | definition (03 #4) | med | S | low | Removes the per-edge `action` boilerplate; explicit per handler; `data` stays the fsm payload. |
|
|
77
|
+
| 15 | Typed `meta` (`WorkflowFSMConfig`) | definition (03 #5) | low | S | low | Editor feedback only; kept because it is trivially cheap. |
|
|
78
|
+
| 16 | `deno fmt` / `deno lint` clean + `deno task check` | ops (04 #4) | low | S | low | Red baseline blocks using them as `Verify:` commands. Markdown reflow is the owner's call. |
|
|
79
|
+
|
|
80
|
+
Deliberately **omitted** as low-value at the stated scale or out of scope: inbox/history
|
|
81
|
+
retention purge (revisit when the tables grow), snapshot-into-instance (deferred by the
|
|
82
|
+
design brief itself), a "latest registered version" resolver for `create()`, a
|
|
83
|
+
transactional `Jobs.create` in steve (unnecessary once the fence exists — the at-least-once
|
|
84
|
+
model is fine when duplicates are no-ops), a partial unique index on waiting
|
|
85
|
+
`correlation_token`s (would turn a definition mistake into a failed advance instead of a
|
|
86
|
+
matcher decision), and the fsm debug lines emitted per advance (filterable).
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## Recommended first sprint (do these 5 first)
|
|
91
|
+
|
|
92
|
+
**1. Fencing token (Rank 1 — `01-durability.md` #1).** First because everything else in
|
|
93
|
+
the sprint stands on it. Migration `1.2.0` adds `seq integer NOT NULL DEFAULT 0` to
|
|
94
|
+
`__workflow_instances`; every settle-point write in `runAdvance` bumps it; every advance
|
|
95
|
+
payload carries `kind` (`start` / `effect` / `timeout` / `signal`) and `expected_seq`;
|
|
96
|
+
every effect payload carries `seq`. `runAdvance` no-ops (debug log, no history) when the
|
|
97
|
+
row's `seq` differs or the kind's precondition fails (`start` needs `pending`, `effect`
|
|
98
|
+
needs `running`, `timeout`/`signal` need `waiting`); `runEffect` skips the handler on a
|
|
99
|
+
stale `seq`; `failEffectJob` ignores stale jobs. `create()` moves inside a transaction.
|
|
100
|
+
Jobs queued by 2.0.x (no `expected_seq`) are treated as unfenced — a shim in the same
|
|
101
|
+
spirit as `payloadTenantId`, removed together. Scope: one migration, `types.ts`,
|
|
102
|
+
`driver.ts`, `workflow.ts`, payload call sites in scheduler/correlator, a new
|
|
103
|
+
`tests/durability.test.ts`.
|
|
104
|
+
|
|
105
|
+
**2. Correlator semantics (Rank 2 — `02-correlation.md` #1, #2).** Second because it is
|
|
106
|
+
the first bug real traffic will produce, and its fenced poke needs Rank 1. Look the token
|
|
107
|
+
up among _live_ instances; defer (leave unprocessed) when the instance is not yet
|
|
108
|
+
`waiting` or its node has no `MATCHED` edge; mark processed only for unknown/terminal
|
|
109
|
+
tokens or a matcher `false`; on matcher `true` enqueue `{ kind: "signal", inbox_id,
|
|
110
|
+
expected_seq }` and let the advance apply `MATCHED` and mark the inbox row processed in
|
|
111
|
+
its own transaction; per-tick dedupe by instance; a throwing matcher leaves the row for
|
|
112
|
+
the next tick. Delete the correlator's instance `UPDATE`.
|
|
113
|
+
|
|
114
|
+
**3. Scheduler poke + stale-`pending` re-poke (Rank 4 — `01-durability.md` #2).** After
|
|
115
|
+
Rank 2, because the stale-`pending` re-poke is only safe once `create()` is the sole
|
|
116
|
+
producer of `pending`. The tick becomes two `SELECT`s and a loop of fenced pokes; the
|
|
117
|
+
advance re-validates `waiting && wake_at <= now()` under the lock and clears `wake_at`
|
|
118
|
+
itself. `claimDueWakeUps` is deleted; a partial index for the `pending` scan rides in
|
|
119
|
+
migration `1.2.0`.
|
|
120
|
+
|
|
121
|
+
**4. Job-failure observation (Rank 3 — `01-durability.md` #3).** Independent of 2 and 3,
|
|
122
|
+
needs only Rank 1. `onDone(JOB_TYPE_ADVANCE)`: `failed` → instance `failed` with the
|
|
123
|
+
attempt's error message; `expired` → re-dispatch the identical payload, bounded by
|
|
124
|
+
`redispatchLimit` (3). Effect types: `expired` → same bounded re-dispatch instead of
|
|
125
|
+
failing. New `advanceMaxAttempts` (10). This is what turns "steve marks it expired" into
|
|
126
|
+
recovery.
|
|
127
|
+
|
|
128
|
+
**5. Docs pass (Rank 5 — `04-docs-tests-ops.md` #1).** Last, so it documents what shipped:
|
|
129
|
+
the at-least-once/fenced model, the poke ticks, the signal deferral rules, `autoCleanup:
|
|
130
|
+
true` in the README example, the new options, and the cron/steve noop-handler hazards.
|
|
131
|
+
|
|
132
|
+
> Sequencing: T01 → T02 → T03; T01 → T04; all four → T05. Rank 6 (`cancel`/`retry`) is the
|
|
133
|
+
> natural first backlog pick — it is small and its safety comes from Rank 1.
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## Cross-cutting themes
|
|
138
|
+
|
|
139
|
+
- **Exactly-once assumed, at-least-once delivered.** The comment at `workflow.ts:152–154`
|
|
140
|
+
("the advance is idempotent (cursor-aware)") is the single assumption behind 01 #1, #2,
|
|
141
|
+
#3 and the "retries" wording in 04 #1. Steve is honest about its model
|
|
142
|
+
(`_mark-expired.ts:6–8`); the driver just has no way to tell a fresh job from a stale
|
|
143
|
+
one. The fence is the one primitive that makes every other fix a local change.
|
|
144
|
+
|
|
145
|
+
- **"The advance is the only writer."** After the sprint, ticks never mutate instance
|
|
146
|
+
rows; `runAdvance` (and `cancel`/`retry`) do, under `FOR UPDATE`, bumping `seq`. That
|
|
147
|
+
invariant is why duplicate pokes are harmless, why the correlator no longer needs its
|
|
148
|
+
`pending` flip, and why the inbox row can be marked processed atomically with the
|
|
149
|
+
transition. It is worth stating in `AGENTS.md` as convention #8.
|
|
150
|
+
|
|
151
|
+
- **Signals are events, not requests.** They arrive whenever they like — before the wait,
|
|
152
|
+
during a delay, twice in a minute, after completion. 02 #1/#2 and 03 #1 (the validator's
|
|
153
|
+
`matcher ⇒ MATCHED` rule) are the same theme from the runtime and definition sides.
|
|
154
|
+
|
|
155
|
+
- **Docs describe intent, code describes behavior.** Three durability sentences and the
|
|
156
|
+
missing `autoCleanup` mention (04 #1) are not carelessness — they describe the design the
|
|
157
|
+
code was meant to implement. Fix the code, then make the docs true; do not fix the docs
|
|
158
|
+
first.
|
|
159
|
+
|
|
160
|
+
- **Small additive API completions.** `cancel`/`retry`, `correlationToken` from a handler,
|
|
161
|
+
context defaults, the double-attach guard, typed `meta` — each is a few dozen lines and
|
|
162
|
+
none changes an existing signature (the ignored `_client` parameter on the internal
|
|
163
|
+
enqueue methods is the one deliberate removal).
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## Dependency / sequencing notes
|
|
168
|
+
|
|
169
|
+
```mermaid
|
|
170
|
+
flowchart TD
|
|
171
|
+
T01["T01 fence: seq + expected_seq + kind"]
|
|
172
|
+
T02["T02 correlator: defer / MATCHED check / processed-in-advance"]
|
|
173
|
+
T03["T03 scheduler poke + stale-pending re-poke"]
|
|
174
|
+
T04["T04 advance onDone + expired re-dispatch"]
|
|
175
|
+
T05["T05 docs pass"]
|
|
176
|
+
T07["T07 cancel() / retry()"]
|
|
177
|
+
T08["T08 HandlerResult.correlationToken"]
|
|
178
|
+
T17["T17 HandlerResult.context (approved)"]
|
|
179
|
+
T10["T10 one Workflow per Jobs guard"]
|
|
180
|
+
T11["T11 tenant-per-call runtime (deferred)"]
|
|
181
|
+
T06["T06 validator rules"]
|
|
182
|
+
T09["T09 context defaults in create()"]
|
|
183
|
+
T12["T12 migration table_schema"]
|
|
184
|
+
T13["T13 driver tests"]
|
|
185
|
+
T14["T14 typed meta"]
|
|
186
|
+
T15["T15 fmt / lint / check task (approved)"]
|
|
187
|
+
T16["T16 release (human)"]
|
|
188
|
+
|
|
189
|
+
T01 --> T02 --> T03
|
|
190
|
+
T01 --> T04
|
|
191
|
+
T02 --> T05
|
|
192
|
+
T03 --> T05
|
|
193
|
+
T04 --> T05
|
|
194
|
+
T01 --> T07
|
|
195
|
+
T01 --> T08 --> T17
|
|
196
|
+
T10 --> T11
|
|
197
|
+
T05 --> T16
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Reading: T01 precedes everything durability-related. T03 must follow T02 — its
|
|
201
|
+
stale-`pending` re-poke would otherwise re-drive rows the old correlator flipped to
|
|
202
|
+
`pending` and reset their timers. T04 only needs T01. T05 closes the sprint. T06, T09,
|
|
203
|
+
T10, T12, T13, T14, T15 are independent and can land in any order or alongside the sprint;
|
|
204
|
+
T08 and T17 both touch `HandlerResult`, so T17 follows T08. T11 is deferred (guard only) — see the decisions log.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Completeness check
|
|
209
|
+
|
|
210
|
+
- **01 #2 and 02 #1 both redesign tick semantics** and must agree on the invariant "the
|
|
211
|
+
advance is the only writer." They do — but the dependency is real and directional (T02
|
|
212
|
+
before T03), and is recorded in `PROGRESS.md`'s `Deps` column.
|
|
213
|
+
- **03 #1's `matcher ⇒ MATCHED` rule and 02 #1's "defer when the node lacks `MATCHED`"
|
|
214
|
+
overlap by design:** the validator catches the definition error (a matcher with nowhere
|
|
215
|
+
to go), the correlator handles the legitimate case (a timer-only delay node). Neither
|
|
216
|
+
makes the other redundant.
|
|
217
|
+
- **04 #1 cannot be written before the sprint's behavior lands** — it is intentionally the
|
|
218
|
+
last task, and its "Done when" greps for the sentences that must disappear.
|
|
219
|
+
- **Not examined:** throughput and index behavior at scale (out of scope per the design
|
|
220
|
+
brief), the npm build output under Node (tests are Deno-only; a one-off smoke run of the
|
|
221
|
+
built package against Node is worth doing before the release, T16), and
|
|
222
|
+
`@marianmeres/migrate`'s own semantics (`up("latest")` / `down` are exercised by the
|
|
223
|
+
existing migration tests and behaved).
|
|
224
|
+
- **Verified as fine, no finding:** registry/validator structure, fsm hook semantics
|
|
225
|
+
(`onEnter` skipped on resume; transitions run synchronously inside the advance
|
|
226
|
+
transaction), history table and `getHistory`, the migration ledger and the append-only
|
|
227
|
+
rule for migration files, `Cron.migrate` referenced in the README (exists,
|
|
228
|
+
`cron.ts:1222`), tenant scoping on all three tables, and the `SKIP LOCKED` claims.
|
|
229
|
+
|
|
230
|
+
Source documents: [`01-durability.md`](./01-durability.md),
|
|
231
|
+
[`02-correlation.md`](./02-correlation.md),
|
|
232
|
+
[`03-definition-and-api.md`](./03-definition-and-api.md),
|
|
233
|
+
[`04-docs-tests-ops.md`](./04-docs-tests-ops.md).
|