@marianmeres/workflow 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +207 -94
- package/API.md +277 -157
- package/README.md +182 -83
- package/dist/correlator.d.ts +31 -14
- package/dist/correlator.js +130 -73
- package/dist/definition.d.ts +5 -1
- package/dist/definition.js +54 -1
- package/dist/driver.d.ts +28 -23
- package/dist/driver.js +223 -59
- package/dist/migrations/1_1_0.d.ts +11 -0
- package/dist/migrations/1_1_0.js +61 -0
- package/dist/migrations/1_2_0.d.ts +16 -0
- package/dist/migrations/1_2_0.js +44 -0
- package/dist/migrations/index.js +5 -1
- package/dist/mod.d.ts +8 -5
- package/dist/mod.js +7 -4
- package/dist/persistence/history.d.ts +1 -1
- package/dist/persistence/history.js +3 -3
- package/dist/persistence/inbox.d.ts +9 -3
- package/dist/persistence/inbox.js +16 -7
- package/dist/persistence/instances.d.ts +46 -13
- package/dist/persistence/instances.js +78 -33
- package/dist/scheduler.d.ts +33 -9
- package/dist/scheduler.js +45 -21
- package/dist/types.d.ts +126 -13
- package/dist/types.js +28 -2
- package/dist/workflow.d.ts +76 -4
- package/dist/workflow.js +237 -35
- package/docs/_archive/analysis/00-overview-and-roadmap.md +233 -0
- package/docs/_archive/analysis/01-durability.md +404 -0
- package/docs/_archive/analysis/02-correlation.md +228 -0
- package/docs/_archive/analysis/03-definition-and-api.md +248 -0
- package/docs/_archive/analysis/04-docs-tests-ops.md +191 -0
- package/docs/_archive/analysis/PROGRESS.md +172 -0
- package/docs/_archive/analysis/README.md +38 -0
- package/package.json +8 -7
package/dist/mod.js
CHANGED
|
@@ -23,7 +23,10 @@
|
|
|
23
23
|
* const pool = new pg.Pool({ /* ... *\/ });
|
|
24
24
|
* await createMigrate(pool).up("latest");
|
|
25
25
|
*
|
|
26
|
-
*
|
|
26
|
+
* // `autoCleanup` runs steve's reaper, which is what turns a crashed worker
|
|
27
|
+
* // into an `expired` job the framework can re-dispatch. Without it such a
|
|
28
|
+
* // job stays `running` forever and its instance never moves again.
|
|
29
|
+
* const jobs = new Jobs({ db: pool, autoCleanup: true });
|
|
27
30
|
* const wf = new Workflow({
|
|
28
31
|
* db: pool,
|
|
29
32
|
* jobs,
|
|
@@ -39,8 +42,8 @@ export { Workflow } from "./workflow.js";
|
|
|
39
42
|
export { WorkflowScheduler, } from "./scheduler.js";
|
|
40
43
|
export { WorkflowInboxCorrelator, } from "./correlator.js";
|
|
41
44
|
export { WorkflowRegistry } from "./registry.js";
|
|
42
|
-
export {
|
|
45
|
+
export { defKey, validateDefinition } from "./definition.js";
|
|
43
46
|
export { createMigrate } from "./migrations/index.js";
|
|
44
47
|
export { getHistory } from "./persistence/history.js";
|
|
45
|
-
export { effectJobType, JOB_TYPE_ADVANCE, PURE_ENTER_EVENT
|
|
46
|
-
export {
|
|
48
|
+
export { effectJobType, JOB_TYPE_ADVANCE, PURE_ENTER_EVENT } from "./driver.js";
|
|
49
|
+
export { DEFAULT_TENANT_ID, EXECUTION_STATE, HISTORY_EVENT, JOB_TYPE_EFFECT_PREFIX, } from "./types.js";
|
|
@@ -7,7 +7,7 @@ type Executor = pg.Pool | pg.PoolClient | pg.Client;
|
|
|
7
7
|
* transaction as the state change it describes.
|
|
8
8
|
*/
|
|
9
9
|
export declare function appendHistory(exec: Executor, input: {
|
|
10
|
-
|
|
10
|
+
tenant_id: string;
|
|
11
11
|
instance_id: string;
|
|
12
12
|
event_type: HistoryEventType;
|
|
13
13
|
from_node?: string | null;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const SELECT_COLUMNS = `id,
|
|
1
|
+
const SELECT_COLUMNS = `id, tenant_id, instance_id, at, event_type, from_node, to_node, data`;
|
|
2
2
|
/**
|
|
3
3
|
* Appends a single audit row to `__workflow_history`. Accepts a pool, a
|
|
4
4
|
* client, or a transaction-bound client — call this inside the same
|
|
@@ -6,9 +6,9 @@ const SELECT_COLUMNS = `id, project_id, instance_id, at, event_type, from_node,
|
|
|
6
6
|
*/
|
|
7
7
|
export async function appendHistory(exec, input) {
|
|
8
8
|
await exec.query(`INSERT INTO __workflow_history
|
|
9
|
-
(
|
|
9
|
+
(tenant_id, instance_id, event_type, from_node, to_node, data)
|
|
10
10
|
VALUES ($1, $2, $3, $4, $5, $6::jsonb)`, [
|
|
11
|
-
input.
|
|
11
|
+
input.tenant_id,
|
|
12
12
|
input.instance_id,
|
|
13
13
|
input.event_type,
|
|
14
14
|
input.from_node ?? null,
|
|
@@ -6,18 +6,24 @@ type Executor = pg.Pool | pg.PoolClient | pg.Client;
|
|
|
6
6
|
* pick it up on its next tick and attempt to match it to a waiting instance.
|
|
7
7
|
*/
|
|
8
8
|
export declare function appendInbox(exec: Executor, input: {
|
|
9
|
-
|
|
9
|
+
tenant_id: string;
|
|
10
10
|
source: string;
|
|
11
11
|
correlation_token: string;
|
|
12
12
|
payload: Record<string, unknown>;
|
|
13
13
|
}): Promise<InboxRow>;
|
|
14
14
|
/**
|
|
15
|
-
* Claims a batch of unprocessed inbox rows for the
|
|
15
|
+
* Claims a batch of unprocessed inbox rows for the tenant. Uses
|
|
16
16
|
* `FOR UPDATE SKIP LOCKED` so two concurrent correlator workers don't grab
|
|
17
17
|
* the same row. Caller MUST be inside a transaction and either mark them
|
|
18
18
|
* processed or roll back.
|
|
19
19
|
*/
|
|
20
|
-
export declare function claimUnprocessed(client: pg.PoolClient | pg.Client,
|
|
20
|
+
export declare function claimUnprocessed(client: pg.PoolClient | pg.Client, tenant_id: string, limit?: number): Promise<InboxRow[]>;
|
|
21
|
+
/**
|
|
22
|
+
* Loads a single inbox row with a row-level lock — caller MUST be inside a
|
|
23
|
+
* transaction. Used by the advance to mark delivery in the same transaction as
|
|
24
|
+
* the transition it triggers.
|
|
25
|
+
*/
|
|
26
|
+
export declare function lockInboxRow(client: pg.PoolClient | pg.Client, id: string): Promise<InboxRow | null>;
|
|
21
27
|
/** Marks an inbox row as processed (sets `processed_at = now()`). */
|
|
22
28
|
export declare function markProcessed(exec: Executor, id: string): Promise<void>;
|
|
23
29
|
export {};
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
const SELECT_COLUMNS = `
|
|
2
|
-
id,
|
|
2
|
+
id, tenant_id, received_at, source, correlation_token, payload, processed_at
|
|
3
3
|
`;
|
|
4
4
|
/**
|
|
5
5
|
* Inserts an external signal into `__workflow_inbox`. The correlator will
|
|
6
6
|
* pick it up on its next tick and attempt to match it to a waiting instance.
|
|
7
7
|
*/
|
|
8
8
|
export async function appendInbox(exec, input) {
|
|
9
|
-
const r = await exec.query(`INSERT INTO __workflow_inbox (
|
|
9
|
+
const r = await exec.query(`INSERT INTO __workflow_inbox (tenant_id, source, correlation_token, payload)
|
|
10
10
|
VALUES ($1, $2, $3, $4::jsonb)
|
|
11
11
|
RETURNING ${SELECT_COLUMNS}`, [
|
|
12
|
-
input.
|
|
12
|
+
input.tenant_id,
|
|
13
13
|
input.source,
|
|
14
14
|
input.correlation_token,
|
|
15
15
|
JSON.stringify(input.payload),
|
|
@@ -17,20 +17,29 @@ export async function appendInbox(exec, input) {
|
|
|
17
17
|
return r.rows[0];
|
|
18
18
|
}
|
|
19
19
|
/**
|
|
20
|
-
* Claims a batch of unprocessed inbox rows for the
|
|
20
|
+
* Claims a batch of unprocessed inbox rows for the tenant. Uses
|
|
21
21
|
* `FOR UPDATE SKIP LOCKED` so two concurrent correlator workers don't grab
|
|
22
22
|
* the same row. Caller MUST be inside a transaction and either mark them
|
|
23
23
|
* processed or roll back.
|
|
24
24
|
*/
|
|
25
|
-
export async function claimUnprocessed(client,
|
|
25
|
+
export async function claimUnprocessed(client, tenant_id, limit = 100) {
|
|
26
26
|
const r = await client.query(`SELECT ${SELECT_COLUMNS} FROM __workflow_inbox
|
|
27
|
-
WHERE
|
|
27
|
+
WHERE tenant_id = $1
|
|
28
28
|
AND processed_at IS NULL
|
|
29
29
|
ORDER BY received_at
|
|
30
30
|
LIMIT $2
|
|
31
|
-
FOR UPDATE SKIP LOCKED`, [
|
|
31
|
+
FOR UPDATE SKIP LOCKED`, [tenant_id, limit]);
|
|
32
32
|
return r.rows;
|
|
33
33
|
}
|
|
34
|
+
/**
|
|
35
|
+
* Loads a single inbox row with a row-level lock — caller MUST be inside a
|
|
36
|
+
* transaction. Used by the advance to mark delivery in the same transaction as
|
|
37
|
+
* the transition it triggers.
|
|
38
|
+
*/
|
|
39
|
+
export async function lockInboxRow(client, id) {
|
|
40
|
+
const r = await client.query(`SELECT ${SELECT_COLUMNS} FROM __workflow_inbox WHERE id = $1 FOR UPDATE`, [id]);
|
|
41
|
+
return r.rows[0] ?? null;
|
|
42
|
+
}
|
|
34
43
|
/** Marks an inbox row as processed (sets `processed_at = now()`). */
|
|
35
44
|
export async function markProcessed(exec, id) {
|
|
36
45
|
await exec.query(`UPDATE __workflow_inbox SET processed_at = now() WHERE id = $1`, [id]);
|
|
@@ -7,7 +7,7 @@ type Executor = pg.Pool | pg.PoolClient | pg.Client;
|
|
|
7
7
|
* enqueuing the first `workflow.advance` job.
|
|
8
8
|
*/
|
|
9
9
|
export declare function createInstance(exec: Executor, input: {
|
|
10
|
-
|
|
10
|
+
tenant_id: string;
|
|
11
11
|
definition_id: string;
|
|
12
12
|
definition_version: string;
|
|
13
13
|
cursor: string;
|
|
@@ -21,6 +21,10 @@ export declare function lockInstance(client: pg.PoolClient | pg.Client, id: stri
|
|
|
21
21
|
/**
|
|
22
22
|
* Partial-update of a workflow instance row. Only fields present in `patch`
|
|
23
23
|
* are written; `updated_at` is bumped automatically.
|
|
24
|
+
*
|
|
25
|
+
* `options.bumpSeq` advances the fencing token in the same `UPDATE` — set it on
|
|
26
|
+
* settle-point writes, so every job issued against the pre-write row becomes
|
|
27
|
+
* recognizably stale.
|
|
24
28
|
*/
|
|
25
29
|
export declare function updateInstance(exec: Executor, id: string, patch: {
|
|
26
30
|
cursor?: string;
|
|
@@ -29,22 +33,51 @@ export declare function updateInstance(exec: Executor, id: string, patch: {
|
|
|
29
33
|
execution_state?: ExecutionState;
|
|
30
34
|
wake_at?: Date | null;
|
|
31
35
|
correlation_token?: string | null;
|
|
36
|
+
}, options?: {
|
|
37
|
+
bumpSeq?: boolean;
|
|
32
38
|
}): Promise<WorkflowInstanceRow>;
|
|
39
|
+
/** The minimum a scheduler poke needs: who to poke, and the fence to poke with. */
|
|
40
|
+
export interface InstancePoke {
|
|
41
|
+
id: string;
|
|
42
|
+
seq: number;
|
|
43
|
+
}
|
|
33
44
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
45
|
+
* Instances whose timer has expired. Read-only on purpose: the scheduler only
|
|
46
|
+
* pokes an advance per row, and the advance — which re-checks the due-ness under
|
|
47
|
+
* the row lock — is what writes the state and clears `wake_at`. A row poked but
|
|
48
|
+
* not yet advanced stays due and is poked again next tick; the fence turns the
|
|
49
|
+
* duplicate into a no-op. Nothing is left stranded by a crash mid-tick.
|
|
50
|
+
*/
|
|
51
|
+
export declare function selectDueWakeUps(exec: Executor, tenant_id: string, limit?: number): Promise<InstancePoke[]>;
|
|
52
|
+
/**
|
|
53
|
+
* Instances stuck in `pending` for longer than `older_than_sec` — the residue of
|
|
54
|
+
* a crash between `create()`'s commit and steve's (separate-connection) job
|
|
55
|
+
* insert, i.e. rows no job will ever pick up.
|
|
37
56
|
*
|
|
38
|
-
*
|
|
57
|
+
* Only meaningful because `create()` is the sole producer of `pending`: every
|
|
58
|
+
* other write path settles the row into a state this scan ignores. A row that is
|
|
59
|
+
* merely waiting on a slow queue is re-poked too, harmlessly — the second
|
|
60
|
+
* advance to reach it is fenced out.
|
|
39
61
|
*/
|
|
40
|
-
export declare function
|
|
41
|
-
id: string;
|
|
42
|
-
correlation_token: string | null;
|
|
43
|
-
}>>;
|
|
62
|
+
export declare function selectStalePending(exec: Executor, tenant_id: string, older_than_sec: number, limit?: number): Promise<InstancePoke[]>;
|
|
44
63
|
/**
|
|
45
|
-
* Finds the
|
|
46
|
-
* Used by the inbox correlator
|
|
47
|
-
*
|
|
64
|
+
* Finds the live (non-terminal) instance owning this correlation token, if any.
|
|
65
|
+
* Used by the inbox correlator — the instance may be in any live execution
|
|
66
|
+
* state, not just `waiting`: a signal that arrives early is deferred, not
|
|
67
|
+
* dropped.
|
|
68
|
+
*
|
|
69
|
+
* The contract is one live instance per token; `ORDER BY created_at` only makes
|
|
70
|
+
* a violation deterministic (oldest wins) rather than arbitrary. Does not lock —
|
|
71
|
+
* the advance re-reads under a lock and is fenced by `seq`.
|
|
72
|
+
*/
|
|
73
|
+
export declare function findByCorrelation(exec: Executor, tenant_id: string, correlation_token: string): Promise<WorkflowInstanceRow | null>;
|
|
74
|
+
/**
|
|
75
|
+
* Finds the most recent terminal instance owning this correlation token, if any.
|
|
76
|
+
* Only consulted when {@link findByCorrelation} misses, to tell "the instance is
|
|
77
|
+
* over" apart from "nobody ever owned this token".
|
|
78
|
+
*
|
|
79
|
+
* Note that a *completed* or *cancelled* instance has its token cleared, so in
|
|
80
|
+
* practice this finds the failed ones.
|
|
48
81
|
*/
|
|
49
|
-
export declare function
|
|
82
|
+
export declare function findTerminalByCorrelation(exec: Executor, tenant_id: string, correlation_token: string): Promise<WorkflowInstanceRow | null>;
|
|
50
83
|
export {};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { EXECUTION_STATE, } from "../types.js";
|
|
2
2
|
const SELECT_COLUMNS = `
|
|
3
|
-
id,
|
|
3
|
+
id, tenant_id, definition_id, definition_version,
|
|
4
4
|
cursor, previous_cursor, context, execution_state,
|
|
5
|
-
wake_at, correlation_token, created_at, updated_at
|
|
5
|
+
wake_at, correlation_token, seq, created_at, updated_at
|
|
6
6
|
`;
|
|
7
7
|
/**
|
|
8
8
|
* Inserts a new workflow instance row in `execution_state='pending'` at the
|
|
@@ -11,10 +11,10 @@ const SELECT_COLUMNS = `
|
|
|
11
11
|
*/
|
|
12
12
|
export async function createInstance(exec, input) {
|
|
13
13
|
const r = await exec.query(`INSERT INTO __workflow_instances
|
|
14
|
-
(
|
|
14
|
+
(tenant_id, definition_id, definition_version, cursor, context, execution_state, correlation_token)
|
|
15
15
|
VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7)
|
|
16
16
|
RETURNING ${SELECT_COLUMNS}`, [
|
|
17
|
-
input.
|
|
17
|
+
input.tenant_id,
|
|
18
18
|
input.definition_id,
|
|
19
19
|
input.definition_version,
|
|
20
20
|
input.cursor,
|
|
@@ -37,8 +37,12 @@ export async function lockInstance(client, id) {
|
|
|
37
37
|
/**
|
|
38
38
|
* Partial-update of a workflow instance row. Only fields present in `patch`
|
|
39
39
|
* are written; `updated_at` is bumped automatically.
|
|
40
|
+
*
|
|
41
|
+
* `options.bumpSeq` advances the fencing token in the same `UPDATE` — set it on
|
|
42
|
+
* settle-point writes, so every job issued against the pre-write row becomes
|
|
43
|
+
* recognizably stale.
|
|
40
44
|
*/
|
|
41
|
-
export async function updateInstance(exec, id, patch) {
|
|
45
|
+
export async function updateInstance(exec, id, patch, options = {}) {
|
|
42
46
|
const sets = [];
|
|
43
47
|
const values = [];
|
|
44
48
|
let i = 1;
|
|
@@ -66,6 +70,8 @@ export async function updateInstance(exec, id, patch) {
|
|
|
66
70
|
sets.push(`correlation_token = $${i++}`);
|
|
67
71
|
values.push(patch.correlation_token);
|
|
68
72
|
}
|
|
73
|
+
if (options.bumpSeq)
|
|
74
|
+
sets.push(`seq = seq + 1`);
|
|
69
75
|
sets.push(`updated_at = now()`);
|
|
70
76
|
values.push(id);
|
|
71
77
|
const r = await exec.query(`UPDATE __workflow_instances SET ${sets.join(", ")} WHERE id = $${i}
|
|
@@ -73,40 +79,79 @@ export async function updateInstance(exec, id, patch) {
|
|
|
73
79
|
return r.rows[0];
|
|
74
80
|
}
|
|
75
81
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
82
|
+
* Instances whose timer has expired. Read-only on purpose: the scheduler only
|
|
83
|
+
* pokes an advance per row, and the advance — which re-checks the due-ness under
|
|
84
|
+
* the row lock — is what writes the state and clears `wake_at`. A row poked but
|
|
85
|
+
* not yet advanced stays due and is poked again next tick; the fence turns the
|
|
86
|
+
* duplicate into a no-op. Nothing is left stranded by a crash mid-tick.
|
|
87
|
+
*/
|
|
88
|
+
export async function selectDueWakeUps(exec, tenant_id, limit = 100) {
|
|
89
|
+
const r = await exec.query(`SELECT id, seq FROM __workflow_instances
|
|
90
|
+
WHERE tenant_id = $1
|
|
91
|
+
AND execution_state = $2
|
|
92
|
+
AND wake_at IS NOT NULL
|
|
93
|
+
AND wake_at <= now()
|
|
94
|
+
ORDER BY wake_at
|
|
95
|
+
LIMIT $3`, [tenant_id, EXECUTION_STATE.WAITING, limit]);
|
|
96
|
+
return r.rows;
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Instances stuck in `pending` for longer than `older_than_sec` — the residue of
|
|
100
|
+
* a crash between `create()`'s commit and steve's (separate-connection) job
|
|
101
|
+
* insert, i.e. rows no job will ever pick up.
|
|
79
102
|
*
|
|
80
|
-
*
|
|
103
|
+
* Only meaningful because `create()` is the sole producer of `pending`: every
|
|
104
|
+
* other write path settles the row into a state this scan ignores. A row that is
|
|
105
|
+
* merely waiting on a slow queue is re-poked too, harmlessly — the second
|
|
106
|
+
* advance to reach it is fenced out.
|
|
81
107
|
*/
|
|
82
|
-
export async function
|
|
83
|
-
const r = await exec.query(`
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
WHERE project_id = $1
|
|
90
|
-
AND execution_state = $3
|
|
91
|
-
AND wake_at IS NOT NULL
|
|
92
|
-
AND wake_at <= now()
|
|
93
|
-
ORDER BY wake_at
|
|
94
|
-
LIMIT $4
|
|
95
|
-
FOR UPDATE SKIP LOCKED
|
|
96
|
-
)
|
|
97
|
-
RETURNING id, correlation_token`, [project_id, EXECUTION_STATE.PENDING, EXECUTION_STATE.WAITING, limit]);
|
|
108
|
+
export async function selectStalePending(exec, tenant_id, older_than_sec, limit = 100) {
|
|
109
|
+
const r = await exec.query(`SELECT id, seq FROM __workflow_instances
|
|
110
|
+
WHERE tenant_id = $1
|
|
111
|
+
AND execution_state = $2
|
|
112
|
+
AND updated_at < now() - make_interval(secs => $3::float8)
|
|
113
|
+
ORDER BY updated_at
|
|
114
|
+
LIMIT $4`, [tenant_id, EXECUTION_STATE.PENDING, older_than_sec, limit]);
|
|
98
115
|
return r.rows;
|
|
99
116
|
}
|
|
117
|
+
const TERMINAL_STATES = [
|
|
118
|
+
EXECUTION_STATE.COMPLETED,
|
|
119
|
+
EXECUTION_STATE.FAILED,
|
|
120
|
+
EXECUTION_STATE.CANCELLED,
|
|
121
|
+
];
|
|
100
122
|
/**
|
|
101
|
-
* Finds the
|
|
102
|
-
* Used by the inbox correlator
|
|
103
|
-
*
|
|
123
|
+
* Finds the live (non-terminal) instance owning this correlation token, if any.
|
|
124
|
+
* Used by the inbox correlator — the instance may be in any live execution
|
|
125
|
+
* state, not just `waiting`: a signal that arrives early is deferred, not
|
|
126
|
+
* dropped.
|
|
127
|
+
*
|
|
128
|
+
* The contract is one live instance per token; `ORDER BY created_at` only makes
|
|
129
|
+
* a violation deterministic (oldest wins) rather than arbitrary. Does not lock —
|
|
130
|
+
* the advance re-reads under a lock and is fenced by `seq`.
|
|
104
131
|
*/
|
|
105
|
-
export async function
|
|
132
|
+
export async function findByCorrelation(exec, tenant_id, correlation_token) {
|
|
106
133
|
const r = await exec.query(`SELECT ${SELECT_COLUMNS} FROM __workflow_instances
|
|
107
|
-
WHERE
|
|
108
|
-
AND
|
|
109
|
-
AND
|
|
110
|
-
|
|
134
|
+
WHERE tenant_id = $1
|
|
135
|
+
AND correlation_token = $2
|
|
136
|
+
AND NOT (execution_state = ANY($3))
|
|
137
|
+
ORDER BY created_at
|
|
138
|
+
LIMIT 1`, [tenant_id, correlation_token, TERMINAL_STATES]);
|
|
139
|
+
return r.rows[0] ?? null;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Finds the most recent terminal instance owning this correlation token, if any.
|
|
143
|
+
* Only consulted when {@link findByCorrelation} misses, to tell "the instance is
|
|
144
|
+
* over" apart from "nobody ever owned this token".
|
|
145
|
+
*
|
|
146
|
+
* Note that a *completed* or *cancelled* instance has its token cleared, so in
|
|
147
|
+
* practice this finds the failed ones.
|
|
148
|
+
*/
|
|
149
|
+
export async function findTerminalByCorrelation(exec, tenant_id, correlation_token) {
|
|
150
|
+
const r = await exec.query(`SELECT ${SELECT_COLUMNS} FROM __workflow_instances
|
|
151
|
+
WHERE tenant_id = $1
|
|
152
|
+
AND correlation_token = $2
|
|
153
|
+
AND execution_state = ANY($3)
|
|
154
|
+
ORDER BY created_at DESC
|
|
155
|
+
LIMIT 1`, [tenant_id, correlation_token, TERMINAL_STATES]);
|
|
111
156
|
return r.rows[0] ?? null;
|
|
112
157
|
}
|
package/dist/scheduler.d.ts
CHANGED
|
@@ -8,23 +8,43 @@ export interface WorkflowSchedulerOptions {
|
|
|
8
8
|
* The consumer owns the lifecycle: `await cron.start(N)` / `await cron.stop()`.
|
|
9
9
|
*/
|
|
10
10
|
cron: Cron;
|
|
11
|
-
/** The Workflow whose instances this scheduler wakes (used for `db`, `
|
|
11
|
+
/** The Workflow whose instances this scheduler wakes (used for `db`, `tenantId`, `enqueueAdvance`). */
|
|
12
12
|
workflow: Workflow;
|
|
13
13
|
/** Cron expression for the tick. Default: every minute. */
|
|
14
14
|
tickExpression?: string;
|
|
15
15
|
/** IANA timezone for the cron expression. */
|
|
16
16
|
timezone?: string | null;
|
|
17
|
-
/** Maximum number of
|
|
17
|
+
/** Maximum number of rows poked per tick, per scan. Default: 100. */
|
|
18
18
|
tickBatchSize?: number;
|
|
19
|
-
/** Custom name for the registered cron job. Default: `workflow.scheduler.<
|
|
19
|
+
/** Custom name for the registered cron job. Default: `workflow.scheduler.<tenantId>`. */
|
|
20
20
|
tickName?: string;
|
|
21
|
+
/**
|
|
22
|
+
* Age in seconds at which a `pending` instance is assumed stranded and
|
|
23
|
+
* re-poked with a fresh `start` advance. Covers the window between
|
|
24
|
+
* `create()`'s commit and steve's job insert, which happens on its own
|
|
25
|
+
* connection and can be lost to a crash. Default: 300. `0` disables the scan.
|
|
26
|
+
*/
|
|
27
|
+
stalePendingSec?: number;
|
|
28
|
+
}
|
|
29
|
+
/** What one scheduler tick poked. */
|
|
30
|
+
export interface SchedulerTickResult {
|
|
31
|
+
/** Due `waiting` rows poked with a `timeout` advance. */
|
|
32
|
+
woken: number;
|
|
33
|
+
/** Stranded `pending` rows re-poked with a `start` advance. */
|
|
34
|
+
repoked: number;
|
|
21
35
|
}
|
|
22
36
|
/**
|
|
23
37
|
* Cron-driven scheduler that wakes time-suspended workflow instances.
|
|
24
38
|
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
39
|
+
* Each tick is read-only against `__workflow_instances`: it selects due rows and
|
|
40
|
+
* enqueues one `workflow.advance` poke per row, carrying the row's `seq` as the
|
|
41
|
+
* fence. The advance re-checks the precondition under the row lock and is the
|
|
42
|
+
* only thing that writes — so a crash anywhere in a tick costs nothing but a
|
|
43
|
+
* repeated poke next tick, and the repeat is fenced out.
|
|
44
|
+
*
|
|
45
|
+
* Two scans per tick: due timers (`waiting` with an expired `wake_at`) and, at
|
|
46
|
+
* `stalePendingSec`, instances stuck in `pending` because the job that should
|
|
47
|
+
* have started them never made it into the queue.
|
|
28
48
|
*
|
|
29
49
|
* The scheduler does not own its `Cron`. The caller constructs a `Cron`, passes
|
|
30
50
|
* it in, and runs its lifecycle (`start` / `stop`). Call `register()` once
|
|
@@ -33,9 +53,10 @@ export interface WorkflowSchedulerOptions {
|
|
|
33
53
|
export declare class WorkflowScheduler {
|
|
34
54
|
#private;
|
|
35
55
|
readonly cron: Cron;
|
|
36
|
-
readonly
|
|
56
|
+
readonly tenantId: string;
|
|
37
57
|
readonly tickExpression: string;
|
|
38
58
|
readonly tickName: string;
|
|
59
|
+
readonly stalePendingSec: number;
|
|
39
60
|
constructor(options: WorkflowSchedulerOptions);
|
|
40
61
|
/**
|
|
41
62
|
* Registers the tick on the injected `Cron`. Idempotent — re-registering an
|
|
@@ -44,6 +65,9 @@ export declare class WorkflowScheduler {
|
|
|
44
65
|
register(): Promise<void>;
|
|
45
66
|
/** Removes the tick from the injected `Cron`. */
|
|
46
67
|
unregister(): Promise<void>;
|
|
47
|
-
/**
|
|
48
|
-
|
|
68
|
+
/**
|
|
69
|
+
* Runs one tick immediately. Exposed for testing and on-demand wakes. The
|
|
70
|
+
* counts are pokes issued, not instances moved — the advance decides that.
|
|
71
|
+
*/
|
|
72
|
+
tickOnce(): Promise<SchedulerTickResult>;
|
|
49
73
|
}
|
package/dist/scheduler.js
CHANGED
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { clog } from "./log.js";
|
|
2
|
-
import {
|
|
2
|
+
import { selectDueWakeUps, selectStalePending, } from "./persistence/instances.js";
|
|
3
|
+
import { TIMEOUT_EVENT } from "./types.js";
|
|
3
4
|
const DEFAULT_TICK_EXPRESSION = "* * * * *";
|
|
5
|
+
const DEFAULT_STALE_PENDING_SEC = 300;
|
|
4
6
|
/**
|
|
5
7
|
* Cron-driven scheduler that wakes time-suspended workflow instances.
|
|
6
8
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
9
|
+
* Each tick is read-only against `__workflow_instances`: it selects due rows and
|
|
10
|
+
* enqueues one `workflow.advance` poke per row, carrying the row's `seq` as the
|
|
11
|
+
* fence. The advance re-checks the precondition under the row lock and is the
|
|
12
|
+
* only thing that writes — so a crash anywhere in a tick costs nothing but a
|
|
13
|
+
* repeated poke next tick, and the repeat is fenced out.
|
|
14
|
+
*
|
|
15
|
+
* Two scans per tick: due timers (`waiting` with an expired `wake_at`) and, at
|
|
16
|
+
* `stalePendingSec`, instances stuck in `pending` because the job that should
|
|
17
|
+
* have started them never made it into the queue.
|
|
10
18
|
*
|
|
11
19
|
* The scheduler does not own its `Cron`. The caller constructs a `Cron`, passes
|
|
12
20
|
* it in, and runs its lifecycle (`start` / `stop`). Call `register()` once
|
|
@@ -14,20 +22,22 @@ const DEFAULT_TICK_EXPRESSION = "* * * * *";
|
|
|
14
22
|
*/
|
|
15
23
|
export class WorkflowScheduler {
|
|
16
24
|
cron;
|
|
17
|
-
|
|
25
|
+
tenantId;
|
|
18
26
|
tickExpression;
|
|
19
27
|
tickName;
|
|
28
|
+
stalePendingSec;
|
|
20
29
|
#workflow;
|
|
21
30
|
#tickBatchSize;
|
|
22
31
|
#timezone;
|
|
23
32
|
constructor(options) {
|
|
24
33
|
this.cron = options.cron;
|
|
25
34
|
this.#workflow = options.workflow;
|
|
26
|
-
this.
|
|
35
|
+
this.tenantId = options.workflow.tenantId;
|
|
27
36
|
this.tickExpression = options.tickExpression ?? DEFAULT_TICK_EXPRESSION;
|
|
28
|
-
this.tickName = options.tickName ?? `workflow.scheduler.${this.
|
|
37
|
+
this.tickName = options.tickName ?? `workflow.scheduler.${this.tenantId}`;
|
|
29
38
|
this.#tickBatchSize = options.tickBatchSize ?? 100;
|
|
30
39
|
this.#timezone = options.timezone;
|
|
40
|
+
this.stalePendingSec = options.stalePendingSec ?? DEFAULT_STALE_PENDING_SEC;
|
|
31
41
|
}
|
|
32
42
|
/**
|
|
33
43
|
* Registers the tick on the injected `Cron`. Idempotent — re-registering an
|
|
@@ -44,24 +54,38 @@ export class WorkflowScheduler {
|
|
|
44
54
|
async unregister() {
|
|
45
55
|
await this.cron.unregister(this.tickName);
|
|
46
56
|
}
|
|
47
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Runs one tick immediately. Exposed for testing and on-demand wakes. The
|
|
59
|
+
* counts are pokes issued, not instances moved — the advance decides that.
|
|
60
|
+
*/
|
|
48
61
|
async tickOnce() {
|
|
49
62
|
return await this.#tick();
|
|
50
63
|
}
|
|
51
64
|
async #tick() {
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
clog.debug?.(`scheduler: claimed ${claimed.length} due wake-ups`);
|
|
56
|
-
for (const row of claimed) {
|
|
57
|
-
const payload = {
|
|
58
|
-
project_id: this.projectId,
|
|
59
|
-
instance_id: row.id,
|
|
60
|
-
timeout: true,
|
|
61
|
-
outcome: "TIMEOUT",
|
|
62
|
-
};
|
|
63
|
-
await this.#workflow.enqueueAdvance(this.#workflow.db, payload);
|
|
65
|
+
const due = await selectDueWakeUps(this.#workflow.db, this.tenantId, this.#tickBatchSize);
|
|
66
|
+
for (const row of due) {
|
|
67
|
+
await this.#poke(row, { kind: "timeout", outcome: TIMEOUT_EVENT });
|
|
64
68
|
}
|
|
65
|
-
|
|
69
|
+
if (due.length)
|
|
70
|
+
clog.debug?.(`scheduler: poked ${due.length} due wake-ups`);
|
|
71
|
+
let stale = [];
|
|
72
|
+
if (this.stalePendingSec > 0) {
|
|
73
|
+
stale = await selectStalePending(this.#workflow.db, this.tenantId, this.stalePendingSec, this.#tickBatchSize);
|
|
74
|
+
for (const row of stale) {
|
|
75
|
+
await this.#poke(row, { kind: "start" });
|
|
76
|
+
}
|
|
77
|
+
if (stale.length) {
|
|
78
|
+
clog.debug?.(`scheduler: re-poked ${stale.length} pending instances`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return { woken: due.length, repoked: stale.length };
|
|
82
|
+
}
|
|
83
|
+
#poke(row, what) {
|
|
84
|
+
return this.#workflow.enqueueAdvance(this.#workflow.db, {
|
|
85
|
+
tenant_id: this.tenantId,
|
|
86
|
+
instance_id: row.id,
|
|
87
|
+
expected_seq: row.seq,
|
|
88
|
+
...what,
|
|
89
|
+
});
|
|
66
90
|
}
|
|
67
91
|
}
|