notifkit 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-GRXzgPvW.mjs +89 -0
- package/dist/bulk-BvRfz6Of.mjs +77 -0
- package/dist/index.d.mts +1100 -8883
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-BENOY7HG.mjs} +49 -75
- package/dist/{main-BiFiOX9D.mjs → main-BpnuFX62.mjs} +119 -112
- package/dist/{main-Dz2sUh3c.mjs → main-BqVH2BHj.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-C_5VFIwp.mjs} +135 -79
- package/dist/main-D6od9N8h.mjs +312 -0
- package/dist/main-D84OX0ux.mjs +811 -0
- package/dist/{main-39DtKrX4.mjs → main-HH-WwsMM.mjs} +395 -184
- package/dist/{main-B3iQrl3t.mjs → main-HY_TDs61.mjs} +149 -134
- package/dist/{src-QiTu5e_a.mjs → src-CI7FGf3h.mjs} +1220 -651
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +17 -17
- package/src/client.ts +15 -5
- package/src/config/index.ts +26 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +2 -2
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +61 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { and, eq, inArray } from "drizzle-orm";
|
|
2
|
+
import { suppressions } from "@/db/schema.js";
|
|
3
|
+
import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
|
|
4
|
+
import { normaliseTarget } from "@/shared/index.js";
|
|
5
|
+
|
|
6
|
+
export type BlockReason =
|
|
7
|
+
"suppressed" | "recipient_not_found" | "user_opted_out" | "channel_disabled" | "contact_inactive";
|
|
8
|
+
|
|
9
|
+
/** The fields of a parked `notification.dispatched` payload the gate reads. */
|
|
10
|
+
interface ParkedTask {
|
|
11
|
+
taskId: string;
|
|
12
|
+
projectId: string;
|
|
13
|
+
recipientId: string;
|
|
14
|
+
channel: string;
|
|
15
|
+
templateId?: string;
|
|
16
|
+
destination?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Re-checks parked sends at the moment they are released.
|
|
21
|
+
*
|
|
22
|
+
* The engine clears a message against suppressions, opt-outs and contacts when
|
|
23
|
+
* it first sees it. A send parked for later — a `sendAt`, a quiet-hours
|
|
24
|
+
* deferral, a retry — can sit for hours or days, and whatever the person did in
|
|
25
|
+
* between (unsubscribed, complained, hard-bounced, removed the address) must
|
|
26
|
+
* still stop it. Without this the scheduler sent to them anyway.
|
|
27
|
+
*
|
|
28
|
+
* Reads Postgres directly rather than through the repositories' caches: this
|
|
29
|
+
* runs once per poll, not once per message, and a stale "still subscribed" is
|
|
30
|
+
* the very thing it exists to prevent.
|
|
31
|
+
*
|
|
32
|
+
* Returns the blocked tasks with the reason. Throws if a lookup fails, so the
|
|
33
|
+
* caller holds the batch rather than sending it unchecked.
|
|
34
|
+
*/
|
|
35
|
+
export async function findBlockedAtRelease(
|
|
36
|
+
db: any,
|
|
37
|
+
tasks: ParkedTask[],
|
|
38
|
+
): Promise<Map<string, BlockReason>> {
|
|
39
|
+
const blocked = new Map<string, BlockReason>();
|
|
40
|
+
|
|
41
|
+
const byProject = new Map<string, ParkedTask[]>();
|
|
42
|
+
for (const t of tasks) {
|
|
43
|
+
if (!t?.projectId || !t.recipientId) continue;
|
|
44
|
+
if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
|
|
45
|
+
byProject.get(t.projectId)!.push(t);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
for (const [projectId, group] of byProject) {
|
|
49
|
+
const recipientIds = [...new Set(group.map((t) => t.recipientId))];
|
|
50
|
+
const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))] as string[];
|
|
51
|
+
const targets = [
|
|
52
|
+
...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean)),
|
|
53
|
+
] as string[];
|
|
54
|
+
|
|
55
|
+
const templateRepo = new TemplateRepository(db);
|
|
56
|
+
const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
|
|
57
|
+
targets.length > 0
|
|
58
|
+
? db
|
|
59
|
+
.select({ channel: suppressions.channel, target: suppressions.target })
|
|
60
|
+
.from(suppressions)
|
|
61
|
+
.where(
|
|
62
|
+
and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets)),
|
|
63
|
+
)
|
|
64
|
+
: [],
|
|
65
|
+
new UserRepository(db).findRecordsByIds(projectId, recipientIds),
|
|
66
|
+
new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
|
|
67
|
+
Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id))),
|
|
68
|
+
]);
|
|
69
|
+
|
|
70
|
+
const suppressed = new Set(
|
|
71
|
+
(suppressedRows as { channel: string; target: string }[]).map(
|
|
72
|
+
(r) => `${r.channel}:${normaliseTarget(r.target)}`,
|
|
73
|
+
),
|
|
74
|
+
);
|
|
75
|
+
const userById = new Map(users.map((u) => [u.userId, u]));
|
|
76
|
+
const topicsByTemplate = new Map<string, string[]>();
|
|
77
|
+
templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
|
|
78
|
+
|
|
79
|
+
for (const t of group) {
|
|
80
|
+
const topics = t.templateId ? (topicsByTemplate.get(t.templateId) ?? []) : [];
|
|
81
|
+
const reason = decide(t, topics, suppressed, userById.get(t.recipientId), contactsByUser);
|
|
82
|
+
if (reason) blocked.set(t.taskId, reason);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return blocked;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function decide(
|
|
90
|
+
t: ParkedTask,
|
|
91
|
+
topics: string[],
|
|
92
|
+
suppressed: Set<string>,
|
|
93
|
+
user:
|
|
94
|
+
| { preferences: { topics?: Record<string, boolean>; channels?: Record<string, boolean> } }
|
|
95
|
+
| undefined,
|
|
96
|
+
contactsByUser: Map<string, { channel: string; target: string; preferences?: any }[]>,
|
|
97
|
+
): BlockReason | undefined {
|
|
98
|
+
// Checked first and for every priority, as the engine does: a suppression
|
|
99
|
+
// records that the address asked us to stop or is dead.
|
|
100
|
+
if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) {
|
|
101
|
+
return "suppressed";
|
|
102
|
+
}
|
|
103
|
+
if (!user) return "recipient_not_found";
|
|
104
|
+
if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
|
|
105
|
+
if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
|
|
106
|
+
|
|
107
|
+
if (t.destination) {
|
|
108
|
+
const contact = (contactsByUser.get(t.recipientId) ?? []).find(
|
|
109
|
+
(c) => c.channel === t.channel && c.target === t.destination,
|
|
110
|
+
);
|
|
111
|
+
if (!contact) return "contact_inactive";
|
|
112
|
+
if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) {
|
|
113
|
+
return "user_opted_out";
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
3
|
-
import {
|
|
4
|
-
import { RedisClient,
|
|
3
|
+
import type { Logger } from "@/index.js";
|
|
4
|
+
import type { RedisClient, Redis } from "@/index.js";
|
|
5
5
|
import {
|
|
6
|
-
StreamConsumer,
|
|
7
|
-
PendingMessageScanner,
|
|
6
|
+
type StreamConsumer,
|
|
7
|
+
type PendingMessageScanner,
|
|
8
8
|
StreamProducer,
|
|
9
9
|
type StreamMessage,
|
|
10
10
|
} from "@/index.js";
|
|
11
11
|
import { BaseWorker } from "@/index.js";
|
|
12
12
|
import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/index.js";
|
|
13
|
-
import { type StreamName } from "@/contracts/streams.js";
|
|
14
|
-
import { createDatabase } from "@/db/index.js";
|
|
15
13
|
import {
|
|
16
14
|
workflowInstances,
|
|
17
15
|
workflowSteps,
|
|
@@ -26,13 +24,14 @@ import {
|
|
|
26
24
|
type WorkflowContext,
|
|
27
25
|
type WorkflowStepContext,
|
|
28
26
|
} from "@/workflows/index.js";
|
|
29
|
-
import {
|
|
30
|
-
type WorkerOptions,
|
|
31
|
-
LUA_SCHEDULER_POLL,
|
|
32
|
-
LUA_RELEASE_LOCK,
|
|
33
|
-
LUA_RENEW_LOCK,
|
|
34
|
-
} from "@/shared/index.js";
|
|
27
|
+
import { type WorkerOptions } from "@/shared/index.js";
|
|
35
28
|
import { startHealthReporter } from "@/workers/index.js";
|
|
29
|
+
import {
|
|
30
|
+
createStreamConsumer,
|
|
31
|
+
createWorkerRuntime,
|
|
32
|
+
shutdownWorker,
|
|
33
|
+
uniqueConsumerId,
|
|
34
|
+
} from "@/workers/bootstrap.js";
|
|
36
35
|
|
|
37
36
|
/** How long a workflow instance lock is held before it self-expires. */
|
|
38
37
|
const WORKFLOW_LOCK_TTL_SECONDS = 60;
|
|
@@ -40,10 +39,48 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
|
|
|
40
39
|
const WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;
|
|
41
40
|
/** How long a claimed workflow timer stays invisible to other pollers. */
|
|
42
41
|
const WORKFLOW_TIMER_VISIBILITY_MS = 60_000;
|
|
42
|
+
/** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
|
|
43
|
+
export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
|
|
44
|
+
/** How soon a resume that found its instance busy is tried again. */
|
|
45
|
+
const WORKFLOW_RESUME_RETRY_MS = 2_000;
|
|
46
|
+
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
47
|
+
|
|
48
|
+
/** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
|
|
49
|
+
const DURATION_UNIT_SECONDS: Record<string, number> = {
|
|
50
|
+
ms: 0.001,
|
|
51
|
+
s: 1,
|
|
52
|
+
m: 60,
|
|
53
|
+
h: 3_600,
|
|
54
|
+
d: 86_400,
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
59
|
+
* read exactly; anything that is not a number and a unit throws. The old
|
|
60
|
+
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
61
|
+
* parseInt, so "1.5h" meant one hour.
|
|
62
|
+
*/
|
|
63
|
+
export function parseDurationSeconds(value: string, what: string): number {
|
|
64
|
+
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
65
|
+
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
66
|
+
return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]!]!;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function durationToMs(value: string, what: string): number {
|
|
70
|
+
return Math.round(parseDurationSeconds(value, what) * 1000);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Thrown at a step boundary once the instance has been canceled. */
|
|
74
|
+
class WorkflowCanceledError extends Error {
|
|
75
|
+
constructor() {
|
|
76
|
+
super("Workflow instance was canceled");
|
|
77
|
+
this.name = "WorkflowCanceledError";
|
|
78
|
+
}
|
|
79
|
+
}
|
|
43
80
|
|
|
44
81
|
loadEnv();
|
|
45
82
|
const config = readBaseConfig();
|
|
46
|
-
let logger:
|
|
83
|
+
let logger: Logger;
|
|
47
84
|
let redis: RedisClient;
|
|
48
85
|
let sql: any;
|
|
49
86
|
let db: any;
|
|
@@ -90,12 +127,15 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
90
127
|
}
|
|
91
128
|
|
|
92
129
|
override async stop(): Promise<void> {
|
|
130
|
+
// The flush interval keeps running until in-flight runs settle: a step's
|
|
131
|
+
// notify is waiting on it, and clearing it first left the drain to the 30s
|
|
132
|
+
// stop timeout with the instance lock still being renewed.
|
|
133
|
+
await super.stop();
|
|
93
134
|
if (this.flushTimer) {
|
|
94
135
|
clearInterval(this.flushTimer);
|
|
95
136
|
this.flushTimer = null;
|
|
96
137
|
}
|
|
97
138
|
await this.flushWorkerBuffers();
|
|
98
|
-
await super.stop();
|
|
99
139
|
}
|
|
100
140
|
|
|
101
141
|
private async flushWorkerBuffers(): Promise<void> {
|
|
@@ -125,6 +165,29 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
125
165
|
}
|
|
126
166
|
}
|
|
127
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Re-delivers a resume through the timer ZSET a few seconds from now. The
|
|
170
|
+
* poll loop publishes it once the instance is pending again, and drops it if
|
|
171
|
+
* the instance has finished. The member carries no timestamp, so repeated
|
|
172
|
+
* deferrals of one instance collapse into a single entry. It is marked
|
|
173
|
+
* `deferred` so it never equals a `wait` timer's member: sharing one let the
|
|
174
|
+
* ZADD here re-score a sleep's wake-up to seconds from now, and the early
|
|
175
|
+
* resume then re-suspended with no timer left to wake it.
|
|
176
|
+
*/
|
|
177
|
+
private async deferResume(payload: any): Promise<void> {
|
|
178
|
+
await this.redisCli.zadd(
|
|
179
|
+
WORKFLOW_TIMERS_KEY,
|
|
180
|
+
Date.now() + WORKFLOW_RESUME_RETRY_MS,
|
|
181
|
+
JSON.stringify({
|
|
182
|
+
instanceId: payload.instanceId,
|
|
183
|
+
name: payload.name,
|
|
184
|
+
projectId: payload.projectId,
|
|
185
|
+
input: payload.input ?? {},
|
|
186
|
+
deferred: true,
|
|
187
|
+
}),
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
|
|
128
191
|
async process(message: StreamMessage): Promise<void> {
|
|
129
192
|
const { event } = message;
|
|
130
193
|
const publishPromises: Promise<void>[] = [];
|
|
@@ -195,7 +258,12 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
195
258
|
"NX",
|
|
196
259
|
);
|
|
197
260
|
if (!acquired) {
|
|
198
|
-
|
|
261
|
+
// Another process is mid-run on this instance — typically still
|
|
262
|
+
// persisting the very suspension this resume answers, when the awaited
|
|
263
|
+
// event arrived quickly. Acking here dropped the resume, and with the
|
|
264
|
+
// waiter already consumed nothing would ever wake the instance again.
|
|
265
|
+
this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
|
|
266
|
+
await this.deferResume(payload);
|
|
199
267
|
return;
|
|
200
268
|
}
|
|
201
269
|
|
|
@@ -203,7 +271,7 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
203
271
|
// outlives the TTL lets a second resume execute the same steps in parallel.
|
|
204
272
|
const renewTimer = setInterval(() => {
|
|
205
273
|
void this.redisCli
|
|
206
|
-
.
|
|
274
|
+
.renewLock(lockKey, lockToken, WORKFLOW_LOCK_TTL_SECONDS)
|
|
207
275
|
.catch((err: unknown) => {
|
|
208
276
|
this.logger.warn({ err, instanceId }, "failed to renew workflow lock");
|
|
209
277
|
});
|
|
@@ -231,15 +299,40 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
231
299
|
instance = rows[0]!;
|
|
232
300
|
}
|
|
233
301
|
|
|
302
|
+
if (instance.status === "running") {
|
|
303
|
+
// Its last holder died mid-run (the reaper will set it back to
|
|
304
|
+
// pending) or has not finished suspending. Either way this resume
|
|
305
|
+
// is still owed.
|
|
306
|
+
await this.deferResume(payload);
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
234
309
|
if (instance.status !== "pending") {
|
|
235
310
|
this.logger.info({ instanceId }, "Workflow is not pending, skipping");
|
|
236
311
|
return;
|
|
237
312
|
}
|
|
238
313
|
|
|
239
|
-
|
|
314
|
+
// Conditional, so a cancel that lands between the read above and here
|
|
315
|
+
// is not overwritten.
|
|
316
|
+
const claimed = await this.dbConn
|
|
240
317
|
.update(workflowInstances)
|
|
241
318
|
.set({ status: "running" })
|
|
242
|
-
.where(eq(workflowInstances.id, instanceId))
|
|
319
|
+
.where(and(eq(workflowInstances.id, instanceId), eq(workflowInstances.status, "pending")))
|
|
320
|
+
.returning({ id: workflowInstances.id });
|
|
321
|
+
if (claimed.length === 0) {
|
|
322
|
+
this.logger.info({ instanceId }, "Workflow is no longer pending, skipping");
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// A cancel can arrive while the handler runs. Checked before each new
|
|
327
|
+
// step, so a canceled instance stops at the next step boundary.
|
|
328
|
+
const stopIfCanceled = async () => {
|
|
329
|
+
const [row] = await this.dbConn
|
|
330
|
+
.select({ status: workflowInstances.status })
|
|
331
|
+
.from(workflowInstances)
|
|
332
|
+
.where(eq(workflowInstances.id, instanceId))
|
|
333
|
+
.limit(1);
|
|
334
|
+
if (row?.status === "canceled") throw new WorkflowCanceledError();
|
|
335
|
+
};
|
|
243
336
|
|
|
244
337
|
// Load existing steps
|
|
245
338
|
const existingSteps = await this.dbConn
|
|
@@ -257,6 +350,7 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
257
350
|
notify: async (args) => {
|
|
258
351
|
const stepId = String(currentStepIndex++);
|
|
259
352
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
353
|
+
await stopIfCanceled();
|
|
260
354
|
|
|
261
355
|
// The step payload is the same shape as notify(), but the wire event
|
|
262
356
|
// is not — translate rather than spread.
|
|
@@ -265,6 +359,9 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
265
359
|
instance!.input,
|
|
266
360
|
payload.projectId,
|
|
267
361
|
`wf-${instanceId}-${stepId}`,
|
|
362
|
+
// The contract wants a uuid, which the API always issues; anything
|
|
363
|
+
// else would fail validation and drop the notification.
|
|
364
|
+
UUID_RE.test(instanceId) ? instanceId : undefined,
|
|
268
365
|
);
|
|
269
366
|
|
|
270
367
|
const result = await new Promise<{ messageId: string; notificationId: string }>(
|
|
@@ -301,15 +398,18 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
301
398
|
},
|
|
302
399
|
wait: async (duration) => {
|
|
303
400
|
const stepId = String(currentStepIndex++);
|
|
304
|
-
if (stepOutputMap.has(stepId))
|
|
401
|
+
if (stepOutputMap.has(stepId)) {
|
|
402
|
+
// A resume that arrives early — a duplicate trigger, a replayed
|
|
403
|
+
// message — must not cut the sleep short. The timer is still set.
|
|
404
|
+
const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
|
|
405
|
+
if (Number.isFinite(until) && until > Date.now()) {
|
|
406
|
+
throw new SuspendExecutionError("wait", { duration });
|
|
407
|
+
}
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
await stopIfCanceled();
|
|
305
411
|
|
|
306
|
-
|
|
307
|
-
let ms = 0;
|
|
308
|
-
if (duration.endsWith("d")) ms = parseInt(duration) * 24 * 60 * 60 * 1000;
|
|
309
|
-
else if (duration.endsWith("h")) ms = parseInt(duration) * 60 * 60 * 1000;
|
|
310
|
-
else if (duration.endsWith("m")) ms = parseInt(duration) * 60 * 1000;
|
|
311
|
-
else if (duration.endsWith("s")) ms = parseInt(duration) * 1000;
|
|
312
|
-
else throw new Error(`Invalid wait duration: ${duration}`);
|
|
412
|
+
const ms = durationToMs(duration, "wait duration");
|
|
313
413
|
|
|
314
414
|
const resumeAt = Date.now() + ms;
|
|
315
415
|
|
|
@@ -317,7 +417,7 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
317
417
|
// If the process dies after this point the source event is retried;
|
|
318
418
|
// if it dies after the database write, the timer is already durable.
|
|
319
419
|
await this.redisCli.zadd(
|
|
320
|
-
|
|
420
|
+
WORKFLOW_TIMERS_KEY,
|
|
321
421
|
resumeAt,
|
|
322
422
|
JSON.stringify({
|
|
323
423
|
instanceId: instance!.id,
|
|
@@ -341,27 +441,28 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
341
441
|
const stepId = String(currentStepIndex++);
|
|
342
442
|
if (stepOutputMap.has(stepId)) {
|
|
343
443
|
const out = stepOutputMap.get(stepId);
|
|
344
|
-
|
|
444
|
+
// Still waiting: neither the event nor the timeout has filled the
|
|
445
|
+
// step in, so this resume is spurious.
|
|
446
|
+
if (out === null || out === undefined) {
|
|
447
|
+
throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
448
|
+
}
|
|
449
|
+
if (typeof out === "object" && (out as any).timedOut === true) {
|
|
345
450
|
return null;
|
|
346
451
|
}
|
|
347
452
|
return out;
|
|
348
453
|
}
|
|
454
|
+
await stopIfCanceled();
|
|
349
455
|
|
|
350
456
|
options = options || {};
|
|
351
457
|
options.timeout = options.timeout || "24h";
|
|
352
458
|
options.match = options.match || {};
|
|
353
459
|
|
|
354
|
-
|
|
355
|
-
if (options.timeout.endsWith("d")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1000;
|
|
356
|
-
else if (options.timeout.endsWith("h")) ms = parseInt(options.timeout) * 60 * 60 * 1000;
|
|
357
|
-
else if (options.timeout.endsWith("m")) ms = parseInt(options.timeout) * 60 * 1000;
|
|
358
|
-
else if (options.timeout.endsWith("s")) ms = parseInt(options.timeout) * 1000;
|
|
359
|
-
else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);
|
|
460
|
+
const ms = durationToMs(options.timeout, "waitForEvent timeout");
|
|
360
461
|
|
|
361
462
|
const resumeAt = Date.now() + ms;
|
|
362
463
|
|
|
363
464
|
await this.redisCli.zadd(
|
|
364
|
-
|
|
465
|
+
WORKFLOW_TIMERS_KEY,
|
|
365
466
|
resumeAt,
|
|
366
467
|
JSON.stringify({
|
|
367
468
|
instanceId: instance!.id,
|
|
@@ -374,16 +475,8 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
374
475
|
}),
|
|
375
476
|
);
|
|
376
477
|
|
|
377
|
-
//
|
|
378
|
-
|
|
379
|
-
instanceId: instance!.id,
|
|
380
|
-
projectId: payload.projectId,
|
|
381
|
-
eventName: eventName,
|
|
382
|
-
matchCriteria: options.match,
|
|
383
|
-
expiresAt: new Date(resumeAt),
|
|
384
|
-
});
|
|
385
|
-
|
|
386
|
-
// Register step as pending event
|
|
478
|
+
// The pending step before the waiter: an event that matches the
|
|
479
|
+
// waiter the moment it exists must find a step to record itself in.
|
|
387
480
|
await this.dbConn.insert(workflowSteps).values({
|
|
388
481
|
instanceId: instance!.id,
|
|
389
482
|
projectId: payload.projectId,
|
|
@@ -392,11 +485,20 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
392
485
|
output: null, // this will be updated by event worker or timeout
|
|
393
486
|
});
|
|
394
487
|
|
|
488
|
+
await this.dbConn.insert(workflowWaiters).values({
|
|
489
|
+
instanceId: instance!.id,
|
|
490
|
+
projectId: payload.projectId,
|
|
491
|
+
eventName: eventName,
|
|
492
|
+
matchCriteria: options.match,
|
|
493
|
+
expiresAt: new Date(resumeAt),
|
|
494
|
+
});
|
|
495
|
+
|
|
395
496
|
throw new SuspendExecutionError("waitForEvent", { eventName });
|
|
396
497
|
},
|
|
397
498
|
run: async (stepName, fn) => {
|
|
398
499
|
const stepId = String(currentStepIndex++);
|
|
399
500
|
if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
|
|
501
|
+
await stopIfCanceled();
|
|
400
502
|
|
|
401
503
|
const result = await fn();
|
|
402
504
|
await this.dbConn.insert(workflowSteps).values({
|
|
@@ -415,33 +517,35 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
415
517
|
event: (instance!.input as any) || { user: { id: "unknown" } },
|
|
416
518
|
};
|
|
417
519
|
|
|
520
|
+
// Only from `running`: a cancel that landed mid-run must stand.
|
|
521
|
+
const finish = (status: "completed" | "pending" | "failed") =>
|
|
522
|
+
this.dbConn
|
|
523
|
+
.update(workflowInstances)
|
|
524
|
+
.set({ status })
|
|
525
|
+
.where(
|
|
526
|
+
and(eq(workflowInstances.id, instance!.id), eq(workflowInstances.status, "running")),
|
|
527
|
+
);
|
|
528
|
+
|
|
418
529
|
try {
|
|
419
530
|
await handler(ctx);
|
|
420
531
|
// If we reach here, workflow completed
|
|
421
|
-
await
|
|
422
|
-
.update(workflowInstances)
|
|
423
|
-
.set({ status: "completed" })
|
|
424
|
-
.where(eq(workflowInstances.id, instance!.id));
|
|
532
|
+
await finish("completed");
|
|
425
533
|
this.logger.info({ instanceId: instance!.id }, "Workflow completed successfully");
|
|
426
534
|
} catch (err: any) {
|
|
427
|
-
if (err instanceof
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
.where(eq(workflowInstances.id, instance!.id));
|
|
535
|
+
if (err instanceof WorkflowCanceledError) {
|
|
536
|
+
this.logger.info({ instanceId: instance!.id }, "Workflow canceled mid-run, stopping");
|
|
537
|
+
} else if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
|
|
538
|
+
await finish("pending");
|
|
432
539
|
this.logger.info({ instanceId: instance!.id, reason: err.reason }, "Workflow suspended");
|
|
433
540
|
} else {
|
|
434
541
|
this.logger.error({ err, instanceId: instance!.id }, "Workflow failed");
|
|
435
|
-
await
|
|
436
|
-
.update(workflowInstances)
|
|
437
|
-
.set({ status: "failed" })
|
|
438
|
-
.where(eq(workflowInstances.id, instance!.id));
|
|
542
|
+
await finish("failed");
|
|
439
543
|
}
|
|
440
544
|
}
|
|
441
545
|
} finally {
|
|
442
546
|
clearInterval(renewTimer);
|
|
443
547
|
// Compare-and-delete: never release a lock a later process re-acquired.
|
|
444
|
-
await this.redisCli.
|
|
548
|
+
await this.redisCli.releaseLock(lockKey, lockToken);
|
|
445
549
|
}
|
|
446
550
|
|
|
447
551
|
await Promise.all(publishPromises);
|
|
@@ -456,11 +560,7 @@ export function __injectForTests(r: any, d: any, wp: any, np: any) {
|
|
|
456
560
|
}
|
|
457
561
|
|
|
458
562
|
export async function startWorkflowWorker() {
|
|
459
|
-
logger =
|
|
460
|
-
redis = new RedisClient({ url: config.REDIS_URL, name: "workflow", logger });
|
|
461
|
-
const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "workflow", logger });
|
|
462
|
-
sql = dbData.sql;
|
|
463
|
-
db = dbData.db;
|
|
563
|
+
({ logger, redis, sql, db } = createWorkerRuntime(config, "workflow", "workflow-worker"));
|
|
464
564
|
notificationProducer = new StreamProducer({
|
|
465
565
|
redis: redis.native,
|
|
466
566
|
stream: STREAMS.INBOUND_NORMAL,
|
|
@@ -471,23 +571,15 @@ export async function startWorkflowWorker() {
|
|
|
471
571
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
472
572
|
logger,
|
|
473
573
|
});
|
|
474
|
-
consumer =
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
pendingScanner = new PendingMessageScanner({
|
|
485
|
-
redis: redis.native,
|
|
486
|
-
stream: STREAMS.WORKFLOW_INBOUND as StreamName,
|
|
487
|
-
group: CONSUMER_GROUPS.WORKFLOW as any,
|
|
488
|
-
consumer: `workflow-${process.pid}`,
|
|
489
|
-
logger,
|
|
490
|
-
});
|
|
574
|
+
({ consumer, pendingScanner } = createStreamConsumer(
|
|
575
|
+
{ redis, logger },
|
|
576
|
+
{
|
|
577
|
+
stream: STREAMS.WORKFLOW_INBOUND,
|
|
578
|
+
group: CONSUMER_GROUPS.WORKFLOW,
|
|
579
|
+
consumer: uniqueConsumerId("workflow"),
|
|
580
|
+
batchSize: config.WORKER_CONCURRENCY,
|
|
581
|
+
},
|
|
582
|
+
));
|
|
491
583
|
|
|
492
584
|
worker = new WorkflowWorker({
|
|
493
585
|
consumer,
|
|
@@ -505,14 +597,12 @@ export async function startWorkflowWorker() {
|
|
|
505
597
|
void (async () => {
|
|
506
598
|
try {
|
|
507
599
|
const now = Date.now();
|
|
508
|
-
const tasks =
|
|
509
|
-
|
|
510
|
-
1,
|
|
511
|
-
"notif:workflow:timers",
|
|
600
|
+
const tasks = await redis.native.schedulerPoll(
|
|
601
|
+
WORKFLOW_TIMERS_KEY,
|
|
512
602
|
now,
|
|
513
603
|
100,
|
|
514
604
|
WORKFLOW_TIMER_VISIBILITY_MS,
|
|
515
|
-
)
|
|
605
|
+
);
|
|
516
606
|
|
|
517
607
|
for (const taskStr of tasks) {
|
|
518
608
|
const task = JSON.parse(taskStr);
|
|
@@ -525,8 +615,13 @@ export async function startWorkflowWorker() {
|
|
|
525
615
|
.where(eq(workflowInstances.id, task.instanceId))
|
|
526
616
|
.limit(1)
|
|
527
617
|
)[0];
|
|
528
|
-
if (
|
|
529
|
-
|
|
618
|
+
if (
|
|
619
|
+
!inst ||
|
|
620
|
+
inst.status === "completed" ||
|
|
621
|
+
inst.status === "failed" ||
|
|
622
|
+
inst.status === "canceled"
|
|
623
|
+
) {
|
|
624
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
530
625
|
continue;
|
|
531
626
|
}
|
|
532
627
|
// A timer can be observed while the worker is still persisting the
|
|
@@ -536,7 +631,27 @@ export async function startWorkflowWorker() {
|
|
|
536
631
|
if (inst.status === "running") continue;
|
|
537
632
|
|
|
538
633
|
if (task.isEventTimeout) {
|
|
539
|
-
//
|
|
634
|
+
// A timeout for one waitForEvent step. If that step already has an
|
|
635
|
+
// output the event came first and this timer is stale; acting on it
|
|
636
|
+
// would time out a later wait on the same event name.
|
|
637
|
+
if (task.stepId !== undefined) {
|
|
638
|
+
const [own] = await db
|
|
639
|
+
.select({ output: workflowSteps.output })
|
|
640
|
+
.from(workflowSteps)
|
|
641
|
+
.where(
|
|
642
|
+
and(
|
|
643
|
+
eq(workflowSteps.instanceId, task.instanceId),
|
|
644
|
+
eq(workflowSteps.stepIndex, String(task.stepId)),
|
|
645
|
+
),
|
|
646
|
+
)
|
|
647
|
+
.limit(1);
|
|
648
|
+
if (!own || own.output !== null) {
|
|
649
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
650
|
+
continue;
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
// Clean up only this step's waiter
|
|
540
655
|
const deleted = await db
|
|
541
656
|
.delete(workflowWaiters)
|
|
542
657
|
.where(
|
|
@@ -550,7 +665,7 @@ export async function startWorkflowWorker() {
|
|
|
550
665
|
// If the waiter was already deleted (by EventWorker when event arrived before timeout),
|
|
551
666
|
// this timer is stale. Clean up from Redis and skip re-resuming the workflow.
|
|
552
667
|
if (deleted.length === 0) {
|
|
553
|
-
await redis.native.zrem(
|
|
668
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
554
669
|
continue;
|
|
555
670
|
}
|
|
556
671
|
|
|
@@ -564,7 +679,10 @@ export async function startWorkflowWorker() {
|
|
|
564
679
|
eq(workflowSteps.action, "waitForEvent"),
|
|
565
680
|
),
|
|
566
681
|
);
|
|
567
|
-
const pendingStep =
|
|
682
|
+
const pendingStep =
|
|
683
|
+
task.stepId !== undefined
|
|
684
|
+
? steps.find((s: any) => s.stepIndex === String(task.stepId) && s.output === null)
|
|
685
|
+
: steps.find((s: any) => s.output === null);
|
|
568
686
|
if (pendingStep) {
|
|
569
687
|
await db
|
|
570
688
|
.update(workflowSteps)
|
|
@@ -576,7 +694,7 @@ export async function startWorkflowWorker() {
|
|
|
576
694
|
await workflowProducer.publish(
|
|
577
695
|
buildStreamEvent("workflow.resumed", task, "scheduler", undefined),
|
|
578
696
|
);
|
|
579
|
-
await redis.native.zrem(
|
|
697
|
+
await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
|
|
580
698
|
logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
|
|
581
699
|
}
|
|
582
700
|
} catch (err) {
|
|
@@ -616,12 +734,14 @@ export async function startWorkflowWorker() {
|
|
|
616
734
|
}
|
|
617
735
|
|
|
618
736
|
export async function stopWorkflowWorker(): Promise<void> {
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
737
|
+
await shutdownWorker("workflow worker", {
|
|
738
|
+
logger,
|
|
739
|
+
timers: [healthInterval, pollInterval, reaperInterval],
|
|
740
|
+
worker,
|
|
741
|
+
sql,
|
|
742
|
+
redis,
|
|
743
|
+
});
|
|
744
|
+
healthInterval = null;
|
|
745
|
+
pollInterval = null;
|
|
746
|
+
reaperInterval = null;
|
|
627
747
|
}
|