notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -0,0 +1,117 @@
1
+ import { and, eq, inArray } from "drizzle-orm";
2
+ import { suppressions } from "@/db/schema.js";
3
+ import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
4
+ import { normaliseTarget } from "@/shared/index.js";
5
+
6
+ export type BlockReason =
7
+ "suppressed" | "recipient_not_found" | "user_opted_out" | "channel_disabled" | "contact_inactive";
8
+
9
+ /** The fields of a parked `notification.dispatched` payload the gate reads. */
10
+ interface ParkedTask {
11
+ taskId: string;
12
+ projectId: string;
13
+ recipientId: string;
14
+ channel: string;
15
+ templateId?: string;
16
+ destination?: string;
17
+ }
18
+
19
+ /**
20
+ * Re-checks parked sends at the moment they are released.
21
+ *
22
+ * The engine clears a message against suppressions, opt-outs and contacts when
23
+ * it first sees it. A send parked for later — a `sendAt`, a quiet-hours
24
+ * deferral, a retry — can sit for hours or days, and whatever the person did in
25
+ * between (unsubscribed, complained, hard-bounced, removed the address) must
26
+ * still stop it. Without this the scheduler sent to them anyway.
27
+ *
28
+ * Reads Postgres directly rather than through the repositories' caches: this
29
+ * runs once per poll, not once per message, and a stale "still subscribed" is
30
+ * the very thing it exists to prevent.
31
+ *
32
+ * Returns the blocked tasks with the reason. Throws if a lookup fails, so the
33
+ * caller holds the batch rather than sending it unchecked.
34
+ */
35
+ export async function findBlockedAtRelease(
36
+ db: any,
37
+ tasks: ParkedTask[],
38
+ ): Promise<Map<string, BlockReason>> {
39
+ const blocked = new Map<string, BlockReason>();
40
+
41
+ const byProject = new Map<string, ParkedTask[]>();
42
+ for (const t of tasks) {
43
+ if (!t?.projectId || !t.recipientId) continue;
44
+ if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
45
+ byProject.get(t.projectId)!.push(t);
46
+ }
47
+
48
+ for (const [projectId, group] of byProject) {
49
+ const recipientIds = [...new Set(group.map((t) => t.recipientId))];
50
+ const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))] as string[];
51
+ const targets = [
52
+ ...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean)),
53
+ ] as string[];
54
+
55
+ const templateRepo = new TemplateRepository(db);
56
+ const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
57
+ targets.length > 0
58
+ ? db
59
+ .select({ channel: suppressions.channel, target: suppressions.target })
60
+ .from(suppressions)
61
+ .where(
62
+ and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets)),
63
+ )
64
+ : [],
65
+ new UserRepository(db).findRecordsByIds(projectId, recipientIds),
66
+ new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
67
+ Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id))),
68
+ ]);
69
+
70
+ const suppressed = new Set(
71
+ (suppressedRows as { channel: string; target: string }[]).map(
72
+ (r) => `${r.channel}:${normaliseTarget(r.target)}`,
73
+ ),
74
+ );
75
+ const userById = new Map(users.map((u) => [u.userId, u]));
76
+ const topicsByTemplate = new Map<string, string[]>();
77
+ templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
78
+
79
+ for (const t of group) {
80
+ const topics = t.templateId ? (topicsByTemplate.get(t.templateId) ?? []) : [];
81
+ const reason = decide(t, topics, suppressed, userById.get(t.recipientId), contactsByUser);
82
+ if (reason) blocked.set(t.taskId, reason);
83
+ }
84
+ }
85
+
86
+ return blocked;
87
+ }
88
+
89
+ function decide(
90
+ t: ParkedTask,
91
+ topics: string[],
92
+ suppressed: Set<string>,
93
+ user:
94
+ | { preferences: { topics?: Record<string, boolean>; channels?: Record<string, boolean> } }
95
+ | undefined,
96
+ contactsByUser: Map<string, { channel: string; target: string; preferences?: any }[]>,
97
+ ): BlockReason | undefined {
98
+ // Checked first and for every priority, as the engine does: a suppression
99
+ // records that the address asked us to stop or is dead.
100
+ if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) {
101
+ return "suppressed";
102
+ }
103
+ if (!user) return "recipient_not_found";
104
+ if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
105
+ if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
106
+
107
+ if (t.destination) {
108
+ const contact = (contactsByUser.get(t.recipientId) ?? []).find(
109
+ (c) => c.channel === t.channel && c.target === t.destination,
110
+ );
111
+ if (!contact) return "contact_inactive";
112
+ if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) {
113
+ return "user_opted_out";
114
+ }
115
+ }
116
+ return undefined;
117
+ }
@@ -1,17 +1,15 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { loadEnv, readBaseConfig } from "@/index.js";
3
- import { createLogger } from "@/index.js";
4
- import { RedisClient, type Redis } from "@/index.js";
3
+ import type { Logger } from "@/index.js";
4
+ import type { RedisClient, Redis } from "@/index.js";
5
5
  import {
6
- StreamConsumer,
7
- PendingMessageScanner,
6
+ type StreamConsumer,
7
+ type PendingMessageScanner,
8
8
  StreamProducer,
9
9
  type StreamMessage,
10
10
  } from "@/index.js";
11
11
  import { BaseWorker } from "@/index.js";
12
12
  import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/index.js";
13
- import { type StreamName } from "@/contracts/streams.js";
14
- import { createDatabase } from "@/db/index.js";
15
13
  import {
16
14
  workflowInstances,
17
15
  workflowSteps,
@@ -26,13 +24,14 @@ import {
26
24
  type WorkflowContext,
27
25
  type WorkflowStepContext,
28
26
  } from "@/workflows/index.js";
29
- import {
30
- type WorkerOptions,
31
- LUA_SCHEDULER_POLL,
32
- LUA_RELEASE_LOCK,
33
- LUA_RENEW_LOCK,
34
- } from "@/shared/index.js";
27
+ import { type WorkerOptions } from "@/shared/index.js";
35
28
  import { startHealthReporter } from "@/workers/index.js";
29
+ import {
30
+ createStreamConsumer,
31
+ createWorkerRuntime,
32
+ shutdownWorker,
33
+ uniqueConsumerId,
34
+ } from "@/workers/bootstrap.js";
36
35
 
37
36
  /** How long a workflow instance lock is held before it self-expires. */
38
37
  const WORKFLOW_LOCK_TTL_SECONDS = 60;
@@ -40,10 +39,48 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
40
39
  const WORKFLOW_LOCK_RENEW_MS = (WORKFLOW_LOCK_TTL_SECONDS / 3) * 1000;
41
40
  /** How long a claimed workflow timer stays invisible to other pollers. */
42
41
  const WORKFLOW_TIMER_VISIBILITY_MS = 60_000;
42
+ /** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
43
+ export const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
44
+ /** How soon a resume that found its instance busy is tried again. */
45
+ const WORKFLOW_RESUME_RETRY_MS = 2_000;
46
+ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
47
+
48
+ /** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
49
+ const DURATION_UNIT_SECONDS: Record<string, number> = {
50
+ ms: 0.001,
51
+ s: 1,
52
+ m: 60,
53
+ h: 3_600,
54
+ d: 86_400,
55
+ };
56
+
57
+ /**
58
+ * `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
59
+ * read exactly; anything that is not a number and a unit throws. The old
60
+ * parser matched on the last character, so "500ms" meant 500 seconds, and ran
61
+ * parseInt, so "1.5h" meant one hour.
62
+ */
63
+ export function parseDurationSeconds(value: string, what: string): number {
64
+ const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
65
+ if (!match) throw new Error(`Invalid ${what}: ${value}`);
66
+ return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]!]!;
67
+ }
68
+
69
+ function durationToMs(value: string, what: string): number {
70
+ return Math.round(parseDurationSeconds(value, what) * 1000);
71
+ }
72
+
73
+ /** Thrown at a step boundary once the instance has been canceled. */
74
+ class WorkflowCanceledError extends Error {
75
+ constructor() {
76
+ super("Workflow instance was canceled");
77
+ this.name = "WorkflowCanceledError";
78
+ }
79
+ }
43
80
 
44
81
  loadEnv();
45
82
  const config = readBaseConfig();
46
- let logger: ReturnType<typeof createLogger>;
83
+ let logger: Logger;
47
84
  let redis: RedisClient;
48
85
  let sql: any;
49
86
  let db: any;
@@ -90,12 +127,15 @@ export class WorkflowWorker extends BaseWorker {
90
127
  }
91
128
 
92
129
  override async stop(): Promise<void> {
130
+ // The flush interval keeps running until in-flight runs settle: a step's
131
+ // notify is waiting on it, and clearing it first left the drain to the 30s
132
+ // stop timeout with the instance lock still being renewed.
133
+ await super.stop();
93
134
  if (this.flushTimer) {
94
135
  clearInterval(this.flushTimer);
95
136
  this.flushTimer = null;
96
137
  }
97
138
  await this.flushWorkerBuffers();
98
- await super.stop();
99
139
  }
100
140
 
101
141
  private async flushWorkerBuffers(): Promise<void> {
@@ -125,6 +165,29 @@ export class WorkflowWorker extends BaseWorker {
125
165
  }
126
166
  }
127
167
 
168
+ /**
169
+ * Re-delivers a resume through the timer ZSET a few seconds from now. The
170
+ * poll loop publishes it once the instance is pending again, and drops it if
171
+ * the instance has finished. The member carries no timestamp, so repeated
172
+ * deferrals of one instance collapse into a single entry. It is marked
173
+ * `deferred` so it never equals a `wait` timer's member: sharing one let the
174
+ * ZADD here re-score a sleep's wake-up to seconds from now, and the early
175
+ * resume then re-suspended with no timer left to wake it.
176
+ */
177
+ private async deferResume(payload: any): Promise<void> {
178
+ await this.redisCli.zadd(
179
+ WORKFLOW_TIMERS_KEY,
180
+ Date.now() + WORKFLOW_RESUME_RETRY_MS,
181
+ JSON.stringify({
182
+ instanceId: payload.instanceId,
183
+ name: payload.name,
184
+ projectId: payload.projectId,
185
+ input: payload.input ?? {},
186
+ deferred: true,
187
+ }),
188
+ );
189
+ }
190
+
128
191
  async process(message: StreamMessage): Promise<void> {
129
192
  const { event } = message;
130
193
  const publishPromises: Promise<void>[] = [];
@@ -195,7 +258,12 @@ export class WorkflowWorker extends BaseWorker {
195
258
  "NX",
196
259
  );
197
260
  if (!acquired) {
198
- this.logger.info({ instanceId }, "Workflow is locked by another process, skipping");
261
+ // Another process is mid-run on this instance — typically still
262
+ // persisting the very suspension this resume answers, when the awaited
263
+ // event arrived quickly. Acking here dropped the resume, and with the
264
+ // waiter already consumed nothing would ever wake the instance again.
265
+ this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
266
+ await this.deferResume(payload);
199
267
  return;
200
268
  }
201
269
 
@@ -203,7 +271,7 @@ export class WorkflowWorker extends BaseWorker {
203
271
  // outlives the TTL lets a second resume execute the same steps in parallel.
204
272
  const renewTimer = setInterval(() => {
205
273
  void this.redisCli
206
- .eval(LUA_RENEW_LOCK, 1, lockKey, lockToken, String(WORKFLOW_LOCK_TTL_SECONDS))
274
+ .renewLock(lockKey, lockToken, WORKFLOW_LOCK_TTL_SECONDS)
207
275
  .catch((err: unknown) => {
208
276
  this.logger.warn({ err, instanceId }, "failed to renew workflow lock");
209
277
  });
@@ -231,15 +299,40 @@ export class WorkflowWorker extends BaseWorker {
231
299
  instance = rows[0]!;
232
300
  }
233
301
 
302
+ if (instance.status === "running") {
303
+ // Its last holder died mid-run (the reaper will set it back to
304
+ // pending) or has not finished suspending. Either way this resume
305
+ // is still owed.
306
+ await this.deferResume(payload);
307
+ return;
308
+ }
234
309
  if (instance.status !== "pending") {
235
310
  this.logger.info({ instanceId }, "Workflow is not pending, skipping");
236
311
  return;
237
312
  }
238
313
 
239
- await this.dbConn
314
+ // Conditional, so a cancel that lands between the read above and here
315
+ // is not overwritten.
316
+ const claimed = await this.dbConn
240
317
  .update(workflowInstances)
241
318
  .set({ status: "running" })
242
- .where(eq(workflowInstances.id, instanceId));
319
+ .where(and(eq(workflowInstances.id, instanceId), eq(workflowInstances.status, "pending")))
320
+ .returning({ id: workflowInstances.id });
321
+ if (claimed.length === 0) {
322
+ this.logger.info({ instanceId }, "Workflow is no longer pending, skipping");
323
+ return;
324
+ }
325
+
326
+ // A cancel can arrive while the handler runs. Checked before each new
327
+ // step, so a canceled instance stops at the next step boundary.
328
+ const stopIfCanceled = async () => {
329
+ const [row] = await this.dbConn
330
+ .select({ status: workflowInstances.status })
331
+ .from(workflowInstances)
332
+ .where(eq(workflowInstances.id, instanceId))
333
+ .limit(1);
334
+ if (row?.status === "canceled") throw new WorkflowCanceledError();
335
+ };
243
336
 
244
337
  // Load existing steps
245
338
  const existingSteps = await this.dbConn
@@ -257,6 +350,7 @@ export class WorkflowWorker extends BaseWorker {
257
350
  notify: async (args) => {
258
351
  const stepId = String(currentStepIndex++);
259
352
  if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
353
+ await stopIfCanceled();
260
354
 
261
355
  // The step payload is the same shape as notify(), but the wire event
262
356
  // is not — translate rather than spread.
@@ -265,6 +359,9 @@ export class WorkflowWorker extends BaseWorker {
265
359
  instance!.input,
266
360
  payload.projectId,
267
361
  `wf-${instanceId}-${stepId}`,
362
+ // The contract wants a uuid, which the API always issues; anything
363
+ // else would fail validation and drop the notification.
364
+ UUID_RE.test(instanceId) ? instanceId : undefined,
268
365
  );
269
366
 
270
367
  const result = await new Promise<{ messageId: string; notificationId: string }>(
@@ -301,15 +398,18 @@ export class WorkflowWorker extends BaseWorker {
301
398
  },
302
399
  wait: async (duration) => {
303
400
  const stepId = String(currentStepIndex++);
304
- if (stepOutputMap.has(stepId)) return;
401
+ if (stepOutputMap.has(stepId)) {
402
+ // A resume that arrives early — a duplicate trigger, a replayed
403
+ // message — must not cut the sleep short. The timer is still set.
404
+ const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
405
+ if (Number.isFinite(until) && until > Date.now()) {
406
+ throw new SuspendExecutionError("wait", { duration });
407
+ }
408
+ return;
409
+ }
410
+ await stopIfCanceled();
305
411
 
306
- // Simple duration parse (e.g. '2h' -> ms)
307
- let ms = 0;
308
- if (duration.endsWith("d")) ms = parseInt(duration) * 24 * 60 * 60 * 1000;
309
- else if (duration.endsWith("h")) ms = parseInt(duration) * 60 * 60 * 1000;
310
- else if (duration.endsWith("m")) ms = parseInt(duration) * 60 * 1000;
311
- else if (duration.endsWith("s")) ms = parseInt(duration) * 1000;
312
- else throw new Error(`Invalid wait duration: ${duration}`);
412
+ const ms = durationToMs(duration, "wait duration");
313
413
 
314
414
  const resumeAt = Date.now() + ms;
315
415
 
@@ -317,7 +417,7 @@ export class WorkflowWorker extends BaseWorker {
317
417
  // If the process dies after this point the source event is retried;
318
418
  // if it dies after the database write, the timer is already durable.
319
419
  await this.redisCli.zadd(
320
- "notif:workflow:timers",
420
+ WORKFLOW_TIMERS_KEY,
321
421
  resumeAt,
322
422
  JSON.stringify({
323
423
  instanceId: instance!.id,
@@ -341,27 +441,28 @@ export class WorkflowWorker extends BaseWorker {
341
441
  const stepId = String(currentStepIndex++);
342
442
  if (stepOutputMap.has(stepId)) {
343
443
  const out = stepOutputMap.get(stepId);
344
- if (out && typeof out === "object" && (out as any).timedOut === true) {
444
+ // Still waiting: neither the event nor the timeout has filled the
445
+ // step in, so this resume is spurious.
446
+ if (out === null || out === undefined) {
447
+ throw new SuspendExecutionError("waitForEvent", { eventName });
448
+ }
449
+ if (typeof out === "object" && (out as any).timedOut === true) {
345
450
  return null;
346
451
  }
347
452
  return out;
348
453
  }
454
+ await stopIfCanceled();
349
455
 
350
456
  options = options || {};
351
457
  options.timeout = options.timeout || "24h";
352
458
  options.match = options.match || {};
353
459
 
354
- let ms = 0;
355
- if (options.timeout.endsWith("d")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1000;
356
- else if (options.timeout.endsWith("h")) ms = parseInt(options.timeout) * 60 * 60 * 1000;
357
- else if (options.timeout.endsWith("m")) ms = parseInt(options.timeout) * 60 * 1000;
358
- else if (options.timeout.endsWith("s")) ms = parseInt(options.timeout) * 1000;
359
- else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);
460
+ const ms = durationToMs(options.timeout, "waitForEvent timeout");
360
461
 
361
462
  const resumeAt = Date.now() + ms;
362
463
 
363
464
  await this.redisCli.zadd(
364
- "notif:workflow:timers",
465
+ WORKFLOW_TIMERS_KEY,
365
466
  resumeAt,
366
467
  JSON.stringify({
367
468
  instanceId: instance!.id,
@@ -374,16 +475,8 @@ export class WorkflowWorker extends BaseWorker {
374
475
  }),
375
476
  );
376
477
 
377
- // Register waiter
378
- await this.dbConn.insert(workflowWaiters).values({
379
- instanceId: instance!.id,
380
- projectId: payload.projectId,
381
- eventName: eventName,
382
- matchCriteria: options.match,
383
- expiresAt: new Date(resumeAt),
384
- });
385
-
386
- // Register step as pending event
478
+ // The pending step before the waiter: an event that matches the
479
+ // waiter the moment it exists must find a step to record itself in.
387
480
  await this.dbConn.insert(workflowSteps).values({
388
481
  instanceId: instance!.id,
389
482
  projectId: payload.projectId,
@@ -392,11 +485,20 @@ export class WorkflowWorker extends BaseWorker {
392
485
  output: null, // this will be updated by event worker or timeout
393
486
  });
394
487
 
488
+ await this.dbConn.insert(workflowWaiters).values({
489
+ instanceId: instance!.id,
490
+ projectId: payload.projectId,
491
+ eventName: eventName,
492
+ matchCriteria: options.match,
493
+ expiresAt: new Date(resumeAt),
494
+ });
495
+
395
496
  throw new SuspendExecutionError("waitForEvent", { eventName });
396
497
  },
397
498
  run: async (stepName, fn) => {
398
499
  const stepId = String(currentStepIndex++);
399
500
  if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
501
+ await stopIfCanceled();
400
502
 
401
503
  const result = await fn();
402
504
  await this.dbConn.insert(workflowSteps).values({
@@ -415,33 +517,35 @@ export class WorkflowWorker extends BaseWorker {
415
517
  event: (instance!.input as any) || { user: { id: "unknown" } },
416
518
  };
417
519
 
520
+ // Only from `running`: a cancel that landed mid-run must stand.
521
+ const finish = (status: "completed" | "pending" | "failed") =>
522
+ this.dbConn
523
+ .update(workflowInstances)
524
+ .set({ status })
525
+ .where(
526
+ and(eq(workflowInstances.id, instance!.id), eq(workflowInstances.status, "running")),
527
+ );
528
+
418
529
  try {
419
530
  await handler(ctx);
420
531
  // If we reach here, workflow completed
421
- await this.dbConn
422
- .update(workflowInstances)
423
- .set({ status: "completed" })
424
- .where(eq(workflowInstances.id, instance!.id));
532
+ await finish("completed");
425
533
  this.logger.info({ instanceId: instance!.id }, "Workflow completed successfully");
426
534
  } catch (err: any) {
427
- if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
428
- await this.dbConn
429
- .update(workflowInstances)
430
- .set({ status: "pending" })
431
- .where(eq(workflowInstances.id, instance!.id));
535
+ if (err instanceof WorkflowCanceledError) {
536
+ this.logger.info({ instanceId: instance!.id }, "Workflow canceled mid-run, stopping");
537
+ } else if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
538
+ await finish("pending");
432
539
  this.logger.info({ instanceId: instance!.id, reason: err.reason }, "Workflow suspended");
433
540
  } else {
434
541
  this.logger.error({ err, instanceId: instance!.id }, "Workflow failed");
435
- await this.dbConn
436
- .update(workflowInstances)
437
- .set({ status: "failed" })
438
- .where(eq(workflowInstances.id, instance!.id));
542
+ await finish("failed");
439
543
  }
440
544
  }
441
545
  } finally {
442
546
  clearInterval(renewTimer);
443
547
  // Compare-and-delete: never release a lock a later process re-acquired.
444
- await this.redisCli.eval(LUA_RELEASE_LOCK, 1, lockKey, lockToken);
548
+ await this.redisCli.releaseLock(lockKey, lockToken);
445
549
  }
446
550
 
447
551
  await Promise.all(publishPromises);
@@ -456,11 +560,7 @@ export function __injectForTests(r: any, d: any, wp: any, np: any) {
456
560
  }
457
561
 
458
562
  export async function startWorkflowWorker() {
459
- logger = createLogger({ name: "workflow-worker", level: config.LOG_LEVEL });
460
- redis = new RedisClient({ url: config.REDIS_URL, name: "workflow", logger });
461
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "workflow", logger });
462
- sql = dbData.sql;
463
- db = dbData.db;
563
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "workflow", "workflow-worker"));
464
564
  notificationProducer = new StreamProducer({
465
565
  redis: redis.native,
466
566
  stream: STREAMS.INBOUND_NORMAL,
@@ -471,23 +571,15 @@ export async function startWorkflowWorker() {
471
571
  stream: STREAMS.WORKFLOW_INBOUND,
472
572
  logger,
473
573
  });
474
- consumer = new StreamConsumer({
475
- redis: redis.native,
476
- stream: STREAMS.WORKFLOW_INBOUND as StreamName,
477
- group: CONSUMER_GROUPS.WORKFLOW as any,
478
- consumer: `workflow-${process.pid}`,
479
- dlqStream: STREAMS.DEAD_LETTER,
480
- batchSize: config.WORKER_CONCURRENCY,
481
- logger,
482
- });
483
-
484
- pendingScanner = new PendingMessageScanner({
485
- redis: redis.native,
486
- stream: STREAMS.WORKFLOW_INBOUND as StreamName,
487
- group: CONSUMER_GROUPS.WORKFLOW as any,
488
- consumer: `workflow-${process.pid}`,
489
- logger,
490
- });
574
+ ({ consumer, pendingScanner } = createStreamConsumer(
575
+ { redis, logger },
576
+ {
577
+ stream: STREAMS.WORKFLOW_INBOUND,
578
+ group: CONSUMER_GROUPS.WORKFLOW,
579
+ consumer: uniqueConsumerId("workflow"),
580
+ batchSize: config.WORKER_CONCURRENCY,
581
+ },
582
+ ));
491
583
 
492
584
  worker = new WorkflowWorker({
493
585
  consumer,
@@ -505,14 +597,12 @@ export async function startWorkflowWorker() {
505
597
  void (async () => {
506
598
  try {
507
599
  const now = Date.now();
508
- const tasks = (await redis.native.eval(
509
- LUA_SCHEDULER_POLL,
510
- 1,
511
- "notif:workflow:timers",
600
+ const tasks = await redis.native.schedulerPoll(
601
+ WORKFLOW_TIMERS_KEY,
512
602
  now,
513
603
  100,
514
604
  WORKFLOW_TIMER_VISIBILITY_MS,
515
- )) as string[];
605
+ );
516
606
 
517
607
  for (const taskStr of tasks) {
518
608
  const task = JSON.parse(taskStr);
@@ -525,8 +615,13 @@ export async function startWorkflowWorker() {
525
615
  .where(eq(workflowInstances.id, task.instanceId))
526
616
  .limit(1)
527
617
  )[0];
528
- if (!inst || inst.status === "completed" || inst.status === "failed") {
529
- await redis.native.zrem("notif:workflow:timers", taskStr);
618
+ if (
619
+ !inst ||
620
+ inst.status === "completed" ||
621
+ inst.status === "failed" ||
622
+ inst.status === "canceled"
623
+ ) {
624
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
530
625
  continue;
531
626
  }
532
627
  // A timer can be observed while the worker is still persisting the
@@ -536,7 +631,27 @@ export async function startWorkflowWorker() {
536
631
  if (inst.status === "running") continue;
537
632
 
538
633
  if (task.isEventTimeout) {
539
- // It's a timeout for waitForEvent. Clean up only this step's waiter
634
+ // A timeout for one waitForEvent step. If that step already has an
635
+ // output the event came first and this timer is stale; acting on it
636
+ // would time out a later wait on the same event name.
637
+ if (task.stepId !== undefined) {
638
+ const [own] = await db
639
+ .select({ output: workflowSteps.output })
640
+ .from(workflowSteps)
641
+ .where(
642
+ and(
643
+ eq(workflowSteps.instanceId, task.instanceId),
644
+ eq(workflowSteps.stepIndex, String(task.stepId)),
645
+ ),
646
+ )
647
+ .limit(1);
648
+ if (!own || own.output !== null) {
649
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
650
+ continue;
651
+ }
652
+ }
653
+
654
+ // Clean up only this step's waiter
540
655
  const deleted = await db
541
656
  .delete(workflowWaiters)
542
657
  .where(
@@ -550,7 +665,7 @@ export async function startWorkflowWorker() {
550
665
  // If the waiter was already deleted (by EventWorker when event arrived before timeout),
551
666
  // this timer is stale. Clean up from Redis and skip re-resuming the workflow.
552
667
  if (deleted.length === 0) {
553
- await redis.native.zrem("notif:workflow:timers", taskStr);
668
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
554
669
  continue;
555
670
  }
556
671
 
@@ -564,7 +679,10 @@ export async function startWorkflowWorker() {
564
679
  eq(workflowSteps.action, "waitForEvent"),
565
680
  ),
566
681
  );
567
- const pendingStep = steps.find((s: any) => s.output === null);
682
+ const pendingStep =
683
+ task.stepId !== undefined
684
+ ? steps.find((s: any) => s.stepIndex === String(task.stepId) && s.output === null)
685
+ : steps.find((s: any) => s.output === null);
568
686
  if (pendingStep) {
569
687
  await db
570
688
  .update(workflowSteps)
@@ -576,7 +694,7 @@ export async function startWorkflowWorker() {
576
694
  await workflowProducer.publish(
577
695
  buildStreamEvent("workflow.resumed", task, "scheduler", undefined),
578
696
  );
579
- await redis.native.zrem("notif:workflow:timers", taskStr);
697
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
580
698
  logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
581
699
  }
582
700
  } catch (err) {
@@ -616,12 +734,14 @@ export async function startWorkflowWorker() {
616
734
  }
617
735
 
618
736
  export async function stopWorkflowWorker(): Promise<void> {
619
- logger?.info("shutdown initiated");
620
- if (healthInterval) clearInterval(healthInterval);
621
- if (pollInterval) clearInterval(pollInterval);
622
- if (reaperInterval) clearInterval(reaperInterval);
623
- if (worker) await worker.stop();
624
- if (sql) await sql.end();
625
- if (redis) await redis.disconnect();
626
- logger?.info("workflow worker stopped");
737
+ await shutdownWorker("workflow worker", {
738
+ logger,
739
+ timers: [healthInterval, pollInterval, reaperInterval],
740
+ worker,
741
+ sql,
742
+ redis,
743
+ });
744
+ healthInterval = null;
745
+ pollInterval = null;
746
+ reaperInterval = null;
627
747
  }