notifkit 0.1.10 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  2. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  3. package/dashboard/dist/index.html +2 -2
  4. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  5. package/dist/bulk-BvRfz6Of.mjs +77 -0
  6. package/dist/index.d.mts +1100 -8883
  7. package/dist/index.mjs +2 -38
  8. package/dist/{main-C9s-2ArH.mjs → main-BENOY7HG.mjs} +49 -75
  9. package/dist/{main-BiFiOX9D.mjs → main-BpnuFX62.mjs} +119 -112
  10. package/dist/{main-Dz2sUh3c.mjs → main-BqVH2BHj.mjs} +57 -54
  11. package/dist/{main-BgmT1MPe.mjs → main-C_5VFIwp.mjs} +135 -79
  12. package/dist/main-D6od9N8h.mjs +312 -0
  13. package/dist/main-D84OX0ux.mjs +811 -0
  14. package/dist/{main-39DtKrX4.mjs → main-HH-WwsMM.mjs} +395 -184
  15. package/dist/{main-B3iQrl3t.mjs → main-HY_TDs61.mjs} +149 -134
  16. package/dist/{src-QiTu5e_a.mjs → src-CI7FGf3h.mjs} +1220 -651
  17. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  18. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  19. package/drizzle/meta/_journal.json +14 -0
  20. package/package.json +17 -17
  21. package/src/client.ts +15 -5
  22. package/src/config/index.ts +26 -3
  23. package/src/contracts/events/notification-ai-pending.ts +12 -0
  24. package/src/contracts/events/notification-delivered.ts +7 -0
  25. package/src/contracts/events/notification-dispatched.ts +21 -2
  26. package/src/contracts/events/notification-enriched.ts +16 -0
  27. package/src/contracts/events/notification-failed.ts +2 -0
  28. package/src/contracts/events/notification-requested.ts +2 -0
  29. package/src/contracts/registry.ts +2 -2
  30. package/src/contracts/sdk.ts +77 -29
  31. package/src/db/bulk.ts +132 -0
  32. package/src/db/index.ts +2 -2
  33. package/src/db/schema.ts +3 -50
  34. package/src/idempotency/index.ts +60 -6
  35. package/src/logger/index.ts +0 -12
  36. package/src/queue/index.ts +255 -76
  37. package/src/rate-limiter/index.ts +217 -16
  38. package/src/redis/index.ts +265 -0
  39. package/src/repositories/index.ts +171 -82
  40. package/src/server.ts +61 -41
  41. package/src/services/ai/main.ts +83 -70
  42. package/src/services/api/admin-static.ts +41 -12
  43. package/src/services/api/handlers.ts +357 -159
  44. package/src/services/api/http.ts +34 -7
  45. package/src/services/api/main.ts +104 -67
  46. package/src/services/api/router.ts +18 -5
  47. package/src/services/delivery/main.ts +495 -230
  48. package/src/services/delivery/throttle.ts +42 -59
  49. package/src/services/engine/main.ts +206 -152
  50. package/src/services/enricher/main.ts +155 -107
  51. package/src/services/events/main.ts +100 -55
  52. package/src/services/scheduler/main.ts +151 -79
  53. package/src/services/scheduler/send-gate.ts +117 -0
  54. package/src/services/workflow/main.ts +220 -100
  55. package/src/shared/batch-processor.ts +65 -34
  56. package/src/shared/dataloader.ts +58 -22
  57. package/src/shared/events.ts +3 -0
  58. package/src/shared/index.ts +0 -1
  59. package/src/shared/utils.ts +2 -47
  60. package/src/templates/index.ts +0 -67
  61. package/src/transport/index.ts +6 -0
  62. package/src/workers/bootstrap.ts +124 -0
  63. package/src/workers/index.ts +99 -24
  64. package/src/workflows/sdk.ts +2 -0
  65. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  66. package/dist/index.d.mts.map +0 -1
  67. package/dist/index.mjs.map +0 -1
  68. package/dist/main-39DtKrX4.mjs.map +0 -1
  69. package/dist/main-B3iQrl3t.mjs.map +0 -1
  70. package/dist/main-BM8DH3v3.mjs +0 -632
  71. package/dist/main-BM8DH3v3.mjs.map +0 -1
  72. package/dist/main-BgmT1MPe.mjs.map +0 -1
  73. package/dist/main-BiFiOX9D.mjs.map +0 -1
  74. package/dist/main-C9s-2ArH.mjs.map +0 -1
  75. package/dist/main-Cqo3rSb5.mjs +0 -224
  76. package/dist/main-Cqo3rSb5.mjs.map +0 -1
  77. package/dist/main-Dz2sUh3c.mjs.map +0 -1
  78. package/dist/src-QiTu5e_a.mjs.map +0 -1
@@ -1,4 +1,5 @@
1
- import { A as RedisClient, H as LUA_RENEW_LOCK, Qt as CONSUMER_GROUPS, V as LUA_RELEASE_LOCK, W as LUA_SCHEDULER_POLL, X as StreamConsumer, Y as PendingMessageScanner, Z as StreamProducer, Zt as buildStreamEvent, _t as workflowSteps, a as SuspendExecutionError, at as createDatabase, c as BaseWorker, gn as loadEnv, gt as workflowInstances, ht as workflowDefinitions, i as workflowRegistry, o as buildStepNotifyPayload, rn as STREAMS, tt as createLogger, u as startHealthReporter, vn as readBaseConfig, vt as workflowWaiters } from "./src-QiTu5e_a.mjs";
1
+ import { K as StreamProducer, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, Yt as STREAMS, a as SuspendExecutionError, c as BaseWorker, cn as loadEnv, ct as workflowInstances, d as startHealthReporter, i as workflowRegistry, lt as workflowSteps, o as buildStepNotifyPayload, st as workflowDefinitions, un as readBaseConfig, ut as workflowWaiters } from "./src-CI7FGf3h.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime } from "./bootstrap-GRXzgPvW.mjs";
2
3
  import { randomUUID } from "node:crypto";
3
4
  import { and, eq } from "drizzle-orm";
4
5
  //#region src/services/workflow/main.ts
@@ -8,6 +9,40 @@ const WORKFLOW_LOCK_TTL_SECONDS = 60;
8
9
  const WORKFLOW_LOCK_RENEW_MS = WORKFLOW_LOCK_TTL_SECONDS / 3 * 1e3;
9
10
  /** How long a claimed workflow timer stays invisible to other pollers. */
10
11
  const WORKFLOW_TIMER_VISIBILITY_MS = 6e4;
12
+ /** Wake-ups for sleeping instances: waits, event timeouts, deferred resumes. */
13
+ const WORKFLOW_TIMERS_KEY = "notif:workflow:timers";
14
+ /** How soon a resume that found its instance busy is tried again. */
15
+ const WORKFLOW_RESUME_RETRY_MS = 2e3;
16
+ const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
17
+ /** Seconds per unit. Everything is read as seconds first, then scheduled in ms. */
18
+ const DURATION_UNIT_SECONDS = {
19
+ ms: .001,
20
+ s: 1,
21
+ m: 60,
22
+ h: 3600,
23
+ d: 86400
24
+ };
25
+ /**
26
+ * `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
27
+ * read exactly; anything that is not a number and a unit throws. The old
28
+ * parser matched on the last character, so "500ms" meant 500 seconds, and ran
29
+ * parseInt, so "1.5h" meant one hour.
30
+ */
31
+ function parseDurationSeconds(value, what) {
32
+ const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
33
+ if (!match) throw new Error(`Invalid ${what}: ${value}`);
34
+ return Number(match[1]) * DURATION_UNIT_SECONDS[match[2]];
35
+ }
36
+ function durationToMs(value, what) {
37
+ return Math.round(parseDurationSeconds(value, what) * 1e3);
38
+ }
39
+ /** Thrown at a step boundary once the instance has been canceled. */
40
+ var WorkflowCanceledError = class extends Error {
41
+ constructor() {
42
+ super("Workflow instance was canceled");
43
+ this.name = "WorkflowCanceledError";
44
+ }
45
+ };
11
46
  loadEnv();
12
47
  const config = readBaseConfig();
13
48
  let logger;
@@ -38,12 +73,12 @@ var WorkflowWorker = class extends BaseWorker {
38
73
  this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 100);
39
74
  }
40
75
  async stop() {
76
+ await super.stop();
41
77
  if (this.flushTimer) {
42
78
  clearInterval(this.flushTimer);
43
79
  this.flushTimer = null;
44
80
  }
45
81
  await this.flushWorkerBuffers();
46
- await super.stop();
47
82
  }
48
83
  async flushWorkerBuffers() {
49
84
  if (this.eventBuffer.length === 0) return;
@@ -71,6 +106,24 @@ var WorkflowWorker = class extends BaseWorker {
71
106
  for (const e of events) e.reject(err);
72
107
  }
73
108
  }
109
+ /**
110
+ * Re-delivers a resume through the timer ZSET a few seconds from now. The
111
+ * poll loop publishes it once the instance is pending again, and drops it if
112
+ * the instance has finished. The member carries no timestamp, so repeated
113
+ * deferrals of one instance collapse into a single entry. It is marked
114
+ * `deferred` so it never equals a `wait` timer's member: sharing one let the
115
+ * ZADD here re-score a sleep's wake-up to seconds from now, and the early
116
+ * resume then re-suspended with no timer left to wake it.
117
+ */
118
+ async deferResume(payload) {
119
+ await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, Date.now() + WORKFLOW_RESUME_RETRY_MS, JSON.stringify({
120
+ instanceId: payload.instanceId,
121
+ name: payload.name,
122
+ projectId: payload.projectId,
123
+ input: payload.input ?? {},
124
+ deferred: true
125
+ }));
126
+ }
74
127
  async process(message) {
75
128
  const { event } = message;
76
129
  const publishPromises = [];
@@ -107,11 +160,12 @@ var WorkflowWorker = class extends BaseWorker {
107
160
  const lockKey = `lock:workflow:${instanceId}`;
108
161
  const lockToken = randomUUID();
109
162
  if (!await this.redisCli.set(lockKey, lockToken, "EX", WORKFLOW_LOCK_TTL_SECONDS, "NX")) {
110
- this.logger.info({ instanceId }, "Workflow is locked by another process, skipping");
163
+ this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
164
+ await this.deferResume(payload);
111
165
  return;
112
166
  }
113
167
  const renewTimer = setInterval(() => {
114
- this.redisCli.eval(LUA_RENEW_LOCK, 1, lockKey, lockToken, String(WORKFLOW_LOCK_TTL_SECONDS)).catch((err) => {
168
+ this.redisCli.renewLock(lockKey, lockToken, WORKFLOW_LOCK_TTL_SECONDS).catch((err) => {
115
169
  this.logger.warn({
116
170
  err,
117
171
  instanceId
@@ -127,11 +181,22 @@ var WorkflowWorker = class extends BaseWorker {
127
181
  status: "pending",
128
182
  input: payload.input || {}
129
183
  }).returning())[0];
184
+ if (instance.status === "running") {
185
+ await this.deferResume(payload);
186
+ return;
187
+ }
130
188
  if (instance.status !== "pending") {
131
189
  this.logger.info({ instanceId }, "Workflow is not pending, skipping");
132
190
  return;
133
191
  }
134
- await this.dbConn.update(workflowInstances).set({ status: "running" }).where(eq(workflowInstances.id, instanceId));
192
+ if ((await this.dbConn.update(workflowInstances).set({ status: "running" }).where(and(eq(workflowInstances.id, instanceId), eq(workflowInstances.status, "pending"))).returning({ id: workflowInstances.id })).length === 0) {
193
+ this.logger.info({ instanceId }, "Workflow is no longer pending, skipping");
194
+ return;
195
+ }
196
+ const stopIfCanceled = async () => {
197
+ const [row] = await this.dbConn.select({ status: workflowInstances.status }).from(workflowInstances).where(eq(workflowInstances.id, instanceId)).limit(1);
198
+ if (row?.status === "canceled") throw new WorkflowCanceledError();
199
+ };
135
200
  const existingSteps = await this.dbConn.select().from(workflowSteps).where(eq(workflowSteps.instanceId, instanceId));
136
201
  const stepOutputMap = /* @__PURE__ */ new Map();
137
202
  for (const s of existingSteps) stepOutputMap.set(s.stepIndex, s.output);
@@ -141,7 +206,8 @@ var WorkflowWorker = class extends BaseWorker {
141
206
  notify: async (args) => {
142
207
  const stepId = String(currentStepIndex++);
143
208
  if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
144
- const requested = buildStepNotifyPayload(args, instance.input, payload.projectId, `wf-${instanceId}-${stepId}`);
209
+ await stopIfCanceled();
210
+ const requested = buildStepNotifyPayload(args, instance.input, payload.projectId, `wf-${instanceId}-${stepId}`, UUID_RE.test(instanceId) ? instanceId : void 0);
145
211
  const result = await new Promise((resolve, reject) => {
146
212
  this.eventBuffer.push({
147
213
  producer: this.notificationProducer,
@@ -166,15 +232,15 @@ var WorkflowWorker = class extends BaseWorker {
166
232
  },
167
233
  wait: async (duration) => {
168
234
  const stepId = String(currentStepIndex++);
169
- if (stepOutputMap.has(stepId)) return;
170
- let ms = 0;
171
- if (duration.endsWith("d")) ms = parseInt(duration) * 24 * 60 * 60 * 1e3;
172
- else if (duration.endsWith("h")) ms = parseInt(duration) * 60 * 60 * 1e3;
173
- else if (duration.endsWith("m")) ms = parseInt(duration) * 60 * 1e3;
174
- else if (duration.endsWith("s")) ms = parseInt(duration) * 1e3;
175
- else throw new Error(`Invalid wait duration: ${duration}`);
235
+ if (stepOutputMap.has(stepId)) {
236
+ const until = Number(stepOutputMap.get(stepId)?.scheduledAt);
237
+ if (Number.isFinite(until) && until > Date.now()) throw new SuspendExecutionError("wait", { duration });
238
+ return;
239
+ }
240
+ await stopIfCanceled();
241
+ const ms = durationToMs(duration, "wait duration");
176
242
  const resumeAt = Date.now() + ms;
177
- await this.redisCli.zadd("notif:workflow:timers", resumeAt, JSON.stringify({
243
+ await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
178
244
  instanceId: instance.id,
179
245
  name: instance.name,
180
246
  projectId: payload.projectId,
@@ -193,20 +259,17 @@ var WorkflowWorker = class extends BaseWorker {
193
259
  const stepId = String(currentStepIndex++);
194
260
  if (stepOutputMap.has(stepId)) {
195
261
  const out = stepOutputMap.get(stepId);
196
- if (out && typeof out === "object" && out.timedOut === true) return null;
262
+ if (out === null || out === void 0) throw new SuspendExecutionError("waitForEvent", { eventName });
263
+ if (typeof out === "object" && out.timedOut === true) return null;
197
264
  return out;
198
265
  }
266
+ await stopIfCanceled();
199
267
  options = options || {};
200
268
  options.timeout = options.timeout || "24h";
201
269
  options.match = options.match || {};
202
- let ms = 0;
203
- if (options.timeout.endsWith("d")) ms = parseInt(options.timeout) * 24 * 60 * 60 * 1e3;
204
- else if (options.timeout.endsWith("h")) ms = parseInt(options.timeout) * 60 * 60 * 1e3;
205
- else if (options.timeout.endsWith("m")) ms = parseInt(options.timeout) * 60 * 1e3;
206
- else if (options.timeout.endsWith("s")) ms = parseInt(options.timeout) * 1e3;
207
- else throw new Error(`Invalid waitForEvent timeout: ${options.timeout}`);
270
+ const ms = durationToMs(options.timeout, "waitForEvent timeout");
208
271
  const resumeAt = Date.now() + ms;
209
- await this.redisCli.zadd("notif:workflow:timers", resumeAt, JSON.stringify({
272
+ await this.redisCli.zadd(WORKFLOW_TIMERS_KEY, resumeAt, JSON.stringify({
210
273
  instanceId: instance.id,
211
274
  name: instance.name,
212
275
  projectId: payload.projectId,
@@ -215,13 +278,6 @@ var WorkflowWorker = class extends BaseWorker {
215
278
  eventName,
216
279
  stepId
217
280
  }));
218
- await this.dbConn.insert(workflowWaiters).values({
219
- instanceId: instance.id,
220
- projectId: payload.projectId,
221
- eventName,
222
- matchCriteria: options.match,
223
- expiresAt: new Date(resumeAt)
224
- });
225
281
  await this.dbConn.insert(workflowSteps).values({
226
282
  instanceId: instance.id,
227
283
  projectId: payload.projectId,
@@ -229,11 +285,19 @@ var WorkflowWorker = class extends BaseWorker {
229
285
  action: "waitForEvent",
230
286
  output: null
231
287
  });
288
+ await this.dbConn.insert(workflowWaiters).values({
289
+ instanceId: instance.id,
290
+ projectId: payload.projectId,
291
+ eventName,
292
+ matchCriteria: options.match,
293
+ expiresAt: new Date(resumeAt)
294
+ });
232
295
  throw new SuspendExecutionError("waitForEvent", { eventName });
233
296
  },
234
297
  run: async (stepName, fn) => {
235
298
  const stepId = String(currentStepIndex++);
236
299
  if (stepOutputMap.has(stepId)) return stepOutputMap.get(stepId);
300
+ await stopIfCanceled();
237
301
  const result = await fn();
238
302
  await this.dbConn.insert(workflowSteps).values({
239
303
  instanceId: instance.id,
@@ -247,13 +311,15 @@ var WorkflowWorker = class extends BaseWorker {
247
311
  },
248
312
  event: instance.input || { user: { id: "unknown" } }
249
313
  };
314
+ const finish = (status) => this.dbConn.update(workflowInstances).set({ status }).where(and(eq(workflowInstances.id, instance.id), eq(workflowInstances.status, "running")));
250
315
  try {
251
316
  await handler(ctx);
252
- await this.dbConn.update(workflowInstances).set({ status: "completed" }).where(eq(workflowInstances.id, instance.id));
317
+ await finish("completed");
253
318
  this.logger.info({ instanceId: instance.id }, "Workflow completed successfully");
254
319
  } catch (err) {
255
- if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
256
- await this.dbConn.update(workflowInstances).set({ status: "pending" }).where(eq(workflowInstances.id, instance.id));
320
+ if (err instanceof WorkflowCanceledError) this.logger.info({ instanceId: instance.id }, "Workflow canceled mid-run, stopping");
321
+ else if (err instanceof SuspendExecutionError || err.name === "SuspendExecutionError") {
322
+ await finish("pending");
257
323
  this.logger.info({
258
324
  instanceId: instance.id,
259
325
  reason: err.reason
@@ -263,33 +329,18 @@ var WorkflowWorker = class extends BaseWorker {
263
329
  err,
264
330
  instanceId: instance.id
265
331
  }, "Workflow failed");
266
- await this.dbConn.update(workflowInstances).set({ status: "failed" }).where(eq(workflowInstances.id, instance.id));
332
+ await finish("failed");
267
333
  }
268
334
  }
269
335
  } finally {
270
336
  clearInterval(renewTimer);
271
- await this.redisCli.eval(LUA_RELEASE_LOCK, 1, lockKey, lockToken);
337
+ await this.redisCli.releaseLock(lockKey, lockToken);
272
338
  }
273
339
  await Promise.all(publishPromises);
274
340
  }
275
341
  };
276
342
  async function startWorkflowWorker() {
277
- logger = createLogger({
278
- name: "workflow-worker",
279
- level: config.LOG_LEVEL
280
- });
281
- redis = new RedisClient({
282
- url: config.REDIS_URL,
283
- name: "workflow",
284
- logger
285
- });
286
- const dbData = createDatabase({
287
- url: config.DATABASE_URL,
288
- applicationName: "workflow",
289
- logger
290
- });
291
- sql$1 = dbData.sql;
292
- db = dbData.db;
343
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "workflow", "workflow-worker"));
293
344
  notificationProducer = new StreamProducer({
294
345
  redis: redis.native,
295
346
  stream: STREAMS.INBOUND_NORMAL,
@@ -300,22 +351,15 @@ async function startWorkflowWorker() {
300
351
  stream: STREAMS.WORKFLOW_INBOUND,
301
352
  logger
302
353
  });
303
- consumer = new StreamConsumer({
304
- redis: redis.native,
305
- stream: STREAMS.WORKFLOW_INBOUND,
306
- group: CONSUMER_GROUPS.WORKFLOW,
307
- consumer: `workflow-${process.pid}`,
308
- dlqStream: STREAMS.DEAD_LETTER,
309
- batchSize: config.WORKER_CONCURRENCY,
354
+ ({consumer, pendingScanner} = createStreamConsumer({
355
+ redis,
310
356
  logger
311
- });
312
- pendingScanner = new PendingMessageScanner({
313
- redis: redis.native,
357
+ }, {
314
358
  stream: STREAMS.WORKFLOW_INBOUND,
315
359
  group: CONSUMER_GROUPS.WORKFLOW,
316
- consumer: `workflow-${process.pid}`,
317
- logger
318
- });
360
+ consumer: uniqueConsumerId("workflow"),
361
+ batchSize: config.WORKER_CONCURRENCY
362
+ }));
319
363
  worker = new WorkflowWorker({
320
364
  consumer,
321
365
  pendingScanner,
@@ -330,25 +374,33 @@ async function startWorkflowWorker() {
330
374
  (async () => {
331
375
  try {
332
376
  const now = Date.now();
333
- const tasks = await redis.native.eval(LUA_SCHEDULER_POLL, 1, "notif:workflow:timers", now, 100, WORKFLOW_TIMER_VISIBILITY_MS);
377
+ const tasks = await redis.native.schedulerPoll(WORKFLOW_TIMERS_KEY, now, 100, WORKFLOW_TIMER_VISIBILITY_MS);
334
378
  for (const taskStr of tasks) {
335
379
  const task = JSON.parse(taskStr);
336
380
  const inst = (await db.select().from(workflowInstances).where(eq(workflowInstances.id, task.instanceId)).limit(1))[0];
337
- if (!inst || inst.status === "completed" || inst.status === "failed") {
338
- await redis.native.zrem("notif:workflow:timers", taskStr);
381
+ if (!inst || inst.status === "completed" || inst.status === "failed" || inst.status === "canceled") {
382
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
339
383
  continue;
340
384
  }
341
385
  if (inst.status === "running") continue;
342
386
  if (task.isEventTimeout) {
387
+ if (task.stepId !== void 0) {
388
+ const [own] = await db.select({ output: workflowSteps.output }).from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.stepIndex, String(task.stepId)))).limit(1);
389
+ if (!own || own.output !== null) {
390
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
391
+ continue;
392
+ }
393
+ }
343
394
  if ((await db.delete(workflowWaiters).where(and(eq(workflowWaiters.instanceId, task.instanceId), eq(workflowWaiters.eventName, task.eventName))).returning()).length === 0) {
344
- await redis.native.zrem("notif:workflow:timers", taskStr);
395
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
345
396
  continue;
346
397
  }
347
- const pendingStep = (await db.select().from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.action, "waitForEvent")))).find((s) => s.output === null);
398
+ const steps = await db.select().from(workflowSteps).where(and(eq(workflowSteps.instanceId, task.instanceId), eq(workflowSteps.action, "waitForEvent")));
399
+ const pendingStep = task.stepId !== void 0 ? steps.find((s) => s.stepIndex === String(task.stepId) && s.output === null) : steps.find((s) => s.output === null);
348
400
  if (pendingStep) await db.update(workflowSteps).set({ output: { timedOut: true } }).where(eq(workflowSteps.id, pendingStep.id));
349
401
  }
350
402
  await workflowProducer.publish(buildStreamEvent("workflow.resumed", task, "scheduler", void 0));
351
- await redis.native.zrem("notif:workflow:timers", taskStr);
403
+ await redis.native.zrem(WORKFLOW_TIMERS_KEY, taskStr);
352
404
  logger.info({ instanceId: task.instanceId }, "Workflow resumed from timer");
353
405
  }
354
406
  } catch (err) {
@@ -377,16 +429,20 @@ async function startWorkflowWorker() {
377
429
  await worker.start();
378
430
  }
379
431
  async function stopWorkflowWorker() {
380
- logger?.info("shutdown initiated");
381
- if (healthInterval) clearInterval(healthInterval);
382
- if (pollInterval) clearInterval(pollInterval);
383
- if (reaperInterval) clearInterval(reaperInterval);
384
- if (worker) await worker.stop();
385
- if (sql$1) await sql$1.end();
386
- if (redis) await redis.disconnect();
387
- logger?.info("workflow worker stopped");
432
+ await shutdownWorker("workflow worker", {
433
+ logger,
434
+ timers: [
435
+ healthInterval,
436
+ pollInterval,
437
+ reaperInterval
438
+ ],
439
+ worker,
440
+ sql: sql$1,
441
+ redis
442
+ });
443
+ healthInterval = null;
444
+ pollInterval = null;
445
+ reaperInterval = null;
388
446
  }
389
447
  //#endregion
390
448
  export { startWorkflowWorker, stopWorkflowWorker };
391
-
392
- //# sourceMappingURL=main-BgmT1MPe.mjs.map
@@ -0,0 +1,312 @@
1
+ import { C as ContactRepository, D as TemplateRepository, O as UserRepository, Sn as normaliseTarget, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, Yt as STREAMS, c as BaseWorker, cn as loadEnv, d as startHealthReporter, en as registry, it as suppressions, rt as scheduledPayloads, un as readBaseConfig, wn as globalEmitter, xn as getPriorityBucket } from "./src-CI7FGf3h.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-GRXzgPvW.mjs";
3
+ import { randomUUID } from "node:crypto";
4
+ import { and, asc, eq, gt, inArray, or, sql } from "drizzle-orm";
5
+ //#region src/services/scheduler/send-gate.ts
6
+ /**
7
+ * Re-checks parked sends at the moment they are released.
8
+ *
9
+ * The engine clears a message against suppressions, opt-outs and contacts when
10
+ * it first sees it. A send parked for later — a `sendAt`, a quiet-hours
11
+ * deferral, a retry — can sit for hours or days, and whatever the person did in
12
+ * between (unsubscribed, complained, hard-bounced, removed the address) must
13
+ * still stop it. Without this the scheduler sent to them anyway.
14
+ *
15
+ * Reads Postgres directly rather than through the repositories' caches: this
16
+ * runs once per poll, not once per message, and a stale "still subscribed" is
17
+ * the very thing it exists to prevent.
18
+ *
19
+ * Returns the blocked tasks with the reason. Throws if a lookup fails, so the
20
+ * caller holds the batch rather than sending it unchecked.
21
+ */
22
+ async function findBlockedAtRelease(db, tasks) {
23
+ const blocked = /* @__PURE__ */ new Map();
24
+ const byProject = /* @__PURE__ */ new Map();
25
+ for (const t of tasks) {
26
+ if (!t?.projectId || !t.recipientId) continue;
27
+ if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
28
+ byProject.get(t.projectId).push(t);
29
+ }
30
+ for (const [projectId, group] of byProject) {
31
+ const recipientIds = [...new Set(group.map((t) => t.recipientId))];
32
+ const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))];
33
+ const targets = [...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean))];
34
+ const templateRepo = new TemplateRepository(db);
35
+ const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
36
+ targets.length > 0 ? db.select({
37
+ channel: suppressions.channel,
38
+ target: suppressions.target
39
+ }).from(suppressions).where(and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets))) : [],
40
+ new UserRepository(db).findRecordsByIds(projectId, recipientIds),
41
+ new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
42
+ Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id)))
43
+ ]);
44
+ const suppressed = new Set(suppressedRows.map((r) => `${r.channel}:${normaliseTarget(r.target)}`));
45
+ const userById = new Map(users.map((u) => [u.userId, u]));
46
+ const topicsByTemplate = /* @__PURE__ */ new Map();
47
+ templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
48
+ for (const t of group) {
49
+ const reason = decide(t, t.templateId ? topicsByTemplate.get(t.templateId) ?? [] : [], suppressed, userById.get(t.recipientId), contactsByUser);
50
+ if (reason) blocked.set(t.taskId, reason);
51
+ }
52
+ }
53
+ return blocked;
54
+ }
55
+ function decide(t, topics, suppressed, user, contactsByUser) {
56
+ if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) return "suppressed";
57
+ if (!user) return "recipient_not_found";
58
+ if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
59
+ if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
60
+ if (t.destination) {
61
+ const contact = (contactsByUser.get(t.recipientId) ?? []).find((c) => c.channel === t.channel && c.target === t.destination);
62
+ if (!contact) return "contact_inactive";
63
+ if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) return "user_opted_out";
64
+ }
65
+ }
66
+ //#endregion
67
+ //#region src/services/scheduler/main.ts
68
+ loadEnv();
69
+ const config = readBaseConfig();
70
+ let logger;
71
+ let redis;
72
+ let sql$1;
73
+ let db;
74
+ let consumer;
75
+ let pendingScanner;
76
+ let worker;
77
+ let healthInterval = null;
78
+ let pollTimeout = null;
79
+ let isPolling = false;
80
+ var SchedulerWorker = class extends BaseWorker {
81
+ registry;
82
+ redisCli;
83
+ constructor(options) {
84
+ super(options);
85
+ this.registry = options.registry;
86
+ this.redisCli = options.redis;
87
+ }
88
+ async process(message) {
89
+ const { event } = message;
90
+ const payloadResult = this.registry.safeParsePayload("notification.scheduled", event.payload);
91
+ if (!payloadResult.success) {
92
+ this.logger.warn({
93
+ messageId: message.id,
94
+ issues: payloadResult.error.issues
95
+ }, "invalid notification.scheduled payload — skipping");
96
+ return;
97
+ }
98
+ const scheduled = payloadResult.data;
99
+ const scheduledAt = new Date(scheduled.scheduledAt).getTime();
100
+ const taskData = JSON.stringify({
101
+ taskId: scheduled.taskId,
102
+ enrichedEventId: scheduled.enrichedEventId,
103
+ traceId: event.metadata.traceId
104
+ });
105
+ const shard = parseInt(scheduled.taskId.slice(-1), 16) || 0;
106
+ const zsetKey = `notif:scheduled:zset:${shard}`;
107
+ await this.redisCli.zadd(zsetKey, scheduledAt, taskData);
108
+ this.logger.debug({
109
+ taskId: scheduled.taskId,
110
+ scheduledAt: scheduled.scheduledAt,
111
+ shard
112
+ }, "scheduled task queued in ZSET");
113
+ }
114
+ };
115
+ async function executeSchedulerPoll(redis, outboundProducers, logger, db) {
116
+ const lockKey = "notif:lock:scheduler:poll";
117
+ const lockOwner = randomUUID();
118
+ let acquired = false;
119
+ try {
120
+ if (await redis.set(lockKey, lockOwner, "EX", 30, "NX") !== "OK") return false;
121
+ acquired = true;
122
+ const now = Date.now();
123
+ const visibilityTimeout = 6e4;
124
+ const perShardLimit = 100;
125
+ const pipeline = redis.pipeline();
126
+ for (let i = 0; i < 16; i++) {
127
+ const shardKey = `notif:scheduled:zset:${i}`;
128
+ pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
129
+ }
130
+ const perShard = (await pipeline.exec())?.map((res) => res[0] ? [] : res[1] ?? []) ?? [];
131
+ const tasks = perShard.flat();
132
+ const anyShardFull = perShard.some((shard) => shard.length >= perShardLimit);
133
+ if (tasks.length === 0) return false;
134
+ const parsedTasks = tasks.map((t) => {
135
+ try {
136
+ return {
137
+ taskStr: t,
138
+ ...JSON.parse(t)
139
+ };
140
+ } catch (err) {
141
+ logger.error({
142
+ taskStr: t,
143
+ err
144
+ }, "failed to parse scheduled task JSON");
145
+ return null;
146
+ }
147
+ }).filter(Boolean);
148
+ if (parsedTasks.length === 0) return false;
149
+ const taskIds = parsedTasks.map((t) => t.taskId);
150
+ const dbPayloads = await db.select().from(scheduledPayloads).where(inArray(scheduledPayloads.taskId, taskIds));
151
+ const payloadMap = new Map(dbPayloads.map((row) => [row.taskId, row.payload]));
152
+ const blocked = await findBlockedAtRelease(db, [...payloadMap.values()]);
153
+ const batchedEvents = {
154
+ critical: [],
155
+ normal: [],
156
+ low: []
157
+ };
158
+ const cleanupPipeline = redis.pipeline();
159
+ const dbCleanupIds = [];
160
+ const released = /* @__PURE__ */ new Set();
161
+ for (let i = 0; i < parsedTasks.length; i++) {
162
+ const { taskStr, taskId, traceId } = parsedTasks[i];
163
+ if (released.has(taskId)) {
164
+ cleanupPipeline.zrem(`notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`, taskStr);
165
+ continue;
166
+ }
167
+ const payload = payloadMap.get(taskId);
168
+ if (!payload) {
169
+ logger.warn({ taskId }, "scheduled payload not found in Postgres — skipping release");
170
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
171
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
172
+ continue;
173
+ }
174
+ const dispatchPayload = payload;
175
+ const blockReason = blocked.get(taskId);
176
+ if (blockReason) {
177
+ logger.info({
178
+ taskId,
179
+ reason: blockReason
180
+ }, "parked send no longer allowed — dropping");
181
+ globalEmitter.emit("notification:skipped", {
182
+ projectId: dispatchPayload.projectId,
183
+ eventId: dispatchPayload.enrichedEventId,
184
+ recipientId: dispatchPayload.recipientId,
185
+ reason: blockReason
186
+ });
187
+ released.add(taskId);
188
+ dbCleanupIds.push(taskId);
189
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
190
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
191
+ continue;
192
+ }
193
+ batchedEvents[getPriorityBucket(dispatchPayload.priority)].push(buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId));
194
+ released.add(taskId);
195
+ dbCleanupIds.push(taskId);
196
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
197
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
198
+ }
199
+ for (const p of Object.keys(batchedEvents)) if (batchedEvents[p].length > 0) await (outboundProducers[p] ?? outboundProducers["normal"]).publishBatch(batchedEvents[p]);
200
+ if (dbCleanupIds.length > 0) try {
201
+ const sendAt = sql`${scheduledPayloads.payload}->>'scheduledAt'`;
202
+ await db.delete(scheduledPayloads).where(or(...dbCleanupIds.map((taskId) => {
203
+ const released = payloadMap.get(taskId)?.scheduledAt;
204
+ return and(eq(scheduledPayloads.taskId, taskId), typeof released === "string" ? sql`${sendAt} = ${released}` : sql`${sendAt} IS NULL`);
205
+ })));
206
+ } catch (err) {
207
+ logger.error({ err }, "failed to delete scheduled payloads from Postgres, continuing with Redis cleanup");
208
+ }
209
+ await cleanupPipeline.exec();
210
+ logger.info({ count: parsedTasks.length }, "scheduled tasks released to outbound");
211
+ return anyShardFull;
212
+ } catch (err) {
213
+ logger.error({ err }, "error in scheduler polling loop");
214
+ return false;
215
+ } finally {
216
+ if (acquired) await redis.releaseLock(lockKey, lockOwner).catch(() => {});
217
+ }
218
+ }
219
+ /**
220
+ * Present while the scheduler ZSETs are known to be intact. Redis losing its
221
+ * data (a flush, a restart without persistence, failover to an empty replica)
222
+ * takes this key with it, which is how the poll loop notices.
223
+ */
224
+ const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
225
+ /**
226
+ * Re-queues every scheduled send still waiting in Postgres. Each row keeps its
227
+ * payload, and with it the send time, so the ZSETs can be rebuilt exactly;
228
+ * a legacy row without a send time is released on the next poll.
229
+ *
230
+ * Only run when the sentinel is missing: a rebuilt member has no trace id and
231
+ * so never matches a surviving original, and the poll only de-duplicates
232
+ * copies that happen to be claimed together.
233
+ */
234
+ async function rebuildScheduledQueue(redis, db, logger, pageSize = 1e3) {
235
+ let after;
236
+ let total = 0;
237
+ for (;;) {
238
+ const rows = await db.select({
239
+ taskId: scheduledPayloads.taskId,
240
+ payload: scheduledPayloads.payload
241
+ }).from(scheduledPayloads).where(after === void 0 ? void 0 : gt(scheduledPayloads.taskId, after)).orderBy(asc(scheduledPayloads.taskId)).limit(pageSize);
242
+ if (rows.length === 0) break;
243
+ const pipeline = redis.pipeline();
244
+ for (const { taskId, payload } of rows) {
245
+ const sendAt = Date.parse(payload?.scheduledAt ?? "");
246
+ const member = JSON.stringify({
247
+ taskId,
248
+ enrichedEventId: payload?.enrichedEventId
249
+ });
250
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
251
+ pipeline.zadd(`notif:scheduled:zset:${shard}`, "NX", Number.isNaN(sendAt) ? Date.now() : sendAt, member);
252
+ }
253
+ await pipeline.exec();
254
+ total += rows.length;
255
+ after = rows[rows.length - 1].taskId;
256
+ if (rows.length < pageSize) break;
257
+ }
258
+ await redis.set(SCHEDULER_SENTINEL_KEY, "1");
259
+ if (total > 0) logger.warn({ count: total }, "rebuilt scheduled queue from Postgres");
260
+ return total;
261
+ }
262
+ async function startSchedulerWorker() {
263
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "scheduler"));
264
+ ({consumer, pendingScanner} = createStreamConsumer({
265
+ redis,
266
+ logger
267
+ }, {
268
+ stream: STREAMS.SCHEDULED,
269
+ group: CONSUMER_GROUPS.SCHEDULER,
270
+ consumer: uniqueConsumerId("scheduler"),
271
+ batchSize: config.WORKER_CONCURRENCY
272
+ }));
273
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
274
+ worker = new SchedulerWorker({
275
+ consumer,
276
+ pendingScanner,
277
+ logger,
278
+ concurrency: config.WORKER_CONCURRENCY,
279
+ registry,
280
+ redis: redis.native
281
+ });
282
+ isPolling = true;
283
+ const pollLoop = async () => {
284
+ if (!isPolling) return;
285
+ try {
286
+ if (!await redis.native.exists("notif:scheduler:intact")) await rebuildScheduledQueue(redis.native, db, logger);
287
+ const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
288
+ if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5e3);
289
+ } catch (err) {
290
+ logger.error({ err }, "scheduler poll loop error");
291
+ if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), 5e3);
292
+ }
293
+ };
294
+ pollLoop();
295
+ healthInterval = startHealthReporter("scheduler", worker, redis, logger);
296
+ logger.info({ env: config.NODE_ENV }, "scheduler starting");
297
+ await worker.start();
298
+ }
299
+ async function stopSchedulerWorker() {
300
+ isPolling = false;
301
+ await shutdownWorker("scheduler", {
302
+ logger,
303
+ timers: [healthInterval, pollTimeout],
304
+ worker,
305
+ sql: sql$1,
306
+ redis
307
+ });
308
+ healthInterval = null;
309
+ pollTimeout = null;
310
+ }
311
+ //#endregion
312
+ export { startSchedulerWorker, stopSchedulerWorker };