notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -1,24 +1,29 @@
1
1
  import { loadEnv, readBaseConfig } from "@/index.js";
2
- import { createLogger } from "@/index.js";
3
- import { RedisClient } from "@/index.js";
2
+ import type { Logger } from "@/index.js";
3
+ import type { RedisClient } from "@/index.js";
4
4
  import {
5
- StreamConsumer,
6
- PendingMessageScanner,
5
+ type StreamConsumer,
6
+ type PendingMessageScanner,
7
7
  StreamProducer,
8
8
  type StreamMessage,
9
9
  } from "@/index.js";
10
10
  import { BaseWorker } from "@/index.js";
11
11
  import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/index.js";
12
- import { type StreamName } from "@/contracts/streams.js";
13
- import { createDatabase } from "@/db/index.js";
14
- import { workflowWaiters, workflowSteps, workflowInstances, messageLogs } from "@/db/schema.js";
12
+ import { insertMessageLogs } from "@/db/bulk.js";
13
+ import { workflowWaiters, workflowSteps, workflowInstances } from "@/db/schema.js";
15
14
  import { eq, and, or, isNull, gt } from "drizzle-orm";
16
15
  import { type WorkerOptions } from "@/shared/index.js";
17
16
  import { startHealthReporter } from "@/workers/index.js";
17
+ import {
18
+ createStreamConsumer,
19
+ createWorkerRuntime,
20
+ shutdownWorker,
21
+ uniqueConsumerId,
22
+ } from "@/workers/bootstrap.js";
18
23
 
19
24
  loadEnv();
20
25
  const config = readBaseConfig();
21
- let logger: ReturnType<typeof createLogger>;
26
+ let logger: Logger;
22
27
  let redis: RedisClient;
23
28
  let sql: any;
24
29
  let db: any;
@@ -46,12 +51,23 @@ export class EventWorker extends BaseWorker {
46
51
  reject: (err: any) => void;
47
52
  }[] = [];
48
53
  private flushInterval: NodeJS.Timeout | null = null;
49
- private isFlushing = false;
54
+ private flushesInFlight = 0;
55
+ private static readonly MAX_CONCURRENT_FLUSHES = 4;
56
+ /**
57
+ * Buffered logs that trigger a flush without waiting for the interval.
58
+ *
59
+ * Every message holds its worker slot until its log row is written, so a
60
+ * purely timed flush caps the worker at `concurrency / LOG_FLUSH_INTERVAL_MS`
61
+ * rows per second — far below what one delivery worker produces. Flushing
62
+ * once half the slots are waiting keeps the other half reading.
63
+ */
64
+ private readonly flushThreshold: number;
50
65
 
51
66
  constructor(options: EventWorkerOptions) {
52
67
  super(options);
53
68
  this.dbConn = options.db;
54
69
  this.workflowProducer = options.workflowProducer;
70
+ this.flushThreshold = Math.max(1, Math.min(500, Math.floor(this.concurrency / 2)));
55
71
  this.startFlushInterval();
56
72
  }
57
73
 
@@ -62,9 +78,12 @@ export class EventWorker extends BaseWorker {
62
78
  }
63
79
 
64
80
  private async flushLogs() {
65
- if (this.isFlushing || (this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0))
81
+ if (
82
+ this.flushesInFlight >= EventWorker.MAX_CONCURRENT_FLUSHES ||
83
+ (this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0)
84
+ )
66
85
  return;
67
- this.isFlushing = true;
86
+ this.flushesInFlight++;
68
87
 
69
88
  const startTime = Date.now();
70
89
  const batch = this.messageLogBuffer;
@@ -75,13 +94,13 @@ export class EventWorker extends BaseWorker {
75
94
 
76
95
  try {
77
96
  if (batch.length > 0) {
78
- await this.dbConn
79
- .insert(messageLogs)
80
- .values(batch.map((b) => b.log))
81
- .onConflictDoNothing(); // Idempotent insert
97
+ await insertMessageLogs(
98
+ this.dbConn,
99
+ batch.map((b) => b.log),
100
+ ); // Idempotent insert
82
101
  for (const b of batch) b.resolve();
83
102
 
84
- this.logger.info(
103
+ this.logger.debug(
85
104
  {
86
105
  batchSize: batch.length,
87
106
  latencyMs: Date.now() - startTime,
@@ -112,14 +131,31 @@ export class EventWorker extends BaseWorker {
112
131
  for (const b of batch) b.reject(error);
113
132
  for (const e of events) e.reject(error);
114
133
  } finally {
115
- this.isFlushing = false;
134
+ this.flushesInFlight--;
135
+ // Rows that arrived while every flush slot was busy would otherwise wait
136
+ // out the interval with their worker slots held.
137
+ if (this.messageLogBuffer.length >= this.flushThreshold) {
138
+ void this.flushLogs();
139
+ }
116
140
  }
117
141
  }
118
142
 
119
143
  override async stop(): Promise<void> {
120
- if (this.flushInterval) clearInterval(this.flushInterval);
144
+ // The interval keeps running until in-flight messages settle: each of them
145
+ // is waiting on a flush, and clearing it first left them to the 30s stop
146
+ // timeout.
121
147
  await super.stop();
122
- await this.flushLogs();
148
+ if (this.flushInterval) clearInterval(this.flushInterval);
149
+ const deadline = Date.now() + 10_000;
150
+ while (
151
+ (this.messageLogBuffer.length > 0 ||
152
+ this.eventBuffer.length > 0 ||
153
+ this.flushesInFlight > 0) &&
154
+ Date.now() < deadline
155
+ ) {
156
+ await this.flushLogs();
157
+ await new Promise((resolve) => setTimeout(resolve, 10));
158
+ }
123
159
  }
124
160
 
125
161
  override async process(message: StreamMessage): Promise<void> {
@@ -150,26 +186,42 @@ export class EventWorker extends BaseWorker {
150
186
 
151
187
  const kind = event.type === "notification.dispatched" ? "dispatched" : "attempt";
152
188
 
153
- publishPromises.push(
154
- new Promise((resolve, reject) => {
189
+ const row = (
190
+ rowKind: string,
191
+ rowStatus: string,
192
+ providerMessageId: string | null,
193
+ timestamp?: string,
194
+ ) =>
195
+ new Promise<void>((resolve, reject) => {
155
196
  this.messageLogBuffer.push({
156
197
  log: {
157
198
  projectId: payload.projectId,
158
199
  taskId: payload.taskId,
159
- providerMessageId: payload.providerMessageId || null,
200
+ providerMessageId,
160
201
  channel: payload.channel,
161
202
  attempt: payload.attempt || 1,
162
- kind,
163
- status,
203
+ kind: rowKind,
204
+ status: rowStatus,
164
205
  templateId: payload.templateId || null,
165
206
  workflowInstanceId: payload.workflowInstanceId || null,
166
207
  campaignId: payload.campaignId || null,
208
+ timestamp,
167
209
  },
168
210
  resolve,
169
211
  reject,
170
212
  });
171
- }),
172
- );
213
+ });
214
+
215
+ // An outcome carrying `dispatchedAt` also stands in for the attempt's
216
+ // dispatched entry, which delivery folded into it. Stamped with the
217
+ // dispatch time so the log still reads dispatched → outcome.
218
+ if (kind === "attempt" && payload.dispatchedAt) {
219
+ publishPromises.push(row("dispatched", "dispatched", null, payload.dispatchedAt));
220
+ }
221
+ publishPromises.push(row(kind, status, payload.providerMessageId || null));
222
+ if (this.messageLogBuffer.length >= this.flushThreshold) {
223
+ void this.flushLogs();
224
+ }
173
225
  await Promise.all(publishPromises);
174
226
  return;
175
227
  }
@@ -312,39 +364,36 @@ export class EventWorker extends BaseWorker {
312
364
  }
313
365
 
314
366
  export async function startEventWorker() {
315
- logger = createLogger({ name: "event-worker", level: config.LOG_LEVEL });
316
- redis = new RedisClient({ url: config.REDIS_URL, name: "events", logger });
317
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "events", logger });
318
- sql = dbData.sql;
319
- db = dbData.db;
367
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "events", "event-worker"));
320
368
  workflowProducer = new StreamProducer({
321
369
  redis: redis.native,
322
370
  stream: STREAMS.WORKFLOW_INBOUND,
323
371
  logger,
324
372
  });
325
- consumer = new StreamConsumer({
326
- redis: redis.native,
327
- stream: STREAMS.EVENTS_INBOUND as StreamName,
328
- group: CONSUMER_GROUPS.EVENTS as any,
329
- consumer: `events-${process.pid}`,
330
- dlqStream: STREAMS.DEAD_LETTER,
331
- batchSize: config.WORKER_CONCURRENCY,
332
- logger,
333
- });
334
-
335
- pendingScanner = new PendingMessageScanner({
336
- redis: redis.native,
337
- stream: STREAMS.EVENTS_INBOUND as StreamName,
338
- group: CONSUMER_GROUPS.EVENTS as any,
339
- consumer: `events-${process.pid}`,
340
- logger,
341
- });
373
+ // Delivery emits two events per message, so this worker has to keep at least
374
+ // as many in flight as delivery does or its stream grows without bound.
375
+ const eventsConcurrency = Math.max(
376
+ config.WORKER_CONCURRENCY,
377
+ config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY,
378
+ );
379
+ ({ consumer, pendingScanner } = createStreamConsumer(
380
+ { redis, logger },
381
+ {
382
+ stream: STREAMS.EVENTS_INBOUND,
383
+ group: CONSUMER_GROUPS.EVENTS,
384
+ consumer: uniqueConsumerId("events"),
385
+ batchSize: eventsConcurrency,
386
+ bufferAcks: true,
387
+ // Log rows are not latency-sensitive; let reads fill up.
388
+ coalesceMs: 10,
389
+ },
390
+ ));
342
391
 
343
392
  worker = new EventWorker({
344
393
  consumer,
345
394
  pendingScanner,
346
395
  logger,
347
- concurrency: config.WORKER_CONCURRENCY,
396
+ concurrency: eventsConcurrency,
348
397
  db,
349
398
  workflowProducer,
350
399
  });
@@ -356,10 +405,6 @@ export async function startEventWorker() {
356
405
  }
357
406
 
358
407
  export async function stopEventWorker(): Promise<void> {
359
- logger?.info("shutdown initiated");
360
- if (healthInterval) clearInterval(healthInterval);
361
- if (worker) await worker.stop();
362
- if (sql) await sql.end();
363
- if (redis) await redis.disconnect();
364
- logger?.info("event worker stopped");
408
+ await shutdownWorker("event worker", { logger, timers: [healthInterval], worker, sql, redis });
409
+ healthInterval = null;
365
410
  }
@@ -1,13 +1,8 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { loadEnv, readBaseConfig } from "@/index.js";
3
- import { createLogger } from "@/index.js";
4
- import { RedisClient, type Redis } from "@/index.js";
5
- import {
6
- StreamConsumer,
7
- PendingMessageScanner,
8
- StreamProducer,
9
- type StreamMessage,
10
- } from "@/index.js";
3
+ import type { Logger } from "@/index.js";
4
+ import type { RedisClient, Redis } from "@/index.js";
5
+ import { type StreamConsumer, type PendingMessageScanner, type StreamMessage } from "@/index.js";
11
6
  import { BaseWorker } from "@/index.js";
12
7
  import {
13
8
  STREAMS,
@@ -16,17 +11,24 @@ import {
16
11
  buildStreamEvent,
17
12
  type NotificationScheduledPayload,
18
13
  } from "@/index.js";
19
- import { getPriorityBucket, type WorkerOptions, LUA_SCHEDULER_POLL } from "@/shared/index.js";
20
- import { createDatabase } from "@/db/index.js";
14
+ import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
21
15
  import { scheduledPayloads } from "@/db/schema.js";
22
- import { inArray } from "drizzle-orm";
16
+ import { and, asc, eq, gt, inArray, or, sql as drizzleSql } from "drizzle-orm";
23
17
  import { startHealthReporter } from "@/workers/index.js";
18
+ import {
19
+ createPriorityProducers,
20
+ createStreamConsumer,
21
+ createWorkerRuntime,
22
+ shutdownWorker,
23
+ uniqueConsumerId,
24
+ } from "@/workers/bootstrap.js";
25
+ import { findBlockedAtRelease } from "./send-gate.js";
24
26
  // ─── Bootstrap ─────────────────────────────────────────────────────────────
25
27
 
26
28
  loadEnv();
27
29
  const config = readBaseConfig();
28
30
 
29
- let logger: ReturnType<typeof createLogger>;
31
+ let logger: Logger;
30
32
  let redis: RedisClient;
31
33
  let sql: any;
32
34
  let db: any;
@@ -105,14 +107,8 @@ export async function executeSchedulerPoll(
105
107
  const pipeline = redis.pipeline();
106
108
 
107
109
  for (let i = 0; i < 16; i++) {
108
- pipeline.eval(
109
- LUA_SCHEDULER_POLL,
110
- 1,
111
- `notif:scheduled:zset:${i}`,
112
- now,
113
- perShardLimit,
114
- visibilityTimeout,
115
- );
110
+ const shardKey = `notif:scheduled:zset:${i}`;
111
+ pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
116
112
  }
117
113
 
118
114
  const results = await pipeline.exec();
@@ -143,12 +139,13 @@ export async function executeSchedulerPoll(
143
139
  .where(inArray(scheduledPayloads.taskId, taskIds));
144
140
  const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload]));
145
141
 
146
- const batchedEvents: Record<
147
- "critical" | "high" | "normal" | "low",
148
- Omit<any, "id" | "timestamp">[]
149
- > = {
142
+ // What the person did while this sat parked still applies. A failed
143
+ // lookup throws to the catch below: the claimed tasks reappear after the
144
+ // visibility timeout instead of going out unchecked.
145
+ const blocked = await findBlockedAtRelease(db, [...payloadMap.values()] as any[]);
146
+
147
+ const batchedEvents: Record<"critical" | "normal" | "low", Omit<any, "id" | "timestamp">[]> = {
150
148
  critical: [],
151
- high: [],
152
149
  normal: [],
153
150
  low: [],
154
151
  };
@@ -156,8 +153,20 @@ export async function executeSchedulerPoll(
156
153
  const cleanupPipeline = redis.pipeline();
157
154
  const dbCleanupIds: string[] = [];
158
155
 
156
+ // After a rebuild the same task can sit in the ZSET twice (the original
157
+ // member carries a trace id Postgres does not keep), so release each task
158
+ // once and just clear any second copy.
159
+ const released = new Set<string>();
160
+
159
161
  for (let i = 0; i < parsedTasks.length; i++) {
160
162
  const { taskStr, taskId, traceId } = parsedTasks[i];
163
+ if (released.has(taskId)) {
164
+ cleanupPipeline.zrem(
165
+ `notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`,
166
+ taskStr,
167
+ );
168
+ continue;
169
+ }
161
170
  const payload = payloadMap.get(taskId);
162
171
 
163
172
  if (!payload) {
@@ -168,12 +177,30 @@ export async function executeSchedulerPoll(
168
177
  }
169
178
 
170
179
  const dispatchPayload = payload as any;
180
+
181
+ const blockReason = blocked.get(taskId);
182
+ if (blockReason) {
183
+ logger.info({ taskId, reason: blockReason }, "parked send no longer allowed — dropping");
184
+ globalEmitter.emit("notification:skipped", {
185
+ projectId: dispatchPayload.projectId,
186
+ eventId: dispatchPayload.enrichedEventId,
187
+ recipientId: dispatchPayload.recipientId,
188
+ reason: blockReason,
189
+ });
190
+ released.add(taskId);
191
+ dbCleanupIds.push(taskId);
192
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
193
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
194
+ continue;
195
+ }
196
+
171
197
  const p = getPriorityBucket(dispatchPayload.priority);
172
198
 
173
199
  batchedEvents[p].push(
174
200
  buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId),
175
201
  );
176
202
 
203
+ released.add(taskId);
177
204
  dbCleanupIds.push(taskId);
178
205
  const shard = parseInt(taskId.slice(-1), 16) || 0;
179
206
  cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
@@ -191,7 +218,24 @@ export async function executeSchedulerPoll(
191
218
 
192
219
  if (dbCleanupIds.length > 0) {
193
220
  try {
194
- await db.delete(scheduledPayloads).where(inArray(scheduledPayloads.taskId, dbCleanupIds));
221
+ // Only the payload that was released. By now delivery may have
222
+ // throttled the task and parked it again under the same id with a
223
+ // later send time; deleting by id alone removed that newer payload,
224
+ // and its ZSET entry was then dropped as canceled — the send was lost.
225
+ const sendAt = drizzleSql`${scheduledPayloads.payload}->>'scheduledAt'`;
226
+ await db.delete(scheduledPayloads).where(
227
+ or(
228
+ ...dbCleanupIds.map((taskId) => {
229
+ const released = (payloadMap.get(taskId) as any)?.scheduledAt;
230
+ return and(
231
+ eq(scheduledPayloads.taskId, taskId),
232
+ typeof released === "string"
233
+ ? drizzleSql`${sendAt} = ${released}`
234
+ : drizzleSql`${sendAt} IS NULL`,
235
+ );
236
+ }),
237
+ ),
238
+ );
195
239
  } catch (err) {
196
240
  logger.error(
197
241
  { err },
@@ -212,52 +256,81 @@ export async function executeSchedulerPoll(
212
256
  return false;
213
257
  } finally {
214
258
  if (acquired) {
215
- // Delete the lock only if we still own it
216
- const lua = `
217
- if redis.call("get", KEYS[1]) == ARGV[1] then
218
- return redis.call("del", KEYS[1])
219
- else
220
- return 0
221
- end
222
- `;
223
- await redis.eval(lua, 1, lockKey, lockOwner).catch(() => {});
259
+ await redis.releaseLock(lockKey, lockOwner).catch(() => {});
224
260
  }
225
261
  }
226
262
  }
227
263
 
228
- export async function startSchedulerWorker() {
229
- logger = createLogger({ name: "scheduler", level: config.LOG_LEVEL });
230
- redis = new RedisClient({ url: config.REDIS_URL, name: "scheduler", logger });
231
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "scheduler", logger });
232
- sql = dbData.sql;
233
- db = dbData.db;
234
- consumer = new StreamConsumer({
235
- redis: redis.native,
236
- stream: STREAMS.SCHEDULED,
237
- group: CONSUMER_GROUPS.SCHEDULER,
238
- consumer: `scheduler-${process.pid}`,
239
- dlqStream: STREAMS.DEAD_LETTER,
240
- batchSize: config.WORKER_CONCURRENCY,
241
- logger,
242
- });
264
+ /**
265
+ * Present while the scheduler ZSETs are known to be intact. Redis losing its
266
+ * data (a flush, a restart without persistence, failover to an empty replica)
267
+ * takes this key with it, which is how the poll loop notices.
268
+ */
269
+ export const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
270
+
271
+ /**
272
+ * Re-queues every scheduled send still waiting in Postgres. Each row keeps its
273
+ * payload, and with it the send time, so the ZSETs can be rebuilt exactly;
274
+ * a legacy row without a send time is released on the next poll.
275
+ *
276
+ * Only run when the sentinel is missing: a rebuilt member has no trace id and
277
+ * so never matches a surviving original, and the poll only de-duplicates
278
+ * copies that happen to be claimed together.
279
+ */
280
+ export async function rebuildScheduledQueue(
281
+ redis: Redis,
282
+ db: any,
283
+ logger: Logger,
284
+ pageSize = 1000,
285
+ ): Promise<number> {
286
+ let after: string | undefined;
287
+ let total = 0;
288
+ for (;;) {
289
+ const rows: { taskId: string; payload: any }[] = await db
290
+ .select({ taskId: scheduledPayloads.taskId, payload: scheduledPayloads.payload })
291
+ .from(scheduledPayloads)
292
+ .where(after === undefined ? undefined : gt(scheduledPayloads.taskId, after))
293
+ .orderBy(asc(scheduledPayloads.taskId))
294
+ .limit(pageSize);
295
+ if (rows.length === 0) break;
243
296
 
244
- pendingScanner = new PendingMessageScanner({
245
- redis: redis.native,
246
- stream: STREAMS.SCHEDULED,
247
- group: CONSUMER_GROUPS.SCHEDULER,
248
- consumer: `scheduler-${process.pid}`,
249
- logger,
250
- });
297
+ const pipeline = redis.pipeline();
298
+ for (const { taskId, payload } of rows) {
299
+ const sendAt = Date.parse(payload?.scheduledAt ?? "");
300
+ const member = JSON.stringify({ taskId, enrichedEventId: payload?.enrichedEventId });
301
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
302
+ pipeline.zadd(
303
+ `notif:scheduled:zset:${shard}`,
304
+ "NX",
305
+ Number.isNaN(sendAt) ? Date.now() : sendAt,
306
+ member,
307
+ );
308
+ }
309
+ await pipeline.exec();
251
310
 
252
- const outboundProducers = {
253
- critical: new StreamProducer({
254
- redis: redis.native,
255
- stream: STREAMS.OUTBOUND_CRITICAL,
256
- logger,
257
- }),
258
- normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),
259
- low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),
260
- };
311
+ total += rows.length;
312
+ after = rows[rows.length - 1]!.taskId;
313
+ if (rows.length < pageSize) break;
314
+ }
315
+ // Last, so a crash part-way leaves the sentinel missing and the next poll retries.
316
+ await redis.set(SCHEDULER_SENTINEL_KEY, "1");
317
+ if (total > 0) logger.warn({ count: total }, "rebuilt scheduled queue from Postgres");
318
+ return total;
319
+ }
320
+
321
+ export async function startSchedulerWorker() {
322
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "scheduler"));
323
+ ({ consumer, pendingScanner } = createStreamConsumer(
324
+ { redis, logger },
325
+ {
326
+ stream: STREAMS.SCHEDULED,
327
+ group: CONSUMER_GROUPS.SCHEDULER,
328
+ consumer: uniqueConsumerId("scheduler"),
329
+ batchSize: config.WORKER_CONCURRENCY,
330
+ },
331
+ ));
332
+
333
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
261
334
 
262
335
  // ─── Stage 4a: Scheduled notification processor ─────────────────────────────
263
336
  //
@@ -278,6 +351,9 @@ export async function startSchedulerWorker() {
278
351
  const pollLoop = async (): Promise<void> => {
279
352
  if (!isPolling) return;
280
353
  try {
354
+ if (!(await redis.native.exists(SCHEDULER_SENTINEL_KEY))) {
355
+ await rebuildScheduledQueue(redis.native, db, logger);
356
+ }
281
357
  const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
282
358
  if (isPolling) {
283
359
  pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5000);
@@ -302,18 +378,14 @@ export async function startSchedulerWorker() {
302
378
  // ─── Shutdown ──────────────────────────────────────────────────────────────
303
379
 
304
380
  export async function stopSchedulerWorker(): Promise<void> {
305
- logger?.info("shutdown initiated");
306
- if (healthInterval) {
307
- clearInterval(healthInterval);
308
- healthInterval = null;
309
- }
310
- if (pollTimeout) {
311
- clearTimeout(pollTimeout);
312
- pollTimeout = null;
313
- }
314
381
  isPolling = false;
315
- if (worker) await worker.stop();
316
- if (sql) await sql.end();
317
- if (redis) await redis.disconnect();
318
- logger?.info("scheduler stopped");
382
+ await shutdownWorker("scheduler", {
383
+ logger,
384
+ timers: [healthInterval, pollTimeout],
385
+ worker,
386
+ sql,
387
+ redis,
388
+ });
389
+ healthInterval = null;
390
+ pollTimeout = null;
319
391
  }