notifkit 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +37 -1
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-omGPIBny.mjs +89 -0
  6. package/dist/bulk-C4hTcnLu.mjs +77 -0
  7. package/dist/index.d.mts +1160 -8886
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
  10. package/dist/main-B5BwjIMu.mjs +811 -0
  11. package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
  12. package/dist/main-Bes11Ad3.mjs +312 -0
  13. package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
  14. package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
  15. package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
  16. package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
  17. package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +21 -18
  22. package/scripts/create-admin.mjs +8 -1
  23. package/scripts/migrate.mjs +43 -0
  24. package/src/client.ts +15 -5
  25. package/src/config/index.ts +37 -3
  26. package/src/contracts/events/notification-ai-pending.ts +12 -0
  27. package/src/contracts/events/notification-delivered.ts +7 -0
  28. package/src/contracts/events/notification-dispatched.ts +21 -2
  29. package/src/contracts/events/notification-enriched.ts +16 -0
  30. package/src/contracts/events/notification-failed.ts +2 -0
  31. package/src/contracts/events/notification-requested.ts +2 -0
  32. package/src/contracts/registry.ts +2 -2
  33. package/src/contracts/sdk.ts +77 -29
  34. package/src/db/bulk.ts +132 -0
  35. package/src/db/index.ts +174 -7
  36. package/src/db/schema.ts +3 -50
  37. package/src/idempotency/index.ts +60 -6
  38. package/src/logger/index.ts +0 -12
  39. package/src/queue/index.ts +255 -76
  40. package/src/rate-limiter/index.ts +217 -16
  41. package/src/redis/index.ts +265 -0
  42. package/src/repositories/index.ts +171 -82
  43. package/src/server.ts +64 -41
  44. package/src/services/ai/main.ts +83 -70
  45. package/src/services/api/admin-static.ts +41 -12
  46. package/src/services/api/handlers.ts +357 -159
  47. package/src/services/api/http.ts +34 -7
  48. package/src/services/api/main.ts +104 -67
  49. package/src/services/api/router.ts +18 -5
  50. package/src/services/delivery/main.ts +495 -230
  51. package/src/services/delivery/throttle.ts +42 -59
  52. package/src/services/engine/main.ts +206 -152
  53. package/src/services/enricher/main.ts +155 -107
  54. package/src/services/events/main.ts +100 -55
  55. package/src/services/scheduler/main.ts +151 -79
  56. package/src/services/scheduler/send-gate.ts +117 -0
  57. package/src/services/workflow/main.ts +220 -100
  58. package/src/shared/batch-processor.ts +65 -34
  59. package/src/shared/dataloader.ts +58 -22
  60. package/src/shared/events.ts +3 -0
  61. package/src/shared/index.ts +0 -1
  62. package/src/shared/utils.ts +2 -47
  63. package/src/templates/index.ts +0 -67
  64. package/src/transport/index.ts +6 -0
  65. package/src/workers/bootstrap.ts +124 -0
  66. package/src/workers/index.ts +99 -24
  67. package/src/workflows/sdk.ts +2 -0
  68. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  69. package/dist/index.d.mts.map +0 -1
  70. package/dist/index.mjs.map +0 -1
  71. package/dist/main-39DtKrX4.mjs.map +0 -1
  72. package/dist/main-B3iQrl3t.mjs.map +0 -1
  73. package/dist/main-BM8DH3v3.mjs +0 -632
  74. package/dist/main-BM8DH3v3.mjs.map +0 -1
  75. package/dist/main-BgmT1MPe.mjs.map +0 -1
  76. package/dist/main-BiFiOX9D.mjs.map +0 -1
  77. package/dist/main-C9s-2ArH.mjs.map +0 -1
  78. package/dist/main-Cqo3rSb5.mjs +0 -224
  79. package/dist/main-Cqo3rSb5.mjs.map +0 -1
  80. package/dist/main-Dz2sUh3c.mjs.map +0 -1
  81. package/dist/src-QiTu5e_a.mjs.map +0 -1
@@ -1,13 +1,13 @@
1
1
  import { loadEnv, parseConfig, baseConfigSchema } from "@/index.js";
2
- import { createLogger } from "@/index.js";
3
- import { RedisClient, type Redis } from "@/index.js";
2
+ import type { Logger } from "@/index.js";
3
+ import type { RedisClient, Redis } from "@/index.js";
4
4
  import {
5
- StreamConsumer,
6
- PendingMessageScanner,
5
+ type StreamConsumer,
6
+ type PendingMessageScanner,
7
7
  type StreamMessage,
8
8
  StreamProducer,
9
9
  } from "@/index.js";
10
- import { BaseWorker } from "@/index.js";
10
+ import { BaseWorker, type ProcessResult } from "@/index.js";
11
11
  import {
12
12
  STREAMS,
13
13
  OUTBOUND_STREAMS,
@@ -15,16 +15,27 @@ import {
15
15
  registry,
16
16
  buildStreamEvent,
17
17
  type NotificationDispatchedPayload,
18
+ type DispatchedTaskPayload,
19
+ type RenderedContent,
20
+ PUBSUB_CHANNELS,
18
21
  } from "@/index.js";
19
- import { type StreamName } from "@/contracts/streams.js";
20
- import { createDatabase } from "@/db/index.js";
22
+ import { upsertOutboxProviderIds } from "@/db/bulk.js";
21
23
  import { deliveryOutbox, scheduledPayloads } from "@/db/schema.js";
22
- import { ContactRepository, IdempotencyGuard } from "@/index.js";
24
+ import { sql as drizzleSql } from "drizzle-orm";
25
+ import { ContactRepository, IdempotencyGuard, TemplateRepository } from "@/index.js";
26
+ import { renderWithTemplate, TemplateCache } from "@/templates/index.js";
23
27
  import { transportRegistry } from "@/index.js";
24
28
  import { globalEmitter, getPriorityBucket, type WorkerOptions } from "@/shared/index.js";
25
29
  import { startHealthReporter, NonRetryableError } from "@/workers/index.js";
30
+ import {
31
+ createPriorityProducers,
32
+ createStreamConsumer,
33
+ createWorkerRuntime,
34
+ shutdownWorker,
35
+ uniqueConsumerId,
36
+ } from "@/workers/bootstrap.js";
26
37
  import { BatchProcessor, CircuitBreaker } from "@/shared/index.js";
27
- import { throttleProvider } from "./throttle.js";
38
+ import { ProviderThrottle } from "./throttle.js";
28
39
  import { metrics } from "@/metrics/index.js";
29
40
 
30
41
  // ─── App-specific config ────────────────────────────────────────────────────
@@ -36,7 +47,7 @@ const deliveryConfigSchema = baseConfigSchema.extend({});
36
47
  loadEnv();
37
48
  const config = parseConfig(deliveryConfigSchema, process.env);
38
49
 
39
- let logger: ReturnType<typeof createLogger>;
50
+ let logger: Logger;
40
51
  let redis: RedisClient;
41
52
  let sql: any;
42
53
  let db: any;
@@ -47,6 +58,7 @@ let worker: BaseWorker;
47
58
  let scheduledProducer: StreamProducer;
48
59
  let enrichedProducers: Record<string, StreamProducer>;
49
60
  let healthInterval: NodeJS.Timeout | null = null;
61
+ let subscriber: Redis | null = null;
50
62
 
51
63
  (global as any)._telemetry = (global as any)._telemetry || {
52
64
  count: 0,
@@ -59,6 +71,48 @@ let healthInterval: NodeJS.Timeout | null = null;
59
71
  flushCount: 0,
60
72
  };
61
73
 
74
+ /** How long a send may run before its dispatched log entry is published alone. */
75
+ const DISPATCH_LOG_DELAY_MS = 1_000;
76
+
77
+ /**
78
+ * Wait before re-trying a send that failed at the provider, by prior failures.
79
+ * The first step matches the circuit breaker's reset timeout, so a retry of a
80
+ * task the open breaker turned away reaches the provider's recovery probe
81
+ * rather than the same closed door.
82
+ */
83
+ const RETRY_BACKOFF_MS = [30_000, 120_000, 600_000];
84
+
85
+ /**
86
+ * How many times a send may be put back for want of provider capacity before
87
+ * it is failed. Separate from `maxAttempts`, which counts sends the provider
88
+ * refused: waiting in line is not a failed attempt, and a campaign several
89
+ * windows deeper than the provider's limit has to be able to wait that long.
90
+ */
91
+ export const MAX_THROTTLE_DEFERRALS = 50;
92
+
93
+ /**
94
+ * Names the provider at `transports[i]` for its rate limit and circuit breaker.
95
+ * The rate limit's budget lives in Redis and is shared by every delivery
96
+ * process, so the name must come out the same everywhere: the class name,
97
+ * numbered when a channel has two providers of one class.
98
+ */
99
+ function providerKey(channel: string, transports: any[], i: number): string {
100
+ const name = transports[i].constructor?.name ?? "provider";
101
+ const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
102
+ return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`;
103
+ }
104
+
105
+ /** A provider's refusal, keeping whether it said another attempt could work. */
106
+ function providerFailure(res: { error?: string; retryable?: boolean }): Error {
107
+ const err = new Error(res.error ?? "Transport failed");
108
+ (err as any).retryable = res.retryable;
109
+ return err;
110
+ }
111
+
112
+ export function retryDelayMs(priorFailures: number): number {
113
+ return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)]!;
114
+ }
115
+
62
116
  export interface DeliveryWorkerOptions extends WorkerOptions {
63
117
  transportRegistry: any;
64
118
  idempotency: any;
@@ -69,12 +123,14 @@ export interface DeliveryWorkerOptions extends WorkerOptions {
69
123
  eventsProducer: any;
70
124
  globalEmitter: any;
71
125
  db: any;
126
+ providerThrottle?: ProviderThrottle;
127
+ /** Renders tasks that arrive without `renderedContent`. */
128
+ templateCache?: TemplateCache;
72
129
  }
73
130
 
74
131
  export class DeliveryWorker extends BaseWorker {
75
132
  private readonly transportRegistry: any;
76
133
  private readonly idempotency: any;
77
- private readonly redisCli: Redis;
78
134
  private readonly scheduledProducer: any;
79
135
  private readonly enrichedProducers: any;
80
136
  private readonly contactRepo: any;
@@ -86,49 +142,63 @@ export class DeliveryWorker extends BaseWorker {
86
142
  private outboxUpdateProcessor: BatchProcessor<any, void>;
87
143
  private outboxInsertProcessor: BatchProcessor<any, boolean>;
88
144
  private breakers = new Map<string, CircuitBreaker>();
145
+ private readonly providerThrottle: ProviderThrottle;
146
+ private readonly templateCache?: TemplateCache;
89
147
 
90
148
  constructor(options: DeliveryWorkerOptions) {
91
149
  super(options);
92
150
  this.transportRegistry = options.transportRegistry;
93
151
  this.idempotency = options.idempotency;
94
- this.redisCli = options.redis;
95
152
  this.scheduledProducer = options.scheduledProducer;
96
153
  this.enrichedProducers = options.enrichedProducers;
97
154
  this.contactRepo = options.contactRepo;
98
155
  this.eventsProducer = options.eventsProducer;
99
156
  this.globalEmitter = options.globalEmitter;
100
157
  this.db = options.db;
158
+ this.providerThrottle =
159
+ options.providerThrottle ?? new ProviderThrottle(options.redis, this.logger);
160
+ this.templateCache = options.templateCache;
161
+
162
+ // Every message waits on these flushes before it is acked, so they bound
163
+ // the worker's throughput: allow a few in flight and let batches grow with
164
+ // the configured concurrency.
165
+ const batchSize = Math.max(10, Math.min(options.concurrency ?? 50, 500));
166
+ const flushConcurrency = 4;
167
+
168
+ this.eventProcessor = new BatchProcessor<any, void>(
169
+ batchSize,
170
+ 5,
171
+ async (events) => {
172
+ await this.eventsProducer.publishBatch(events);
173
+ return events.map(() => undefined as void);
174
+ },
175
+ flushConcurrency,
176
+ );
177
+
178
+ this.outboxUpdateProcessor = new BatchProcessor<any, void>(
179
+ batchSize,
180
+ 5,
181
+ async (updates) => {
182
+ const values = updates.map((update) => ({
183
+ taskId: update.taskId,
184
+ channel: update.channel,
185
+ destination: update.destination,
186
+ providerMessageId: update.providerMessageId,
187
+ }));
188
+
189
+ const tDbUpdateStart = Date.now();
190
+ await upsertOutboxProviderIds(this.db, values).catch((e: any) =>
191
+ this.logger.error({ err: e }, "background update failed"),
192
+ );
101
193
 
102
- this.eventProcessor = new BatchProcessor<any, void>(1000, 100, async (events) => {
103
- await this.eventsProducer.publishBatch(events);
104
- return events.map(() => undefined as void);
105
- });
106
-
107
- this.outboxUpdateProcessor = new BatchProcessor<any, void>(500, 100, async (updates) => {
108
- const { sql } = await import("drizzle-orm");
109
- const values = updates.map((update) => ({
110
- taskId: update.taskId,
111
- channel: update.channel,
112
- destination: update.destination,
113
- providerMessageId: update.providerMessageId,
114
- }));
115
-
116
- const tDbUpdateStart = Date.now();
117
- await this.db
118
- .insert(deliveryOutbox)
119
- .values(values)
120
- .onConflictDoUpdate({
121
- target: [deliveryOutbox.taskId, deliveryOutbox.channel, deliveryOutbox.destination],
122
- set: { providerMessageId: sql`EXCLUDED.provider_message_id` },
123
- })
124
- .catch((e: any) => this.logger.error({ err: e }, "background update failed"));
125
-
126
- (global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
127
- (global as any)._telemetry.flushCount++;
128
- return updates.map(() => undefined as void);
129
- });
194
+ (global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
195
+ (global as any)._telemetry.flushCount++;
196
+ return updates.map(() => undefined as void);
197
+ },
198
+ flushConcurrency,
199
+ );
130
200
 
131
- this.outboxInsertProcessor = new BatchProcessor(500, 10, async (tasks) => {
201
+ this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
132
202
  const values = tasks.map((task: any) => ({
133
203
  taskId: task.taskId,
134
204
  channel: task.channel,
@@ -149,6 +219,68 @@ export class DeliveryWorker extends BaseWorker {
149
219
  await super.stop();
150
220
  }
151
221
 
222
+ /**
223
+ * Renders a task the engine dispatched unrendered. Done here, after the
224
+ * throttle and the lease, so the body is built once per actual send and never
225
+ * travels through the outbound stream.
226
+ */
227
+ private async render(task: NotificationDispatchedPayload): Promise<RenderedContent> {
228
+ if (!task.templateId) return renderWithTemplate(null, task.templateVariables);
229
+ const template = this.templateCache
230
+ ? await this.templateCache.getCachedTemplate(task.projectId, task.templateId)
231
+ : null;
232
+ if (!template) {
233
+ // The engine checked it existed; it was deleted since. Retrying won't help.
234
+ throw new NonRetryableError(`template "${task.templateId}" not found at send time`);
235
+ }
236
+ return renderWithTemplate(template, task.templateVariables);
237
+ }
238
+
239
+ /**
240
+ * Parks a task in Postgres and the scheduler until `sendAt`. The payload
241
+ * carries the send time, so the scheduler can rebuild it if Redis loses its
242
+ * data.
243
+ */
244
+ private async reschedule(
245
+ task: DispatchedTaskPayload,
246
+ sendAt: Date,
247
+ reason: string,
248
+ ): Promise<void> {
249
+ const scheduledAt = sendAt.toISOString();
250
+ await this.db
251
+ .insert(scheduledPayloads)
252
+ .values({ taskId: task.taskId, payload: { ...task, scheduledAt } })
253
+ .onConflictDoUpdate({
254
+ target: scheduledPayloads.taskId,
255
+ set: { payload: drizzleSql`EXCLUDED.payload` },
256
+ });
257
+
258
+ await this.scheduledProducer.publish(
259
+ buildStreamEvent(
260
+ "notification.scheduled",
261
+ {
262
+ projectId: task.projectId,
263
+ enrichedEventId: task.enrichedEventId,
264
+ taskId: task.taskId,
265
+ scheduledAt,
266
+ },
267
+ "delivery",
268
+ `${task.taskId}:${reason}:${Date.now()}`,
269
+ ),
270
+ );
271
+ }
272
+
273
+ /** Takes a slot from the provider's rate limit, if it has one. */
274
+ private async providerGate(
275
+ channel: string,
276
+ transports: any[],
277
+ i: number,
278
+ ): Promise<{ allowed: boolean; retryAfterMs: number }> {
279
+ const limits = transports[i].limits;
280
+ if (!limits) return { allowed: true, retryAfterMs: 0 };
281
+ return this.providerThrottle.check(providerKey(channel, transports, i), limits);
282
+ }
283
+
152
284
  private getBreaker(name: string): CircuitBreaker {
153
285
  let breaker = this.breakers.get(name);
154
286
  if (!breaker) {
@@ -158,7 +290,7 @@ export class DeliveryWorker extends BaseWorker {
158
290
  return breaker;
159
291
  }
160
292
 
161
- async process(message: StreamMessage, attempt: number = 1): Promise<void> {
293
+ async process(message: StreamMessage, attempt: number = 1): Promise<ProcessResult> {
162
294
  const { event } = message;
163
295
  const publishPromises: Promise<void>[] = [];
164
296
 
@@ -172,6 +304,11 @@ export class DeliveryWorker extends BaseWorker {
172
304
  }
173
305
 
174
306
  const task = payloadResult.data as NotificationDispatchedPayload;
307
+ // Attempts made on earlier trips through the scheduler, plus this one.
308
+ const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
309
+ // What gets parked for a retry: the task as it arrived, before rendering
310
+ // and the transport's abort signal were attached to it.
311
+ const original: DispatchedTaskPayload = { ...task };
175
312
 
176
313
  const fallbackToNextChannel = async (reason: string) => {
177
314
  if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
@@ -191,6 +328,9 @@ export class DeliveryWorker extends BaseWorker {
191
328
  recipient: task.recipient,
192
329
  scheduledAt: undefined,
193
330
  fallbackChain: remainingChain.length > 0 ? remainingChain : undefined,
331
+ campaignId: task.campaignId,
332
+ workflowInstanceId: task.workflowInstanceId,
333
+ throttleCounted: true,
194
334
  };
195
335
 
196
336
  const p = getPriorityBucket(task.priority);
@@ -241,10 +381,9 @@ export class DeliveryWorker extends BaseWorker {
241
381
  failureReason: "no transport registered for channel",
242
382
  failureCode: "no_transport",
243
383
  retryable: false,
244
- attempt,
384
+ attempt: attemptNo,
245
385
  templateId: task.templateId,
246
- workflowInstanceId:
247
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
386
+ workflowInstanceId: task.workflowInstanceId,
248
387
  campaignId: task.campaignId,
249
388
  },
250
389
  "delivery",
@@ -255,137 +394,216 @@ export class DeliveryWorker extends BaseWorker {
255
394
  return;
256
395
  }
257
396
 
258
- const limitConfig = transports[0].limits;
259
- if (limitConfig) {
260
- const { allowed, retryAfterMs } = await throttleProvider(
261
- this.redisCli,
262
- task.channel,
263
- limitConfig,
264
- this.logger,
265
- );
266
- if (!allowed) {
267
- task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
268
- const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
397
+ // Each provider has its own budget: start at the first with room left. The
398
+ // channel is out of capacity only once every provider for it is.
399
+ let startAt = -1;
400
+ let retryAfterMs = Infinity;
401
+ let windowMs = Infinity;
402
+ for (let i = 0; i < transports.length; i++) {
403
+ const gate = await this.providerGate(task.channel, transports, i);
404
+ if (gate.allowed) {
405
+ startAt = i;
406
+ break;
407
+ }
408
+ retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs);
409
+ windowMs = Math.min(windowMs, (transports[i].limits?.windowSeconds ?? 0) * 1000);
410
+ }
411
+ if (startAt === -1) {
412
+ task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
269
413
 
270
- if (task.throttleAttemptCount > maxAttempts) {
271
- this.logger.warn(
272
- { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount },
273
- "provider rate limit max attempts exceeded",
414
+ if (task.throttleAttemptCount > MAX_THROTTLE_DEFERRALS) {
415
+ this.logger.warn(
416
+ { messageId: message.id, taskId: task.taskId, attempts: task.throttleAttemptCount },
417
+ "provider rate limit max attempts exceeded",
418
+ );
419
+ const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
420
+ if (!fallbackTriggered) {
421
+ this.globalEmitter.emit(
422
+ "delivery:failed",
423
+ task.taskId,
424
+ "provider throttle exceeded",
425
+ task.channel,
426
+ task.projectId,
427
+ );
428
+ await this.eventProcessor.add(
429
+ buildStreamEvent(
430
+ "notification.failed",
431
+ {
432
+ projectId: task.projectId,
433
+ taskId: task.taskId,
434
+ enrichedEventId: task.enrichedEventId,
435
+ recipientId: task.recipientId,
436
+ channel: task.channel,
437
+ failureReason: "provider rate limit max attempts exceeded",
438
+ failureCode: "provider_throttle_exceeded",
439
+ retryable: false,
440
+ attempt: attemptNo,
441
+ templateId: task.templateId,
442
+ workflowInstanceId: task.workflowInstanceId,
443
+ campaignId: task.campaignId,
444
+ },
445
+ "delivery",
446
+ event.metadata.traceId,
447
+ ),
274
448
  );
275
- const fallbackTriggered = await fallbackToNextChannel("provider_throttle_exceeded");
276
- if (!fallbackTriggered) {
277
- this.globalEmitter.emit(
278
- "delivery:failed",
279
- task.taskId,
280
- "provider throttle exceeded",
281
- task.channel,
282
- task.projectId,
283
- );
284
- await this.eventProcessor.add(
285
- buildStreamEvent(
286
- "notification.failed",
287
- {
288
- projectId: task.projectId,
289
- taskId: task.taskId,
290
- enrichedEventId: task.enrichedEventId,
291
- recipientId: task.recipientId,
292
- channel: task.channel,
293
- failureReason: "provider rate limit max attempts exceeded",
294
- failureCode: "provider_throttle_exceeded",
295
- retryable: false,
296
- attempt,
297
- templateId: task.templateId,
298
- workflowInstanceId:
299
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
300
- campaignId: task.campaignId,
301
- },
302
- "delivery",
303
- event.metadata.traceId,
304
- ),
305
- );
306
- }
307
- return;
308
449
  }
309
-
310
- const { sql } = await import("drizzle-orm");
311
- await this.db
312
- .insert(scheduledPayloads)
313
- .values({
314
- taskId: task.taskId,
315
- payload: task,
316
- })
317
- .onConflictDoUpdate({
318
- target: scheduledPayloads.taskId,
319
- set: { payload: sql`EXCLUDED.payload` },
320
- });
321
-
322
- await this.scheduledProducer.publish(
323
- buildStreamEvent(
324
- "notification.scheduled",
325
- {
326
- projectId: task.projectId,
327
- enrichedEventId: task.enrichedEventId,
328
- taskId: task.taskId,
329
- scheduledAt: new Date(Date.now() + retryAfterMs).toISOString(),
330
- throttleAttemptCount: task.throttleAttemptCount,
331
- },
332
- "delivery",
333
- `${task.taskId}:throttle:${Date.now()}`,
334
- ),
335
- );
336
-
337
450
  return;
338
451
  }
452
+
453
+ // Spread over more windows the longer a task has waited. Every task turned
454
+ // away in one window used to come back at the same boundary, where the
455
+ // limit again let one window's worth through and bounced the rest.
456
+ const spreadMs = Number.isFinite(windowMs)
457
+ ? Math.random() * windowMs * Math.min(task.throttleAttemptCount, 10)
458
+ : 0;
459
+ await this.reschedule(
460
+ { ...original, throttleAttemptCount: task.throttleAttemptCount },
461
+ new Date(Date.now() + retryAfterMs + spreadMs),
462
+ "throttle",
463
+ );
464
+ return;
339
465
  }
340
466
 
341
- const tInsertStart = Date.now();
342
467
  const idempotencyKey = task.taskId;
343
- if (!(await this.idempotency.checkAndMark(idempotencyKey, 60))) {
468
+ const lease = this.idempotency.acquireLease
469
+ ? await this.idempotency.acquireLease(idempotencyKey, 60)
470
+ : (await this.idempotency.checkAndMark(idempotencyKey, 60))
471
+ ? "acquired"
472
+ : "locked";
473
+
474
+ if (lease === "completed") {
344
475
  this.logger.info(
345
- { messageId: message.id, taskId: task.taskId, channel: task.channel, attempt },
346
- "duplicate delivery — skipping",
476
+ { messageId: message.id, taskId: task.taskId, channel: task.channel },
477
+ "duplicate delivery — already completed, acking message",
347
478
  );
348
479
  return;
349
480
  }
350
481
 
351
- try {
352
- await this.outboxInsertProcessor.add(task);
353
- const insertTime = Date.now() - tInsertStart;
482
+ if (lease === "locked") {
483
+ this.logger.info(
484
+ { messageId: message.id, taskId: task.taskId, channel: task.channel, attempt },
485
+ "duplicate delivery — skipping and leaving pending in stream",
486
+ );
487
+ return { ack: false };
488
+ }
354
489
 
355
- publishPromises.push(
356
- this.eventProcessor.add(
357
- buildStreamEvent(
358
- "notification.dispatched",
359
- {
360
- projectId: task.projectId,
361
- taskId: task.taskId,
362
- enrichedEventId: task.enrichedEventId,
363
- recipientId: task.recipientId,
364
- channel: task.channel,
365
- templateId: task.templateId,
366
- attempt,
367
- workflowInstanceId:
368
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
369
- campaignId: task.campaignId,
370
- },
371
- "delivery",
372
- event.metadata.traceId,
373
- ),
490
+ const tDispatchStart = Date.now();
491
+ const MAX_DISPATCH_BUDGET_MS = 25_000; // Strictly shorter than lock lease (30s)
492
+
493
+ // The attempt's `dispatched` log entry normally rides on its outcome event,
494
+ // one stream entry instead of two. A send still running after
495
+ // DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
496
+ // visible while it is in flight, as it always was.
497
+ let dispatchStarted = false;
498
+ let dispatchLogged = false;
499
+ let dispatchedAt = "";
500
+ let dispatchLogTimer: NodeJS.Timeout | undefined;
501
+ const logDispatched = (): Promise<void> | undefined => {
502
+ clearTimeout(dispatchLogTimer);
503
+ if (!dispatchStarted || dispatchLogged) return undefined;
504
+ dispatchLogged = true;
505
+ const logged = this.eventProcessor.add(
506
+ buildStreamEvent(
507
+ "notification.dispatched",
508
+ {
509
+ projectId: task.projectId,
510
+ taskId: task.taskId,
511
+ enrichedEventId: task.enrichedEventId,
512
+ recipientId: task.recipientId,
513
+ channel: task.channel,
514
+ templateId: task.templateId,
515
+ attempt: attemptNo,
516
+ workflowInstanceId: task.workflowInstanceId,
517
+ campaignId: task.campaignId,
518
+ },
519
+ "delivery",
520
+ event.metadata.traceId,
374
521
  ),
375
522
  );
523
+ publishPromises.push(logged);
524
+ return logged;
525
+ };
526
+ /** Outcome payload, carrying the dispatched entry if it has not gone out. */
527
+ const outcome = <T extends Record<string, unknown>>(payload: T): T => {
528
+ clearTimeout(dispatchLogTimer);
529
+ if (dispatchLogged) return payload;
530
+ dispatchLogged = true;
531
+ return { ...payload, attempt: attemptNo, dispatchedAt };
532
+ };
533
+
534
+ let retryScheduled = false;
535
+ /**
536
+ * Re-queues a send the provider turned away, unless it said the failure is
537
+ * permanent or the task is out of attempts. An outage, a timeout or an open
538
+ * circuit is exactly what a later attempt can get past; failing the task
539
+ * outright turned every blip into lost mail. The attempt stays in the log as
540
+ * dispatched, and the task's outcome is whatever its last attempt comes to.
541
+ */
542
+ const retryLater = async (failure: { retryable?: boolean; error?: string }) => {
543
+ const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
544
+ if (failure.retryable === false || attemptNo >= maxAttempts) return false;
545
+ const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0);
546
+ await this.reschedule(
547
+ { ...original, deliveryAttemptCount: attemptNo },
548
+ new Date(Date.now() + delayMs),
549
+ "retry",
550
+ );
551
+ retryScheduled = true;
552
+ metrics.deliveryFailed.inc({ channel: task.channel, reason: "retry_scheduled" });
553
+ this.logger.warn(
554
+ {
555
+ taskId: task.taskId,
556
+ channel: task.channel,
557
+ attempt: attemptNo,
558
+ maxAttempts,
559
+ delayMs,
560
+ error: failure.error,
561
+ },
562
+ "provider send failed — retrying later",
563
+ );
564
+ return true;
565
+ };
566
+
567
+ try {
568
+ if (!task.renderedContent) {
569
+ task.renderedContent = await this.render(task);
570
+ }
571
+
572
+ dispatchStarted = true;
573
+ dispatchedAt = new Date().toISOString();
574
+ dispatchLogTimer = setTimeout(() => {
575
+ void logDispatched();
576
+ }, DISPATCH_LOG_DELAY_MS);
376
577
 
377
578
  // For push: we just need to send to the pre-resolved destination,
378
579
  // but if it's invalid, we deactivate it.
379
580
  if (task.channel === "push") {
380
581
  let lastResult: any = { success: false, error: "No transports" };
381
582
 
382
- for (const transport of transports) {
583
+ for (let i = startAt; i < transports.length; i++) {
584
+ const transport = transports[i];
585
+ // The starting provider's slot was taken above. Failing over to a
586
+ // later one spends that provider's own budget, or skips it if spent.
587
+ if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
588
+ continue;
589
+ }
590
+ const elapsedMs = Date.now() - tDispatchStart;
591
+ const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
592
+ if (remainingBudgetMs <= 0) {
593
+ this.logger.warn(
594
+ { taskId: task.taskId, channel: task.channel },
595
+ "dispatch budget exhausted before transport call — aborting to protect lock lease",
596
+ );
597
+ break;
598
+ }
599
+ const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
600
+ const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
601
+
383
602
  try {
384
603
  const tProv = Date.now();
385
- const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`);
604
+ const breaker = this.getBreaker(providerKey(task.channel, transports, i));
386
605
 
387
606
  lastResult = await breaker.execute(async () => {
388
- const timeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
389
607
  const controller = new AbortController();
390
608
  const timeout = setTimeout(() => {
391
609
  controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
@@ -403,7 +621,7 @@ export class DeliveryWorker extends BaseWorker {
403
621
  }),
404
622
  ]);
405
623
  if (!res.success && !res.invalidToken) {
406
- throw new Error(res.error ?? "Transport failed");
624
+ throw providerFailure(res);
407
625
  }
408
626
  return res;
409
627
  } finally {
@@ -414,11 +632,19 @@ export class DeliveryWorker extends BaseWorker {
414
632
  (global as any)._telemetry.provider += Date.now() - tProv;
415
633
  if (lastResult.success || lastResult.invalidToken) break;
416
634
  } catch (err: any) {
417
- lastResult = { success: false, error: err.message };
635
+ lastResult = { success: false, error: err.message, retryable: err.retryable };
418
636
  }
419
637
  }
420
638
 
421
639
  if (lastResult.invalidToken) {
640
+ publishPromises.push(
641
+ this.outboxUpdateProcessor.add({
642
+ taskId: task.taskId,
643
+ channel: task.channel,
644
+ destination: task.destination,
645
+ providerMessageId: null,
646
+ }),
647
+ );
422
648
  await this.contactRepo.deactivate(
423
649
  task.projectId,
424
650
  task.recipientId,
@@ -442,7 +668,7 @@ export class DeliveryWorker extends BaseWorker {
442
668
  this.eventProcessor.add(
443
669
  buildStreamEvent(
444
670
  "notification.failed",
445
- {
671
+ outcome({
446
672
  projectId: task.projectId,
447
673
  taskId: task.taskId,
448
674
  enrichedEventId: task.enrichedEventId,
@@ -451,12 +677,11 @@ export class DeliveryWorker extends BaseWorker {
451
677
  failureReason: "invalid_token",
452
678
  failureCode: "push_failure",
453
679
  retryable: false,
454
- attempt,
680
+ attempt: attemptNo,
455
681
  templateId: task.templateId,
456
- workflowInstanceId:
457
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
682
+ workflowInstanceId: task.workflowInstanceId,
458
683
  campaignId: task.campaignId,
459
- },
684
+ }),
460
685
  "delivery",
461
686
  event.metadata.traceId,
462
687
  ),
@@ -497,7 +722,7 @@ export class DeliveryWorker extends BaseWorker {
497
722
  this.eventProcessor.add(
498
723
  buildStreamEvent(
499
724
  "notification.delivered",
500
- {
725
+ outcome({
501
726
  projectId: task.projectId,
502
727
  taskId: task.taskId,
503
728
  enrichedEventId: task.enrichedEventId,
@@ -505,16 +730,15 @@ export class DeliveryWorker extends BaseWorker {
505
730
  deliveredAt: new Date().toISOString(),
506
731
  providerMessageId,
507
732
  templateId: task.templateId,
508
- workflowInstanceId:
509
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
733
+ workflowInstanceId: task.workflowInstanceId,
510
734
  campaignId: task.campaignId,
511
- },
735
+ }),
512
736
  "delivery",
513
737
  event.metadata.traceId,
514
738
  ),
515
739
  ),
516
740
  );
517
- } else {
741
+ } else if (!(await retryLater(lastResult))) {
518
742
  this.logger.warn(
519
743
  { taskId: task.taskId, error: lastResult.error },
520
744
  "push delivery failed",
@@ -528,11 +752,20 @@ export class DeliveryWorker extends BaseWorker {
528
752
  );
529
753
  metrics.deliveryFailed.inc({ channel: task.channel, reason: "push_error" });
530
754
 
755
+ publishPromises.push(
756
+ this.outboxUpdateProcessor.add({
757
+ taskId: task.taskId,
758
+ channel: task.channel,
759
+ destination: task.destination,
760
+ providerMessageId: null,
761
+ }),
762
+ );
763
+
531
764
  publishPromises.push(
532
765
  this.eventProcessor.add(
533
766
  buildStreamEvent(
534
767
  "notification.failed",
535
- {
768
+ outcome({
536
769
  projectId: task.projectId,
537
770
  taskId: task.taskId,
538
771
  enrichedEventId: task.enrichedEventId,
@@ -541,12 +774,11 @@ export class DeliveryWorker extends BaseWorker {
541
774
  failureReason: lastResult.error ?? "push delivery failed",
542
775
  failureCode: "push_failure",
543
776
  retryable: false,
544
- attempt,
777
+ attempt: attemptNo,
545
778
  templateId: task.templateId,
546
- workflowInstanceId:
547
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
779
+ workflowInstanceId: task.workflowInstanceId,
548
780
  campaignId: task.campaignId,
549
- },
781
+ }),
550
782
  "delivery",
551
783
  event.metadata.traceId,
552
784
  ),
@@ -561,13 +793,30 @@ export class DeliveryWorker extends BaseWorker {
561
793
  } else {
562
794
  // Other channels (email, sms, webhook, whatsapp) use the pre-resolved destination
563
795
  let result: any = { success: false, error: "No transports" };
564
- for (const transport of transports) {
796
+ for (let i = startAt; i < transports.length; i++) {
797
+ const transport = transports[i];
798
+ // The starting provider's slot was taken above. Failing over to a
799
+ // later one spends that provider's own budget, or skips it if spent.
800
+ if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
801
+ continue;
802
+ }
803
+ const elapsedMs = Date.now() - tDispatchStart;
804
+ const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
805
+ if (remainingBudgetMs <= 0) {
806
+ this.logger.warn(
807
+ { taskId: task.taskId, channel: task.channel },
808
+ "dispatch budget exhausted before transport call — aborting to protect lock lease",
809
+ );
810
+ break;
811
+ }
812
+ const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
813
+ const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
814
+
565
815
  try {
566
816
  const tProv = Date.now();
567
- const breaker = this.getBreaker(`${task.channel}:${transport.constructor.name}`);
817
+ const breaker = this.getBreaker(providerKey(task.channel, transports, i));
568
818
 
569
819
  result = await breaker.execute(async () => {
570
- const timeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
571
820
  const controller = new AbortController();
572
821
  const timeout = setTimeout(() => {
573
822
  controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
@@ -585,7 +834,7 @@ export class DeliveryWorker extends BaseWorker {
585
834
  }),
586
835
  ]);
587
836
  if (!res.success) {
588
- throw new Error(res.error ?? "Transport failed");
837
+ throw providerFailure(res);
589
838
  }
590
839
  return res;
591
840
  } finally {
@@ -596,7 +845,7 @@ export class DeliveryWorker extends BaseWorker {
596
845
  (global as any)._telemetry.provider += Date.now() - tProv;
597
846
  if (result.success) break;
598
847
  } catch (err: any) {
599
- result = { success: false, error: err.message };
848
+ result = { success: false, error: err.message, retryable: err.retryable };
600
849
  }
601
850
  }
602
851
 
@@ -626,7 +875,7 @@ export class DeliveryWorker extends BaseWorker {
626
875
  this.eventProcessor.add(
627
876
  buildStreamEvent(
628
877
  "notification.delivered",
629
- {
878
+ outcome({
630
879
  projectId: task.projectId,
631
880
  taskId: task.taskId,
632
881
  enrichedEventId: task.enrichedEventId,
@@ -634,21 +883,29 @@ export class DeliveryWorker extends BaseWorker {
634
883
  deliveredAt: new Date().toISOString(),
635
884
  providerMessageId,
636
885
  templateId: task.templateId,
637
- workflowInstanceId:
638
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
886
+ workflowInstanceId: task.workflowInstanceId,
639
887
  campaignId: task.campaignId,
640
- },
888
+ }),
641
889
  "delivery",
642
890
  event.metadata.traceId,
643
891
  ),
644
892
  ),
645
893
  );
646
- } else {
894
+ } else if (!(await retryLater(result))) {
895
+ publishPromises.push(
896
+ this.outboxUpdateProcessor.add({
897
+ taskId: task.taskId,
898
+ channel: task.channel,
899
+ destination: task.destination,
900
+ providerMessageId: null,
901
+ }),
902
+ );
903
+
647
904
  publishPromises.push(
648
905
  this.eventProcessor.add(
649
906
  buildStreamEvent(
650
907
  "notification.failed",
651
- {
908
+ outcome({
652
909
  projectId: task.projectId,
653
910
  taskId: task.taskId,
654
911
  enrichedEventId: task.enrichedEventId,
@@ -656,12 +913,11 @@ export class DeliveryWorker extends BaseWorker {
656
913
  failureReason: result.error ?? "delivery failed completely",
657
914
  failureCode: "provider_error",
658
915
  retryable: false,
659
- attempt,
916
+ attempt: attemptNo,
660
917
  templateId: task.templateId,
661
- workflowInstanceId:
662
- event.metadata.source === "workflow" ? event.metadata.traceId : undefined,
918
+ workflowInstanceId: task.workflowInstanceId,
663
919
  campaignId: task.campaignId,
664
- },
920
+ }),
665
921
  "delivery",
666
922
  event.metadata.traceId,
667
923
  ),
@@ -669,7 +925,7 @@ export class DeliveryWorker extends BaseWorker {
669
925
  );
670
926
  }
671
927
 
672
- if (!result.success) {
928
+ if (!result.success && !retryScheduled) {
673
929
  this.logger.warn(
674
930
  { taskId: task.taskId, channel: task.channel, error: result.error },
675
931
  "delivery failed completely across providers",
@@ -687,14 +943,17 @@ export class DeliveryWorker extends BaseWorker {
687
943
  if (!fallbackTriggered) {
688
944
  throw new NonRetryableError(result.error ?? "delivery failed");
689
945
  }
690
- } else {
691
- this.logger.info(
946
+ } else if (result.success) {
947
+ this.logger.debug(
692
948
  { taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
693
949
  "notification delivered",
694
950
  );
695
951
  }
696
952
  }
697
953
 
954
+ // Every branch above publishes an outcome; this only guards one that doesn't.
955
+ void logDispatched();
956
+
698
957
  const tFlushStart = Date.now();
699
958
  await Promise.all(publishPromises).catch((err) => {
700
959
  this.logger.error(
@@ -704,7 +963,14 @@ export class DeliveryWorker extends BaseWorker {
704
963
  });
705
964
  const flushTime = Date.now() - tFlushStart;
706
965
 
707
- await this.idempotency.markProcessed(idempotencyKey);
966
+ if (retryScheduled) {
967
+ // Released so the re-queued attempt can take the lease.
968
+ await this.idempotency.unmark(idempotencyKey).catch(() => {});
969
+ } else {
970
+ void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
971
+ this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
972
+ });
973
+ }
708
974
 
709
975
  const t = ((global as any)._telemetry = (global as any)._telemetry || {
710
976
  count: 0,
@@ -714,7 +980,7 @@ export class DeliveryWorker extends BaseWorker {
714
980
  ack: 0,
715
981
  });
716
982
  t.count++;
717
- t.insert += insertTime;
983
+ t.insert += 0;
718
984
  t.flush += flushTime;
719
985
 
720
986
  if (t.count % 1000 === 0) {
@@ -731,6 +997,8 @@ export class DeliveryWorker extends BaseWorker {
731
997
  t.flushCount = 0;
732
998
  }
733
999
  } catch (err) {
1000
+ // The attempt happened even though it did not finish; keep it in the log.
1001
+ logDispatched()?.catch(() => {});
734
1002
  await this.idempotency.unmark(idempotencyKey).catch(() => {});
735
1003
  throw err;
736
1004
  }
@@ -738,21 +1006,22 @@ export class DeliveryWorker extends BaseWorker {
738
1006
  }
739
1007
 
740
1008
  export async function startDeliveryWorker() {
741
- logger = createLogger({ name: "delivery", level: config.LOG_LEVEL });
742
-
743
- redis = new RedisClient({ url: config.REDIS_URL, name: "delivery", logger });
744
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "delivery", logger });
745
- sql = dbData.sql;
746
- db = dbData.db;
747
- consumer = new StreamConsumer({
748
- redis: redis.native,
749
- stream: OUTBOUND_STREAMS as unknown as StreamName[],
750
- group: CONSUMER_GROUPS.DELIVERY,
751
- consumer: `delivery-${process.pid}`,
752
- dlqStream: STREAMS.DEAD_LETTER,
753
- batchSize: config.WORKER_CONCURRENCY,
754
- logger,
755
- });
1009
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "delivery"));
1010
+ // Delivery spends most of its time waiting on providers, so it needs far
1011
+ // more messages in flight than the CPU-bound stages do.
1012
+ const deliveryConcurrency = config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY;
1013
+ const consumerId = uniqueConsumerId("delivery");
1014
+ ({ consumer, pendingScanner } = createStreamConsumer(
1015
+ { redis, logger },
1016
+ {
1017
+ stream: OUTBOUND_STREAMS,
1018
+ group: CONSUMER_GROUPS.DELIVERY,
1019
+ consumer: consumerId,
1020
+ batchSize: deliveryConcurrency,
1021
+ bufferAcks: true,
1022
+ coalesceMs: 2,
1023
+ },
1024
+ ));
756
1025
 
757
1026
  scheduledProducer = new StreamProducer({
758
1027
  redis: redis.native,
@@ -760,30 +1029,15 @@ export async function startDeliveryWorker() {
760
1029
  logger,
761
1030
  });
762
1031
 
763
- pendingScanner = new PendingMessageScanner({
764
- redis: redis.native,
765
- stream: OUTBOUND_STREAMS as unknown as StreamName[],
766
- group: CONSUMER_GROUPS.DELIVERY,
767
- consumer: `delivery-${process.pid}`,
768
- logger,
769
- });
770
-
771
- enrichedProducers = {
772
- critical: new StreamProducer({
773
- redis: redis.native,
774
- stream: STREAMS.ENRICHED_CRITICAL,
775
- logger,
776
- }),
777
- normal: new StreamProducer({ redis: redis.native, stream: STREAMS.ENRICHED_NORMAL, logger }),
778
- low: new StreamProducer({ redis: redis.native, stream: STREAMS.ENRICHED_LOW, logger }),
779
- };
1032
+ enrichedProducers = createPriorityProducers(redis.native, logger, "ENRICHED");
780
1033
 
781
1034
  const contactRepo = new ContactRepository(db);
1035
+ const templateCache = new TemplateCache(new TemplateRepository(db));
782
1036
 
783
1037
  const idempotency = new IdempotencyGuard({
784
1038
  redis: redis.native,
785
1039
  keyPrefix: "notif:processed:delivery",
786
- ttlSeconds: 86_400,
1040
+ ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
787
1041
  });
788
1042
 
789
1043
  // ─── Stage 3: Delivery Worker ───────────────────────────────────────────────
@@ -804,7 +1058,7 @@ export async function startDeliveryWorker() {
804
1058
  consumer,
805
1059
  pendingScanner,
806
1060
  logger,
807
- concurrency: config.WORKER_CONCURRENCY,
1061
+ concurrency: deliveryConcurrency,
808
1062
  transportRegistry,
809
1063
  idempotency,
810
1064
  redis: redis.native,
@@ -814,8 +1068,14 @@ export async function startDeliveryWorker() {
814
1068
  eventsProducer,
815
1069
  globalEmitter,
816
1070
  db,
1071
+ templateCache,
817
1072
  });
818
1073
 
1074
+ // Delivery renders templates now, so it has to hear about edits too.
1075
+ subscriber = redis.native.duplicate({ enableAutoPipelining: false });
1076
+ await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
1077
+ subscriber.on("message", (_channel: string, key: string) => templateCache.invalidateKey(key));
1078
+
819
1079
  // ─── Health check interval ──────────────────────────────────────────────────
820
1080
 
821
1081
  healthInterval = startHealthReporter("delivery", worker, redis, logger);
@@ -827,16 +1087,21 @@ export async function startDeliveryWorker() {
827
1087
  await worker.start();
828
1088
  }
829
1089
 
1090
+ export function getDeliveryWorker(): DeliveryWorker | undefined {
1091
+ return worker as DeliveryWorker | undefined;
1092
+ }
1093
+
830
1094
  // ─── Shutdown ──────────────────────────────────────────────────────────────
831
1095
 
832
1096
  export async function stopDeliveryWorker(): Promise<void> {
833
- logger?.info("shutdown initiated");
834
- if (healthInterval) {
835
- clearInterval(healthInterval);
836
- healthInterval = null;
837
- }
838
- if (worker) await worker.stop();
839
- if (sql) await sql.end();
840
- if (redis) await redis.disconnect();
841
- logger?.info("delivery stopped");
1097
+ await shutdownWorker("delivery", {
1098
+ logger,
1099
+ timers: [healthInterval],
1100
+ subscriber,
1101
+ worker,
1102
+ sql,
1103
+ redis,
1104
+ });
1105
+ healthInterval = null;
1106
+ subscriber = null;
842
1107
  }