notifkit 0.1.15 → 0.1.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dashboard/dist/assets/index-C-DJYu0_.css +1 -0
  2. package/dashboard/dist/assets/index-CzFZcEon.js +318 -0
  3. package/dashboard/dist/index.html +2 -2
  4. package/dist/{bootstrap-BOmT69OH.mjs → bootstrap-BMDRPjgp.mjs} +1 -1
  5. package/dist/{bulk-ZPiOJRx-.mjs → bulk-CJ5QXiPa.mjs} +1 -1
  6. package/dist/index.d.mts +71 -27
  7. package/dist/index.mjs +2 -2
  8. package/dist/{main-CCqcFDDe.mjs → main-6_bXQBDH.mjs} +21 -41
  9. package/dist/{main-BdZvBgV7.mjs → main-8nP3qmCT.mjs} +22 -48
  10. package/dist/{main-C0m9-Eos.mjs → main-B82bsFoy.mjs} +183 -334
  11. package/dist/{main-CTjaZ7ZD.mjs → main-CpaHF1Zw.mjs} +13 -15
  12. package/dist/{main-JEXBLP2A.mjs → main-DULD8-th.mjs} +8 -6
  13. package/dist/{main-C6S3EsJ3.mjs → main-ZW5RaiOK.mjs} +77 -64
  14. package/dist/{main-g33jpia4.mjs → main-g1KEcXXp.mjs} +676 -582
  15. package/dist/{main-24J4U8yP.mjs → main-mGIT8jv4.mjs} +4 -4
  16. package/dist/schedule-or-dispatch-Cd7GLfc3.mjs +52 -0
  17. package/dist/{src-9LXITcNi.mjs → src-BE_EUpZ6.mjs} +93 -93
  18. package/package.json +1 -1
  19. package/scripts/create-project.mjs +6 -2
  20. package/src/client.ts +3 -3
  21. package/src/config/index.ts +1 -0
  22. package/src/contracts/common.ts +2 -0
  23. package/src/contracts/envelope.ts +1 -1
  24. package/src/contracts/events/notification-created.ts +2 -0
  25. package/src/contracts/events/notification-delivered.ts +1 -1
  26. package/src/contracts/events/notification-failed.ts +1 -1
  27. package/src/contracts/events/notification-requested.ts +3 -2
  28. package/src/contracts/helpers.ts +1 -2
  29. package/src/contracts/index.ts +1 -0
  30. package/src/contracts/registry.ts +1 -0
  31. package/src/contracts/sdk.ts +8 -37
  32. package/src/db/schema.ts +8 -13
  33. package/src/idempotency/index.ts +6 -8
  34. package/src/metrics/index.ts +2 -0
  35. package/src/rate-limiter/api-rate-limiter.ts +208 -0
  36. package/src/rate-limiter/index.ts +3 -411
  37. package/src/rate-limiter/project-settings-cache.ts +99 -0
  38. package/src/rate-limiter/user-throttle.ts +100 -0
  39. package/src/redis/index.ts +10 -3
  40. package/src/repositories/admin-user.ts +83 -0
  41. package/src/repositories/contact.ts +359 -0
  42. package/src/repositories/index.ts +9 -1416
  43. package/src/repositories/preference.ts +35 -0
  44. package/src/repositories/project.ts +110 -0
  45. package/src/repositories/segment.ts +16 -0
  46. package/src/repositories/template.ts +72 -0
  47. package/src/repositories/types.ts +49 -0
  48. package/src/repositories/user.ts +644 -0
  49. package/src/repositories/workflow.ts +64 -0
  50. package/src/services/ai/main.ts +29 -71
  51. package/src/services/api/admin-static.ts +4 -1
  52. package/src/services/api/handlers/auth.ts +109 -0
  53. package/src/services/api/handlers/campaigns.ts +191 -0
  54. package/src/services/api/handlers/events.ts +99 -0
  55. package/src/services/api/handlers/notifications.ts +356 -0
  56. package/src/services/api/handlers/projects.ts +116 -0
  57. package/src/services/api/handlers/shared.ts +262 -0
  58. package/src/services/api/handlers/suppressions.ts +95 -0
  59. package/src/services/api/handlers/system.ts +314 -0
  60. package/src/services/api/handlers/templates.ts +97 -0
  61. package/src/services/api/handlers/unsubscribe.ts +215 -0
  62. package/src/services/api/handlers/users.ts +262 -0
  63. package/src/services/api/handlers/workflows.ts +150 -0
  64. package/src/services/api/handlers.ts +25 -2190
  65. package/src/services/api/health.ts +133 -0
  66. package/src/services/api/http.ts +12 -0
  67. package/src/services/api/main.ts +32 -322
  68. package/src/services/api/router.ts +1 -1
  69. package/src/services/api/webhooks.ts +172 -0
  70. package/src/services/delivery/main.ts +256 -498
  71. package/src/services/engine/main.ts +31 -247
  72. package/src/services/engine/quiet-hours.ts +151 -0
  73. package/src/services/engine/schedule-or-dispatch.ts +104 -0
  74. package/src/services/enricher/main.ts +9 -21
  75. package/src/services/events/main.ts +2 -2
  76. package/src/services/scheduler/main.ts +16 -25
  77. package/src/services/workflow/duration.ts +24 -0
  78. package/src/services/workflow/main.ts +6 -164
  79. package/src/services/workflow/timers.ts +157 -0
  80. package/src/shared/events.ts +3 -2
  81. package/src/templates/render.ts +1 -0
  82. package/src/workers/index.ts +7 -4
  83. package/src/workflows/sdk.ts +1 -0
  84. package/dashboard/dist/assets/index-ggpJKZqZ.js +0 -315
  85. package/dashboard/dist/assets/index-s-SXGIwv.css +0 -1
@@ -1,4 +1,4 @@
1
- import { loadEnv, parseConfig, baseConfigSchema } from "@/index.js";
1
+ import { loadEnv, readBaseConfig } from "@/index.js";
2
2
  import type { Logger } from "@/index.js";
3
3
  import type { RedisClient, Redis } from "@/index.js";
4
4
  import {
@@ -39,14 +39,10 @@ import { publishDeliveryEvents } from "./live-events.js";
39
39
  import { ProviderThrottle } from "./throttle.js";
40
40
  import { metrics } from "@/metrics/index.js";
41
41
 
42
- // ─── App-specific config ────────────────────────────────────────────────────
43
-
44
- const deliveryConfigSchema = baseConfigSchema.extend({});
45
-
46
42
  // ─── Bootstrap ─────────────────────────────────────────────────────────────
47
43
 
48
44
  loadEnv();
49
- const config = parseConfig(deliveryConfigSchema, process.env);
45
+ const config = readBaseConfig();
50
46
 
51
47
  let logger: Logger;
52
48
  let redis: RedisClient;
@@ -62,17 +58,6 @@ let healthInterval: NodeJS.Timeout | null = null;
62
58
  let subscriber: Redis | null = null;
63
59
  let stopLiveEvents: (() => void) | null = null;
64
60
 
65
- (global as any)._telemetry = (global as any)._telemetry || {
66
- count: 0,
67
- insert: 0,
68
- provider: 0,
69
- flush: 0,
70
- ack: 0,
71
- dequeue: 0,
72
- dbupdate: 0,
73
- flushCount: 0,
74
- };
75
-
76
61
  /** How long a send may run before its dispatched log entry is published alone. */
77
62
  const DISPATCH_LOG_DELAY_MS = 1_000;
78
63
 
@@ -188,13 +173,9 @@ export class DeliveryWorker extends BaseWorker {
188
173
  providerMessageId: update.providerMessageId,
189
174
  }));
190
175
 
191
- const tDbUpdateStart = Date.now();
192
176
  await upsertOutboxProviderIds(this.db, values).catch((e: any) =>
193
177
  this.logger.error({ err: e }, "background update failed"),
194
178
  );
195
-
196
- (global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
197
- (global as any)._telemetry.flushCount++;
198
179
  return updates.map(() => undefined as void);
199
180
  },
200
181
  flushConcurrency,
@@ -312,6 +293,95 @@ export class DeliveryWorker extends BaseWorker {
312
293
  // and the transport's abort signal were attached to it.
313
294
  const original: DispatchedTaskPayload = { ...task };
314
295
 
296
+ // The attempt's `dispatched` log entry normally rides on its outcome event,
297
+ // one stream entry instead of two. A send still running after
298
+ // DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
299
+ // visible while it is in flight, as it always was.
300
+ let dispatchStarted = false;
301
+ let dispatchLogged = false;
302
+ let dispatchedAt = "";
303
+ let dispatchLogTimer: NodeJS.Timeout | undefined;
304
+ const logDispatched = (): Promise<void> | undefined => {
305
+ clearTimeout(dispatchLogTimer);
306
+ if (!dispatchStarted || dispatchLogged) return undefined;
307
+ dispatchLogged = true;
308
+ const logged = this.eventProcessor.add(
309
+ buildStreamEvent(
310
+ "notification.dispatched",
311
+ {
312
+ projectId: task.projectId,
313
+ taskId: task.taskId,
314
+ enrichedEventId: task.enrichedEventId,
315
+ recipientId: task.recipientId,
316
+ channel: task.channel,
317
+ templateId: task.templateId,
318
+ attempt: attemptNo,
319
+ workflowInstanceId: task.workflowInstanceId,
320
+ campaignId: task.campaignId,
321
+ },
322
+ "delivery",
323
+ event.metadata.traceId,
324
+ ),
325
+ );
326
+ publishPromises.push(logged);
327
+ return logged;
328
+ };
329
+ /** Outcome payload, carrying the dispatched entry if it has not gone out. */
330
+ const outcome = <T extends Record<string, unknown>>(payload: T): T => {
331
+ clearTimeout(dispatchLogTimer);
332
+ if (dispatchLogged) return payload;
333
+ dispatchLogged = true;
334
+ return { ...payload, attempt: attemptNo, dispatchedAt };
335
+ };
336
+ /**
337
+ * Queues the task's `notification.failed` event; once sending has started it
338
+ * is the attempt's outcome. Only the non-push provider failure leaves out
339
+ * `recipientId`.
340
+ */
341
+ const failed = (failureReason: string, failureCode: string, withRecipientId = true) => {
342
+ const payload = {
343
+ projectId: task.projectId,
344
+ taskId: task.taskId,
345
+ enrichedEventId: task.enrichedEventId,
346
+ ...(withRecipientId ? { recipientId: task.recipientId } : {}),
347
+ channel: task.channel,
348
+ failureReason,
349
+ failureCode,
350
+ retryable: false,
351
+ attempt: attemptNo,
352
+ templateId: task.templateId,
353
+ workflowInstanceId: task.workflowInstanceId,
354
+ campaignId: task.campaignId,
355
+ };
356
+ return this.eventProcessor.add(
357
+ buildStreamEvent(
358
+ "notification.failed",
359
+ dispatchStarted ? outcome(payload) : payload,
360
+ "delivery",
361
+ event.metadata.traceId,
362
+ ),
363
+ );
364
+ };
365
+ const delivered = (providerMessageId: string) =>
366
+ this.eventProcessor.add(
367
+ buildStreamEvent(
368
+ "notification.delivered",
369
+ outcome({
370
+ projectId: task.projectId,
371
+ taskId: task.taskId,
372
+ enrichedEventId: task.enrichedEventId,
373
+ channel: task.channel,
374
+ deliveredAt: new Date().toISOString(),
375
+ providerMessageId,
376
+ templateId: task.templateId,
377
+ workflowInstanceId: task.workflowInstanceId,
378
+ campaignId: task.campaignId,
379
+ }),
380
+ "delivery",
381
+ event.metadata.traceId,
382
+ ),
383
+ );
384
+
315
385
  const fallbackToNextChannel = async (reason: string) => {
316
386
  if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
317
387
  const nextChannel = task.fallbackChain[0];
@@ -371,27 +441,7 @@ export class DeliveryWorker extends BaseWorker {
371
441
  task.channel,
372
442
  task.projectId,
373
443
  );
374
- await this.eventProcessor.add(
375
- buildStreamEvent(
376
- "notification.failed",
377
- {
378
- projectId: task.projectId,
379
- taskId: task.taskId,
380
- enrichedEventId: task.enrichedEventId,
381
- recipientId: task.recipientId,
382
- channel: task.channel,
383
- failureReason: "no transport registered for channel",
384
- failureCode: "no_transport",
385
- retryable: false,
386
- attempt: attemptNo,
387
- templateId: task.templateId,
388
- workflowInstanceId: task.workflowInstanceId,
389
- campaignId: task.campaignId,
390
- },
391
- "delivery",
392
- event.metadata.traceId,
393
- ),
394
- );
444
+ await failed("no transport registered for channel", "no_transport");
395
445
  }
396
446
  return;
397
447
  }
@@ -427,27 +477,7 @@ export class DeliveryWorker extends BaseWorker {
427
477
  task.channel,
428
478
  task.projectId,
429
479
  );
430
- await this.eventProcessor.add(
431
- buildStreamEvent(
432
- "notification.failed",
433
- {
434
- projectId: task.projectId,
435
- taskId: task.taskId,
436
- enrichedEventId: task.enrichedEventId,
437
- recipientId: task.recipientId,
438
- channel: task.channel,
439
- failureReason: "provider rate limit max attempts exceeded",
440
- failureCode: "provider_throttle_exceeded",
441
- retryable: false,
442
- attempt: attemptNo,
443
- templateId: task.templateId,
444
- workflowInstanceId: task.workflowInstanceId,
445
- campaignId: task.campaignId,
446
- },
447
- "delivery",
448
- event.metadata.traceId,
449
- ),
450
- );
480
+ await failed("provider rate limit max attempts exceeded", "provider_throttle_exceeded");
451
481
  }
452
482
  return;
453
483
  }
@@ -492,47 +522,6 @@ export class DeliveryWorker extends BaseWorker {
492
522
  const tDispatchStart = Date.now();
493
523
  const MAX_DISPATCH_BUDGET_MS = 25_000; // Strictly shorter than lock lease (30s)
494
524
 
495
- // The attempt's `dispatched` log entry normally rides on its outcome event,
496
- // one stream entry instead of two. A send still running after
497
- // DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
498
- // visible while it is in flight, as it always was.
499
- let dispatchStarted = false;
500
- let dispatchLogged = false;
501
- let dispatchedAt = "";
502
- let dispatchLogTimer: NodeJS.Timeout | undefined;
503
- const logDispatched = (): Promise<void> | undefined => {
504
- clearTimeout(dispatchLogTimer);
505
- if (!dispatchStarted || dispatchLogged) return undefined;
506
- dispatchLogged = true;
507
- const logged = this.eventProcessor.add(
508
- buildStreamEvent(
509
- "notification.dispatched",
510
- {
511
- projectId: task.projectId,
512
- taskId: task.taskId,
513
- enrichedEventId: task.enrichedEventId,
514
- recipientId: task.recipientId,
515
- channel: task.channel,
516
- templateId: task.templateId,
517
- attempt: attemptNo,
518
- workflowInstanceId: task.workflowInstanceId,
519
- campaignId: task.campaignId,
520
- },
521
- "delivery",
522
- event.metadata.traceId,
523
- ),
524
- );
525
- publishPromises.push(logged);
526
- return logged;
527
- };
528
- /** Outcome payload, carrying the dispatched entry if it has not gone out. */
529
- const outcome = <T extends Record<string, unknown>>(payload: T): T => {
530
- clearTimeout(dispatchLogTimer);
531
- if (dispatchLogged) return payload;
532
- dispatchLogged = true;
533
- return { ...payload, attempt: attemptNo, dispatchedAt };
534
- };
535
-
536
525
  let retryScheduled = false;
537
526
  /**
538
527
  * Re-queues a send the provider turned away, unless it said the failure is
@@ -577,383 +566,183 @@ export class DeliveryWorker extends BaseWorker {
577
566
  void logDispatched();
578
567
  }, DISPATCH_LOG_DELAY_MS);
579
568
 
580
- // For push: we just need to send to the pre-resolved destination,
581
- // but if it's invalid, we deactivate it.
582
- if (task.channel === "push") {
583
- let lastResult: any = { success: false, error: "No transports" };
584
-
585
- for (let i = startAt; i < transports.length; i++) {
586
- const transport = transports[i];
587
- // The starting provider's slot was taken above. Failing over to a
588
- // later one spends that provider's own budget, or skips it if spent.
589
- if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
590
- continue;
591
- }
592
- const elapsedMs = Date.now() - tDispatchStart;
593
- const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
594
- if (remainingBudgetMs <= 0) {
595
- this.logger.warn(
596
- { taskId: task.taskId, channel: task.channel },
597
- "dispatch budget exhausted before transport call — aborting to protect lock lease",
598
- );
599
- break;
600
- }
601
- const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
602
- const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
603
-
604
- try {
605
- const tProv = Date.now();
606
- const breaker = this.getBreaker(providerKey(task.channel, transports, i));
607
-
608
- lastResult = await breaker.execute(async () => {
609
- const controller = new AbortController();
610
- const timeout = setTimeout(() => {
611
- controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
612
- }, timeoutMs);
613
-
614
- try {
615
- (task as any).signal = controller.signal;
616
- const res: any = await Promise.race([
617
- transport.send(task),
618
- new Promise((_, reject) => {
619
- if (controller.signal.aborted) return reject(controller.signal.reason);
620
- controller.signal.addEventListener("abort", () =>
621
- reject(controller.signal.reason),
622
- );
623
- }),
624
- ]);
625
- // A permanent rejection is the provider working: returned, not
626
- // thrown, so the breaker does not count it.
627
- if (!res.success && !res.invalidToken && res.retryable !== false) {
628
- throw providerFailure(res);
629
- }
630
- return res;
631
- } finally {
632
- clearTimeout(timeout);
633
- }
634
- });
635
-
636
- (global as any)._telemetry.provider += Date.now() - tProv;
637
- if (lastResult.success || lastResult.invalidToken) break;
638
- } catch (err: any) {
639
- lastResult = { success: false, error: err.message, retryable: err.retryable };
640
- }
569
+ // Push alone treats a rejected token as an answer: the token is
570
+ // deactivated rather than the next provider tried.
571
+ const isPush = task.channel === "push";
572
+ let result: any = { success: false, error: "No transports" };
573
+ for (let i = startAt; i < transports.length; i++) {
574
+ const transport = transports[i];
575
+ // The starting provider's slot was taken above. Failing over to a
576
+ // later one spends that provider's own budget, or skips it if spent.
577
+ if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
578
+ continue;
641
579
  }
642
-
643
- if (lastResult.invalidToken) {
644
- publishPromises.push(
645
- this.outboxUpdateProcessor.add({
646
- taskId: task.taskId,
647
- channel: task.channel,
648
- destination: task.destination,
649
- providerMessageId: null,
650
- }),
651
- );
652
- await this.contactRepo.deactivate(
653
- task.projectId,
654
- task.recipientId,
655
- task.channel as any,
656
- task.destination,
657
- );
658
- this.logger.info(
659
- { channel: task.channel, target: task.destination },
660
- "deactivated invalid contact target",
661
- );
662
- this.globalEmitter.emit(
663
- "delivery:failed",
664
- task.taskId,
665
- "invalidToken",
666
- task.channel,
667
- task.projectId,
580
+ const elapsedMs = Date.now() - tDispatchStart;
581
+ const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
582
+ if (remainingBudgetMs <= 0) {
583
+ this.logger.warn(
584
+ { taskId: task.taskId, channel: task.channel },
585
+ "dispatch budget exhausted before transport call — aborting to protect lock lease",
668
586
  );
669
- metrics.deliveryFailed.inc({ channel: task.channel, reason: "invalid_token" });
670
-
671
- publishPromises.push(
672
- this.eventProcessor.add(
673
- buildStreamEvent(
674
- "notification.failed",
675
- outcome({
676
- projectId: task.projectId,
677
- taskId: task.taskId,
678
- enrichedEventId: task.enrichedEventId,
679
- recipientId: task.recipientId,
680
- channel: task.channel,
681
- failureReason: "invalid_token",
682
- failureCode: "push_failure",
683
- retryable: false,
684
- attempt: attemptNo,
685
- templateId: task.templateId,
686
- workflowInstanceId: task.workflowInstanceId,
687
- campaignId: task.campaignId,
587
+ break;
588
+ }
589
+ const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
590
+ const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
591
+
592
+ try {
593
+ const breaker = this.getBreaker(providerKey(task.channel, transports, i));
594
+
595
+ result = await breaker.execute(async () => {
596
+ const controller = new AbortController();
597
+ const timeout = setTimeout(() => {
598
+ controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
599
+ }, timeoutMs);
600
+
601
+ try {
602
+ (task as any).signal = controller.signal;
603
+ const res: any = await Promise.race([
604
+ transport.send(task),
605
+ new Promise((_, reject) => {
606
+ if (controller.signal.aborted) return reject(controller.signal.reason);
607
+ controller.signal.addEventListener("abort", () =>
608
+ reject(controller.signal.reason),
609
+ );
688
610
  }),
689
- "delivery",
690
- event.metadata.traceId,
691
- ),
692
- ),
693
- );
611
+ ]);
612
+ // A permanent rejection (a bad address) is the provider working:
613
+ // returned, not thrown, so the breaker does not count it. A run
614
+ // of them in one list used to open it for every project.
615
+ if (!res.success && !(isPush && res.invalidToken) && res.retryable !== false) {
616
+ throw providerFailure(res);
617
+ }
618
+ return res;
619
+ } finally {
620
+ clearTimeout(timeout);
621
+ }
622
+ });
623
+
624
+ if (result.success || (isPush && result.invalidToken)) break;
625
+ } catch (err: any) {
626
+ result = { success: false, error: err.message, retryable: err.retryable };
627
+ }
628
+ }
694
629
 
695
- const fallbackTriggered = await fallbackToNextChannel("invalid_token");
696
- if (!fallbackTriggered) {
697
- throw new NonRetryableError("Push delivery failed: invalid token");
698
- }
699
- } else if (lastResult.success) {
700
- // Record providerMessageId in outbox before publishing event
701
- const providerMessageId = lastResult.providerMessageId || "push-success";
702
-
703
- publishPromises.push(
704
- this.outboxUpdateProcessor.add({
705
- taskId: task.taskId,
706
- channel: task.channel,
707
- destination: task.destination,
708
- providerMessageId,
709
- }),
710
- );
630
+ if (isPush && result.invalidToken) {
631
+ publishPromises.push(
632
+ this.outboxUpdateProcessor.add({
633
+ taskId: task.taskId,
634
+ channel: task.channel,
635
+ destination: task.destination,
636
+ providerMessageId: null,
637
+ }),
638
+ );
639
+ await this.contactRepo.deactivate(
640
+ task.projectId,
641
+ task.recipientId,
642
+ task.channel as any,
643
+ task.destination,
644
+ );
645
+ this.logger.info(
646
+ { channel: task.channel, target: task.destination },
647
+ "deactivated invalid contact target",
648
+ );
649
+ this.globalEmitter.emit(
650
+ "delivery:failed",
651
+ task.taskId,
652
+ "invalidToken",
653
+ task.channel,
654
+ task.projectId,
655
+ );
656
+ metrics.deliveryFailed.inc({ channel: task.channel, reason: "invalid_token" });
657
+
658
+ publishPromises.push(failed("invalid_token", "push_failure"));
659
+
660
+ const fallbackTriggered = await fallbackToNextChannel("invalid_token");
661
+ if (!fallbackTriggered) {
662
+ throw new NonRetryableError("Push delivery failed: invalid token");
663
+ }
664
+ } else if (result.success) {
665
+ // Record providerMessageId in outbox before publishing event
666
+ const providerMessageId = result.providerMessageId || (isPush ? "push-success" : "success");
711
667
 
668
+ publishPromises.push(
669
+ this.outboxUpdateProcessor.add({
670
+ taskId: task.taskId,
671
+ channel: task.channel,
672
+ destination: task.destination,
673
+ providerMessageId,
674
+ }),
675
+ );
676
+
677
+ if (isPush) {
712
678
  this.logger.debug(
713
679
  { taskId: task.taskId, messageId: providerMessageId },
714
680
  "push delivered",
715
681
  );
716
- this.globalEmitter.emit(
717
- "delivery:delivered",
718
- task.taskId,
719
- providerMessageId,
720
- task.channel,
721
- task.projectId,
722
- );
723
- metrics.deliverySuccess.inc({ channel: task.channel });
724
-
725
- publishPromises.push(
726
- this.eventProcessor.add(
727
- buildStreamEvent(
728
- "notification.delivered",
729
- outcome({
730
- projectId: task.projectId,
731
- taskId: task.taskId,
732
- enrichedEventId: task.enrichedEventId,
733
- channel: task.channel,
734
- deliveredAt: new Date().toISOString(),
735
- providerMessageId,
736
- templateId: task.templateId,
737
- workflowInstanceId: task.workflowInstanceId,
738
- campaignId: task.campaignId,
739
- }),
740
- "delivery",
741
- event.metadata.traceId,
742
- ),
743
- ),
744
- );
745
- } else if (!(await retryLater(lastResult))) {
746
- this.logger.warn(
747
- { taskId: task.taskId, error: lastResult.error },
748
- "push delivery failed",
749
- );
750
- this.globalEmitter.emit(
751
- "delivery:failed",
752
- task.taskId,
753
- lastResult.error ?? "push delivery failed",
754
- task.channel,
755
- task.projectId,
756
- );
757
- metrics.deliveryFailed.inc({ channel: task.channel, reason: "push_error" });
758
-
759
- publishPromises.push(
760
- this.outboxUpdateProcessor.add({
761
- taskId: task.taskId,
762
- channel: task.channel,
763
- destination: task.destination,
764
- providerMessageId: null,
765
- }),
766
- );
682
+ }
683
+ this.globalEmitter.emit(
684
+ "delivery:delivered",
685
+ task.taskId,
686
+ providerMessageId,
687
+ task.channel,
688
+ task.projectId,
689
+ );
690
+ metrics.deliverySuccess.inc({ channel: task.channel });
767
691
 
768
- publishPromises.push(
769
- this.eventProcessor.add(
770
- buildStreamEvent(
771
- "notification.failed",
772
- outcome({
773
- projectId: task.projectId,
774
- taskId: task.taskId,
775
- enrichedEventId: task.enrichedEventId,
776
- recipientId: task.recipientId,
777
- channel: task.channel,
778
- failureReason: lastResult.error ?? "push delivery failed",
779
- failureCode: "push_failure",
780
- retryable: false,
781
- attempt: attemptNo,
782
- templateId: task.templateId,
783
- workflowInstanceId: task.workflowInstanceId,
784
- campaignId: task.campaignId,
785
- }),
786
- "delivery",
787
- event.metadata.traceId,
788
- ),
789
- ),
790
- );
692
+ publishPromises.push(delivered(providerMessageId));
791
693
 
792
- const fallbackTriggered = await fallbackToNextChannel("push_delivery_failed");
793
- if (!fallbackTriggered) {
794
- throw new NonRetryableError(lastResult.error ?? "Push delivery failed");
795
- }
694
+ if (!isPush) {
695
+ this.logger.debug(
696
+ { taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
697
+ "notification delivered",
698
+ );
796
699
  }
797
- } else {
798
- // Other channels (email, sms, webhook, whatsapp) use the pre-resolved destination
799
- let result: any = { success: false, error: "No transports" };
800
- for (let i = startAt; i < transports.length; i++) {
801
- const transport = transports[i];
802
- // The starting provider's slot was taken above. Failing over to a
803
- // later one spends that provider's own budget, or skips it if spent.
804
- if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
805
- continue;
806
- }
807
- const elapsedMs = Date.now() - tDispatchStart;
808
- const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
809
- if (remainingBudgetMs <= 0) {
700
+ } else if (!(await retryLater(result))) {
701
+ const error =
702
+ result.error ?? (isPush ? "push delivery failed" : "delivery failed completely");
703
+ // Push reports the failure before queueing its outcome, other channels after.
704
+ const report = () => {
705
+ if (isPush) {
706
+ this.logger.warn({ taskId: task.taskId, error: result.error }, "push delivery failed");
707
+ } else {
810
708
  this.logger.warn(
811
- { taskId: task.taskId, channel: task.channel },
812
- "dispatch budget exhausted before transport call — aborting to protect lock lease",
709
+ { taskId: task.taskId, channel: task.channel, error: result.error },
710
+ "delivery failed completely across providers",
813
711
  );
814
- break;
815
- }
816
- const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
817
- const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
818
-
819
- try {
820
- const tProv = Date.now();
821
- const breaker = this.getBreaker(providerKey(task.channel, transports, i));
822
-
823
- result = await breaker.execute(async () => {
824
- const controller = new AbortController();
825
- const timeout = setTimeout(() => {
826
- controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
827
- }, timeoutMs);
828
-
829
- try {
830
- (task as any).signal = controller.signal;
831
- const res: any = await Promise.race([
832
- transport.send(task),
833
- new Promise((_, reject) => {
834
- if (controller.signal.aborted) return reject(controller.signal.reason);
835
- controller.signal.addEventListener("abort", () =>
836
- reject(controller.signal.reason),
837
- );
838
- }),
839
- ]);
840
- // A permanent rejection (a bad address) is the provider working:
841
- // returned, not thrown, so the breaker does not count it. A run
842
- // of them in one list used to open it for every project.
843
- if (!res.success && res.retryable !== false) {
844
- throw providerFailure(res);
845
- }
846
- return res;
847
- } finally {
848
- clearTimeout(timeout);
849
- }
850
- });
851
-
852
- (global as any)._telemetry.provider += Date.now() - tProv;
853
- if (result.success) break;
854
- } catch (err: any) {
855
- result = { success: false, error: err.message, retryable: err.retryable };
856
712
  }
857
- }
858
-
859
- if (result.success) {
860
- // Record providerMessageId in outbox before publishing event
861
- const providerMessageId = result.providerMessageId || "success";
862
-
863
- publishPromises.push(
864
- this.outboxUpdateProcessor.add({
865
- taskId: task.taskId,
866
- channel: task.channel,
867
- destination: task.destination,
868
- providerMessageId,
869
- }),
870
- );
871
-
872
- this.globalEmitter.emit(
873
- "delivery:delivered",
874
- task.taskId,
875
- providerMessageId,
876
- task.channel,
877
- task.projectId,
878
- );
879
- metrics.deliverySuccess.inc({ channel: task.channel });
880
-
881
- publishPromises.push(
882
- this.eventProcessor.add(
883
- buildStreamEvent(
884
- "notification.delivered",
885
- outcome({
886
- projectId: task.projectId,
887
- taskId: task.taskId,
888
- enrichedEventId: task.enrichedEventId,
889
- channel: task.channel,
890
- deliveredAt: new Date().toISOString(),
891
- providerMessageId,
892
- templateId: task.templateId,
893
- workflowInstanceId: task.workflowInstanceId,
894
- campaignId: task.campaignId,
895
- }),
896
- "delivery",
897
- event.metadata.traceId,
898
- ),
899
- ),
900
- );
901
- } else if (!(await retryLater(result))) {
902
- publishPromises.push(
903
- this.outboxUpdateProcessor.add({
904
- taskId: task.taskId,
905
- channel: task.channel,
906
- destination: task.destination,
907
- providerMessageId: null,
908
- }),
909
- );
910
-
911
- publishPromises.push(
912
- this.eventProcessor.add(
913
- buildStreamEvent(
914
- "notification.failed",
915
- outcome({
916
- projectId: task.projectId,
917
- taskId: task.taskId,
918
- enrichedEventId: task.enrichedEventId,
919
- channel: task.channel,
920
- failureReason: result.error ?? "delivery failed completely",
921
- failureCode: "provider_error",
922
- retryable: false,
923
- attempt: attemptNo,
924
- templateId: task.templateId,
925
- workflowInstanceId: task.workflowInstanceId,
926
- campaignId: task.campaignId,
927
- }),
928
- "delivery",
929
- event.metadata.traceId,
930
- ),
931
- ),
932
- );
933
- }
934
-
935
- if (!result.success && !retryScheduled) {
936
- this.logger.warn(
937
- { taskId: task.taskId, channel: task.channel, error: result.error },
938
- "delivery failed completely across providers",
939
- );
940
713
  this.globalEmitter.emit(
941
714
  "delivery:failed",
942
715
  task.taskId,
943
- result.error ?? "delivery failed completely",
716
+ error,
944
717
  task.channel,
945
718
  task.projectId,
946
719
  );
947
- metrics.deliveryFailed.inc({ channel: task.channel, reason: "provider_error" });
720
+ metrics.deliveryFailed.inc({
721
+ channel: task.channel,
722
+ reason: isPush ? "push_error" : "provider_error",
723
+ });
724
+ };
948
725
 
949
- const fallbackTriggered = await fallbackToNextChannel("all_providers_failed");
950
- if (!fallbackTriggered) {
951
- throw new NonRetryableError(result.error ?? "delivery failed");
952
- }
953
- } else if (result.success) {
954
- this.logger.debug(
955
- { taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
956
- "notification delivered",
726
+ if (isPush) report();
727
+ publishPromises.push(
728
+ this.outboxUpdateProcessor.add({
729
+ taskId: task.taskId,
730
+ channel: task.channel,
731
+ destination: task.destination,
732
+ providerMessageId: null,
733
+ }),
734
+ );
735
+ publishPromises.push(
736
+ isPush ? failed(error, "push_failure") : failed(error, "provider_error", false),
737
+ );
738
+ if (!isPush) report();
739
+
740
+ const fallbackTriggered = await fallbackToNextChannel(
741
+ isPush ? "push_delivery_failed" : "all_providers_failed",
742
+ );
743
+ if (!fallbackTriggered) {
744
+ throw new NonRetryableError(
745
+ result.error ?? (isPush ? "Push delivery failed" : "delivery failed"),
957
746
  );
958
747
  }
959
748
  }
@@ -961,52 +750,29 @@ export class DeliveryWorker extends BaseWorker {
961
750
  // Every branch above publishes an outcome; this only guards one that doesn't.
962
751
  void logDispatched();
963
752
 
964
- const tFlushStart = Date.now();
965
753
  await Promise.all(publishPromises).catch((err) => {
966
754
  this.logger.error(
967
755
  { err, taskId: task.taskId },
968
756
  "failed to publish post-dispatch events, swallowing error to prevent duplicate delivery",
969
757
  );
970
758
  });
971
- const flushTime = Date.now() - tFlushStart;
972
759
 
973
760
  if (retryScheduled) {
974
761
  // Released so the re-queued attempt can take the lease.
975
- await this.idempotency.unmark(idempotencyKey).catch(() => {});
762
+ await this.idempotency.unmark(idempotencyKey).catch((err: any) => {
763
+ this.logger.warn({ err, taskId: task.taskId }, "failed to release idempotency lock");
764
+ });
976
765
  } else {
977
766
  void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
978
767
  this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
979
768
  });
980
769
  }
981
-
982
- const t = ((global as any)._telemetry = (global as any)._telemetry || {
983
- count: 0,
984
- insert: 0,
985
- provider: 0,
986
- flush: 0,
987
- ack: 0,
988
- });
989
- t.count++;
990
- t.insert += 0;
991
- t.flush += flushTime;
992
-
993
- if (t.count % 1000 === 0) {
994
- this.logger.debug(
995
- `[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1000}ms, DB Insert: ${t.insert / 1000}ms, Provider: ${t.provider / 1000}ms, Wait for Flush: ${t.flush / 1000}ms, Ack: ${t.ack / 1000}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`,
996
- );
997
- t.count = 0;
998
- t.insert = 0;
999
- t.provider = 0;
1000
- t.flush = 0;
1001
- t.dequeue = 0;
1002
- t.ack = 0;
1003
- t.dbupdate = 0;
1004
- t.flushCount = 0;
1005
- }
1006
770
  } catch (err) {
1007
771
  // The attempt happened even though it did not finish; keep it in the log.
1008
772
  logDispatched()?.catch(() => {});
1009
- await this.idempotency.unmark(idempotencyKey).catch(() => {});
773
+ await this.idempotency.unmark(idempotencyKey).catch((e: any) => {
774
+ this.logger.warn({ err: e, taskId: task.taskId }, "failed to release idempotency lock");
775
+ });
1010
776
  throw err;
1011
777
  }
1012
778
  }
@@ -1047,14 +813,6 @@ export async function startDeliveryWorker() {
1047
813
  ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
1048
814
  });
1049
815
 
1050
- // ─── Stage 3: Delivery Worker ───────────────────────────────────────────────
1051
- //
1052
- // Pipeline:
1053
- // 1. Parse payload as notification.dispatched
1054
- // 2. Resolve active device tokens from DB
1055
- // 3. Send via registered transport
1056
- // 4. Deactivate invalid tokens in DB
1057
-
1058
816
  const eventsProducer = new StreamProducer({
1059
817
  redis: redis.native,
1060
818
  stream: STREAMS.EVENTS_INBOUND,
@@ -1083,11 +841,11 @@ export async function startDeliveryWorker() {
1083
841
  await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
1084
842
  subscriber.on("message", (_channel: string, key: string) => templateCache.invalidateKey(key));
1085
843
 
1086
- // ─── Health check interval ──────────────────────────────────────────────────
1087
-
1088
844
  // Outcomes for the API's live event stream, which may be in another process.
1089
845
  stopLiveEvents = publishDeliveryEvents(redis.native, globalEmitter);
1090
846
 
847
+ // ─── Health check interval ──────────────────────────────────────────────────
848
+
1091
849
  healthInterval = startHealthReporter("delivery", worker, redis, logger);
1092
850
 
1093
851
  logger.info(