notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -0,0 +1,811 @@
1
+ import { $ as deliveryOutbox, B as ApiRateLimiter, C as ContactRepository, D as TemplateRepository, J as metrics, Jt as PUBSUB_CHANNELS, K as StreamProducer, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, X as IdempotencyGuard, Yt as STREAMS, _ as TemplateCache, c as BaseWorker, cn as loadEnv, d as startHealthReporter, en as registry, g as transportRegistry, ln as parseConfig, on as baseConfigSchema, qt as OUTBOUND_STREAMS, rt as scheduledPayloads, u as NonRetryableError, vn as CircuitBreaker, wn as globalEmitter, x as renderWithTemplate, xn as getPriorityBucket, yn as BatchProcessor } from "./src-CI7FGf3h.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-GRXzgPvW.mjs";
3
+ import { n as upsertOutboxProviderIds } from "./bulk-BvRfz6Of.mjs";
4
+ import { sql } from "drizzle-orm";
5
+ //#region src/services/delivery/throttle.ts
6
+ /**
7
+ * Provider rate limit enforced through locally leased tokens.
8
+ *
9
+ * A per-message check against one sorted set per channel would serialise every
10
+ * delivery process in the cluster on a single hot key. Instead each process
11
+ * leases a chunk of the window's budget and spends it in memory, contacting
12
+ * Redis only when its chunk runs low. The window is a sliding estimate over
13
+ * two fixed buckets rather than an exact log, the same trade the API ingress
14
+ * limiter already makes.
15
+ */
16
+ var ProviderThrottle = class {
17
+ redis;
18
+ logger;
19
+ limiters = /* @__PURE__ */ new Map();
20
+ constructor(redis, logger) {
21
+ this.redis = redis;
22
+ this.logger = logger;
23
+ }
24
+ async check(channel, config) {
25
+ const windowMs = Math.max(1, Math.round(config.windowSeconds * 1e3));
26
+ let limiter = this.limiters.get(windowMs);
27
+ if (!limiter) {
28
+ limiter = new ApiRateLimiter({
29
+ redis: this.redis,
30
+ windowMs,
31
+ keyPrefix: "rate-limit:provider",
32
+ logger: this.logger
33
+ });
34
+ this.limiters.set(windowMs, limiter);
35
+ }
36
+ if (await limiter.check(channel, config.limit)) return {
37
+ allowed: true,
38
+ retryAfterMs: 0
39
+ };
40
+ const retryAfterMs = windowMs - Date.now() % windowMs;
41
+ this.logger?.warn({
42
+ channel,
43
+ limit: config.limit,
44
+ windowSeconds: config.windowSeconds,
45
+ retryAfterMs
46
+ }, "Provider rate limit hit — task must be rescheduled");
47
+ return {
48
+ allowed: false,
49
+ retryAfterMs
50
+ };
51
+ }
52
+ };
53
+ //#endregion
54
+ //#region src/services/delivery/main.ts
55
+ const deliveryConfigSchema = baseConfigSchema.extend({});
56
+ loadEnv();
57
+ const config = parseConfig(deliveryConfigSchema, process.env);
58
+ let logger;
59
+ let redis;
60
+ let sql$1;
61
+ let db;
62
+ let consumer;
63
+ let pendingScanner;
64
+ let worker;
65
+ let scheduledProducer;
66
+ let enrichedProducers;
67
+ let healthInterval = null;
68
+ let subscriber = null;
69
+ global._telemetry = global._telemetry || {
70
+ count: 0,
71
+ insert: 0,
72
+ provider: 0,
73
+ flush: 0,
74
+ ack: 0,
75
+ dequeue: 0,
76
+ dbupdate: 0,
77
+ flushCount: 0
78
+ };
79
+ /** How long a send may run before its dispatched log entry is published alone. */
80
+ const DISPATCH_LOG_DELAY_MS = 1e3;
81
+ /**
82
+ * Wait before re-trying a send that failed at the provider, by prior failures.
83
+ * The first step matches the circuit breaker's reset timeout, so a retry of a
84
+ * task the open breaker turned away reaches the provider's recovery probe
85
+ * rather than the same closed door.
86
+ */
87
+ const RETRY_BACKOFF_MS = [
88
+ 3e4,
89
+ 12e4,
90
+ 6e5
91
+ ];
92
+ /**
93
+ * How many times a send may be put back for want of provider capacity before
94
+ * it is failed. Separate from `maxAttempts`, which counts sends the provider
95
+ * refused: waiting in line is not a failed attempt, and a campaign several
96
+ * windows deeper than the provider's limit has to be able to wait that long.
97
+ */
98
+ const MAX_THROTTLE_DEFERRALS = 50;
99
+ /**
100
+ * Names the provider at `transports[i]` for its rate limit and circuit breaker.
101
+ * The rate limit's budget lives in Redis and is shared by every delivery
102
+ * process, so the name must come out the same everywhere: the class name,
103
+ * numbered when a channel has two providers of one class.
104
+ */
105
+ function providerKey(channel, transports, i) {
106
+ const name = transports[i].constructor?.name ?? "provider";
107
+ const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
108
+ return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`;
109
+ }
110
+ /** A provider's refusal, keeping whether it said another attempt could work. */
111
+ function providerFailure(res) {
112
+ const err = new Error(res.error ?? "Transport failed");
113
+ err.retryable = res.retryable;
114
+ return err;
115
+ }
116
+ function retryDelayMs(priorFailures) {
117
+ return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)];
118
+ }
119
+ var DeliveryWorker = class extends BaseWorker {
120
+ transportRegistry;
121
+ idempotency;
122
+ scheduledProducer;
123
+ enrichedProducers;
124
+ contactRepo;
125
+ eventsProducer;
126
+ globalEmitter;
127
+ db;
128
+ eventProcessor;
129
+ outboxUpdateProcessor;
130
+ outboxInsertProcessor;
131
+ breakers = /* @__PURE__ */ new Map();
132
+ providerThrottle;
133
+ templateCache;
134
+ constructor(options) {
135
+ super(options);
136
+ this.transportRegistry = options.transportRegistry;
137
+ this.idempotency = options.idempotency;
138
+ this.scheduledProducer = options.scheduledProducer;
139
+ this.enrichedProducers = options.enrichedProducers;
140
+ this.contactRepo = options.contactRepo;
141
+ this.eventsProducer = options.eventsProducer;
142
+ this.globalEmitter = options.globalEmitter;
143
+ this.db = options.db;
144
+ this.providerThrottle = options.providerThrottle ?? new ProviderThrottle(options.redis, this.logger);
145
+ this.templateCache = options.templateCache;
146
+ const batchSize = Math.max(10, Math.min(options.concurrency ?? 50, 500));
147
+ const flushConcurrency = 4;
148
+ this.eventProcessor = new BatchProcessor(batchSize, 5, async (events) => {
149
+ await this.eventsProducer.publishBatch(events);
150
+ return events.map(() => void 0);
151
+ }, flushConcurrency);
152
+ this.outboxUpdateProcessor = new BatchProcessor(batchSize, 5, async (updates) => {
153
+ const values = updates.map((update) => ({
154
+ taskId: update.taskId,
155
+ channel: update.channel,
156
+ destination: update.destination,
157
+ providerMessageId: update.providerMessageId
158
+ }));
159
+ const tDbUpdateStart = Date.now();
160
+ await upsertOutboxProviderIds(this.db, values).catch((e) => this.logger.error({ err: e }, "background update failed"));
161
+ global._telemetry.dbupdate += Date.now() - tDbUpdateStart;
162
+ global._telemetry.flushCount++;
163
+ return updates.map(() => void 0);
164
+ }, flushConcurrency);
165
+ this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
166
+ const values = tasks.map((task) => ({
167
+ taskId: task.taskId,
168
+ channel: task.channel,
169
+ destination: task.destination
170
+ }));
171
+ await this.db.insert(deliveryOutbox).values(values).onConflictDoNothing();
172
+ return tasks.map(() => true);
173
+ });
174
+ }
175
+ async stop() {
176
+ await Promise.all([
177
+ this.eventProcessor.flush(),
178
+ this.outboxUpdateProcessor.flush(),
179
+ this.outboxInsertProcessor.flush()
180
+ ]);
181
+ await super.stop();
182
+ }
183
+ /**
184
+ * Renders a task the engine dispatched unrendered. Done here, after the
185
+ * throttle and the lease, so the body is built once per actual send and never
186
+ * travels through the outbound stream.
187
+ */
188
+ async render(task) {
189
+ if (!task.templateId) return renderWithTemplate(null, task.templateVariables);
190
+ const template = this.templateCache ? await this.templateCache.getCachedTemplate(task.projectId, task.templateId) : null;
191
+ if (!template) throw new NonRetryableError(`template "${task.templateId}" not found at send time`);
192
+ return renderWithTemplate(template, task.templateVariables);
193
+ }
194
+ /**
195
+ * Parks a task in Postgres and the scheduler until `sendAt`. The payload
196
+ * carries the send time, so the scheduler can rebuild it if Redis loses its
197
+ * data.
198
+ */
199
+ async reschedule(task, sendAt, reason) {
200
+ const scheduledAt = sendAt.toISOString();
201
+ await this.db.insert(scheduledPayloads).values({
202
+ taskId: task.taskId,
203
+ payload: {
204
+ ...task,
205
+ scheduledAt
206
+ }
207
+ }).onConflictDoUpdate({
208
+ target: scheduledPayloads.taskId,
209
+ set: { payload: sql`EXCLUDED.payload` }
210
+ });
211
+ await this.scheduledProducer.publish(buildStreamEvent("notification.scheduled", {
212
+ projectId: task.projectId,
213
+ enrichedEventId: task.enrichedEventId,
214
+ taskId: task.taskId,
215
+ scheduledAt
216
+ }, "delivery", `${task.taskId}:${reason}:${Date.now()}`));
217
+ }
218
+ /** Takes a slot from the provider's rate limit, if it has one. */
219
+ async providerGate(channel, transports, i) {
220
+ const limits = transports[i].limits;
221
+ if (!limits) return {
222
+ allowed: true,
223
+ retryAfterMs: 0
224
+ };
225
+ return this.providerThrottle.check(providerKey(channel, transports, i), limits);
226
+ }
227
+ getBreaker(name) {
228
+ let breaker = this.breakers.get(name);
229
+ if (!breaker) {
230
+ breaker = new CircuitBreaker({
231
+ failureThreshold: 5,
232
+ resetTimeoutMs: 3e4
233
+ });
234
+ this.breakers.set(name, breaker);
235
+ }
236
+ return breaker;
237
+ }
238
+ async process(message, attempt = 1) {
239
+ const { event } = message;
240
+ const publishPromises = [];
241
+ const payloadResult = registry.safeParsePayload("notification.dispatched", event.payload);
242
+ if (!payloadResult.success) {
243
+ this.logger.warn({
244
+ messageId: message.id,
245
+ issues: payloadResult.error.issues
246
+ }, "invalid notification.dispatched payload — skipping");
247
+ return;
248
+ }
249
+ const task = payloadResult.data;
250
+ const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
251
+ const original = { ...task };
252
+ const fallbackToNextChannel = async (reason) => {
253
+ if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
254
+ const nextChannel = task.fallbackChain[0];
255
+ const remainingChain = task.fallbackChain.slice(1);
256
+ const fallbackPayload = {
257
+ projectId: task.projectId,
258
+ rawEventId: task.enrichedEventId,
259
+ recipientId: task.recipientId,
260
+ channel: nextChannel,
261
+ priority: task.priority,
262
+ templateId: task.templateId,
263
+ templateVariables: task.templateVariables,
264
+ aiPrompts: task.aiPrompts,
265
+ recipient: task.recipient,
266
+ scheduledAt: void 0,
267
+ fallbackChain: remainingChain.length > 0 ? remainingChain : void 0,
268
+ campaignId: task.campaignId,
269
+ workflowInstanceId: task.workflowInstanceId,
270
+ throttleCounted: true
271
+ };
272
+ const p = getPriorityBucket(task.priority);
273
+ await (this.enrichedProducers[p] ?? this.enrichedProducers["normal"]).publish(buildStreamEvent("notification.enriched", fallbackPayload, "delivery", event.metadata.traceId));
274
+ this.logger.info({
275
+ taskId: task.taskId,
276
+ reason,
277
+ nextChannel,
278
+ traceId: event.metadata.traceId
279
+ }, "Delivery failed completely, rolling over to next channel in fallback chain");
280
+ return true;
281
+ }
282
+ return false;
283
+ };
284
+ const transports = this.transportRegistry.getAll(task.channel);
285
+ if (transports.length === 0) {
286
+ this.logger.warn({
287
+ taskId: task.taskId,
288
+ channel: task.channel
289
+ }, "no transport registered for channel — dropping");
290
+ if (!await fallbackToNextChannel("no_transport")) {
291
+ this.globalEmitter.emit("delivery:failed", task.taskId, "no transport", task.channel, task.projectId);
292
+ await this.eventProcessor.add(buildStreamEvent("notification.failed", {
293
+ projectId: task.projectId,
294
+ taskId: task.taskId,
295
+ enrichedEventId: task.enrichedEventId,
296
+ recipientId: task.recipientId,
297
+ channel: task.channel,
298
+ failureReason: "no transport registered for channel",
299
+ failureCode: "no_transport",
300
+ retryable: false,
301
+ attempt: attemptNo,
302
+ templateId: task.templateId,
303
+ workflowInstanceId: task.workflowInstanceId,
304
+ campaignId: task.campaignId
305
+ }, "delivery", event.metadata.traceId));
306
+ }
307
+ return;
308
+ }
309
+ let startAt = -1;
310
+ let retryAfterMs = Infinity;
311
+ let windowMs = Infinity;
312
+ for (let i = 0; i < transports.length; i++) {
313
+ const gate = await this.providerGate(task.channel, transports, i);
314
+ if (gate.allowed) {
315
+ startAt = i;
316
+ break;
317
+ }
318
+ retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs);
319
+ windowMs = Math.min(windowMs, (transports[i].limits?.windowSeconds ?? 0) * 1e3);
320
+ }
321
+ if (startAt === -1) {
322
+ task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
323
+ if (task.throttleAttemptCount > 50) {
324
+ this.logger.warn({
325
+ messageId: message.id,
326
+ taskId: task.taskId,
327
+ attempts: task.throttleAttemptCount
328
+ }, "provider rate limit max attempts exceeded");
329
+ if (!await fallbackToNextChannel("provider_throttle_exceeded")) {
330
+ this.globalEmitter.emit("delivery:failed", task.taskId, "provider throttle exceeded", task.channel, task.projectId);
331
+ await this.eventProcessor.add(buildStreamEvent("notification.failed", {
332
+ projectId: task.projectId,
333
+ taskId: task.taskId,
334
+ enrichedEventId: task.enrichedEventId,
335
+ recipientId: task.recipientId,
336
+ channel: task.channel,
337
+ failureReason: "provider rate limit max attempts exceeded",
338
+ failureCode: "provider_throttle_exceeded",
339
+ retryable: false,
340
+ attempt: attemptNo,
341
+ templateId: task.templateId,
342
+ workflowInstanceId: task.workflowInstanceId,
343
+ campaignId: task.campaignId
344
+ }, "delivery", event.metadata.traceId));
345
+ }
346
+ return;
347
+ }
348
+ const spreadMs = Number.isFinite(windowMs) ? Math.random() * windowMs * Math.min(task.throttleAttemptCount, 10) : 0;
349
+ await this.reschedule({
350
+ ...original,
351
+ throttleAttemptCount: task.throttleAttemptCount
352
+ }, new Date(Date.now() + retryAfterMs + spreadMs), "throttle");
353
+ return;
354
+ }
355
+ const idempotencyKey = task.taskId;
356
+ const lease = this.idempotency.acquireLease ? await this.idempotency.acquireLease(idempotencyKey, 60) : await this.idempotency.checkAndMark(idempotencyKey, 60) ? "acquired" : "locked";
357
+ if (lease === "completed") {
358
+ this.logger.info({
359
+ messageId: message.id,
360
+ taskId: task.taskId,
361
+ channel: task.channel
362
+ }, "duplicate delivery — already completed, acking message");
363
+ return;
364
+ }
365
+ if (lease === "locked") {
366
+ this.logger.info({
367
+ messageId: message.id,
368
+ taskId: task.taskId,
369
+ channel: task.channel,
370
+ attempt
371
+ }, "duplicate delivery — skipping and leaving pending in stream");
372
+ return { ack: false };
373
+ }
374
+ const tDispatchStart = Date.now();
375
+ const MAX_DISPATCH_BUDGET_MS = 25e3;
376
+ let dispatchStarted = false;
377
+ let dispatchLogged = false;
378
+ let dispatchedAt = "";
379
+ let dispatchLogTimer;
380
+ const logDispatched = () => {
381
+ clearTimeout(dispatchLogTimer);
382
+ if (!dispatchStarted || dispatchLogged) return void 0;
383
+ dispatchLogged = true;
384
+ const logged = this.eventProcessor.add(buildStreamEvent("notification.dispatched", {
385
+ projectId: task.projectId,
386
+ taskId: task.taskId,
387
+ enrichedEventId: task.enrichedEventId,
388
+ recipientId: task.recipientId,
389
+ channel: task.channel,
390
+ templateId: task.templateId,
391
+ attempt: attemptNo,
392
+ workflowInstanceId: task.workflowInstanceId,
393
+ campaignId: task.campaignId
394
+ }, "delivery", event.metadata.traceId));
395
+ publishPromises.push(logged);
396
+ return logged;
397
+ };
398
+ /** Outcome payload, carrying the dispatched entry if it has not gone out. */
399
+ const outcome = (payload) => {
400
+ clearTimeout(dispatchLogTimer);
401
+ if (dispatchLogged) return payload;
402
+ dispatchLogged = true;
403
+ return {
404
+ ...payload,
405
+ attempt: attemptNo,
406
+ dispatchedAt
407
+ };
408
+ };
409
+ let retryScheduled = false;
410
+ /**
411
+ * Re-queues a send the provider turned away, unless it said the failure is
412
+ * permanent or the task is out of attempts. An outage, a timeout or an open
413
+ * circuit is exactly what a later attempt can get past; failing the task
414
+ * outright turned every blip into lost mail. The attempt stays in the log as
415
+ * dispatched, and the task's outcome is whatever its last attempt comes to.
416
+ */
417
+ const retryLater = async (failure) => {
418
+ const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
419
+ if (failure.retryable === false || attemptNo >= maxAttempts) return false;
420
+ const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0);
421
+ await this.reschedule({
422
+ ...original,
423
+ deliveryAttemptCount: attemptNo
424
+ }, new Date(Date.now() + delayMs), "retry");
425
+ retryScheduled = true;
426
+ metrics.deliveryFailed.inc({
427
+ channel: task.channel,
428
+ reason: "retry_scheduled"
429
+ });
430
+ this.logger.warn({
431
+ taskId: task.taskId,
432
+ channel: task.channel,
433
+ attempt: attemptNo,
434
+ maxAttempts,
435
+ delayMs,
436
+ error: failure.error
437
+ }, "provider send failed — retrying later");
438
+ return true;
439
+ };
440
+ try {
441
+ if (!task.renderedContent) task.renderedContent = await this.render(task);
442
+ dispatchStarted = true;
443
+ dispatchedAt = (/* @__PURE__ */ new Date()).toISOString();
444
+ dispatchLogTimer = setTimeout(() => {
445
+ logDispatched();
446
+ }, DISPATCH_LOG_DELAY_MS);
447
+ if (task.channel === "push") {
448
+ let lastResult = {
449
+ success: false,
450
+ error: "No transports"
451
+ };
452
+ for (let i = startAt; i < transports.length; i++) {
453
+ const transport = transports[i];
454
+ if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) continue;
455
+ const elapsedMs = Date.now() - tDispatchStart;
456
+ const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
457
+ if (remainingBudgetMs <= 0) {
458
+ this.logger.warn({
459
+ taskId: task.taskId,
460
+ channel: task.channel
461
+ }, "dispatch budget exhausted before transport call — aborting to protect lock lease");
462
+ break;
463
+ }
464
+ const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
465
+ const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
466
+ try {
467
+ const tProv = Date.now();
468
+ lastResult = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
469
+ const controller = new AbortController();
470
+ const timeout = setTimeout(() => {
471
+ controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
472
+ }, timeoutMs);
473
+ try {
474
+ task.signal = controller.signal;
475
+ const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
476
+ if (controller.signal.aborted) return reject(controller.signal.reason);
477
+ controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
478
+ })]);
479
+ if (!res.success && !res.invalidToken) throw providerFailure(res);
480
+ return res;
481
+ } finally {
482
+ clearTimeout(timeout);
483
+ }
484
+ });
485
+ global._telemetry.provider += Date.now() - tProv;
486
+ if (lastResult.success || lastResult.invalidToken) break;
487
+ } catch (err) {
488
+ lastResult = {
489
+ success: false,
490
+ error: err.message,
491
+ retryable: err.retryable
492
+ };
493
+ }
494
+ }
495
+ if (lastResult.invalidToken) {
496
+ publishPromises.push(this.outboxUpdateProcessor.add({
497
+ taskId: task.taskId,
498
+ channel: task.channel,
499
+ destination: task.destination,
500
+ providerMessageId: null
501
+ }));
502
+ await this.contactRepo.deactivate(task.projectId, task.recipientId, task.channel, task.destination);
503
+ this.logger.info({
504
+ channel: task.channel,
505
+ target: task.destination
506
+ }, "deactivated invalid contact target");
507
+ this.globalEmitter.emit("delivery:failed", task.taskId, "invalidToken", task.channel, task.projectId);
508
+ metrics.deliveryFailed.inc({
509
+ channel: task.channel,
510
+ reason: "invalid_token"
511
+ });
512
+ publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
513
+ projectId: task.projectId,
514
+ taskId: task.taskId,
515
+ enrichedEventId: task.enrichedEventId,
516
+ recipientId: task.recipientId,
517
+ channel: task.channel,
518
+ failureReason: "invalid_token",
519
+ failureCode: "push_failure",
520
+ retryable: false,
521
+ attempt: attemptNo,
522
+ templateId: task.templateId,
523
+ workflowInstanceId: task.workflowInstanceId,
524
+ campaignId: task.campaignId
525
+ }), "delivery", event.metadata.traceId)));
526
+ if (!await fallbackToNextChannel("invalid_token")) throw new NonRetryableError("Push delivery failed: invalid token");
527
+ } else if (lastResult.success) {
528
+ const providerMessageId = lastResult.providerMessageId || "push-success";
529
+ publishPromises.push(this.outboxUpdateProcessor.add({
530
+ taskId: task.taskId,
531
+ channel: task.channel,
532
+ destination: task.destination,
533
+ providerMessageId
534
+ }));
535
+ this.logger.debug({
536
+ taskId: task.taskId,
537
+ messageId: providerMessageId
538
+ }, "push delivered");
539
+ this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
540
+ metrics.deliverySuccess.inc({ channel: task.channel });
541
+ publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
542
+ projectId: task.projectId,
543
+ taskId: task.taskId,
544
+ enrichedEventId: task.enrichedEventId,
545
+ channel: task.channel,
546
+ deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
547
+ providerMessageId,
548
+ templateId: task.templateId,
549
+ workflowInstanceId: task.workflowInstanceId,
550
+ campaignId: task.campaignId
551
+ }), "delivery", event.metadata.traceId)));
552
+ } else if (!await retryLater(lastResult)) {
553
+ this.logger.warn({
554
+ taskId: task.taskId,
555
+ error: lastResult.error
556
+ }, "push delivery failed");
557
+ this.globalEmitter.emit("delivery:failed", task.taskId, lastResult.error ?? "push delivery failed", task.channel, task.projectId);
558
+ metrics.deliveryFailed.inc({
559
+ channel: task.channel,
560
+ reason: "push_error"
561
+ });
562
+ publishPromises.push(this.outboxUpdateProcessor.add({
563
+ taskId: task.taskId,
564
+ channel: task.channel,
565
+ destination: task.destination,
566
+ providerMessageId: null
567
+ }));
568
+ publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
569
+ projectId: task.projectId,
570
+ taskId: task.taskId,
571
+ enrichedEventId: task.enrichedEventId,
572
+ recipientId: task.recipientId,
573
+ channel: task.channel,
574
+ failureReason: lastResult.error ?? "push delivery failed",
575
+ failureCode: "push_failure",
576
+ retryable: false,
577
+ attempt: attemptNo,
578
+ templateId: task.templateId,
579
+ workflowInstanceId: task.workflowInstanceId,
580
+ campaignId: task.campaignId
581
+ }), "delivery", event.metadata.traceId)));
582
+ if (!await fallbackToNextChannel("push_delivery_failed")) throw new NonRetryableError(lastResult.error ?? "Push delivery failed");
583
+ }
584
+ } else {
585
+ let result = {
586
+ success: false,
587
+ error: "No transports"
588
+ };
589
+ for (let i = startAt; i < transports.length; i++) {
590
+ const transport = transports[i];
591
+ if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) continue;
592
+ const elapsedMs = Date.now() - tDispatchStart;
593
+ const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
594
+ if (remainingBudgetMs <= 0) {
595
+ this.logger.warn({
596
+ taskId: task.taskId,
597
+ channel: task.channel
598
+ }, "dispatch budget exhausted before transport call — aborting to protect lock lease");
599
+ break;
600
+ }
601
+ const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
602
+ const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
603
+ try {
604
+ const tProv = Date.now();
605
+ result = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
606
+ const controller = new AbortController();
607
+ const timeout = setTimeout(() => {
608
+ controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
609
+ }, timeoutMs);
610
+ try {
611
+ task.signal = controller.signal;
612
+ const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
613
+ if (controller.signal.aborted) return reject(controller.signal.reason);
614
+ controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
615
+ })]);
616
+ if (!res.success) throw providerFailure(res);
617
+ return res;
618
+ } finally {
619
+ clearTimeout(timeout);
620
+ }
621
+ });
622
+ global._telemetry.provider += Date.now() - tProv;
623
+ if (result.success) break;
624
+ } catch (err) {
625
+ result = {
626
+ success: false,
627
+ error: err.message,
628
+ retryable: err.retryable
629
+ };
630
+ }
631
+ }
632
+ if (result.success) {
633
+ const providerMessageId = result.providerMessageId || "success";
634
+ publishPromises.push(this.outboxUpdateProcessor.add({
635
+ taskId: task.taskId,
636
+ channel: task.channel,
637
+ destination: task.destination,
638
+ providerMessageId
639
+ }));
640
+ this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
641
+ metrics.deliverySuccess.inc({ channel: task.channel });
642
+ publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
643
+ projectId: task.projectId,
644
+ taskId: task.taskId,
645
+ enrichedEventId: task.enrichedEventId,
646
+ channel: task.channel,
647
+ deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
648
+ providerMessageId,
649
+ templateId: task.templateId,
650
+ workflowInstanceId: task.workflowInstanceId,
651
+ campaignId: task.campaignId
652
+ }), "delivery", event.metadata.traceId)));
653
+ } else if (!await retryLater(result)) {
654
+ publishPromises.push(this.outboxUpdateProcessor.add({
655
+ taskId: task.taskId,
656
+ channel: task.channel,
657
+ destination: task.destination,
658
+ providerMessageId: null
659
+ }));
660
+ publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
661
+ projectId: task.projectId,
662
+ taskId: task.taskId,
663
+ enrichedEventId: task.enrichedEventId,
664
+ channel: task.channel,
665
+ failureReason: result.error ?? "delivery failed completely",
666
+ failureCode: "provider_error",
667
+ retryable: false,
668
+ attempt: attemptNo,
669
+ templateId: task.templateId,
670
+ workflowInstanceId: task.workflowInstanceId,
671
+ campaignId: task.campaignId
672
+ }), "delivery", event.metadata.traceId)));
673
+ }
674
+ if (!result.success && !retryScheduled) {
675
+ this.logger.warn({
676
+ taskId: task.taskId,
677
+ channel: task.channel,
678
+ error: result.error
679
+ }, "delivery failed completely across providers");
680
+ this.globalEmitter.emit("delivery:failed", task.taskId, result.error ?? "delivery failed completely", task.channel, task.projectId);
681
+ metrics.deliveryFailed.inc({
682
+ channel: task.channel,
683
+ reason: "provider_error"
684
+ });
685
+ if (!await fallbackToNextChannel("all_providers_failed")) throw new NonRetryableError(result.error ?? "delivery failed");
686
+ } else if (result.success) this.logger.debug({
687
+ taskId: task.taskId,
688
+ channel: task.channel,
689
+ messageId: result.providerMessageId
690
+ }, "notification delivered");
691
+ }
692
+ logDispatched();
693
+ const tFlushStart = Date.now();
694
+ await Promise.all(publishPromises).catch((err) => {
695
+ this.logger.error({
696
+ err,
697
+ taskId: task.taskId
698
+ }, "failed to publish post-dispatch events, swallowing error to prevent duplicate delivery");
699
+ });
700
+ const flushTime = Date.now() - tFlushStart;
701
+ if (retryScheduled) await this.idempotency.unmark(idempotencyKey).catch(() => {});
702
+ else this.idempotency.markProcessed(idempotencyKey).catch((err) => {
703
+ this.logger.warn({
704
+ err,
705
+ taskId: task.taskId
706
+ }, "failed to upgrade idempotency TTL");
707
+ });
708
+ const t = global._telemetry = global._telemetry || {
709
+ count: 0,
710
+ insert: 0,
711
+ provider: 0,
712
+ flush: 0,
713
+ ack: 0
714
+ };
715
+ t.count++;
716
+ t.insert += 0;
717
+ t.flush += flushTime;
718
+ if (t.count % 1e3 === 0) {
719
+ this.logger.debug(`[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1e3}ms, DB Insert: ${t.insert / 1e3}ms, Provider: ${t.provider / 1e3}ms, Wait for Flush: ${t.flush / 1e3}ms, Ack: ${t.ack / 1e3}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`);
720
+ t.count = 0;
721
+ t.insert = 0;
722
+ t.provider = 0;
723
+ t.flush = 0;
724
+ t.dequeue = 0;
725
+ t.ack = 0;
726
+ t.dbupdate = 0;
727
+ t.flushCount = 0;
728
+ }
729
+ } catch (err) {
730
+ logDispatched()?.catch(() => {});
731
+ await this.idempotency.unmark(idempotencyKey).catch(() => {});
732
+ throw err;
733
+ }
734
+ }
735
+ };
736
+ async function startDeliveryWorker() {
737
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "delivery"));
738
+ const deliveryConcurrency = config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY;
739
+ const consumerId = uniqueConsumerId("delivery");
740
+ ({consumer, pendingScanner} = createStreamConsumer({
741
+ redis,
742
+ logger
743
+ }, {
744
+ stream: OUTBOUND_STREAMS,
745
+ group: CONSUMER_GROUPS.DELIVERY,
746
+ consumer: consumerId,
747
+ batchSize: deliveryConcurrency,
748
+ bufferAcks: true,
749
+ coalesceMs: 2
750
+ }));
751
+ scheduledProducer = new StreamProducer({
752
+ redis: redis.native,
753
+ stream: STREAMS.SCHEDULED,
754
+ logger
755
+ });
756
+ enrichedProducers = createPriorityProducers(redis.native, logger, "ENRICHED");
757
+ const contactRepo = new ContactRepository(db);
758
+ const templateCache = new TemplateCache(new TemplateRepository(db));
759
+ const idempotency = new IdempotencyGuard({
760
+ redis: redis.native,
761
+ keyPrefix: "notif:processed:delivery",
762
+ ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS
763
+ });
764
+ const eventsProducer = new StreamProducer({
765
+ redis: redis.native,
766
+ stream: STREAMS.EVENTS_INBOUND,
767
+ logger
768
+ });
769
+ worker = new DeliveryWorker({
770
+ consumer,
771
+ pendingScanner,
772
+ logger,
773
+ concurrency: deliveryConcurrency,
774
+ transportRegistry,
775
+ idempotency,
776
+ redis: redis.native,
777
+ scheduledProducer,
778
+ enrichedProducers,
779
+ contactRepo,
780
+ eventsProducer,
781
+ globalEmitter,
782
+ db,
783
+ templateCache
784
+ });
785
+ subscriber = redis.native.duplicate({ enableAutoPipelining: false });
786
+ await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
787
+ subscriber.on("message", (_channel, key) => templateCache.invalidateKey(key));
788
+ healthInterval = startHealthReporter("delivery", worker, redis, logger);
789
+ logger.info({
790
+ env: config.NODE_ENV,
791
+ channels: transportRegistry.registeredChannels()
792
+ }, "delivery starting");
793
+ await worker.start();
794
+ }
795
+ function getDeliveryWorker() {
796
+ return worker;
797
+ }
798
+ async function stopDeliveryWorker() {
799
+ await shutdownWorker("delivery", {
800
+ logger,
801
+ timers: [healthInterval],
802
+ subscriber,
803
+ worker,
804
+ sql: sql$1,
805
+ redis
806
+ });
807
+ healthInterval = null;
808
+ subscriber = null;
809
+ }
810
+ //#endregion
811
+ export { DeliveryWorker, MAX_THROTTLE_DEFERRALS, getDeliveryWorker, retryDelayMs, startDeliveryWorker, stopDeliveryWorker };