notifkit 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/index-C-DJYu0_.css +1 -0
- package/dashboard/dist/assets/index-CzFZcEon.js +318 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/{bootstrap-m107TukN.mjs → bootstrap-BMDRPjgp.mjs} +1 -1
- package/dist/{bulk-xAd_biYV.mjs → bulk-CJ5QXiPa.mjs} +1 -1
- package/dist/index.d.mts +71 -27
- package/dist/index.mjs +2 -2
- package/dist/{main-CWIBUEc7.mjs → main-6_bXQBDH.mjs} +21 -41
- package/dist/{main-WkxXwP5q.mjs → main-8nP3qmCT.mjs} +29 -49
- package/dist/{main-HT5c6jyf.mjs → main-B82bsFoy.mjs} +183 -334
- package/dist/{main-C87TSAqM.mjs → main-CpaHF1Zw.mjs} +13 -15
- package/dist/{main-DRHL2bst.mjs → main-DULD8-th.mjs} +8 -6
- package/dist/{main-D_62f9fH.mjs → main-ZW5RaiOK.mjs} +77 -64
- package/dist/{main-5kVwbbnW.mjs → main-g1KEcXXp.mjs} +676 -582
- package/dist/{main-C0MjK1Au.mjs → main-mGIT8jv4.mjs} +4 -4
- package/dist/schedule-or-dispatch-Cd7GLfc3.mjs +52 -0
- package/dist/{src-DYqdD694.mjs → src-BE_EUpZ6.mjs} +93 -93
- package/package.json +1 -1
- package/scripts/create-admin.mjs +0 -0
- package/scripts/create-project.mjs +6 -2
- package/scripts/migrate.mjs +0 -0
- package/scripts/notifkit.mjs +0 -0
- package/src/client.ts +3 -3
- package/src/config/index.ts +1 -0
- package/src/contracts/common.ts +2 -0
- package/src/contracts/envelope.ts +1 -1
- package/src/contracts/events/notification-created.ts +2 -0
- package/src/contracts/events/notification-delivered.ts +1 -1
- package/src/contracts/events/notification-failed.ts +1 -1
- package/src/contracts/events/notification-requested.ts +3 -2
- package/src/contracts/helpers.ts +1 -2
- package/src/contracts/index.ts +1 -0
- package/src/contracts/registry.ts +1 -0
- package/src/contracts/sdk.ts +8 -37
- package/src/db/schema.ts +8 -13
- package/src/idempotency/index.ts +6 -8
- package/src/metrics/index.ts +2 -0
- package/src/rate-limiter/api-rate-limiter.ts +208 -0
- package/src/rate-limiter/index.ts +3 -411
- package/src/rate-limiter/project-settings-cache.ts +99 -0
- package/src/rate-limiter/user-throttle.ts +100 -0
- package/src/redis/index.ts +10 -3
- package/src/repositories/admin-user.ts +83 -0
- package/src/repositories/contact.ts +359 -0
- package/src/repositories/index.ts +9 -1416
- package/src/repositories/preference.ts +35 -0
- package/src/repositories/project.ts +110 -0
- package/src/repositories/segment.ts +16 -0
- package/src/repositories/template.ts +72 -0
- package/src/repositories/types.ts +49 -0
- package/src/repositories/user.ts +644 -0
- package/src/repositories/workflow.ts +64 -0
- package/src/services/ai/main.ts +29 -71
- package/src/services/api/admin-static.ts +4 -1
- package/src/services/api/handlers/auth.ts +109 -0
- package/src/services/api/handlers/campaigns.ts +191 -0
- package/src/services/api/handlers/events.ts +99 -0
- package/src/services/api/handlers/notifications.ts +356 -0
- package/src/services/api/handlers/projects.ts +116 -0
- package/src/services/api/handlers/shared.ts +262 -0
- package/src/services/api/handlers/suppressions.ts +95 -0
- package/src/services/api/handlers/system.ts +314 -0
- package/src/services/api/handlers/templates.ts +97 -0
- package/src/services/api/handlers/unsubscribe.ts +215 -0
- package/src/services/api/handlers/users.ts +262 -0
- package/src/services/api/handlers/workflows.ts +150 -0
- package/src/services/api/handlers.ts +25 -2190
- package/src/services/api/health.ts +133 -0
- package/src/services/api/http.ts +12 -0
- package/src/services/api/main.ts +32 -322
- package/src/services/api/router.ts +1 -1
- package/src/services/api/webhooks.ts +172 -0
- package/src/services/delivery/main.ts +256 -498
- package/src/services/engine/main.ts +31 -235
- package/src/services/engine/quiet-hours.ts +151 -0
- package/src/services/engine/schedule-or-dispatch.ts +104 -0
- package/src/services/enricher/main.ts +9 -21
- package/src/services/events/main.ts +2 -2
- package/src/services/scheduler/main.ts +16 -25
- package/src/services/workflow/duration.ts +24 -0
- package/src/services/workflow/main.ts +6 -164
- package/src/services/workflow/timers.ts +157 -0
- package/src/shared/events.ts +3 -2
- package/src/templates/render.ts +1 -0
- package/src/workers/index.ts +7 -4
- package/src/workflows/sdk.ts +1 -0
- package/dashboard/README.md +0 -29
- package/dashboard/dist/assets/index-ggpJKZqZ.js +0 -315
- package/dashboard/dist/assets/index-s-SXGIwv.css +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadEnv,
|
|
1
|
+
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
2
2
|
import type { Logger } from "@/index.js";
|
|
3
3
|
import type { RedisClient, Redis } from "@/index.js";
|
|
4
4
|
import {
|
|
@@ -39,14 +39,10 @@ import { publishDeliveryEvents } from "./live-events.js";
|
|
|
39
39
|
import { ProviderThrottle } from "./throttle.js";
|
|
40
40
|
import { metrics } from "@/metrics/index.js";
|
|
41
41
|
|
|
42
|
-
// ─── App-specific config ────────────────────────────────────────────────────
|
|
43
|
-
|
|
44
|
-
const deliveryConfigSchema = baseConfigSchema.extend({});
|
|
45
|
-
|
|
46
42
|
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
47
43
|
|
|
48
44
|
loadEnv();
|
|
49
|
-
const config =
|
|
45
|
+
const config = readBaseConfig();
|
|
50
46
|
|
|
51
47
|
let logger: Logger;
|
|
52
48
|
let redis: RedisClient;
|
|
@@ -62,17 +58,6 @@ let healthInterval: NodeJS.Timeout | null = null;
|
|
|
62
58
|
let subscriber: Redis | null = null;
|
|
63
59
|
let stopLiveEvents: (() => void) | null = null;
|
|
64
60
|
|
|
65
|
-
(global as any)._telemetry = (global as any)._telemetry || {
|
|
66
|
-
count: 0,
|
|
67
|
-
insert: 0,
|
|
68
|
-
provider: 0,
|
|
69
|
-
flush: 0,
|
|
70
|
-
ack: 0,
|
|
71
|
-
dequeue: 0,
|
|
72
|
-
dbupdate: 0,
|
|
73
|
-
flushCount: 0,
|
|
74
|
-
};
|
|
75
|
-
|
|
76
61
|
/** How long a send may run before its dispatched log entry is published alone. */
|
|
77
62
|
const DISPATCH_LOG_DELAY_MS = 1_000;
|
|
78
63
|
|
|
@@ -188,13 +173,9 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
188
173
|
providerMessageId: update.providerMessageId,
|
|
189
174
|
}));
|
|
190
175
|
|
|
191
|
-
const tDbUpdateStart = Date.now();
|
|
192
176
|
await upsertOutboxProviderIds(this.db, values).catch((e: any) =>
|
|
193
177
|
this.logger.error({ err: e }, "background update failed"),
|
|
194
178
|
);
|
|
195
|
-
|
|
196
|
-
(global as any)._telemetry.dbupdate += Date.now() - tDbUpdateStart;
|
|
197
|
-
(global as any)._telemetry.flushCount++;
|
|
198
179
|
return updates.map(() => undefined as void);
|
|
199
180
|
},
|
|
200
181
|
flushConcurrency,
|
|
@@ -312,6 +293,95 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
312
293
|
// and the transport's abort signal were attached to it.
|
|
313
294
|
const original: DispatchedTaskPayload = { ...task };
|
|
314
295
|
|
|
296
|
+
// The attempt's `dispatched` log entry normally rides on its outcome event,
|
|
297
|
+
// one stream entry instead of two. A send still running after
|
|
298
|
+
// DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
|
|
299
|
+
// visible while it is in flight, as it always was.
|
|
300
|
+
let dispatchStarted = false;
|
|
301
|
+
let dispatchLogged = false;
|
|
302
|
+
let dispatchedAt = "";
|
|
303
|
+
let dispatchLogTimer: NodeJS.Timeout | undefined;
|
|
304
|
+
const logDispatched = (): Promise<void> | undefined => {
|
|
305
|
+
clearTimeout(dispatchLogTimer);
|
|
306
|
+
if (!dispatchStarted || dispatchLogged) return undefined;
|
|
307
|
+
dispatchLogged = true;
|
|
308
|
+
const logged = this.eventProcessor.add(
|
|
309
|
+
buildStreamEvent(
|
|
310
|
+
"notification.dispatched",
|
|
311
|
+
{
|
|
312
|
+
projectId: task.projectId,
|
|
313
|
+
taskId: task.taskId,
|
|
314
|
+
enrichedEventId: task.enrichedEventId,
|
|
315
|
+
recipientId: task.recipientId,
|
|
316
|
+
channel: task.channel,
|
|
317
|
+
templateId: task.templateId,
|
|
318
|
+
attempt: attemptNo,
|
|
319
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
320
|
+
campaignId: task.campaignId,
|
|
321
|
+
},
|
|
322
|
+
"delivery",
|
|
323
|
+
event.metadata.traceId,
|
|
324
|
+
),
|
|
325
|
+
);
|
|
326
|
+
publishPromises.push(logged);
|
|
327
|
+
return logged;
|
|
328
|
+
};
|
|
329
|
+
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
330
|
+
const outcome = <T extends Record<string, unknown>>(payload: T): T => {
|
|
331
|
+
clearTimeout(dispatchLogTimer);
|
|
332
|
+
if (dispatchLogged) return payload;
|
|
333
|
+
dispatchLogged = true;
|
|
334
|
+
return { ...payload, attempt: attemptNo, dispatchedAt };
|
|
335
|
+
};
|
|
336
|
+
/**
|
|
337
|
+
* Queues the task's `notification.failed` event; once sending has started it
|
|
338
|
+
* is the attempt's outcome. Only the non-push provider failure leaves out
|
|
339
|
+
* `recipientId`.
|
|
340
|
+
*/
|
|
341
|
+
const failed = (failureReason: string, failureCode: string, withRecipientId = true) => {
|
|
342
|
+
const payload = {
|
|
343
|
+
projectId: task.projectId,
|
|
344
|
+
taskId: task.taskId,
|
|
345
|
+
enrichedEventId: task.enrichedEventId,
|
|
346
|
+
...(withRecipientId ? { recipientId: task.recipientId } : {}),
|
|
347
|
+
channel: task.channel,
|
|
348
|
+
failureReason,
|
|
349
|
+
failureCode,
|
|
350
|
+
retryable: false,
|
|
351
|
+
attempt: attemptNo,
|
|
352
|
+
templateId: task.templateId,
|
|
353
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
354
|
+
campaignId: task.campaignId,
|
|
355
|
+
};
|
|
356
|
+
return this.eventProcessor.add(
|
|
357
|
+
buildStreamEvent(
|
|
358
|
+
"notification.failed",
|
|
359
|
+
dispatchStarted ? outcome(payload) : payload,
|
|
360
|
+
"delivery",
|
|
361
|
+
event.metadata.traceId,
|
|
362
|
+
),
|
|
363
|
+
);
|
|
364
|
+
};
|
|
365
|
+
const delivered = (providerMessageId: string) =>
|
|
366
|
+
this.eventProcessor.add(
|
|
367
|
+
buildStreamEvent(
|
|
368
|
+
"notification.delivered",
|
|
369
|
+
outcome({
|
|
370
|
+
projectId: task.projectId,
|
|
371
|
+
taskId: task.taskId,
|
|
372
|
+
enrichedEventId: task.enrichedEventId,
|
|
373
|
+
channel: task.channel,
|
|
374
|
+
deliveredAt: new Date().toISOString(),
|
|
375
|
+
providerMessageId,
|
|
376
|
+
templateId: task.templateId,
|
|
377
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
378
|
+
campaignId: task.campaignId,
|
|
379
|
+
}),
|
|
380
|
+
"delivery",
|
|
381
|
+
event.metadata.traceId,
|
|
382
|
+
),
|
|
383
|
+
);
|
|
384
|
+
|
|
315
385
|
const fallbackToNextChannel = async (reason: string) => {
|
|
316
386
|
if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
|
|
317
387
|
const nextChannel = task.fallbackChain[0];
|
|
@@ -371,27 +441,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
371
441
|
task.channel,
|
|
372
442
|
task.projectId,
|
|
373
443
|
);
|
|
374
|
-
await
|
|
375
|
-
buildStreamEvent(
|
|
376
|
-
"notification.failed",
|
|
377
|
-
{
|
|
378
|
-
projectId: task.projectId,
|
|
379
|
-
taskId: task.taskId,
|
|
380
|
-
enrichedEventId: task.enrichedEventId,
|
|
381
|
-
recipientId: task.recipientId,
|
|
382
|
-
channel: task.channel,
|
|
383
|
-
failureReason: "no transport registered for channel",
|
|
384
|
-
failureCode: "no_transport",
|
|
385
|
-
retryable: false,
|
|
386
|
-
attempt: attemptNo,
|
|
387
|
-
templateId: task.templateId,
|
|
388
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
389
|
-
campaignId: task.campaignId,
|
|
390
|
-
},
|
|
391
|
-
"delivery",
|
|
392
|
-
event.metadata.traceId,
|
|
393
|
-
),
|
|
394
|
-
);
|
|
444
|
+
await failed("no transport registered for channel", "no_transport");
|
|
395
445
|
}
|
|
396
446
|
return;
|
|
397
447
|
}
|
|
@@ -427,27 +477,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
427
477
|
task.channel,
|
|
428
478
|
task.projectId,
|
|
429
479
|
);
|
|
430
|
-
await
|
|
431
|
-
buildStreamEvent(
|
|
432
|
-
"notification.failed",
|
|
433
|
-
{
|
|
434
|
-
projectId: task.projectId,
|
|
435
|
-
taskId: task.taskId,
|
|
436
|
-
enrichedEventId: task.enrichedEventId,
|
|
437
|
-
recipientId: task.recipientId,
|
|
438
|
-
channel: task.channel,
|
|
439
|
-
failureReason: "provider rate limit max attempts exceeded",
|
|
440
|
-
failureCode: "provider_throttle_exceeded",
|
|
441
|
-
retryable: false,
|
|
442
|
-
attempt: attemptNo,
|
|
443
|
-
templateId: task.templateId,
|
|
444
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
445
|
-
campaignId: task.campaignId,
|
|
446
|
-
},
|
|
447
|
-
"delivery",
|
|
448
|
-
event.metadata.traceId,
|
|
449
|
-
),
|
|
450
|
-
);
|
|
480
|
+
await failed("provider rate limit max attempts exceeded", "provider_throttle_exceeded");
|
|
451
481
|
}
|
|
452
482
|
return;
|
|
453
483
|
}
|
|
@@ -492,47 +522,6 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
492
522
|
const tDispatchStart = Date.now();
|
|
493
523
|
const MAX_DISPATCH_BUDGET_MS = 25_000; // Strictly shorter than lock lease (30s)
|
|
494
524
|
|
|
495
|
-
// The attempt's `dispatched` log entry normally rides on its outcome event,
|
|
496
|
-
// one stream entry instead of two. A send still running after
|
|
497
|
-
// DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
|
|
498
|
-
// visible while it is in flight, as it always was.
|
|
499
|
-
let dispatchStarted = false;
|
|
500
|
-
let dispatchLogged = false;
|
|
501
|
-
let dispatchedAt = "";
|
|
502
|
-
let dispatchLogTimer: NodeJS.Timeout | undefined;
|
|
503
|
-
const logDispatched = (): Promise<void> | undefined => {
|
|
504
|
-
clearTimeout(dispatchLogTimer);
|
|
505
|
-
if (!dispatchStarted || dispatchLogged) return undefined;
|
|
506
|
-
dispatchLogged = true;
|
|
507
|
-
const logged = this.eventProcessor.add(
|
|
508
|
-
buildStreamEvent(
|
|
509
|
-
"notification.dispatched",
|
|
510
|
-
{
|
|
511
|
-
projectId: task.projectId,
|
|
512
|
-
taskId: task.taskId,
|
|
513
|
-
enrichedEventId: task.enrichedEventId,
|
|
514
|
-
recipientId: task.recipientId,
|
|
515
|
-
channel: task.channel,
|
|
516
|
-
templateId: task.templateId,
|
|
517
|
-
attempt: attemptNo,
|
|
518
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
519
|
-
campaignId: task.campaignId,
|
|
520
|
-
},
|
|
521
|
-
"delivery",
|
|
522
|
-
event.metadata.traceId,
|
|
523
|
-
),
|
|
524
|
-
);
|
|
525
|
-
publishPromises.push(logged);
|
|
526
|
-
return logged;
|
|
527
|
-
};
|
|
528
|
-
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
529
|
-
const outcome = <T extends Record<string, unknown>>(payload: T): T => {
|
|
530
|
-
clearTimeout(dispatchLogTimer);
|
|
531
|
-
if (dispatchLogged) return payload;
|
|
532
|
-
dispatchLogged = true;
|
|
533
|
-
return { ...payload, attempt: attemptNo, dispatchedAt };
|
|
534
|
-
};
|
|
535
|
-
|
|
536
525
|
let retryScheduled = false;
|
|
537
526
|
/**
|
|
538
527
|
* Re-queues a send the provider turned away, unless it said the failure is
|
|
@@ -577,383 +566,183 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
577
566
|
void logDispatched();
|
|
578
567
|
}, DISPATCH_LOG_DELAY_MS);
|
|
579
568
|
|
|
580
|
-
//
|
|
581
|
-
//
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
continue;
|
|
591
|
-
}
|
|
592
|
-
const elapsedMs = Date.now() - tDispatchStart;
|
|
593
|
-
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
594
|
-
if (remainingBudgetMs <= 0) {
|
|
595
|
-
this.logger.warn(
|
|
596
|
-
{ taskId: task.taskId, channel: task.channel },
|
|
597
|
-
"dispatch budget exhausted before transport call — aborting to protect lock lease",
|
|
598
|
-
);
|
|
599
|
-
break;
|
|
600
|
-
}
|
|
601
|
-
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
602
|
-
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
603
|
-
|
|
604
|
-
try {
|
|
605
|
-
const tProv = Date.now();
|
|
606
|
-
const breaker = this.getBreaker(providerKey(task.channel, transports, i));
|
|
607
|
-
|
|
608
|
-
lastResult = await breaker.execute(async () => {
|
|
609
|
-
const controller = new AbortController();
|
|
610
|
-
const timeout = setTimeout(() => {
|
|
611
|
-
controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
612
|
-
}, timeoutMs);
|
|
613
|
-
|
|
614
|
-
try {
|
|
615
|
-
(task as any).signal = controller.signal;
|
|
616
|
-
const res: any = await Promise.race([
|
|
617
|
-
transport.send(task),
|
|
618
|
-
new Promise((_, reject) => {
|
|
619
|
-
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
620
|
-
controller.signal.addEventListener("abort", () =>
|
|
621
|
-
reject(controller.signal.reason),
|
|
622
|
-
);
|
|
623
|
-
}),
|
|
624
|
-
]);
|
|
625
|
-
// A permanent rejection is the provider working: returned, not
|
|
626
|
-
// thrown, so the breaker does not count it.
|
|
627
|
-
if (!res.success && !res.invalidToken && res.retryable !== false) {
|
|
628
|
-
throw providerFailure(res);
|
|
629
|
-
}
|
|
630
|
-
return res;
|
|
631
|
-
} finally {
|
|
632
|
-
clearTimeout(timeout);
|
|
633
|
-
}
|
|
634
|
-
});
|
|
635
|
-
|
|
636
|
-
(global as any)._telemetry.provider += Date.now() - tProv;
|
|
637
|
-
if (lastResult.success || lastResult.invalidToken) break;
|
|
638
|
-
} catch (err: any) {
|
|
639
|
-
lastResult = { success: false, error: err.message, retryable: err.retryable };
|
|
640
|
-
}
|
|
569
|
+
// Push alone treats a rejected token as an answer: the token is
|
|
570
|
+
// deactivated rather than the next provider tried.
|
|
571
|
+
const isPush = task.channel === "push";
|
|
572
|
+
let result: any = { success: false, error: "No transports" };
|
|
573
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
574
|
+
const transport = transports[i];
|
|
575
|
+
// The starting provider's slot was taken above. Failing over to a
|
|
576
|
+
// later one spends that provider's own budget, or skips it if spent.
|
|
577
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
|
|
578
|
+
continue;
|
|
641
579
|
}
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
destination: task.destination,
|
|
649
|
-
providerMessageId: null,
|
|
650
|
-
}),
|
|
651
|
-
);
|
|
652
|
-
await this.contactRepo.deactivate(
|
|
653
|
-
task.projectId,
|
|
654
|
-
task.recipientId,
|
|
655
|
-
task.channel as any,
|
|
656
|
-
task.destination,
|
|
657
|
-
);
|
|
658
|
-
this.logger.info(
|
|
659
|
-
{ channel: task.channel, target: task.destination },
|
|
660
|
-
"deactivated invalid contact target",
|
|
661
|
-
);
|
|
662
|
-
this.globalEmitter.emit(
|
|
663
|
-
"delivery:failed",
|
|
664
|
-
task.taskId,
|
|
665
|
-
"invalidToken",
|
|
666
|
-
task.channel,
|
|
667
|
-
task.projectId,
|
|
580
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
581
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
582
|
+
if (remainingBudgetMs <= 0) {
|
|
583
|
+
this.logger.warn(
|
|
584
|
+
{ taskId: task.taskId, channel: task.channel },
|
|
585
|
+
"dispatch budget exhausted before transport call — aborting to protect lock lease",
|
|
668
586
|
);
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
587
|
+
break;
|
|
588
|
+
}
|
|
589
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
590
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
591
|
+
|
|
592
|
+
try {
|
|
593
|
+
const breaker = this.getBreaker(providerKey(task.channel, transports, i));
|
|
594
|
+
|
|
595
|
+
result = await breaker.execute(async () => {
|
|
596
|
+
const controller = new AbortController();
|
|
597
|
+
const timeout = setTimeout(() => {
|
|
598
|
+
controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
599
|
+
}, timeoutMs);
|
|
600
|
+
|
|
601
|
+
try {
|
|
602
|
+
(task as any).signal = controller.signal;
|
|
603
|
+
const res: any = await Promise.race([
|
|
604
|
+
transport.send(task),
|
|
605
|
+
new Promise((_, reject) => {
|
|
606
|
+
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
607
|
+
controller.signal.addEventListener("abort", () =>
|
|
608
|
+
reject(controller.signal.reason),
|
|
609
|
+
);
|
|
688
610
|
}),
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
611
|
+
]);
|
|
612
|
+
// A permanent rejection (a bad address) is the provider working:
|
|
613
|
+
// returned, not thrown, so the breaker does not count it. A run
|
|
614
|
+
// of them in one list used to open it for every project.
|
|
615
|
+
if (!res.success && !(isPush && res.invalidToken) && res.retryable !== false) {
|
|
616
|
+
throw providerFailure(res);
|
|
617
|
+
}
|
|
618
|
+
return res;
|
|
619
|
+
} finally {
|
|
620
|
+
clearTimeout(timeout);
|
|
621
|
+
}
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
if (result.success || (isPush && result.invalidToken)) break;
|
|
625
|
+
} catch (err: any) {
|
|
626
|
+
result = { success: false, error: err.message, retryable: err.retryable };
|
|
627
|
+
}
|
|
628
|
+
}
|
|
694
629
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
630
|
+
if (isPush && result.invalidToken) {
|
|
631
|
+
publishPromises.push(
|
|
632
|
+
this.outboxUpdateProcessor.add({
|
|
633
|
+
taskId: task.taskId,
|
|
634
|
+
channel: task.channel,
|
|
635
|
+
destination: task.destination,
|
|
636
|
+
providerMessageId: null,
|
|
637
|
+
}),
|
|
638
|
+
);
|
|
639
|
+
await this.contactRepo.deactivate(
|
|
640
|
+
task.projectId,
|
|
641
|
+
task.recipientId,
|
|
642
|
+
task.channel as any,
|
|
643
|
+
task.destination,
|
|
644
|
+
);
|
|
645
|
+
this.logger.info(
|
|
646
|
+
{ channel: task.channel, target: task.destination },
|
|
647
|
+
"deactivated invalid contact target",
|
|
648
|
+
);
|
|
649
|
+
this.globalEmitter.emit(
|
|
650
|
+
"delivery:failed",
|
|
651
|
+
task.taskId,
|
|
652
|
+
"invalidToken",
|
|
653
|
+
task.channel,
|
|
654
|
+
task.projectId,
|
|
655
|
+
);
|
|
656
|
+
metrics.deliveryFailed.inc({ channel: task.channel, reason: "invalid_token" });
|
|
657
|
+
|
|
658
|
+
publishPromises.push(failed("invalid_token", "push_failure"));
|
|
659
|
+
|
|
660
|
+
const fallbackTriggered = await fallbackToNextChannel("invalid_token");
|
|
661
|
+
if (!fallbackTriggered) {
|
|
662
|
+
throw new NonRetryableError("Push delivery failed: invalid token");
|
|
663
|
+
}
|
|
664
|
+
} else if (result.success) {
|
|
665
|
+
// Record providerMessageId in outbox before publishing event
|
|
666
|
+
const providerMessageId = result.providerMessageId || (isPush ? "push-success" : "success");
|
|
711
667
|
|
|
668
|
+
publishPromises.push(
|
|
669
|
+
this.outboxUpdateProcessor.add({
|
|
670
|
+
taskId: task.taskId,
|
|
671
|
+
channel: task.channel,
|
|
672
|
+
destination: task.destination,
|
|
673
|
+
providerMessageId,
|
|
674
|
+
}),
|
|
675
|
+
);
|
|
676
|
+
|
|
677
|
+
if (isPush) {
|
|
712
678
|
this.logger.debug(
|
|
713
679
|
{ taskId: task.taskId, messageId: providerMessageId },
|
|
714
680
|
"push delivered",
|
|
715
681
|
);
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
publishPromises.push(
|
|
726
|
-
this.eventProcessor.add(
|
|
727
|
-
buildStreamEvent(
|
|
728
|
-
"notification.delivered",
|
|
729
|
-
outcome({
|
|
730
|
-
projectId: task.projectId,
|
|
731
|
-
taskId: task.taskId,
|
|
732
|
-
enrichedEventId: task.enrichedEventId,
|
|
733
|
-
channel: task.channel,
|
|
734
|
-
deliveredAt: new Date().toISOString(),
|
|
735
|
-
providerMessageId,
|
|
736
|
-
templateId: task.templateId,
|
|
737
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
738
|
-
campaignId: task.campaignId,
|
|
739
|
-
}),
|
|
740
|
-
"delivery",
|
|
741
|
-
event.metadata.traceId,
|
|
742
|
-
),
|
|
743
|
-
),
|
|
744
|
-
);
|
|
745
|
-
} else if (!(await retryLater(lastResult))) {
|
|
746
|
-
this.logger.warn(
|
|
747
|
-
{ taskId: task.taskId, error: lastResult.error },
|
|
748
|
-
"push delivery failed",
|
|
749
|
-
);
|
|
750
|
-
this.globalEmitter.emit(
|
|
751
|
-
"delivery:failed",
|
|
752
|
-
task.taskId,
|
|
753
|
-
lastResult.error ?? "push delivery failed",
|
|
754
|
-
task.channel,
|
|
755
|
-
task.projectId,
|
|
756
|
-
);
|
|
757
|
-
metrics.deliveryFailed.inc({ channel: task.channel, reason: "push_error" });
|
|
758
|
-
|
|
759
|
-
publishPromises.push(
|
|
760
|
-
this.outboxUpdateProcessor.add({
|
|
761
|
-
taskId: task.taskId,
|
|
762
|
-
channel: task.channel,
|
|
763
|
-
destination: task.destination,
|
|
764
|
-
providerMessageId: null,
|
|
765
|
-
}),
|
|
766
|
-
);
|
|
682
|
+
}
|
|
683
|
+
this.globalEmitter.emit(
|
|
684
|
+
"delivery:delivered",
|
|
685
|
+
task.taskId,
|
|
686
|
+
providerMessageId,
|
|
687
|
+
task.channel,
|
|
688
|
+
task.projectId,
|
|
689
|
+
);
|
|
690
|
+
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
767
691
|
|
|
768
|
-
|
|
769
|
-
this.eventProcessor.add(
|
|
770
|
-
buildStreamEvent(
|
|
771
|
-
"notification.failed",
|
|
772
|
-
outcome({
|
|
773
|
-
projectId: task.projectId,
|
|
774
|
-
taskId: task.taskId,
|
|
775
|
-
enrichedEventId: task.enrichedEventId,
|
|
776
|
-
recipientId: task.recipientId,
|
|
777
|
-
channel: task.channel,
|
|
778
|
-
failureReason: lastResult.error ?? "push delivery failed",
|
|
779
|
-
failureCode: "push_failure",
|
|
780
|
-
retryable: false,
|
|
781
|
-
attempt: attemptNo,
|
|
782
|
-
templateId: task.templateId,
|
|
783
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
784
|
-
campaignId: task.campaignId,
|
|
785
|
-
}),
|
|
786
|
-
"delivery",
|
|
787
|
-
event.metadata.traceId,
|
|
788
|
-
),
|
|
789
|
-
),
|
|
790
|
-
);
|
|
692
|
+
publishPromises.push(delivered(providerMessageId));
|
|
791
693
|
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
694
|
+
if (!isPush) {
|
|
695
|
+
this.logger.debug(
|
|
696
|
+
{ taskId: task.taskId, channel: task.channel, messageId: result.providerMessageId },
|
|
697
|
+
"notification delivered",
|
|
698
|
+
);
|
|
796
699
|
}
|
|
797
|
-
} else {
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
continue;
|
|
806
|
-
}
|
|
807
|
-
const elapsedMs = Date.now() - tDispatchStart;
|
|
808
|
-
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
809
|
-
if (remainingBudgetMs <= 0) {
|
|
700
|
+
} else if (!(await retryLater(result))) {
|
|
701
|
+
const error =
|
|
702
|
+
result.error ?? (isPush ? "push delivery failed" : "delivery failed completely");
|
|
703
|
+
// Push reports the failure before queueing its outcome, other channels after.
|
|
704
|
+
const report = () => {
|
|
705
|
+
if (isPush) {
|
|
706
|
+
this.logger.warn({ taskId: task.taskId, error: result.error }, "push delivery failed");
|
|
707
|
+
} else {
|
|
810
708
|
this.logger.warn(
|
|
811
|
-
{ taskId: task.taskId, channel: task.channel },
|
|
812
|
-
"
|
|
709
|
+
{ taskId: task.taskId, channel: task.channel, error: result.error },
|
|
710
|
+
"delivery failed completely across providers",
|
|
813
711
|
);
|
|
814
|
-
break;
|
|
815
|
-
}
|
|
816
|
-
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 10_000;
|
|
817
|
-
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
818
|
-
|
|
819
|
-
try {
|
|
820
|
-
const tProv = Date.now();
|
|
821
|
-
const breaker = this.getBreaker(providerKey(task.channel, transports, i));
|
|
822
|
-
|
|
823
|
-
result = await breaker.execute(async () => {
|
|
824
|
-
const controller = new AbortController();
|
|
825
|
-
const timeout = setTimeout(() => {
|
|
826
|
-
controller.abort(new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
827
|
-
}, timeoutMs);
|
|
828
|
-
|
|
829
|
-
try {
|
|
830
|
-
(task as any).signal = controller.signal;
|
|
831
|
-
const res: any = await Promise.race([
|
|
832
|
-
transport.send(task),
|
|
833
|
-
new Promise((_, reject) => {
|
|
834
|
-
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
835
|
-
controller.signal.addEventListener("abort", () =>
|
|
836
|
-
reject(controller.signal.reason),
|
|
837
|
-
);
|
|
838
|
-
}),
|
|
839
|
-
]);
|
|
840
|
-
// A permanent rejection (a bad address) is the provider working:
|
|
841
|
-
// returned, not thrown, so the breaker does not count it. A run
|
|
842
|
-
// of them in one list used to open it for every project.
|
|
843
|
-
if (!res.success && res.retryable !== false) {
|
|
844
|
-
throw providerFailure(res);
|
|
845
|
-
}
|
|
846
|
-
return res;
|
|
847
|
-
} finally {
|
|
848
|
-
clearTimeout(timeout);
|
|
849
|
-
}
|
|
850
|
-
});
|
|
851
|
-
|
|
852
|
-
(global as any)._telemetry.provider += Date.now() - tProv;
|
|
853
|
-
if (result.success) break;
|
|
854
|
-
} catch (err: any) {
|
|
855
|
-
result = { success: false, error: err.message, retryable: err.retryable };
|
|
856
712
|
}
|
|
857
|
-
}
|
|
858
|
-
|
|
859
|
-
if (result.success) {
|
|
860
|
-
// Record providerMessageId in outbox before publishing event
|
|
861
|
-
const providerMessageId = result.providerMessageId || "success";
|
|
862
|
-
|
|
863
|
-
publishPromises.push(
|
|
864
|
-
this.outboxUpdateProcessor.add({
|
|
865
|
-
taskId: task.taskId,
|
|
866
|
-
channel: task.channel,
|
|
867
|
-
destination: task.destination,
|
|
868
|
-
providerMessageId,
|
|
869
|
-
}),
|
|
870
|
-
);
|
|
871
|
-
|
|
872
|
-
this.globalEmitter.emit(
|
|
873
|
-
"delivery:delivered",
|
|
874
|
-
task.taskId,
|
|
875
|
-
providerMessageId,
|
|
876
|
-
task.channel,
|
|
877
|
-
task.projectId,
|
|
878
|
-
);
|
|
879
|
-
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
880
|
-
|
|
881
|
-
publishPromises.push(
|
|
882
|
-
this.eventProcessor.add(
|
|
883
|
-
buildStreamEvent(
|
|
884
|
-
"notification.delivered",
|
|
885
|
-
outcome({
|
|
886
|
-
projectId: task.projectId,
|
|
887
|
-
taskId: task.taskId,
|
|
888
|
-
enrichedEventId: task.enrichedEventId,
|
|
889
|
-
channel: task.channel,
|
|
890
|
-
deliveredAt: new Date().toISOString(),
|
|
891
|
-
providerMessageId,
|
|
892
|
-
templateId: task.templateId,
|
|
893
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
894
|
-
campaignId: task.campaignId,
|
|
895
|
-
}),
|
|
896
|
-
"delivery",
|
|
897
|
-
event.metadata.traceId,
|
|
898
|
-
),
|
|
899
|
-
),
|
|
900
|
-
);
|
|
901
|
-
} else if (!(await retryLater(result))) {
|
|
902
|
-
publishPromises.push(
|
|
903
|
-
this.outboxUpdateProcessor.add({
|
|
904
|
-
taskId: task.taskId,
|
|
905
|
-
channel: task.channel,
|
|
906
|
-
destination: task.destination,
|
|
907
|
-
providerMessageId: null,
|
|
908
|
-
}),
|
|
909
|
-
);
|
|
910
|
-
|
|
911
|
-
publishPromises.push(
|
|
912
|
-
this.eventProcessor.add(
|
|
913
|
-
buildStreamEvent(
|
|
914
|
-
"notification.failed",
|
|
915
|
-
outcome({
|
|
916
|
-
projectId: task.projectId,
|
|
917
|
-
taskId: task.taskId,
|
|
918
|
-
enrichedEventId: task.enrichedEventId,
|
|
919
|
-
channel: task.channel,
|
|
920
|
-
failureReason: result.error ?? "delivery failed completely",
|
|
921
|
-
failureCode: "provider_error",
|
|
922
|
-
retryable: false,
|
|
923
|
-
attempt: attemptNo,
|
|
924
|
-
templateId: task.templateId,
|
|
925
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
926
|
-
campaignId: task.campaignId,
|
|
927
|
-
}),
|
|
928
|
-
"delivery",
|
|
929
|
-
event.metadata.traceId,
|
|
930
|
-
),
|
|
931
|
-
),
|
|
932
|
-
);
|
|
933
|
-
}
|
|
934
|
-
|
|
935
|
-
if (!result.success && !retryScheduled) {
|
|
936
|
-
this.logger.warn(
|
|
937
|
-
{ taskId: task.taskId, channel: task.channel, error: result.error },
|
|
938
|
-
"delivery failed completely across providers",
|
|
939
|
-
);
|
|
940
713
|
this.globalEmitter.emit(
|
|
941
714
|
"delivery:failed",
|
|
942
715
|
task.taskId,
|
|
943
|
-
|
|
716
|
+
error,
|
|
944
717
|
task.channel,
|
|
945
718
|
task.projectId,
|
|
946
719
|
);
|
|
947
|
-
metrics.deliveryFailed.inc({
|
|
720
|
+
metrics.deliveryFailed.inc({
|
|
721
|
+
channel: task.channel,
|
|
722
|
+
reason: isPush ? "push_error" : "provider_error",
|
|
723
|
+
});
|
|
724
|
+
};
|
|
948
725
|
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
726
|
+
if (isPush) report();
|
|
727
|
+
publishPromises.push(
|
|
728
|
+
this.outboxUpdateProcessor.add({
|
|
729
|
+
taskId: task.taskId,
|
|
730
|
+
channel: task.channel,
|
|
731
|
+
destination: task.destination,
|
|
732
|
+
providerMessageId: null,
|
|
733
|
+
}),
|
|
734
|
+
);
|
|
735
|
+
publishPromises.push(
|
|
736
|
+
isPush ? failed(error, "push_failure") : failed(error, "provider_error", false),
|
|
737
|
+
);
|
|
738
|
+
if (!isPush) report();
|
|
739
|
+
|
|
740
|
+
const fallbackTriggered = await fallbackToNextChannel(
|
|
741
|
+
isPush ? "push_delivery_failed" : "all_providers_failed",
|
|
742
|
+
);
|
|
743
|
+
if (!fallbackTriggered) {
|
|
744
|
+
throw new NonRetryableError(
|
|
745
|
+
result.error ?? (isPush ? "Push delivery failed" : "delivery failed"),
|
|
957
746
|
);
|
|
958
747
|
}
|
|
959
748
|
}
|
|
@@ -961,52 +750,29 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
961
750
|
// Every branch above publishes an outcome; this only guards one that doesn't.
|
|
962
751
|
void logDispatched();
|
|
963
752
|
|
|
964
|
-
const tFlushStart = Date.now();
|
|
965
753
|
await Promise.all(publishPromises).catch((err) => {
|
|
966
754
|
this.logger.error(
|
|
967
755
|
{ err, taskId: task.taskId },
|
|
968
756
|
"failed to publish post-dispatch events, swallowing error to prevent duplicate delivery",
|
|
969
757
|
);
|
|
970
758
|
});
|
|
971
|
-
const flushTime = Date.now() - tFlushStart;
|
|
972
759
|
|
|
973
760
|
if (retryScheduled) {
|
|
974
761
|
// Released so the re-queued attempt can take the lease.
|
|
975
|
-
await this.idempotency.unmark(idempotencyKey).catch(() => {
|
|
762
|
+
await this.idempotency.unmark(idempotencyKey).catch((err: any) => {
|
|
763
|
+
this.logger.warn({ err, taskId: task.taskId }, "failed to release idempotency lock");
|
|
764
|
+
});
|
|
976
765
|
} else {
|
|
977
766
|
void this.idempotency.markProcessed(idempotencyKey).catch((err: any) => {
|
|
978
767
|
this.logger.warn({ err, taskId: task.taskId }, "failed to upgrade idempotency TTL");
|
|
979
768
|
});
|
|
980
769
|
}
|
|
981
|
-
|
|
982
|
-
const t = ((global as any)._telemetry = (global as any)._telemetry || {
|
|
983
|
-
count: 0,
|
|
984
|
-
insert: 0,
|
|
985
|
-
provider: 0,
|
|
986
|
-
flush: 0,
|
|
987
|
-
ack: 0,
|
|
988
|
-
});
|
|
989
|
-
t.count++;
|
|
990
|
-
t.insert += 0;
|
|
991
|
-
t.flush += flushTime;
|
|
992
|
-
|
|
993
|
-
if (t.count % 1000 === 0) {
|
|
994
|
-
this.logger.debug(
|
|
995
|
-
`[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1000}ms, DB Insert: ${t.insert / 1000}ms, Provider: ${t.provider / 1000}ms, Wait for Flush: ${t.flush / 1000}ms, Ack: ${t.ack / 1000}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`,
|
|
996
|
-
);
|
|
997
|
-
t.count = 0;
|
|
998
|
-
t.insert = 0;
|
|
999
|
-
t.provider = 0;
|
|
1000
|
-
t.flush = 0;
|
|
1001
|
-
t.dequeue = 0;
|
|
1002
|
-
t.ack = 0;
|
|
1003
|
-
t.dbupdate = 0;
|
|
1004
|
-
t.flushCount = 0;
|
|
1005
|
-
}
|
|
1006
770
|
} catch (err) {
|
|
1007
771
|
// The attempt happened even though it did not finish; keep it in the log.
|
|
1008
772
|
logDispatched()?.catch(() => {});
|
|
1009
|
-
await this.idempotency.unmark(idempotencyKey).catch(() => {
|
|
773
|
+
await this.idempotency.unmark(idempotencyKey).catch((e: any) => {
|
|
774
|
+
this.logger.warn({ err: e, taskId: task.taskId }, "failed to release idempotency lock");
|
|
775
|
+
});
|
|
1010
776
|
throw err;
|
|
1011
777
|
}
|
|
1012
778
|
}
|
|
@@ -1047,14 +813,6 @@ export async function startDeliveryWorker() {
|
|
|
1047
813
|
ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
|
|
1048
814
|
});
|
|
1049
815
|
|
|
1050
|
-
// ─── Stage 3: Delivery Worker ───────────────────────────────────────────────
|
|
1051
|
-
//
|
|
1052
|
-
// Pipeline:
|
|
1053
|
-
// 1. Parse payload as notification.dispatched
|
|
1054
|
-
// 2. Resolve active device tokens from DB
|
|
1055
|
-
// 3. Send via registered transport
|
|
1056
|
-
// 4. Deactivate invalid tokens in DB
|
|
1057
|
-
|
|
1058
816
|
const eventsProducer = new StreamProducer({
|
|
1059
817
|
redis: redis.native,
|
|
1060
818
|
stream: STREAMS.EVENTS_INBOUND,
|
|
@@ -1083,11 +841,11 @@ export async function startDeliveryWorker() {
|
|
|
1083
841
|
await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
|
|
1084
842
|
subscriber.on("message", (_channel: string, key: string) => templateCache.invalidateKey(key));
|
|
1085
843
|
|
|
1086
|
-
// ─── Health check interval ──────────────────────────────────────────────────
|
|
1087
|
-
|
|
1088
844
|
// Outcomes for the API's live event stream, which may be in another process.
|
|
1089
845
|
stopLiveEvents = publishDeliveryEvents(redis.native, globalEmitter);
|
|
1090
846
|
|
|
847
|
+
// ─── Health check interval ──────────────────────────────────────────────────
|
|
848
|
+
|
|
1091
849
|
healthInterval = startHealthReporter("delivery", worker, redis, logger);
|
|
1092
850
|
|
|
1093
851
|
logger.info(
|