notifkit 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/index-C-DJYu0_.css +1 -0
- package/dashboard/dist/assets/index-CzFZcEon.js +318 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/{bootstrap-BOmT69OH.mjs → bootstrap-BMDRPjgp.mjs} +1 -1
- package/dist/{bulk-ZPiOJRx-.mjs → bulk-CJ5QXiPa.mjs} +1 -1
- package/dist/index.d.mts +71 -27
- package/dist/index.mjs +2 -2
- package/dist/{main-CCqcFDDe.mjs → main-6_bXQBDH.mjs} +21 -41
- package/dist/{main-BdZvBgV7.mjs → main-8nP3qmCT.mjs} +22 -48
- package/dist/{main-C0m9-Eos.mjs → main-B82bsFoy.mjs} +183 -334
- package/dist/{main-CTjaZ7ZD.mjs → main-CpaHF1Zw.mjs} +13 -15
- package/dist/{main-JEXBLP2A.mjs → main-DULD8-th.mjs} +8 -6
- package/dist/{main-C6S3EsJ3.mjs → main-ZW5RaiOK.mjs} +77 -64
- package/dist/{main-g33jpia4.mjs → main-g1KEcXXp.mjs} +676 -582
- package/dist/{main-24J4U8yP.mjs → main-mGIT8jv4.mjs} +4 -4
- package/dist/schedule-or-dispatch-Cd7GLfc3.mjs +52 -0
- package/dist/{src-9LXITcNi.mjs → src-BE_EUpZ6.mjs} +93 -93
- package/package.json +1 -1
- package/scripts/create-project.mjs +6 -2
- package/src/client.ts +3 -3
- package/src/config/index.ts +1 -0
- package/src/contracts/common.ts +2 -0
- package/src/contracts/envelope.ts +1 -1
- package/src/contracts/events/notification-created.ts +2 -0
- package/src/contracts/events/notification-delivered.ts +1 -1
- package/src/contracts/events/notification-failed.ts +1 -1
- package/src/contracts/events/notification-requested.ts +3 -2
- package/src/contracts/helpers.ts +1 -2
- package/src/contracts/index.ts +1 -0
- package/src/contracts/registry.ts +1 -0
- package/src/contracts/sdk.ts +8 -37
- package/src/db/schema.ts +8 -13
- package/src/idempotency/index.ts +6 -8
- package/src/metrics/index.ts +2 -0
- package/src/rate-limiter/api-rate-limiter.ts +208 -0
- package/src/rate-limiter/index.ts +3 -411
- package/src/rate-limiter/project-settings-cache.ts +99 -0
- package/src/rate-limiter/user-throttle.ts +100 -0
- package/src/redis/index.ts +10 -3
- package/src/repositories/admin-user.ts +83 -0
- package/src/repositories/contact.ts +359 -0
- package/src/repositories/index.ts +9 -1416
- package/src/repositories/preference.ts +35 -0
- package/src/repositories/project.ts +110 -0
- package/src/repositories/segment.ts +16 -0
- package/src/repositories/template.ts +72 -0
- package/src/repositories/types.ts +49 -0
- package/src/repositories/user.ts +644 -0
- package/src/repositories/workflow.ts +64 -0
- package/src/services/ai/main.ts +29 -71
- package/src/services/api/admin-static.ts +4 -1
- package/src/services/api/handlers/auth.ts +109 -0
- package/src/services/api/handlers/campaigns.ts +191 -0
- package/src/services/api/handlers/events.ts +99 -0
- package/src/services/api/handlers/notifications.ts +356 -0
- package/src/services/api/handlers/projects.ts +116 -0
- package/src/services/api/handlers/shared.ts +262 -0
- package/src/services/api/handlers/suppressions.ts +95 -0
- package/src/services/api/handlers/system.ts +314 -0
- package/src/services/api/handlers/templates.ts +97 -0
- package/src/services/api/handlers/unsubscribe.ts +215 -0
- package/src/services/api/handlers/users.ts +262 -0
- package/src/services/api/handlers/workflows.ts +150 -0
- package/src/services/api/handlers.ts +25 -2190
- package/src/services/api/health.ts +133 -0
- package/src/services/api/http.ts +12 -0
- package/src/services/api/main.ts +32 -322
- package/src/services/api/router.ts +1 -1
- package/src/services/api/webhooks.ts +172 -0
- package/src/services/delivery/main.ts +256 -498
- package/src/services/engine/main.ts +31 -247
- package/src/services/engine/quiet-hours.ts +151 -0
- package/src/services/engine/schedule-or-dispatch.ts +104 -0
- package/src/services/enricher/main.ts +9 -21
- package/src/services/events/main.ts +2 -2
- package/src/services/scheduler/main.ts +16 -25
- package/src/services/workflow/duration.ts +24 -0
- package/src/services/workflow/main.ts +6 -164
- package/src/services/workflow/timers.ts +157 -0
- package/src/shared/events.ts +3 -2
- package/src/templates/render.ts +1 -0
- package/src/workers/index.ts +7 -4
- package/src/workflows/sdk.ts +1 -0
- package/dashboard/dist/assets/index-ggpJKZqZ.js +0 -315
- package/dashboard/dist/assets/index-s-SXGIwv.css +0 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { $t as
|
|
2
|
-
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-
|
|
3
|
-
import { n as upsertOutboxProviderIds } from "./bulk-
|
|
1
|
+
import { $t as OUTBOUND_STREAMS, An as globalEmitter, D as ContactRepository, Dn as getPriorityBucket, E as TemplateRepository, H as ApiRateLimiter, J as StreamProducer, Q as IdempotencyGuard, Tn as BatchProcessor, X as metrics, Xt as CONSUMER_GROUPS, Yt as buildStreamEvent, _ as TemplateCache, at as deliveryOutbox, c as BaseWorker, d as startHealthReporter, en as PUBSUB_CHANNELS, g as transportRegistry, gn as readBaseConfig, lt as scheduledPayloads, mn as loadEnv, on as registry, tn as STREAMS, u as NonRetryableError, wn as CircuitBreaker, x as renderWithTemplate } from "./src-BE_EUpZ6.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-BMDRPjgp.mjs";
|
|
3
|
+
import { n as upsertOutboxProviderIds } from "./bulk-CJ5QXiPa.mjs";
|
|
4
4
|
import { sql } from "drizzle-orm";
|
|
5
5
|
//#region src/services/delivery/live-events.ts
|
|
6
6
|
/**
|
|
@@ -101,9 +101,8 @@ var ProviderThrottle = class {
|
|
|
101
101
|
};
|
|
102
102
|
//#endregion
|
|
103
103
|
//#region src/services/delivery/main.ts
|
|
104
|
-
const deliveryConfigSchema = baseConfigSchema.extend({});
|
|
105
104
|
loadEnv();
|
|
106
|
-
const config =
|
|
105
|
+
const config = readBaseConfig();
|
|
107
106
|
let logger;
|
|
108
107
|
let redis;
|
|
109
108
|
let sql$1;
|
|
@@ -116,16 +115,6 @@ let enrichedProducers;
|
|
|
116
115
|
let healthInterval = null;
|
|
117
116
|
let subscriber = null;
|
|
118
117
|
let stopLiveEvents = null;
|
|
119
|
-
global._telemetry = global._telemetry || {
|
|
120
|
-
count: 0,
|
|
121
|
-
insert: 0,
|
|
122
|
-
provider: 0,
|
|
123
|
-
flush: 0,
|
|
124
|
-
ack: 0,
|
|
125
|
-
dequeue: 0,
|
|
126
|
-
dbupdate: 0,
|
|
127
|
-
flushCount: 0
|
|
128
|
-
};
|
|
129
118
|
/** How long a send may run before its dispatched log entry is published alone. */
|
|
130
119
|
const DISPATCH_LOG_DELAY_MS = 1e3;
|
|
131
120
|
/**
|
|
@@ -206,10 +195,7 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
206
195
|
destination: update.destination,
|
|
207
196
|
providerMessageId: update.providerMessageId
|
|
208
197
|
}));
|
|
209
|
-
const tDbUpdateStart = Date.now();
|
|
210
198
|
await upsertOutboxProviderIds(this.db, values).catch((e) => this.logger.error({ err: e }, "background update failed"));
|
|
211
|
-
global._telemetry.dbupdate += Date.now() - tDbUpdateStart;
|
|
212
|
-
global._telemetry.flushCount++;
|
|
213
199
|
return updates.map(() => void 0);
|
|
214
200
|
}, flushConcurrency);
|
|
215
201
|
this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
|
|
@@ -299,6 +285,72 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
299
285
|
const task = payloadResult.data;
|
|
300
286
|
const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
|
|
301
287
|
const original = { ...task };
|
|
288
|
+
let dispatchStarted = false;
|
|
289
|
+
let dispatchLogged = false;
|
|
290
|
+
let dispatchedAt = "";
|
|
291
|
+
let dispatchLogTimer;
|
|
292
|
+
const logDispatched = () => {
|
|
293
|
+
clearTimeout(dispatchLogTimer);
|
|
294
|
+
if (!dispatchStarted || dispatchLogged) return void 0;
|
|
295
|
+
dispatchLogged = true;
|
|
296
|
+
const logged = this.eventProcessor.add(buildStreamEvent("notification.dispatched", {
|
|
297
|
+
projectId: task.projectId,
|
|
298
|
+
taskId: task.taskId,
|
|
299
|
+
enrichedEventId: task.enrichedEventId,
|
|
300
|
+
recipientId: task.recipientId,
|
|
301
|
+
channel: task.channel,
|
|
302
|
+
templateId: task.templateId,
|
|
303
|
+
attempt: attemptNo,
|
|
304
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
305
|
+
campaignId: task.campaignId
|
|
306
|
+
}, "delivery", event.metadata.traceId));
|
|
307
|
+
publishPromises.push(logged);
|
|
308
|
+
return logged;
|
|
309
|
+
};
|
|
310
|
+
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
311
|
+
const outcome = (payload) => {
|
|
312
|
+
clearTimeout(dispatchLogTimer);
|
|
313
|
+
if (dispatchLogged) return payload;
|
|
314
|
+
dispatchLogged = true;
|
|
315
|
+
return {
|
|
316
|
+
...payload,
|
|
317
|
+
attempt: attemptNo,
|
|
318
|
+
dispatchedAt
|
|
319
|
+
};
|
|
320
|
+
};
|
|
321
|
+
/**
|
|
322
|
+
* Queues the task's `notification.failed` event; once sending has started it
|
|
323
|
+
* is the attempt's outcome. Only the non-push provider failure leaves out
|
|
324
|
+
* `recipientId`.
|
|
325
|
+
*/
|
|
326
|
+
const failed = (failureReason, failureCode, withRecipientId = true) => {
|
|
327
|
+
const payload = {
|
|
328
|
+
projectId: task.projectId,
|
|
329
|
+
taskId: task.taskId,
|
|
330
|
+
enrichedEventId: task.enrichedEventId,
|
|
331
|
+
...withRecipientId ? { recipientId: task.recipientId } : {},
|
|
332
|
+
channel: task.channel,
|
|
333
|
+
failureReason,
|
|
334
|
+
failureCode,
|
|
335
|
+
retryable: false,
|
|
336
|
+
attempt: attemptNo,
|
|
337
|
+
templateId: task.templateId,
|
|
338
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
339
|
+
campaignId: task.campaignId
|
|
340
|
+
};
|
|
341
|
+
return this.eventProcessor.add(buildStreamEvent("notification.failed", dispatchStarted ? outcome(payload) : payload, "delivery", event.metadata.traceId));
|
|
342
|
+
};
|
|
343
|
+
const delivered = (providerMessageId) => this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
|
|
344
|
+
projectId: task.projectId,
|
|
345
|
+
taskId: task.taskId,
|
|
346
|
+
enrichedEventId: task.enrichedEventId,
|
|
347
|
+
channel: task.channel,
|
|
348
|
+
deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
349
|
+
providerMessageId,
|
|
350
|
+
templateId: task.templateId,
|
|
351
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
352
|
+
campaignId: task.campaignId
|
|
353
|
+
}), "delivery", event.metadata.traceId));
|
|
302
354
|
const fallbackToNextChannel = async (reason) => {
|
|
303
355
|
if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
|
|
304
356
|
const nextChannel = task.fallbackChain[0];
|
|
@@ -339,20 +391,7 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
339
391
|
}, "no transport registered for channel — dropping");
|
|
340
392
|
if (!await fallbackToNextChannel("no_transport")) {
|
|
341
393
|
this.globalEmitter.emit("delivery:failed", task.taskId, "no transport", task.channel, task.projectId);
|
|
342
|
-
await
|
|
343
|
-
projectId: task.projectId,
|
|
344
|
-
taskId: task.taskId,
|
|
345
|
-
enrichedEventId: task.enrichedEventId,
|
|
346
|
-
recipientId: task.recipientId,
|
|
347
|
-
channel: task.channel,
|
|
348
|
-
failureReason: "no transport registered for channel",
|
|
349
|
-
failureCode: "no_transport",
|
|
350
|
-
retryable: false,
|
|
351
|
-
attempt: attemptNo,
|
|
352
|
-
templateId: task.templateId,
|
|
353
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
354
|
-
campaignId: task.campaignId
|
|
355
|
-
}, "delivery", event.metadata.traceId));
|
|
394
|
+
await failed("no transport registered for channel", "no_transport");
|
|
356
395
|
}
|
|
357
396
|
return;
|
|
358
397
|
}
|
|
@@ -378,20 +417,7 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
378
417
|
}, "provider rate limit max attempts exceeded");
|
|
379
418
|
if (!await fallbackToNextChannel("provider_throttle_exceeded")) {
|
|
380
419
|
this.globalEmitter.emit("delivery:failed", task.taskId, "provider throttle exceeded", task.channel, task.projectId);
|
|
381
|
-
await
|
|
382
|
-
projectId: task.projectId,
|
|
383
|
-
taskId: task.taskId,
|
|
384
|
-
enrichedEventId: task.enrichedEventId,
|
|
385
|
-
recipientId: task.recipientId,
|
|
386
|
-
channel: task.channel,
|
|
387
|
-
failureReason: "provider rate limit max attempts exceeded",
|
|
388
|
-
failureCode: "provider_throttle_exceeded",
|
|
389
|
-
retryable: false,
|
|
390
|
-
attempt: attemptNo,
|
|
391
|
-
templateId: task.templateId,
|
|
392
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
393
|
-
campaignId: task.campaignId
|
|
394
|
-
}, "delivery", event.metadata.traceId));
|
|
420
|
+
await failed("provider rate limit max attempts exceeded", "provider_throttle_exceeded");
|
|
395
421
|
}
|
|
396
422
|
return;
|
|
397
423
|
}
|
|
@@ -423,39 +449,6 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
423
449
|
}
|
|
424
450
|
const tDispatchStart = Date.now();
|
|
425
451
|
const MAX_DISPATCH_BUDGET_MS = 25e3;
|
|
426
|
-
let dispatchStarted = false;
|
|
427
|
-
let dispatchLogged = false;
|
|
428
|
-
let dispatchedAt = "";
|
|
429
|
-
let dispatchLogTimer;
|
|
430
|
-
const logDispatched = () => {
|
|
431
|
-
clearTimeout(dispatchLogTimer);
|
|
432
|
-
if (!dispatchStarted || dispatchLogged) return void 0;
|
|
433
|
-
dispatchLogged = true;
|
|
434
|
-
const logged = this.eventProcessor.add(buildStreamEvent("notification.dispatched", {
|
|
435
|
-
projectId: task.projectId,
|
|
436
|
-
taskId: task.taskId,
|
|
437
|
-
enrichedEventId: task.enrichedEventId,
|
|
438
|
-
recipientId: task.recipientId,
|
|
439
|
-
channel: task.channel,
|
|
440
|
-
templateId: task.templateId,
|
|
441
|
-
attempt: attemptNo,
|
|
442
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
443
|
-
campaignId: task.campaignId
|
|
444
|
-
}, "delivery", event.metadata.traceId));
|
|
445
|
-
publishPromises.push(logged);
|
|
446
|
-
return logged;
|
|
447
|
-
};
|
|
448
|
-
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
449
|
-
const outcome = (payload) => {
|
|
450
|
-
clearTimeout(dispatchLogTimer);
|
|
451
|
-
if (dispatchLogged) return payload;
|
|
452
|
-
dispatchLogged = true;
|
|
453
|
-
return {
|
|
454
|
-
...payload,
|
|
455
|
-
attempt: attemptNo,
|
|
456
|
-
dispatchedAt
|
|
457
|
-
};
|
|
458
|
-
};
|
|
459
452
|
let retryScheduled = false;
|
|
460
453
|
/**
|
|
461
454
|
* Re-queues a send the provider turned away, unless it said the failure is
|
|
@@ -494,291 +487,147 @@ var DeliveryWorker = class extends BaseWorker {
|
|
|
494
487
|
dispatchLogTimer = setTimeout(() => {
|
|
495
488
|
logDispatched();
|
|
496
489
|
}, DISPATCH_LOG_DELAY_MS);
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
this.logger.warn({
|
|
509
|
-
taskId: task.taskId,
|
|
510
|
-
channel: task.channel
|
|
511
|
-
}, "dispatch budget exhausted before transport call — aborting to protect lock lease");
|
|
512
|
-
break;
|
|
513
|
-
}
|
|
514
|
-
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
|
|
515
|
-
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
516
|
-
try {
|
|
517
|
-
const tProv = Date.now();
|
|
518
|
-
lastResult = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
|
|
519
|
-
const controller = new AbortController();
|
|
520
|
-
const timeout = setTimeout(() => {
|
|
521
|
-
controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
522
|
-
}, timeoutMs);
|
|
523
|
-
try {
|
|
524
|
-
task.signal = controller.signal;
|
|
525
|
-
const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
|
|
526
|
-
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
527
|
-
controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
|
|
528
|
-
})]);
|
|
529
|
-
if (!res.success && !res.invalidToken && res.retryable !== false) throw providerFailure(res);
|
|
530
|
-
return res;
|
|
531
|
-
} finally {
|
|
532
|
-
clearTimeout(timeout);
|
|
533
|
-
}
|
|
534
|
-
});
|
|
535
|
-
global._telemetry.provider += Date.now() - tProv;
|
|
536
|
-
if (lastResult.success || lastResult.invalidToken) break;
|
|
537
|
-
} catch (err) {
|
|
538
|
-
lastResult = {
|
|
539
|
-
success: false,
|
|
540
|
-
error: err.message,
|
|
541
|
-
retryable: err.retryable
|
|
542
|
-
};
|
|
543
|
-
}
|
|
544
|
-
}
|
|
545
|
-
if (lastResult.invalidToken) {
|
|
546
|
-
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
547
|
-
taskId: task.taskId,
|
|
548
|
-
channel: task.channel,
|
|
549
|
-
destination: task.destination,
|
|
550
|
-
providerMessageId: null
|
|
551
|
-
}));
|
|
552
|
-
await this.contactRepo.deactivate(task.projectId, task.recipientId, task.channel, task.destination);
|
|
553
|
-
this.logger.info({
|
|
554
|
-
channel: task.channel,
|
|
555
|
-
target: task.destination
|
|
556
|
-
}, "deactivated invalid contact target");
|
|
557
|
-
this.globalEmitter.emit("delivery:failed", task.taskId, "invalidToken", task.channel, task.projectId);
|
|
558
|
-
metrics.deliveryFailed.inc({
|
|
559
|
-
channel: task.channel,
|
|
560
|
-
reason: "invalid_token"
|
|
561
|
-
});
|
|
562
|
-
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
|
|
563
|
-
projectId: task.projectId,
|
|
564
|
-
taskId: task.taskId,
|
|
565
|
-
enrichedEventId: task.enrichedEventId,
|
|
566
|
-
recipientId: task.recipientId,
|
|
567
|
-
channel: task.channel,
|
|
568
|
-
failureReason: "invalid_token",
|
|
569
|
-
failureCode: "push_failure",
|
|
570
|
-
retryable: false,
|
|
571
|
-
attempt: attemptNo,
|
|
572
|
-
templateId: task.templateId,
|
|
573
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
574
|
-
campaignId: task.campaignId
|
|
575
|
-
}), "delivery", event.metadata.traceId)));
|
|
576
|
-
if (!await fallbackToNextChannel("invalid_token")) throw new NonRetryableError("Push delivery failed: invalid token");
|
|
577
|
-
} else if (lastResult.success) {
|
|
578
|
-
const providerMessageId = lastResult.providerMessageId || "push-success";
|
|
579
|
-
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
580
|
-
taskId: task.taskId,
|
|
581
|
-
channel: task.channel,
|
|
582
|
-
destination: task.destination,
|
|
583
|
-
providerMessageId
|
|
584
|
-
}));
|
|
585
|
-
this.logger.debug({
|
|
586
|
-
taskId: task.taskId,
|
|
587
|
-
messageId: providerMessageId
|
|
588
|
-
}, "push delivered");
|
|
589
|
-
this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
|
|
590
|
-
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
591
|
-
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
|
|
592
|
-
projectId: task.projectId,
|
|
593
|
-
taskId: task.taskId,
|
|
594
|
-
enrichedEventId: task.enrichedEventId,
|
|
595
|
-
channel: task.channel,
|
|
596
|
-
deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
597
|
-
providerMessageId,
|
|
598
|
-
templateId: task.templateId,
|
|
599
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
600
|
-
campaignId: task.campaignId
|
|
601
|
-
}), "delivery", event.metadata.traceId)));
|
|
602
|
-
} else if (!await retryLater(lastResult)) {
|
|
490
|
+
const isPush = task.channel === "push";
|
|
491
|
+
let result = {
|
|
492
|
+
success: false,
|
|
493
|
+
error: "No transports"
|
|
494
|
+
};
|
|
495
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
496
|
+
const transport = transports[i];
|
|
497
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) continue;
|
|
498
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
499
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
500
|
+
if (remainingBudgetMs <= 0) {
|
|
603
501
|
this.logger.warn({
|
|
604
502
|
taskId: task.taskId,
|
|
605
|
-
|
|
606
|
-
}, "
|
|
607
|
-
|
|
608
|
-
metrics.deliveryFailed.inc({
|
|
609
|
-
channel: task.channel,
|
|
610
|
-
reason: "push_error"
|
|
611
|
-
});
|
|
612
|
-
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
613
|
-
taskId: task.taskId,
|
|
614
|
-
channel: task.channel,
|
|
615
|
-
destination: task.destination,
|
|
616
|
-
providerMessageId: null
|
|
617
|
-
}));
|
|
618
|
-
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
|
|
619
|
-
projectId: task.projectId,
|
|
620
|
-
taskId: task.taskId,
|
|
621
|
-
enrichedEventId: task.enrichedEventId,
|
|
622
|
-
recipientId: task.recipientId,
|
|
623
|
-
channel: task.channel,
|
|
624
|
-
failureReason: lastResult.error ?? "push delivery failed",
|
|
625
|
-
failureCode: "push_failure",
|
|
626
|
-
retryable: false,
|
|
627
|
-
attempt: attemptNo,
|
|
628
|
-
templateId: task.templateId,
|
|
629
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
630
|
-
campaignId: task.campaignId
|
|
631
|
-
}), "delivery", event.metadata.traceId)));
|
|
632
|
-
if (!await fallbackToNextChannel("push_delivery_failed")) throw new NonRetryableError(lastResult.error ?? "Push delivery failed");
|
|
503
|
+
channel: task.channel
|
|
504
|
+
}, "dispatch budget exhausted before transport call — aborting to protect lock lease");
|
|
505
|
+
break;
|
|
633
506
|
}
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
task.signal = controller.signal;
|
|
662
|
-
const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
|
|
663
|
-
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
664
|
-
controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
|
|
665
|
-
})]);
|
|
666
|
-
if (!res.success && res.retryable !== false) throw providerFailure(res);
|
|
667
|
-
return res;
|
|
668
|
-
} finally {
|
|
669
|
-
clearTimeout(timeout);
|
|
670
|
-
}
|
|
671
|
-
});
|
|
672
|
-
global._telemetry.provider += Date.now() - tProv;
|
|
673
|
-
if (result.success) break;
|
|
674
|
-
} catch (err) {
|
|
675
|
-
result = {
|
|
676
|
-
success: false,
|
|
677
|
-
error: err.message,
|
|
678
|
-
retryable: err.retryable
|
|
679
|
-
};
|
|
680
|
-
}
|
|
507
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
|
|
508
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
509
|
+
try {
|
|
510
|
+
result = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
|
|
511
|
+
const controller = new AbortController();
|
|
512
|
+
const timeout = setTimeout(() => {
|
|
513
|
+
controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
514
|
+
}, timeoutMs);
|
|
515
|
+
try {
|
|
516
|
+
task.signal = controller.signal;
|
|
517
|
+
const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
|
|
518
|
+
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
519
|
+
controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
|
|
520
|
+
})]);
|
|
521
|
+
if (!res.success && !(isPush && res.invalidToken) && res.retryable !== false) throw providerFailure(res);
|
|
522
|
+
return res;
|
|
523
|
+
} finally {
|
|
524
|
+
clearTimeout(timeout);
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
if (result.success || isPush && result.invalidToken) break;
|
|
528
|
+
} catch (err) {
|
|
529
|
+
result = {
|
|
530
|
+
success: false,
|
|
531
|
+
error: err.message,
|
|
532
|
+
retryable: err.retryable
|
|
533
|
+
};
|
|
681
534
|
}
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
535
|
+
}
|
|
536
|
+
if (isPush && result.invalidToken) {
|
|
537
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
538
|
+
taskId: task.taskId,
|
|
539
|
+
channel: task.channel,
|
|
540
|
+
destination: task.destination,
|
|
541
|
+
providerMessageId: null
|
|
542
|
+
}));
|
|
543
|
+
await this.contactRepo.deactivate(task.projectId, task.recipientId, task.channel, task.destination);
|
|
544
|
+
this.logger.info({
|
|
545
|
+
channel: task.channel,
|
|
546
|
+
target: task.destination
|
|
547
|
+
}, "deactivated invalid contact target");
|
|
548
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, "invalidToken", task.channel, task.projectId);
|
|
549
|
+
metrics.deliveryFailed.inc({
|
|
550
|
+
channel: task.channel,
|
|
551
|
+
reason: "invalid_token"
|
|
552
|
+
});
|
|
553
|
+
publishPromises.push(failed("invalid_token", "push_failure"));
|
|
554
|
+
if (!await fallbackToNextChannel("invalid_token")) throw new NonRetryableError("Push delivery failed: invalid token");
|
|
555
|
+
} else if (result.success) {
|
|
556
|
+
const providerMessageId = result.providerMessageId || (isPush ? "push-success" : "success");
|
|
557
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
558
|
+
taskId: task.taskId,
|
|
559
|
+
channel: task.channel,
|
|
560
|
+
destination: task.destination,
|
|
561
|
+
providerMessageId
|
|
562
|
+
}));
|
|
563
|
+
if (isPush) this.logger.debug({
|
|
564
|
+
taskId: task.taskId,
|
|
565
|
+
messageId: providerMessageId
|
|
566
|
+
}, "push delivered");
|
|
567
|
+
this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
|
|
568
|
+
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
569
|
+
publishPromises.push(delivered(providerMessageId));
|
|
570
|
+
if (!isPush) this.logger.debug({
|
|
571
|
+
taskId: task.taskId,
|
|
572
|
+
channel: task.channel,
|
|
573
|
+
messageId: result.providerMessageId
|
|
574
|
+
}, "notification delivered");
|
|
575
|
+
} else if (!await retryLater(result)) {
|
|
576
|
+
const error = result.error ?? (isPush ? "push delivery failed" : "delivery failed completely");
|
|
577
|
+
const report = () => {
|
|
578
|
+
if (isPush) this.logger.warn({
|
|
712
579
|
taskId: task.taskId,
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
failureCode: "provider_error",
|
|
717
|
-
retryable: false,
|
|
718
|
-
attempt: attemptNo,
|
|
719
|
-
templateId: task.templateId,
|
|
720
|
-
workflowInstanceId: task.workflowInstanceId,
|
|
721
|
-
campaignId: task.campaignId
|
|
722
|
-
}), "delivery", event.metadata.traceId)));
|
|
723
|
-
}
|
|
724
|
-
if (!result.success && !retryScheduled) {
|
|
725
|
-
this.logger.warn({
|
|
580
|
+
error: result.error
|
|
581
|
+
}, "push delivery failed");
|
|
582
|
+
else this.logger.warn({
|
|
726
583
|
taskId: task.taskId,
|
|
727
584
|
channel: task.channel,
|
|
728
585
|
error: result.error
|
|
729
586
|
}, "delivery failed completely across providers");
|
|
730
|
-
this.globalEmitter.emit("delivery:failed", task.taskId,
|
|
587
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, error, task.channel, task.projectId);
|
|
731
588
|
metrics.deliveryFailed.inc({
|
|
732
589
|
channel: task.channel,
|
|
733
|
-
reason: "provider_error"
|
|
590
|
+
reason: isPush ? "push_error" : "provider_error"
|
|
734
591
|
});
|
|
735
|
-
|
|
736
|
-
|
|
592
|
+
};
|
|
593
|
+
if (isPush) report();
|
|
594
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
737
595
|
taskId: task.taskId,
|
|
738
596
|
channel: task.channel,
|
|
739
|
-
|
|
740
|
-
|
|
597
|
+
destination: task.destination,
|
|
598
|
+
providerMessageId: null
|
|
599
|
+
}));
|
|
600
|
+
publishPromises.push(isPush ? failed(error, "push_failure") : failed(error, "provider_error", false));
|
|
601
|
+
if (!isPush) report();
|
|
602
|
+
if (!await fallbackToNextChannel(isPush ? "push_delivery_failed" : "all_providers_failed")) throw new NonRetryableError(result.error ?? (isPush ? "Push delivery failed" : "delivery failed"));
|
|
741
603
|
}
|
|
742
604
|
logDispatched();
|
|
743
|
-
const tFlushStart = Date.now();
|
|
744
605
|
await Promise.all(publishPromises).catch((err) => {
|
|
745
606
|
this.logger.error({
|
|
746
607
|
err,
|
|
747
608
|
taskId: task.taskId
|
|
748
609
|
}, "failed to publish post-dispatch events, swallowing error to prevent duplicate delivery");
|
|
749
610
|
});
|
|
750
|
-
|
|
751
|
-
|
|
611
|
+
if (retryScheduled) await this.idempotency.unmark(idempotencyKey).catch((err) => {
|
|
612
|
+
this.logger.warn({
|
|
613
|
+
err,
|
|
614
|
+
taskId: task.taskId
|
|
615
|
+
}, "failed to release idempotency lock");
|
|
616
|
+
});
|
|
752
617
|
else this.idempotency.markProcessed(idempotencyKey).catch((err) => {
|
|
753
618
|
this.logger.warn({
|
|
754
619
|
err,
|
|
755
620
|
taskId: task.taskId
|
|
756
621
|
}, "failed to upgrade idempotency TTL");
|
|
757
622
|
});
|
|
758
|
-
const t = global._telemetry = global._telemetry || {
|
|
759
|
-
count: 0,
|
|
760
|
-
insert: 0,
|
|
761
|
-
provider: 0,
|
|
762
|
-
flush: 0,
|
|
763
|
-
ack: 0
|
|
764
|
-
};
|
|
765
|
-
t.count++;
|
|
766
|
-
t.insert += 0;
|
|
767
|
-
t.flush += flushTime;
|
|
768
|
-
if (t.count % 1e3 === 0) {
|
|
769
|
-
this.logger.debug(`[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1e3}ms, DB Insert: ${t.insert / 1e3}ms, Provider: ${t.provider / 1e3}ms, Wait for Flush: ${t.flush / 1e3}ms, Ack: ${t.ack / 1e3}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`);
|
|
770
|
-
t.count = 0;
|
|
771
|
-
t.insert = 0;
|
|
772
|
-
t.provider = 0;
|
|
773
|
-
t.flush = 0;
|
|
774
|
-
t.dequeue = 0;
|
|
775
|
-
t.ack = 0;
|
|
776
|
-
t.dbupdate = 0;
|
|
777
|
-
t.flushCount = 0;
|
|
778
|
-
}
|
|
779
623
|
} catch (err) {
|
|
780
624
|
logDispatched()?.catch(() => {});
|
|
781
|
-
await this.idempotency.unmark(idempotencyKey).catch(() => {
|
|
625
|
+
await this.idempotency.unmark(idempotencyKey).catch((e) => {
|
|
626
|
+
this.logger.warn({
|
|
627
|
+
err: e,
|
|
628
|
+
taskId: task.taskId
|
|
629
|
+
}, "failed to release idempotency lock");
|
|
630
|
+
});
|
|
782
631
|
throw err;
|
|
783
632
|
}
|
|
784
633
|
}
|