notifkit 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-GRXzgPvW.mjs +89 -0
- package/dist/bulk-BvRfz6Of.mjs +77 -0
- package/dist/index.d.mts +1100 -8883
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-BENOY7HG.mjs} +49 -75
- package/dist/{main-BiFiOX9D.mjs → main-BpnuFX62.mjs} +119 -112
- package/dist/{main-Dz2sUh3c.mjs → main-BqVH2BHj.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-C_5VFIwp.mjs} +135 -79
- package/dist/main-D6od9N8h.mjs +312 -0
- package/dist/main-D84OX0ux.mjs +811 -0
- package/dist/{main-39DtKrX4.mjs → main-HH-WwsMM.mjs} +395 -184
- package/dist/{main-B3iQrl3t.mjs → main-HY_TDs61.mjs} +149 -134
- package/dist/{src-QiTu5e_a.mjs → src-CI7FGf3h.mjs} +1220 -651
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +17 -17
- package/src/client.ts +15 -5
- package/src/config/index.ts +26 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +2 -2
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +61 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -0,0 +1,811 @@
|
|
|
1
|
+
import { $ as deliveryOutbox, B as ApiRateLimiter, C as ContactRepository, D as TemplateRepository, J as metrics, Jt as PUBSUB_CHANNELS, K as StreamProducer, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, X as IdempotencyGuard, Yt as STREAMS, _ as TemplateCache, c as BaseWorker, cn as loadEnv, d as startHealthReporter, en as registry, g as transportRegistry, ln as parseConfig, on as baseConfigSchema, qt as OUTBOUND_STREAMS, rt as scheduledPayloads, u as NonRetryableError, vn as CircuitBreaker, wn as globalEmitter, x as renderWithTemplate, xn as getPriorityBucket, yn as BatchProcessor } from "./src-CI7FGf3h.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-GRXzgPvW.mjs";
|
|
3
|
+
import { n as upsertOutboxProviderIds } from "./bulk-BvRfz6Of.mjs";
|
|
4
|
+
import { sql } from "drizzle-orm";
|
|
5
|
+
//#region src/services/delivery/throttle.ts
|
|
6
|
+
/**
|
|
7
|
+
* Provider rate limit enforced through locally leased tokens.
|
|
8
|
+
*
|
|
9
|
+
* A per-message check against one sorted set per channel would serialise every
|
|
10
|
+
* delivery process in the cluster on a single hot key. Instead each process
|
|
11
|
+
* leases a chunk of the window's budget and spends it in memory, contacting
|
|
12
|
+
* Redis only when its chunk runs low. The window is a sliding estimate over
|
|
13
|
+
* two fixed buckets rather than an exact log, the same trade the API ingress
|
|
14
|
+
* limiter already makes.
|
|
15
|
+
*/
|
|
16
|
+
var ProviderThrottle = class {
|
|
17
|
+
redis;
|
|
18
|
+
logger;
|
|
19
|
+
limiters = /* @__PURE__ */ new Map();
|
|
20
|
+
constructor(redis, logger) {
|
|
21
|
+
this.redis = redis;
|
|
22
|
+
this.logger = logger;
|
|
23
|
+
}
|
|
24
|
+
async check(channel, config) {
|
|
25
|
+
const windowMs = Math.max(1, Math.round(config.windowSeconds * 1e3));
|
|
26
|
+
let limiter = this.limiters.get(windowMs);
|
|
27
|
+
if (!limiter) {
|
|
28
|
+
limiter = new ApiRateLimiter({
|
|
29
|
+
redis: this.redis,
|
|
30
|
+
windowMs,
|
|
31
|
+
keyPrefix: "rate-limit:provider",
|
|
32
|
+
logger: this.logger
|
|
33
|
+
});
|
|
34
|
+
this.limiters.set(windowMs, limiter);
|
|
35
|
+
}
|
|
36
|
+
if (await limiter.check(channel, config.limit)) return {
|
|
37
|
+
allowed: true,
|
|
38
|
+
retryAfterMs: 0
|
|
39
|
+
};
|
|
40
|
+
const retryAfterMs = windowMs - Date.now() % windowMs;
|
|
41
|
+
this.logger?.warn({
|
|
42
|
+
channel,
|
|
43
|
+
limit: config.limit,
|
|
44
|
+
windowSeconds: config.windowSeconds,
|
|
45
|
+
retryAfterMs
|
|
46
|
+
}, "Provider rate limit hit — task must be rescheduled");
|
|
47
|
+
return {
|
|
48
|
+
allowed: false,
|
|
49
|
+
retryAfterMs
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
//#endregion
|
|
54
|
+
//#region src/services/delivery/main.ts
|
|
55
|
+
const deliveryConfigSchema = baseConfigSchema.extend({});
|
|
56
|
+
loadEnv();
|
|
57
|
+
const config = parseConfig(deliveryConfigSchema, process.env);
|
|
58
|
+
let logger;
|
|
59
|
+
let redis;
|
|
60
|
+
let sql$1;
|
|
61
|
+
let db;
|
|
62
|
+
let consumer;
|
|
63
|
+
let pendingScanner;
|
|
64
|
+
let worker;
|
|
65
|
+
let scheduledProducer;
|
|
66
|
+
let enrichedProducers;
|
|
67
|
+
let healthInterval = null;
|
|
68
|
+
let subscriber = null;
|
|
69
|
+
global._telemetry = global._telemetry || {
|
|
70
|
+
count: 0,
|
|
71
|
+
insert: 0,
|
|
72
|
+
provider: 0,
|
|
73
|
+
flush: 0,
|
|
74
|
+
ack: 0,
|
|
75
|
+
dequeue: 0,
|
|
76
|
+
dbupdate: 0,
|
|
77
|
+
flushCount: 0
|
|
78
|
+
};
|
|
79
|
+
/** How long a send may run before its dispatched log entry is published alone. */
|
|
80
|
+
const DISPATCH_LOG_DELAY_MS = 1e3;
|
|
81
|
+
/**
|
|
82
|
+
* Wait before re-trying a send that failed at the provider, by prior failures.
|
|
83
|
+
* The first step matches the circuit breaker's reset timeout, so a retry of a
|
|
84
|
+
* task the open breaker turned away reaches the provider's recovery probe
|
|
85
|
+
* rather than the same closed door.
|
|
86
|
+
*/
|
|
87
|
+
const RETRY_BACKOFF_MS = [
|
|
88
|
+
3e4,
|
|
89
|
+
12e4,
|
|
90
|
+
6e5
|
|
91
|
+
];
|
|
92
|
+
/**
|
|
93
|
+
* How many times a send may be put back for want of provider capacity before
|
|
94
|
+
* it is failed. Separate from `maxAttempts`, which counts sends the provider
|
|
95
|
+
* refused: waiting in line is not a failed attempt, and a campaign several
|
|
96
|
+
* windows deeper than the provider's limit has to be able to wait that long.
|
|
97
|
+
*/
|
|
98
|
+
const MAX_THROTTLE_DEFERRALS = 50;
|
|
99
|
+
/**
|
|
100
|
+
* Names the provider at `transports[i]` for its rate limit and circuit breaker.
|
|
101
|
+
* The rate limit's budget lives in Redis and is shared by every delivery
|
|
102
|
+
* process, so the name must come out the same everywhere: the class name,
|
|
103
|
+
* numbered when a channel has two providers of one class.
|
|
104
|
+
*/
|
|
105
|
+
function providerKey(channel, transports, i) {
|
|
106
|
+
const name = transports[i].constructor?.name ?? "provider";
|
|
107
|
+
const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
|
|
108
|
+
return nth === 0 ? `${channel}:${name}` : `${channel}:${name}#${nth + 1}`;
|
|
109
|
+
}
|
|
110
|
+
/** A provider's refusal, keeping whether it said another attempt could work. */
|
|
111
|
+
function providerFailure(res) {
|
|
112
|
+
const err = new Error(res.error ?? "Transport failed");
|
|
113
|
+
err.retryable = res.retryable;
|
|
114
|
+
return err;
|
|
115
|
+
}
|
|
116
|
+
function retryDelayMs(priorFailures) {
|
|
117
|
+
return RETRY_BACKOFF_MS[Math.min(priorFailures, RETRY_BACKOFF_MS.length - 1)];
|
|
118
|
+
}
|
|
119
|
+
var DeliveryWorker = class extends BaseWorker {
|
|
120
|
+
transportRegistry;
|
|
121
|
+
idempotency;
|
|
122
|
+
scheduledProducer;
|
|
123
|
+
enrichedProducers;
|
|
124
|
+
contactRepo;
|
|
125
|
+
eventsProducer;
|
|
126
|
+
globalEmitter;
|
|
127
|
+
db;
|
|
128
|
+
eventProcessor;
|
|
129
|
+
outboxUpdateProcessor;
|
|
130
|
+
outboxInsertProcessor;
|
|
131
|
+
breakers = /* @__PURE__ */ new Map();
|
|
132
|
+
providerThrottle;
|
|
133
|
+
templateCache;
|
|
134
|
+
constructor(options) {
|
|
135
|
+
super(options);
|
|
136
|
+
this.transportRegistry = options.transportRegistry;
|
|
137
|
+
this.idempotency = options.idempotency;
|
|
138
|
+
this.scheduledProducer = options.scheduledProducer;
|
|
139
|
+
this.enrichedProducers = options.enrichedProducers;
|
|
140
|
+
this.contactRepo = options.contactRepo;
|
|
141
|
+
this.eventsProducer = options.eventsProducer;
|
|
142
|
+
this.globalEmitter = options.globalEmitter;
|
|
143
|
+
this.db = options.db;
|
|
144
|
+
this.providerThrottle = options.providerThrottle ?? new ProviderThrottle(options.redis, this.logger);
|
|
145
|
+
this.templateCache = options.templateCache;
|
|
146
|
+
const batchSize = Math.max(10, Math.min(options.concurrency ?? 50, 500));
|
|
147
|
+
const flushConcurrency = 4;
|
|
148
|
+
this.eventProcessor = new BatchProcessor(batchSize, 5, async (events) => {
|
|
149
|
+
await this.eventsProducer.publishBatch(events);
|
|
150
|
+
return events.map(() => void 0);
|
|
151
|
+
}, flushConcurrency);
|
|
152
|
+
this.outboxUpdateProcessor = new BatchProcessor(batchSize, 5, async (updates) => {
|
|
153
|
+
const values = updates.map((update) => ({
|
|
154
|
+
taskId: update.taskId,
|
|
155
|
+
channel: update.channel,
|
|
156
|
+
destination: update.destination,
|
|
157
|
+
providerMessageId: update.providerMessageId
|
|
158
|
+
}));
|
|
159
|
+
const tDbUpdateStart = Date.now();
|
|
160
|
+
await upsertOutboxProviderIds(this.db, values).catch((e) => this.logger.error({ err: e }, "background update failed"));
|
|
161
|
+
global._telemetry.dbupdate += Date.now() - tDbUpdateStart;
|
|
162
|
+
global._telemetry.flushCount++;
|
|
163
|
+
return updates.map(() => void 0);
|
|
164
|
+
}, flushConcurrency);
|
|
165
|
+
this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
|
|
166
|
+
const values = tasks.map((task) => ({
|
|
167
|
+
taskId: task.taskId,
|
|
168
|
+
channel: task.channel,
|
|
169
|
+
destination: task.destination
|
|
170
|
+
}));
|
|
171
|
+
await this.db.insert(deliveryOutbox).values(values).onConflictDoNothing();
|
|
172
|
+
return tasks.map(() => true);
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
async stop() {
|
|
176
|
+
await Promise.all([
|
|
177
|
+
this.eventProcessor.flush(),
|
|
178
|
+
this.outboxUpdateProcessor.flush(),
|
|
179
|
+
this.outboxInsertProcessor.flush()
|
|
180
|
+
]);
|
|
181
|
+
await super.stop();
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Renders a task the engine dispatched unrendered. Done here, after the
|
|
185
|
+
* throttle and the lease, so the body is built once per actual send and never
|
|
186
|
+
* travels through the outbound stream.
|
|
187
|
+
*/
|
|
188
|
+
async render(task) {
|
|
189
|
+
if (!task.templateId) return renderWithTemplate(null, task.templateVariables);
|
|
190
|
+
const template = this.templateCache ? await this.templateCache.getCachedTemplate(task.projectId, task.templateId) : null;
|
|
191
|
+
if (!template) throw new NonRetryableError(`template "${task.templateId}" not found at send time`);
|
|
192
|
+
return renderWithTemplate(template, task.templateVariables);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Parks a task in Postgres and the scheduler until `sendAt`. The payload
|
|
196
|
+
* carries the send time, so the scheduler can rebuild it if Redis loses its
|
|
197
|
+
* data.
|
|
198
|
+
*/
|
|
199
|
+
async reschedule(task, sendAt, reason) {
|
|
200
|
+
const scheduledAt = sendAt.toISOString();
|
|
201
|
+
await this.db.insert(scheduledPayloads).values({
|
|
202
|
+
taskId: task.taskId,
|
|
203
|
+
payload: {
|
|
204
|
+
...task,
|
|
205
|
+
scheduledAt
|
|
206
|
+
}
|
|
207
|
+
}).onConflictDoUpdate({
|
|
208
|
+
target: scheduledPayloads.taskId,
|
|
209
|
+
set: { payload: sql`EXCLUDED.payload` }
|
|
210
|
+
});
|
|
211
|
+
await this.scheduledProducer.publish(buildStreamEvent("notification.scheduled", {
|
|
212
|
+
projectId: task.projectId,
|
|
213
|
+
enrichedEventId: task.enrichedEventId,
|
|
214
|
+
taskId: task.taskId,
|
|
215
|
+
scheduledAt
|
|
216
|
+
}, "delivery", `${task.taskId}:${reason}:${Date.now()}`));
|
|
217
|
+
}
|
|
218
|
+
/** Takes a slot from the provider's rate limit, if it has one. */
|
|
219
|
+
async providerGate(channel, transports, i) {
|
|
220
|
+
const limits = transports[i].limits;
|
|
221
|
+
if (!limits) return {
|
|
222
|
+
allowed: true,
|
|
223
|
+
retryAfterMs: 0
|
|
224
|
+
};
|
|
225
|
+
return this.providerThrottle.check(providerKey(channel, transports, i), limits);
|
|
226
|
+
}
|
|
227
|
+
getBreaker(name) {
|
|
228
|
+
let breaker = this.breakers.get(name);
|
|
229
|
+
if (!breaker) {
|
|
230
|
+
breaker = new CircuitBreaker({
|
|
231
|
+
failureThreshold: 5,
|
|
232
|
+
resetTimeoutMs: 3e4
|
|
233
|
+
});
|
|
234
|
+
this.breakers.set(name, breaker);
|
|
235
|
+
}
|
|
236
|
+
return breaker;
|
|
237
|
+
}
|
|
238
|
+
async process(message, attempt = 1) {
|
|
239
|
+
const { event } = message;
|
|
240
|
+
const publishPromises = [];
|
|
241
|
+
const payloadResult = registry.safeParsePayload("notification.dispatched", event.payload);
|
|
242
|
+
if (!payloadResult.success) {
|
|
243
|
+
this.logger.warn({
|
|
244
|
+
messageId: message.id,
|
|
245
|
+
issues: payloadResult.error.issues
|
|
246
|
+
}, "invalid notification.dispatched payload — skipping");
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const task = payloadResult.data;
|
|
250
|
+
const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
|
|
251
|
+
const original = { ...task };
|
|
252
|
+
const fallbackToNextChannel = async (reason) => {
|
|
253
|
+
if (task.fallbackChain && task.fallbackChain.length > 0 && task.recipient) {
|
|
254
|
+
const nextChannel = task.fallbackChain[0];
|
|
255
|
+
const remainingChain = task.fallbackChain.slice(1);
|
|
256
|
+
const fallbackPayload = {
|
|
257
|
+
projectId: task.projectId,
|
|
258
|
+
rawEventId: task.enrichedEventId,
|
|
259
|
+
recipientId: task.recipientId,
|
|
260
|
+
channel: nextChannel,
|
|
261
|
+
priority: task.priority,
|
|
262
|
+
templateId: task.templateId,
|
|
263
|
+
templateVariables: task.templateVariables,
|
|
264
|
+
aiPrompts: task.aiPrompts,
|
|
265
|
+
recipient: task.recipient,
|
|
266
|
+
scheduledAt: void 0,
|
|
267
|
+
fallbackChain: remainingChain.length > 0 ? remainingChain : void 0,
|
|
268
|
+
campaignId: task.campaignId,
|
|
269
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
270
|
+
throttleCounted: true
|
|
271
|
+
};
|
|
272
|
+
const p = getPriorityBucket(task.priority);
|
|
273
|
+
await (this.enrichedProducers[p] ?? this.enrichedProducers["normal"]).publish(buildStreamEvent("notification.enriched", fallbackPayload, "delivery", event.metadata.traceId));
|
|
274
|
+
this.logger.info({
|
|
275
|
+
taskId: task.taskId,
|
|
276
|
+
reason,
|
|
277
|
+
nextChannel,
|
|
278
|
+
traceId: event.metadata.traceId
|
|
279
|
+
}, "Delivery failed completely, rolling over to next channel in fallback chain");
|
|
280
|
+
return true;
|
|
281
|
+
}
|
|
282
|
+
return false;
|
|
283
|
+
};
|
|
284
|
+
const transports = this.transportRegistry.getAll(task.channel);
|
|
285
|
+
if (transports.length === 0) {
|
|
286
|
+
this.logger.warn({
|
|
287
|
+
taskId: task.taskId,
|
|
288
|
+
channel: task.channel
|
|
289
|
+
}, "no transport registered for channel — dropping");
|
|
290
|
+
if (!await fallbackToNextChannel("no_transport")) {
|
|
291
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, "no transport", task.channel, task.projectId);
|
|
292
|
+
await this.eventProcessor.add(buildStreamEvent("notification.failed", {
|
|
293
|
+
projectId: task.projectId,
|
|
294
|
+
taskId: task.taskId,
|
|
295
|
+
enrichedEventId: task.enrichedEventId,
|
|
296
|
+
recipientId: task.recipientId,
|
|
297
|
+
channel: task.channel,
|
|
298
|
+
failureReason: "no transport registered for channel",
|
|
299
|
+
failureCode: "no_transport",
|
|
300
|
+
retryable: false,
|
|
301
|
+
attempt: attemptNo,
|
|
302
|
+
templateId: task.templateId,
|
|
303
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
304
|
+
campaignId: task.campaignId
|
|
305
|
+
}, "delivery", event.metadata.traceId));
|
|
306
|
+
}
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
let startAt = -1;
|
|
310
|
+
let retryAfterMs = Infinity;
|
|
311
|
+
let windowMs = Infinity;
|
|
312
|
+
for (let i = 0; i < transports.length; i++) {
|
|
313
|
+
const gate = await this.providerGate(task.channel, transports, i);
|
|
314
|
+
if (gate.allowed) {
|
|
315
|
+
startAt = i;
|
|
316
|
+
break;
|
|
317
|
+
}
|
|
318
|
+
retryAfterMs = Math.min(retryAfterMs, gate.retryAfterMs);
|
|
319
|
+
windowMs = Math.min(windowMs, (transports[i].limits?.windowSeconds ?? 0) * 1e3);
|
|
320
|
+
}
|
|
321
|
+
if (startAt === -1) {
|
|
322
|
+
task.throttleAttemptCount = (task.throttleAttemptCount ?? 0) + 1;
|
|
323
|
+
if (task.throttleAttemptCount > 50) {
|
|
324
|
+
this.logger.warn({
|
|
325
|
+
messageId: message.id,
|
|
326
|
+
taskId: task.taskId,
|
|
327
|
+
attempts: task.throttleAttemptCount
|
|
328
|
+
}, "provider rate limit max attempts exceeded");
|
|
329
|
+
if (!await fallbackToNextChannel("provider_throttle_exceeded")) {
|
|
330
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, "provider throttle exceeded", task.channel, task.projectId);
|
|
331
|
+
await this.eventProcessor.add(buildStreamEvent("notification.failed", {
|
|
332
|
+
projectId: task.projectId,
|
|
333
|
+
taskId: task.taskId,
|
|
334
|
+
enrichedEventId: task.enrichedEventId,
|
|
335
|
+
recipientId: task.recipientId,
|
|
336
|
+
channel: task.channel,
|
|
337
|
+
failureReason: "provider rate limit max attempts exceeded",
|
|
338
|
+
failureCode: "provider_throttle_exceeded",
|
|
339
|
+
retryable: false,
|
|
340
|
+
attempt: attemptNo,
|
|
341
|
+
templateId: task.templateId,
|
|
342
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
343
|
+
campaignId: task.campaignId
|
|
344
|
+
}, "delivery", event.metadata.traceId));
|
|
345
|
+
}
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
const spreadMs = Number.isFinite(windowMs) ? Math.random() * windowMs * Math.min(task.throttleAttemptCount, 10) : 0;
|
|
349
|
+
await this.reschedule({
|
|
350
|
+
...original,
|
|
351
|
+
throttleAttemptCount: task.throttleAttemptCount
|
|
352
|
+
}, new Date(Date.now() + retryAfterMs + spreadMs), "throttle");
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
const idempotencyKey = task.taskId;
|
|
356
|
+
const lease = this.idempotency.acquireLease ? await this.idempotency.acquireLease(idempotencyKey, 60) : await this.idempotency.checkAndMark(idempotencyKey, 60) ? "acquired" : "locked";
|
|
357
|
+
if (lease === "completed") {
|
|
358
|
+
this.logger.info({
|
|
359
|
+
messageId: message.id,
|
|
360
|
+
taskId: task.taskId,
|
|
361
|
+
channel: task.channel
|
|
362
|
+
}, "duplicate delivery — already completed, acking message");
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
if (lease === "locked") {
|
|
366
|
+
this.logger.info({
|
|
367
|
+
messageId: message.id,
|
|
368
|
+
taskId: task.taskId,
|
|
369
|
+
channel: task.channel,
|
|
370
|
+
attempt
|
|
371
|
+
}, "duplicate delivery — skipping and leaving pending in stream");
|
|
372
|
+
return { ack: false };
|
|
373
|
+
}
|
|
374
|
+
const tDispatchStart = Date.now();
|
|
375
|
+
const MAX_DISPATCH_BUDGET_MS = 25e3;
|
|
376
|
+
let dispatchStarted = false;
|
|
377
|
+
let dispatchLogged = false;
|
|
378
|
+
let dispatchedAt = "";
|
|
379
|
+
let dispatchLogTimer;
|
|
380
|
+
const logDispatched = () => {
|
|
381
|
+
clearTimeout(dispatchLogTimer);
|
|
382
|
+
if (!dispatchStarted || dispatchLogged) return void 0;
|
|
383
|
+
dispatchLogged = true;
|
|
384
|
+
const logged = this.eventProcessor.add(buildStreamEvent("notification.dispatched", {
|
|
385
|
+
projectId: task.projectId,
|
|
386
|
+
taskId: task.taskId,
|
|
387
|
+
enrichedEventId: task.enrichedEventId,
|
|
388
|
+
recipientId: task.recipientId,
|
|
389
|
+
channel: task.channel,
|
|
390
|
+
templateId: task.templateId,
|
|
391
|
+
attempt: attemptNo,
|
|
392
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
393
|
+
campaignId: task.campaignId
|
|
394
|
+
}, "delivery", event.metadata.traceId));
|
|
395
|
+
publishPromises.push(logged);
|
|
396
|
+
return logged;
|
|
397
|
+
};
|
|
398
|
+
/** Outcome payload, carrying the dispatched entry if it has not gone out. */
|
|
399
|
+
const outcome = (payload) => {
|
|
400
|
+
clearTimeout(dispatchLogTimer);
|
|
401
|
+
if (dispatchLogged) return payload;
|
|
402
|
+
dispatchLogged = true;
|
|
403
|
+
return {
|
|
404
|
+
...payload,
|
|
405
|
+
attempt: attemptNo,
|
|
406
|
+
dispatchedAt
|
|
407
|
+
};
|
|
408
|
+
};
|
|
409
|
+
let retryScheduled = false;
|
|
410
|
+
/**
|
|
411
|
+
* Re-queues a send the provider turned away, unless it said the failure is
|
|
412
|
+
* permanent or the task is out of attempts. An outage, a timeout or an open
|
|
413
|
+
* circuit is exactly what a later attempt can get past; failing the task
|
|
414
|
+
* outright turned every blip into lost mail. The attempt stays in the log as
|
|
415
|
+
* dispatched, and the task's outcome is whatever its last attempt comes to.
|
|
416
|
+
*/
|
|
417
|
+
const retryLater = async (failure) => {
|
|
418
|
+
const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
|
|
419
|
+
if (failure.retryable === false || attemptNo >= maxAttempts) return false;
|
|
420
|
+
const delayMs = retryDelayMs(task.deliveryAttemptCount ?? 0);
|
|
421
|
+
await this.reschedule({
|
|
422
|
+
...original,
|
|
423
|
+
deliveryAttemptCount: attemptNo
|
|
424
|
+
}, new Date(Date.now() + delayMs), "retry");
|
|
425
|
+
retryScheduled = true;
|
|
426
|
+
metrics.deliveryFailed.inc({
|
|
427
|
+
channel: task.channel,
|
|
428
|
+
reason: "retry_scheduled"
|
|
429
|
+
});
|
|
430
|
+
this.logger.warn({
|
|
431
|
+
taskId: task.taskId,
|
|
432
|
+
channel: task.channel,
|
|
433
|
+
attempt: attemptNo,
|
|
434
|
+
maxAttempts,
|
|
435
|
+
delayMs,
|
|
436
|
+
error: failure.error
|
|
437
|
+
}, "provider send failed — retrying later");
|
|
438
|
+
return true;
|
|
439
|
+
};
|
|
440
|
+
try {
|
|
441
|
+
if (!task.renderedContent) task.renderedContent = await this.render(task);
|
|
442
|
+
dispatchStarted = true;
|
|
443
|
+
dispatchedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
444
|
+
dispatchLogTimer = setTimeout(() => {
|
|
445
|
+
logDispatched();
|
|
446
|
+
}, DISPATCH_LOG_DELAY_MS);
|
|
447
|
+
if (task.channel === "push") {
|
|
448
|
+
let lastResult = {
|
|
449
|
+
success: false,
|
|
450
|
+
error: "No transports"
|
|
451
|
+
};
|
|
452
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
453
|
+
const transport = transports[i];
|
|
454
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) continue;
|
|
455
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
456
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
457
|
+
if (remainingBudgetMs <= 0) {
|
|
458
|
+
this.logger.warn({
|
|
459
|
+
taskId: task.taskId,
|
|
460
|
+
channel: task.channel
|
|
461
|
+
}, "dispatch budget exhausted before transport call — aborting to protect lock lease");
|
|
462
|
+
break;
|
|
463
|
+
}
|
|
464
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
|
|
465
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
466
|
+
try {
|
|
467
|
+
const tProv = Date.now();
|
|
468
|
+
lastResult = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
|
|
469
|
+
const controller = new AbortController();
|
|
470
|
+
const timeout = setTimeout(() => {
|
|
471
|
+
controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
472
|
+
}, timeoutMs);
|
|
473
|
+
try {
|
|
474
|
+
task.signal = controller.signal;
|
|
475
|
+
const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
|
|
476
|
+
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
477
|
+
controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
|
|
478
|
+
})]);
|
|
479
|
+
if (!res.success && !res.invalidToken) throw providerFailure(res);
|
|
480
|
+
return res;
|
|
481
|
+
} finally {
|
|
482
|
+
clearTimeout(timeout);
|
|
483
|
+
}
|
|
484
|
+
});
|
|
485
|
+
global._telemetry.provider += Date.now() - tProv;
|
|
486
|
+
if (lastResult.success || lastResult.invalidToken) break;
|
|
487
|
+
} catch (err) {
|
|
488
|
+
lastResult = {
|
|
489
|
+
success: false,
|
|
490
|
+
error: err.message,
|
|
491
|
+
retryable: err.retryable
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
if (lastResult.invalidToken) {
|
|
496
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
497
|
+
taskId: task.taskId,
|
|
498
|
+
channel: task.channel,
|
|
499
|
+
destination: task.destination,
|
|
500
|
+
providerMessageId: null
|
|
501
|
+
}));
|
|
502
|
+
await this.contactRepo.deactivate(task.projectId, task.recipientId, task.channel, task.destination);
|
|
503
|
+
this.logger.info({
|
|
504
|
+
channel: task.channel,
|
|
505
|
+
target: task.destination
|
|
506
|
+
}, "deactivated invalid contact target");
|
|
507
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, "invalidToken", task.channel, task.projectId);
|
|
508
|
+
metrics.deliveryFailed.inc({
|
|
509
|
+
channel: task.channel,
|
|
510
|
+
reason: "invalid_token"
|
|
511
|
+
});
|
|
512
|
+
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
|
|
513
|
+
projectId: task.projectId,
|
|
514
|
+
taskId: task.taskId,
|
|
515
|
+
enrichedEventId: task.enrichedEventId,
|
|
516
|
+
recipientId: task.recipientId,
|
|
517
|
+
channel: task.channel,
|
|
518
|
+
failureReason: "invalid_token",
|
|
519
|
+
failureCode: "push_failure",
|
|
520
|
+
retryable: false,
|
|
521
|
+
attempt: attemptNo,
|
|
522
|
+
templateId: task.templateId,
|
|
523
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
524
|
+
campaignId: task.campaignId
|
|
525
|
+
}), "delivery", event.metadata.traceId)));
|
|
526
|
+
if (!await fallbackToNextChannel("invalid_token")) throw new NonRetryableError("Push delivery failed: invalid token");
|
|
527
|
+
} else if (lastResult.success) {
|
|
528
|
+
const providerMessageId = lastResult.providerMessageId || "push-success";
|
|
529
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
530
|
+
taskId: task.taskId,
|
|
531
|
+
channel: task.channel,
|
|
532
|
+
destination: task.destination,
|
|
533
|
+
providerMessageId
|
|
534
|
+
}));
|
|
535
|
+
this.logger.debug({
|
|
536
|
+
taskId: task.taskId,
|
|
537
|
+
messageId: providerMessageId
|
|
538
|
+
}, "push delivered");
|
|
539
|
+
this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
|
|
540
|
+
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
541
|
+
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
|
|
542
|
+
projectId: task.projectId,
|
|
543
|
+
taskId: task.taskId,
|
|
544
|
+
enrichedEventId: task.enrichedEventId,
|
|
545
|
+
channel: task.channel,
|
|
546
|
+
deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
547
|
+
providerMessageId,
|
|
548
|
+
templateId: task.templateId,
|
|
549
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
550
|
+
campaignId: task.campaignId
|
|
551
|
+
}), "delivery", event.metadata.traceId)));
|
|
552
|
+
} else if (!await retryLater(lastResult)) {
|
|
553
|
+
this.logger.warn({
|
|
554
|
+
taskId: task.taskId,
|
|
555
|
+
error: lastResult.error
|
|
556
|
+
}, "push delivery failed");
|
|
557
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, lastResult.error ?? "push delivery failed", task.channel, task.projectId);
|
|
558
|
+
metrics.deliveryFailed.inc({
|
|
559
|
+
channel: task.channel,
|
|
560
|
+
reason: "push_error"
|
|
561
|
+
});
|
|
562
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
563
|
+
taskId: task.taskId,
|
|
564
|
+
channel: task.channel,
|
|
565
|
+
destination: task.destination,
|
|
566
|
+
providerMessageId: null
|
|
567
|
+
}));
|
|
568
|
+
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
|
|
569
|
+
projectId: task.projectId,
|
|
570
|
+
taskId: task.taskId,
|
|
571
|
+
enrichedEventId: task.enrichedEventId,
|
|
572
|
+
recipientId: task.recipientId,
|
|
573
|
+
channel: task.channel,
|
|
574
|
+
failureReason: lastResult.error ?? "push delivery failed",
|
|
575
|
+
failureCode: "push_failure",
|
|
576
|
+
retryable: false,
|
|
577
|
+
attempt: attemptNo,
|
|
578
|
+
templateId: task.templateId,
|
|
579
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
580
|
+
campaignId: task.campaignId
|
|
581
|
+
}), "delivery", event.metadata.traceId)));
|
|
582
|
+
if (!await fallbackToNextChannel("push_delivery_failed")) throw new NonRetryableError(lastResult.error ?? "Push delivery failed");
|
|
583
|
+
}
|
|
584
|
+
} else {
|
|
585
|
+
let result = {
|
|
586
|
+
success: false,
|
|
587
|
+
error: "No transports"
|
|
588
|
+
};
|
|
589
|
+
for (let i = startAt; i < transports.length; i++) {
|
|
590
|
+
const transport = transports[i];
|
|
591
|
+
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) continue;
|
|
592
|
+
const elapsedMs = Date.now() - tDispatchStart;
|
|
593
|
+
const remainingBudgetMs = Math.max(0, MAX_DISPATCH_BUDGET_MS - elapsedMs);
|
|
594
|
+
if (remainingBudgetMs <= 0) {
|
|
595
|
+
this.logger.warn({
|
|
596
|
+
taskId: task.taskId,
|
|
597
|
+
channel: task.channel
|
|
598
|
+
}, "dispatch budget exhausted before transport call — aborting to protect lock lease");
|
|
599
|
+
break;
|
|
600
|
+
}
|
|
601
|
+
const configuredTimeoutMs = task.deliveryOptions?.timeoutMs ?? 1e4;
|
|
602
|
+
const timeoutMs = Math.min(configuredTimeoutMs, remainingBudgetMs);
|
|
603
|
+
try {
|
|
604
|
+
const tProv = Date.now();
|
|
605
|
+
result = await this.getBreaker(providerKey(task.channel, transports, i)).execute(async () => {
|
|
606
|
+
const controller = new AbortController();
|
|
607
|
+
const timeout = setTimeout(() => {
|
|
608
|
+
controller.abort(/* @__PURE__ */ new Error(`Transport timeout after ${timeoutMs}ms`));
|
|
609
|
+
}, timeoutMs);
|
|
610
|
+
try {
|
|
611
|
+
task.signal = controller.signal;
|
|
612
|
+
const res = await Promise.race([transport.send(task), new Promise((_, reject) => {
|
|
613
|
+
if (controller.signal.aborted) return reject(controller.signal.reason);
|
|
614
|
+
controller.signal.addEventListener("abort", () => reject(controller.signal.reason));
|
|
615
|
+
})]);
|
|
616
|
+
if (!res.success) throw providerFailure(res);
|
|
617
|
+
return res;
|
|
618
|
+
} finally {
|
|
619
|
+
clearTimeout(timeout);
|
|
620
|
+
}
|
|
621
|
+
});
|
|
622
|
+
global._telemetry.provider += Date.now() - tProv;
|
|
623
|
+
if (result.success) break;
|
|
624
|
+
} catch (err) {
|
|
625
|
+
result = {
|
|
626
|
+
success: false,
|
|
627
|
+
error: err.message,
|
|
628
|
+
retryable: err.retryable
|
|
629
|
+
};
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
if (result.success) {
|
|
633
|
+
const providerMessageId = result.providerMessageId || "success";
|
|
634
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
635
|
+
taskId: task.taskId,
|
|
636
|
+
channel: task.channel,
|
|
637
|
+
destination: task.destination,
|
|
638
|
+
providerMessageId
|
|
639
|
+
}));
|
|
640
|
+
this.globalEmitter.emit("delivery:delivered", task.taskId, providerMessageId, task.channel, task.projectId);
|
|
641
|
+
metrics.deliverySuccess.inc({ channel: task.channel });
|
|
642
|
+
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.delivered", outcome({
|
|
643
|
+
projectId: task.projectId,
|
|
644
|
+
taskId: task.taskId,
|
|
645
|
+
enrichedEventId: task.enrichedEventId,
|
|
646
|
+
channel: task.channel,
|
|
647
|
+
deliveredAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
648
|
+
providerMessageId,
|
|
649
|
+
templateId: task.templateId,
|
|
650
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
651
|
+
campaignId: task.campaignId
|
|
652
|
+
}), "delivery", event.metadata.traceId)));
|
|
653
|
+
} else if (!await retryLater(result)) {
|
|
654
|
+
publishPromises.push(this.outboxUpdateProcessor.add({
|
|
655
|
+
taskId: task.taskId,
|
|
656
|
+
channel: task.channel,
|
|
657
|
+
destination: task.destination,
|
|
658
|
+
providerMessageId: null
|
|
659
|
+
}));
|
|
660
|
+
publishPromises.push(this.eventProcessor.add(buildStreamEvent("notification.failed", outcome({
|
|
661
|
+
projectId: task.projectId,
|
|
662
|
+
taskId: task.taskId,
|
|
663
|
+
enrichedEventId: task.enrichedEventId,
|
|
664
|
+
channel: task.channel,
|
|
665
|
+
failureReason: result.error ?? "delivery failed completely",
|
|
666
|
+
failureCode: "provider_error",
|
|
667
|
+
retryable: false,
|
|
668
|
+
attempt: attemptNo,
|
|
669
|
+
templateId: task.templateId,
|
|
670
|
+
workflowInstanceId: task.workflowInstanceId,
|
|
671
|
+
campaignId: task.campaignId
|
|
672
|
+
}), "delivery", event.metadata.traceId)));
|
|
673
|
+
}
|
|
674
|
+
if (!result.success && !retryScheduled) {
|
|
675
|
+
this.logger.warn({
|
|
676
|
+
taskId: task.taskId,
|
|
677
|
+
channel: task.channel,
|
|
678
|
+
error: result.error
|
|
679
|
+
}, "delivery failed completely across providers");
|
|
680
|
+
this.globalEmitter.emit("delivery:failed", task.taskId, result.error ?? "delivery failed completely", task.channel, task.projectId);
|
|
681
|
+
metrics.deliveryFailed.inc({
|
|
682
|
+
channel: task.channel,
|
|
683
|
+
reason: "provider_error"
|
|
684
|
+
});
|
|
685
|
+
if (!await fallbackToNextChannel("all_providers_failed")) throw new NonRetryableError(result.error ?? "delivery failed");
|
|
686
|
+
} else if (result.success) this.logger.debug({
|
|
687
|
+
taskId: task.taskId,
|
|
688
|
+
channel: task.channel,
|
|
689
|
+
messageId: result.providerMessageId
|
|
690
|
+
}, "notification delivered");
|
|
691
|
+
}
|
|
692
|
+
logDispatched();
|
|
693
|
+
const tFlushStart = Date.now();
|
|
694
|
+
await Promise.all(publishPromises).catch((err) => {
|
|
695
|
+
this.logger.error({
|
|
696
|
+
err,
|
|
697
|
+
taskId: task.taskId
|
|
698
|
+
}, "failed to publish post-dispatch events, swallowing error to prevent duplicate delivery");
|
|
699
|
+
});
|
|
700
|
+
const flushTime = Date.now() - tFlushStart;
|
|
701
|
+
if (retryScheduled) await this.idempotency.unmark(idempotencyKey).catch(() => {});
|
|
702
|
+
else this.idempotency.markProcessed(idempotencyKey).catch((err) => {
|
|
703
|
+
this.logger.warn({
|
|
704
|
+
err,
|
|
705
|
+
taskId: task.taskId
|
|
706
|
+
}, "failed to upgrade idempotency TTL");
|
|
707
|
+
});
|
|
708
|
+
const t = global._telemetry = global._telemetry || {
|
|
709
|
+
count: 0,
|
|
710
|
+
insert: 0,
|
|
711
|
+
provider: 0,
|
|
712
|
+
flush: 0,
|
|
713
|
+
ack: 0
|
|
714
|
+
};
|
|
715
|
+
t.count++;
|
|
716
|
+
t.insert += 0;
|
|
717
|
+
t.flush += flushTime;
|
|
718
|
+
if (t.count % 1e3 === 0) {
|
|
719
|
+
this.logger.debug(`[Metrics 1000 msgs] Dequeue: ${t.dequeue / 1e3}ms, DB Insert: ${t.insert / 1e3}ms, Provider: ${t.provider / 1e3}ms, Wait for Flush: ${t.flush / 1e3}ms, Ack: ${t.ack / 1e3}ms | DB Update (avg per flush): ${t.dbupdate / Math.max(1, t.flushCount)}ms`);
|
|
720
|
+
t.count = 0;
|
|
721
|
+
t.insert = 0;
|
|
722
|
+
t.provider = 0;
|
|
723
|
+
t.flush = 0;
|
|
724
|
+
t.dequeue = 0;
|
|
725
|
+
t.ack = 0;
|
|
726
|
+
t.dbupdate = 0;
|
|
727
|
+
t.flushCount = 0;
|
|
728
|
+
}
|
|
729
|
+
} catch (err) {
|
|
730
|
+
logDispatched()?.catch(() => {});
|
|
731
|
+
await this.idempotency.unmark(idempotencyKey).catch(() => {});
|
|
732
|
+
throw err;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
};
|
|
736
|
+
async function startDeliveryWorker() {
|
|
737
|
+
({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "delivery"));
|
|
738
|
+
const deliveryConcurrency = config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY;
|
|
739
|
+
const consumerId = uniqueConsumerId("delivery");
|
|
740
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
741
|
+
redis,
|
|
742
|
+
logger
|
|
743
|
+
}, {
|
|
744
|
+
stream: OUTBOUND_STREAMS,
|
|
745
|
+
group: CONSUMER_GROUPS.DELIVERY,
|
|
746
|
+
consumer: consumerId,
|
|
747
|
+
batchSize: deliveryConcurrency,
|
|
748
|
+
bufferAcks: true,
|
|
749
|
+
coalesceMs: 2
|
|
750
|
+
}));
|
|
751
|
+
scheduledProducer = new StreamProducer({
|
|
752
|
+
redis: redis.native,
|
|
753
|
+
stream: STREAMS.SCHEDULED,
|
|
754
|
+
logger
|
|
755
|
+
});
|
|
756
|
+
enrichedProducers = createPriorityProducers(redis.native, logger, "ENRICHED");
|
|
757
|
+
const contactRepo = new ContactRepository(db);
|
|
758
|
+
const templateCache = new TemplateCache(new TemplateRepository(db));
|
|
759
|
+
const idempotency = new IdempotencyGuard({
|
|
760
|
+
redis: redis.native,
|
|
761
|
+
keyPrefix: "notif:processed:delivery",
|
|
762
|
+
ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS
|
|
763
|
+
});
|
|
764
|
+
const eventsProducer = new StreamProducer({
|
|
765
|
+
redis: redis.native,
|
|
766
|
+
stream: STREAMS.EVENTS_INBOUND,
|
|
767
|
+
logger
|
|
768
|
+
});
|
|
769
|
+
worker = new DeliveryWorker({
|
|
770
|
+
consumer,
|
|
771
|
+
pendingScanner,
|
|
772
|
+
logger,
|
|
773
|
+
concurrency: deliveryConcurrency,
|
|
774
|
+
transportRegistry,
|
|
775
|
+
idempotency,
|
|
776
|
+
redis: redis.native,
|
|
777
|
+
scheduledProducer,
|
|
778
|
+
enrichedProducers,
|
|
779
|
+
contactRepo,
|
|
780
|
+
eventsProducer,
|
|
781
|
+
globalEmitter,
|
|
782
|
+
db,
|
|
783
|
+
templateCache
|
|
784
|
+
});
|
|
785
|
+
subscriber = redis.native.duplicate({ enableAutoPipelining: false });
|
|
786
|
+
await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
|
|
787
|
+
subscriber.on("message", (_channel, key) => templateCache.invalidateKey(key));
|
|
788
|
+
healthInterval = startHealthReporter("delivery", worker, redis, logger);
|
|
789
|
+
logger.info({
|
|
790
|
+
env: config.NODE_ENV,
|
|
791
|
+
channels: transportRegistry.registeredChannels()
|
|
792
|
+
}, "delivery starting");
|
|
793
|
+
await worker.start();
|
|
794
|
+
}
|
|
795
|
+
function getDeliveryWorker() {
|
|
796
|
+
return worker;
|
|
797
|
+
}
|
|
798
|
+
async function stopDeliveryWorker() {
|
|
799
|
+
await shutdownWorker("delivery", {
|
|
800
|
+
logger,
|
|
801
|
+
timers: [healthInterval],
|
|
802
|
+
subscriber,
|
|
803
|
+
worker,
|
|
804
|
+
sql: sql$1,
|
|
805
|
+
redis
|
|
806
|
+
});
|
|
807
|
+
healthInterval = null;
|
|
808
|
+
subscriber = null;
|
|
809
|
+
}
|
|
810
|
+
//#endregion
|
|
811
|
+
export { DeliveryWorker, MAX_THROTTLE_DEFERRALS, getDeliveryWorker, retryDelayMs, startDeliveryWorker, stopDeliveryWorker };
|