notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -1,7 +1,16 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { C as ContactRepository, D as TemplateRepository, Jt as CONSUMER_GROUPS, O as UserRepository, On as globalEmitter, Tn as getPriorityBucket, X as IdempotencyGuard, Xt as INBOUND_STREAMS, _ as TemplateCache, c as BaseWorker, d as startHealthReporter, fn as loadEnv, in as registry, mn as readBaseConfig, qt as buildStreamEvent, w as PreferenceRepository } from "./src-CIPTzq9i.mjs";
|
|
2
|
+
import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-omGPIBny.mjs";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
2
4
|
//#region src/services/enricher/main.ts
|
|
3
5
|
loadEnv();
|
|
4
6
|
const config = readBaseConfig();
|
|
7
|
+
/**
|
|
8
|
+
* A caller's own idempotency key (X-Idempotency-Key, or a workflow step's) is a
|
|
9
|
+
* documented 24-hour promise. Without one the key is the event's id, which only
|
|
10
|
+
* a redelivery can bring back, so the guard's shorter TTL is enough.
|
|
11
|
+
*/
|
|
12
|
+
const CALLER_KEY_TTL_SECONDS = 86400;
|
|
13
|
+
const dedupeTtl = (raw) => raw.idempotencyKey ? CALLER_KEY_TTL_SECONDS : void 0;
|
|
5
14
|
let logger;
|
|
6
15
|
let redis;
|
|
7
16
|
let sql;
|
|
@@ -17,6 +26,8 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
17
26
|
prefRepo;
|
|
18
27
|
contactRepo;
|
|
19
28
|
templateCache;
|
|
29
|
+
/** Engine in this process, when the pipeline is fused. See `setInlineNext`. */
|
|
30
|
+
inlineNext = null;
|
|
20
31
|
userBatch = [];
|
|
21
32
|
batchTimer = null;
|
|
22
33
|
eventBuffer = [];
|
|
@@ -31,10 +42,10 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
31
42
|
resolve,
|
|
32
43
|
reject
|
|
33
44
|
});
|
|
34
|
-
if (this.contactBatch.length >=
|
|
45
|
+
if (this.contactBatch.length >= Math.min(this.concurrency, 50)) {
|
|
35
46
|
if (this.contactBatchTimer) clearTimeout(this.contactBatchTimer);
|
|
36
47
|
this.flushContactBatch();
|
|
37
|
-
} else if (!this.contactBatchTimer) this.contactBatchTimer = setTimeout(() => void this.flushContactBatch(),
|
|
48
|
+
} else if (!this.contactBatchTimer) this.contactBatchTimer = setTimeout(() => void this.flushContactBatch(), 5);
|
|
38
49
|
});
|
|
39
50
|
}
|
|
40
51
|
async flushContactBatch() {
|
|
@@ -65,9 +76,19 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
65
76
|
this.prefRepo = options.prefRepo;
|
|
66
77
|
this.contactRepo = options.contactRepo;
|
|
67
78
|
this.templateCache = options.templateCache;
|
|
68
|
-
this.flushTimer = setInterval(() => void this.flushWorkerBuffers(),
|
|
79
|
+
this.flushTimer = setInterval(() => void this.flushWorkerBuffers(), 5);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Hands enriched events for single-user notifications straight to `next`
|
|
83
|
+
* instead of the enriched stream. Fan-outs still go through the stream: an
|
|
84
|
+
* inline chain holds this worker's slot until the last send completes, and a
|
|
85
|
+
* segment would hold one slot for thousands of them.
|
|
86
|
+
*/
|
|
87
|
+
setInlineNext(next) {
|
|
88
|
+
this.inlineNext = next;
|
|
69
89
|
}
|
|
70
90
|
async stop() {
|
|
91
|
+
await super.stop();
|
|
71
92
|
if (this.batchTimer) {
|
|
72
93
|
clearTimeout(this.batchTimer);
|
|
73
94
|
this.batchTimer = null;
|
|
@@ -83,7 +104,6 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
83
104
|
await this.flushContactBatch();
|
|
84
105
|
await this.flushUserBatch();
|
|
85
106
|
await this.flushWorkerBuffers();
|
|
86
|
-
await super.stop();
|
|
87
107
|
}
|
|
88
108
|
async flushWorkerBuffers() {
|
|
89
109
|
if (this.eventBuffer.length === 0) return;
|
|
@@ -112,12 +132,12 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
112
132
|
resolve,
|
|
113
133
|
reject
|
|
114
134
|
});
|
|
115
|
-
if (this.userBatch.length >=
|
|
135
|
+
if (this.userBatch.length >= Math.min(this.concurrency, 50)) {
|
|
116
136
|
if (this.batchTimer) clearTimeout(this.batchTimer);
|
|
117
137
|
this.flushUserBatch();
|
|
118
138
|
} else if (!this.batchTimer) this.batchTimer = setTimeout(() => {
|
|
119
139
|
this.flushUserBatch();
|
|
120
|
-
},
|
|
140
|
+
}, 5);
|
|
121
141
|
});
|
|
122
142
|
}
|
|
123
143
|
async flushUserBatch() {
|
|
@@ -163,8 +183,7 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
163
183
|
try {
|
|
164
184
|
const profile = await this.userRepo.findRecordById(raw.projectId, raw.recipientId);
|
|
165
185
|
if (!profile) return;
|
|
166
|
-
const
|
|
167
|
-
const optedOutTypes = new Set(prefs.filter((p) => !p.optedIn).map((p) => p.eventType));
|
|
186
|
+
const topics = (raw.templateId ? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId) : null)?.topics ?? [];
|
|
168
187
|
const enrichedPayload = {
|
|
169
188
|
projectId: raw.projectId,
|
|
170
189
|
rawEventId: event.id,
|
|
@@ -179,7 +198,7 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
179
198
|
locale: profile.language ?? "en",
|
|
180
199
|
timezone: profile.timezone ?? "UTC",
|
|
181
200
|
preferences: {
|
|
182
|
-
optedOut:
|
|
201
|
+
optedOut: topics.some((t) => profile.preferences.topics?.[t] === false),
|
|
183
202
|
channels: Object.entries(profile.preferences.channels ?? {}).filter(([_, enabled]) => !enabled).map(([channel]) => channel),
|
|
184
203
|
quietHours: profile.preferences.quietHours
|
|
185
204
|
}
|
|
@@ -196,19 +215,20 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
196
215
|
reject
|
|
197
216
|
});
|
|
198
217
|
}));
|
|
199
|
-
this.logger.
|
|
218
|
+
this.logger.debug({
|
|
200
219
|
messageId: message.id,
|
|
201
220
|
eventId: event.id,
|
|
202
221
|
recipientId: raw.recipientId
|
|
203
222
|
}, "event enriched");
|
|
204
223
|
} catch (err) {
|
|
224
|
+
await this.idempotency.unmark(dedupeId).catch(() => {});
|
|
205
225
|
throw err;
|
|
206
226
|
}
|
|
207
227
|
await Promise.all(publishPromises).catch(async (err) => {
|
|
208
228
|
await this.idempotency.unmark(dedupeId).catch(() => {});
|
|
209
229
|
throw err;
|
|
210
230
|
});
|
|
211
|
-
await this.idempotency.markProcessed(dedupeId);
|
|
231
|
+
await this.idempotency.markProcessed(dedupeId, dedupeTtl(raw));
|
|
212
232
|
return;
|
|
213
233
|
}
|
|
214
234
|
const raw = requestedResult.data;
|
|
@@ -232,80 +252,97 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
232
252
|
} else this.logger.warn({ target: raw.target }, "Segment/topic resolution not fully implemented");
|
|
233
253
|
const maxUsers = readBaseConfig().SEGMENT_MAX_USERS;
|
|
234
254
|
if (userIds.length > maxUsers) {
|
|
255
|
+
const reason = `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`;
|
|
235
256
|
this.logger.error({
|
|
236
257
|
count: userIds.length,
|
|
237
258
|
max: maxUsers,
|
|
238
259
|
projectId: raw.projectId,
|
|
239
260
|
eventId: event.id
|
|
240
261
|
}, "Segment fan-out exceeds maximum allowed limit");
|
|
241
|
-
|
|
242
|
-
await
|
|
243
|
-
projectId: raw.projectId,
|
|
244
|
-
rawEventId: event.id,
|
|
245
|
-
error: `Segment fan-out of ${userIds.length} exceeds limit of ${maxUsers}`
|
|
246
|
-
}, "enricher", event.metadata.traceId));
|
|
262
|
+
globalEmitter.emit("notification:failed", event.id, reason, event.type);
|
|
263
|
+
await this.idempotency.markProcessed(dedupeId, dedupeTtl(raw));
|
|
247
264
|
return;
|
|
248
265
|
}
|
|
249
266
|
const topics = (raw.templateId ? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId) : null)?.topics ?? [];
|
|
250
267
|
const channels = raw.channels && raw.channels.length > 0 ? raw.channels : ["email"];
|
|
251
268
|
const isFallback = raw.fallback === true;
|
|
252
|
-
const channelsToProcess = isFallback ? [channels[0]] : channels;
|
|
253
|
-
const fallbackChain = isFallback ? channels.slice(1) : void 0;
|
|
254
269
|
const chunkArray = (arr, size) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size));
|
|
255
270
|
const chunks = chunkArray(userIds, 500);
|
|
271
|
+
const inline = raw.target.type === "user" ? this.inlineNext : null;
|
|
256
272
|
for (const chunk of chunks) {
|
|
257
273
|
const profiles = (await Promise.all(chunk.map((id) => this.loadUser(raw.projectId, id)))).filter(Boolean);
|
|
258
274
|
const contactsByUser = await this.loadContacts(raw.projectId, profiles.map((profile) => profile.userId));
|
|
259
275
|
const batchedEvents = {
|
|
260
276
|
critical: [],
|
|
261
|
-
high: [],
|
|
262
277
|
normal: [],
|
|
263
278
|
low: []
|
|
264
279
|
};
|
|
265
|
-
for (const profile of profiles)
|
|
266
|
-
const
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
280
|
+
for (const profile of profiles) {
|
|
281
|
+
const contacts = contactsByUser.get(profile.userId) ?? [];
|
|
282
|
+
const usable = (channel) => contacts.filter((contact) => contact.channel === channel && !topics.some((t) => contact.preferences?.topics?.[t] === false));
|
|
283
|
+
let userChannels = channels;
|
|
284
|
+
let fallbackChain;
|
|
285
|
+
if (isFallback) {
|
|
286
|
+
const first = channels.findIndex((c) => usable(c).length > 0);
|
|
287
|
+
if (first === -1) {
|
|
288
|
+
this.logger.info({
|
|
289
|
+
recipientId: profile.userId,
|
|
290
|
+
channels
|
|
291
|
+
}, "no active contact on any fallback channel");
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
userChannels = [channels[first]];
|
|
295
|
+
fallbackChain = channels.slice(first + 1);
|
|
274
296
|
}
|
|
275
|
-
for (const
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
297
|
+
for (const channel of userChannels) {
|
|
298
|
+
const channelContacts = usable(channel);
|
|
299
|
+
const resolved = channel === "push" ? [void 0] : channelContacts;
|
|
300
|
+
if (resolved.length === 0) {
|
|
301
|
+
this.logger.info({
|
|
302
|
+
recipientId: profile.userId,
|
|
303
|
+
channel
|
|
304
|
+
}, "no active contact for channel");
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
for (const contact of resolved) {
|
|
308
|
+
const destination = contact?.target;
|
|
309
|
+
const enrichedPayload = {
|
|
310
|
+
projectId: raw.projectId,
|
|
311
|
+
rawEventId: event.id,
|
|
312
|
+
recipientId: profile.userId,
|
|
313
|
+
channel,
|
|
314
|
+
priority: "normal",
|
|
315
|
+
templateId: raw.templateId,
|
|
316
|
+
templateVariables: raw.data,
|
|
317
|
+
aiPrompts: raw.aiPrompts,
|
|
318
|
+
recipient: {
|
|
319
|
+
id: profile.userId,
|
|
320
|
+
email: channel === "email" ? destination ?? profile.email ?? void 0 : profile.email ?? void 0,
|
|
321
|
+
phone: channel === "sms" || channel === "whatsapp" ? destination : void 0,
|
|
322
|
+
webhook: channel === "webhook" ? destination : void 0,
|
|
323
|
+
telegram: channel === "telegram" ? destination : void 0,
|
|
324
|
+
discord: channel === "discord" ? destination : void 0,
|
|
325
|
+
slack: channel === "slack" ? destination : void 0,
|
|
326
|
+
locale: profile.language ?? "en",
|
|
327
|
+
timezone: profile.timezone ?? "UTC",
|
|
328
|
+
preferences: {
|
|
329
|
+
optedOut: topics.some((t) => profile.preferences.topics?.[t] === false),
|
|
330
|
+
channels: Object.entries(profile.preferences.channels ?? {}).filter(([_, enabled]) => !enabled).map(([channel]) => channel),
|
|
331
|
+
quietHours: profile.preferences.quietHours
|
|
332
|
+
}
|
|
333
|
+
},
|
|
334
|
+
scheduledAt: raw.scheduledAt,
|
|
335
|
+
fallbackChain: fallbackChain?.length ? fallbackChain : void 0,
|
|
336
|
+
contactId: contact?.id,
|
|
337
|
+
destination,
|
|
338
|
+
campaignId: raw.campaignId,
|
|
339
|
+
workflowInstanceId: raw.workflowInstanceId
|
|
340
|
+
};
|
|
341
|
+
const msgPriority = raw.priority ?? "normal";
|
|
342
|
+
const p = getPriorityBucket(msgPriority);
|
|
343
|
+
enrichedPayload.priority = msgPriority;
|
|
344
|
+
batchedEvents[p].push(buildStreamEvent("notification.enriched", enrichedPayload, "enricher", event.metadata.traceId));
|
|
345
|
+
}
|
|
309
346
|
}
|
|
310
347
|
}
|
|
311
348
|
for (const p of [
|
|
@@ -314,86 +351,64 @@ var EnricherWorker = class extends BaseWorker {
|
|
|
314
351
|
"low"
|
|
315
352
|
]) if (batchedEvents[p].length > 0) {
|
|
316
353
|
const producer = this.producers[p] ?? this.producers.normal;
|
|
317
|
-
for (const ev of batchedEvents[p])
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
354
|
+
for (const ev of batchedEvents[p]) {
|
|
355
|
+
if (inline) {
|
|
356
|
+
publishPromises.push(inline({
|
|
357
|
+
...ev,
|
|
358
|
+
id: randomUUID(),
|
|
359
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
360
|
+
}));
|
|
361
|
+
continue;
|
|
362
|
+
}
|
|
363
|
+
publishPromises.push(new Promise((resolve, reject) => {
|
|
364
|
+
this.eventBuffer.push({
|
|
365
|
+
producer,
|
|
366
|
+
event: ev,
|
|
367
|
+
resolve,
|
|
368
|
+
reject
|
|
369
|
+
});
|
|
370
|
+
}));
|
|
371
|
+
}
|
|
372
|
+
if (this.eventBuffer.length >= Math.min(this.concurrency, 50)) this.flushWorkerBuffers();
|
|
325
373
|
}
|
|
326
374
|
}
|
|
327
|
-
this.logger.
|
|
375
|
+
this.logger.debug({
|
|
328
376
|
messageId: message.id,
|
|
329
377
|
eventId: event.id,
|
|
330
378
|
target: raw.target.type,
|
|
331
379
|
traceId: event.metadata.traceId
|
|
332
380
|
}, "event enriched");
|
|
333
381
|
} catch (err) {
|
|
382
|
+
await this.idempotency.unmark(dedupeId).catch(() => {});
|
|
334
383
|
throw err;
|
|
335
384
|
}
|
|
336
385
|
await Promise.all(publishPromises).catch(async (err) => {
|
|
337
386
|
await this.idempotency.unmark(dedupeId).catch(() => {});
|
|
338
387
|
throw err;
|
|
339
388
|
});
|
|
340
|
-
await this.idempotency.markProcessed(dedupeId);
|
|
389
|
+
await this.idempotency.markProcessed(dedupeId, dedupeTtl(raw));
|
|
341
390
|
}
|
|
342
391
|
};
|
|
343
392
|
async function startEnricherWorker() {
|
|
344
|
-
logger =
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
url: config.REDIS_URL,
|
|
350
|
-
name: "enricher",
|
|
393
|
+
({logger, redis, sql, db} = createWorkerRuntime(config, "enricher"));
|
|
394
|
+
const enricherConcurrency = config.PIPELINE_FUSED ? Math.max(config.WORKER_CONCURRENCY, config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY) : config.WORKER_CONCURRENCY;
|
|
395
|
+
const consumerId = uniqueConsumerId("enricher");
|
|
396
|
+
({consumer, pendingScanner} = createStreamConsumer({
|
|
397
|
+
redis,
|
|
351
398
|
logger
|
|
352
|
-
}
|
|
353
|
-
const dbData = createDatabase({
|
|
354
|
-
url: config.DATABASE_URL,
|
|
355
|
-
applicationName: "enricher",
|
|
356
|
-
logger
|
|
357
|
-
});
|
|
358
|
-
sql = dbData.sql;
|
|
359
|
-
db = dbData.db;
|
|
360
|
-
consumer = new StreamConsumer({
|
|
361
|
-
redis: redis.native,
|
|
362
|
-
stream: INBOUND_STREAMS,
|
|
363
|
-
group: CONSUMER_GROUPS.ENRICHER,
|
|
364
|
-
consumer: `enricher-${process.pid}`,
|
|
365
|
-
dlqStream: STREAMS.DEAD_LETTER,
|
|
366
|
-
batchSize: config.WORKER_CONCURRENCY,
|
|
367
|
-
logger
|
|
368
|
-
});
|
|
369
|
-
pendingScanner = new PendingMessageScanner({
|
|
370
|
-
redis: redis.native,
|
|
399
|
+
}, {
|
|
371
400
|
stream: INBOUND_STREAMS,
|
|
372
401
|
group: CONSUMER_GROUPS.ENRICHER,
|
|
373
|
-
consumer:
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
stream: STREAMS.ENRICHED_CRITICAL,
|
|
380
|
-
logger
|
|
381
|
-
}),
|
|
382
|
-
normal: new StreamProducer({
|
|
383
|
-
redis: redis.native,
|
|
384
|
-
stream: STREAMS.ENRICHED_NORMAL,
|
|
385
|
-
logger
|
|
386
|
-
}),
|
|
387
|
-
low: new StreamProducer({
|
|
388
|
-
redis: redis.native,
|
|
389
|
-
stream: STREAMS.ENRICHED_LOW,
|
|
390
|
-
logger
|
|
391
|
-
})
|
|
392
|
-
};
|
|
402
|
+
consumer: consumerId,
|
|
403
|
+
batchSize: enricherConcurrency,
|
|
404
|
+
bufferAcks: true,
|
|
405
|
+
coalesceMs: 2
|
|
406
|
+
}));
|
|
407
|
+
const producers = createPriorityProducers(redis.native, logger, "ENRICHED");
|
|
393
408
|
const idempotency = new IdempotencyGuard({
|
|
394
409
|
redis: redis.native,
|
|
395
410
|
keyPrefix: "notif:processed:enricher",
|
|
396
|
-
ttlSeconds:
|
|
411
|
+
ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS
|
|
397
412
|
});
|
|
398
413
|
const userRepo = new UserRepository(db);
|
|
399
414
|
const prefRepo = new PreferenceRepository(db);
|
|
@@ -404,7 +419,7 @@ async function startEnricherWorker() {
|
|
|
404
419
|
pendingScanner,
|
|
405
420
|
logger,
|
|
406
421
|
maxRetriesBeforeDlq: 5,
|
|
407
|
-
concurrency:
|
|
422
|
+
concurrency: enricherConcurrency,
|
|
408
423
|
producers,
|
|
409
424
|
idempotency,
|
|
410
425
|
userRepo,
|
|
@@ -416,18 +431,18 @@ async function startEnricherWorker() {
|
|
|
416
431
|
logger.info({ env: config.NODE_ENV }, "enricher starting");
|
|
417
432
|
await worker.start();
|
|
418
433
|
}
|
|
434
|
+
function getEnricherWorker() {
|
|
435
|
+
return worker;
|
|
436
|
+
}
|
|
419
437
|
async function stopEnricherWorker() {
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
logger?.info("enricher stopped");
|
|
438
|
+
await shutdownWorker("enricher", {
|
|
439
|
+
logger,
|
|
440
|
+
timers: [healthInterval],
|
|
441
|
+
worker,
|
|
442
|
+
sql,
|
|
443
|
+
redis
|
|
444
|
+
});
|
|
445
|
+
healthInterval = null;
|
|
429
446
|
}
|
|
430
447
|
//#endregion
|
|
431
|
-
export { startEnricherWorker, stopEnricherWorker };
|
|
432
|
-
|
|
433
|
-
//# sourceMappingURL=main-B3iQrl3t.mjs.map
|
|
448
|
+
export { EnricherWorker, getEnricherWorker, startEnricherWorker, stopEnricherWorker };
|