notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
3
|
-
import {
|
|
4
|
-
import { RedisClient,
|
|
3
|
+
import type { Logger } from "@/index.js";
|
|
4
|
+
import type { RedisClient, Redis } from "@/index.js";
|
|
5
5
|
import {
|
|
6
|
-
StreamConsumer,
|
|
7
|
-
PendingMessageScanner,
|
|
6
|
+
type StreamConsumer,
|
|
7
|
+
type PendingMessageScanner,
|
|
8
8
|
StreamProducer,
|
|
9
9
|
type StreamMessage,
|
|
10
10
|
} from "@/index.js";
|
|
11
|
-
import { BaseWorker } from "@/index.js";
|
|
11
|
+
import { BaseWorker, type ProcessResult } from "@/index.js";
|
|
12
12
|
import {
|
|
13
13
|
STREAMS,
|
|
14
14
|
ENRICHED_STREAMS,
|
|
@@ -17,13 +17,12 @@ import {
|
|
|
17
17
|
registry,
|
|
18
18
|
buildStreamEvent,
|
|
19
19
|
type NotificationEnrichedPayload,
|
|
20
|
-
type
|
|
20
|
+
type DispatchedTaskPayload,
|
|
21
|
+
type StreamEvent,
|
|
21
22
|
} from "@/index.js";
|
|
22
|
-
import { type StreamName } from "@/contracts/streams.js";
|
|
23
23
|
import { IdempotencyGuard } from "@/index.js";
|
|
24
24
|
import { UserThrottle, ProjectSettingsCache } from "@/index.js";
|
|
25
25
|
import { TemplateRepository, ContactRepository, ProjectRepository } from "@/index.js";
|
|
26
|
-
import { createDatabase } from "@/db/index.js";
|
|
27
26
|
import { scheduledPayloads, suppressions } from "@/db/schema.js";
|
|
28
27
|
import { and, eq } from "drizzle-orm";
|
|
29
28
|
import {
|
|
@@ -32,10 +31,18 @@ import {
|
|
|
32
31
|
normaliseTarget,
|
|
33
32
|
type WorkerOptions,
|
|
34
33
|
DataLoader,
|
|
34
|
+
LRUCache,
|
|
35
35
|
} from "@/shared/index.js";
|
|
36
|
-
import {
|
|
36
|
+
import { TemplateCache } from "@/templates/index.js";
|
|
37
37
|
import { buildUnsubscribeHeaders } from "@/unsubscribe/index.js";
|
|
38
38
|
import { startHealthReporter } from "@/workers/index.js";
|
|
39
|
+
import {
|
|
40
|
+
createPriorityProducers,
|
|
41
|
+
createStreamConsumer,
|
|
42
|
+
createWorkerRuntime,
|
|
43
|
+
shutdownWorker,
|
|
44
|
+
uniqueConsumerId,
|
|
45
|
+
} from "@/workers/bootstrap.js";
|
|
39
46
|
|
|
40
47
|
function localTimeToUtc(
|
|
41
48
|
year: number,
|
|
@@ -177,12 +184,19 @@ export function isInQuietHours(
|
|
|
177
184
|
return { inQuietHours: true, nextActiveTime: candidateTime };
|
|
178
185
|
}
|
|
179
186
|
|
|
187
|
+
/** Idempotency stand-in for inline messages; see EngineWorker.process. */
|
|
188
|
+
const NO_IDEMPOTENCY = {
|
|
189
|
+
checkAndMark: async (_id: string, _ttl?: number) => true,
|
|
190
|
+
markProcessed: async (_id: string, _ttl?: number) => {},
|
|
191
|
+
unmark: async (_id: string) => {},
|
|
192
|
+
};
|
|
193
|
+
|
|
180
194
|
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
181
195
|
|
|
182
196
|
loadEnv();
|
|
183
197
|
const config = readBaseConfig();
|
|
184
198
|
|
|
185
|
-
let logger:
|
|
199
|
+
let logger: Logger;
|
|
186
200
|
let redis: RedisClient;
|
|
187
201
|
let sql: any;
|
|
188
202
|
let db: any;
|
|
@@ -223,6 +237,8 @@ export class EngineWorker extends BaseWorker {
|
|
|
223
237
|
private readonly globalEmitter: any;
|
|
224
238
|
private readonly contactRepo: any;
|
|
225
239
|
private readonly db: any;
|
|
240
|
+
/** Delivery in this process, when the pipeline is fused. */
|
|
241
|
+
private inlineNext: ((event: StreamEvent) => Promise<void>) | null = null;
|
|
226
242
|
|
|
227
243
|
constructor(options: EngineWorkerOptions) {
|
|
228
244
|
super(options);
|
|
@@ -240,28 +256,36 @@ export class EngineWorker extends BaseWorker {
|
|
|
240
256
|
this.db = options.db;
|
|
241
257
|
}
|
|
242
258
|
|
|
259
|
+
/** Hands due tasks straight to `next` instead of the outbound stream. */
|
|
260
|
+
setInlineNext(next: ((event: StreamEvent) => Promise<void>) | null): void {
|
|
261
|
+
this.inlineNext = next;
|
|
262
|
+
}
|
|
263
|
+
|
|
243
264
|
private readonly contactsLoader = new DataLoader<
|
|
244
265
|
{ projectId: string; recipientId: string },
|
|
245
266
|
any[]
|
|
246
|
-
>(
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
267
|
+
>(
|
|
268
|
+
async (keys) => {
|
|
269
|
+
const byProject = new Map<string, string[]>();
|
|
270
|
+
for (const key of keys) {
|
|
271
|
+
if (!byProject.has(key.projectId)) byProject.set(key.projectId, []);
|
|
272
|
+
byProject.get(key.projectId)!.push(key.recipientId);
|
|
273
|
+
}
|
|
252
274
|
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
275
|
+
const resultsByProjectAndUser = new Map<string, Map<string, any[]>>();
|
|
276
|
+
for (const [projectId, userIds] of byProject) {
|
|
277
|
+
const activeContactsMap = await this.contactRepo.findActiveByUserIds(projectId, userIds);
|
|
278
|
+
resultsByProjectAndUser.set(projectId, activeContactsMap);
|
|
279
|
+
}
|
|
258
280
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
281
|
+
return keys.map((key) => {
|
|
282
|
+
const projectMap = resultsByProjectAndUser.get(key.projectId);
|
|
283
|
+
if (!projectMap) return [];
|
|
284
|
+
return projectMap.get(key.recipientId) || [];
|
|
285
|
+
});
|
|
286
|
+
},
|
|
287
|
+
{ batchDelayMs: 2, maxBatchSize: 500 },
|
|
288
|
+
);
|
|
265
289
|
|
|
266
290
|
/**
|
|
267
291
|
* Suppressed destinations for one (project, channel), as a normalised set.
|
|
@@ -272,6 +296,8 @@ export class EngineWorker extends BaseWorker {
|
|
|
272
296
|
* suppression written mid-batch takes effect on the next batch — acceptable,
|
|
273
297
|
* since the webhook that writes it is itself minutes behind the send.
|
|
274
298
|
*/
|
|
299
|
+
private readonly suppressionsCache = new LRUCache<string, Set<string>>(1000, 10_000);
|
|
300
|
+
|
|
275
301
|
private readonly suppressionsLoader = new DataLoader<
|
|
276
302
|
{ projectId: string; channel: string },
|
|
277
303
|
Set<string>
|
|
@@ -280,6 +306,13 @@ export class EngineWorker extends BaseWorker {
|
|
|
280
306
|
for (const key of keys) {
|
|
281
307
|
const cacheKey = `${key.projectId}:${key.channel}`;
|
|
282
308
|
if (results.has(cacheKey)) continue;
|
|
309
|
+
|
|
310
|
+
const cached = this.suppressionsCache.get(cacheKey);
|
|
311
|
+
if (cached !== undefined) {
|
|
312
|
+
results.set(cacheKey, cached);
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
|
|
283
316
|
try {
|
|
284
317
|
const rows = await this.db
|
|
285
318
|
.select({ target: suppressions.target })
|
|
@@ -290,10 +323,9 @@ export class EngineWorker extends BaseWorker {
|
|
|
290
323
|
eq(suppressions.channel, key.channel as any),
|
|
291
324
|
),
|
|
292
325
|
);
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
);
|
|
326
|
+
const set = new Set<string>(rows.map((r: { target: string }) => normaliseTarget(r.target)));
|
|
327
|
+
this.suppressionsCache.set(cacheKey, set);
|
|
328
|
+
results.set(cacheKey, set);
|
|
297
329
|
} catch (err) {
|
|
298
330
|
// A suppression lookup that fails must not silently become "nothing
|
|
299
331
|
// is suppressed" — that would resume mailing people who opted out.
|
|
@@ -307,7 +339,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
307
339
|
return keys.map((key) => results.get(`${key.projectId}:${key.channel}`) ?? new Set<string>());
|
|
308
340
|
});
|
|
309
341
|
|
|
310
|
-
async process(message: StreamMessage): Promise<
|
|
342
|
+
async process(message: StreamMessage): Promise<ProcessResult> {
|
|
311
343
|
const { event } = message;
|
|
312
344
|
|
|
313
345
|
const payloadResult = this.registry.safeParsePayload("notification.enriched", event.payload);
|
|
@@ -322,17 +354,27 @@ export class EngineWorker extends BaseWorker {
|
|
|
322
354
|
const enriched = payloadResult.data as NotificationEnrichedPayload;
|
|
323
355
|
|
|
324
356
|
// Idempotency
|
|
325
|
-
|
|
357
|
+
// The enricher emits one event per resolved contact, so two addresses on one
|
|
358
|
+
// channel are two distinct messages here.
|
|
359
|
+
const idempotencyKey =
|
|
360
|
+
`${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}` +
|
|
361
|
+
(enriched.contactId ? `:${enriched.contactId}` : "");
|
|
326
362
|
let customTtl: number | undefined;
|
|
327
363
|
if (enriched.scheduledAt) {
|
|
328
364
|
const msUntil = new Date(enriched.scheduledAt).getTime() - Date.now();
|
|
329
365
|
if (msUntil > 0) {
|
|
330
|
-
//
|
|
331
|
-
customTtl =
|
|
366
|
+
// Held until the send is due, then as long as any other marker.
|
|
367
|
+
customTtl = config.IDEMPOTENCY_TTL_SECONDS + Math.ceil(msUntil / 1000);
|
|
332
368
|
}
|
|
333
369
|
}
|
|
334
370
|
|
|
335
|
-
|
|
371
|
+
// Inline (fused) runs skip the engine's own marker. They are already
|
|
372
|
+
// exclusive under the enricher's lock, the send itself is guarded by
|
|
373
|
+
// delivery's lease on the taskId, and the throttle counts a message once
|
|
374
|
+
// however often it is retried. Stream-read messages keep the guard.
|
|
375
|
+
const idempotency = message.inline ? NO_IDEMPOTENCY : this.idempotency;
|
|
376
|
+
|
|
377
|
+
if (!(await idempotency.checkAndMark(idempotencyKey, customTtl ?? 60))) {
|
|
336
378
|
this.logger.debug({ messageId: message.id, eventId: event.id }, "duplicate — skipping");
|
|
337
379
|
return;
|
|
338
380
|
}
|
|
@@ -350,7 +392,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
350
392
|
recipientId: enriched.recipientId,
|
|
351
393
|
reason: "user_opted_out",
|
|
352
394
|
});
|
|
353
|
-
await
|
|
395
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
354
396
|
return;
|
|
355
397
|
}
|
|
356
398
|
|
|
@@ -365,14 +407,20 @@ export class EngineWorker extends BaseWorker {
|
|
|
365
407
|
recipientId: enriched.recipientId,
|
|
366
408
|
reason: "channel_disabled",
|
|
367
409
|
});
|
|
368
|
-
await
|
|
410
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
369
411
|
return;
|
|
370
412
|
}
|
|
371
413
|
|
|
372
414
|
// Quiet hours check
|
|
373
415
|
const qh = enriched.recipient.preferences.quietHours;
|
|
374
416
|
if (qh && qh.length > 0 && enriched.priority !== "critical") {
|
|
375
|
-
|
|
417
|
+
// Judged at the moment the message would go out, not the moment the
|
|
418
|
+
// engine sees it: a send booked for next week must not be pulled
|
|
419
|
+
// forward to tonight's quiet-hours end, and one booked for 3am must
|
|
420
|
+
// wait until the quiet hours are over.
|
|
421
|
+
const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : new Date();
|
|
422
|
+
const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : new Date();
|
|
423
|
+
const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt);
|
|
376
424
|
if (qhResult.inQuietHours && qhResult.nextActiveTime) {
|
|
377
425
|
this.logger.info(
|
|
378
426
|
{
|
|
@@ -402,16 +450,16 @@ export class EngineWorker extends BaseWorker {
|
|
|
402
450
|
);
|
|
403
451
|
}
|
|
404
452
|
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
enriched.priority,
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
453
|
+
// A channel fallback was counted when the notification was first sent.
|
|
454
|
+
const throttleResult = enriched.throttleCounted
|
|
455
|
+
? { allowed: true, count: 0, limit: 0 }
|
|
456
|
+
: await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, {
|
|
457
|
+
limit: projectThrottle.throttleLimit,
|
|
458
|
+
windowHours: projectThrottle.throttleWindowHours,
|
|
459
|
+
scheduledAt: enriched.scheduledAt,
|
|
460
|
+
// A replayed message must not be counted against the user twice.
|
|
461
|
+
messageId: idempotencyKey,
|
|
462
|
+
});
|
|
415
463
|
if (!throttleResult.allowed) {
|
|
416
464
|
this.logger.info(
|
|
417
465
|
{
|
|
@@ -428,12 +476,11 @@ export class EngineWorker extends BaseWorker {
|
|
|
428
476
|
enriched.recipientId,
|
|
429
477
|
throttleResult.count,
|
|
430
478
|
);
|
|
431
|
-
await
|
|
479
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
432
480
|
return;
|
|
433
481
|
}
|
|
434
482
|
|
|
435
|
-
//
|
|
436
|
-
// Gather AI Prompts
|
|
483
|
+
// Template-level AI prompts, overridden by any on the request.
|
|
437
484
|
let dbTemplate = null;
|
|
438
485
|
if (enriched.templateId) {
|
|
439
486
|
dbTemplate = await this.templateCache.getCachedTemplate(
|
|
@@ -451,7 +498,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
451
498
|
recipientId: enriched.recipientId,
|
|
452
499
|
reason: "template_not_found",
|
|
453
500
|
});
|
|
454
|
-
await
|
|
501
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
455
502
|
return;
|
|
456
503
|
}
|
|
457
504
|
}
|
|
@@ -464,49 +511,30 @@ export class EngineWorker extends BaseWorker {
|
|
|
464
511
|
// resulting opt-out applies to.
|
|
465
512
|
const templateTopics: string[] = dbTemplate?.topics ?? [];
|
|
466
513
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
{
|
|
492
|
-
messageId: message.id,
|
|
493
|
-
eventId: event.id,
|
|
494
|
-
recipientId: enriched.recipientId,
|
|
495
|
-
traceId: event.metadata.traceId,
|
|
496
|
-
},
|
|
497
|
-
"task routed to AI worker",
|
|
498
|
-
);
|
|
499
|
-
await this.idempotency.markProcessed(idempotencyKey, customTtl);
|
|
500
|
-
return;
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
// Render template
|
|
504
|
-
const rendered = renderWithTemplate(dbTemplate, enriched.templateVariables);
|
|
505
|
-
|
|
506
|
-
const allContacts = await this.contactsLoader.load({
|
|
507
|
-
projectId: enriched.projectId,
|
|
508
|
-
recipientId: enriched.recipientId,
|
|
509
|
-
});
|
|
514
|
+
// AI-generated messages take the same gates as any other — contacts,
|
|
515
|
+
// topic opt-outs and suppressions below — and are handed to the AI worker
|
|
516
|
+
// per contact once they have passed them.
|
|
517
|
+
const useAi = Object.keys(aiPrompts).length > 0;
|
|
518
|
+
|
|
519
|
+
// Rendering happens in delivery, just before the send, so the rendered
|
|
520
|
+
// body does not travel through the outbound stream.
|
|
521
|
+
|
|
522
|
+
// The enricher already resolved the contact; only events without one
|
|
523
|
+
// (push, channel fallbacks, the AI worker) need the user's contacts.
|
|
524
|
+
const allContacts =
|
|
525
|
+
enriched.contactId && enriched.destination
|
|
526
|
+
? [
|
|
527
|
+
{
|
|
528
|
+
id: enriched.contactId,
|
|
529
|
+
channel: enriched.channel,
|
|
530
|
+
target: enriched.destination,
|
|
531
|
+
active: true,
|
|
532
|
+
},
|
|
533
|
+
]
|
|
534
|
+
: await this.contactsLoader.load({
|
|
535
|
+
projectId: enriched.projectId,
|
|
536
|
+
recipientId: enriched.recipientId,
|
|
537
|
+
});
|
|
510
538
|
const activeContacts = allContacts.filter(
|
|
511
539
|
(c: any) => c.channel === enriched.channel && c.active,
|
|
512
540
|
);
|
|
@@ -522,7 +550,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
522
550
|
recipientId: enriched.recipientId,
|
|
523
551
|
reason: "no_active_contacts",
|
|
524
552
|
});
|
|
525
|
-
await
|
|
553
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
526
554
|
return;
|
|
527
555
|
}
|
|
528
556
|
|
|
@@ -532,7 +560,8 @@ export class EngineWorker extends BaseWorker {
|
|
|
532
560
|
});
|
|
533
561
|
|
|
534
562
|
for (const contact of activeContacts) {
|
|
535
|
-
|
|
563
|
+
// This address opted out of one of the template's topics.
|
|
564
|
+
if (templateTopics.some((t) => contact.preferences?.topics?.[t] === false)) {
|
|
536
565
|
continue;
|
|
537
566
|
}
|
|
538
567
|
|
|
@@ -595,7 +624,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
595
624
|
})
|
|
596
625
|
: undefined;
|
|
597
626
|
|
|
598
|
-
const taskPayload:
|
|
627
|
+
const taskPayload: DispatchedTaskPayload = {
|
|
599
628
|
projectId: enriched.projectId,
|
|
600
629
|
taskId,
|
|
601
630
|
enrichedEventId: event.id,
|
|
@@ -606,7 +635,6 @@ export class EngineWorker extends BaseWorker {
|
|
|
606
635
|
templateVariables: enriched.templateVariables,
|
|
607
636
|
aiPrompts: enriched.aiPrompts,
|
|
608
637
|
recipient: enriched.recipient,
|
|
609
|
-
renderedContent: rendered,
|
|
610
638
|
destination: resolvedDestination,
|
|
611
639
|
deliveryOptions: {
|
|
612
640
|
maxAttempts: 3,
|
|
@@ -615,8 +643,42 @@ export class EngineWorker extends BaseWorker {
|
|
|
615
643
|
},
|
|
616
644
|
fallbackChain: enriched.fallbackChain,
|
|
617
645
|
campaignId: enriched.campaignId,
|
|
646
|
+
workflowInstanceId: enriched.workflowInstanceId,
|
|
618
647
|
};
|
|
619
648
|
|
|
649
|
+
if (useAi) {
|
|
650
|
+
await this.aiPendingProducer.publish(
|
|
651
|
+
buildStreamEvent(
|
|
652
|
+
"notification.ai_pending",
|
|
653
|
+
{
|
|
654
|
+
projectId: enriched.projectId,
|
|
655
|
+
enrichedEventId: event.id,
|
|
656
|
+
recipientId: enriched.recipientId,
|
|
657
|
+
channel: enriched.channel,
|
|
658
|
+
priority: enriched.priority,
|
|
659
|
+
templateId: enriched.templateId,
|
|
660
|
+
templateVariables: enriched.templateVariables,
|
|
661
|
+
recipient: enriched.recipient,
|
|
662
|
+
aiPrompts,
|
|
663
|
+
scheduledAt: enriched.scheduledAt,
|
|
664
|
+
fallbackChain: enriched.fallbackChain,
|
|
665
|
+
taskId,
|
|
666
|
+
destination: resolvedDestination,
|
|
667
|
+
deliveryHeaders: unsubscribeHeaders,
|
|
668
|
+
campaignId: enriched.campaignId,
|
|
669
|
+
workflowInstanceId: enriched.workflowInstanceId,
|
|
670
|
+
},
|
|
671
|
+
"engine",
|
|
672
|
+
event.metadata.traceId,
|
|
673
|
+
),
|
|
674
|
+
);
|
|
675
|
+
this.logger.debug(
|
|
676
|
+
{ messageId: message.id, taskId, traceId: event.metadata.traceId },
|
|
677
|
+
"task routed to AI worker",
|
|
678
|
+
);
|
|
679
|
+
continue;
|
|
680
|
+
}
|
|
681
|
+
|
|
620
682
|
const envelope = buildStreamEvent(
|
|
621
683
|
"notification.dispatched",
|
|
622
684
|
taskPayload as Record<string, unknown>,
|
|
@@ -653,7 +715,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
653
715
|
);
|
|
654
716
|
|
|
655
717
|
await this.scheduledProducer.publish(scheduledEnvelope);
|
|
656
|
-
this.logger.
|
|
718
|
+
this.logger.debug(
|
|
657
719
|
{
|
|
658
720
|
messageId: message.id,
|
|
659
721
|
taskId,
|
|
@@ -666,8 +728,16 @@ export class EngineWorker extends BaseWorker {
|
|
|
666
728
|
const p = getPriorityBucket(enriched.priority);
|
|
667
729
|
const outboundProducer = this.outboundProducers[p] ?? this.outboundProducers["normal"]!;
|
|
668
730
|
|
|
669
|
-
|
|
670
|
-
|
|
731
|
+
if (this.inlineNext) {
|
|
732
|
+
await this.inlineNext({
|
|
733
|
+
...envelope,
|
|
734
|
+
id: randomUUID(),
|
|
735
|
+
timestamp: new Date().toISOString(),
|
|
736
|
+
} as StreamEvent);
|
|
737
|
+
} else {
|
|
738
|
+
await outboundProducer.publish(envelope);
|
|
739
|
+
}
|
|
740
|
+
this.logger.debug(
|
|
671
741
|
{
|
|
672
742
|
messageId: message.id,
|
|
673
743
|
taskId,
|
|
@@ -678,9 +748,9 @@ export class EngineWorker extends BaseWorker {
|
|
|
678
748
|
);
|
|
679
749
|
}
|
|
680
750
|
}
|
|
681
|
-
await
|
|
751
|
+
await idempotency.markProcessed(idempotencyKey, customTtl);
|
|
682
752
|
} catch (err) {
|
|
683
|
-
await
|
|
753
|
+
await idempotency.unmark(idempotencyKey).catch(() => {});
|
|
684
754
|
throw err;
|
|
685
755
|
}
|
|
686
756
|
}
|
|
@@ -689,41 +759,24 @@ export class EngineWorker extends BaseWorker {
|
|
|
689
759
|
let subscriber: any = null;
|
|
690
760
|
|
|
691
761
|
export async function startEngineWorker() {
|
|
692
|
-
logger =
|
|
693
|
-
redis = new RedisClient({ url: config.REDIS_URL, name: "engine", logger });
|
|
694
|
-
const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "engine", logger });
|
|
695
|
-
sql = dbData.sql;
|
|
696
|
-
db = dbData.db;
|
|
762
|
+
({ logger, redis, sql, db } = createWorkerRuntime(config, "engine"));
|
|
697
763
|
templateRepo = new TemplateRepository(db);
|
|
698
764
|
templateCache = new TemplateCache(templateRepo);
|
|
699
765
|
const contactRepo = new ContactRepository(db);
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
consumer: `engine-${process.pid}`,
|
|
715
|
-
logger,
|
|
716
|
-
});
|
|
717
|
-
|
|
718
|
-
const outboundProducers = {
|
|
719
|
-
critical: new StreamProducer({
|
|
720
|
-
redis: redis.native,
|
|
721
|
-
stream: STREAMS.OUTBOUND_CRITICAL,
|
|
722
|
-
logger,
|
|
723
|
-
}),
|
|
724
|
-
normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),
|
|
725
|
-
low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),
|
|
726
|
-
};
|
|
766
|
+
const consumerId = uniqueConsumerId("engine");
|
|
767
|
+
({ consumer, pendingScanner } = createStreamConsumer(
|
|
768
|
+
{ redis, logger },
|
|
769
|
+
{
|
|
770
|
+
stream: ENRICHED_STREAMS,
|
|
771
|
+
group: CONSUMER_GROUPS.ENGINE,
|
|
772
|
+
consumer: consumerId,
|
|
773
|
+
batchSize: config.WORKER_CONCURRENCY,
|
|
774
|
+
bufferAcks: true,
|
|
775
|
+
coalesceMs: 2,
|
|
776
|
+
},
|
|
777
|
+
));
|
|
778
|
+
|
|
779
|
+
const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
|
|
727
780
|
|
|
728
781
|
const scheduledProducer = new StreamProducer({
|
|
729
782
|
redis: redis.native,
|
|
@@ -740,7 +793,7 @@ export async function startEngineWorker() {
|
|
|
740
793
|
const idempotency = new IdempotencyGuard({
|
|
741
794
|
redis: redis.native,
|
|
742
795
|
keyPrefix: "notif:processed:engine",
|
|
743
|
-
ttlSeconds:
|
|
796
|
+
ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
|
|
744
797
|
});
|
|
745
798
|
|
|
746
799
|
const throttle = new UserThrottle({
|
|
@@ -785,7 +838,7 @@ export async function startEngineWorker() {
|
|
|
785
838
|
db,
|
|
786
839
|
});
|
|
787
840
|
|
|
788
|
-
subscriber = redis.native.duplicate();
|
|
841
|
+
subscriber = redis.native.duplicate({ enableAutoPipelining: false });
|
|
789
842
|
await subscriber.subscribe(
|
|
790
843
|
PUBSUB_CHANNELS.TEMPLATE_INVALIDATED,
|
|
791
844
|
PUBSUB_CHANNELS.PROJECT_INVALIDATED,
|
|
@@ -808,20 +861,21 @@ export async function startEngineWorker() {
|
|
|
808
861
|
await worker.start();
|
|
809
862
|
}
|
|
810
863
|
|
|
864
|
+
export function getEngineWorker(): EngineWorker | undefined {
|
|
865
|
+
return worker as EngineWorker | undefined;
|
|
866
|
+
}
|
|
867
|
+
|
|
811
868
|
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
812
869
|
|
|
813
870
|
export async function stopEngineWorker(): Promise<void> {
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
if (sql) await sql.end();
|
|
825
|
-
if (redis) await redis.disconnect();
|
|
826
|
-
logger?.info("engine stopped");
|
|
871
|
+
await shutdownWorker("engine", {
|
|
872
|
+
logger,
|
|
873
|
+
timers: [healthInterval],
|
|
874
|
+
subscriber,
|
|
875
|
+
worker,
|
|
876
|
+
sql,
|
|
877
|
+
redis,
|
|
878
|
+
});
|
|
879
|
+
healthInterval = null;
|
|
880
|
+
subscriber = null;
|
|
827
881
|
}
|