notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -1,14 +1,14 @@
1
1
  import { randomUUID } from "node:crypto";
2
2
  import { loadEnv, readBaseConfig } from "@/index.js";
3
- import { createLogger } from "@/index.js";
4
- import { RedisClient, type Redis } from "@/index.js";
3
+ import type { Logger } from "@/index.js";
4
+ import type { RedisClient, Redis } from "@/index.js";
5
5
  import {
6
- StreamConsumer,
7
- PendingMessageScanner,
6
+ type StreamConsumer,
7
+ type PendingMessageScanner,
8
8
  StreamProducer,
9
9
  type StreamMessage,
10
10
  } from "@/index.js";
11
- import { BaseWorker } from "@/index.js";
11
+ import { BaseWorker, type ProcessResult } from "@/index.js";
12
12
  import {
13
13
  STREAMS,
14
14
  ENRICHED_STREAMS,
@@ -17,13 +17,12 @@ import {
17
17
  registry,
18
18
  buildStreamEvent,
19
19
  type NotificationEnrichedPayload,
20
- type NotificationDispatchedPayload,
20
+ type DispatchedTaskPayload,
21
+ type StreamEvent,
21
22
  } from "@/index.js";
22
- import { type StreamName } from "@/contracts/streams.js";
23
23
  import { IdempotencyGuard } from "@/index.js";
24
24
  import { UserThrottle, ProjectSettingsCache } from "@/index.js";
25
25
  import { TemplateRepository, ContactRepository, ProjectRepository } from "@/index.js";
26
- import { createDatabase } from "@/db/index.js";
27
26
  import { scheduledPayloads, suppressions } from "@/db/schema.js";
28
27
  import { and, eq } from "drizzle-orm";
29
28
  import {
@@ -32,10 +31,18 @@ import {
32
31
  normaliseTarget,
33
32
  type WorkerOptions,
34
33
  DataLoader,
34
+ LRUCache,
35
35
  } from "@/shared/index.js";
36
- import { renderWithTemplate, TemplateCache } from "@/templates/index.js";
36
+ import { TemplateCache } from "@/templates/index.js";
37
37
  import { buildUnsubscribeHeaders } from "@/unsubscribe/index.js";
38
38
  import { startHealthReporter } from "@/workers/index.js";
39
+ import {
40
+ createPriorityProducers,
41
+ createStreamConsumer,
42
+ createWorkerRuntime,
43
+ shutdownWorker,
44
+ uniqueConsumerId,
45
+ } from "@/workers/bootstrap.js";
39
46
 
40
47
  function localTimeToUtc(
41
48
  year: number,
@@ -177,12 +184,19 @@ export function isInQuietHours(
177
184
  return { inQuietHours: true, nextActiveTime: candidateTime };
178
185
  }
179
186
 
187
+ /** Idempotency stand-in for inline messages; see EngineWorker.process. */
188
+ const NO_IDEMPOTENCY = {
189
+ checkAndMark: async (_id: string, _ttl?: number) => true,
190
+ markProcessed: async (_id: string, _ttl?: number) => {},
191
+ unmark: async (_id: string) => {},
192
+ };
193
+
180
194
  // ─── Bootstrap ─────────────────────────────────────────────────────────────
181
195
 
182
196
  loadEnv();
183
197
  const config = readBaseConfig();
184
198
 
185
- let logger: ReturnType<typeof createLogger>;
199
+ let logger: Logger;
186
200
  let redis: RedisClient;
187
201
  let sql: any;
188
202
  let db: any;
@@ -223,6 +237,8 @@ export class EngineWorker extends BaseWorker {
223
237
  private readonly globalEmitter: any;
224
238
  private readonly contactRepo: any;
225
239
  private readonly db: any;
240
+ /** Delivery in this process, when the pipeline is fused. */
241
+ private inlineNext: ((event: StreamEvent) => Promise<void>) | null = null;
226
242
 
227
243
  constructor(options: EngineWorkerOptions) {
228
244
  super(options);
@@ -240,28 +256,36 @@ export class EngineWorker extends BaseWorker {
240
256
  this.db = options.db;
241
257
  }
242
258
 
259
+ /** Hands due tasks straight to `next` instead of the outbound stream. */
260
+ setInlineNext(next: ((event: StreamEvent) => Promise<void>) | null): void {
261
+ this.inlineNext = next;
262
+ }
263
+
243
264
  private readonly contactsLoader = new DataLoader<
244
265
  { projectId: string; recipientId: string },
245
266
  any[]
246
- >(async (keys) => {
247
- const byProject = new Map<string, string[]>();
248
- for (const key of keys) {
249
- if (!byProject.has(key.projectId)) byProject.set(key.projectId, []);
250
- byProject.get(key.projectId)!.push(key.recipientId);
251
- }
267
+ >(
268
+ async (keys) => {
269
+ const byProject = new Map<string, string[]>();
270
+ for (const key of keys) {
271
+ if (!byProject.has(key.projectId)) byProject.set(key.projectId, []);
272
+ byProject.get(key.projectId)!.push(key.recipientId);
273
+ }
252
274
 
253
- const resultsByProjectAndUser = new Map<string, Map<string, any[]>>();
254
- for (const [projectId, userIds] of byProject) {
255
- const activeContactsMap = await this.contactRepo.findActiveByUserIds(projectId, userIds);
256
- resultsByProjectAndUser.set(projectId, activeContactsMap);
257
- }
275
+ const resultsByProjectAndUser = new Map<string, Map<string, any[]>>();
276
+ for (const [projectId, userIds] of byProject) {
277
+ const activeContactsMap = await this.contactRepo.findActiveByUserIds(projectId, userIds);
278
+ resultsByProjectAndUser.set(projectId, activeContactsMap);
279
+ }
258
280
 
259
- return keys.map((key) => {
260
- const projectMap = resultsByProjectAndUser.get(key.projectId);
261
- if (!projectMap) return [];
262
- return projectMap.get(key.recipientId) || [];
263
- });
264
- });
281
+ return keys.map((key) => {
282
+ const projectMap = resultsByProjectAndUser.get(key.projectId);
283
+ if (!projectMap) return [];
284
+ return projectMap.get(key.recipientId) || [];
285
+ });
286
+ },
287
+ { batchDelayMs: 2, maxBatchSize: 500 },
288
+ );
265
289
 
266
290
  /**
267
291
  * Suppressed destinations for one (project, channel), as a normalised set.
@@ -272,6 +296,8 @@ export class EngineWorker extends BaseWorker {
272
296
  * suppression written mid-batch takes effect on the next batch — acceptable,
273
297
  * since the webhook that writes it is itself minutes behind the send.
274
298
  */
299
+ private readonly suppressionsCache = new LRUCache<string, Set<string>>(1000, 10_000);
300
+
275
301
  private readonly suppressionsLoader = new DataLoader<
276
302
  { projectId: string; channel: string },
277
303
  Set<string>
@@ -280,6 +306,13 @@ export class EngineWorker extends BaseWorker {
280
306
  for (const key of keys) {
281
307
  const cacheKey = `${key.projectId}:${key.channel}`;
282
308
  if (results.has(cacheKey)) continue;
309
+
310
+ const cached = this.suppressionsCache.get(cacheKey);
311
+ if (cached !== undefined) {
312
+ results.set(cacheKey, cached);
313
+ continue;
314
+ }
315
+
283
316
  try {
284
317
  const rows = await this.db
285
318
  .select({ target: suppressions.target })
@@ -290,10 +323,9 @@ export class EngineWorker extends BaseWorker {
290
323
  eq(suppressions.channel, key.channel as any),
291
324
  ),
292
325
  );
293
- results.set(
294
- cacheKey,
295
- new Set(rows.map((r: { target: string }) => normaliseTarget(r.target))),
296
- );
326
+ const set = new Set<string>(rows.map((r: { target: string }) => normaliseTarget(r.target)));
327
+ this.suppressionsCache.set(cacheKey, set);
328
+ results.set(cacheKey, set);
297
329
  } catch (err) {
298
330
  // A suppression lookup that fails must not silently become "nothing
299
331
  // is suppressed" — that would resume mailing people who opted out.
@@ -307,7 +339,7 @@ export class EngineWorker extends BaseWorker {
307
339
  return keys.map((key) => results.get(`${key.projectId}:${key.channel}`) ?? new Set<string>());
308
340
  });
309
341
 
310
- async process(message: StreamMessage): Promise<void> {
342
+ async process(message: StreamMessage): Promise<ProcessResult> {
311
343
  const { event } = message;
312
344
 
313
345
  const payloadResult = this.registry.safeParsePayload("notification.enriched", event.payload);
@@ -322,17 +354,27 @@ export class EngineWorker extends BaseWorker {
322
354
  const enriched = payloadResult.data as NotificationEnrichedPayload;
323
355
 
324
356
  // Idempotency
325
- const idempotencyKey = `${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}`;
357
+ // The enricher emits one event per resolved contact, so two addresses on one
358
+ // channel are two distinct messages here.
359
+ const idempotencyKey =
360
+ `${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}` +
361
+ (enriched.contactId ? `:${enriched.contactId}` : "");
326
362
  let customTtl: number | undefined;
327
363
  if (enriched.scheduledAt) {
328
364
  const msUntil = new Date(enriched.scheduledAt).getTime() - Date.now();
329
365
  if (msUntil > 0) {
330
- // base 24h (86400) + schedule time
331
- customTtl = 86400 + Math.ceil(msUntil / 1000);
366
+ // Held until the send is due, then as long as any other marker.
367
+ customTtl = config.IDEMPOTENCY_TTL_SECONDS + Math.ceil(msUntil / 1000);
332
368
  }
333
369
  }
334
370
 
335
- if (!(await this.idempotency.checkAndMark(idempotencyKey, 60))) {
371
+ // Inline (fused) runs skip the engine's own marker. They are already
372
+ // exclusive under the enricher's lock, the send itself is guarded by
373
+ // delivery's lease on the taskId, and the throttle counts a message once
374
+ // however often it is retried. Stream-read messages keep the guard.
375
+ const idempotency = message.inline ? NO_IDEMPOTENCY : this.idempotency;
376
+
377
+ if (!(await idempotency.checkAndMark(idempotencyKey, customTtl ?? 60))) {
336
378
  this.logger.debug({ messageId: message.id, eventId: event.id }, "duplicate — skipping");
337
379
  return;
338
380
  }
@@ -350,7 +392,7 @@ export class EngineWorker extends BaseWorker {
350
392
  recipientId: enriched.recipientId,
351
393
  reason: "user_opted_out",
352
394
  });
353
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
395
+ await idempotency.markProcessed(idempotencyKey, customTtl);
354
396
  return;
355
397
  }
356
398
 
@@ -365,14 +407,20 @@ export class EngineWorker extends BaseWorker {
365
407
  recipientId: enriched.recipientId,
366
408
  reason: "channel_disabled",
367
409
  });
368
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
410
+ await idempotency.markProcessed(idempotencyKey, customTtl);
369
411
  return;
370
412
  }
371
413
 
372
414
  // Quiet hours check
373
415
  const qh = enriched.recipient.preferences.quietHours;
374
416
  if (qh && qh.length > 0 && enriched.priority !== "critical") {
375
- const qhResult = isInQuietHours(enriched.recipient.timezone, qh);
417
+ // Judged at the moment the message would go out, not the moment the
418
+ // engine sees it: a send booked for next week must not be pulled
419
+ // forward to tonight's quiet-hours end, and one booked for 3am must
420
+ // wait until the quiet hours are over.
421
+ const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : new Date();
422
+ const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : new Date();
423
+ const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt);
376
424
  if (qhResult.inQuietHours && qhResult.nextActiveTime) {
377
425
  this.logger.info(
378
426
  {
@@ -402,16 +450,16 @@ export class EngineWorker extends BaseWorker {
402
450
  );
403
451
  }
404
452
 
405
- const throttleResult = await this.throttle.check(
406
- enriched.projectId,
407
- enriched.recipientId,
408
- enriched.priority,
409
- {
410
- limit: projectThrottle.throttleLimit,
411
- windowHours: projectThrottle.throttleWindowHours,
412
- scheduledAt: enriched.scheduledAt,
413
- },
414
- );
453
+ // A channel fallback was counted when the notification was first sent.
454
+ const throttleResult = enriched.throttleCounted
455
+ ? { allowed: true, count: 0, limit: 0 }
456
+ : await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, {
457
+ limit: projectThrottle.throttleLimit,
458
+ windowHours: projectThrottle.throttleWindowHours,
459
+ scheduledAt: enriched.scheduledAt,
460
+ // A replayed message must not be counted against the user twice.
461
+ messageId: idempotencyKey,
462
+ });
415
463
  if (!throttleResult.allowed) {
416
464
  this.logger.info(
417
465
  {
@@ -428,12 +476,11 @@ export class EngineWorker extends BaseWorker {
428
476
  enriched.recipientId,
429
477
  throttleResult.count,
430
478
  );
431
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
479
+ await idempotency.markProcessed(idempotencyKey, customTtl);
432
480
  return;
433
481
  }
434
482
 
435
- // Gather AI Prompts
436
- // Gather AI Prompts
483
+ // Template-level AI prompts, overridden by any on the request.
437
484
  let dbTemplate = null;
438
485
  if (enriched.templateId) {
439
486
  dbTemplate = await this.templateCache.getCachedTemplate(
@@ -451,7 +498,7 @@ export class EngineWorker extends BaseWorker {
451
498
  recipientId: enriched.recipientId,
452
499
  reason: "template_not_found",
453
500
  });
454
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
501
+ await idempotency.markProcessed(idempotencyKey, customTtl);
455
502
  return;
456
503
  }
457
504
  }
@@ -464,49 +511,30 @@ export class EngineWorker extends BaseWorker {
464
511
  // resulting opt-out applies to.
465
512
  const templateTopics: string[] = dbTemplate?.topics ?? [];
466
513
 
467
- if (Object.keys(aiPrompts).length > 0) {
468
- const aiPendingPayload = {
469
- projectId: enriched.projectId,
470
- enrichedEventId: event.id,
471
- recipientId: enriched.recipientId,
472
- channel: enriched.channel,
473
- priority: enriched.priority,
474
- templateId: enriched.templateId,
475
- templateVariables: enriched.templateVariables,
476
- recipient: enriched.recipient,
477
- aiPrompts,
478
- scheduledAt: enriched.scheduledAt,
479
- fallbackChain: enriched.fallbackChain,
480
- };
481
-
482
- const aiPendingEnvelope = buildStreamEvent(
483
- "notification.ai_pending",
484
- aiPendingPayload as Record<string, unknown>,
485
- "engine",
486
- event.metadata.traceId,
487
- );
488
-
489
- await this.aiPendingProducer.publish(aiPendingEnvelope);
490
- this.logger.info(
491
- {
492
- messageId: message.id,
493
- eventId: event.id,
494
- recipientId: enriched.recipientId,
495
- traceId: event.metadata.traceId,
496
- },
497
- "task routed to AI worker",
498
- );
499
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
500
- return;
501
- }
502
-
503
- // Render template
504
- const rendered = renderWithTemplate(dbTemplate, enriched.templateVariables);
505
-
506
- const allContacts = await this.contactsLoader.load({
507
- projectId: enriched.projectId,
508
- recipientId: enriched.recipientId,
509
- });
514
+ // AI-generated messages take the same gates as any other — contacts,
515
+ // topic opt-outs and suppressions below — and are handed to the AI worker
516
+ // per contact once they have passed them.
517
+ const useAi = Object.keys(aiPrompts).length > 0;
518
+
519
+ // Rendering happens in delivery, just before the send, so the rendered
520
+ // body does not travel through the outbound stream.
521
+
522
+ // The enricher already resolved the contact; only events without one
523
+ // (push, channel fallbacks, the AI worker) need the user's contacts.
524
+ const allContacts =
525
+ enriched.contactId && enriched.destination
526
+ ? [
527
+ {
528
+ id: enriched.contactId,
529
+ channel: enriched.channel,
530
+ target: enriched.destination,
531
+ active: true,
532
+ },
533
+ ]
534
+ : await this.contactsLoader.load({
535
+ projectId: enriched.projectId,
536
+ recipientId: enriched.recipientId,
537
+ });
510
538
  const activeContacts = allContacts.filter(
511
539
  (c: any) => c.channel === enriched.channel && c.active,
512
540
  );
@@ -522,7 +550,7 @@ export class EngineWorker extends BaseWorker {
522
550
  recipientId: enriched.recipientId,
523
551
  reason: "no_active_contacts",
524
552
  });
525
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
553
+ await idempotency.markProcessed(idempotencyKey, customTtl);
526
554
  return;
527
555
  }
528
556
 
@@ -532,7 +560,8 @@ export class EngineWorker extends BaseWorker {
532
560
  });
533
561
 
534
562
  for (const contact of activeContacts) {
535
- if (contact.preferences?.optedOut) {
563
+ // This address opted out of one of the template's topics.
564
+ if (templateTopics.some((t) => contact.preferences?.topics?.[t] === false)) {
536
565
  continue;
537
566
  }
538
567
 
@@ -595,7 +624,7 @@ export class EngineWorker extends BaseWorker {
595
624
  })
596
625
  : undefined;
597
626
 
598
- const taskPayload: NotificationDispatchedPayload = {
627
+ const taskPayload: DispatchedTaskPayload = {
599
628
  projectId: enriched.projectId,
600
629
  taskId,
601
630
  enrichedEventId: event.id,
@@ -606,7 +635,6 @@ export class EngineWorker extends BaseWorker {
606
635
  templateVariables: enriched.templateVariables,
607
636
  aiPrompts: enriched.aiPrompts,
608
637
  recipient: enriched.recipient,
609
- renderedContent: rendered,
610
638
  destination: resolvedDestination,
611
639
  deliveryOptions: {
612
640
  maxAttempts: 3,
@@ -615,8 +643,42 @@ export class EngineWorker extends BaseWorker {
615
643
  },
616
644
  fallbackChain: enriched.fallbackChain,
617
645
  campaignId: enriched.campaignId,
646
+ workflowInstanceId: enriched.workflowInstanceId,
618
647
  };
619
648
 
649
+ if (useAi) {
650
+ await this.aiPendingProducer.publish(
651
+ buildStreamEvent(
652
+ "notification.ai_pending",
653
+ {
654
+ projectId: enriched.projectId,
655
+ enrichedEventId: event.id,
656
+ recipientId: enriched.recipientId,
657
+ channel: enriched.channel,
658
+ priority: enriched.priority,
659
+ templateId: enriched.templateId,
660
+ templateVariables: enriched.templateVariables,
661
+ recipient: enriched.recipient,
662
+ aiPrompts,
663
+ scheduledAt: enriched.scheduledAt,
664
+ fallbackChain: enriched.fallbackChain,
665
+ taskId,
666
+ destination: resolvedDestination,
667
+ deliveryHeaders: unsubscribeHeaders,
668
+ campaignId: enriched.campaignId,
669
+ workflowInstanceId: enriched.workflowInstanceId,
670
+ },
671
+ "engine",
672
+ event.metadata.traceId,
673
+ ),
674
+ );
675
+ this.logger.debug(
676
+ { messageId: message.id, taskId, traceId: event.metadata.traceId },
677
+ "task routed to AI worker",
678
+ );
679
+ continue;
680
+ }
681
+
620
682
  const envelope = buildStreamEvent(
621
683
  "notification.dispatched",
622
684
  taskPayload as Record<string, unknown>,
@@ -653,7 +715,7 @@ export class EngineWorker extends BaseWorker {
653
715
  );
654
716
 
655
717
  await this.scheduledProducer.publish(scheduledEnvelope);
656
- this.logger.info(
718
+ this.logger.debug(
657
719
  {
658
720
  messageId: message.id,
659
721
  taskId,
@@ -666,8 +728,16 @@ export class EngineWorker extends BaseWorker {
666
728
  const p = getPriorityBucket(enriched.priority);
667
729
  const outboundProducer = this.outboundProducers[p] ?? this.outboundProducers["normal"]!;
668
730
 
669
- await outboundProducer.publish(envelope);
670
- this.logger.info(
731
+ if (this.inlineNext) {
732
+ await this.inlineNext({
733
+ ...envelope,
734
+ id: randomUUID(),
735
+ timestamp: new Date().toISOString(),
736
+ } as StreamEvent);
737
+ } else {
738
+ await outboundProducer.publish(envelope);
739
+ }
740
+ this.logger.debug(
671
741
  {
672
742
  messageId: message.id,
673
743
  taskId,
@@ -678,9 +748,9 @@ export class EngineWorker extends BaseWorker {
678
748
  );
679
749
  }
680
750
  }
681
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
751
+ await idempotency.markProcessed(idempotencyKey, customTtl);
682
752
  } catch (err) {
683
- await this.idempotency.unmark(idempotencyKey).catch(() => {});
753
+ await idempotency.unmark(idempotencyKey).catch(() => {});
684
754
  throw err;
685
755
  }
686
756
  }
@@ -689,41 +759,24 @@ export class EngineWorker extends BaseWorker {
689
759
  let subscriber: any = null;
690
760
 
691
761
  export async function startEngineWorker() {
692
- logger = createLogger({ name: "engine", level: config.LOG_LEVEL });
693
- redis = new RedisClient({ url: config.REDIS_URL, name: "engine", logger });
694
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "engine", logger });
695
- sql = dbData.sql;
696
- db = dbData.db;
762
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "engine"));
697
763
  templateRepo = new TemplateRepository(db);
698
764
  templateCache = new TemplateCache(templateRepo);
699
765
  const contactRepo = new ContactRepository(db);
700
- consumer = new StreamConsumer({
701
- redis: redis.native,
702
- stream: ENRICHED_STREAMS as unknown as StreamName[],
703
- group: CONSUMER_GROUPS.ENGINE,
704
- consumer: `engine-${process.pid}`,
705
- dlqStream: STREAMS.DEAD_LETTER,
706
- batchSize: config.WORKER_CONCURRENCY,
707
- logger,
708
- });
709
-
710
- pendingScanner = new PendingMessageScanner({
711
- redis: redis.native,
712
- stream: ENRICHED_STREAMS as unknown as StreamName[],
713
- group: CONSUMER_GROUPS.ENGINE,
714
- consumer: `engine-${process.pid}`,
715
- logger,
716
- });
717
-
718
- const outboundProducers = {
719
- critical: new StreamProducer({
720
- redis: redis.native,
721
- stream: STREAMS.OUTBOUND_CRITICAL,
722
- logger,
723
- }),
724
- normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),
725
- low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),
726
- };
766
+ const consumerId = uniqueConsumerId("engine");
767
+ ({ consumer, pendingScanner } = createStreamConsumer(
768
+ { redis, logger },
769
+ {
770
+ stream: ENRICHED_STREAMS,
771
+ group: CONSUMER_GROUPS.ENGINE,
772
+ consumer: consumerId,
773
+ batchSize: config.WORKER_CONCURRENCY,
774
+ bufferAcks: true,
775
+ coalesceMs: 2,
776
+ },
777
+ ));
778
+
779
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
727
780
 
728
781
  const scheduledProducer = new StreamProducer({
729
782
  redis: redis.native,
@@ -740,7 +793,7 @@ export async function startEngineWorker() {
740
793
  const idempotency = new IdempotencyGuard({
741
794
  redis: redis.native,
742
795
  keyPrefix: "notif:processed:engine",
743
- ttlSeconds: 86_400,
796
+ ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
744
797
  });
745
798
 
746
799
  const throttle = new UserThrottle({
@@ -785,7 +838,7 @@ export async function startEngineWorker() {
785
838
  db,
786
839
  });
787
840
 
788
- subscriber = redis.native.duplicate();
841
+ subscriber = redis.native.duplicate({ enableAutoPipelining: false });
789
842
  await subscriber.subscribe(
790
843
  PUBSUB_CHANNELS.TEMPLATE_INVALIDATED,
791
844
  PUBSUB_CHANNELS.PROJECT_INVALIDATED,
@@ -808,20 +861,21 @@ export async function startEngineWorker() {
808
861
  await worker.start();
809
862
  }
810
863
 
864
+ export function getEngineWorker(): EngineWorker | undefined {
865
+ return worker as EngineWorker | undefined;
866
+ }
867
+
811
868
  // ─── Shutdown ──────────────────────────────────────────────────────────────
812
869
 
813
870
  export async function stopEngineWorker(): Promise<void> {
814
- logger?.info("shutdown initiated");
815
- if (healthInterval) {
816
- clearInterval(healthInterval);
817
- healthInterval = null;
818
- }
819
- if (subscriber) {
820
- subscriber.disconnect();
821
- subscriber = null;
822
- }
823
- if (worker) await worker.stop();
824
- if (sql) await sql.end();
825
- if (redis) await redis.disconnect();
826
- logger?.info("engine stopped");
871
+ await shutdownWorker("engine", {
872
+ logger,
873
+ timers: [healthInterval],
874
+ subscriber,
875
+ worker,
876
+ sql,
877
+ redis,
878
+ });
879
+ healthInterval = null;
880
+ subscriber = null;
827
881
  }