notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -1,4 +1,5 @@
1
- import { $t as ENRICHED_STREAMS, A as RedisClient, E as TemplateRepository, G as getPriorityBucket, J as globalEmitter, K as normaliseTarget, L as DataLoader, M as UserThrottle, Qt as CONSUMER_GROUPS, S as ContactRepository, X as StreamConsumer, Y as PendingMessageScanner, Z as StreamProducer, Zt as buildStreamEvent, at as createDatabase, b as renderWithTemplate, c as BaseWorker, cn as registry, d as buildUnsubscribeHeaders, dt as scheduledPayloads, ft as suppressions, g as TemplateCache, gn as loadEnv, it as IdempotencyGuard, j as ProjectSettingsCache, nn as PUBSUB_CHANNELS, rn as STREAMS, tt as createLogger, u as startHealthReporter, vn as readBaseConfig, w as ProjectRepository } from "./src-CGmXRfsM.mjs";
1
+ import { C as ContactRepository, Cn as LRUCache, D as TemplateRepository, Gt as ENRICHED_STREAMS, H as UserThrottle, Jt as PUBSUB_CHANNELS, K as StreamProducer, Sn as normaliseTarget, T as ProjectRepository, Ut as buildStreamEvent, V as ProjectSettingsCache, Wt as CONSUMER_GROUPS, X as IdempotencyGuard, Yt as STREAMS, _ as TemplateCache, _n as DataLoader, c as BaseWorker, cn as loadEnv, d as startHealthReporter, en as registry, f as buildUnsubscribeHeaders, it as suppressions, rt as scheduledPayloads, un as readBaseConfig, wn as globalEmitter, xn as getPriorityBucket } from "./src-CI7FGf3h.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-GRXzgPvW.mjs";
2
3
  import { randomUUID } from "node:crypto";
3
4
  import { and, eq } from "drizzle-orm";
4
5
  //#region src/services/engine/main.ts
@@ -94,6 +95,12 @@ function isInQuietHours(timezone, quietHours, fromDate = /* @__PURE__ */ new Dat
94
95
  nextActiveTime: candidateTime
95
96
  };
96
97
  }
98
+ /** Idempotency stand-in for inline messages; see EngineWorker.process. */
99
+ const NO_IDEMPOTENCY = {
100
+ checkAndMark: async (_id, _ttl) => true,
101
+ markProcessed: async (_id, _ttl) => {},
102
+ unmark: async (_id) => {}
103
+ };
97
104
  loadEnv();
98
105
  const config = readBaseConfig();
99
106
  let logger;
@@ -119,6 +126,8 @@ var EngineWorker = class extends BaseWorker {
119
126
  globalEmitter;
120
127
  contactRepo;
121
128
  db;
129
+ /** Delivery in this process, when the pipeline is fused. */
130
+ inlineNext = null;
122
131
  constructor(options) {
123
132
  super(options);
124
133
  this.registry = options.registry;
@@ -134,6 +143,10 @@ var EngineWorker = class extends BaseWorker {
134
143
  this.contactRepo = options.contactRepo;
135
144
  this.db = options.db;
136
145
  }
146
+ /** Hands due tasks straight to `next` instead of the outbound stream. */
147
+ setInlineNext(next) {
148
+ this.inlineNext = next;
149
+ }
137
150
  contactsLoader = new DataLoader(async (keys) => {
138
151
  const byProject = /* @__PURE__ */ new Map();
139
152
  for (const key of keys) {
@@ -150,6 +163,9 @@ var EngineWorker = class extends BaseWorker {
150
163
  if (!projectMap) return [];
151
164
  return projectMap.get(key.recipientId) || [];
152
165
  });
166
+ }, {
167
+ batchDelayMs: 2,
168
+ maxBatchSize: 500
153
169
  });
154
170
  /**
155
171
  * Suppressed destinations for one (project, channel), as a normalised set.
@@ -160,14 +176,22 @@ var EngineWorker = class extends BaseWorker {
160
176
  * suppression written mid-batch takes effect on the next batch — acceptable,
161
177
  * since the webhook that writes it is itself minutes behind the send.
162
178
  */
179
+ suppressionsCache = new LRUCache(1e3, 1e4);
163
180
  suppressionsLoader = new DataLoader(async (keys) => {
164
181
  const results = /* @__PURE__ */ new Map();
165
182
  for (const key of keys) {
166
183
  const cacheKey = `${key.projectId}:${key.channel}`;
167
184
  if (results.has(cacheKey)) continue;
185
+ const cached = this.suppressionsCache.get(cacheKey);
186
+ if (cached !== void 0) {
187
+ results.set(cacheKey, cached);
188
+ continue;
189
+ }
168
190
  try {
169
191
  const rows = await this.db.select({ target: suppressions.target }).from(suppressions).where(and(eq(suppressions.projectId, key.projectId), eq(suppressions.channel, key.channel)));
170
- results.set(cacheKey, new Set(rows.map((r) => normaliseTarget(r.target))));
192
+ const set = new Set(rows.map((r) => normaliseTarget(r.target)));
193
+ this.suppressionsCache.set(cacheKey, set);
194
+ results.set(cacheKey, set);
171
195
  } catch (err) {
172
196
  this.logger.error({
173
197
  err,
@@ -190,13 +214,14 @@ var EngineWorker = class extends BaseWorker {
190
214
  return;
191
215
  }
192
216
  const enriched = payloadResult.data;
193
- const idempotencyKey = `${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}`;
217
+ const idempotencyKey = `${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}` + (enriched.contactId ? `:${enriched.contactId}` : "");
194
218
  let customTtl;
195
219
  if (enriched.scheduledAt) {
196
220
  const msUntil = new Date(enriched.scheduledAt).getTime() - Date.now();
197
- if (msUntil > 0) customTtl = 86400 + Math.ceil(msUntil / 1e3);
221
+ if (msUntil > 0) customTtl = config.IDEMPOTENCY_TTL_SECONDS + Math.ceil(msUntil / 1e3);
198
222
  }
199
- if (!await this.idempotency.checkAndMark(idempotencyKey, 60)) {
223
+ const idempotency = message.inline ? NO_IDEMPOTENCY : this.idempotency;
224
+ if (!await idempotency.checkAndMark(idempotencyKey, customTtl ?? 60)) {
200
225
  this.logger.debug({
201
226
  messageId: message.id,
202
227
  eventId: event.id
@@ -216,7 +241,7 @@ var EngineWorker = class extends BaseWorker {
216
241
  recipientId: enriched.recipientId,
217
242
  reason: "user_opted_out"
218
243
  });
219
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
244
+ await idempotency.markProcessed(idempotencyKey, customTtl);
220
245
  return;
221
246
  }
222
247
  if (enriched.recipient.preferences.channels?.includes(enriched.channel)) {
@@ -231,12 +256,14 @@ var EngineWorker = class extends BaseWorker {
231
256
  recipientId: enriched.recipientId,
232
257
  reason: "channel_disabled"
233
258
  });
234
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
259
+ await idempotency.markProcessed(idempotencyKey, customTtl);
235
260
  return;
236
261
  }
237
262
  const qh = enriched.recipient.preferences.quietHours;
238
263
  if (qh && qh.length > 0 && enriched.priority !== "critical") {
239
- const qhResult = isInQuietHours(enriched.recipient.timezone, qh);
264
+ const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : /* @__PURE__ */ new Date();
265
+ const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : /* @__PURE__ */ new Date();
266
+ const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt);
240
267
  if (qhResult.inQuietHours && qhResult.nextActiveTime) {
241
268
  this.logger.info({
242
269
  messageId: message.id,
@@ -258,10 +285,15 @@ var EngineWorker = class extends BaseWorker {
258
285
  projectId: enriched.projectId
259
286
  }, "could not read project throttle settings — falling back to the global default");
260
287
  }
261
- const throttleResult = await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, {
288
+ const throttleResult = enriched.throttleCounted ? {
289
+ allowed: true,
290
+ count: 0,
291
+ limit: 0
292
+ } : await this.throttle.check(enriched.projectId, enriched.recipientId, enriched.priority, {
262
293
  limit: projectThrottle.throttleLimit,
263
294
  windowHours: projectThrottle.throttleWindowHours,
264
- scheduledAt: enriched.scheduledAt
295
+ scheduledAt: enriched.scheduledAt,
296
+ messageId: idempotencyKey
265
297
  });
266
298
  if (!throttleResult.allowed) {
267
299
  this.logger.info({
@@ -272,7 +304,7 @@ var EngineWorker = class extends BaseWorker {
272
304
  priority: enriched.priority
273
305
  }, "user throttled — dropping");
274
306
  this.globalEmitter.emit("notification:throttled", enriched.recipientId, throttleResult.count);
275
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
307
+ await idempotency.markProcessed(idempotencyKey, customTtl);
276
308
  return;
277
309
  }
278
310
  let dbTemplate = null;
@@ -289,7 +321,7 @@ var EngineWorker = class extends BaseWorker {
289
321
  recipientId: enriched.recipientId,
290
322
  reason: "template_not_found"
291
323
  });
292
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
324
+ await idempotency.markProcessed(idempotencyKey, customTtl);
293
325
  return;
294
326
  }
295
327
  }
@@ -298,33 +330,13 @@ var EngineWorker = class extends BaseWorker {
298
330
  ...enriched.aiPrompts ?? {}
299
331
  };
300
332
  const templateTopics = dbTemplate?.topics ?? [];
301
- if (Object.keys(aiPrompts).length > 0) {
302
- const aiPendingPayload = {
303
- projectId: enriched.projectId,
304
- enrichedEventId: event.id,
305
- recipientId: enriched.recipientId,
306
- channel: enriched.channel,
307
- priority: enriched.priority,
308
- templateId: enriched.templateId,
309
- templateVariables: enriched.templateVariables,
310
- recipient: enriched.recipient,
311
- aiPrompts,
312
- scheduledAt: enriched.scheduledAt,
313
- fallbackChain: enriched.fallbackChain
314
- };
315
- const aiPendingEnvelope = buildStreamEvent("notification.ai_pending", aiPendingPayload, "engine", event.metadata.traceId);
316
- await this.aiPendingProducer.publish(aiPendingEnvelope);
317
- this.logger.info({
318
- messageId: message.id,
319
- eventId: event.id,
320
- recipientId: enriched.recipientId,
321
- traceId: event.metadata.traceId
322
- }, "task routed to AI worker");
323
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
324
- return;
325
- }
326
- const rendered = renderWithTemplate(dbTemplate, enriched.templateVariables);
327
- const activeContacts = (await this.contactsLoader.load({
333
+ const useAi = Object.keys(aiPrompts).length > 0;
334
+ const activeContacts = (enriched.contactId && enriched.destination ? [{
335
+ id: enriched.contactId,
336
+ channel: enriched.channel,
337
+ target: enriched.destination,
338
+ active: true
339
+ }] : await this.contactsLoader.load({
328
340
  projectId: enriched.projectId,
329
341
  recipientId: enriched.recipientId
330
342
  })).filter((c) => c.channel === enriched.channel && c.active);
@@ -340,7 +352,7 @@ var EngineWorker = class extends BaseWorker {
340
352
  recipientId: enriched.recipientId,
341
353
  reason: "no_active_contacts"
342
354
  });
343
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
355
+ await idempotency.markProcessed(idempotencyKey, customTtl);
344
356
  return;
345
357
  }
346
358
  const suppressedTargets = await this.suppressionsLoader.load({
@@ -348,7 +360,7 @@ var EngineWorker = class extends BaseWorker {
348
360
  channel: enriched.channel
349
361
  });
350
362
  for (const contact of activeContacts) {
351
- if (contact.preferences?.optedOut) continue;
363
+ if (templateTopics.some((t) => contact.preferences?.topics?.[t] === false)) continue;
352
364
  if (contact.target && suppressedTargets.has(normaliseTarget(contact.target))) {
353
365
  this.logger.info({
354
366
  messageId: message.id,
@@ -391,7 +403,6 @@ var EngineWorker = class extends BaseWorker {
391
403
  templateVariables: enriched.templateVariables,
392
404
  aiPrompts: enriched.aiPrompts,
393
405
  recipient: enriched.recipient,
394
- renderedContent: rendered,
395
406
  destination: resolvedDestination,
396
407
  deliveryOptions: {
397
408
  maxAttempts: 3,
@@ -399,8 +410,35 @@ var EngineWorker = class extends BaseWorker {
399
410
  ...unsubscribeHeaders ? { headers: unsubscribeHeaders } : {}
400
411
  },
401
412
  fallbackChain: enriched.fallbackChain,
402
- campaignId: enriched.campaignId
413
+ campaignId: enriched.campaignId,
414
+ workflowInstanceId: enriched.workflowInstanceId
403
415
  };
416
+ if (useAi) {
417
+ await this.aiPendingProducer.publish(buildStreamEvent("notification.ai_pending", {
418
+ projectId: enriched.projectId,
419
+ enrichedEventId: event.id,
420
+ recipientId: enriched.recipientId,
421
+ channel: enriched.channel,
422
+ priority: enriched.priority,
423
+ templateId: enriched.templateId,
424
+ templateVariables: enriched.templateVariables,
425
+ recipient: enriched.recipient,
426
+ aiPrompts,
427
+ scheduledAt: enriched.scheduledAt,
428
+ fallbackChain: enriched.fallbackChain,
429
+ taskId,
430
+ destination: resolvedDestination,
431
+ deliveryHeaders: unsubscribeHeaders,
432
+ campaignId: enriched.campaignId,
433
+ workflowInstanceId: enriched.workflowInstanceId
434
+ }, "engine", event.metadata.traceId));
435
+ this.logger.debug({
436
+ messageId: message.id,
437
+ taskId,
438
+ traceId: event.metadata.traceId
439
+ }, "task routed to AI worker");
440
+ continue;
441
+ }
404
442
  const envelope = buildStreamEvent("notification.dispatched", taskPayload, "engine", event.metadata.traceId);
405
443
  const now = Date.now();
406
444
  if ((enriched.scheduledAt ? new Date(enriched.scheduledAt).getTime() : now) > now) {
@@ -418,7 +456,7 @@ var EngineWorker = class extends BaseWorker {
418
456
  scheduledAt: enriched.scheduledAt
419
457
  }, "engine", event.metadata.traceId);
420
458
  await this.scheduledProducer.publish(scheduledEnvelope);
421
- this.logger.info({
459
+ this.logger.debug({
422
460
  messageId: message.id,
423
461
  taskId,
424
462
  scheduledAt: enriched.scheduledAt,
@@ -426,8 +464,14 @@ var EngineWorker = class extends BaseWorker {
426
464
  }, "task scheduled and payload cached");
427
465
  } else {
428
466
  const p = getPriorityBucket(enriched.priority);
429
- await (this.outboundProducers[p] ?? this.outboundProducers["normal"]).publish(envelope);
430
- this.logger.info({
467
+ const outboundProducer = this.outboundProducers[p] ?? this.outboundProducers["normal"];
468
+ if (this.inlineNext) await this.inlineNext({
469
+ ...envelope,
470
+ id: randomUUID(),
471
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
472
+ });
473
+ else await outboundProducer.publish(envelope);
474
+ this.logger.debug({
431
475
  messageId: message.id,
432
476
  taskId,
433
477
  recipientId: enriched.recipientId,
@@ -435,67 +479,32 @@ var EngineWorker = class extends BaseWorker {
435
479
  }, "task dispatched");
436
480
  }
437
481
  }
438
- await this.idempotency.markProcessed(idempotencyKey, customTtl);
482
+ await idempotency.markProcessed(idempotencyKey, customTtl);
439
483
  } catch (err) {
440
- await this.idempotency.unmark(idempotencyKey).catch(() => {});
484
+ await idempotency.unmark(idempotencyKey).catch(() => {});
441
485
  throw err;
442
486
  }
443
487
  }
444
488
  };
445
489
  let subscriber = null;
446
490
  async function startEngineWorker() {
447
- logger = createLogger({
448
- name: "engine",
449
- level: config.LOG_LEVEL
450
- });
451
- redis = new RedisClient({
452
- url: config.REDIS_URL,
453
- name: "engine",
454
- logger
455
- });
456
- const dbData = createDatabase({
457
- url: config.DATABASE_URL,
458
- applicationName: "engine",
459
- logger
460
- });
461
- sql$1 = dbData.sql;
462
- db = dbData.db;
491
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "engine"));
463
492
  templateRepo = new TemplateRepository(db);
464
493
  templateCache = new TemplateCache(templateRepo);
465
494
  const contactRepo = new ContactRepository(db);
466
- consumer = new StreamConsumer({
467
- redis: redis.native,
468
- stream: ENRICHED_STREAMS,
469
- group: CONSUMER_GROUPS.ENGINE,
470
- consumer: `engine-${process.pid}`,
471
- dlqStream: STREAMS.DEAD_LETTER,
472
- batchSize: config.WORKER_CONCURRENCY,
495
+ const consumerId = uniqueConsumerId("engine");
496
+ ({consumer, pendingScanner} = createStreamConsumer({
497
+ redis,
473
498
  logger
474
- });
475
- pendingScanner = new PendingMessageScanner({
476
- redis: redis.native,
499
+ }, {
477
500
  stream: ENRICHED_STREAMS,
478
501
  group: CONSUMER_GROUPS.ENGINE,
479
- consumer: `engine-${process.pid}`,
480
- logger
481
- });
482
- const outboundProducers = {
483
- critical: new StreamProducer({
484
- redis: redis.native,
485
- stream: STREAMS.OUTBOUND_CRITICAL,
486
- logger
487
- }),
488
- normal: new StreamProducer({
489
- redis: redis.native,
490
- stream: STREAMS.OUTBOUND_NORMAL,
491
- logger
492
- }),
493
- low: new StreamProducer({
494
- redis: redis.native,
495
- stream: STREAMS.OUTBOUND_LOW,
496
- logger
497
- })
498
- };
502
+ consumer: consumerId,
503
+ batchSize: config.WORKER_CONCURRENCY,
504
+ bufferAcks: true,
505
+ coalesceMs: 2
506
+ }));
507
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
499
508
  const scheduledProducer = new StreamProducer({
500
509
  redis: redis.native,
501
510
  stream: STREAMS.SCHEDULED,
@@ -509,7 +518,7 @@ async function startEngineWorker() {
509
518
  const idempotency = new IdempotencyGuard({
510
519
  redis: redis.native,
511
520
  keyPrefix: "notif:processed:engine",
512
- ttlSeconds: 86400
521
+ ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS
513
522
  });
514
523
  const throttle = new UserThrottle({
515
524
  redis: redis.native,
@@ -535,7 +544,7 @@ async function startEngineWorker() {
535
544
  contactRepo,
536
545
  db
537
546
  });
538
- subscriber = redis.native.duplicate();
547
+ subscriber = redis.native.duplicate({ enableAutoPipelining: false });
539
548
  await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED, PUBSUB_CHANNELS.PROJECT_INVALIDATED);
540
549
  subscriber.on("message", (channel, message) => {
541
550
  if (channel === PUBSUB_CHANNELS.TEMPLATE_INVALIDATED) {
@@ -550,22 +559,20 @@ async function startEngineWorker() {
550
559
  logger.info({ env: config.NODE_ENV }, "engine starting");
551
560
  await worker.start();
552
561
  }
562
+ function getEngineWorker() {
563
+ return worker;
564
+ }
553
565
  async function stopEngineWorker() {
554
- logger?.info("shutdown initiated");
555
- if (healthInterval) {
556
- clearInterval(healthInterval);
557
- healthInterval = null;
558
- }
559
- if (subscriber) {
560
- subscriber.disconnect();
561
- subscriber = null;
562
- }
563
- if (worker) await worker.stop();
564
- if (sql$1) await sql$1.end();
565
- if (redis) await redis.disconnect();
566
- logger?.info("engine stopped");
566
+ await shutdownWorker("engine", {
567
+ logger,
568
+ timers: [healthInterval],
569
+ subscriber,
570
+ worker,
571
+ sql: sql$1,
572
+ redis
573
+ });
574
+ healthInterval = null;
575
+ subscriber = null;
567
576
  }
568
577
  //#endregion
569
- export { startEngineWorker, stopEngineWorker };
570
-
571
- //# sourceMappingURL=main-BPbPnBIG.mjs.map
578
+ export { EngineWorker, getEngineWorker, isInQuietHours, startEngineWorker, stopEngineWorker };
@@ -1,4 +1,6 @@
1
- import { A as RedisClient, Qt as CONSUMER_GROUPS, X as StreamConsumer, Y as PendingMessageScanner, Z as StreamProducer, Zt as buildStreamEvent, _t as workflowSteps, at as createDatabase, c as BaseWorker, ct as messageLogs, gn as loadEnv, gt as workflowInstances, rn as STREAMS, tt as createLogger, u as startHealthReporter, vn as readBaseConfig, vt as workflowWaiters } from "./src-CGmXRfsM.mjs";
1
+ import { K as StreamProducer, Ut as buildStreamEvent, Wt as CONSUMER_GROUPS, Yt as STREAMS, c as BaseWorker, cn as loadEnv, ct as workflowInstances, d as startHealthReporter, lt as workflowSteps, un as readBaseConfig, ut as workflowWaiters } from "./src-CI7FGf3h.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime } from "./bootstrap-GRXzgPvW.mjs";
3
+ import { t as insertMessageLogs } from "./bulk-BvRfz6Of.mjs";
2
4
  import { and, eq, gt, isNull, or } from "drizzle-orm";
3
5
  //#region src/services/events/main.ts
4
6
  loadEnv();
@@ -12,17 +14,28 @@ let pendingScanner;
12
14
  let worker;
13
15
  let healthInterval = null;
14
16
  let workflowProducer;
15
- var EventWorker = class extends BaseWorker {
17
+ var EventWorker = class EventWorker extends BaseWorker {
16
18
  dbConn;
17
19
  workflowProducer;
18
20
  messageLogBuffer = [];
19
21
  eventBuffer = [];
20
22
  flushInterval = null;
21
- isFlushing = false;
23
+ flushesInFlight = 0;
24
+ static MAX_CONCURRENT_FLUSHES = 4;
25
+ /**
26
+ * Buffered logs that trigger a flush without waiting for the interval.
27
+ *
28
+ * Every message holds its worker slot until its log row is written, so a
29
+ * purely timed flush caps the worker at `concurrency / LOG_FLUSH_INTERVAL_MS`
30
+ * rows per second — far below what one delivery worker produces. Flushing
31
+ * once half the slots are waiting keeps the other half reading.
32
+ */
33
+ flushThreshold;
22
34
  constructor(options) {
23
35
  super(options);
24
36
  this.dbConn = options.db;
25
37
  this.workflowProducer = options.workflowProducer;
38
+ this.flushThreshold = Math.max(1, Math.min(500, Math.floor(this.concurrency / 2)));
26
39
  this.startFlushInterval();
27
40
  }
28
41
  startFlushInterval() {
@@ -31,8 +44,8 @@ var EventWorker = class extends BaseWorker {
31
44
  }, config.LOG_FLUSH_INTERVAL_MS);
32
45
  }
33
46
  async flushLogs() {
34
- if (this.isFlushing || this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0) return;
35
- this.isFlushing = true;
47
+ if (this.flushesInFlight >= EventWorker.MAX_CONCURRENT_FLUSHES || this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0) return;
48
+ this.flushesInFlight++;
36
49
  const startTime = Date.now();
37
50
  const batch = this.messageLogBuffer;
38
51
  this.messageLogBuffer = [];
@@ -40,9 +53,9 @@ var EventWorker = class extends BaseWorker {
40
53
  this.eventBuffer = [];
41
54
  try {
42
55
  if (batch.length > 0) {
43
- await this.dbConn.insert(messageLogs).values(batch.map((b) => b.log)).onConflictDoNothing();
56
+ await insertMessageLogs(this.dbConn, batch.map((b) => b.log));
44
57
  for (const b of batch) b.resolve();
45
- this.logger.info({
58
+ this.logger.debug({
46
59
  batchSize: batch.length,
47
60
  latencyMs: Date.now() - startTime
48
61
  }, "flushed delivery logs to database");
@@ -66,13 +79,18 @@ var EventWorker = class extends BaseWorker {
66
79
  for (const b of batch) b.reject(error);
67
80
  for (const e of events) e.reject(error);
68
81
  } finally {
69
- this.isFlushing = false;
82
+ this.flushesInFlight--;
83
+ if (this.messageLogBuffer.length >= this.flushThreshold) this.flushLogs();
70
84
  }
71
85
  }
72
86
  async stop() {
73
- if (this.flushInterval) clearInterval(this.flushInterval);
74
87
  await super.stop();
75
- await this.flushLogs();
88
+ if (this.flushInterval) clearInterval(this.flushInterval);
89
+ const deadline = Date.now() + 1e4;
90
+ while ((this.messageLogBuffer.length > 0 || this.eventBuffer.length > 0 || this.flushesInFlight > 0) && Date.now() < deadline) {
91
+ await this.flushLogs();
92
+ await new Promise((resolve) => setTimeout(resolve, 10));
93
+ }
76
94
  }
77
95
  async process(message) {
78
96
  const { event } = message;
@@ -89,24 +107,28 @@ var EventWorker = class extends BaseWorker {
89
107
  }
90
108
  const status = event.type === "notification.dispatched" ? "dispatched" : event.type === "notification.delivered" ? "delivered" : "failed";
91
109
  const kind = event.type === "notification.dispatched" ? "dispatched" : "attempt";
92
- publishPromises.push(new Promise((resolve, reject) => {
110
+ const row = (rowKind, rowStatus, providerMessageId, timestamp) => new Promise((resolve, reject) => {
93
111
  this.messageLogBuffer.push({
94
112
  log: {
95
113
  projectId: payload.projectId,
96
114
  taskId: payload.taskId,
97
- providerMessageId: payload.providerMessageId || null,
115
+ providerMessageId,
98
116
  channel: payload.channel,
99
117
  attempt: payload.attempt || 1,
100
- kind,
101
- status,
118
+ kind: rowKind,
119
+ status: rowStatus,
102
120
  templateId: payload.templateId || null,
103
121
  workflowInstanceId: payload.workflowInstanceId || null,
104
- campaignId: payload.campaignId || null
122
+ campaignId: payload.campaignId || null,
123
+ timestamp
105
124
  },
106
125
  resolve,
107
126
  reject
108
127
  });
109
- }));
128
+ });
129
+ if (kind === "attempt" && payload.dispatchedAt) publishPromises.push(row("dispatched", "dispatched", null, payload.dispatchedAt));
130
+ publishPromises.push(row(kind, status, payload.providerMessageId || null));
131
+ if (this.messageLogBuffer.length >= this.flushThreshold) this.flushLogs();
110
132
  await Promise.all(publishPromises);
111
133
  return;
112
134
  }
@@ -171,48 +193,29 @@ var EventWorker = class extends BaseWorker {
171
193
  }
172
194
  };
173
195
  async function startEventWorker() {
174
- logger = createLogger({
175
- name: "event-worker",
176
- level: config.LOG_LEVEL
177
- });
178
- redis = new RedisClient({
179
- url: config.REDIS_URL,
180
- name: "events",
181
- logger
182
- });
183
- const dbData = createDatabase({
184
- url: config.DATABASE_URL,
185
- applicationName: "events",
186
- logger
187
- });
188
- sql$1 = dbData.sql;
189
- db = dbData.db;
196
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "events", "event-worker"));
190
197
  workflowProducer = new StreamProducer({
191
198
  redis: redis.native,
192
199
  stream: STREAMS.WORKFLOW_INBOUND,
193
200
  logger
194
201
  });
195
- consumer = new StreamConsumer({
196
- redis: redis.native,
197
- stream: STREAMS.EVENTS_INBOUND,
198
- group: CONSUMER_GROUPS.EVENTS,
199
- consumer: `events-${process.pid}`,
200
- dlqStream: STREAMS.DEAD_LETTER,
201
- batchSize: config.WORKER_CONCURRENCY,
202
+ const eventsConcurrency = Math.max(config.WORKER_CONCURRENCY, config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY);
203
+ ({consumer, pendingScanner} = createStreamConsumer({
204
+ redis,
202
205
  logger
203
- });
204
- pendingScanner = new PendingMessageScanner({
205
- redis: redis.native,
206
+ }, {
206
207
  stream: STREAMS.EVENTS_INBOUND,
207
208
  group: CONSUMER_GROUPS.EVENTS,
208
- consumer: `events-${process.pid}`,
209
- logger
210
- });
209
+ consumer: uniqueConsumerId("events"),
210
+ batchSize: eventsConcurrency,
211
+ bufferAcks: true,
212
+ coalesceMs: 10
213
+ }));
211
214
  worker = new EventWorker({
212
215
  consumer,
213
216
  pendingScanner,
214
217
  logger,
215
- concurrency: config.WORKER_CONCURRENCY,
218
+ concurrency: eventsConcurrency,
216
219
  db,
217
220
  workflowProducer
218
221
  });
@@ -221,14 +224,14 @@ async function startEventWorker() {
221
224
  await worker.start();
222
225
  }
223
226
  async function stopEventWorker() {
224
- logger?.info("shutdown initiated");
225
- if (healthInterval) clearInterval(healthInterval);
226
- if (worker) await worker.stop();
227
- if (sql$1) await sql$1.end();
228
- if (redis) await redis.disconnect();
229
- logger?.info("event worker stopped");
227
+ await shutdownWorker("event worker", {
228
+ logger,
229
+ timers: [healthInterval],
230
+ worker,
231
+ sql: sql$1,
232
+ redis
233
+ });
234
+ healthInterval = null;
230
235
  }
231
236
  //#endregion
232
237
  export { startEventWorker, stopEventWorker };
233
-
234
- //# sourceMappingURL=main-4Wyge9ms.mjs.map