notifkit 0.1.9 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +66 -72
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-GRXzgPvW.mjs +89 -0
  6. package/dist/bulk-BvRfz6Of.mjs +77 -0
  7. package/dist/index.d.mts +1107 -8885
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
  10. package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
  11. package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
  12. package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
  13. package/dist/main-D6od9N8h.mjs +312 -0
  14. package/dist/main-D84OX0ux.mjs +811 -0
  15. package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
  16. package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
  17. package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +17 -17
  22. package/scripts/create-admin.mjs +8 -11
  23. package/src/client.ts +15 -5
  24. package/src/config/index.ts +26 -3
  25. package/src/contracts/events/notification-ai-pending.ts +12 -0
  26. package/src/contracts/events/notification-delivered.ts +7 -0
  27. package/src/contracts/events/notification-dispatched.ts +21 -2
  28. package/src/contracts/events/notification-enriched.ts +16 -0
  29. package/src/contracts/events/notification-failed.ts +2 -0
  30. package/src/contracts/events/notification-requested.ts +2 -0
  31. package/src/contracts/registry.ts +2 -2
  32. package/src/contracts/sdk.ts +77 -29
  33. package/src/db/bulk.ts +132 -0
  34. package/src/db/index.ts +2 -2
  35. package/src/db/schema.ts +3 -50
  36. package/src/idempotency/index.ts +60 -6
  37. package/src/logger/index.ts +0 -12
  38. package/src/queue/index.ts +255 -76
  39. package/src/rate-limiter/index.ts +217 -16
  40. package/src/redis/index.ts +265 -0
  41. package/src/repositories/index.ts +171 -82
  42. package/src/server.ts +71 -41
  43. package/src/services/ai/main.ts +83 -70
  44. package/src/services/api/admin-static.ts +41 -12
  45. package/src/services/api/handlers.ts +357 -159
  46. package/src/services/api/http.ts +34 -7
  47. package/src/services/api/main.ts +104 -67
  48. package/src/services/api/router.ts +18 -5
  49. package/src/services/delivery/main.ts +495 -230
  50. package/src/services/delivery/throttle.ts +42 -59
  51. package/src/services/engine/main.ts +206 -152
  52. package/src/services/enricher/main.ts +155 -107
  53. package/src/services/events/main.ts +100 -55
  54. package/src/services/scheduler/main.ts +151 -79
  55. package/src/services/scheduler/send-gate.ts +117 -0
  56. package/src/services/workflow/main.ts +220 -100
  57. package/src/shared/batch-processor.ts +65 -34
  58. package/src/shared/dataloader.ts +58 -22
  59. package/src/shared/events.ts +3 -0
  60. package/src/shared/index.ts +0 -1
  61. package/src/shared/utils.ts +2 -47
  62. package/src/templates/index.ts +0 -67
  63. package/src/transport/index.ts +6 -0
  64. package/src/workers/bootstrap.ts +124 -0
  65. package/src/workers/index.ts +99 -24
  66. package/src/workflows/sdk.ts +2 -0
  67. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  68. package/dist/index.d.mts.map +0 -1
  69. package/dist/index.mjs.map +0 -1
  70. package/dist/main-4Wyge9ms.mjs.map +0 -1
  71. package/dist/main-AO0JaY5z.mjs.map +0 -1
  72. package/dist/main-BMYSYcv2.mjs +0 -224
  73. package/dist/main-BMYSYcv2.mjs.map +0 -1
  74. package/dist/main-BPbPnBIG.mjs.map +0 -1
  75. package/dist/main-CyeQ_yDt.mjs +0 -632
  76. package/dist/main-CyeQ_yDt.mjs.map +0 -1
  77. package/dist/main-D19Wxs0x.mjs.map +0 -1
  78. package/dist/main-DSmh1e-V.mjs.map +0 -1
  79. package/dist/main-JdOvJ70b.mjs.map +0 -1
  80. package/dist/src-CGmXRfsM.mjs.map +0 -1
@@ -1,14 +1,13 @@
1
- import { randomUUID } from "node:crypto";
2
1
  import { loadEnv, readBaseConfig } from "@/index.js";
3
- import { createLogger } from "@/index.js";
4
- import { RedisClient, type Redis } from "@/index.js";
2
+ import type { Logger } from "@/index.js";
3
+ import type { RedisClient, Redis } from "@/index.js";
5
4
  import {
6
- StreamConsumer,
7
- PendingMessageScanner,
5
+ type StreamConsumer,
6
+ type PendingMessageScanner,
8
7
  StreamProducer,
9
8
  type StreamMessage,
10
9
  } from "@/index.js";
11
- import { BaseWorker } from "@/index.js";
10
+ import { BaseWorker, type ProcessResult } from "@/index.js";
12
11
  import {
13
12
  STREAMS,
14
13
  CONSUMER_GROUPS,
@@ -20,21 +19,26 @@ import {
20
19
  AI_DEFAULTS,
21
20
  } from "@/index.js";
22
21
  import { generateText } from "ai";
23
- import { type StreamName } from "@/contracts/streams.js";
24
22
  import { IdempotencyGuard } from "@/index.js";
25
23
  import { TemplateRepository } from "@/index.js";
26
- import { createDatabase } from "@/db/index.js";
27
24
  import { scheduledPayloads } from "@/db/schema.js";
28
25
  import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
29
26
  import { renderWithTemplate, TemplateCache } from "@/templates/index.js";
30
27
  import { startHealthReporter } from "@/workers/index.js";
28
+ import {
29
+ createPriorityProducers,
30
+ createStreamConsumer,
31
+ createWorkerRuntime,
32
+ shutdownWorker,
33
+ uniqueConsumerId,
34
+ } from "@/workers/bootstrap.js";
31
35
 
32
36
  // ─── Bootstrap ─────────────────────────────────────────────────────────────
33
37
 
34
38
  loadEnv();
35
39
  const config = readBaseConfig();
36
40
 
37
- let logger: ReturnType<typeof createLogger>;
41
+ let logger: Logger;
38
42
  let redis: RedisClient;
39
43
  let sql: any;
40
44
  let db: any;
@@ -110,7 +114,7 @@ export class AiWorker extends BaseWorker {
110
114
  this.outboundProducers = options.outboundProducers;
111
115
  this.db = options.db;
112
116
  }
113
- async process(message: StreamMessage): Promise<void> {
117
+ async process(message: StreamMessage): Promise<ProcessResult> {
114
118
  const { event } = message;
115
119
 
116
120
  const payloadResult = this.registry.safeParsePayload("notification.ai_pending", event.payload);
@@ -124,21 +128,44 @@ export class AiWorker extends BaseWorker {
124
128
 
125
129
  const pending = payloadResult.data as NotificationAiPendingPayload;
126
130
 
127
- // Idempotency
128
- const idempotencyKey = `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}:ai`;
129
- if (!(await this.idempotency.checkAndMark(idempotencyKey))) {
131
+ // Stable across replays: the engine's task id when it sent one. A fresh id
132
+ // per attempt made a replay after a crash a second, unrelated task that
133
+ // delivery's lease could not recognise — the recipient got it twice.
134
+ const taskId =
135
+ pending.taskId ?? `${pending.enrichedEventId}:${pending.recipientId}:${pending.channel}`;
136
+
137
+ // Idempotency. The lease has to outlast every generation this message may
138
+ // run, or a second worker takes it over mid-call and bills the prompts again.
139
+ const idempotencyKey = `${taskId}:ai`;
140
+ const aiConfig = getAiConfig();
141
+ const maxPrompts = aiConfig.maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;
142
+ const leaseSeconds =
143
+ Math.ceil(((aiConfig.timeoutMs ?? AI_DEFAULTS.timeoutMs) * maxPrompts) / 1000) + 30;
144
+ const lease = this.idempotency.acquireLease
145
+ ? await this.idempotency.acquireLease(idempotencyKey, leaseSeconds)
146
+ : (await this.idempotency.checkAndMark(idempotencyKey, leaseSeconds))
147
+ ? "acquired"
148
+ : "locked";
149
+
150
+ if (lease === "completed") {
130
151
  this.logger.debug(
131
152
  { messageId: message.id, eventId: event.id },
132
- "duplicate ai task — skipping",
153
+ "ai task already completed — acking message",
133
154
  );
134
155
  return;
135
156
  }
157
+
158
+ if (lease === "locked") {
159
+ this.logger.debug(
160
+ { messageId: message.id, eventId: event.id },
161
+ "duplicate ai task — lock held by another worker, leaving pending in stream",
162
+ );
163
+ return { ack: false };
164
+ }
136
165
  try {
137
166
  // Execute AI prompts. Each key is a separate billed model call, so the
138
167
  // count is capped rather than being driven by whatever the caller sent.
139
168
  const promptEntries = Object.entries(pending.aiPrompts);
140
- const maxPrompts =
141
- getAiConfig().maxPromptsPerNotification ?? AI_DEFAULTS.maxPromptsPerNotification;
142
169
  if (promptEntries.length > maxPrompts) {
143
170
  this.logger.warn(
144
171
  { messageId: message.id, requested: promptEntries.length, maxPrompts },
@@ -160,9 +187,9 @@ export class AiWorker extends BaseWorker {
160
187
 
161
188
  const rendered = renderWithTemplate(dbTemplate, finalVars);
162
189
 
163
- const taskId = randomUUID();
164
190
  const destination =
165
- pending.channel === "email"
191
+ pending.destination ??
192
+ (pending.channel === "email"
166
193
  ? pending.recipient.email
167
194
  : pending.channel === "sms" || pending.channel === "whatsapp"
168
195
  ? pending.recipient.phone
@@ -176,7 +203,7 @@ export class AiWorker extends BaseWorker {
176
203
  ? pending.recipient.slack
177
204
  : pending.channel === "push"
178
205
  ? (pending.recipient.pushTokens?.[0] ?? pending.recipient.pushToken)
179
- : undefined;
206
+ : undefined);
180
207
  const resolvedDestination =
181
208
  destination ?? (pending.channel === "push" ? undefined : pending.recipientId);
182
209
 
@@ -196,8 +223,11 @@ export class AiWorker extends BaseWorker {
196
223
  deliveryOptions: {
197
224
  maxAttempts: 3,
198
225
  timeoutMs: 10_000,
226
+ ...(pending.deliveryHeaders ? { headers: pending.deliveryHeaders } : {}),
199
227
  },
200
228
  fallbackChain: pending.fallbackChain,
229
+ campaignId: pending.campaignId,
230
+ workflowInstanceId: pending.workflowInstanceId,
201
231
  };
202
232
 
203
233
  const envelope = buildStreamEvent(
@@ -212,10 +242,15 @@ export class AiWorker extends BaseWorker {
212
242
  const scheduledAt = pending.scheduledAt ? new Date(pending.scheduledAt).getTime() : now;
213
243
 
214
244
  if (scheduledAt > now) {
215
- await this.db.insert(scheduledPayloads).values({
216
- taskId,
217
- payload: taskPayload,
218
- });
245
+ await this.db
246
+ .insert(scheduledPayloads)
247
+ .values({
248
+ taskId,
249
+ // The send time rides in the payload so the scheduler can re-queue
250
+ // this from Postgres if Redis loses its data.
251
+ payload: { ...taskPayload, scheduledAt: pending.scheduledAt },
252
+ })
253
+ .onConflictDoNothing();
219
254
 
220
255
  const scheduledEnvelope = buildStreamEvent(
221
256
  "notification.scheduled",
@@ -254,6 +289,7 @@ export class AiWorker extends BaseWorker {
254
289
  "task dispatched after AI generation",
255
290
  );
256
291
  }
292
+ await this.idempotency.markProcessed?.(idempotencyKey);
257
293
  } catch (err) {
258
294
  await this.idempotency.unmark(idempotencyKey);
259
295
  if (err instanceof PermanentAiError || (err as Error)?.name === "PermanentAiError") {
@@ -276,40 +312,20 @@ export class AiWorker extends BaseWorker {
276
312
  }
277
313
 
278
314
  export async function startAiWorker() {
279
- logger = createLogger({ name: "ai", level: config.LOG_LEVEL });
280
- redis = new RedisClient({ url: config.REDIS_URL, name: "ai", logger });
281
- const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "ai", logger });
282
- sql = dbData.sql;
283
- db = dbData.db;
315
+ ({ logger, redis, sql, db } = createWorkerRuntime(config, "ai"));
284
316
  templateRepo = new TemplateRepository(db);
285
317
  templateCache = new TemplateCache(templateRepo);
286
- consumer = new StreamConsumer({
287
- redis: redis.native,
288
- stream: STREAMS.AI_PENDING as StreamName,
289
- group: CONSUMER_GROUPS.AI,
290
- consumer: `ai-${process.pid}`,
291
- dlqStream: STREAMS.DEAD_LETTER,
292
- batchSize: config.WORKER_CONCURRENCY,
293
- logger,
294
- });
295
-
296
- pendingScanner = new PendingMessageScanner({
297
- redis: redis.native,
298
- stream: STREAMS.AI_PENDING as StreamName,
299
- group: CONSUMER_GROUPS.AI,
300
- consumer: `ai-${process.pid}`,
301
- logger,
302
- });
303
-
304
- const outboundProducers = {
305
- critical: new StreamProducer({
306
- redis: redis.native,
307
- stream: STREAMS.OUTBOUND_CRITICAL,
308
- logger,
309
- }),
310
- normal: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_NORMAL, logger }),
311
- low: new StreamProducer({ redis: redis.native, stream: STREAMS.OUTBOUND_LOW, logger }),
312
- };
318
+ ({ consumer, pendingScanner } = createStreamConsumer(
319
+ { redis, logger },
320
+ {
321
+ stream: STREAMS.AI_PENDING,
322
+ group: CONSUMER_GROUPS.AI,
323
+ consumer: uniqueConsumerId("ai"),
324
+ batchSize: config.WORKER_CONCURRENCY,
325
+ },
326
+ ));
327
+
328
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
313
329
 
314
330
  const scheduledProducer = new StreamProducer({
315
331
  redis: redis.native,
@@ -320,7 +336,7 @@ export async function startAiWorker() {
320
336
  const idempotency = new IdempotencyGuard({
321
337
  redis: redis.native,
322
338
  keyPrefix: "notif:processed:ai",
323
- ttlSeconds: 86_400,
339
+ ttlSeconds: config.IDEMPOTENCY_TTL_SECONDS,
324
340
  });
325
341
 
326
342
  // AI generation
@@ -372,7 +388,7 @@ export async function startAiWorker() {
372
388
  db,
373
389
  });
374
390
 
375
- subscriber = redis.native.duplicate();
391
+ subscriber = redis.native.duplicate({ enableAutoPipelining: false });
376
392
  await subscriber.subscribe("template.invalidated");
377
393
  subscriber.on("message", (channel: string, message: string) => {
378
394
  if (channel === "template.invalidated") {
@@ -388,17 +404,14 @@ export async function startAiWorker() {
388
404
  }
389
405
 
390
406
  export async function stopAiWorker(): Promise<void> {
391
- logger?.info("shutdown initiated");
392
- if (healthInterval) {
393
- clearInterval(healthInterval);
394
- healthInterval = null;
395
- }
396
- if (subscriber) {
397
- subscriber.disconnect();
398
- subscriber = null;
399
- }
400
- if (worker) await worker.stop();
401
- if (sql) await sql.end();
402
- if (redis) await redis.disconnect();
403
- logger?.info("ai stopped");
407
+ await shutdownWorker("ai", {
408
+ logger,
409
+ timers: [healthInterval],
410
+ subscriber,
411
+ worker,
412
+ sql,
413
+ redis,
414
+ });
415
+ healthInterval = null;
416
+ subscriber = null;
404
417
  }
@@ -1,6 +1,7 @@
1
1
  import type { IncomingMessage, ServerResponse } from "node:http";
2
- import { resolve, extname, join, relative, isAbsolute } from "node:path";
2
+ import { resolve, extname, join, relative, isAbsolute, dirname } from "node:path";
3
3
  import { existsSync, statSync, createReadStream } from "node:fs";
4
+ import { fileURLToPath } from "node:url";
4
5
  import { request as httpRequest } from "node:http";
5
6
 
6
7
  const MIME_TYPES: Record<string, string> = {
@@ -23,18 +24,46 @@ const MIME_TYPES: Record<string, string> = {
23
24
  ".map": "application/json; charset=utf-8",
24
25
  };
25
26
 
26
- // Possible output directories where built dashboard static files might reside
27
- const POSSIBLE_DASHBOARD_DIRS = [
28
- resolve(process.cwd(), "dashboard", "dist"),
29
- resolve(process.cwd(), "dashboard", "out"),
30
- resolve(process.cwd(), "dist", "admin"),
31
- resolve(process.cwd(), "dist", "dashboard"),
27
+ // Where a built dashboard may sit, relative to a base directory.
28
+ const DASHBOARD_SUBDIRS = [
29
+ ["dashboard", "dist"],
30
+ ["dashboard", "out"],
31
+ ["dist", "admin"],
32
+ ["dist", "dashboard"],
32
33
  ];
33
34
 
34
- function getDashboardDir(): string | null {
35
- for (const dir of POSSIBLE_DASHBOARD_DIRS) {
36
- if (existsSync(dir) && existsSync(join(dir, "index.html"))) {
37
- return dir;
35
+ const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
36
+
37
+ /** The nearest ancestor of `dir` holding a package.json: notifkit's own root. */
38
+ function packageRoot(dir: string): string | null {
39
+ for (let d = dir; ; d = dirname(d)) {
40
+ if (existsSync(join(d, "package.json"))) return d;
41
+ if (dirname(d) === d) return null;
42
+ }
43
+ }
44
+
45
+ /**
46
+ * Locates the built dashboard.
47
+ *
48
+ * The package ships `dashboard/dist`, so it is looked for beside this module
49
+ * first. Resolving it from the working directory alone found it only when the
50
+ * process was started from a notifkit checkout; installed as a dependency, the
51
+ * working directory is the host app's and /admin reported "not built".
52
+ * `NOTIFKIT_DASHBOARD_DIR` overrides both.
53
+ */
54
+ export function findDashboardDir(
55
+ moduleDir: string = MODULE_DIR,
56
+ cwd: string = process.cwd(),
57
+ env: Record<string, string | undefined> = process.env,
58
+ ): string | null {
59
+ const override = env.NOTIFKIT_DASHBOARD_DIR;
60
+ if (override && existsSync(join(override, "index.html"))) return resolve(override);
61
+
62
+ const bases = [packageRoot(moduleDir), cwd].filter((b): b is string => b !== null);
63
+ for (const base of bases) {
64
+ for (const sub of DASHBOARD_SUBDIRS) {
65
+ const dir = resolve(base, ...sub);
66
+ if (existsSync(join(dir, "index.html"))) return dir;
38
67
  }
39
68
  }
40
69
  return null;
@@ -111,7 +140,7 @@ export async function handleAdminRequest(
111
140
  }
112
141
  }
113
142
 
114
- const dashboardDir = getDashboardDir();
143
+ const dashboardDir = findDashboardDir();
115
144
  if (!dashboardDir) {
116
145
  // If dashboard build not found, return friendly message
117
146
  res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" });