notifkit 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +37 -1
  2. package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
  3. package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
  4. package/dashboard/dist/index.html +2 -2
  5. package/dist/bootstrap-omGPIBny.mjs +89 -0
  6. package/dist/bulk-C4hTcnLu.mjs +77 -0
  7. package/dist/index.d.mts +1160 -8886
  8. package/dist/index.mjs +2 -38
  9. package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
  10. package/dist/main-B5BwjIMu.mjs +811 -0
  11. package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
  12. package/dist/main-Bes11Ad3.mjs +312 -0
  13. package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
  14. package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
  15. package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
  16. package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
  17. package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
  18. package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
  19. package/drizzle/0007_workflow_status_canceled.sql +3 -0
  20. package/drizzle/meta/_journal.json +14 -0
  21. package/package.json +21 -18
  22. package/scripts/create-admin.mjs +8 -1
  23. package/scripts/migrate.mjs +43 -0
  24. package/src/client.ts +15 -5
  25. package/src/config/index.ts +37 -3
  26. package/src/contracts/events/notification-ai-pending.ts +12 -0
  27. package/src/contracts/events/notification-delivered.ts +7 -0
  28. package/src/contracts/events/notification-dispatched.ts +21 -2
  29. package/src/contracts/events/notification-enriched.ts +16 -0
  30. package/src/contracts/events/notification-failed.ts +2 -0
  31. package/src/contracts/events/notification-requested.ts +2 -0
  32. package/src/contracts/registry.ts +2 -2
  33. package/src/contracts/sdk.ts +77 -29
  34. package/src/db/bulk.ts +132 -0
  35. package/src/db/index.ts +174 -7
  36. package/src/db/schema.ts +3 -50
  37. package/src/idempotency/index.ts +60 -6
  38. package/src/logger/index.ts +0 -12
  39. package/src/queue/index.ts +255 -76
  40. package/src/rate-limiter/index.ts +217 -16
  41. package/src/redis/index.ts +265 -0
  42. package/src/repositories/index.ts +171 -82
  43. package/src/server.ts +64 -41
  44. package/src/services/ai/main.ts +83 -70
  45. package/src/services/api/admin-static.ts +41 -12
  46. package/src/services/api/handlers.ts +357 -159
  47. package/src/services/api/http.ts +34 -7
  48. package/src/services/api/main.ts +104 -67
  49. package/src/services/api/router.ts +18 -5
  50. package/src/services/delivery/main.ts +495 -230
  51. package/src/services/delivery/throttle.ts +42 -59
  52. package/src/services/engine/main.ts +206 -152
  53. package/src/services/enricher/main.ts +155 -107
  54. package/src/services/events/main.ts +100 -55
  55. package/src/services/scheduler/main.ts +151 -79
  56. package/src/services/scheduler/send-gate.ts +117 -0
  57. package/src/services/workflow/main.ts +220 -100
  58. package/src/shared/batch-processor.ts +65 -34
  59. package/src/shared/dataloader.ts +58 -22
  60. package/src/shared/events.ts +3 -0
  61. package/src/shared/index.ts +0 -1
  62. package/src/shared/utils.ts +2 -47
  63. package/src/templates/index.ts +0 -67
  64. package/src/transport/index.ts +6 -0
  65. package/src/workers/bootstrap.ts +124 -0
  66. package/src/workers/index.ts +99 -24
  67. package/src/workflows/sdk.ts +2 -0
  68. package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
  69. package/dist/index.d.mts.map +0 -1
  70. package/dist/index.mjs.map +0 -1
  71. package/dist/main-39DtKrX4.mjs.map +0 -1
  72. package/dist/main-B3iQrl3t.mjs.map +0 -1
  73. package/dist/main-BM8DH3v3.mjs +0 -632
  74. package/dist/main-BM8DH3v3.mjs.map +0 -1
  75. package/dist/main-BgmT1MPe.mjs.map +0 -1
  76. package/dist/main-BiFiOX9D.mjs.map +0 -1
  77. package/dist/main-C9s-2ArH.mjs.map +0 -1
  78. package/dist/main-Cqo3rSb5.mjs +0 -224
  79. package/dist/main-Cqo3rSb5.mjs.map +0 -1
  80. package/dist/main-Dz2sUh3c.mjs.map +0 -1
  81. package/dist/src-QiTu5e_a.mjs.map +0 -1
@@ -0,0 +1,312 @@
1
+ import { $t as STREAMS, C as ContactRepository, D as TemplateRepository, En as normaliseTarget, Jt as CONSUMER_GROUPS, O as UserRepository, On as globalEmitter, Tn as getPriorityBucket, c as BaseWorker, ct as suppressions, d as startHealthReporter, fn as loadEnv, in as registry, mn as readBaseConfig, qt as buildStreamEvent, st as scheduledPayloads } from "./src-CIPTzq9i.mjs";
2
+ import { a as uniqueConsumerId, i as shutdownWorker, n as createStreamConsumer, r as createWorkerRuntime, t as createPriorityProducers } from "./bootstrap-omGPIBny.mjs";
3
+ import { randomUUID } from "node:crypto";
4
+ import { and, asc, eq, gt, inArray, or, sql } from "drizzle-orm";
5
+ //#region src/services/scheduler/send-gate.ts
6
+ /**
7
+ * Re-checks parked sends at the moment they are released.
8
+ *
9
+ * The engine clears a message against suppressions, opt-outs and contacts when
10
+ * it first sees it. A send parked for later — a `sendAt`, a quiet-hours
11
+ * deferral, a retry — can sit for hours or days, and whatever the person did in
12
+ * between (unsubscribed, complained, hard-bounced, removed the address) must
13
+ * still stop it. Without this the scheduler sent to them anyway.
14
+ *
15
+ * Reads Postgres directly rather than through the repositories' caches: this
16
+ * runs once per poll, not once per message, and a stale "still subscribed" is
17
+ * the very thing it exists to prevent.
18
+ *
19
+ * Returns the blocked tasks with the reason. Throws if a lookup fails, so the
20
+ * caller holds the batch rather than sending it unchecked.
21
+ */
22
+ async function findBlockedAtRelease(db, tasks) {
23
+ const blocked = /* @__PURE__ */ new Map();
24
+ const byProject = /* @__PURE__ */ new Map();
25
+ for (const t of tasks) {
26
+ if (!t?.projectId || !t.recipientId) continue;
27
+ if (!byProject.has(t.projectId)) byProject.set(t.projectId, []);
28
+ byProject.get(t.projectId).push(t);
29
+ }
30
+ for (const [projectId, group] of byProject) {
31
+ const recipientIds = [...new Set(group.map((t) => t.recipientId))];
32
+ const templateIds = [...new Set(group.map((t) => t.templateId).filter(Boolean))];
33
+ const targets = [...new Set(group.map((t) => t.destination && normaliseTarget(t.destination)).filter(Boolean))];
34
+ const templateRepo = new TemplateRepository(db);
35
+ const [suppressedRows, users, contactsByUser, templates] = await Promise.all([
36
+ targets.length > 0 ? db.select({
37
+ channel: suppressions.channel,
38
+ target: suppressions.target
39
+ }).from(suppressions).where(and(eq(suppressions.projectId, projectId), inArray(suppressions.target, targets))) : [],
40
+ new UserRepository(db).findRecordsByIds(projectId, recipientIds),
41
+ new ContactRepository(db).findActiveByUserIds(projectId, recipientIds),
42
+ Promise.all(templateIds.map((id) => templateRepo.findById(projectId, id)))
43
+ ]);
44
+ const suppressed = new Set(suppressedRows.map((r) => `${r.channel}:${normaliseTarget(r.target)}`));
45
+ const userById = new Map(users.map((u) => [u.userId, u]));
46
+ const topicsByTemplate = /* @__PURE__ */ new Map();
47
+ templateIds.forEach((id, i) => topicsByTemplate.set(id, templates[i]?.topics ?? []));
48
+ for (const t of group) {
49
+ const reason = decide(t, t.templateId ? topicsByTemplate.get(t.templateId) ?? [] : [], suppressed, userById.get(t.recipientId), contactsByUser);
50
+ if (reason) blocked.set(t.taskId, reason);
51
+ }
52
+ }
53
+ return blocked;
54
+ }
55
+ function decide(t, topics, suppressed, user, contactsByUser) {
56
+ if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) return "suppressed";
57
+ if (!user) return "recipient_not_found";
58
+ if (topics.some((topic) => user.preferences.topics?.[topic] === false)) return "user_opted_out";
59
+ if (user.preferences.channels?.[t.channel] === false) return "channel_disabled";
60
+ if (t.destination) {
61
+ const contact = (contactsByUser.get(t.recipientId) ?? []).find((c) => c.channel === t.channel && c.target === t.destination);
62
+ if (!contact) return "contact_inactive";
63
+ if (topics.some((topic) => contact.preferences?.topics?.[topic] === false)) return "user_opted_out";
64
+ }
65
+ }
66
+ //#endregion
67
+ //#region src/services/scheduler/main.ts
68
+ loadEnv();
69
+ const config = readBaseConfig();
70
+ let logger;
71
+ let redis;
72
+ let sql$1;
73
+ let db;
74
+ let consumer;
75
+ let pendingScanner;
76
+ let worker;
77
+ let healthInterval = null;
78
+ let pollTimeout = null;
79
+ let isPolling = false;
80
+ var SchedulerWorker = class extends BaseWorker {
81
+ registry;
82
+ redisCli;
83
+ constructor(options) {
84
+ super(options);
85
+ this.registry = options.registry;
86
+ this.redisCli = options.redis;
87
+ }
88
+ async process(message) {
89
+ const { event } = message;
90
+ const payloadResult = this.registry.safeParsePayload("notification.scheduled", event.payload);
91
+ if (!payloadResult.success) {
92
+ this.logger.warn({
93
+ messageId: message.id,
94
+ issues: payloadResult.error.issues
95
+ }, "invalid notification.scheduled payload — skipping");
96
+ return;
97
+ }
98
+ const scheduled = payloadResult.data;
99
+ const scheduledAt = new Date(scheduled.scheduledAt).getTime();
100
+ const taskData = JSON.stringify({
101
+ taskId: scheduled.taskId,
102
+ enrichedEventId: scheduled.enrichedEventId,
103
+ traceId: event.metadata.traceId
104
+ });
105
+ const shard = parseInt(scheduled.taskId.slice(-1), 16) || 0;
106
+ const zsetKey = `notif:scheduled:zset:${shard}`;
107
+ await this.redisCli.zadd(zsetKey, scheduledAt, taskData);
108
+ this.logger.debug({
109
+ taskId: scheduled.taskId,
110
+ scheduledAt: scheduled.scheduledAt,
111
+ shard
112
+ }, "scheduled task queued in ZSET");
113
+ }
114
+ };
115
+ async function executeSchedulerPoll(redis, outboundProducers, logger, db) {
116
+ const lockKey = "notif:lock:scheduler:poll";
117
+ const lockOwner = randomUUID();
118
+ let acquired = false;
119
+ try {
120
+ if (await redis.set(lockKey, lockOwner, "EX", 30, "NX") !== "OK") return false;
121
+ acquired = true;
122
+ const now = Date.now();
123
+ const visibilityTimeout = 6e4;
124
+ const perShardLimit = 100;
125
+ const pipeline = redis.pipeline();
126
+ for (let i = 0; i < 16; i++) {
127
+ const shardKey = `notif:scheduled:zset:${i}`;
128
+ pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
129
+ }
130
+ const perShard = (await pipeline.exec())?.map((res) => res[0] ? [] : res[1] ?? []) ?? [];
131
+ const tasks = perShard.flat();
132
+ const anyShardFull = perShard.some((shard) => shard.length >= perShardLimit);
133
+ if (tasks.length === 0) return false;
134
+ const parsedTasks = tasks.map((t) => {
135
+ try {
136
+ return {
137
+ taskStr: t,
138
+ ...JSON.parse(t)
139
+ };
140
+ } catch (err) {
141
+ logger.error({
142
+ taskStr: t,
143
+ err
144
+ }, "failed to parse scheduled task JSON");
145
+ return null;
146
+ }
147
+ }).filter(Boolean);
148
+ if (parsedTasks.length === 0) return false;
149
+ const taskIds = parsedTasks.map((t) => t.taskId);
150
+ const dbPayloads = await db.select().from(scheduledPayloads).where(inArray(scheduledPayloads.taskId, taskIds));
151
+ const payloadMap = new Map(dbPayloads.map((row) => [row.taskId, row.payload]));
152
+ const blocked = await findBlockedAtRelease(db, [...payloadMap.values()]);
153
+ const batchedEvents = {
154
+ critical: [],
155
+ normal: [],
156
+ low: []
157
+ };
158
+ const cleanupPipeline = redis.pipeline();
159
+ const dbCleanupIds = [];
160
+ const released = /* @__PURE__ */ new Set();
161
+ for (let i = 0; i < parsedTasks.length; i++) {
162
+ const { taskStr, taskId, traceId } = parsedTasks[i];
163
+ if (released.has(taskId)) {
164
+ cleanupPipeline.zrem(`notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`, taskStr);
165
+ continue;
166
+ }
167
+ const payload = payloadMap.get(taskId);
168
+ if (!payload) {
169
+ logger.warn({ taskId }, "scheduled payload not found in Postgres — skipping release");
170
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
171
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
172
+ continue;
173
+ }
174
+ const dispatchPayload = payload;
175
+ const blockReason = blocked.get(taskId);
176
+ if (blockReason) {
177
+ logger.info({
178
+ taskId,
179
+ reason: blockReason
180
+ }, "parked send no longer allowed — dropping");
181
+ globalEmitter.emit("notification:skipped", {
182
+ projectId: dispatchPayload.projectId,
183
+ eventId: dispatchPayload.enrichedEventId,
184
+ recipientId: dispatchPayload.recipientId,
185
+ reason: blockReason
186
+ });
187
+ released.add(taskId);
188
+ dbCleanupIds.push(taskId);
189
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
190
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
191
+ continue;
192
+ }
193
+ batchedEvents[getPriorityBucket(dispatchPayload.priority)].push(buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId));
194
+ released.add(taskId);
195
+ dbCleanupIds.push(taskId);
196
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
197
+ cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
198
+ }
199
+ for (const p of Object.keys(batchedEvents)) if (batchedEvents[p].length > 0) await (outboundProducers[p] ?? outboundProducers["normal"]).publishBatch(batchedEvents[p]);
200
+ if (dbCleanupIds.length > 0) try {
201
+ const sendAt = sql`${scheduledPayloads.payload}->>'scheduledAt'`;
202
+ await db.delete(scheduledPayloads).where(or(...dbCleanupIds.map((taskId) => {
203
+ const released = payloadMap.get(taskId)?.scheduledAt;
204
+ return and(eq(scheduledPayloads.taskId, taskId), typeof released === "string" ? sql`${sendAt} = ${released}` : sql`${sendAt} IS NULL`);
205
+ })));
206
+ } catch (err) {
207
+ logger.error({ err }, "failed to delete scheduled payloads from Postgres, continuing with Redis cleanup");
208
+ }
209
+ await cleanupPipeline.exec();
210
+ logger.info({ count: parsedTasks.length }, "scheduled tasks released to outbound");
211
+ return anyShardFull;
212
+ } catch (err) {
213
+ logger.error({ err }, "error in scheduler polling loop");
214
+ return false;
215
+ } finally {
216
+ if (acquired) await redis.releaseLock(lockKey, lockOwner).catch(() => {});
217
+ }
218
+ }
219
+ /**
220
+ * Present while the scheduler ZSETs are known to be intact. Redis losing its
221
+ * data (a flush, a restart without persistence, failover to an empty replica)
222
+ * takes this key with it, which is how the poll loop notices.
223
+ */
224
+ const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
225
+ /**
226
+ * Re-queues every scheduled send still waiting in Postgres. Each row keeps its
227
+ * payload, and with it the send time, so the ZSETs can be rebuilt exactly;
228
+ * a legacy row without a send time is released on the next poll.
229
+ *
230
+ * Only run when the sentinel is missing: a rebuilt member has no trace id and
231
+ * so never matches a surviving original, and the poll only de-duplicates
232
+ * copies that happen to be claimed together.
233
+ */
234
+ async function rebuildScheduledQueue(redis, db, logger, pageSize = 1e3) {
235
+ let after;
236
+ let total = 0;
237
+ for (;;) {
238
+ const rows = await db.select({
239
+ taskId: scheduledPayloads.taskId,
240
+ payload: scheduledPayloads.payload
241
+ }).from(scheduledPayloads).where(after === void 0 ? void 0 : gt(scheduledPayloads.taskId, after)).orderBy(asc(scheduledPayloads.taskId)).limit(pageSize);
242
+ if (rows.length === 0) break;
243
+ const pipeline = redis.pipeline();
244
+ for (const { taskId, payload } of rows) {
245
+ const sendAt = Date.parse(payload?.scheduledAt ?? "");
246
+ const member = JSON.stringify({
247
+ taskId,
248
+ enrichedEventId: payload?.enrichedEventId
249
+ });
250
+ const shard = parseInt(taskId.slice(-1), 16) || 0;
251
+ pipeline.zadd(`notif:scheduled:zset:${shard}`, "NX", Number.isNaN(sendAt) ? Date.now() : sendAt, member);
252
+ }
253
+ await pipeline.exec();
254
+ total += rows.length;
255
+ after = rows[rows.length - 1].taskId;
256
+ if (rows.length < pageSize) break;
257
+ }
258
+ await redis.set(SCHEDULER_SENTINEL_KEY, "1");
259
+ if (total > 0) logger.warn({ count: total }, "rebuilt scheduled queue from Postgres");
260
+ return total;
261
+ }
262
+ async function startSchedulerWorker() {
263
+ ({logger, redis, sql: sql$1, db} = createWorkerRuntime(config, "scheduler"));
264
+ ({consumer, pendingScanner} = createStreamConsumer({
265
+ redis,
266
+ logger
267
+ }, {
268
+ stream: STREAMS.SCHEDULED,
269
+ group: CONSUMER_GROUPS.SCHEDULER,
270
+ consumer: uniqueConsumerId("scheduler"),
271
+ batchSize: config.WORKER_CONCURRENCY
272
+ }));
273
+ const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
274
+ worker = new SchedulerWorker({
275
+ consumer,
276
+ pendingScanner,
277
+ logger,
278
+ concurrency: config.WORKER_CONCURRENCY,
279
+ registry,
280
+ redis: redis.native
281
+ });
282
+ isPolling = true;
283
+ const pollLoop = async () => {
284
+ if (!isPolling) return;
285
+ try {
286
+ if (!await redis.native.exists("notif:scheduler:intact")) await rebuildScheduledQueue(redis.native, db, logger);
287
+ const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
288
+ if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5e3);
289
+ } catch (err) {
290
+ logger.error({ err }, "scheduler poll loop error");
291
+ if (isPolling) pollTimeout = setTimeout(() => void pollLoop(), 5e3);
292
+ }
293
+ };
294
+ pollLoop();
295
+ healthInterval = startHealthReporter("scheduler", worker, redis, logger);
296
+ logger.info({ env: config.NODE_ENV }, "scheduler starting");
297
+ await worker.start();
298
+ }
299
+ async function stopSchedulerWorker() {
300
+ isPolling = false;
301
+ await shutdownWorker("scheduler", {
302
+ logger,
303
+ timers: [healthInterval, pollTimeout],
304
+ worker,
305
+ sql: sql$1,
306
+ redis
307
+ });
308
+ healthInterval = null;
309
+ pollTimeout = null;
310
+ }
311
+ //#endregion
312
+ export { startSchedulerWorker, stopSchedulerWorker };