notifkit 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/{bootstrap-P63acBG9.mjs → bootstrap-CHOj1rDv.mjs} +1 -1
- package/dist/{bulk-Beblt3ny.mjs → bulk-CAsYwF-9.mjs} +1 -1
- package/dist/index.d.mts +29 -111
- package/dist/index.mjs +2 -2
- package/dist/{main-Dk92nti8.mjs → main-B88bnGHO.mjs} +5 -12
- package/dist/{main-BIahf4f9.mjs → main-BDTDp4Bg.mjs} +35 -16
- package/dist/{main-C3XcLoCb.mjs → main-C9ekO6RS.mjs} +3 -3
- package/dist/{main-_I4b1cmP.mjs → main-CBeKd8wJ.mjs} +3 -3
- package/dist/{main-NHUvynW2.mjs → main-CVTZoGzA.mjs} +4 -9
- package/dist/{main-CktEylpL.mjs → main-CYxNbgX5.mjs} +23 -94
- package/dist/{main-DB0bUh2r.mjs → main-CcGiccDe.mjs} +5 -14
- package/dist/{main-Lcbg7aEn.mjs → main-DGF3WtFw.mjs} +8 -58
- package/dist/{schedule-or-dispatch-C5qfxQCv.mjs → schedule-or-dispatch-BF85qhMh.mjs} +1 -1
- package/dist/{src-BRatuWJS.mjs → src-aiRAj2xr.mjs} +72 -175
- package/drizzle/0009_retention_time_indexes.sql +4 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +1 -1
- package/src/client.ts +6 -6
- package/src/config/index.ts +8 -10
- package/src/contracts/sdk.ts +26 -11
- package/src/db/bulk.ts +3 -10
- package/src/db/schema.ts +3 -0
- package/src/idempotency/index.ts +12 -53
- package/src/queue/index.ts +1 -12
- package/src/rate-limiter/api-rate-limiter.ts +8 -27
- package/src/rate-limiter/project-settings-cache.ts +4 -16
- package/src/redis/index.ts +3 -51
- package/src/repositories/contact.ts +0 -2
- package/src/server.ts +1 -4
- package/src/services/api/handlers/auth.ts +3 -11
- package/src/services/api/handlers/campaigns.ts +4 -15
- package/src/services/api/handlers/events.ts +1 -2
- package/src/services/api/handlers/notifications.ts +6 -16
- package/src/services/api/handlers/projects.ts +1 -3
- package/src/services/api/handlers/shared.ts +7 -29
- package/src/services/api/handlers/system.ts +1 -1
- package/src/services/api/handlers/unsubscribe.ts +7 -30
- package/src/services/api/handlers/users.ts +1 -2
- package/src/services/api/http.ts +5 -10
- package/src/services/api/main.ts +27 -63
- package/src/services/api/webhooks.ts +8 -19
- package/src/services/auth/index.ts +2 -21
- package/src/services/delivery/main.ts +25 -89
- package/src/services/engine/main.ts +33 -69
- package/src/services/engine/schedule-or-dispatch.ts +7 -4
- package/src/services/enricher/main.ts +19 -49
- package/src/services/events/main.ts +3 -12
- package/src/services/scheduler/main.ts +26 -32
- package/src/services/scheduler/retention.ts +47 -0
- package/src/services/scheduler/send-gate.ts +6 -17
- package/src/services/workflow/duration.ts +1 -6
- package/src/services/workflow/main.ts +3 -9
- package/src/services/workflow/timers.ts +2 -1
- package/src/shared/batch-processor.ts +3 -7
- package/src/shared/utils.ts +2 -8
- package/src/transport/index.ts +0 -4
- package/src/unsubscribe/index.ts +3 -11
- package/src/workers/index.ts +8 -27
|
@@ -27,16 +27,15 @@ import {
|
|
|
27
27
|
uniqueConsumerId,
|
|
28
28
|
} from "@/workers/bootstrap.js";
|
|
29
29
|
|
|
30
|
-
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
31
|
-
|
|
32
30
|
loadEnv();
|
|
33
31
|
const config = readBaseConfig();
|
|
34
32
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
*
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
const chunkArray = <T>(arr: T[], size: number): T[][] =>
|
|
34
|
+
Array.from({ length: Math.ceil(arr.length / size) }, (_, i) =>
|
|
35
|
+
arr.slice(i * size, i * size + size),
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
// A caller's own idempotency key is promised 24 h; an event id only needs to outlive redelivery.
|
|
40
39
|
const CALLER_KEY_TTL_SECONDS = 86_400;
|
|
41
40
|
const dedupeTtl = (raw: { idempotencyKey?: string }) =>
|
|
42
41
|
raw.idempotencyKey ? CALLER_KEY_TTL_SECONDS : undefined;
|
|
@@ -141,19 +140,15 @@ export class EnricherWorker extends BaseWorker {
|
|
|
141
140
|
}
|
|
142
141
|
|
|
143
142
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
* inline chain holds this worker's slot until the last send completes, and a
|
|
147
|
-
* segment would hold one slot for thousands of them.
|
|
143
|
+
* Single-user notifications skip the enriched stream. Fan-outs keep it, since an
|
|
144
|
+
* inline chain holds this worker's slot until its last send completes.
|
|
148
145
|
*/
|
|
149
146
|
setInlineNext(next: ((event: StreamEvent) => Promise<void>) | null): void {
|
|
150
147
|
this.inlineNext = next;
|
|
151
148
|
}
|
|
152
149
|
|
|
153
150
|
override async stop(): Promise<void> {
|
|
154
|
-
//
|
|
155
|
-
// waiting on one of these flushes, and clearing them first left the drain
|
|
156
|
-
// to the 30s stop timeout.
|
|
151
|
+
// Keep the flush timers until in-flight messages settle; they wait on them.
|
|
157
152
|
await super.stop();
|
|
158
153
|
if (this.batchTimer) {
|
|
159
154
|
clearTimeout(this.batchTimer);
|
|
@@ -268,9 +263,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
268
263
|
const profile = await this.userRepo.findRecordById(raw.projectId, raw.recipientId);
|
|
269
264
|
if (!profile) return;
|
|
270
265
|
|
|
271
|
-
// Opt-outs are keyed on the template's topics,
|
|
272
|
-
// path. `event.type` is always "notification.created", which no user
|
|
273
|
-
// ever opts out of, so checking it here let every opt-out through.
|
|
266
|
+
// Opt-outs are keyed on the template's topics, not the event type.
|
|
274
267
|
const template = raw.templateId
|
|
275
268
|
? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId)
|
|
276
269
|
: null;
|
|
@@ -297,8 +290,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
297
290
|
quietHours: profile.preferences.quietHours,
|
|
298
291
|
},
|
|
299
292
|
},
|
|
300
|
-
//
|
|
301
|
-
// which predates campaigns and carries no label to attribute to.
|
|
293
|
+
// The legacy payload has no campaign.
|
|
302
294
|
scheduledAt: raw.scheduledAt,
|
|
303
295
|
};
|
|
304
296
|
|
|
@@ -336,7 +328,6 @@ export class EnricherWorker extends BaseWorker {
|
|
|
336
328
|
return;
|
|
337
329
|
}
|
|
338
330
|
|
|
339
|
-
// Handle new notification.requested
|
|
340
331
|
const raw = (requestedResult as any).data;
|
|
341
332
|
const dedupeId = `${raw.projectId}:${raw.idempotencyKey ?? event.id}`;
|
|
342
333
|
if (!(await this.idempotency.checkAndMark(dedupeId, 60))) return;
|
|
@@ -355,7 +346,6 @@ export class EnricherWorker extends BaseWorker {
|
|
|
355
346
|
this.logger.info({ topic: raw.target.topic, count: userIds.length }, "Resolved topic");
|
|
356
347
|
} else {
|
|
357
348
|
this.logger.warn({ target: raw.target }, "Segment/topic resolution not fully implemented");
|
|
358
|
-
// Stub: maybe resolve later
|
|
359
349
|
}
|
|
360
350
|
|
|
361
351
|
const maxUsers = readBaseConfig().SEGMENT_MAX_USERS;
|
|
@@ -365,17 +355,13 @@ export class EnricherWorker extends BaseWorker {
|
|
|
365
355
|
{ count: userIds.length, max: maxUsers, projectId: raw.projectId, eventId: event.id },
|
|
366
356
|
"Segment fan-out exceeds maximum allowed limit",
|
|
367
357
|
);
|
|
368
|
-
// Reported
|
|
369
|
-
// the enriched stream is not a payload the engine can read, so the
|
|
370
|
-
// refusal used to vanish there without a trace.
|
|
358
|
+
// Reported via the emitter: the engine can't read a failure on the enriched stream.
|
|
371
359
|
globalEmitter.emit("notification:failed", event.id, reason, event.type);
|
|
372
360
|
await this.idempotency.markProcessed(dedupeId, dedupeTtl(raw));
|
|
373
361
|
return;
|
|
374
362
|
}
|
|
375
363
|
|
|
376
|
-
//
|
|
377
|
-
// `event.type` here is "notification.requested", which no user ever sets a
|
|
378
|
-
// preference against, so keying on it silently disabled every opt-out.
|
|
364
|
+
// Opt-outs are keyed on the template's topics, not the event type.
|
|
379
365
|
const template = raw.templateId
|
|
380
366
|
? await this.templateCache.getCachedTemplate(raw.projectId, raw.templateId)
|
|
381
367
|
: null;
|
|
@@ -385,11 +371,6 @@ export class EnricherWorker extends BaseWorker {
|
|
|
385
371
|
raw.channels && raw.channels.length > 0 ? raw.channels : ["email"];
|
|
386
372
|
const isFallback = (raw as any).fallback === true;
|
|
387
373
|
|
|
388
|
-
const chunkArray = <T>(arr: T[], size: number) =>
|
|
389
|
-
Array.from({ length: Math.ceil(arr.length / size) }, (v, i) =>
|
|
390
|
-
arr.slice(i * size, i * size + size),
|
|
391
|
-
);
|
|
392
|
-
|
|
393
374
|
const chunks = chunkArray(userIds, 500);
|
|
394
375
|
const inline = raw.target.type === "user" ? this.inlineNext : null;
|
|
395
376
|
|
|
@@ -413,8 +394,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
413
394
|
|
|
414
395
|
for (const profile of profiles) {
|
|
415
396
|
const contacts = contactsByUser.get(profile.userId) ?? [];
|
|
416
|
-
//
|
|
417
|
-
// of this template's topics.
|
|
397
|
+
// The user's addresses on a channel, minus those opted out of this template's topics.
|
|
418
398
|
const usable = (channel: string) =>
|
|
419
399
|
contacts.filter(
|
|
420
400
|
(contact: any) =>
|
|
@@ -422,9 +402,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
422
402
|
!topics.some((t) => contact.preferences?.topics?.[t] === false),
|
|
423
403
|
);
|
|
424
404
|
|
|
425
|
-
//
|
|
426
|
-
// start at the first channel this user can be reached on and chain the
|
|
427
|
-
// rest behind it — a user with no email address still gets the SMS.
|
|
405
|
+
// With fallback, start at the first reachable channel and chain the rest behind it.
|
|
428
406
|
let userChannels: NotificationChannel[] = channels;
|
|
429
407
|
let fallbackChain: NotificationChannel[] | undefined;
|
|
430
408
|
if (isFallback) {
|
|
@@ -442,8 +420,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
442
420
|
|
|
443
421
|
for (const channel of userChannels) {
|
|
444
422
|
const channelContacts = usable(channel);
|
|
445
|
-
// Push resolves its
|
|
446
|
-
// remains current. Other channels need one task per address.
|
|
423
|
+
// Push resolves its tokens at send time; other channels get one task per address.
|
|
447
424
|
const resolved: ({ id: string; target: string } | undefined)[] =
|
|
448
425
|
channel === "push" ? [undefined] : channelContacts;
|
|
449
426
|
if (resolved.length === 0) {
|
|
@@ -488,8 +465,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
488
465
|
},
|
|
489
466
|
scheduledAt: raw.scheduledAt,
|
|
490
467
|
fallbackChain: fallbackChain?.length ? fallbackChain : undefined,
|
|
491
|
-
//
|
|
492
|
-
// user's contacts a second time.
|
|
468
|
+
// Saves the engine a second contacts read.
|
|
493
469
|
contactId: contact?.id,
|
|
494
470
|
destination,
|
|
495
471
|
campaignId: raw.campaignId,
|
|
@@ -550,8 +526,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
550
526
|
"event enriched",
|
|
551
527
|
);
|
|
552
528
|
} catch (err) {
|
|
553
|
-
// Release the marker, or
|
|
554
|
-
// duplicate and acked without ever being processed.
|
|
529
|
+
// Release the marker, or the retry is acked as a duplicate.
|
|
555
530
|
await this.idempotency.unmark(dedupeId).catch(() => {});
|
|
556
531
|
throw err;
|
|
557
532
|
}
|
|
@@ -567,8 +542,7 @@ export class EnricherWorker extends BaseWorker {
|
|
|
567
542
|
|
|
568
543
|
export async function startEnricherWorker() {
|
|
569
544
|
({ logger, redis, sql, db } = createWorkerRuntime(config, "enricher"));
|
|
570
|
-
//
|
|
571
|
-
// it needs delivery's concurrency, not the CPU-bound default.
|
|
545
|
+
// Fused, the enricher's slot is held through delivery, so it needs delivery's concurrency.
|
|
572
546
|
const enricherConcurrency = config.PIPELINE_FUSED
|
|
573
547
|
? Math.max(config.WORKER_CONCURRENCY, config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY)
|
|
574
548
|
: config.WORKER_CONCURRENCY;
|
|
@@ -610,8 +584,6 @@ export async function startEnricherWorker() {
|
|
|
610
584
|
templateCache,
|
|
611
585
|
});
|
|
612
586
|
|
|
613
|
-
// ─── Health check interval ──────────────────────────────────────────────────
|
|
614
|
-
|
|
615
587
|
healthInterval = startHealthReporter("enricher", worker, redis, logger);
|
|
616
588
|
|
|
617
589
|
logger.info({ env: config.NODE_ENV }, "enricher starting");
|
|
@@ -622,8 +594,6 @@ export function getEnricherWorker(): EnricherWorker | undefined {
|
|
|
622
594
|
return worker as EnricherWorker | undefined;
|
|
623
595
|
}
|
|
624
596
|
|
|
625
|
-
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
626
|
-
|
|
627
597
|
export async function stopEnricherWorker(): Promise<void> {
|
|
628
598
|
await shutdownWorker("enricher", { logger, timers: [healthInterval], worker, sql, redis });
|
|
629
599
|
healthInterval = null;
|
|
@@ -132,8 +132,7 @@ export class EventWorker extends BaseWorker {
|
|
|
132
132
|
for (const e of events) e.reject(error);
|
|
133
133
|
} finally {
|
|
134
134
|
this.flushesInFlight--;
|
|
135
|
-
//
|
|
136
|
-
// out the interval with their worker slots held.
|
|
135
|
+
// Flush rows that queued while every slot was busy, instead of waiting out the interval.
|
|
137
136
|
if (this.messageLogBuffer.length >= this.flushThreshold) {
|
|
138
137
|
void this.flushLogs();
|
|
139
138
|
}
|
|
@@ -141,9 +140,7 @@ export class EventWorker extends BaseWorker {
|
|
|
141
140
|
}
|
|
142
141
|
|
|
143
142
|
override async stop(): Promise<void> {
|
|
144
|
-
//
|
|
145
|
-
// is waiting on a flush, and clearing it first left them to the 30s stop
|
|
146
|
-
// timeout.
|
|
143
|
+
// Keep the flush interval until in-flight messages settle; they wait on it.
|
|
147
144
|
await super.stop();
|
|
148
145
|
if (this.flushInterval) clearInterval(this.flushInterval);
|
|
149
146
|
const deadline = Date.now() + 10_000;
|
|
@@ -240,7 +237,6 @@ export class EventWorker extends BaseWorker {
|
|
|
240
237
|
return;
|
|
241
238
|
}
|
|
242
239
|
|
|
243
|
-
// Find matching waiters
|
|
244
240
|
const now = new Date();
|
|
245
241
|
const waiters = await this.dbConn
|
|
246
242
|
.select()
|
|
@@ -256,7 +252,6 @@ export class EventWorker extends BaseWorker {
|
|
|
256
252
|
const data: Record<string, any> = eventData && typeof eventData === "object" ? eventData : {};
|
|
257
253
|
|
|
258
254
|
for (const waiter of waiters) {
|
|
259
|
-
// Check match criteria
|
|
260
255
|
let matched = true;
|
|
261
256
|
const criteria = waiter.matchCriteria as Record<string, any>;
|
|
262
257
|
if (criteria && typeof criteria === "object") {
|
|
@@ -303,7 +298,6 @@ export class EventWorker extends BaseWorker {
|
|
|
303
298
|
continue;
|
|
304
299
|
}
|
|
305
300
|
|
|
306
|
-
// Find the step that is waiting
|
|
307
301
|
const steps = await this.dbConn
|
|
308
302
|
.select()
|
|
309
303
|
.from(workflowSteps)
|
|
@@ -318,7 +312,6 @@ export class EventWorker extends BaseWorker {
|
|
|
318
312
|
const pendingStep = steps.find((s: any) => s.output === null);
|
|
319
313
|
|
|
320
314
|
if (pendingStep) {
|
|
321
|
-
// Update the step with the event payload
|
|
322
315
|
await this.dbConn
|
|
323
316
|
.update(workflowSteps)
|
|
324
317
|
.set({ output: data })
|
|
@@ -351,11 +344,9 @@ export class EventWorker extends BaseWorker {
|
|
|
351
344
|
}),
|
|
352
345
|
);
|
|
353
346
|
|
|
354
|
-
// Delete the waiter
|
|
355
347
|
await this.dbConn.delete(workflowWaiters).where(eq(workflowWaiters.id, waiter.id));
|
|
356
348
|
|
|
357
|
-
// The timer
|
|
358
|
-
// instance status and drops the entry if it is no longer pending.
|
|
349
|
+
// The timer entry is left; the poll loop drops it once the instance isn't pending.
|
|
359
350
|
}
|
|
360
351
|
}
|
|
361
352
|
|
|
@@ -2,7 +2,13 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
3
3
|
import type { Logger } from "@/index.js";
|
|
4
4
|
import type { RedisClient, Redis } from "@/index.js";
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
type StreamConsumer,
|
|
7
|
+
type PendingMessageScanner,
|
|
8
|
+
type StreamMessage,
|
|
9
|
+
type StreamProducer,
|
|
10
|
+
} from "@/index.js";
|
|
11
|
+
import type { Db, Sql } from "@/db/index.js";
|
|
6
12
|
import { BaseWorker } from "@/index.js";
|
|
7
13
|
import {
|
|
8
14
|
STREAMS,
|
|
@@ -23,15 +29,15 @@ import {
|
|
|
23
29
|
uniqueConsumerId,
|
|
24
30
|
} from "@/workers/bootstrap.js";
|
|
25
31
|
import { findBlockedAtRelease } from "./send-gate.js";
|
|
26
|
-
|
|
32
|
+
import { maybePruneDeliveryLogs } from "./retention.js";
|
|
27
33
|
|
|
28
34
|
loadEnv();
|
|
29
35
|
const config = readBaseConfig();
|
|
30
36
|
|
|
31
37
|
let logger: Logger;
|
|
32
38
|
let redis: RedisClient;
|
|
33
|
-
let sql:
|
|
34
|
-
let db:
|
|
39
|
+
let sql: Sql;
|
|
40
|
+
let db: Db;
|
|
35
41
|
|
|
36
42
|
let consumer: StreamConsumer;
|
|
37
43
|
let pendingScanner: PendingMessageScanner;
|
|
@@ -90,9 +96,11 @@ export class SchedulerWorker extends BaseWorker {
|
|
|
90
96
|
|
|
91
97
|
export async function executeSchedulerPoll(
|
|
92
98
|
redis: Redis,
|
|
93
|
-
outboundProducers:
|
|
94
|
-
|
|
95
|
-
|
|
99
|
+
outboundProducers: Partial<
|
|
100
|
+
Record<"critical" | "normal" | "low", Pick<StreamProducer, "publishBatch">>
|
|
101
|
+
>,
|
|
102
|
+
logger: Pick<Logger, "info" | "warn" | "error">,
|
|
103
|
+
db: Db,
|
|
96
104
|
): Promise<boolean> {
|
|
97
105
|
const lockKey = "notif:lock:scheduler:poll";
|
|
98
106
|
const lockOwner = randomUUID();
|
|
@@ -117,8 +125,7 @@ export async function executeSchedulerPoll(
|
|
|
117
125
|
const results = await pipeline.exec();
|
|
118
126
|
const perShard = results?.map((res: any) => (res[0] ? [] : ((res[1] ?? []) as string[]))) ?? [];
|
|
119
127
|
const tasks = perShard.flat();
|
|
120
|
-
// A shard
|
|
121
|
-
// what tells the caller to poll again rather than wait for the next tick.
|
|
128
|
+
// A full shard may have more due tasks behind it.
|
|
122
129
|
const anyShardFull = perShard.some((shard) => shard.length >= perShardLimit);
|
|
123
130
|
|
|
124
131
|
if (tasks.length === 0) return false;
|
|
@@ -142,9 +149,7 @@ export async function executeSchedulerPoll(
|
|
|
142
149
|
.where(inArray(scheduledPayloads.taskId, taskIds));
|
|
143
150
|
const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload]));
|
|
144
151
|
|
|
145
|
-
//
|
|
146
|
-
// lookup throws to the catch below: the claimed tasks reappear after the
|
|
147
|
-
// visibility timeout instead of going out unchecked.
|
|
152
|
+
// Throws on a failed lookup, so claimed tasks come back after the visibility timeout.
|
|
148
153
|
const blocked = await findBlockedAtRelease(db, [...payloadMap.values()] as any[]);
|
|
149
154
|
|
|
150
155
|
const batchedEvents: Record<"critical" | "normal" | "low", Omit<any, "id" | "timestamp">[]> = {
|
|
@@ -156,9 +161,7 @@ export async function executeSchedulerPoll(
|
|
|
156
161
|
const cleanupPipeline = redis.pipeline();
|
|
157
162
|
const dbCleanupIds: string[] = [];
|
|
158
163
|
|
|
159
|
-
// After a rebuild
|
|
160
|
-
// member carries a trace id Postgres does not keep), so release each task
|
|
161
|
-
// once and just clear any second copy.
|
|
164
|
+
// After a rebuild a task can be in the ZSET twice (one copy has a trace id); release it once.
|
|
162
165
|
const released = new Set<string>();
|
|
163
166
|
|
|
164
167
|
for (let i = 0; i < parsedTasks.length; i++) {
|
|
@@ -203,9 +206,6 @@ export async function executeSchedulerPoll(
|
|
|
203
206
|
cleanupPipeline.zrem(zsetKey(shardOf(taskId)), taskStr);
|
|
204
207
|
}
|
|
205
208
|
|
|
206
|
-
// Every bucket, not a hardcoded subset: a task binned into one that is not
|
|
207
|
-
// published here has already been added to the cleanup list, so it would be
|
|
208
|
-
// deleted from Redis and Postgres without ever being sent.
|
|
209
209
|
for (const p of Object.keys(batchedEvents) as (keyof typeof batchedEvents)[]) {
|
|
210
210
|
if (batchedEvents[p].length > 0) {
|
|
211
211
|
const producer = outboundProducers[p] ?? outboundProducers["normal"]!;
|
|
@@ -215,10 +215,8 @@ export async function executeSchedulerPoll(
|
|
|
215
215
|
|
|
216
216
|
if (dbCleanupIds.length > 0) {
|
|
217
217
|
try {
|
|
218
|
-
//
|
|
219
|
-
//
|
|
220
|
-
// later send time; deleting by id alone removed that newer payload,
|
|
221
|
-
// and its ZSET entry was then dropped as canceled — the send was lost.
|
|
218
|
+
// Match the released send time: delivery may already have re-parked
|
|
219
|
+
// this task id with a later one.
|
|
222
220
|
const sendAt = drizzleSql`${scheduledPayloads.payload}->>'scheduledAt'`;
|
|
223
221
|
await db.delete(scheduledPayloads).where(
|
|
224
222
|
or(
|
|
@@ -244,9 +242,7 @@ export async function executeSchedulerPoll(
|
|
|
244
242
|
|
|
245
243
|
logger.info({ count: parsedTasks.length }, "scheduled tasks released to outbound");
|
|
246
244
|
|
|
247
|
-
//
|
|
248
|
-
// test was `tasks.length === 100` against a total drawn from 16 shards, so
|
|
249
|
-
// a real backlog reported "nothing more" and waited for the next tick.
|
|
245
|
+
// True tells the caller to poll again right away.
|
|
250
246
|
return anyShardFull;
|
|
251
247
|
} catch (err) {
|
|
252
248
|
logger.error({ err }, "error in scheduler polling loop");
|
|
@@ -276,8 +272,8 @@ export const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
|
|
|
276
272
|
*/
|
|
277
273
|
export async function rebuildScheduledQueue(
|
|
278
274
|
redis: Redis,
|
|
279
|
-
db:
|
|
280
|
-
logger: Logger,
|
|
275
|
+
db: Db,
|
|
276
|
+
logger: Pick<Logger, "info" | "warn" | "error">,
|
|
281
277
|
pageSize = 1000,
|
|
282
278
|
): Promise<number> {
|
|
283
279
|
let after: string | undefined;
|
|
@@ -337,7 +333,6 @@ export async function startSchedulerWorker() {
|
|
|
337
333
|
redis: redis.native,
|
|
338
334
|
});
|
|
339
335
|
|
|
340
|
-
// ─── Polling Loop ─────────────────────────────────────────────────────────
|
|
341
336
|
isPolling = true;
|
|
342
337
|
const pollLoop = async (): Promise<void> => {
|
|
343
338
|
if (!isPolling) return;
|
|
@@ -345,6 +340,9 @@ export async function startSchedulerWorker() {
|
|
|
345
340
|
if (!(await redis.native.exists(SCHEDULER_SENTINEL_KEY))) {
|
|
346
341
|
await rebuildScheduledQueue(redis.native, db, logger);
|
|
347
342
|
}
|
|
343
|
+
if (config.LOG_RETENTION_DAYS) {
|
|
344
|
+
void maybePruneDeliveryLogs(redis.native, db, logger, config.LOG_RETENTION_DAYS);
|
|
345
|
+
}
|
|
348
346
|
const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
|
|
349
347
|
if (isPolling) {
|
|
350
348
|
pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5000);
|
|
@@ -358,16 +356,12 @@ export async function startSchedulerWorker() {
|
|
|
358
356
|
};
|
|
359
357
|
void pollLoop();
|
|
360
358
|
|
|
361
|
-
// ─── Health check interval ──────────────────────────────────────────────────
|
|
362
|
-
|
|
363
359
|
healthInterval = startHealthReporter("scheduler", worker, redis, logger);
|
|
364
360
|
|
|
365
361
|
logger.info({ env: config.NODE_ENV }, "scheduler starting");
|
|
366
362
|
await worker.start();
|
|
367
363
|
}
|
|
368
364
|
|
|
369
|
-
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
370
|
-
|
|
371
365
|
export async function stopSchedulerWorker(): Promise<void> {
|
|
372
366
|
isPolling = false;
|
|
373
367
|
await shutdownWorker("scheduler", {
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { sql } from "drizzle-orm";
|
|
2
|
+
import type { Redis } from "ioredis";
|
|
3
|
+
import type { Logger } from "@/logger/index.js";
|
|
4
|
+
import type { Db } from "@/db/index.js";
|
|
5
|
+
import { deliveryOutbox, messageLogs } from "@/db/schema.js";
|
|
6
|
+
|
|
7
|
+
const BATCH = 5000;
|
|
8
|
+
const PRUNE_LOCK_KEY = "notif:lock:scheduler:prune";
|
|
9
|
+
|
|
10
|
+
/** Deletes delivery log and outbox rows older than `retentionDays`, in batches. */
|
|
11
|
+
export async function pruneDeliveryLogs(db: Db, retentionDays: number): Promise<number> {
|
|
12
|
+
const cutoff = new Date(Date.now() - retentionDays * 86_400_000).toISOString();
|
|
13
|
+
const tables = [
|
|
14
|
+
[messageLogs, messageLogs.timestamp],
|
|
15
|
+
[deliveryOutbox, deliveryOutbox.createdAt],
|
|
16
|
+
] as const;
|
|
17
|
+
|
|
18
|
+
let total = 0;
|
|
19
|
+
for (const [table, column] of tables) {
|
|
20
|
+
for (;;) {
|
|
21
|
+
// Small batches keep each delete's locks and WAL short.
|
|
22
|
+
const rows = await db.execute(
|
|
23
|
+
sql`DELETE FROM ${table} WHERE ctid IN (SELECT ctid FROM ${table} WHERE ${column} < ${cutoff}::timestamptz LIMIT ${BATCH}) RETURNING 1`,
|
|
24
|
+
);
|
|
25
|
+
total += rows.length;
|
|
26
|
+
if (rows.length < BATCH) break;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return total;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Runs the prune once per hour across all scheduler processes. */
|
|
33
|
+
export async function maybePruneDeliveryLogs(
|
|
34
|
+
redis: Pick<Redis, "set">,
|
|
35
|
+
db: Db,
|
|
36
|
+
logger: Pick<Logger, "info" | "error">,
|
|
37
|
+
retentionDays: number,
|
|
38
|
+
): Promise<void> {
|
|
39
|
+
try {
|
|
40
|
+
// Never released: the key expiring is what allows the next run.
|
|
41
|
+
if ((await redis.set(PRUNE_LOCK_KEY, "1", "EX", 3600, "NX")) !== "OK") return;
|
|
42
|
+
const deleted = await pruneDeliveryLogs(db, retentionDays);
|
|
43
|
+
if (deleted > 0) logger.info({ deleted, retentionDays }, "pruned old delivery logs");
|
|
44
|
+
} catch (err) {
|
|
45
|
+
logger.error({ err }, "delivery log prune failed");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { and, eq, inArray } from "drizzle-orm";
|
|
2
2
|
import { suppressions } from "@/db/schema.js";
|
|
3
|
+
import type { Db } from "@/db/index.js";
|
|
3
4
|
import { ContactRepository, TemplateRepository, UserRepository } from "@/repositories/index.js";
|
|
4
5
|
import { normaliseTarget } from "@/shared/index.js";
|
|
5
6
|
|
|
@@ -17,23 +18,12 @@ interface ParkedTask {
|
|
|
17
18
|
}
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
|
-
* Re-checks parked sends at
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
* it first sees it. A send parked for later — a `sendAt`, a quiet-hours
|
|
24
|
-
* deferral, a retry — can sit for hours or days, and whatever the person did in
|
|
25
|
-
* between (unsubscribed, complained, hard-bounced, removed the address) must
|
|
26
|
-
* still stop it. Without this the scheduler sent to them anyway.
|
|
27
|
-
*
|
|
28
|
-
* Reads Postgres directly rather than through the repositories' caches: this
|
|
29
|
-
* runs once per poll, not once per message, and a stale "still subscribed" is
|
|
30
|
-
* the very thing it exists to prevent.
|
|
31
|
-
*
|
|
32
|
-
* Returns the blocked tasks with the reason. Throws if a lookup fails, so the
|
|
33
|
-
* caller holds the batch rather than sending it unchecked.
|
|
21
|
+
* Re-checks parked sends at release: an unsubscribe, bounce or removed address
|
|
22
|
+
* since parking must still stop them. Reads Postgres, not the caches, and
|
|
23
|
+
* throws on a failed lookup so the batch is held rather than sent unchecked.
|
|
34
24
|
*/
|
|
35
25
|
export async function findBlockedAtRelease(
|
|
36
|
-
db:
|
|
26
|
+
db: Db,
|
|
37
27
|
tasks: ParkedTask[],
|
|
38
28
|
): Promise<Map<string, BlockReason>> {
|
|
39
29
|
const blocked = new Map<string, BlockReason>();
|
|
@@ -95,8 +85,7 @@ function decide(
|
|
|
95
85
|
| undefined,
|
|
96
86
|
contactsByUser: Map<string, { channel: string; target: string; preferences?: any }[]>,
|
|
97
87
|
): BlockReason | undefined {
|
|
98
|
-
//
|
|
99
|
-
// records that the address asked us to stop or is dead.
|
|
88
|
+
// First and for every priority, as in the engine.
|
|
100
89
|
if (t.destination && suppressed.has(`${t.channel}:${normaliseTarget(t.destination)}`)) {
|
|
101
90
|
return "suppressed";
|
|
102
91
|
}
|
|
@@ -7,12 +7,7 @@ const DURATION_UNIT_SECONDS: Record<string, number> = {
|
|
|
7
7
|
d: 86_400,
|
|
8
8
|
};
|
|
9
9
|
|
|
10
|
-
/**
|
|
11
|
-
* `"500ms"`, `"0.5s"`, `"15m"`, `"1.5h"`, `"2.5d"` to seconds. Decimals are
|
|
12
|
-
* read exactly; anything that is not a number and a unit throws. The old
|
|
13
|
-
* parser matched on the last character, so "500ms" meant 500 seconds, and ran
|
|
14
|
-
* parseInt, so "1.5h" meant one hour.
|
|
15
|
-
*/
|
|
10
|
+
/** `"500ms"`, `"1.5h"`, `"2.5d"` to seconds. Anything else throws. */
|
|
16
11
|
export function parseDurationSeconds(value: string, what: string): number {
|
|
17
12
|
const match = /^(\d+(?:\.\d+)?|\.\d+)(ms|s|m|h|d)$/.exec(value.trim());
|
|
18
13
|
if (!match) throw new Error(`Invalid ${what}: ${value}`);
|
|
@@ -100,9 +100,7 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
override async stop(): Promise<void> {
|
|
103
|
-
//
|
|
104
|
-
// notify is waiting on it, and clearing it first left the drain to the 30s
|
|
105
|
-
// stop timeout with the instance lock still being renewed.
|
|
103
|
+
// Keep the flush interval until in-flight runs settle; a step's notify waits on it.
|
|
106
104
|
await super.stop();
|
|
107
105
|
if (this.flushTimer) {
|
|
108
106
|
clearInterval(this.flushTimer);
|
|
@@ -178,7 +176,6 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
178
176
|
|
|
179
177
|
let handler = workflowRegistry.get(name);
|
|
180
178
|
if (!handler) {
|
|
181
|
-
// Fallback: check dynamic JSON workflows
|
|
182
179
|
const defRows = await this.dbConn
|
|
183
180
|
.select()
|
|
184
181
|
.from(workflowDefinitions)
|
|
@@ -229,10 +226,8 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
229
226
|
"NX",
|
|
230
227
|
);
|
|
231
228
|
if (!acquired) {
|
|
232
|
-
// Another process is mid-run
|
|
233
|
-
//
|
|
234
|
-
// event arrived quickly. Acking here dropped the resume, and with the
|
|
235
|
-
// waiter already consumed nothing would ever wake the instance again.
|
|
229
|
+
// Another process is mid-run, often still saving the suspension this resume
|
|
230
|
+
// answers. The waiter is already consumed, so the resume must not be dropped.
|
|
236
231
|
this.logger.info({ instanceId }, "Workflow is locked by another process, deferring resume");
|
|
237
232
|
await this.deferResume(payload);
|
|
238
233
|
return;
|
|
@@ -305,7 +300,6 @@ export class WorkflowWorker extends BaseWorker {
|
|
|
305
300
|
if (row?.status === "canceled") throw new WorkflowCanceledError();
|
|
306
301
|
};
|
|
307
302
|
|
|
308
|
-
// Load existing steps
|
|
309
303
|
const existingSteps = await this.dbConn
|
|
310
304
|
.select()
|
|
311
305
|
.from(workflowSteps)
|
|
@@ -147,7 +147,8 @@ export async function reapStuckInstances({
|
|
|
147
147
|
await db
|
|
148
148
|
.update(workflowInstances)
|
|
149
149
|
.set({ status: "pending" })
|
|
150
|
-
|
|
150
|
+
// It may have finished or been canceled since the select above.
|
|
151
|
+
.where(and(eq(workflowInstances.id, inst.id), eq(workflowInstances.status, "running")));
|
|
151
152
|
logger.info({ instanceId: inst.id }, "Reaped stuck workflow instance (lock expired)");
|
|
152
153
|
}
|
|
153
154
|
}
|
|
@@ -5,13 +5,9 @@ export interface BatchItem<T, R> {
|
|
|
5
5
|
}
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* since the first one arrived. Up to `maxConcurrent` flushes may be in flight
|
|
12
|
-
* at once: with a single flight, every caller queued behind a slow flush holds
|
|
13
|
-
* its worker slot for the whole round trip, which caps a worker's throughput at
|
|
14
|
-
* `maxSize / flushLatency` no matter how much concurrency it was given.
|
|
8
|
+
* Batches `add()` calls into `flushFn` calls, at `maxSize` items or after
|
|
9
|
+
* `maxWaitMs`. Several flushes may run at once so one slow flush doesn't cap
|
|
10
|
+
* the worker's throughput.
|
|
15
11
|
*/
|
|
16
12
|
export class BatchProcessor<T, R = void> {
|
|
17
13
|
private buffer: BatchItem<T, R>[] = [];
|
package/src/shared/utils.ts
CHANGED
|
@@ -4,14 +4,8 @@ export function getPriorityBucket(priority: string | undefined): "critical" | "n
|
|
|
4
4
|
}
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* Canonical
|
|
8
|
-
*
|
|
9
|
-
* Both the writer (the provider webhook) and the reader (the engine's
|
|
10
|
-
* pre-dispatch gate) must agree on this, or an unsubscribe recorded as
|
|
11
|
-
* `Bob@Example.com` will not match a send addressed to `bob@example.com` and
|
|
12
|
-
* the person keeps receiving mail. Case folding is safe for email domains and
|
|
13
|
-
* for the local part in every mailbox provider in practice; phone numbers and
|
|
14
|
-
* push tokens are case-sensitive and are only trimmed.
|
|
7
|
+
* Canonical address for suppression lookups; writers and readers must both use
|
|
8
|
+
* it. Emails are lower-cased; phone numbers and push tokens are case-sensitive.
|
|
15
9
|
*/
|
|
16
10
|
export function normaliseTarget(target: string): string {
|
|
17
11
|
const trimmed = target.trim();
|
package/src/transport/index.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import type { NotificationChannel, NotificationDispatchedPayload } from "@/index.js";
|
|
2
2
|
|
|
3
|
-
// ─── Core interface ──────────────────────────────────────────────────────────
|
|
4
|
-
//
|
|
5
3
|
// Implement this interface to add a new channel (Email, SMS, APNS, Web Push …).
|
|
6
4
|
// Register the implementation with transportRegistry at app startup.
|
|
7
5
|
|
|
@@ -67,8 +65,6 @@ export interface Transport {
|
|
|
67
65
|
) => Promise<WebhookEvent[]>;
|
|
68
66
|
}
|
|
69
67
|
|
|
70
|
-
// ─── TransportRegistry ───────────────────────────────────────────────────────
|
|
71
|
-
|
|
72
68
|
class TransportRegistry {
|
|
73
69
|
private readonly transports = new Map<string, { transport: Transport; priority: number }[]>();
|
|
74
70
|
|
package/src/unsubscribe/index.ts
CHANGED
|
@@ -1,18 +1,10 @@
|
|
|
1
1
|
import { createHmac, timingSafeEqual } from "node:crypto";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* One-click unsubscribe (RFC 8058).
|
|
4
|
+
* One-click unsubscribe (RFC 8058), required by Gmail and Yahoo for bulk mail.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* and because deliverability is reputation on the *sending domain*, that
|
|
9
|
-
* eventually drags transactional mail down with it.
|
|
10
|
-
*
|
|
11
|
-
* The link has to work from an inbox, years later, with no session — so the
|
|
12
|
-
* token carries its own claim and is signed rather than looked up. Nothing is
|
|
13
|
-
* stored per message, and there is no expiry: an unsubscribe link that has
|
|
14
|
-
* stopped working is worse than useless, because the recipient's next move is
|
|
15
|
-
* the spam button.
|
|
6
|
+
* The token is signed, not stored, and never expires: the link has to keep
|
|
7
|
+
* working from an old email, and a dead one sends people to the spam button.
|
|
16
8
|
*/
|
|
17
9
|
|
|
18
10
|
export interface UnsubscribeClaim {
|