notifkit 0.1.17 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/{bootstrap-P63acBG9.mjs → bootstrap-Cs1XIp6b.mjs} +1 -1
- package/dist/{bulk-Beblt3ny.mjs → bulk-4pRIiG2o.mjs} +1 -1
- package/dist/index.d.mts +29 -111
- package/dist/index.mjs +2 -2
- package/dist/{main-DB0bUh2r.mjs → main-BN7ZfvxN.mjs} +5 -14
- package/dist/{main-C3XcLoCb.mjs → main-Bl0S4GDd.mjs} +3 -3
- package/dist/{main-_I4b1cmP.mjs → main-C5DiM7_r.mjs} +3 -3
- package/dist/{main-Lcbg7aEn.mjs → main-DVzp0DLn.mjs} +8 -58
- package/dist/{main-NHUvynW2.mjs → main-O9iZ6iz2.mjs} +4 -9
- package/dist/{main-BIahf4f9.mjs → main-Ppr7Pq1g.mjs} +35 -16
- package/dist/{main-Dk92nti8.mjs → main-yL5VinQ5.mjs} +5 -12
- package/dist/{main-CktEylpL.mjs → main-zgxAlH3l.mjs} +23 -94
- package/dist/{schedule-or-dispatch-C5qfxQCv.mjs → schedule-or-dispatch-BwCZWS_P.mjs} +1 -1
- package/dist/{src-BRatuWJS.mjs → src-C38w4jR4.mjs} +61 -173
- package/drizzle/0009_retention_time_indexes.sql +4 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +1 -1
- package/src/client.ts +6 -6
- package/src/config/index.ts +8 -10
- package/src/contracts/sdk.ts +9 -9
- package/src/db/bulk.ts +3 -10
- package/src/db/schema.ts +3 -0
- package/src/idempotency/index.ts +12 -53
- package/src/queue/index.ts +1 -12
- package/src/rate-limiter/api-rate-limiter.ts +8 -27
- package/src/rate-limiter/project-settings-cache.ts +4 -16
- package/src/redis/index.ts +3 -51
- package/src/repositories/contact.ts +0 -2
- package/src/server.ts +1 -4
- package/src/services/api/handlers/auth.ts +3 -11
- package/src/services/api/handlers/campaigns.ts +4 -15
- package/src/services/api/handlers/events.ts +1 -2
- package/src/services/api/handlers/notifications.ts +6 -16
- package/src/services/api/handlers/projects.ts +1 -3
- package/src/services/api/handlers/shared.ts +7 -29
- package/src/services/api/handlers/system.ts +1 -1
- package/src/services/api/handlers/unsubscribe.ts +7 -30
- package/src/services/api/handlers/users.ts +1 -2
- package/src/services/api/http.ts +5 -10
- package/src/services/api/main.ts +27 -63
- package/src/services/api/webhooks.ts +8 -19
- package/src/services/auth/index.ts +2 -21
- package/src/services/delivery/main.ts +25 -89
- package/src/services/engine/main.ts +33 -69
- package/src/services/engine/schedule-or-dispatch.ts +7 -4
- package/src/services/enricher/main.ts +19 -49
- package/src/services/events/main.ts +3 -12
- package/src/services/scheduler/main.ts +26 -32
- package/src/services/scheduler/retention.ts +47 -0
- package/src/services/scheduler/send-gate.ts +6 -17
- package/src/services/workflow/duration.ts +1 -6
- package/src/services/workflow/main.ts +3 -9
- package/src/services/workflow/timers.ts +2 -1
- package/src/shared/batch-processor.ts +3 -7
- package/src/shared/utils.ts +2 -8
- package/src/transport/index.ts +0 -4
- package/src/unsubscribe/index.ts +3 -11
- package/src/workers/index.ts +8 -27
|
@@ -20,7 +20,8 @@ import {
|
|
|
20
20
|
PUBSUB_CHANNELS,
|
|
21
21
|
} from "@/index.js";
|
|
22
22
|
import { upsertOutboxProviderIds } from "@/db/bulk.js";
|
|
23
|
-
import {
|
|
23
|
+
import type { Db } from "@/db/index.js";
|
|
24
|
+
import { scheduledPayloads } from "@/db/schema.js";
|
|
24
25
|
import { sql as drizzleSql } from "drizzle-orm";
|
|
25
26
|
import { ContactRepository, IdempotencyGuard, TemplateRepository } from "@/index.js";
|
|
26
27
|
import { renderWithTemplate, TemplateCache } from "@/templates/index.js";
|
|
@@ -39,8 +40,6 @@ import { publishDeliveryEvents } from "./live-events.js";
|
|
|
39
40
|
import { ProviderThrottle } from "./throttle.js";
|
|
40
41
|
import { metrics } from "@/metrics/index.js";
|
|
41
42
|
|
|
42
|
-
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
43
|
-
|
|
44
43
|
loadEnv();
|
|
45
44
|
const config = readBaseConfig();
|
|
46
45
|
|
|
@@ -61,28 +60,14 @@ let stopLiveEvents: (() => void) | null = null;
|
|
|
61
60
|
/** How long a send may run before its dispatched log entry is published alone. */
|
|
62
61
|
const DISPATCH_LOG_DELAY_MS = 1_000;
|
|
63
62
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
* The first step matches the circuit breaker's reset timeout, so a retry of a
|
|
67
|
-
* task the open breaker turned away reaches the provider's recovery probe
|
|
68
|
-
* rather than the same closed door.
|
|
69
|
-
*/
|
|
63
|
+
// Backoff by prior failures. The first step equals the breaker's reset timeout,
|
|
64
|
+
// so a retry reaches the half-open probe instead of the still-open breaker.
|
|
70
65
|
const RETRY_BACKOFF_MS = [30_000, 120_000, 600_000];
|
|
71
66
|
|
|
72
|
-
|
|
73
|
-
* How many times a send may be put back for want of provider capacity before
|
|
74
|
-
* it is failed. Separate from `maxAttempts`, which counts sends the provider
|
|
75
|
-
* refused: waiting in line is not a failed attempt, and a campaign several
|
|
76
|
-
* windows deeper than the provider's limit has to be able to wait that long.
|
|
77
|
-
*/
|
|
67
|
+
// Deferrals for provider capacity. Not counted against maxAttempts: waiting is not a failure.
|
|
78
68
|
export const MAX_THROTTLE_DEFERRALS = 50;
|
|
79
69
|
|
|
80
|
-
|
|
81
|
-
* Names the provider at `transports[i]` for its rate limit and circuit breaker.
|
|
82
|
-
* The rate limit's budget lives in Redis and is shared by every delivery
|
|
83
|
-
* process, so the name must come out the same everywhere: the class name,
|
|
84
|
-
* numbered when a channel has two providers of one class.
|
|
85
|
-
*/
|
|
70
|
+
// Stable across processes because the rate limit budget is shared in Redis.
|
|
86
71
|
function providerKey(channel: string, transports: any[], i: number): string {
|
|
87
72
|
const name = transports[i].constructor?.name ?? "provider";
|
|
88
73
|
const nth = transports.slice(0, i).filter((t) => t.constructor?.name === name).length;
|
|
@@ -108,8 +93,8 @@ export interface DeliveryWorkerOptions extends WorkerOptions {
|
|
|
108
93
|
enrichedProducers: any;
|
|
109
94
|
contactRepo: any;
|
|
110
95
|
eventsProducer: any;
|
|
111
|
-
globalEmitter:
|
|
112
|
-
db:
|
|
96
|
+
globalEmitter: typeof globalEmitter;
|
|
97
|
+
db: Db;
|
|
113
98
|
providerThrottle?: ProviderThrottle;
|
|
114
99
|
/** Renders tasks that arrive without `renderedContent`. */
|
|
115
100
|
templateCache?: TemplateCache;
|
|
@@ -122,12 +107,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
122
107
|
private readonly enrichedProducers: any;
|
|
123
108
|
private readonly contactRepo: any;
|
|
124
109
|
private readonly eventsProducer: any;
|
|
125
|
-
private readonly globalEmitter:
|
|
126
|
-
private readonly db:
|
|
110
|
+
private readonly globalEmitter: typeof globalEmitter;
|
|
111
|
+
private readonly db: Db;
|
|
127
112
|
|
|
128
113
|
private eventProcessor: BatchProcessor<any, void>;
|
|
129
114
|
private outboxUpdateProcessor: BatchProcessor<any, void>;
|
|
130
|
-
private outboxInsertProcessor: BatchProcessor<any, boolean>;
|
|
131
115
|
private breakers = new Map<string, CircuitBreaker>();
|
|
132
116
|
private readonly providerThrottle: ProviderThrottle;
|
|
133
117
|
private readonly templateCache?: TemplateCache;
|
|
@@ -146,9 +130,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
146
130
|
options.providerThrottle ?? new ProviderThrottle(options.redis, this.logger);
|
|
147
131
|
this.templateCache = options.templateCache;
|
|
148
132
|
|
|
149
|
-
//
|
|
150
|
-
// the worker's throughput: allow a few in flight and let batches grow with
|
|
151
|
-
// the configured concurrency.
|
|
133
|
+
// Acks wait on these flushes, so they cap throughput.
|
|
152
134
|
const batchSize = Math.max(10, Math.min(options.concurrency ?? 50, 500));
|
|
153
135
|
const flushConcurrency = 4;
|
|
154
136
|
|
|
@@ -180,33 +162,14 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
180
162
|
},
|
|
181
163
|
flushConcurrency,
|
|
182
164
|
);
|
|
183
|
-
|
|
184
|
-
this.outboxInsertProcessor = new BatchProcessor(batchSize, 5, async (tasks) => {
|
|
185
|
-
const values = tasks.map((task: any) => ({
|
|
186
|
-
taskId: task.taskId,
|
|
187
|
-
channel: task.channel,
|
|
188
|
-
destination: task.destination,
|
|
189
|
-
}));
|
|
190
|
-
|
|
191
|
-
await this.db.insert(deliveryOutbox).values(values).onConflictDoNothing();
|
|
192
|
-
return tasks.map(() => true);
|
|
193
|
-
});
|
|
194
165
|
}
|
|
195
166
|
|
|
196
167
|
override async stop(): Promise<void> {
|
|
197
|
-
await Promise.all([
|
|
198
|
-
this.eventProcessor.flush(),
|
|
199
|
-
this.outboxUpdateProcessor.flush(),
|
|
200
|
-
this.outboxInsertProcessor.flush(),
|
|
201
|
-
]);
|
|
168
|
+
await Promise.all([this.eventProcessor.flush(), this.outboxUpdateProcessor.flush()]);
|
|
202
169
|
await super.stop();
|
|
203
170
|
}
|
|
204
171
|
|
|
205
|
-
/**
|
|
206
|
-
* Renders a task the engine dispatched unrendered. Done here, after the
|
|
207
|
-
* throttle and the lease, so the body is built once per actual send and never
|
|
208
|
-
* travels through the outbound stream.
|
|
209
|
-
*/
|
|
172
|
+
/** Rendered after the throttle and lease, so once per real send. */
|
|
210
173
|
private async render(task: NotificationDispatchedPayload): Promise<RenderedContent> {
|
|
211
174
|
if (!task.templateId) return renderWithTemplate(null, task.templateVariables);
|
|
212
175
|
const template = this.templateCache
|
|
@@ -219,11 +182,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
219
182
|
return renderWithTemplate(template, task.templateVariables);
|
|
220
183
|
}
|
|
221
184
|
|
|
222
|
-
/**
|
|
223
|
-
* Parks a task in Postgres and the scheduler until `sendAt`. The payload
|
|
224
|
-
* carries the send time, so the scheduler can rebuild it if Redis loses its
|
|
225
|
-
* data.
|
|
226
|
-
*/
|
|
185
|
+
/** Parks a task until `sendAt`. Postgres keeps the send time so Redis can be rebuilt. */
|
|
227
186
|
private async reschedule(
|
|
228
187
|
task: DispatchedTaskPayload,
|
|
229
188
|
sendAt: Date,
|
|
@@ -289,14 +248,11 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
289
248
|
const task = payloadResult.data as NotificationDispatchedPayload;
|
|
290
249
|
// Attempts made on earlier trips through the scheduler, plus this one.
|
|
291
250
|
const attemptNo = (task.deliveryAttemptCount ?? 0) + attempt;
|
|
292
|
-
//
|
|
293
|
-
// and the transport's abort signal were attached to it.
|
|
251
|
+
// Parked on retry: the task before rendering and the abort signal were added.
|
|
294
252
|
const original: DispatchedTaskPayload = { ...task };
|
|
295
253
|
|
|
296
|
-
// The
|
|
297
|
-
//
|
|
298
|
-
// DISPATCH_LOG_DELAY_MS is logged on its own, so a slow or stuck attempt is
|
|
299
|
-
// visible while it is in flight, as it always was.
|
|
254
|
+
// The `dispatched` entry rides on the outcome event; a send slower than
|
|
255
|
+
// DISPATCH_LOG_DELAY_MS logs it alone so stuck sends stay visible.
|
|
300
256
|
let dispatchStarted = false;
|
|
301
257
|
let dispatchLogged = false;
|
|
302
258
|
let dispatchedAt = "";
|
|
@@ -333,11 +289,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
333
289
|
dispatchLogged = true;
|
|
334
290
|
return { ...payload, attempt: attemptNo, dispatchedAt };
|
|
335
291
|
};
|
|
336
|
-
/**
|
|
337
|
-
* Queues the task's `notification.failed` event; once sending has started it
|
|
338
|
-
* is the attempt's outcome. Only the non-push provider failure leaves out
|
|
339
|
-
* `recipientId`.
|
|
340
|
-
*/
|
|
292
|
+
/** Queues `notification.failed`. Only the non-push provider failure omits recipientId. */
|
|
341
293
|
const failed = (failureReason: string, failureCode: string, withRecipientId = true) => {
|
|
342
294
|
const payload = {
|
|
343
295
|
projectId: task.projectId,
|
|
@@ -482,9 +434,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
482
434
|
return;
|
|
483
435
|
}
|
|
484
436
|
|
|
485
|
-
// Spread over more windows the longer a task
|
|
486
|
-
// away in one window used to come back at the same boundary, where the
|
|
487
|
-
// limit again let one window's worth through and bounced the rest.
|
|
437
|
+
// Spread retries over more windows the longer a task waits, so they don't all return at once.
|
|
488
438
|
const spreadMs = Number.isFinite(windowMs)
|
|
489
439
|
? Math.random() * windowMs * Math.min(task.throttleAttemptCount, 10)
|
|
490
440
|
: 0;
|
|
@@ -520,16 +470,10 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
520
470
|
}
|
|
521
471
|
|
|
522
472
|
const tDispatchStart = Date.now();
|
|
523
|
-
const MAX_DISPATCH_BUDGET_MS = 25_000; //
|
|
473
|
+
const MAX_DISPATCH_BUDGET_MS = 25_000; // well under the 60 s lease taken above
|
|
524
474
|
|
|
525
475
|
let retryScheduled = false;
|
|
526
|
-
/**
|
|
527
|
-
* Re-queues a send the provider turned away, unless it said the failure is
|
|
528
|
-
* permanent or the task is out of attempts. An outage, a timeout or an open
|
|
529
|
-
* circuit is exactly what a later attempt can get past; failing the task
|
|
530
|
-
* outright turned every blip into lost mail. The attempt stays in the log as
|
|
531
|
-
* dispatched, and the task's outcome is whatever its last attempt comes to.
|
|
532
|
-
*/
|
|
476
|
+
/** Re-queues a failed send unless the provider called it permanent or attempts ran out. */
|
|
533
477
|
const retryLater = async (failure: { retryable?: boolean; error?: string }) => {
|
|
534
478
|
const maxAttempts = task.deliveryOptions?.maxAttempts ?? 3;
|
|
535
479
|
if (failure.retryable === false || attemptNo >= maxAttempts) return false;
|
|
@@ -572,8 +516,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
572
516
|
let result: any = { success: false, error: "No transports" };
|
|
573
517
|
for (let i = startAt; i < transports.length; i++) {
|
|
574
518
|
const transport = transports[i];
|
|
575
|
-
// The
|
|
576
|
-
// later one spends that provider's own budget, or skips it if spent.
|
|
519
|
+
// The first provider's slot was taken above; later ones spend their own.
|
|
577
520
|
if (i > startAt && !(await this.providerGate(task.channel, transports, i)).allowed) {
|
|
578
521
|
continue;
|
|
579
522
|
}
|
|
@@ -609,9 +552,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
609
552
|
);
|
|
610
553
|
}),
|
|
611
554
|
]);
|
|
612
|
-
//
|
|
613
|
-
// returned, not thrown, so the breaker does not count it. A run
|
|
614
|
-
// of them in one list used to open it for every project.
|
|
555
|
+
// Permanent rejections (bad address) are returned, not thrown, so the breaker ignores them.
|
|
615
556
|
if (!res.success && !(isPush && res.invalidToken) && res.retryable !== false) {
|
|
616
557
|
throw providerFailure(res);
|
|
617
558
|
}
|
|
@@ -780,8 +721,7 @@ export class DeliveryWorker extends BaseWorker {
|
|
|
780
721
|
|
|
781
722
|
export async function startDeliveryWorker() {
|
|
782
723
|
({ logger, redis, sql, db } = createWorkerRuntime(config, "delivery"));
|
|
783
|
-
//
|
|
784
|
-
// more messages in flight than the CPU-bound stages do.
|
|
724
|
+
// Mostly waiting on providers, so it runs more in flight than CPU-bound stages.
|
|
785
725
|
const deliveryConcurrency = config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY;
|
|
786
726
|
const consumerId = uniqueConsumerId("delivery");
|
|
787
727
|
({ consumer, pendingScanner } = createStreamConsumer(
|
|
@@ -836,7 +776,7 @@ export async function startDeliveryWorker() {
|
|
|
836
776
|
templateCache,
|
|
837
777
|
});
|
|
838
778
|
|
|
839
|
-
// Delivery renders templates
|
|
779
|
+
// Delivery renders templates, so it needs template invalidations too.
|
|
840
780
|
subscriber = redis.native.duplicate({ enableAutoPipelining: false });
|
|
841
781
|
await subscriber.subscribe(PUBSUB_CHANNELS.TEMPLATE_INVALIDATED);
|
|
842
782
|
subscriber.on("message", (_channel: string, key: string) => templateCache.invalidateKey(key));
|
|
@@ -844,8 +784,6 @@ export async function startDeliveryWorker() {
|
|
|
844
784
|
// Outcomes for the API's live event stream, which may be in another process.
|
|
845
785
|
stopLiveEvents = publishDeliveryEvents(redis.native, globalEmitter);
|
|
846
786
|
|
|
847
|
-
// ─── Health check interval ──────────────────────────────────────────────────
|
|
848
|
-
|
|
849
787
|
healthInterval = startHealthReporter("delivery", worker, redis, logger);
|
|
850
788
|
|
|
851
789
|
logger.info(
|
|
@@ -859,8 +797,6 @@ export function getDeliveryWorker(): DeliveryWorker | undefined {
|
|
|
859
797
|
return worker as DeliveryWorker | undefined;
|
|
860
798
|
}
|
|
861
799
|
|
|
862
|
-
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
863
|
-
|
|
864
800
|
export async function stopDeliveryWorker(): Promise<void> {
|
|
865
801
|
await shutdownWorker("delivery", {
|
|
866
802
|
logger,
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import type { EventEmitter } from "node:events";
|
|
3
|
+
import type { Db } from "@/db/index.js";
|
|
2
4
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
3
5
|
import type { Logger } from "@/index.js";
|
|
4
6
|
import type { RedisClient, Redis } from "@/index.js";
|
|
@@ -54,8 +56,6 @@ const NO_IDEMPOTENCY = {
|
|
|
54
56
|
unmark: async (_id: string) => {},
|
|
55
57
|
};
|
|
56
58
|
|
|
57
|
-
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
58
|
-
|
|
59
59
|
loadEnv();
|
|
60
60
|
const config = readBaseConfig();
|
|
61
61
|
|
|
@@ -72,33 +72,36 @@ let pendingScanner: PendingMessageScanner;
|
|
|
72
72
|
let worker: BaseWorker;
|
|
73
73
|
let healthInterval: NodeJS.Timeout | null = null;
|
|
74
74
|
|
|
75
|
+
type Publisher = Pick<StreamProducer, "publish">;
|
|
76
|
+
|
|
75
77
|
export interface EngineWorkerOptions extends WorkerOptions {
|
|
76
78
|
registry: EventRegistry;
|
|
77
|
-
idempotency:
|
|
78
|
-
throttle:
|
|
79
|
+
idempotency: Pick<IdempotencyGuard, "checkAndMark" | "markProcessed" | "unmark">;
|
|
80
|
+
throttle: Pick<UserThrottle, "check">;
|
|
79
81
|
projectSettings: ProjectSettingsCache;
|
|
80
82
|
redis: Redis;
|
|
81
83
|
templateCache: TemplateCache;
|
|
82
|
-
aiPendingProducer:
|
|
83
|
-
scheduledProducer:
|
|
84
|
-
outboundProducers:
|
|
85
|
-
globalEmitter:
|
|
84
|
+
aiPendingProducer: Publisher;
|
|
85
|
+
scheduledProducer: Publisher;
|
|
86
|
+
outboundProducers: Partial<Record<"critical" | "normal" | "low", Publisher>>;
|
|
87
|
+
globalEmitter: Pick<EventEmitter, "emit">;
|
|
88
|
+
// Left loose: tests reach into it to swap its mock.
|
|
86
89
|
contactRepo: any;
|
|
87
|
-
db:
|
|
90
|
+
db: Db;
|
|
88
91
|
}
|
|
89
92
|
|
|
90
93
|
export class EngineWorker extends BaseWorker {
|
|
91
94
|
private readonly registry: EventRegistry;
|
|
92
|
-
private readonly idempotency:
|
|
93
|
-
private readonly throttle:
|
|
95
|
+
private readonly idempotency: EngineWorkerOptions["idempotency"];
|
|
96
|
+
private readonly throttle: EngineWorkerOptions["throttle"];
|
|
94
97
|
private readonly projectSettings: ProjectSettingsCache;
|
|
95
98
|
private readonly templateCache: TemplateCache;
|
|
96
|
-
private readonly aiPendingProducer:
|
|
97
|
-
private readonly scheduledProducer:
|
|
98
|
-
private readonly outboundProducers:
|
|
99
|
-
private readonly globalEmitter:
|
|
99
|
+
private readonly aiPendingProducer: Publisher;
|
|
100
|
+
private readonly scheduledProducer: Publisher;
|
|
101
|
+
private readonly outboundProducers: EngineWorkerOptions["outboundProducers"];
|
|
102
|
+
private readonly globalEmitter: EngineWorkerOptions["globalEmitter"];
|
|
100
103
|
private readonly contactRepo: any;
|
|
101
|
-
private readonly db:
|
|
104
|
+
private readonly db: Db;
|
|
102
105
|
/** Delivery in this process, when the pipeline is fused. */
|
|
103
106
|
private inlineNext: ((event: StreamEvent) => Promise<void>) | null = null;
|
|
104
107
|
|
|
@@ -148,15 +151,8 @@ export class EngineWorker extends BaseWorker {
|
|
|
148
151
|
{ batchDelayMs: 2, maxBatchSize: 500 },
|
|
149
152
|
);
|
|
150
153
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
*
|
|
154
|
-
* Loaded per project+channel rather than per address: a campaign is thousands
|
|
155
|
-
* of messages against one channel, so this collapses to a single query for
|
|
156
|
-
* the whole batch. Cached for the loader's lifetime of a tick, which means a
|
|
157
|
-
* suppression written mid-batch takes effect on the next batch — acceptable,
|
|
158
|
-
* since the webhook that writes it is itself minutes behind the send.
|
|
159
|
-
*/
|
|
154
|
+
// One query per (project, channel), not per address: a campaign is thousands
|
|
155
|
+
// of sends on one channel. Cached 10s; webhooks that suppress lag far more.
|
|
160
156
|
private readonly suppressionsCache = new LRUCache<string, Set<string>>(1000, 10_000);
|
|
161
157
|
|
|
162
158
|
private readonly suppressionsLoader = new DataLoader<
|
|
@@ -188,8 +184,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
188
184
|
this.suppressionsCache.set(cacheKey, set);
|
|
189
185
|
results.set(cacheKey, set);
|
|
190
186
|
} catch (err) {
|
|
191
|
-
//
|
|
192
|
-
// is suppressed" — that would resume mailing people who opted out.
|
|
187
|
+
// Failing open here would mail people who opted out.
|
|
193
188
|
this.logger.error(
|
|
194
189
|
{ err, projectId: key.projectId, channel: key.channel },
|
|
195
190
|
"suppression lookup failed — holding message",
|
|
@@ -214,9 +209,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
214
209
|
|
|
215
210
|
const enriched = payloadResult.data;
|
|
216
211
|
|
|
217
|
-
//
|
|
218
|
-
// The enricher emits one event per resolved contact, so two addresses on one
|
|
219
|
-
// channel are two distinct messages here.
|
|
212
|
+
// One event per resolved contact, so two addresses on a channel are two messages.
|
|
220
213
|
const idempotencyKey =
|
|
221
214
|
`${enriched.rawEventId}:${enriched.recipientId}:${enriched.channel}` +
|
|
222
215
|
(enriched.contactId ? `:${enriched.contactId}` : "");
|
|
@@ -229,19 +222,16 @@ export class EngineWorker extends BaseWorker {
|
|
|
229
222
|
}
|
|
230
223
|
}
|
|
231
224
|
|
|
232
|
-
// Inline
|
|
233
|
-
// exclusive under the enricher's lock, the send itself is guarded by
|
|
234
|
-
// delivery's lease on the taskId, and the throttle counts a message once
|
|
235
|
-
// however often it is retried. Stream-read messages keep the guard.
|
|
225
|
+
// Inline runs are already exclusive under the enricher's lock and delivery's lease.
|
|
236
226
|
const idempotency = message.inline ? NO_IDEMPOTENCY : this.idempotency;
|
|
237
227
|
|
|
238
|
-
if
|
|
228
|
+
// A short lease: if this process dies, the redelivery must not find it.
|
|
229
|
+
if (!(await idempotency.checkAndMark(idempotencyKey, 60))) {
|
|
239
230
|
this.logger.debug({ messageId: message.id, eventId: event.id }, "duplicate — skipping");
|
|
240
231
|
return;
|
|
241
232
|
}
|
|
242
233
|
|
|
243
234
|
try {
|
|
244
|
-
// Opt-in check
|
|
245
235
|
if (enriched.recipient.preferences.optedOut) {
|
|
246
236
|
this.logger.info(
|
|
247
237
|
{ messageId: message.id, recipientId: enriched.recipientId, eventType: event.type },
|
|
@@ -272,13 +262,9 @@ export class EngineWorker extends BaseWorker {
|
|
|
272
262
|
return;
|
|
273
263
|
}
|
|
274
264
|
|
|
275
|
-
// Quiet hours check
|
|
276
265
|
const qh = enriched.recipient.preferences.quietHours;
|
|
277
266
|
if (qh && qh.length > 0 && enriched.priority !== "critical") {
|
|
278
|
-
// Judged at
|
|
279
|
-
// engine sees it: a send booked for next week must not be pulled
|
|
280
|
-
// forward to tonight's quiet-hours end, and one booked for 3am must
|
|
281
|
-
// wait until the quiet hours are over.
|
|
267
|
+
// Judged at send time, so a send booked for next week isn't pulled forward.
|
|
282
268
|
const requestedAt = enriched.scheduledAt ? new Date(enriched.scheduledAt) : new Date();
|
|
283
269
|
const sendAt = requestedAt.getTime() > Date.now() ? requestedAt : new Date();
|
|
284
270
|
const qhResult = isInQuietHours(enriched.recipient.timezone, qh, sendAt);
|
|
@@ -295,9 +281,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
295
281
|
}
|
|
296
282
|
}
|
|
297
283
|
|
|
298
|
-
//
|
|
299
|
-
// a lookup failure must not drop the notification, so fall back rather
|
|
300
|
-
// than propagate.
|
|
284
|
+
// A failed settings lookup falls back to the global default, never drops the send.
|
|
301
285
|
let projectThrottle: { throttleLimit: number | null; throttleWindowHours: number | null } = {
|
|
302
286
|
throttleLimit: null,
|
|
303
287
|
throttleWindowHours: null,
|
|
@@ -368,18 +352,10 @@ export class EngineWorker extends BaseWorker {
|
|
|
368
352
|
...(enriched.aiPrompts ?? {}),
|
|
369
353
|
};
|
|
370
354
|
|
|
371
|
-
// Drives whether this message gets an unsubscribe header, and what the
|
|
372
|
-
// resulting opt-out applies to.
|
|
373
355
|
const templateTopics: string[] = dbTemplate?.topics ?? [];
|
|
374
|
-
|
|
375
|
-
// AI-generated messages take the same gates as any other — contacts,
|
|
376
|
-
// topic opt-outs and suppressions below — and are handed to the AI worker
|
|
377
|
-
// per contact once they have passed them.
|
|
356
|
+
// AI sends pass the same gates below before going to the AI worker.
|
|
378
357
|
const useAi = Object.keys(aiPrompts).length > 0;
|
|
379
358
|
|
|
380
|
-
// Rendering happens in delivery, just before the send, so the rendered
|
|
381
|
-
// body does not travel through the outbound stream.
|
|
382
|
-
|
|
383
359
|
// The enricher already resolved the contact; only events without one
|
|
384
360
|
// (push, channel fallbacks, the AI worker) need the user's contacts.
|
|
385
361
|
const allContacts =
|
|
@@ -426,9 +402,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
426
402
|
continue;
|
|
427
403
|
}
|
|
428
404
|
|
|
429
|
-
//
|
|
430
|
-
// records that the person asked us to stop, or that the address is
|
|
431
|
-
// dead. Sending anyway is what gets a domain blocked.
|
|
405
|
+
// Suppression beats every other gate, `critical` included.
|
|
432
406
|
if (contact.target && suppressedTargets.has(normaliseTarget(contact.target))) {
|
|
433
407
|
this.logger.info(
|
|
434
408
|
{
|
|
@@ -450,11 +424,7 @@ export class EngineWorker extends BaseWorker {
|
|
|
450
424
|
const taskId = `${enriched.rawEventId}:${contact.id || randomUUID()}`;
|
|
451
425
|
const resolvedDestination = contact.target;
|
|
452
426
|
|
|
453
|
-
//
|
|
454
|
-
// them. A template with no topic is transactional by this codebase's
|
|
455
|
-
// own convention — a password reset, a receipt — and putting an
|
|
456
|
-
// unsubscribe button on those invites people to switch off mail they
|
|
457
|
-
// actually need, with no topic to scope the opt-out to anyway.
|
|
427
|
+
// Only mail with a topic gets unsubscribe headers; no topic means transactional.
|
|
458
428
|
const unsubscribeHeaders =
|
|
459
429
|
enriched.channel === "email" &&
|
|
460
430
|
templateTopics.length > 0 &&
|
|
@@ -604,12 +574,10 @@ export async function startEngineWorker() {
|
|
|
604
574
|
|
|
605
575
|
const throttle = new UserThrottle({
|
|
606
576
|
redis: redis.native,
|
|
607
|
-
maxPerHour:
|
|
577
|
+
maxPerHour: config.RATE_LIMIT_PER_HOUR,
|
|
608
578
|
});
|
|
609
579
|
|
|
610
|
-
//
|
|
611
|
-
// notification. Stale entries expire on their own; the pub/sub subscriber
|
|
612
|
-
// below only makes an admin's change take effect sooner.
|
|
580
|
+
// Read once per send, so cached; the pub/sub subscriber below drops stale entries early.
|
|
613
581
|
const projectRepo = new ProjectRepository(db);
|
|
614
582
|
const projectSettings = new ProjectSettingsCache((projectId) =>
|
|
615
583
|
projectRepo.findThrottleSettings(projectId),
|
|
@@ -649,8 +617,6 @@ export async function startEngineWorker() {
|
|
|
649
617
|
}
|
|
650
618
|
});
|
|
651
619
|
|
|
652
|
-
// ─── Health check interval ──────────────────────────────────────────────────
|
|
653
|
-
|
|
654
620
|
healthInterval = startHealthReporter("engine", worker, redis, logger);
|
|
655
621
|
|
|
656
622
|
logger.info({ env: config.NODE_ENV }, "engine starting");
|
|
@@ -661,8 +627,6 @@ export function getEngineWorker(): EngineWorker | undefined {
|
|
|
661
627
|
return worker as EngineWorker | undefined;
|
|
662
628
|
}
|
|
663
629
|
|
|
664
|
-
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
665
|
-
|
|
666
630
|
export async function stopEngineWorker(): Promise<void> {
|
|
667
631
|
await shutdownWorker("engine", {
|
|
668
632
|
logger,
|
|
@@ -2,12 +2,16 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import type { Logger } from "@/index.js";
|
|
3
3
|
import { buildStreamEvent, type DispatchedTaskPayload, type StreamEvent } from "@/index.js";
|
|
4
4
|
import { scheduledPayloads } from "@/db/schema.js";
|
|
5
|
+
import type { Db } from "@/db/index.js";
|
|
6
|
+
import type { StreamProducer } from "@/queue/index.js";
|
|
5
7
|
import { getPriorityBucket } from "@/shared/index.js";
|
|
6
8
|
|
|
7
9
|
export interface ScheduleOrDispatchDeps {
|
|
8
|
-
db:
|
|
9
|
-
scheduledProducer:
|
|
10
|
-
outboundProducers:
|
|
10
|
+
db: Db;
|
|
11
|
+
scheduledProducer: Pick<StreamProducer, "publish">;
|
|
12
|
+
outboundProducers: Partial<
|
|
13
|
+
Record<"critical" | "normal" | "low", Pick<StreamProducer, "publish">>
|
|
14
|
+
>;
|
|
11
15
|
logger: Logger;
|
|
12
16
|
/** Delivery in this process, when the pipeline is fused. */
|
|
13
17
|
inlineNext?: ((event: StreamEvent) => Promise<void>) | null;
|
|
@@ -41,7 +45,6 @@ export async function scheduleOrDispatch(
|
|
|
41
45
|
opts.traceId,
|
|
42
46
|
);
|
|
43
47
|
|
|
44
|
-
// Route by scheduledAt
|
|
45
48
|
const now = Date.now();
|
|
46
49
|
const scheduledAt = opts.scheduledAt ? new Date(opts.scheduledAt).getTime() : now;
|
|
47
50
|
|