notifkit 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -72
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-GRXzgPvW.mjs +89 -0
- package/dist/bulk-BvRfz6Of.mjs +77 -0
- package/dist/index.d.mts +1107 -8885
- package/dist/index.mjs +2 -38
- package/dist/{main-DSmh1e-V.mjs → main-BENOY7HG.mjs} +49 -75
- package/dist/{main-BPbPnBIG.mjs → main-BpnuFX62.mjs} +119 -112
- package/dist/{main-4Wyge9ms.mjs → main-BqVH2BHj.mjs} +57 -54
- package/dist/{main-D19Wxs0x.mjs → main-C_5VFIwp.mjs} +135 -79
- package/dist/main-D6od9N8h.mjs +312 -0
- package/dist/main-D84OX0ux.mjs +811 -0
- package/dist/{main-JdOvJ70b.mjs → main-HH-WwsMM.mjs} +395 -184
- package/dist/{main-AO0JaY5z.mjs → main-HY_TDs61.mjs} +149 -134
- package/dist/{src-CGmXRfsM.mjs → src-CI7FGf3h.mjs} +1223 -651
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +17 -17
- package/scripts/create-admin.mjs +8 -11
- package/src/client.ts +15 -5
- package/src/config/index.ts +26 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +2 -2
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +71 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-4Wyge9ms.mjs.map +0 -1
- package/dist/main-AO0JaY5z.mjs.map +0 -1
- package/dist/main-BMYSYcv2.mjs +0 -224
- package/dist/main-BMYSYcv2.mjs.map +0 -1
- package/dist/main-BPbPnBIG.mjs.map +0 -1
- package/dist/main-CyeQ_yDt.mjs +0 -632
- package/dist/main-CyeQ_yDt.mjs.map +0 -1
- package/dist/main-D19Wxs0x.mjs.map +0 -1
- package/dist/main-DSmh1e-V.mjs.map +0 -1
- package/dist/main-JdOvJ70b.mjs.map +0 -1
- package/dist/src-CGmXRfsM.mjs.map +0 -1
|
@@ -1,24 +1,29 @@
|
|
|
1
1
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
2
|
-
import {
|
|
3
|
-
import { RedisClient } from "@/index.js";
|
|
2
|
+
import type { Logger } from "@/index.js";
|
|
3
|
+
import type { RedisClient } from "@/index.js";
|
|
4
4
|
import {
|
|
5
|
-
StreamConsumer,
|
|
6
|
-
PendingMessageScanner,
|
|
5
|
+
type StreamConsumer,
|
|
6
|
+
type PendingMessageScanner,
|
|
7
7
|
StreamProducer,
|
|
8
8
|
type StreamMessage,
|
|
9
9
|
} from "@/index.js";
|
|
10
10
|
import { BaseWorker } from "@/index.js";
|
|
11
11
|
import { STREAMS, CONSUMER_GROUPS, buildStreamEvent } from "@/index.js";
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import { workflowWaiters, workflowSteps, workflowInstances, messageLogs } from "@/db/schema.js";
|
|
12
|
+
import { insertMessageLogs } from "@/db/bulk.js";
|
|
13
|
+
import { workflowWaiters, workflowSteps, workflowInstances } from "@/db/schema.js";
|
|
15
14
|
import { eq, and, or, isNull, gt } from "drizzle-orm";
|
|
16
15
|
import { type WorkerOptions } from "@/shared/index.js";
|
|
17
16
|
import { startHealthReporter } from "@/workers/index.js";
|
|
17
|
+
import {
|
|
18
|
+
createStreamConsumer,
|
|
19
|
+
createWorkerRuntime,
|
|
20
|
+
shutdownWorker,
|
|
21
|
+
uniqueConsumerId,
|
|
22
|
+
} from "@/workers/bootstrap.js";
|
|
18
23
|
|
|
19
24
|
loadEnv();
|
|
20
25
|
const config = readBaseConfig();
|
|
21
|
-
let logger:
|
|
26
|
+
let logger: Logger;
|
|
22
27
|
let redis: RedisClient;
|
|
23
28
|
let sql: any;
|
|
24
29
|
let db: any;
|
|
@@ -46,12 +51,23 @@ export class EventWorker extends BaseWorker {
|
|
|
46
51
|
reject: (err: any) => void;
|
|
47
52
|
}[] = [];
|
|
48
53
|
private flushInterval: NodeJS.Timeout | null = null;
|
|
49
|
-
private
|
|
54
|
+
private flushesInFlight = 0;
|
|
55
|
+
private static readonly MAX_CONCURRENT_FLUSHES = 4;
|
|
56
|
+
/**
|
|
57
|
+
* Buffered logs that trigger a flush without waiting for the interval.
|
|
58
|
+
*
|
|
59
|
+
* Every message holds its worker slot until its log row is written, so a
|
|
60
|
+
* purely timed flush caps the worker at `concurrency / LOG_FLUSH_INTERVAL_MS`
|
|
61
|
+
* rows per second — far below what one delivery worker produces. Flushing
|
|
62
|
+
* once half the slots are waiting keeps the other half reading.
|
|
63
|
+
*/
|
|
64
|
+
private readonly flushThreshold: number;
|
|
50
65
|
|
|
51
66
|
constructor(options: EventWorkerOptions) {
|
|
52
67
|
super(options);
|
|
53
68
|
this.dbConn = options.db;
|
|
54
69
|
this.workflowProducer = options.workflowProducer;
|
|
70
|
+
this.flushThreshold = Math.max(1, Math.min(500, Math.floor(this.concurrency / 2)));
|
|
55
71
|
this.startFlushInterval();
|
|
56
72
|
}
|
|
57
73
|
|
|
@@ -62,9 +78,12 @@ export class EventWorker extends BaseWorker {
|
|
|
62
78
|
}
|
|
63
79
|
|
|
64
80
|
private async flushLogs() {
|
|
65
|
-
if (
|
|
81
|
+
if (
|
|
82
|
+
this.flushesInFlight >= EventWorker.MAX_CONCURRENT_FLUSHES ||
|
|
83
|
+
(this.messageLogBuffer.length === 0 && this.eventBuffer.length === 0)
|
|
84
|
+
)
|
|
66
85
|
return;
|
|
67
|
-
this.
|
|
86
|
+
this.flushesInFlight++;
|
|
68
87
|
|
|
69
88
|
const startTime = Date.now();
|
|
70
89
|
const batch = this.messageLogBuffer;
|
|
@@ -75,13 +94,13 @@ export class EventWorker extends BaseWorker {
|
|
|
75
94
|
|
|
76
95
|
try {
|
|
77
96
|
if (batch.length > 0) {
|
|
78
|
-
await
|
|
79
|
-
.
|
|
80
|
-
|
|
81
|
-
|
|
97
|
+
await insertMessageLogs(
|
|
98
|
+
this.dbConn,
|
|
99
|
+
batch.map((b) => b.log),
|
|
100
|
+
); // Idempotent insert
|
|
82
101
|
for (const b of batch) b.resolve();
|
|
83
102
|
|
|
84
|
-
this.logger.
|
|
103
|
+
this.logger.debug(
|
|
85
104
|
{
|
|
86
105
|
batchSize: batch.length,
|
|
87
106
|
latencyMs: Date.now() - startTime,
|
|
@@ -112,14 +131,31 @@ export class EventWorker extends BaseWorker {
|
|
|
112
131
|
for (const b of batch) b.reject(error);
|
|
113
132
|
for (const e of events) e.reject(error);
|
|
114
133
|
} finally {
|
|
115
|
-
this.
|
|
134
|
+
this.flushesInFlight--;
|
|
135
|
+
// Rows that arrived while every flush slot was busy would otherwise wait
|
|
136
|
+
// out the interval with their worker slots held.
|
|
137
|
+
if (this.messageLogBuffer.length >= this.flushThreshold) {
|
|
138
|
+
void this.flushLogs();
|
|
139
|
+
}
|
|
116
140
|
}
|
|
117
141
|
}
|
|
118
142
|
|
|
119
143
|
override async stop(): Promise<void> {
|
|
120
|
-
|
|
144
|
+
// The interval keeps running until in-flight messages settle: each of them
|
|
145
|
+
// is waiting on a flush, and clearing it first left them to the 30s stop
|
|
146
|
+
// timeout.
|
|
121
147
|
await super.stop();
|
|
122
|
-
|
|
148
|
+
if (this.flushInterval) clearInterval(this.flushInterval);
|
|
149
|
+
const deadline = Date.now() + 10_000;
|
|
150
|
+
while (
|
|
151
|
+
(this.messageLogBuffer.length > 0 ||
|
|
152
|
+
this.eventBuffer.length > 0 ||
|
|
153
|
+
this.flushesInFlight > 0) &&
|
|
154
|
+
Date.now() < deadline
|
|
155
|
+
) {
|
|
156
|
+
await this.flushLogs();
|
|
157
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
158
|
+
}
|
|
123
159
|
}
|
|
124
160
|
|
|
125
161
|
override async process(message: StreamMessage): Promise<void> {
|
|
@@ -150,26 +186,42 @@ export class EventWorker extends BaseWorker {
|
|
|
150
186
|
|
|
151
187
|
const kind = event.type === "notification.dispatched" ? "dispatched" : "attempt";
|
|
152
188
|
|
|
153
|
-
|
|
154
|
-
|
|
189
|
+
const row = (
|
|
190
|
+
rowKind: string,
|
|
191
|
+
rowStatus: string,
|
|
192
|
+
providerMessageId: string | null,
|
|
193
|
+
timestamp?: string,
|
|
194
|
+
) =>
|
|
195
|
+
new Promise<void>((resolve, reject) => {
|
|
155
196
|
this.messageLogBuffer.push({
|
|
156
197
|
log: {
|
|
157
198
|
projectId: payload.projectId,
|
|
158
199
|
taskId: payload.taskId,
|
|
159
|
-
providerMessageId
|
|
200
|
+
providerMessageId,
|
|
160
201
|
channel: payload.channel,
|
|
161
202
|
attempt: payload.attempt || 1,
|
|
162
|
-
kind,
|
|
163
|
-
status,
|
|
203
|
+
kind: rowKind,
|
|
204
|
+
status: rowStatus,
|
|
164
205
|
templateId: payload.templateId || null,
|
|
165
206
|
workflowInstanceId: payload.workflowInstanceId || null,
|
|
166
207
|
campaignId: payload.campaignId || null,
|
|
208
|
+
timestamp,
|
|
167
209
|
},
|
|
168
210
|
resolve,
|
|
169
211
|
reject,
|
|
170
212
|
});
|
|
171
|
-
})
|
|
172
|
-
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
// An outcome carrying `dispatchedAt` also stands in for the attempt's
|
|
216
|
+
// dispatched entry, which delivery folded into it. Stamped with the
|
|
217
|
+
// dispatch time so the log still reads dispatched → outcome.
|
|
218
|
+
if (kind === "attempt" && payload.dispatchedAt) {
|
|
219
|
+
publishPromises.push(row("dispatched", "dispatched", null, payload.dispatchedAt));
|
|
220
|
+
}
|
|
221
|
+
publishPromises.push(row(kind, status, payload.providerMessageId || null));
|
|
222
|
+
if (this.messageLogBuffer.length >= this.flushThreshold) {
|
|
223
|
+
void this.flushLogs();
|
|
224
|
+
}
|
|
173
225
|
await Promise.all(publishPromises);
|
|
174
226
|
return;
|
|
175
227
|
}
|
|
@@ -312,39 +364,36 @@ export class EventWorker extends BaseWorker {
|
|
|
312
364
|
}
|
|
313
365
|
|
|
314
366
|
export async function startEventWorker() {
|
|
315
|
-
logger =
|
|
316
|
-
redis = new RedisClient({ url: config.REDIS_URL, name: "events", logger });
|
|
317
|
-
const dbData = createDatabase({ url: config.DATABASE_URL, applicationName: "events", logger });
|
|
318
|
-
sql = dbData.sql;
|
|
319
|
-
db = dbData.db;
|
|
367
|
+
({ logger, redis, sql, db } = createWorkerRuntime(config, "events", "event-worker"));
|
|
320
368
|
workflowProducer = new StreamProducer({
|
|
321
369
|
redis: redis.native,
|
|
322
370
|
stream: STREAMS.WORKFLOW_INBOUND,
|
|
323
371
|
logger,
|
|
324
372
|
});
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
logger,
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
373
|
+
// Delivery emits two events per message, so this worker has to keep at least
|
|
374
|
+
// as many in flight as delivery does or its stream grows without bound.
|
|
375
|
+
const eventsConcurrency = Math.max(
|
|
376
|
+
config.WORKER_CONCURRENCY,
|
|
377
|
+
config.DELIVERY_CONCURRENCY ?? config.WORKER_CONCURRENCY,
|
|
378
|
+
);
|
|
379
|
+
({ consumer, pendingScanner } = createStreamConsumer(
|
|
380
|
+
{ redis, logger },
|
|
381
|
+
{
|
|
382
|
+
stream: STREAMS.EVENTS_INBOUND,
|
|
383
|
+
group: CONSUMER_GROUPS.EVENTS,
|
|
384
|
+
consumer: uniqueConsumerId("events"),
|
|
385
|
+
batchSize: eventsConcurrency,
|
|
386
|
+
bufferAcks: true,
|
|
387
|
+
// Log rows are not latency-sensitive; let reads fill up.
|
|
388
|
+
coalesceMs: 10,
|
|
389
|
+
},
|
|
390
|
+
));
|
|
342
391
|
|
|
343
392
|
worker = new EventWorker({
|
|
344
393
|
consumer,
|
|
345
394
|
pendingScanner,
|
|
346
395
|
logger,
|
|
347
|
-
concurrency:
|
|
396
|
+
concurrency: eventsConcurrency,
|
|
348
397
|
db,
|
|
349
398
|
workflowProducer,
|
|
350
399
|
});
|
|
@@ -356,10 +405,6 @@ export async function startEventWorker() {
|
|
|
356
405
|
}
|
|
357
406
|
|
|
358
407
|
export async function stopEventWorker(): Promise<void> {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
if (worker) await worker.stop();
|
|
362
|
-
if (sql) await sql.end();
|
|
363
|
-
if (redis) await redis.disconnect();
|
|
364
|
-
logger?.info("event worker stopped");
|
|
408
|
+
await shutdownWorker("event worker", { logger, timers: [healthInterval], worker, sql, redis });
|
|
409
|
+
healthInterval = null;
|
|
365
410
|
}
|
|
@@ -1,13 +1,8 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { loadEnv, readBaseConfig } from "@/index.js";
|
|
3
|
-
import {
|
|
4
|
-
import { RedisClient,
|
|
5
|
-
import {
|
|
6
|
-
StreamConsumer,
|
|
7
|
-
PendingMessageScanner,
|
|
8
|
-
StreamProducer,
|
|
9
|
-
type StreamMessage,
|
|
10
|
-
} from "@/index.js";
|
|
3
|
+
import type { Logger } from "@/index.js";
|
|
4
|
+
import type { RedisClient, Redis } from "@/index.js";
|
|
5
|
+
import { type StreamConsumer, type PendingMessageScanner, type StreamMessage } from "@/index.js";
|
|
11
6
|
import { BaseWorker } from "@/index.js";
|
|
12
7
|
import {
|
|
13
8
|
STREAMS,
|
|
@@ -16,17 +11,24 @@ import {
|
|
|
16
11
|
buildStreamEvent,
|
|
17
12
|
type NotificationScheduledPayload,
|
|
18
13
|
} from "@/index.js";
|
|
19
|
-
import { getPriorityBucket, type WorkerOptions
|
|
20
|
-
import { createDatabase } from "@/db/index.js";
|
|
14
|
+
import { getPriorityBucket, globalEmitter, type WorkerOptions } from "@/shared/index.js";
|
|
21
15
|
import { scheduledPayloads } from "@/db/schema.js";
|
|
22
|
-
import { inArray } from "drizzle-orm";
|
|
16
|
+
import { and, asc, eq, gt, inArray, or, sql as drizzleSql } from "drizzle-orm";
|
|
23
17
|
import { startHealthReporter } from "@/workers/index.js";
|
|
18
|
+
import {
|
|
19
|
+
createPriorityProducers,
|
|
20
|
+
createStreamConsumer,
|
|
21
|
+
createWorkerRuntime,
|
|
22
|
+
shutdownWorker,
|
|
23
|
+
uniqueConsumerId,
|
|
24
|
+
} from "@/workers/bootstrap.js";
|
|
25
|
+
import { findBlockedAtRelease } from "./send-gate.js";
|
|
24
26
|
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
25
27
|
|
|
26
28
|
loadEnv();
|
|
27
29
|
const config = readBaseConfig();
|
|
28
30
|
|
|
29
|
-
let logger:
|
|
31
|
+
let logger: Logger;
|
|
30
32
|
let redis: RedisClient;
|
|
31
33
|
let sql: any;
|
|
32
34
|
let db: any;
|
|
@@ -105,14 +107,8 @@ export async function executeSchedulerPoll(
|
|
|
105
107
|
const pipeline = redis.pipeline();
|
|
106
108
|
|
|
107
109
|
for (let i = 0; i < 16; i++) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
1,
|
|
111
|
-
`notif:scheduled:zset:${i}`,
|
|
112
|
-
now,
|
|
113
|
-
perShardLimit,
|
|
114
|
-
visibilityTimeout,
|
|
115
|
-
);
|
|
110
|
+
const shardKey = `notif:scheduled:zset:${i}`;
|
|
111
|
+
pipeline.schedulerPoll(shardKey, now, perShardLimit, visibilityTimeout);
|
|
116
112
|
}
|
|
117
113
|
|
|
118
114
|
const results = await pipeline.exec();
|
|
@@ -143,12 +139,13 @@ export async function executeSchedulerPoll(
|
|
|
143
139
|
.where(inArray(scheduledPayloads.taskId, taskIds));
|
|
144
140
|
const payloadMap = new Map(dbPayloads.map((row: any) => [row.taskId, row.payload]));
|
|
145
141
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
142
|
+
// What the person did while this sat parked still applies. A failed
|
|
143
|
+
// lookup throws to the catch below: the claimed tasks reappear after the
|
|
144
|
+
// visibility timeout instead of going out unchecked.
|
|
145
|
+
const blocked = await findBlockedAtRelease(db, [...payloadMap.values()] as any[]);
|
|
146
|
+
|
|
147
|
+
const batchedEvents: Record<"critical" | "normal" | "low", Omit<any, "id" | "timestamp">[]> = {
|
|
150
148
|
critical: [],
|
|
151
|
-
high: [],
|
|
152
149
|
normal: [],
|
|
153
150
|
low: [],
|
|
154
151
|
};
|
|
@@ -156,8 +153,20 @@ export async function executeSchedulerPoll(
|
|
|
156
153
|
const cleanupPipeline = redis.pipeline();
|
|
157
154
|
const dbCleanupIds: string[] = [];
|
|
158
155
|
|
|
156
|
+
// After a rebuild the same task can sit in the ZSET twice (the original
|
|
157
|
+
// member carries a trace id Postgres does not keep), so release each task
|
|
158
|
+
// once and just clear any second copy.
|
|
159
|
+
const released = new Set<string>();
|
|
160
|
+
|
|
159
161
|
for (let i = 0; i < parsedTasks.length; i++) {
|
|
160
162
|
const { taskStr, taskId, traceId } = parsedTasks[i];
|
|
163
|
+
if (released.has(taskId)) {
|
|
164
|
+
cleanupPipeline.zrem(
|
|
165
|
+
`notif:scheduled:zset:${parseInt(taskId.slice(-1), 16) || 0}`,
|
|
166
|
+
taskStr,
|
|
167
|
+
);
|
|
168
|
+
continue;
|
|
169
|
+
}
|
|
161
170
|
const payload = payloadMap.get(taskId);
|
|
162
171
|
|
|
163
172
|
if (!payload) {
|
|
@@ -168,12 +177,30 @@ export async function executeSchedulerPoll(
|
|
|
168
177
|
}
|
|
169
178
|
|
|
170
179
|
const dispatchPayload = payload as any;
|
|
180
|
+
|
|
181
|
+
const blockReason = blocked.get(taskId);
|
|
182
|
+
if (blockReason) {
|
|
183
|
+
logger.info({ taskId, reason: blockReason }, "parked send no longer allowed — dropping");
|
|
184
|
+
globalEmitter.emit("notification:skipped", {
|
|
185
|
+
projectId: dispatchPayload.projectId,
|
|
186
|
+
eventId: dispatchPayload.enrichedEventId,
|
|
187
|
+
recipientId: dispatchPayload.recipientId,
|
|
188
|
+
reason: blockReason,
|
|
189
|
+
});
|
|
190
|
+
released.add(taskId);
|
|
191
|
+
dbCleanupIds.push(taskId);
|
|
192
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
193
|
+
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
|
|
171
197
|
const p = getPriorityBucket(dispatchPayload.priority);
|
|
172
198
|
|
|
173
199
|
batchedEvents[p].push(
|
|
174
200
|
buildStreamEvent("notification.dispatched", dispatchPayload, "scheduler", traceId),
|
|
175
201
|
);
|
|
176
202
|
|
|
203
|
+
released.add(taskId);
|
|
177
204
|
dbCleanupIds.push(taskId);
|
|
178
205
|
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
179
206
|
cleanupPipeline.zrem(`notif:scheduled:zset:${shard}`, taskStr);
|
|
@@ -191,7 +218,24 @@ export async function executeSchedulerPoll(
|
|
|
191
218
|
|
|
192
219
|
if (dbCleanupIds.length > 0) {
|
|
193
220
|
try {
|
|
194
|
-
|
|
221
|
+
// Only the payload that was released. By now delivery may have
|
|
222
|
+
// throttled the task and parked it again under the same id with a
|
|
223
|
+
// later send time; deleting by id alone removed that newer payload,
|
|
224
|
+
// and its ZSET entry was then dropped as canceled — the send was lost.
|
|
225
|
+
const sendAt = drizzleSql`${scheduledPayloads.payload}->>'scheduledAt'`;
|
|
226
|
+
await db.delete(scheduledPayloads).where(
|
|
227
|
+
or(
|
|
228
|
+
...dbCleanupIds.map((taskId) => {
|
|
229
|
+
const released = (payloadMap.get(taskId) as any)?.scheduledAt;
|
|
230
|
+
return and(
|
|
231
|
+
eq(scheduledPayloads.taskId, taskId),
|
|
232
|
+
typeof released === "string"
|
|
233
|
+
? drizzleSql`${sendAt} = ${released}`
|
|
234
|
+
: drizzleSql`${sendAt} IS NULL`,
|
|
235
|
+
);
|
|
236
|
+
}),
|
|
237
|
+
),
|
|
238
|
+
);
|
|
195
239
|
} catch (err) {
|
|
196
240
|
logger.error(
|
|
197
241
|
{ err },
|
|
@@ -212,52 +256,81 @@ export async function executeSchedulerPoll(
|
|
|
212
256
|
return false;
|
|
213
257
|
} finally {
|
|
214
258
|
if (acquired) {
|
|
215
|
-
|
|
216
|
-
const lua = `
|
|
217
|
-
if redis.call("get", KEYS[1]) == ARGV[1] then
|
|
218
|
-
return redis.call("del", KEYS[1])
|
|
219
|
-
else
|
|
220
|
-
return 0
|
|
221
|
-
end
|
|
222
|
-
`;
|
|
223
|
-
await redis.eval(lua, 1, lockKey, lockOwner).catch(() => {});
|
|
259
|
+
await redis.releaseLock(lockKey, lockOwner).catch(() => {});
|
|
224
260
|
}
|
|
225
261
|
}
|
|
226
262
|
}
|
|
227
263
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
264
|
+
/**
|
|
265
|
+
* Present while the scheduler ZSETs are known to be intact. Redis losing its
|
|
266
|
+
* data (a flush, a restart without persistence, failover to an empty replica)
|
|
267
|
+
* takes this key with it, which is how the poll loop notices.
|
|
268
|
+
*/
|
|
269
|
+
export const SCHEDULER_SENTINEL_KEY = "notif:scheduler:intact";
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Re-queues every scheduled send still waiting in Postgres. Each row keeps its
|
|
273
|
+
* payload, and with it the send time, so the ZSETs can be rebuilt exactly;
|
|
274
|
+
* a legacy row without a send time is released on the next poll.
|
|
275
|
+
*
|
|
276
|
+
* Only run when the sentinel is missing: a rebuilt member has no trace id and
|
|
277
|
+
* so never matches a surviving original, and the poll only de-duplicates
|
|
278
|
+
* copies that happen to be claimed together.
|
|
279
|
+
*/
|
|
280
|
+
export async function rebuildScheduledQueue(
|
|
281
|
+
redis: Redis,
|
|
282
|
+
db: any,
|
|
283
|
+
logger: Logger,
|
|
284
|
+
pageSize = 1000,
|
|
285
|
+
): Promise<number> {
|
|
286
|
+
let after: string | undefined;
|
|
287
|
+
let total = 0;
|
|
288
|
+
for (;;) {
|
|
289
|
+
const rows: { taskId: string; payload: any }[] = await db
|
|
290
|
+
.select({ taskId: scheduledPayloads.taskId, payload: scheduledPayloads.payload })
|
|
291
|
+
.from(scheduledPayloads)
|
|
292
|
+
.where(after === undefined ? undefined : gt(scheduledPayloads.taskId, after))
|
|
293
|
+
.orderBy(asc(scheduledPayloads.taskId))
|
|
294
|
+
.limit(pageSize);
|
|
295
|
+
if (rows.length === 0) break;
|
|
243
296
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
297
|
+
const pipeline = redis.pipeline();
|
|
298
|
+
for (const { taskId, payload } of rows) {
|
|
299
|
+
const sendAt = Date.parse(payload?.scheduledAt ?? "");
|
|
300
|
+
const member = JSON.stringify({ taskId, enrichedEventId: payload?.enrichedEventId });
|
|
301
|
+
const shard = parseInt(taskId.slice(-1), 16) || 0;
|
|
302
|
+
pipeline.zadd(
|
|
303
|
+
`notif:scheduled:zset:${shard}`,
|
|
304
|
+
"NX",
|
|
305
|
+
Number.isNaN(sendAt) ? Date.now() : sendAt,
|
|
306
|
+
member,
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
await pipeline.exec();
|
|
251
310
|
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
311
|
+
total += rows.length;
|
|
312
|
+
after = rows[rows.length - 1]!.taskId;
|
|
313
|
+
if (rows.length < pageSize) break;
|
|
314
|
+
}
|
|
315
|
+
// Last, so a crash part-way leaves the sentinel missing and the next poll retries.
|
|
316
|
+
await redis.set(SCHEDULER_SENTINEL_KEY, "1");
|
|
317
|
+
if (total > 0) logger.warn({ count: total }, "rebuilt scheduled queue from Postgres");
|
|
318
|
+
return total;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
export async function startSchedulerWorker() {
|
|
322
|
+
({ logger, redis, sql, db } = createWorkerRuntime(config, "scheduler"));
|
|
323
|
+
({ consumer, pendingScanner } = createStreamConsumer(
|
|
324
|
+
{ redis, logger },
|
|
325
|
+
{
|
|
326
|
+
stream: STREAMS.SCHEDULED,
|
|
327
|
+
group: CONSUMER_GROUPS.SCHEDULER,
|
|
328
|
+
consumer: uniqueConsumerId("scheduler"),
|
|
329
|
+
batchSize: config.WORKER_CONCURRENCY,
|
|
330
|
+
},
|
|
331
|
+
));
|
|
332
|
+
|
|
333
|
+
const outboundProducers = createPriorityProducers(redis.native, logger, "OUTBOUND");
|
|
261
334
|
|
|
262
335
|
// ─── Stage 4a: Scheduled notification processor ─────────────────────────────
|
|
263
336
|
//
|
|
@@ -278,6 +351,9 @@ export async function startSchedulerWorker() {
|
|
|
278
351
|
const pollLoop = async (): Promise<void> => {
|
|
279
352
|
if (!isPolling) return;
|
|
280
353
|
try {
|
|
354
|
+
if (!(await redis.native.exists(SCHEDULER_SENTINEL_KEY))) {
|
|
355
|
+
await rebuildScheduledQueue(redis.native, db, logger);
|
|
356
|
+
}
|
|
281
357
|
const hasMore = await executeSchedulerPoll(redis.native, outboundProducers, logger, db);
|
|
282
358
|
if (isPolling) {
|
|
283
359
|
pollTimeout = setTimeout(() => void pollLoop(), hasMore ? 0 : 5000);
|
|
@@ -302,18 +378,14 @@ export async function startSchedulerWorker() {
|
|
|
302
378
|
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
303
379
|
|
|
304
380
|
export async function stopSchedulerWorker(): Promise<void> {
|
|
305
|
-
logger?.info("shutdown initiated");
|
|
306
|
-
if (healthInterval) {
|
|
307
|
-
clearInterval(healthInterval);
|
|
308
|
-
healthInterval = null;
|
|
309
|
-
}
|
|
310
|
-
if (pollTimeout) {
|
|
311
|
-
clearTimeout(pollTimeout);
|
|
312
|
-
pollTimeout = null;
|
|
313
|
-
}
|
|
314
381
|
isPolling = false;
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
382
|
+
await shutdownWorker("scheduler", {
|
|
383
|
+
logger,
|
|
384
|
+
timers: [healthInterval, pollTimeout],
|
|
385
|
+
worker,
|
|
386
|
+
sql,
|
|
387
|
+
redis,
|
|
388
|
+
});
|
|
389
|
+
healthInterval = null;
|
|
390
|
+
pollTimeout = null;
|
|
319
391
|
}
|