notifkit 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-omGPIBny.mjs +89 -0
- package/dist/bulk-C4hTcnLu.mjs +77 -0
- package/dist/index.d.mts +1160 -8886
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-B-AZ-xoN.mjs} +49 -75
- package/dist/main-B5BwjIMu.mjs +811 -0
- package/dist/{main-B3iQrl3t.mjs → main-BU27VjDC.mjs} +149 -134
- package/dist/main-Bes11Ad3.mjs +312 -0
- package/dist/{main-BiFiOX9D.mjs → main-BtnGmyaT.mjs} +119 -112
- package/dist/{main-39DtKrX4.mjs → main-BxAeC_k6.mjs} +395 -184
- package/dist/{main-Dz2sUh3c.mjs → main-DAXO-MiH.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-DYGCl7Ja.mjs} +135 -79
- package/dist/{src-QiTu5e_a.mjs → src-CIPTzq9i.mjs} +1351 -661
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +21 -18
- package/scripts/create-admin.mjs +8 -1
- package/scripts/migrate.mjs +43 -0
- package/src/client.ts +15 -5
- package/src/config/index.ts +37 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +174 -7
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +64 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Redis } from "@/index.js";
|
|
2
2
|
import { LRUCache } from "@/shared/index.js";
|
|
3
|
+
import type { Logger } from "@/logger/index.js";
|
|
3
4
|
|
|
4
5
|
import { randomUUID } from "crypto";
|
|
5
6
|
|
|
@@ -125,6 +126,11 @@ export interface ThrottleCheckOptions {
|
|
|
125
126
|
windowHours?: number | null;
|
|
126
127
|
/** Future send time. The window is evaluated at that instant, not at now. */
|
|
127
128
|
scheduledAt?: string;
|
|
129
|
+
/**
|
|
130
|
+
* Stable id of the message being counted. A retry of the same message is
|
|
131
|
+
* then counted once, not once per attempt. Random when omitted.
|
|
132
|
+
*/
|
|
133
|
+
messageId?: string;
|
|
128
134
|
}
|
|
129
135
|
|
|
130
136
|
/** Reject stored values that would make the window meaningless. */
|
|
@@ -172,18 +178,7 @@ export class UserThrottle {
|
|
|
172
178
|
const key = `throttle:${projectId}:user:${userId}`;
|
|
173
179
|
const targetTime = options.scheduledAt ? new Date(options.scheduledAt).getTime() : Date.now();
|
|
174
180
|
const windowStart = targetTime - windowMs;
|
|
175
|
-
const memberId = randomUUID();
|
|
176
|
-
|
|
177
|
-
const LUA_THROTTLE = `
|
|
178
|
-
redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[1])
|
|
179
|
-
local count = redis.call("ZCARD", KEYS[1])
|
|
180
|
-
if tonumber(count) < tonumber(ARGV[2]) then
|
|
181
|
-
redis.call("ZADD", KEYS[1], tonumber(ARGV[3]), ARGV[4])
|
|
182
|
-
redis.call("EXPIRE", KEYS[1], tonumber(ARGV[5]))
|
|
183
|
-
return tonumber(count) + 1
|
|
184
|
-
end
|
|
185
|
-
return tonumber(count) + 1
|
|
186
|
-
`;
|
|
181
|
+
const memberId = options.messageId ?? randomUUID();
|
|
187
182
|
|
|
188
183
|
// The key must outlive the window it is counting. For a future-dated send
|
|
189
184
|
// that means surviving until targetTime plus one more window, so a task
|
|
@@ -194,17 +189,223 @@ export class UserThrottle {
|
|
|
194
189
|
Math.ceil((targetTime - Date.now()) / 1000) + windowSeconds,
|
|
195
190
|
);
|
|
196
191
|
|
|
197
|
-
const count =
|
|
198
|
-
LUA_THROTTLE,
|
|
199
|
-
1,
|
|
192
|
+
const count = await this.redis.throttleUser(
|
|
200
193
|
key,
|
|
201
194
|
windowStart,
|
|
202
195
|
limit,
|
|
203
196
|
targetTime,
|
|
204
197
|
memberId,
|
|
205
198
|
ttlSeconds,
|
|
206
|
-
|
|
199
|
+
Date.now() - windowMs,
|
|
200
|
+
);
|
|
207
201
|
|
|
208
202
|
return { allowed: count <= limit, count, limit };
|
|
209
203
|
}
|
|
210
204
|
}
|
|
205
|
+
|
|
206
|
+
// ─── ApiRateLimiter (In-Memory Token Leasing) ──────────────────────────────
|
|
207
|
+
// Leases token chunks from Redis sliding-window counters into local memory.
|
|
208
|
+
// Eliminates per-request Redis network roundtrips on API ingress.
|
|
209
|
+
|
|
210
|
+
export interface ApiRateLimiterOptions {
|
|
211
|
+
redis: Redis;
|
|
212
|
+
windowMs?: number;
|
|
213
|
+
maxBatchSize?: number;
|
|
214
|
+
logger?: Logger;
|
|
215
|
+
/** Redis key namespace. Distinct limiters must not share one. */
|
|
216
|
+
keyPrefix?: string;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Computes dynamic token lease batch size based on RPM.
|
|
221
|
+
* Targets ~5s of limit capacity so Redis is contacted at most ~12 times/min
|
|
222
|
+
* regardless of instantaneous request rate. Bounded between 1 and maxBatchSize.
|
|
223
|
+
*/
|
|
224
|
+
export function calculateTokenBatchSize(limitRpm: number, maxBatchSize = 500): number {
|
|
225
|
+
if (limitRpm <= 0) return 0;
|
|
226
|
+
if (limitRpm <= 10) return 1;
|
|
227
|
+
if (limitRpm <= 60) return 5;
|
|
228
|
+
const perSecond = limitRpm / 60;
|
|
229
|
+
// 5 s window = perSecond * 5; at least 10 to absorb jitter.
|
|
230
|
+
return Math.min(Math.max(Math.ceil(perSecond * 5), 10), maxBatchSize);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
interface ProjectTokenLeaseState {
|
|
234
|
+
bucketIndex: number;
|
|
235
|
+
tokens: number;
|
|
236
|
+
batchSize: number;
|
|
237
|
+
inFlightLease: Promise<number> | null;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export class ApiRateLimiter {
|
|
241
|
+
private readonly redis: Redis;
|
|
242
|
+
private readonly windowMs: number;
|
|
243
|
+
private readonly maxBatchSize: number;
|
|
244
|
+
private readonly logger?: Logger;
|
|
245
|
+
private readonly keyPrefix: string;
|
|
246
|
+
private readonly leases = new Map<string, ProjectTokenLeaseState>();
|
|
247
|
+
|
|
248
|
+
constructor({
|
|
249
|
+
redis,
|
|
250
|
+
windowMs = 60_000,
|
|
251
|
+
maxBatchSize = 500,
|
|
252
|
+
logger,
|
|
253
|
+
keyPrefix = "rate-limit:api:req",
|
|
254
|
+
}: ApiRateLimiterOptions) {
|
|
255
|
+
this.redis = redis;
|
|
256
|
+
this.windowMs = windowMs;
|
|
257
|
+
this.maxBatchSize = maxBatchSize;
|
|
258
|
+
this.logger = logger;
|
|
259
|
+
this.keyPrefix = keyPrefix;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Evaluates if a request for `projectId` is allowed within its `limitRpm`.
|
|
264
|
+
* Fast-path uses local in-memory token lease (0ms Redis network latency).
|
|
265
|
+
* Refills in the background before tokens are depleted.
|
|
266
|
+
*/
|
|
267
|
+
async check(projectId: string, limitRpm: number): Promise<boolean> {
|
|
268
|
+
if (limitRpm <= 0) return false;
|
|
269
|
+
|
|
270
|
+
const nowMs = Date.now();
|
|
271
|
+
const currentBucket = Math.floor(nowMs / this.windowMs);
|
|
272
|
+
|
|
273
|
+
let state = this.leases.get(projectId);
|
|
274
|
+
if (!state) {
|
|
275
|
+
state = {
|
|
276
|
+
bucketIndex: currentBucket,
|
|
277
|
+
tokens: 0,
|
|
278
|
+
batchSize: calculateTokenBatchSize(limitRpm, this.maxBatchSize),
|
|
279
|
+
inFlightLease: null,
|
|
280
|
+
};
|
|
281
|
+
this.leases.set(projectId, state);
|
|
282
|
+
this.maybePrune(currentBucket);
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Rollover to new bucket window when minute boundary elapses.
|
|
286
|
+
// Kick off a prefill immediately so the first request of the new window
|
|
287
|
+
// doesn't have to wait — it will land before tokens hit 0 again.
|
|
288
|
+
if (state.bucketIndex !== currentBucket) {
|
|
289
|
+
state.bucketIndex = currentBucket;
|
|
290
|
+
state.tokens = 0;
|
|
291
|
+
state.batchSize = calculateTokenBatchSize(limitRpm, this.maxBatchSize);
|
|
292
|
+
state.inFlightLease = null;
|
|
293
|
+
// Fire the first lease for the new window now, don't await it here.
|
|
294
|
+
this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {});
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// 1. Fast Path: Local token available in memory (0ms network latency)
|
|
298
|
+
if (state.tokens > 0) {
|
|
299
|
+
state.tokens--;
|
|
300
|
+
|
|
301
|
+
// Background refill when tokens drop below 25% of batch size
|
|
302
|
+
const refillThreshold = Math.max(1, Math.floor(state.batchSize * 0.25));
|
|
303
|
+
if (state.tokens <= refillThreshold && !state.inFlightLease) {
|
|
304
|
+
this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {});
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
return true;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
// 2. Slow Path: tokens are depleted, block on the in-flight lease or start one.
|
|
311
|
+
// Concurrent requests racing here all join the *same* promise so one Redis
|
|
312
|
+
// round trip serves them. A lease only grants `batchSize` tokens, so a
|
|
313
|
+
// burst larger than that leases again rather than rejecting requests the
|
|
314
|
+
// window still has room for: a request is refused only once Redis itself
|
|
315
|
+
// reports the window spent.
|
|
316
|
+
for (;;) {
|
|
317
|
+
if (!state.inFlightLease) {
|
|
318
|
+
this.refill(projectId, state, limitRpm, nowMs, currentBucket).catch(() => {});
|
|
319
|
+
}
|
|
320
|
+
const granted = state.inFlightLease ? await state.inFlightLease : 0;
|
|
321
|
+
|
|
322
|
+
// Only claim a token if the lease came back for the same bucket we waited on.
|
|
323
|
+
if (state.bucketIndex !== currentBucket) return false;
|
|
324
|
+
if (state.tokens > 0) {
|
|
325
|
+
state.tokens--;
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
if (granted <= 0) return false;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
private async refill(
|
|
333
|
+
projectId: string,
|
|
334
|
+
state: ProjectTokenLeaseState,
|
|
335
|
+
limitRpm: number,
|
|
336
|
+
nowMs: number,
|
|
337
|
+
bucketIndex: number,
|
|
338
|
+
): Promise<number> {
|
|
339
|
+
if (state.inFlightLease) {
|
|
340
|
+
return state.inFlightLease;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const prevBucket = bucketIndex - 1;
|
|
344
|
+
const rlTag = `{${this.keyPrefix}:${projectId || "global"}}`;
|
|
345
|
+
const currentKey = `${rlTag}:${bucketIndex}`;
|
|
346
|
+
const prevKey = `${rlTag}:${prevBucket}`;
|
|
347
|
+
const requested = state.batchSize;
|
|
348
|
+
let inFlightRef: Promise<number> | null = null;
|
|
349
|
+
const executeLease = async (): Promise<number> => {
|
|
350
|
+
try {
|
|
351
|
+
let granted = await this.redis.leaseApiRateLimit(
|
|
352
|
+
currentKey,
|
|
353
|
+
prevKey,
|
|
354
|
+
nowMs,
|
|
355
|
+
this.windowMs,
|
|
356
|
+
limitRpm,
|
|
357
|
+
requested,
|
|
358
|
+
);
|
|
359
|
+
|
|
360
|
+
// On invalid return or negative, fail-open
|
|
361
|
+
if (typeof granted !== "number" || granted < 0) {
|
|
362
|
+
granted = requested;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
if (state.bucketIndex === bucketIndex) {
|
|
366
|
+
state.tokens += granted;
|
|
367
|
+
}
|
|
368
|
+
return granted;
|
|
369
|
+
} catch (err) {
|
|
370
|
+
this.logger?.warn(
|
|
371
|
+
{ err, projectId },
|
|
372
|
+
"project rate limit lease failed — allowing request (fail-open)",
|
|
373
|
+
);
|
|
374
|
+
// Fail-open: credit requested tokens locally to prevent tight retry loops during Redis hiccups
|
|
375
|
+
if (state.bucketIndex === bucketIndex) {
|
|
376
|
+
state.tokens += requested;
|
|
377
|
+
}
|
|
378
|
+
return requested;
|
|
379
|
+
} finally {
|
|
380
|
+
if (state.inFlightLease === inFlightRef) {
|
|
381
|
+
state.inFlightLease = null;
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
const promise = executeLease();
|
|
387
|
+
inFlightRef = promise;
|
|
388
|
+
state.inFlightLease = promise;
|
|
389
|
+
return promise;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
private maybePrune(currentBucket: number): void {
|
|
393
|
+
if (this.leases.size > 2000) {
|
|
394
|
+
for (const [id, s] of this.leases) {
|
|
395
|
+
if (currentBucket - s.bucketIndex > 1) {
|
|
396
|
+
this.leases.delete(id);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** Reset all local cached leases */
|
|
403
|
+
clear(): void {
|
|
404
|
+
this.leases.clear();
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/** Invalidate local lease for a specific project */
|
|
408
|
+
invalidate(projectId: string): void {
|
|
409
|
+
this.leases.delete(projectId);
|
|
410
|
+
}
|
|
411
|
+
}
|
package/src/redis/index.ts
CHANGED
|
@@ -1,6 +1,225 @@
|
|
|
1
1
|
import { Redis, type RedisOptions } from "ioredis";
|
|
2
2
|
import type { Logger } from "@/index.js";
|
|
3
3
|
|
|
4
|
+
declare module "ioredis" {
|
|
5
|
+
interface Redis {
|
|
6
|
+
leaseApiRateLimit(
|
|
7
|
+
currentKey: string,
|
|
8
|
+
prevKey: string,
|
|
9
|
+
now: number | string,
|
|
10
|
+
window: number | string,
|
|
11
|
+
maxReqs: number | string,
|
|
12
|
+
requested: number | string,
|
|
13
|
+
ttl?: number | string,
|
|
14
|
+
): Promise<number>;
|
|
15
|
+
releaseLock(lockKey: string, lockToken: string): Promise<number>;
|
|
16
|
+
renewLock(lockKey: string, lockToken: string, ttlSeconds: number | string): Promise<number>;
|
|
17
|
+
throttleUser(
|
|
18
|
+
key: string,
|
|
19
|
+
windowStart: number | string,
|
|
20
|
+
limit: number | string,
|
|
21
|
+
targetTime: number | string,
|
|
22
|
+
memberId: string,
|
|
23
|
+
ttlSeconds: number | string,
|
|
24
|
+
pruneBefore?: number | string,
|
|
25
|
+
): Promise<number>;
|
|
26
|
+
schedulerPoll(
|
|
27
|
+
key: string,
|
|
28
|
+
maxScore: number | string,
|
|
29
|
+
limit: number | string,
|
|
30
|
+
visibilityTimeout: number | string,
|
|
31
|
+
): Promise<string[]>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface ChainableCommander {
|
|
35
|
+
schedulerPoll(
|
|
36
|
+
key: string,
|
|
37
|
+
maxScore: number | string,
|
|
38
|
+
limit: number | string,
|
|
39
|
+
visibilityTimeout: number | string,
|
|
40
|
+
): this;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Sliding-window token leasing for high-throughput API rate limiting.
|
|
46
|
+
* Atomically checks available capacity and reserves up to N tokens in one round trip.
|
|
47
|
+
*/
|
|
48
|
+
export const LUA_LEASE_API_RATE_LIMIT = `
|
|
49
|
+
local currentKey = KEYS[1]
|
|
50
|
+
local prevKey = KEYS[2]
|
|
51
|
+
local now = tonumber(ARGV[1])
|
|
52
|
+
local window = tonumber(ARGV[2])
|
|
53
|
+
local maxReqs = tonumber(ARGV[3])
|
|
54
|
+
local requested = tonumber(ARGV[4]) or 1
|
|
55
|
+
local ttl = tonumber(ARGV[5]) or (math.ceil((window * 2) / 1000) + 60)
|
|
56
|
+
|
|
57
|
+
if not now or not window or window <= 0 then
|
|
58
|
+
return -1
|
|
59
|
+
end
|
|
60
|
+
if not maxReqs or maxReqs <= 0 then
|
|
61
|
+
return 0
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
if not prevKey then
|
|
65
|
+
local currentBucket = math.floor(now / window)
|
|
66
|
+
local prevBucket = currentBucket - 1
|
|
67
|
+
currentKey = KEYS[1] .. ":" .. currentBucket
|
|
68
|
+
prevKey = KEYS[1] .. ":" .. prevBucket
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
local currentCount = tonumber(redis.call("GET", currentKey) or "0")
|
|
72
|
+
local prevCount = tonumber(redis.call("GET", prevKey) or "0")
|
|
73
|
+
|
|
74
|
+
local timeIntoCurrent = now % window
|
|
75
|
+
local weight = (window - timeIntoCurrent) / window
|
|
76
|
+
local estimated = math.floor(prevCount * weight + currentCount)
|
|
77
|
+
local available = maxReqs - estimated
|
|
78
|
+
|
|
79
|
+
if available <= 0 then
|
|
80
|
+
return 0
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
local toGrant = math.min(requested, available)
|
|
84
|
+
local newCount = redis.call("INCRBY", currentKey, toGrant)
|
|
85
|
+
if newCount == toGrant then
|
|
86
|
+
redis.call("EXPIRE", currentKey, ttl)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return toGrant
|
|
90
|
+
`;
|
|
91
|
+
|
|
92
|
+
/** Release a lock only if we still hold it (value matches our token). */
|
|
93
|
+
export const LUA_RELEASE_LOCK = `
|
|
94
|
+
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
95
|
+
return redis.call('DEL', KEYS[1])
|
|
96
|
+
end
|
|
97
|
+
return 0
|
|
98
|
+
`;
|
|
99
|
+
|
|
100
|
+
/** Extend a lock's TTL only if we still hold it. */
|
|
101
|
+
export const LUA_RENEW_LOCK = `
|
|
102
|
+
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
103
|
+
return redis.call('EXPIRE', KEYS[1], ARGV[2])
|
|
104
|
+
end
|
|
105
|
+
return 0
|
|
106
|
+
`;
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* User-level sliding window notification throttle.
|
|
110
|
+
*
|
|
111
|
+
* Members are sends scored by when they go out, which for a scheduled send is
|
|
112
|
+
* in the future. A send at targetTime is checked against the window ending at
|
|
113
|
+
* it, (windowStart, targetTime], so sends scheduled later do not count against
|
|
114
|
+
* it. Pruning uses pruneBefore (now minus one window), never windowStart: a
|
|
115
|
+
* check for a far-future send must not delete sends that went out today.
|
|
116
|
+
*
|
|
117
|
+
* A member already present was counted by an earlier attempt at the same
|
|
118
|
+
* message (a replay after a crash), so it is allowed again without counting
|
|
119
|
+
* twice. Returns 0 in that case, otherwise the would-be count.
|
|
120
|
+
*
|
|
121
|
+
* ARGV: windowStart, limit, targetTime, member, ttlSeconds, pruneBefore
|
|
122
|
+
*/
|
|
123
|
+
export const LUA_USER_THROTTLE = `
|
|
124
|
+
local limit = tonumber(ARGV[2])
|
|
125
|
+
local targetTime = tonumber(ARGV[3])
|
|
126
|
+
local ttl = tonumber(ARGV[5])
|
|
127
|
+
redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", ARGV[6] or ARGV[1])
|
|
128
|
+
if redis.call("ZSCORE", KEYS[1], ARGV[4]) then
|
|
129
|
+
return 0
|
|
130
|
+
end
|
|
131
|
+
local count = redis.call("ZCOUNT", KEYS[1], "(" .. ARGV[1], targetTime)
|
|
132
|
+
if count < limit then
|
|
133
|
+
redis.call("ZADD", KEYS[1], targetTime, ARGV[4])
|
|
134
|
+
-- Only ever extend: a send today must not cut short the life of a key
|
|
135
|
+
-- that is still holding a send scheduled for next week.
|
|
136
|
+
if redis.call("TTL", KEYS[1]) < ttl then
|
|
137
|
+
redis.call("EXPIRE", KEYS[1], ttl)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
return count + 1
|
|
141
|
+
`;
|
|
142
|
+
|
|
143
|
+
/** Polls scheduler zsets with visibility timeouts. */
|
|
144
|
+
export const LUA_SCHEDULER_POLL = `
|
|
145
|
+
local key = KEYS[1]
|
|
146
|
+
local maxScore = tonumber(ARGV[1])
|
|
147
|
+
local limit = tonumber(ARGV[2])
|
|
148
|
+
local visibilityTimeout = tonumber(ARGV[3]) or 0
|
|
149
|
+
local tasks = redis.call('ZRANGE', key, 0, maxScore, 'BYSCORE', 'LIMIT', 0, limit)
|
|
150
|
+
if #tasks > 0 then
|
|
151
|
+
for i, task in ipairs(tasks) do
|
|
152
|
+
redis.call('ZADD', key, maxScore + visibilityTimeout, task)
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
return tasks
|
|
156
|
+
`;
|
|
157
|
+
|
|
158
|
+
/** Two-phase idempotency lease acquisition. Checks sent/legacy, then attempts SET NX lock. */
|
|
159
|
+
export const LUA_ACQUIRE_LEASE = `
|
|
160
|
+
if redis.call('EXISTS', KEYS[1]) == 1 or redis.call('EXISTS', KEYS[3]) == 1 then
|
|
161
|
+
return 'completed'
|
|
162
|
+
end
|
|
163
|
+
local lockAcquired = redis.call('SET', KEYS[2], '1', 'EX', ARGV[1], 'NX')
|
|
164
|
+
if lockAcquired then
|
|
165
|
+
return 'acquired'
|
|
166
|
+
end
|
|
167
|
+
if redis.call('EXISTS', KEYS[1]) == 1 or redis.call('EXISTS', KEYS[3]) == 1 then
|
|
168
|
+
return 'completed'
|
|
169
|
+
end
|
|
170
|
+
return 'locked'
|
|
171
|
+
`;
|
|
172
|
+
|
|
173
|
+
/** Two-phase idempotency completion marker. Atomically sets completed + legacy and deletes lock. */
|
|
174
|
+
export const LUA_MARK_PROCESSED = `
|
|
175
|
+
redis.call('SET', KEYS[1], '1', 'EX', ARGV[1])
|
|
176
|
+
redis.call('SET', KEYS[2], '1', 'EX', ARGV[1])
|
|
177
|
+
redis.call('DEL', KEYS[3])
|
|
178
|
+
return 1
|
|
179
|
+
`;
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Registers pre-compiled custom commands on an ioredis instance.
|
|
183
|
+
* Calls defineCommand so ioredis uses EVALSHA rather than re-transmitting
|
|
184
|
+
* full script strings over the wire.
|
|
185
|
+
*/
|
|
186
|
+
export function registerCustomCommands(redis: Redis): void {
|
|
187
|
+
redis.defineCommand("leaseApiRateLimit", {
|
|
188
|
+
numberOfKeys: 2,
|
|
189
|
+
lua: LUA_LEASE_API_RATE_LIMIT,
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
redis.defineCommand("releaseLock", {
|
|
193
|
+
numberOfKeys: 1,
|
|
194
|
+
lua: LUA_RELEASE_LOCK,
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
redis.defineCommand("renewLock", {
|
|
198
|
+
numberOfKeys: 1,
|
|
199
|
+
lua: LUA_RENEW_LOCK,
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
redis.defineCommand("throttleUser", {
|
|
203
|
+
numberOfKeys: 1,
|
|
204
|
+
lua: LUA_USER_THROTTLE,
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
redis.defineCommand("schedulerPoll", {
|
|
208
|
+
numberOfKeys: 1,
|
|
209
|
+
lua: LUA_SCHEDULER_POLL,
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
redis.defineCommand("acquireLease", {
|
|
213
|
+
numberOfKeys: 3,
|
|
214
|
+
lua: LUA_ACQUIRE_LEASE,
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
redis.defineCommand("markProcessed", {
|
|
218
|
+
numberOfKeys: 3,
|
|
219
|
+
lua: LUA_MARK_PROCESSED,
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
|
|
4
223
|
export interface RedisClientOptions {
|
|
5
224
|
url: string;
|
|
6
225
|
name?: string;
|
|
@@ -8,9 +227,47 @@ export interface RedisClientOptions {
|
|
|
8
227
|
redisOptions?: Partial<RedisOptions>;
|
|
9
228
|
}
|
|
10
229
|
|
|
230
|
+
const sharedClients = new Map<string, { client: RedisClient; refs: number }>();
|
|
231
|
+
|
|
11
232
|
export class RedisClient {
|
|
12
233
|
readonly native: Redis;
|
|
13
234
|
|
|
235
|
+
/**
|
|
236
|
+
* One command connection per Redis URL for every service in this process.
|
|
237
|
+
*
|
|
238
|
+
* Each service used to open its own, so a single notification's commands
|
|
239
|
+
* reached Redis over three or four sockets, one round trip each. Sharing a
|
|
240
|
+
* connection lets auto-pipelining merge whatever the services issue in the
|
|
241
|
+
* same tick into one write, and Redis spends most of its time per round trip
|
|
242
|
+
* rather than per command. Blocking reads and subscribers still duplicate
|
|
243
|
+
* their own connections.
|
|
244
|
+
*
|
|
245
|
+
* The handle's `disconnect()` releases this caller's reference; the
|
|
246
|
+
* connection closes when the last service lets go.
|
|
247
|
+
*/
|
|
248
|
+
static shared(options: RedisClientOptions): RedisClient {
|
|
249
|
+
let entry = sharedClients.get(options.url);
|
|
250
|
+
if (!entry) {
|
|
251
|
+
entry = { client: new RedisClient({ ...options, name: "notifkit" }), refs: 0 };
|
|
252
|
+
sharedClients.set(options.url, entry);
|
|
253
|
+
}
|
|
254
|
+
entry.refs++;
|
|
255
|
+
|
|
256
|
+
const owner = entry;
|
|
257
|
+
let released = false;
|
|
258
|
+
const handle = Object.create(owner.client) as RedisClient;
|
|
259
|
+
handle.disconnect = async () => {
|
|
260
|
+
if (released) return;
|
|
261
|
+
released = true;
|
|
262
|
+
owner.refs--;
|
|
263
|
+
if (owner.refs === 0) {
|
|
264
|
+
if (sharedClients.get(options.url) === owner) sharedClients.delete(options.url);
|
|
265
|
+
await owner.client.disconnect();
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
return handle;
|
|
269
|
+
}
|
|
270
|
+
|
|
14
271
|
private readonly logger?: Logger;
|
|
15
272
|
private isClosing = false;
|
|
16
273
|
|
|
@@ -22,9 +279,17 @@ export class RedisClient {
|
|
|
22
279
|
enableReadyCheck: true,
|
|
23
280
|
lazyConnect: false,
|
|
24
281
|
connectionName: name,
|
|
282
|
+
// Every worker keeps hundreds of commands in flight (idempotency SETs,
|
|
283
|
+
// throttle scripts, acks). Coalescing the ones issued in the same tick
|
|
284
|
+
// into one write cuts syscalls and Redis read events by an order of
|
|
285
|
+
// magnitude under load. Blocking and subscriber connections opt out
|
|
286
|
+
// when they duplicate this client.
|
|
287
|
+
enableAutoPipelining: true,
|
|
25
288
|
...redisOptions,
|
|
26
289
|
});
|
|
27
290
|
|
|
291
|
+
registerCustomCommands(this.native);
|
|
292
|
+
|
|
28
293
|
this.native.on("connect", () => {
|
|
29
294
|
this.logger?.info({ url: redactUrl(url) }, "redis connected");
|
|
30
295
|
});
|