notifkit 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/dist/assets/{index-7mvXb4bS.js → index-Bsrmo9LE.js} +1 -1
- package/dashboard/dist/assets/index-s-SXGIwv.css +1 -0
- package/dashboard/dist/index.html +2 -2
- package/dist/bootstrap-GRXzgPvW.mjs +89 -0
- package/dist/bulk-BvRfz6Of.mjs +77 -0
- package/dist/index.d.mts +1100 -8883
- package/dist/index.mjs +2 -38
- package/dist/{main-C9s-2ArH.mjs → main-BENOY7HG.mjs} +49 -75
- package/dist/{main-BiFiOX9D.mjs → main-BpnuFX62.mjs} +119 -112
- package/dist/{main-Dz2sUh3c.mjs → main-BqVH2BHj.mjs} +57 -54
- package/dist/{main-BgmT1MPe.mjs → main-C_5VFIwp.mjs} +135 -79
- package/dist/main-D6od9N8h.mjs +312 -0
- package/dist/main-D84OX0ux.mjs +811 -0
- package/dist/{main-39DtKrX4.mjs → main-HH-WwsMM.mjs} +395 -184
- package/dist/{main-B3iQrl3t.mjs → main-HY_TDs61.mjs} +149 -134
- package/dist/{src-QiTu5e_a.mjs → src-CI7FGf3h.mjs} +1220 -651
- package/drizzle/0006_message_logs_redundant_indexes.sql +7 -0
- package/drizzle/0007_workflow_status_canceled.sql +3 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +17 -17
- package/src/client.ts +15 -5
- package/src/config/index.ts +26 -3
- package/src/contracts/events/notification-ai-pending.ts +12 -0
- package/src/contracts/events/notification-delivered.ts +7 -0
- package/src/contracts/events/notification-dispatched.ts +21 -2
- package/src/contracts/events/notification-enriched.ts +16 -0
- package/src/contracts/events/notification-failed.ts +2 -0
- package/src/contracts/events/notification-requested.ts +2 -0
- package/src/contracts/registry.ts +2 -2
- package/src/contracts/sdk.ts +77 -29
- package/src/db/bulk.ts +132 -0
- package/src/db/index.ts +2 -2
- package/src/db/schema.ts +3 -50
- package/src/idempotency/index.ts +60 -6
- package/src/logger/index.ts +0 -12
- package/src/queue/index.ts +255 -76
- package/src/rate-limiter/index.ts +217 -16
- package/src/redis/index.ts +265 -0
- package/src/repositories/index.ts +171 -82
- package/src/server.ts +61 -41
- package/src/services/ai/main.ts +83 -70
- package/src/services/api/admin-static.ts +41 -12
- package/src/services/api/handlers.ts +357 -159
- package/src/services/api/http.ts +34 -7
- package/src/services/api/main.ts +104 -67
- package/src/services/api/router.ts +18 -5
- package/src/services/delivery/main.ts +495 -230
- package/src/services/delivery/throttle.ts +42 -59
- package/src/services/engine/main.ts +206 -152
- package/src/services/enricher/main.ts +155 -107
- package/src/services/events/main.ts +100 -55
- package/src/services/scheduler/main.ts +151 -79
- package/src/services/scheduler/send-gate.ts +117 -0
- package/src/services/workflow/main.ts +220 -100
- package/src/shared/batch-processor.ts +65 -34
- package/src/shared/dataloader.ts +58 -22
- package/src/shared/events.ts +3 -0
- package/src/shared/index.ts +0 -1
- package/src/shared/utils.ts +2 -47
- package/src/templates/index.ts +0 -67
- package/src/transport/index.ts +6 -0
- package/src/workers/bootstrap.ts +124 -0
- package/src/workers/index.ts +99 -24
- package/src/workflows/sdk.ts +2 -0
- package/dashboard/dist/assets/index-o7L-f3-c.css +0 -1
- package/dist/index.d.mts.map +0 -1
- package/dist/index.mjs.map +0 -1
- package/dist/main-39DtKrX4.mjs.map +0 -1
- package/dist/main-B3iQrl3t.mjs.map +0 -1
- package/dist/main-BM8DH3v3.mjs +0 -632
- package/dist/main-BM8DH3v3.mjs.map +0 -1
- package/dist/main-BgmT1MPe.mjs.map +0 -1
- package/dist/main-BiFiOX9D.mjs.map +0 -1
- package/dist/main-C9s-2ArH.mjs.map +0 -1
- package/dist/main-Cqo3rSb5.mjs +0 -224
- package/dist/main-Cqo3rSb5.mjs.map +0 -1
- package/dist/main-Dz2sUh3c.mjs.map +0 -1
- package/dist/src-QiTu5e_a.mjs.map +0 -1
|
@@ -4,64 +4,95 @@ export interface BatchItem<T, R> {
|
|
|
4
4
|
reject: (reason?: any) => void;
|
|
5
5
|
}
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Coalesces individual `add()` calls into batched `flushFn` calls.
|
|
9
|
+
*
|
|
10
|
+
* A batch goes out when `maxSize` items are waiting or `maxWaitMs` has passed
|
|
11
|
+
* since the first one arrived. Up to `maxConcurrent` flushes may be in flight
|
|
12
|
+
* at once: with a single flight, every caller queued behind a slow flush holds
|
|
13
|
+
* its worker slot for the whole round trip, which caps a worker's throughput at
|
|
14
|
+
* `maxSize / flushLatency` no matter how much concurrency it was given.
|
|
15
|
+
*/
|
|
7
16
|
export class BatchProcessor<T, R = void> {
|
|
8
17
|
private buffer: BatchItem<T, R>[] = [];
|
|
9
18
|
private timer: NodeJS.Timeout | null = null;
|
|
10
|
-
private
|
|
19
|
+
private readonly inFlight = new Set<Promise<void>>();
|
|
11
20
|
|
|
12
21
|
constructor(
|
|
13
22
|
private readonly maxSize: number,
|
|
14
23
|
private readonly maxWaitMs: number,
|
|
15
24
|
private readonly flushFn: (items: T[]) => Promise<R[]>,
|
|
25
|
+
private readonly maxConcurrent: number = 1,
|
|
16
26
|
) {}
|
|
17
27
|
|
|
18
28
|
add(item: T): Promise<R> {
|
|
19
29
|
return new Promise((resolve, reject) => {
|
|
20
30
|
this.buffer.push({ item, resolve, reject });
|
|
21
|
-
if (this.buffer.length >= this.maxSize &&
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
void this.flush();
|
|
27
|
-
} else if (!this.timer && !this.isFlushing) {
|
|
28
|
-
this.timer = setTimeout(() => {
|
|
29
|
-
this.timer = null;
|
|
30
|
-
void this.flush();
|
|
31
|
-
}, this.maxWaitMs);
|
|
31
|
+
if (this.buffer.length >= this.maxSize && this.inFlight.size < this.maxConcurrent) {
|
|
32
|
+
this.clearTimer();
|
|
33
|
+
void this.flushOnce();
|
|
34
|
+
} else if (!this.timer && this.inFlight.size < this.maxConcurrent) {
|
|
35
|
+
this.armTimer();
|
|
32
36
|
}
|
|
33
37
|
});
|
|
34
38
|
}
|
|
35
39
|
|
|
40
|
+
/** Flushes everything buffered, waiting for a free flush slot if needed. */
|
|
36
41
|
async flush(): Promise<void> {
|
|
37
|
-
|
|
42
|
+
while (this.buffer.length > 0) {
|
|
43
|
+
while (this.inFlight.size >= this.maxConcurrent) {
|
|
44
|
+
await Promise.race(this.inFlight);
|
|
45
|
+
}
|
|
46
|
+
this.clearTimer();
|
|
47
|
+
await this.flushOnce();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
private flushOnce(): Promise<void> {
|
|
52
|
+
if (this.buffer.length === 0 || this.inFlight.size >= this.maxConcurrent) {
|
|
53
|
+
return Promise.resolve();
|
|
54
|
+
}
|
|
38
55
|
|
|
39
|
-
this.isFlushing = true;
|
|
40
56
|
const batch = this.buffer;
|
|
41
57
|
this.buffer = [];
|
|
42
58
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
} catch (err) {
|
|
50
|
-
for (const b of batch) {
|
|
51
|
-
b.reject(err);
|
|
52
|
-
}
|
|
53
|
-
} finally {
|
|
54
|
-
this.isFlushing = false;
|
|
55
|
-
if (this.buffer.length > 0 && !this.timer) {
|
|
56
|
-
if (this.buffer.length >= this.maxSize) {
|
|
57
|
-
void this.flush();
|
|
58
|
-
} else {
|
|
59
|
-
this.timer = setTimeout(() => {
|
|
60
|
-
this.timer = null;
|
|
61
|
-
void this.flush();
|
|
62
|
-
}, this.maxWaitMs);
|
|
59
|
+
const run = (async () => {
|
|
60
|
+
try {
|
|
61
|
+
const results = await this.flushFn(batch.map((b) => b.item));
|
|
62
|
+
for (let i = 0; i < batch.length; i++) {
|
|
63
|
+
batch[i]!.resolve(results[i] as R);
|
|
63
64
|
}
|
|
65
|
+
} catch (err) {
|
|
66
|
+
for (const b of batch) {
|
|
67
|
+
b.reject(err);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
})();
|
|
71
|
+
|
|
72
|
+
this.inFlight.add(run);
|
|
73
|
+
return run.finally(() => {
|
|
74
|
+
this.inFlight.delete(run);
|
|
75
|
+
// Items that arrived while every slot was busy were not given a timer.
|
|
76
|
+
if (this.buffer.length === 0 || this.timer) return;
|
|
77
|
+
if (this.buffer.length >= this.maxSize) {
|
|
78
|
+
void this.flushOnce();
|
|
79
|
+
} else {
|
|
80
|
+
this.armTimer();
|
|
64
81
|
}
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private armTimer(): void {
|
|
86
|
+
this.timer = setTimeout(() => {
|
|
87
|
+
this.timer = null;
|
|
88
|
+
void this.flushOnce();
|
|
89
|
+
}, this.maxWaitMs);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private clearTimer(): void {
|
|
93
|
+
if (this.timer) {
|
|
94
|
+
clearTimeout(this.timer);
|
|
95
|
+
this.timer = null;
|
|
65
96
|
}
|
|
66
97
|
}
|
|
67
98
|
}
|
package/src/shared/dataloader.ts
CHANGED
|
@@ -1,9 +1,30 @@
|
|
|
1
|
+
export interface DataLoaderOptions {
|
|
2
|
+
/**
|
|
3
|
+
* How long to collect keys before loading. 0 (the default) batches only the
|
|
4
|
+
* keys requested in the same microtask; callers that reach the loader after
|
|
5
|
+
* independent I/O rarely line up that closely, so a few milliseconds turns
|
|
6
|
+
* one query per message into one query per burst.
|
|
7
|
+
*/
|
|
8
|
+
batchDelayMs?: number;
|
|
9
|
+
/** Load immediately once this many keys are waiting. */
|
|
10
|
+
maxBatchSize?: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
1
13
|
export class DataLoader<K, V> {
|
|
2
14
|
private keys: K[] = [];
|
|
3
15
|
private promises: Array<{ resolve: (value: V | Error) => void }> = [];
|
|
4
|
-
private
|
|
16
|
+
private scheduled = false;
|
|
17
|
+
private timer: NodeJS.Timeout | null = null;
|
|
18
|
+
private readonly batchDelayMs: number;
|
|
19
|
+
private readonly maxBatchSize: number;
|
|
5
20
|
|
|
6
|
-
constructor(
|
|
21
|
+
constructor(
|
|
22
|
+
private readonly batchLoadFn: (keys: K[]) => Promise<(V | Error)[]>,
|
|
23
|
+
{ batchDelayMs = 0, maxBatchSize = Infinity }: DataLoaderOptions = {},
|
|
24
|
+
) {
|
|
25
|
+
this.batchDelayMs = batchDelayMs;
|
|
26
|
+
this.maxBatchSize = maxBatchSize;
|
|
27
|
+
}
|
|
7
28
|
|
|
8
29
|
load(key: K): Promise<V> {
|
|
9
30
|
return new Promise((resolve, reject) => {
|
|
@@ -15,27 +36,42 @@ export class DataLoader<K, V> {
|
|
|
15
36
|
},
|
|
16
37
|
});
|
|
17
38
|
|
|
18
|
-
if (
|
|
19
|
-
this.
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
this.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
.then((results) => {
|
|
28
|
-
for (let i = 0; i < currentPromises.length; i++) {
|
|
29
|
-
currentPromises[i]!.resolve(results[i] as V | Error);
|
|
30
|
-
}
|
|
31
|
-
})
|
|
32
|
-
.catch((err) => {
|
|
33
|
-
for (const p of currentPromises) {
|
|
34
|
-
p.resolve(err);
|
|
35
|
-
}
|
|
36
|
-
});
|
|
37
|
-
});
|
|
39
|
+
if (this.keys.length >= this.maxBatchSize) {
|
|
40
|
+
this.dispatch();
|
|
41
|
+
} else if (!this.scheduled) {
|
|
42
|
+
this.scheduled = true;
|
|
43
|
+
if (this.batchDelayMs > 0) {
|
|
44
|
+
this.timer = setTimeout(() => this.dispatch(), this.batchDelayMs);
|
|
45
|
+
} else {
|
|
46
|
+
void Promise.resolve().then(() => this.dispatch());
|
|
47
|
+
}
|
|
38
48
|
}
|
|
39
49
|
});
|
|
40
50
|
}
|
|
51
|
+
|
|
52
|
+
private dispatch(): void {
|
|
53
|
+
if (this.timer) {
|
|
54
|
+
clearTimeout(this.timer);
|
|
55
|
+
this.timer = null;
|
|
56
|
+
}
|
|
57
|
+
this.scheduled = false;
|
|
58
|
+
if (this.keys.length === 0) return;
|
|
59
|
+
|
|
60
|
+
const keysToLoad = this.keys;
|
|
61
|
+
const currentPromises = this.promises;
|
|
62
|
+
this.keys = [];
|
|
63
|
+
this.promises = [];
|
|
64
|
+
|
|
65
|
+
this.batchLoadFn(keysToLoad)
|
|
66
|
+
.then((results) => {
|
|
67
|
+
for (let i = 0; i < currentPromises.length; i++) {
|
|
68
|
+
currentPromises[i]!.resolve(results[i] as V | Error);
|
|
69
|
+
}
|
|
70
|
+
})
|
|
71
|
+
.catch((err) => {
|
|
72
|
+
for (const p of currentPromises) {
|
|
73
|
+
p.resolve(err);
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
}
|
|
41
77
|
}
|
package/src/shared/events.ts
CHANGED
package/src/shared/index.ts
CHANGED
|
@@ -33,7 +33,6 @@ export * from "./cache.js";
|
|
|
33
33
|
export * from "./utils.js";
|
|
34
34
|
export * from "./semaphore.js";
|
|
35
35
|
export * from "./batch-processor.js";
|
|
36
|
-
export * from "./cache.js";
|
|
37
36
|
export * from "./circuit-breaker.js";
|
|
38
37
|
export * from "./dataloader.js";
|
|
39
38
|
export { type WorkerOptions } from "@/workers/index.js";
|
package/src/shared/utils.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
export function getPriorityBucket(
|
|
2
|
-
priority: string | undefined,
|
|
3
|
-
): "critical" | "high" | "normal" | "low" {
|
|
1
|
+
export function getPriorityBucket(priority: string | undefined): "critical" | "normal" | "low" {
|
|
4
2
|
const p = priority || "normal";
|
|
5
|
-
return p === "critical"
|
|
3
|
+
return p === "critical" ? "critical" : p === "low" ? "low" : "normal";
|
|
6
4
|
}
|
|
7
5
|
|
|
8
6
|
/**
|
|
@@ -19,46 +17,3 @@ export function normaliseTarget(target: string): string {
|
|
|
19
17
|
const trimmed = target.trim();
|
|
20
18
|
return trimmed.includes("@") ? trimmed.toLowerCase() : trimmed;
|
|
21
19
|
}
|
|
22
|
-
|
|
23
|
-
export const LUA_SCHEDULER_POLL = `
|
|
24
|
-
local key = KEYS[1]
|
|
25
|
-
local maxScore = tonumber(ARGV[1])
|
|
26
|
-
local limit = tonumber(ARGV[2])
|
|
27
|
-
local visibilityTimeout = tonumber(ARGV[3]) or 0
|
|
28
|
-
local tasks = redis.call('ZRANGE', key, 0, maxScore, 'BYSCORE', 'LIMIT', 0, limit)
|
|
29
|
-
if #tasks > 0 then
|
|
30
|
-
for i, task in ipairs(tasks) do
|
|
31
|
-
redis.call('ZADD', key, maxScore + visibilityTimeout, task)
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
return tasks
|
|
35
|
-
`;
|
|
36
|
-
|
|
37
|
-
export const LUA_SCHEDULER_CLAIM = `
|
|
38
|
-
local payloadKey = KEYS[1]
|
|
39
|
-
local claimedKey = KEYS[2]
|
|
40
|
-
|
|
41
|
-
if redis.call('EXISTS', payloadKey) == 1 then
|
|
42
|
-
redis.call('RENAME', payloadKey, claimedKey)
|
|
43
|
-
return redis.call('GET', claimedKey)
|
|
44
|
-
elseif redis.call('EXISTS', claimedKey) == 1 then
|
|
45
|
-
return redis.call('GET', claimedKey)
|
|
46
|
-
else
|
|
47
|
-
return nil
|
|
48
|
-
end
|
|
49
|
-
`;
|
|
50
|
-
/** Release a lock only if we still hold it (value matches our token). */
|
|
51
|
-
export const LUA_RELEASE_LOCK = `
|
|
52
|
-
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
53
|
-
return redis.call('DEL', KEYS[1])
|
|
54
|
-
end
|
|
55
|
-
return 0
|
|
56
|
-
`;
|
|
57
|
-
|
|
58
|
-
/** Extend a lock's TTL only if we still hold it. */
|
|
59
|
-
export const LUA_RENEW_LOCK = `
|
|
60
|
-
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
|
61
|
-
return redis.call('EXPIRE', KEYS[1], ARGV[2])
|
|
62
|
-
end
|
|
63
|
-
return 0
|
|
64
|
-
`;
|
package/src/templates/index.ts
CHANGED
|
@@ -1,69 +1,2 @@
|
|
|
1
|
-
import type { RenderedContent } from "@/contracts/index.js";
|
|
2
1
|
export * from "./render.js";
|
|
3
2
|
export * from "./cache.js";
|
|
4
|
-
|
|
5
|
-
// ─── Types ──────────────────────────────────────────────────────────────────
|
|
6
|
-
|
|
7
|
-
export interface TemplateContext {
|
|
8
|
-
eventType: string;
|
|
9
|
-
templateVariables: Record<string, unknown>;
|
|
10
|
-
locale: string;
|
|
11
|
-
timezone: string;
|
|
12
|
-
deeplinkScheme: string;
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
export type TemplateRenderer = (ctx: TemplateContext) => RenderedContent;
|
|
16
|
-
|
|
17
|
-
/** Coerce a template variable to a non-empty string, or undefined. */
|
|
18
|
-
function asText(value: unknown): string | undefined {
|
|
19
|
-
return typeof value === "string" && value.length > 0 ? value : undefined;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
// ─── TemplateRegistry ────────────────────────────────────────────────────────
|
|
23
|
-
//
|
|
24
|
-
// Open registry — new event types register a renderer without touching this file.
|
|
25
|
-
// The default renderer falls back to the i18n translation table, so most event
|
|
26
|
-
// types work without a custom renderer.
|
|
27
|
-
|
|
28
|
-
class TemplateRegistry {
|
|
29
|
-
private readonly renderers = new Map<string, TemplateRenderer>();
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* Register a custom renderer for an event type.
|
|
33
|
-
* Overwrites any previous registration for the same type.
|
|
34
|
-
*/
|
|
35
|
-
register(eventType: string, renderer: TemplateRenderer): void {
|
|
36
|
-
this.renderers.set(eventType, renderer);
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/** Render content for the given context, falling back to the i18n table. */
|
|
40
|
-
render(ctx: TemplateContext): RenderedContent {
|
|
41
|
-
const renderer = this.renderers.get(ctx.eventType);
|
|
42
|
-
if (renderer) return renderer(ctx);
|
|
43
|
-
return defaultRenderer(ctx);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
has(eventType: string): boolean {
|
|
47
|
-
return this.renderers.has(eventType);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
registeredTypes(): string[] {
|
|
51
|
-
return [...this.renderers.keys()];
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function defaultRenderer(ctx: TemplateContext): RenderedContent {
|
|
56
|
-
const vars = ctx.templateVariables;
|
|
57
|
-
|
|
58
|
-
// Fall back to any caller-supplied title/body in the payload, then to a generic label.
|
|
59
|
-
const subject = asText(vars.subject ?? vars.title);
|
|
60
|
-
const body = asText(vars.body ?? vars.message) ?? `Notification: ${ctx.eventType}`;
|
|
61
|
-
|
|
62
|
-
return { content: { subject, body } };
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export const templateRegistry = new TemplateRegistry();
|
|
66
|
-
|
|
67
|
-
export function renderTemplate(ctx: TemplateContext): RenderedContent {
|
|
68
|
-
return templateRegistry.render(ctx);
|
|
69
|
-
}
|
package/src/transport/index.ts
CHANGED
|
@@ -10,6 +10,12 @@ export interface DeliveryResult {
|
|
|
10
10
|
providerMessageId?: string;
|
|
11
11
|
invalidToken?: boolean;
|
|
12
12
|
error?: string;
|
|
13
|
+
/**
|
|
14
|
+
* `false` marks a failure another attempt cannot fix — a malformed address,
|
|
15
|
+
* a rejected sender. Anything else (an outage, a timeout, a 5xx) is retried
|
|
16
|
+
* up to the task's `maxAttempts`.
|
|
17
|
+
*/
|
|
18
|
+
retryable?: boolean;
|
|
13
19
|
}
|
|
14
20
|
|
|
15
21
|
export interface WebhookEvent {
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import type { Redis } from "ioredis";
|
|
2
|
+
import { STREAMS, type ConsumerGroup, type StreamName } from "@/contracts/streams.js";
|
|
3
|
+
import { createDatabase, type Db, type Sql } from "@/db/index.js";
|
|
4
|
+
import { createLogger, type LogLevel, type Logger } from "@/logger/index.js";
|
|
5
|
+
import { PendingMessageScanner, StreamConsumer, StreamProducer } from "@/queue/index.js";
|
|
6
|
+
import { RedisClient } from "@/redis/index.js";
|
|
7
|
+
import type { BaseWorker } from "./index.js";
|
|
8
|
+
|
|
9
|
+
// Shared start/stop plumbing for the pipeline services under src/services.
|
|
10
|
+
// Each service still owns its worker and anything specific to it (polling
|
|
11
|
+
// loops, pub/sub subscribers); this only covers the parts they all repeat.
|
|
12
|
+
|
|
13
|
+
export interface WorkerRuntime {
|
|
14
|
+
logger: Logger;
|
|
15
|
+
redis: RedisClient;
|
|
16
|
+
sql: Sql;
|
|
17
|
+
db: Db;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Logger, shared Redis connection and Postgres pool for one service. */
|
|
21
|
+
export function createWorkerRuntime(
|
|
22
|
+
config: { LOG_LEVEL: LogLevel; REDIS_URL: string; DATABASE_URL: string },
|
|
23
|
+
name: string,
|
|
24
|
+
loggerName = name,
|
|
25
|
+
): WorkerRuntime {
|
|
26
|
+
const logger = createLogger({ name: loggerName, level: config.LOG_LEVEL });
|
|
27
|
+
const redis = RedisClient.shared({ url: config.REDIS_URL, name, logger });
|
|
28
|
+
const { sql, db } = createDatabase({
|
|
29
|
+
url: config.DATABASE_URL,
|
|
30
|
+
applicationName: name,
|
|
31
|
+
logger,
|
|
32
|
+
});
|
|
33
|
+
return { logger, redis, sql, db };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A consumer id unique to this process, so replicas sharing a host or PID
|
|
38
|
+
* namespace never read as the same group member.
|
|
39
|
+
*/
|
|
40
|
+
export function uniqueConsumerId(name: string): string {
|
|
41
|
+
const host = process.env.HOSTNAME || process.pid;
|
|
42
|
+
return `${name}-${host}-${Math.random().toString(36).slice(2, 8)}`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A group consumer and the scanner that reclaims its stalled messages. The two
|
|
47
|
+
* must agree on stream, group and consumer id, so they are built together.
|
|
48
|
+
*/
|
|
49
|
+
export function createStreamConsumer(
|
|
50
|
+
runtime: { redis: RedisClient; logger: Logger },
|
|
51
|
+
options: {
|
|
52
|
+
stream: StreamName | readonly StreamName[];
|
|
53
|
+
group: ConsumerGroup;
|
|
54
|
+
consumer: string;
|
|
55
|
+
batchSize: number;
|
|
56
|
+
bufferAcks?: boolean;
|
|
57
|
+
coalesceMs?: number;
|
|
58
|
+
},
|
|
59
|
+
): { consumer: StreamConsumer; pendingScanner: PendingMessageScanner } {
|
|
60
|
+
const { redis, logger } = runtime;
|
|
61
|
+
const stream = (Array.isArray(options.stream) ? [...options.stream] : options.stream) as
|
|
62
|
+
StreamName | StreamName[];
|
|
63
|
+
const consumer = new StreamConsumer({
|
|
64
|
+
redis: redis.native,
|
|
65
|
+
stream,
|
|
66
|
+
group: options.group,
|
|
67
|
+
consumer: options.consumer,
|
|
68
|
+
dlqStream: STREAMS.DEAD_LETTER,
|
|
69
|
+
batchSize: options.batchSize,
|
|
70
|
+
bufferAcks: options.bufferAcks,
|
|
71
|
+
coalesceMs: options.coalesceMs,
|
|
72
|
+
logger,
|
|
73
|
+
});
|
|
74
|
+
const pendingScanner = new PendingMessageScanner({
|
|
75
|
+
redis: redis.native,
|
|
76
|
+
stream,
|
|
77
|
+
group: options.group,
|
|
78
|
+
consumer: options.consumer,
|
|
79
|
+
logger,
|
|
80
|
+
});
|
|
81
|
+
return { consumer, pendingScanner };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export type PriorityProducers = Record<"critical" | "normal" | "low", StreamProducer>;
|
|
85
|
+
|
|
86
|
+
/** One producer per priority lane of a pipeline stage. */
|
|
87
|
+
export function createPriorityProducers(
|
|
88
|
+
redis: Redis,
|
|
89
|
+
logger: Logger,
|
|
90
|
+
stage: "INBOUND" | "ENRICHED" | "OUTBOUND",
|
|
91
|
+
): PriorityProducers {
|
|
92
|
+
const producer = (stream: StreamName) => new StreamProducer({ redis, stream, logger });
|
|
93
|
+
return {
|
|
94
|
+
critical: producer(STREAMS[`${stage}_CRITICAL`]),
|
|
95
|
+
normal: producer(STREAMS[`${stage}_NORMAL`]),
|
|
96
|
+
low: producer(STREAMS[`${stage}_LOW`]),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Stops a service in dependency order: timers and subscribers first so nothing
|
|
102
|
+
* new starts, then the worker drains, then the connections it was using close.
|
|
103
|
+
*/
|
|
104
|
+
export async function shutdownWorker(
|
|
105
|
+
label: string,
|
|
106
|
+
resources: {
|
|
107
|
+
logger?: Logger;
|
|
108
|
+
timers?: (NodeJS.Timeout | null | undefined)[];
|
|
109
|
+
subscriber?: { disconnect(): void } | null;
|
|
110
|
+
worker?: BaseWorker;
|
|
111
|
+
sql?: Sql;
|
|
112
|
+
redis?: RedisClient;
|
|
113
|
+
},
|
|
114
|
+
): Promise<void> {
|
|
115
|
+
const { logger, timers = [], subscriber, worker, sql, redis } = resources;
|
|
116
|
+
logger?.info("shutdown initiated");
|
|
117
|
+
// clearTimeout also cancels intervals in Node.
|
|
118
|
+
for (const timer of timers) if (timer) clearTimeout(timer);
|
|
119
|
+
subscriber?.disconnect();
|
|
120
|
+
if (worker) await worker.stop();
|
|
121
|
+
if (sql) await sql.end();
|
|
122
|
+
if (redis) await redis.disconnect();
|
|
123
|
+
logger?.info(`${label} stopped`);
|
|
124
|
+
}
|