notifkit 0.1.12 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +65 -337
  2. package/dashboard/dist/assets/{index-Bsrmo9LE.js → index-ggpJKZqZ.js} +55 -50
  3. package/dashboard/dist/index.html +1 -1
  4. package/dist/{bootstrap-omGPIBny.mjs → bootstrap-m107TukN.mjs} +1 -1
  5. package/dist/{bulk-C4hTcnLu.mjs → bulk-xAd_biYV.mjs} +1 -1
  6. package/dist/index.d.mts +16 -4
  7. package/dist/index.mjs +2 -2
  8. package/dist/{main-BxAeC_k6.mjs → main-5kVwbbnW.mjs} +157 -76
  9. package/dist/{main-DAXO-MiH.mjs → main-C0MjK1Au.mjs} +3 -3
  10. package/dist/{main-Bes11Ad3.mjs → main-C87TSAqM.mjs} +2 -2
  11. package/dist/{main-B-AZ-xoN.mjs → main-CWIBUEc7.mjs} +2 -2
  12. package/dist/{main-BU27VjDC.mjs → main-DRHL2bst.mjs} +2 -2
  13. package/dist/{main-DYGCl7Ja.mjs → main-D_62f9fH.mjs} +2 -2
  14. package/dist/{main-B5BwjIMu.mjs → main-HT5c6jyf.mjs} +58 -5
  15. package/dist/{main-BtnGmyaT.mjs → main-WkxXwP5q.mjs} +2 -2
  16. package/dist/{src-CIPTzq9i.mjs → src-DYqdD694.mjs} +64 -39
  17. package/drizzle/0008_delivery_outbox_destination_idx.sql +3 -0
  18. package/drizzle/meta/_journal.json +7 -0
  19. package/package.json +5 -2
  20. package/scripts/create-admin.mjs +8 -0
  21. package/scripts/create-project.mjs +1 -0
  22. package/scripts/notifkit.mjs +42 -0
  23. package/src/config/index.ts +6 -3
  24. package/src/contracts/streams.ts +4 -0
  25. package/src/db/schema.ts +3 -0
  26. package/src/queue/index.ts +19 -23
  27. package/src/redis/index.ts +20 -0
  28. package/src/services/api/admin-static.ts +21 -11
  29. package/src/services/api/handlers.ts +51 -43
  30. package/src/services/api/http.ts +12 -9
  31. package/src/services/api/live-events.ts +74 -0
  32. package/src/services/api/main.ts +57 -23
  33. package/src/services/delivery/live-events.ts +68 -0
  34. package/src/services/delivery/main.ts +14 -2
@@ -3,6 +3,7 @@ import { resolve, extname, join, relative, isAbsolute, dirname } from "node:path
3
3
  import { existsSync, statSync, createReadStream } from "node:fs";
4
4
  import { fileURLToPath } from "node:url";
5
5
  import { request as httpRequest } from "node:http";
6
+ import { pipeline } from "node:stream";
6
7
 
7
8
  const MIME_TYPES: Record<string, string> = {
8
9
  ".html": "text/html; charset=utf-8",
@@ -127,14 +128,20 @@ export async function handleAdminRequest(
127
128
  return true;
128
129
  }
129
130
 
130
- // In development, attempt to proxy to local Vite dev server on port 5173 (or VITE_DEV_URL) if active
131
- if (process.env.NODE_ENV !== "production") {
132
- const devProxyUrl =
133
- process.env.VITE_DEV_URL || process.env.ADMIN_DEV_URL || process.env.NEXT_DEV_URL;
134
- const targetPort = devProxyUrl ? parseInt(new URL(devProxyUrl).port, 10) : 5173;
135
- const targetHost = devProxyUrl ? new URL(devProxyUrl).hostname : "127.0.0.1";
136
-
137
- const proxied = await proxyToDevServer(req, res, targetHost, targetPort);
131
+ // Proxy to the dashboard's dev server only when one is named. Defaulting to
132
+ // 127.0.0.1:5173 outside production meant a deployment that never set
133
+ // NODE_ENV sent every /admin request, credentials included, to whatever
134
+ // listened there, and served its reply as the dashboard.
135
+ const devProxyUrl =
136
+ process.env.VITE_DEV_URL || process.env.ADMIN_DEV_URL || process.env.NEXT_DEV_URL;
137
+ if (devProxyUrl && process.env.NODE_ENV !== "production") {
138
+ const target = new URL(devProxyUrl);
139
+ const proxied = await proxyToDevServer(
140
+ req,
141
+ res,
142
+ target.hostname,
143
+ parseInt(target.port, 10) || 80,
144
+ );
138
145
  if (proxied) {
139
146
  return true;
140
147
  }
@@ -163,7 +170,7 @@ export async function handleAdminRequest(
163
170
  <p>The dashboard static bundle has not been built yet.</p>
164
171
  <p>Please build it by running:</p>
165
172
  <p><code>npm run build:dashboard</code></p>
166
- <p>or run the dev server with <code>npm --prefix dashboard run dev</code>.</p>
173
+ <p>or run the dev server with <code>npm --prefix dashboard run dev</code> and start notifkit with <code>VITE_DEV_URL=http://127.0.0.1:5173</code>.</p>
167
174
  </div>
168
175
  </body>
169
176
  </html>
@@ -248,7 +255,10 @@ export async function handleAdminRequest(
248
255
  "Cache-Control": cacheControl,
249
256
  });
250
257
 
251
- const stream = createReadStream(filePath);
252
- stream.pipe(res);
258
+ // pipeline, not pipe: a file that vanished or turned unreadable after the
259
+ // stat above fails the read, and pipe left that error unhandled (crashing
260
+ // the process) and the response open. pipeline destroys the response, so
261
+ // the client sees it cut short, and the error stops here.
262
+ pipeline(createReadStream(filePath), res, () => {});
253
263
  return true;
254
264
  }
@@ -41,6 +41,7 @@ import {
41
41
  } from "@/contracts/index.js";
42
42
  import { readJsonBody, sendJson, sendNoContent, sendValidationError } from "./http.js";
43
43
  import type { RouteContext } from "./router.js";
44
+ import type { LiveDeliveryEvents } from "./live-events.js";
44
45
  import { globalEmitter, getPriorityBucket, normaliseTarget } from "@/shared/index.js";
45
46
  import { metrics } from "@/metrics/index.js";
46
47
  import { eq, desc, and, lte, gte, sql, like, or, isNotNull, inArray } from "drizzle-orm";
@@ -75,6 +76,8 @@ export interface Deps {
75
76
  segmentRepo: SegmentRepository;
76
77
  adminUserRepo?: AdminUserRepository;
77
78
  db: Db;
79
+ /** Delivery outcomes for GET /v1/events/stream. */
80
+ liveEvents: LiveDeliveryEvents;
78
81
  }
79
82
 
80
83
  // Inline user shape (email/phone/pushToken already normalised to string[] by Zod).
@@ -715,37 +718,35 @@ export function createHandlers(deps: Deps) {
715
718
  res: ServerResponse,
716
719
  ctx: RouteContext,
717
720
  ): Promise<void> {
721
+ let open = true;
722
+ let stop: (() => void) | undefined;
723
+ req.on("close", () => {
724
+ open = false;
725
+ stop?.();
726
+ });
727
+
728
+ const unlisten = await deps.liveEvents.listen((e) => {
729
+ // Only events that name this project. One naming none belongs to no
730
+ // tenant, so it goes to nobody rather than to everybody.
731
+ if (!ctx.projectId || e.projectId !== ctx.projectId) return;
732
+ res.write(`event: ${e.event}\n`);
733
+ const data =
734
+ e.event === "delivery:delivered"
735
+ ? { taskId: e.taskId, providerMessageId: e.providerMessageId, channel: e.channel }
736
+ : { taskId: e.taskId, error: e.error, channel: e.channel };
737
+ res.write(`data: ${JSON.stringify(data)}\n\n`);
738
+ });
739
+ if (!open) return unlisten();
740
+ stop = unlisten;
741
+
742
+ // Only once subscribed: a client that has the headers is already listening,
743
+ // so whatever it triggers next cannot be missed.
718
744
  res.writeHead(200, {
719
745
  "Content-Type": "text/event-stream",
720
746
  "Cache-Control": "no-cache",
721
747
  Connection: "keep-alive",
722
748
  });
723
749
  res.write("retry: 10000\n\n");
724
-
725
- const onDelivered = (
726
- taskId: string,
727
- providerMessageId: string,
728
- channel: string,
729
- eventProjectId: string,
730
- ) => {
731
- if (eventProjectId && eventProjectId !== ctx.projectId) return;
732
- res.write(`event: delivery:delivered\n`);
733
- res.write(`data: ${JSON.stringify({ taskId, providerMessageId, channel })}\n\n`);
734
- };
735
-
736
- const onFailed = (taskId: string, error: string, channel: string, eventProjectId: string) => {
737
- if (eventProjectId && eventProjectId !== ctx.projectId) return;
738
- res.write(`event: delivery:failed\n`);
739
- res.write(`data: ${JSON.stringify({ taskId, error, channel })}\n\n`);
740
- };
741
-
742
- globalEmitter.on("delivery:delivered", onDelivered);
743
- globalEmitter.on("delivery:failed", onFailed);
744
-
745
- req.on("close", () => {
746
- globalEmitter.off("delivery:delivered", onDelivered);
747
- globalEmitter.off("delivery:failed", onFailed);
748
- });
749
750
  }
750
751
 
751
752
  // ── GET /v1/notifications/logs — getNotificationLogs ────────────────────
@@ -964,8 +965,9 @@ export function createHandlers(deps: Deps) {
964
965
  });
965
966
  sendJson(res, 201, { name: input.name });
966
967
  } catch (error: any) {
968
+ // Logged, not returned: driver errors name tables, constraints and values.
967
969
  deps.logger.error({ err: error }, "Failed to create workflow definition");
968
- sendJson(res, 500, { error: "internal_error", message: error.message });
970
+ sendJson(res, 500, { error: "internal_error" });
969
971
  }
970
972
  }
971
973
 
@@ -1490,8 +1492,9 @@ export function createHandlers(deps: Deps) {
1490
1492
  await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId);
1491
1493
 
1492
1494
  sendJson(res, 200, { success: true, replayedId: messageId, stream: targetStream });
1493
- } catch (err: any) {
1494
- sendJson(res, 500, { error: "replay_failed", message: err.message });
1495
+ } catch (err) {
1496
+ logger.error({ err, messageId }, "DLQ replay failed");
1497
+ sendJson(res, 500, { error: "replay_failed" });
1495
1498
  }
1496
1499
  }
1497
1500
 
@@ -1511,8 +1514,9 @@ export function createHandlers(deps: Deps) {
1511
1514
  }
1512
1515
  await deps.redis.native.xdel(STREAMS.DEAD_LETTER, messageId);
1513
1516
  sendJson(res, 200, { success: true });
1514
- } catch (err: any) {
1515
- sendJson(res, 500, { error: "delete_failed", message: err.message });
1517
+ } catch (err) {
1518
+ logger.error({ err, messageId }, "DLQ delete failed");
1519
+ sendJson(res, 500, { error: "delete_failed" });
1516
1520
  }
1517
1521
  }
1518
1522
 
@@ -1563,21 +1567,18 @@ export function createHandlers(deps: Deps) {
1563
1567
  const contactTargets = contacts.map((c) => c.target);
1564
1568
  let logs: any[] = [];
1565
1569
  if (contactTargets.length > 0) {
1566
- const outboxRows = await deps.db
1570
+ // One statement: fetching every task id the addresses were ever sent and
1571
+ // passing them back as parameters failed outright past 65,535 sends.
1572
+ const sentTo = deps.db
1567
1573
  .select({ taskId: deliveryOutbox.taskId })
1568
1574
  .from(deliveryOutbox)
1569
1575
  .where(inArray(deliveryOutbox.destination, contactTargets));
1570
- const taskIds = Array.from(new Set(outboxRows.map((r: any) => r.taskId)));
1571
- if (taskIds.length > 0) {
1572
- logs = await deps.db
1573
- .select()
1574
- .from(messageLogs)
1575
- .where(
1576
- and(eq(messageLogs.projectId, ctx.projectId!), inArray(messageLogs.taskId, taskIds)),
1577
- )
1578
- .orderBy(desc(messageLogs.timestamp))
1579
- .limit(50);
1580
- }
1576
+ logs = await deps.db
1577
+ .select()
1578
+ .from(messageLogs)
1579
+ .where(and(eq(messageLogs.projectId, ctx.projectId!), inArray(messageLogs.taskId, sentTo)))
1580
+ .orderBy(desc(messageLogs.timestamp))
1581
+ .limit(50);
1581
1582
  }
1582
1583
 
1583
1584
  sendJson(res, 200, { ...user, contacts, logs });
@@ -2073,7 +2074,14 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
2073
2074
  const { identifier, password } = parsed.data;
2074
2075
  const user = await deps.adminUserRepo.findByEmailOrUsername(identifier);
2075
2076
  const targetHash = user ? user.passwordHash : DUMMY_PASSWORD_HASH;
2076
- const isValid = await verifyPassword(password, targetHash);
2077
+ let isValid = await verifyPassword(password, targetHash);
2078
+ // The startup admin's ADMIN_PASSWORD is stored trimmed (a Windows `set`
2079
+ // leaves a trailing space in it), so one configured with spaces at either
2080
+ // end would otherwise never match what is typed. Only a failed attempt
2081
+ // with such spaces pays the second check.
2082
+ if (!isValid && password.trim() !== password && password.trim()) {
2083
+ isValid = await verifyPassword(password.trim(), targetHash);
2084
+ }
2077
2085
  if (!user || !isValid) {
2078
2086
  sendJson(res, 401, { error: "unauthorized", message: "Invalid credentials" });
2079
2087
  return;
@@ -1,4 +1,5 @@
1
1
  import type { IncomingMessage, ServerResponse } from "node:http";
2
+ import { StringDecoder } from "node:string_decoder";
2
3
  import type { ZodError } from "zod";
3
4
 
4
5
  export async function readRawBody(req: IncomingMessage): Promise<string> {
@@ -23,26 +24,28 @@ export async function readRawBody(req: IncomingMessage): Promise<string> {
23
24
 
24
25
  export async function readJsonBody(req: IncomingMessage): Promise<unknown> {
25
26
  return new Promise<unknown>((resolve, reject) => {
26
- // Read as UTF-8 strings directly: avoids the Buffer.concat + toString() copy
27
- // that readRawBody pays. readRawBody is kept for webhook callers that need
28
- // raw bytes for HMAC verification.
29
- if (typeof req.setEncoding === "function") {
30
- req.setEncoding("utf8");
31
- }
27
+ // Decode chunk by chunk: avoids the Buffer.concat + toString() copy that
28
+ // readRawBody pays. readRawBody is kept for webhook callers that need raw
29
+ // bytes for HMAC verification. The limit counts bytes as they arrive;
30
+ // counting decoded characters let multibyte text through at up to three
31
+ // times the limit. StringDecoder keeps a character split across two
32
+ // chunks intact.
33
+ const decoder = new StringDecoder("utf8");
32
34
  let body = "";
33
35
  let totalLength = 0;
34
36
  const MAX_PAYLOAD_SIZE = 5 * 1024 * 1024;
35
37
  req.on("data", (chunk: string | Buffer) => {
36
- const str = typeof chunk === "string" ? chunk : chunk.toString("utf8");
37
- totalLength += str.length;
38
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
39
+ totalLength += bytes.length;
38
40
  if (totalLength > MAX_PAYLOAD_SIZE) {
39
41
  if (typeof req.destroy === "function") req.destroy();
40
42
  reject(new HttpError(413, "payload_too_large"));
41
43
  return;
42
44
  }
43
- body += str;
45
+ body += decoder.write(bytes);
44
46
  });
45
47
  req.on("end", () => {
48
+ body += decoder.end();
46
49
  const trimmed = body.trim();
47
50
  if (!trimmed) {
48
51
  resolve(undefined);
@@ -0,0 +1,74 @@
1
+ import type { Redis } from "@/redis/index.js";
2
+ import type { Logger } from "@/logger/index.js";
3
+ import { PUBSUB_CHANNELS } from "@/contracts/index.js";
4
+ import type { LiveDeliveryEvent } from "@/services/delivery/live-events.js";
5
+
6
+ export type LiveDeliveryListener = (event: LiveDeliveryEvent) => void;
7
+
8
+ /** What getEventsStream needs: somewhere to listen for delivery outcomes. */
9
+ export interface LiveDeliveryEvents {
10
+ /** Starts calling `listener` for every outcome; the returned function stops it. */
11
+ listen(listener: LiveDeliveryListener): Promise<() => void>;
12
+ }
13
+
14
+ /**
15
+ * Relays delivery outcomes published by delivery workers (in this process or
16
+ * any other) to the API's open event streams.
17
+ *
18
+ * Holds the Redis subscription only while at least one stream is open: the
19
+ * delivery side stops publishing when nobody is subscribed, so an idle API
20
+ * costs the send path nothing.
21
+ */
22
+ export class RedisLiveDeliveryEvents implements LiveDeliveryEvents {
23
+ private subscriber: Redis | null = null;
24
+ private readonly listeners = new Set<LiveDeliveryListener>();
25
+ private subscribed: Promise<unknown> | null = null;
26
+
27
+ constructor(
28
+ private readonly redis: Redis,
29
+ private readonly logger?: Logger,
30
+ ) {}
31
+
32
+ async listen(listener: LiveDeliveryListener): Promise<() => void> {
33
+ this.listeners.add(listener);
34
+ if (!this.subscribed) this.subscribed = this.subscribe();
35
+ await this.subscribed;
36
+ return () => {
37
+ if (!this.listeners.delete(listener) || this.listeners.size > 0) return;
38
+ this.subscribed = null;
39
+ this.subscriber?.unsubscribe(PUBSUB_CHANNELS.DELIVERY_EVENTS).catch(() => {});
40
+ };
41
+ }
42
+
43
+ private async subscribe(): Promise<void> {
44
+ if (!this.subscriber) {
45
+ this.subscriber = this.redis.duplicate({ enableAutoPipelining: false });
46
+ this.subscriber.on("message", (_channel: string, raw: string) => {
47
+ let event: LiveDeliveryEvent;
48
+ try {
49
+ event = JSON.parse(raw);
50
+ } catch {
51
+ return;
52
+ }
53
+ for (const listener of this.listeners) listener(event);
54
+ });
55
+ }
56
+ try {
57
+ await this.subscriber.subscribe(PUBSUB_CHANNELS.DELIVERY_EVENTS);
58
+ // Delivery workers that found nobody listening have gone quiet; wake
59
+ // them now that someone is, so the next outcome is not missed.
60
+ await this.redis.publish(PUBSUB_CHANNELS.DELIVERY_EVENTS_WANTED, "1");
61
+ } catch (err) {
62
+ // The stream stays open and simply carries nothing until a retry.
63
+ this.subscribed = null;
64
+ this.logger?.warn({ err }, "could not subscribe to live delivery events");
65
+ }
66
+ }
67
+
68
+ async close(): Promise<void> {
69
+ this.listeners.clear();
70
+ this.subscribed = null;
71
+ this.subscriber?.disconnect();
72
+ this.subscriber = null;
73
+ }
74
+ }
@@ -18,6 +18,7 @@ import { readJsonBody, readRawBody, sendJson, HttpError } from "./http.js";
18
18
  import { Router } from "./router.js";
19
19
  import { createHandlers } from "./handlers.js";
20
20
  import { handleAdminRequest } from "./admin-static.js";
21
+ import { RedisLiveDeliveryEvents } from "./live-events.js";
21
22
  import { getAdminSession, hashPassword } from "@/services/auth/index.js";
22
23
  import { projects, messageLogs, projectApiKeys, suppressions } from "@/db/schema.js";
23
24
  import { eq, inArray } from "drizzle-orm";
@@ -61,6 +62,7 @@ let authCache = new LRUCache<string, CachedAuthKey>(1000, AUTH_CACHE_TTL_MS);
61
62
  let tokenHashToToken = new Map<string, string>();
62
63
  let authSubscriber: Redis | null = null;
63
64
  let router: Router;
65
+ let liveEvents: RedisLiveDeliveryEvents | null = null;
64
66
  let apiRateLimiter: ApiRateLimiter;
65
67
 
66
68
  export function extractAuthToken(req: Pick<IncomingMessage, "headers">): string | undefined {
@@ -76,17 +78,34 @@ export function extractAuthToken(req: Pick<IncomingMessage, "headers">): string
76
78
  }
77
79
 
78
80
  /**
79
- * Resolves client IP address, respecting reverse proxy headers only when TRUST_PROXY is enabled.
81
+ * What a dashboard user's role may do. `admin_users.role` is free text (the
82
+ * create-admin script takes any value), so only the roles known to grant
83
+ * writes do; anything else, `read_only` included, is read-only.
80
84
  */
81
- export function getClientIp(req: IncomingMessage, trustProxy: boolean): string {
82
- if (trustProxy) {
83
- const forwarded = req.headers["x-forwarded-for"];
84
- if (typeof forwarded === "string") {
85
- const first = forwarded.split(",")[0]?.trim();
86
- if (first) return first;
87
- }
88
- }
89
- return req.socket.remoteAddress || "unknown";
85
+ export function sessionKeyRole(role: string): "admin" | "read_only" {
86
+ return role === "superadmin" || role === "admin" ? "admin" : "read_only";
87
+ }
88
+
89
+ /**
90
+ * Resolves the client address, given how many proxies (TRUST_PROXY) are in
91
+ * front of the API. Each one appends the address it saw to X-Forwarded-For,
92
+ * so the client is `trustedHops` back from the connecting peer. The leftmost
93
+ * entry is the one a client can write itself, so it is used only when the
94
+ * chain is shorter than the configured hops.
95
+ */
96
+ export function getClientIp(req: IncomingMessage, trustedHops: number): string {
97
+ const peer = req.socket.remoteAddress || "unknown";
98
+ const forwarded = req.headers["x-forwarded-for"];
99
+ if (trustedHops <= 0 || typeof forwarded !== "string") return peer;
100
+
101
+ const chain = [
102
+ ...forwarded
103
+ .split(",")
104
+ .map((s) => s.trim())
105
+ .filter(Boolean),
106
+ peer,
107
+ ];
108
+ return chain[Math.max(chain.length - 1 - trustedHops, 0)]!;
90
109
  }
91
110
 
92
111
  async function handleCreateProject(req: IncomingMessage, res: ServerResponse): Promise<void> {
@@ -277,9 +296,11 @@ export async function startApiServer() {
277
296
  }
278
297
  }
279
298
 
299
+ liveEvents = new RedisLiveDeliveryEvents(redis.native, logger);
280
300
  deps = {
281
301
  logger,
282
302
  redis,
303
+ liveEvents,
283
304
  producers,
284
305
  userRepo: new UserRepository(db),
285
306
  contactRepo: new ContactRepository(db),
@@ -443,7 +464,7 @@ export async function startApiServer() {
443
464
  const session = await getAdminSession(redis.native, token);
444
465
  if (session) {
445
466
  isAdminToken = true;
446
- keyRole = session.role === "superadmin" ? "admin" : (session.role as any);
467
+ keyRole = sessionKeyRole(session.role);
447
468
  }
448
469
  } else if (config.ADMIN_API_KEY) {
449
470
  const expectedBuffer = Buffer.from(config.ADMIN_API_KEY);
@@ -559,7 +580,15 @@ export async function startApiServer() {
559
580
  let authorized = false;
560
581
  if (token.startsWith("nk_sess_")) {
561
582
  const session = await getAdminSession(redis.native, token);
562
- if (session) authorized = true;
583
+ if (session) {
584
+ // Creating projects and minting keys hands out admin credentials, so
585
+ // a read-only dashboard user gets to look and nothing more.
586
+ if (sessionKeyRole(session.role) === "read_only" && req.method !== "GET") {
587
+ sendJson(res, 403, { error: "forbidden", message: "Session is read-only" });
588
+ return;
589
+ }
590
+ authorized = true;
591
+ }
563
592
  } else if (config.ADMIN_API_KEY) {
564
593
  const expectedBuffer = Buffer.from(config.ADMIN_API_KEY);
565
594
  const providedBuffer = Buffer.from(token);
@@ -583,11 +612,10 @@ export async function startApiServer() {
583
612
  // an open endpoint cannot be used to hammer the process. Generous, because
584
613
  // a shared corporate egress IP can legitimately produce a burst.
585
614
  if (isPublicUnsubscribe) {
586
- const clientIp = getClientIp(req, Boolean(config.TRUST_PROXY));
615
+ const clientIp = getClientIp(req, config.TRUST_PROXY);
587
616
  try {
588
617
  const key = `rate-limit:api:unsub:${clientIp}`;
589
- const count = await redis.native.incr(key);
590
- if (count === 1) await redis.native.expire(key, 60);
618
+ const count = await redis.native.countRequest(key, 60);
591
619
  if (count > 60) {
592
620
  res.setHeader("Retry-After", "60");
593
621
  sendJson(res, 429, { error: "too_many_requests" });
@@ -603,11 +631,10 @@ export async function startApiServer() {
603
631
  // The login route is unauthenticated and CPU-intensive due to password hashing (scrypt).
604
632
  // Throttle login attempts per IP to prevent credential brute-forcing and CPU exhaustion.
605
633
  if (isPublicAuth) {
606
- const clientIp = getClientIp(req, Boolean(config.TRUST_PROXY));
634
+ const clientIp = getClientIp(req, config.TRUST_PROXY);
607
635
  try {
608
636
  const key = `rate-limit:api:auth:${clientIp}`;
609
- const count = await redis.native.incr(key);
610
- if (count === 1) await redis.native.expire(key, 60);
637
+ const count = await redis.native.countRequest(key, 60);
611
638
  if (count > 10) {
612
639
  res.setHeader("Retry-After", "60");
613
640
  sendJson(res, 429, {
@@ -694,6 +721,7 @@ export async function startApiServer() {
694
721
 
695
722
  if (transport.parseWebhook) {
696
723
  router.post(transport.webhookPath, async (req, res) => {
724
+ let deadline: NodeJS.Timeout | undefined;
697
725
  try {
698
726
  await Promise.race([
699
727
  (async () => {
@@ -835,15 +863,19 @@ export async function startApiServer() {
835
863
  }
836
864
  if (!res.headersSent) sendJson(res, 200, { success: true });
837
865
  })(),
838
- new Promise((_, reject) =>
839
- setTimeout(() => reject(new Error("Webhook processing timeout")), 15000),
840
- ),
866
+ new Promise((_, reject) => {
867
+ deadline = setTimeout(() => reject(new Error("Webhook processing timeout")), 15000);
868
+ }),
841
869
  ]);
842
- } catch (err: any) {
870
+ } catch (err) {
871
+ // The detail is ours to log, not the provider's: providers keep
872
+ // response bodies in their delivery logs.
843
873
  logger.error({ err, channel }, "webhook processing error or timeout");
844
874
  if (!res.headersSent) {
845
- sendJson(res, 500, { error: "internal_error", message: err.message });
875
+ sendJson(res, 500, { error: "internal_error" });
846
876
  }
877
+ } finally {
878
+ clearTimeout(deadline);
847
879
  }
848
880
  });
849
881
  logger.info(`Mounted webhook for ${channel} at ${transport.webhookPath}`);
@@ -873,6 +905,8 @@ export async function stopApiServer(): Promise<void> {
873
905
  authSubscriber.disconnect();
874
906
  authSubscriber = null;
875
907
  }
908
+ await liveEvents?.close();
909
+ liveEvents = null;
876
910
  // These are module-scoped, so a restart in the same process must not inherit
877
911
  // the previous instance's cached auth or health.
878
912
  authCache = new LRUCache<string, CachedAuthKey>(1000, AUTH_CACHE_TTL_MS);
@@ -0,0 +1,68 @@
1
+ import type { EventEmitter } from "node:events";
2
+ import type { Redis } from "@/redis/index.js";
3
+ import { PUBSUB_CHANNELS } from "@/contracts/index.js";
4
+
5
+ /**
6
+ * Delivery outcomes as GET /v1/events/stream relays them. They cross from the
7
+ * delivery worker to the API over Redis pub/sub because the two are separate
8
+ * processes in any deployment that scales them apart; an in-process emitter
9
+ * only ever reached an API that happened to share the delivery's process.
10
+ */
11
+ export type LiveDeliveryEvent =
12
+ | {
13
+ event: "delivery:delivered";
14
+ taskId: string;
15
+ providerMessageId: string;
16
+ channel: string;
17
+ projectId: string;
18
+ }
19
+ | { event: "delivery:failed"; taskId: string; error: string; channel: string; projectId: string };
20
+
21
+ /**
22
+ * How long to stop publishing after a publish nobody was subscribed to. Only
23
+ * a fallback: an API that starts listening says so on DELIVERY_EVENTS_WANTED,
24
+ * which ends the quiet straight away.
25
+ */
26
+ const QUIET_MS = 5_000;
27
+
28
+ /**
29
+ * Forwards the delivery worker's `delivery:*` events to Redis.
30
+ *
31
+ * The API subscribes only while a client has the stream open, so most of the
32
+ * time a publish reaches nobody. PUBLISH says how many received it; after a 0
33
+ * this goes quiet rather than paying a round trip per delivery for no one,
34
+ * until an API announces it is listening.
35
+ */
36
+ export function publishDeliveryEvents(redis: Redis, emitter: EventEmitter): () => void {
37
+ let quietUntil = 0;
38
+ const wanted = redis.duplicate({ enableAutoPipelining: false });
39
+ wanted.on("message", () => (quietUntil = 0));
40
+ wanted.subscribe(PUBSUB_CHANNELS.DELIVERY_EVENTS_WANTED).catch(() => {});
41
+
42
+ const publish = (message: LiveDeliveryEvent) => {
43
+ if (Date.now() < quietUntil) return;
44
+ redis
45
+ .publish(PUBSUB_CHANNELS.DELIVERY_EVENTS, JSON.stringify(message))
46
+ .then((receivers) => {
47
+ if (receivers === 0) quietUntil = Date.now() + QUIET_MS;
48
+ })
49
+ .catch(() => {});
50
+ };
51
+
52
+ const onDelivered = (
53
+ taskId: string,
54
+ providerMessageId: string,
55
+ channel: string,
56
+ projectId: string,
57
+ ) => publish({ event: "delivery:delivered", taskId, providerMessageId, channel, projectId });
58
+ const onFailed = (taskId: string, error: string, channel: string, projectId: string) =>
59
+ publish({ event: "delivery:failed", taskId, error, channel, projectId });
60
+
61
+ emitter.on("delivery:delivered", onDelivered);
62
+ emitter.on("delivery:failed", onFailed);
63
+ return () => {
64
+ emitter.off("delivery:delivered", onDelivered);
65
+ emitter.off("delivery:failed", onFailed);
66
+ wanted.disconnect();
67
+ };
68
+ }
@@ -35,6 +35,7 @@ import {
35
35
  uniqueConsumerId,
36
36
  } from "@/workers/bootstrap.js";
37
37
  import { BatchProcessor, CircuitBreaker } from "@/shared/index.js";
38
+ import { publishDeliveryEvents } from "./live-events.js";
38
39
  import { ProviderThrottle } from "./throttle.js";
39
40
  import { metrics } from "@/metrics/index.js";
40
41
 
@@ -59,6 +60,7 @@ let scheduledProducer: StreamProducer;
59
60
  let enrichedProducers: Record<string, StreamProducer>;
60
61
  let healthInterval: NodeJS.Timeout | null = null;
61
62
  let subscriber: Redis | null = null;
63
+ let stopLiveEvents: (() => void) | null = null;
62
64
 
63
65
  (global as any)._telemetry = (global as any)._telemetry || {
64
66
  count: 0,
@@ -620,7 +622,9 @@ export class DeliveryWorker extends BaseWorker {
620
622
  );
621
623
  }),
622
624
  ]);
623
- if (!res.success && !res.invalidToken) {
625
+ // A permanent rejection is the provider working: returned, not
626
+ // thrown, so the breaker does not count it.
627
+ if (!res.success && !res.invalidToken && res.retryable !== false) {
624
628
  throw providerFailure(res);
625
629
  }
626
630
  return res;
@@ -833,7 +837,10 @@ export class DeliveryWorker extends BaseWorker {
833
837
  );
834
838
  }),
835
839
  ]);
836
- if (!res.success) {
840
+ // A permanent rejection (a bad address) is the provider working:
841
+ // returned, not thrown, so the breaker does not count it. A run
842
+ // of them in one list used to open it for every project.
843
+ if (!res.success && res.retryable !== false) {
837
844
  throw providerFailure(res);
838
845
  }
839
846
  return res;
@@ -1078,6 +1085,9 @@ export async function startDeliveryWorker() {
1078
1085
 
1079
1086
  // ─── Health check interval ──────────────────────────────────────────────────
1080
1087
 
1088
+ // Outcomes for the API's live event stream, which may be in another process.
1089
+ stopLiveEvents = publishDeliveryEvents(redis.native, globalEmitter);
1090
+
1081
1091
  healthInterval = startHealthReporter("delivery", worker, redis, logger);
1082
1092
 
1083
1093
  logger.info(
@@ -1104,4 +1114,6 @@ export async function stopDeliveryWorker(): Promise<void> {
1104
1114
  });
1105
1115
  healthInterval = null;
1106
1116
  subscriber = null;
1117
+ stopLiveEvents?.();
1118
+ stopLiveEvents = null;
1107
1119
  }