notifkit 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/{bootstrap-BMDRPjgp.mjs → bootstrap-Cs1XIp6b.mjs} +1 -1
- package/dist/{bulk-CJ5QXiPa.mjs → bulk-4pRIiG2o.mjs} +1 -1
- package/dist/index.d.mts +37 -106
- package/dist/index.mjs +2 -2
- package/dist/{main-8nP3qmCT.mjs → main-BN7ZfvxN.mjs} +5 -14
- package/dist/{main-6_bXQBDH.mjs → main-Bl0S4GDd.mjs} +3 -3
- package/dist/{main-mGIT8jv4.mjs → main-C5DiM7_r.mjs} +3 -3
- package/dist/{main-B82bsFoy.mjs → main-DVzp0DLn.mjs} +8 -58
- package/dist/{main-ZW5RaiOK.mjs → main-O9iZ6iz2.mjs} +4 -9
- package/dist/{main-CpaHF1Zw.mjs → main-Ppr7Pq1g.mjs} +35 -16
- package/dist/{main-DULD8-th.mjs → main-yL5VinQ5.mjs} +5 -12
- package/dist/{main-g1KEcXXp.mjs → main-zgxAlH3l.mjs} +23 -94
- package/dist/{schedule-or-dispatch-Cd7GLfc3.mjs → schedule-or-dispatch-BwCZWS_P.mjs} +1 -1
- package/dist/{src-BE_EUpZ6.mjs → src-C38w4jR4.mjs} +88 -170
- package/drizzle/0009_retention_time_indexes.sql +4 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +1 -1
- package/src/client.ts +6 -6
- package/src/config/index.ts +9 -11
- package/src/contracts/sdk.ts +9 -9
- package/src/db/bulk.ts +3 -10
- package/src/db/schema.ts +3 -0
- package/src/idempotency/index.ts +12 -53
- package/src/queue/index.ts +9 -12
- package/src/rate-limiter/api-rate-limiter.ts +8 -27
- package/src/rate-limiter/project-settings-cache.ts +4 -16
- package/src/redis/index.ts +3 -51
- package/src/repositories/contact.ts +0 -2
- package/src/server.ts +1 -4
- package/src/services/api/handlers/auth.ts +3 -11
- package/src/services/api/handlers/campaigns.ts +4 -15
- package/src/services/api/handlers/events.ts +1 -2
- package/src/services/api/handlers/notifications.ts +6 -16
- package/src/services/api/handlers/projects.ts +1 -3
- package/src/services/api/handlers/shared.ts +7 -29
- package/src/services/api/handlers/system.ts +1 -1
- package/src/services/api/handlers/unsubscribe.ts +7 -30
- package/src/services/api/handlers/users.ts +1 -2
- package/src/services/api/http.ts +5 -10
- package/src/services/api/main.ts +27 -63
- package/src/services/api/webhooks.ts +8 -19
- package/src/services/auth/index.ts +2 -21
- package/src/services/delivery/main.ts +25 -89
- package/src/services/engine/main.ts +33 -69
- package/src/services/engine/schedule-or-dispatch.ts +7 -4
- package/src/services/enricher/main.ts +19 -49
- package/src/services/events/main.ts +3 -12
- package/src/services/scheduler/main.ts +26 -32
- package/src/services/scheduler/retention.ts +47 -0
- package/src/services/scheduler/send-gate.ts +6 -17
- package/src/services/workflow/duration.ts +1 -6
- package/src/services/workflow/main.ts +3 -9
- package/src/services/workflow/timers.ts +2 -1
- package/src/shared/batch-processor.ts +3 -7
- package/src/shared/utils.ts +2 -8
- package/src/transport/index.ts +0 -4
- package/src/unsubscribe/index.ts +3 -11
- package/src/workers/index.ts +30 -23
|
@@ -37,7 +37,6 @@ export function createNotificationsHandlers(deps: Deps) {
|
|
|
37
37
|
if (!parsed.success) return sendValidationError(res, parsed.error);
|
|
38
38
|
const body = parsed.data;
|
|
39
39
|
|
|
40
|
-
// Resolve the (unresolved) target. Inline users are synced first.
|
|
41
40
|
let targets: NotificationTarget[] = [];
|
|
42
41
|
const inlineUsersToSync: InlineUserLike[] = [];
|
|
43
42
|
if (body.user !== undefined) {
|
|
@@ -51,12 +50,9 @@ export function createNotificationsHandlers(deps: Deps) {
|
|
|
51
50
|
}
|
|
52
51
|
}
|
|
53
52
|
|
|
53
|
+
// A failure must reach the caller: queued without its user, the send is dropped.
|
|
54
54
|
if (inlineUsersToSync.length > 0) {
|
|
55
|
-
|
|
56
|
-
await persistUsers(deps, inlineUsersToSync, ctx.projectId!);
|
|
57
|
-
} catch (err) {
|
|
58
|
-
deps.logger.error({ err }, "inline user bulk sync failed");
|
|
59
|
-
}
|
|
55
|
+
await persistUsers(deps, inlineUsersToSync, ctx.projectId!);
|
|
60
56
|
}
|
|
61
57
|
} else if (body.segment !== undefined) {
|
|
62
58
|
targets.push({ type: "segment", segment: body.segment });
|
|
@@ -207,13 +203,9 @@ export function createNotificationsHandlers(deps: Deps) {
|
|
|
207
203
|
);
|
|
208
204
|
}
|
|
209
205
|
|
|
210
|
-
// One entry per task, newest
|
|
211
|
-
//
|
|
212
|
-
//
|
|
213
|
-
// newest row. The cursor is that row for the first task of the next page,
|
|
214
|
-
// and any task with a row newer than the cursor was listed on an earlier
|
|
215
|
-
// page. Paging on a millisecond timestamp over `limit * 2` rows skipped
|
|
216
|
-
// tasks that shared a timestamp and dropped any that did not fit the slice.
|
|
206
|
+
// One entry per task, at its newest row. Rows are scanned by (timestamp, id)
|
|
207
|
+
// because a batch shares one timestamp; a task with a row newer than the
|
|
208
|
+
// cursor was already listed on an earlier page.
|
|
217
209
|
const start = decodeLogCursor(cursor);
|
|
218
210
|
const tsText = sql<string>`${messageLogs.timestamp}::text`;
|
|
219
211
|
const position = (ts: string, id: string) => sql`(${ts}::timestamptz, ${id}::uuid)`;
|
|
@@ -302,15 +294,13 @@ export function createNotificationsHandlers(deps: Deps) {
|
|
|
302
294
|
.orderBy(desc(messageLogs.timestamp));
|
|
303
295
|
|
|
304
296
|
if (logs.length === 0) {
|
|
305
|
-
// It might still be pending in Redis or scheduled
|
|
306
297
|
return sendJson(res, 404, {
|
|
307
298
|
error: "not_found",
|
|
308
299
|
message: "Task not found or not processed yet",
|
|
309
300
|
});
|
|
310
301
|
}
|
|
311
302
|
|
|
312
|
-
// Newest first, so `status` is the latest known state
|
|
313
|
-
// row Postgres happened to return first.
|
|
303
|
+
// Newest first, so `status` is the latest known state.
|
|
314
304
|
sendJson(res, 200, { status: logs[0]?.status, logs });
|
|
315
305
|
}
|
|
316
306
|
|
|
@@ -47,9 +47,7 @@ export function createProjectsHandlers(deps: Deps) {
|
|
|
47
47
|
// next notification instead of waiting out the cache TTL.
|
|
48
48
|
await deps.redis.native.publish(PUBSUB_CHANNELS.PROJECT_INVALIDATED, id);
|
|
49
49
|
|
|
50
|
-
//
|
|
51
|
-
// hash rather than project — dropping the whole cache is the only way to
|
|
52
|
-
// pick up a new limit promptly. Project updates are rare admin operations.
|
|
50
|
+
// The auth cache is keyed by token, not project, so a new limit needs it all dropped.
|
|
53
51
|
if (parsed.data.rateLimitRpm !== undefined) {
|
|
54
52
|
await deps.redis.native.publish(PUBSUB_CHANNELS.API_KEY_INVALIDATED, "*");
|
|
55
53
|
}
|
|
@@ -31,7 +31,7 @@ export interface Deps {
|
|
|
31
31
|
liveEvents: LiveDeliveryEvents;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
-
//
|
|
34
|
+
// email/phone/pushToken arrive normalised to string[] by Zod.
|
|
35
35
|
export interface InlineUserLike {
|
|
36
36
|
id: string;
|
|
37
37
|
language?: string;
|
|
@@ -53,7 +53,6 @@ interface ResolvedContact {
|
|
|
53
53
|
preferences?: Preferences;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
-
/** Map an inline or user input's contacts and legacy arrays into resolved contact records. */
|
|
57
56
|
export function contactsOf(user: InlineUserLike): ResolvedContact[] {
|
|
58
57
|
const list: ResolvedContact[] = [];
|
|
59
58
|
|
|
@@ -89,7 +88,6 @@ export function contactsOf(user: InlineUserLike): ResolvedContact[] {
|
|
|
89
88
|
return list;
|
|
90
89
|
}
|
|
91
90
|
|
|
92
|
-
/** Persist user records + their contacts in bulk. Shared by addUser and inline notify. */
|
|
93
91
|
export async function persistUsers(
|
|
94
92
|
deps: Deps,
|
|
95
93
|
users: InlineUserLike[],
|
|
@@ -98,9 +96,7 @@ export async function persistUsers(
|
|
|
98
96
|
const usersList = users.map((u) => {
|
|
99
97
|
const primaryEmail = u.email?.[0] ?? u.contacts?.find((c) => c.channel === "email")?.target;
|
|
100
98
|
|
|
101
|
-
// Omitted fields stay omitted
|
|
102
|
-
// stored profile, so an inline `{ id, phone }` on notify must not reset the
|
|
103
|
-
// user's timezone, language or email. Defaults are applied when read.
|
|
99
|
+
// Omitted fields stay omitted, so an inline `{ id, phone }` keeps the stored timezone and email.
|
|
104
100
|
return {
|
|
105
101
|
userId: u.id,
|
|
106
102
|
language: u.language,
|
|
@@ -129,12 +125,7 @@ export async function persistUsers(
|
|
|
129
125
|
});
|
|
130
126
|
}
|
|
131
127
|
|
|
132
|
-
/**
|
|
133
|
-
* A dead-letter entry, read the way StreamConsumer writes it: the original
|
|
134
|
-
* event JSON under `data`, carrying a `dlq` block that records which stream it
|
|
135
|
-
* failed on. Flat `payload`/`eventType` fields are accepted for entries written
|
|
136
|
-
* by hand or by older versions.
|
|
137
|
-
*/
|
|
128
|
+
/** As StreamConsumer writes it: the event JSON under `data` with a `dlq` block. Flat fields are hand-written entries. */
|
|
138
129
|
export interface DlqEntry {
|
|
139
130
|
id: string;
|
|
140
131
|
eventType: string;
|
|
@@ -192,10 +183,7 @@ export function parseDlqEntry(id: string, rawFields: string[]): DlqEntry {
|
|
|
192
183
|
};
|
|
193
184
|
}
|
|
194
185
|
|
|
195
|
-
/**
|
|
196
|
-
* Ids that live in uuid columns. Anything else can match no row, and passing it
|
|
197
|
-
* to Postgres fails the query outright — a 500 for what is simply not found.
|
|
198
|
-
*/
|
|
186
|
+
/** Postgres errors on a non-uuid in a uuid column; callers answer 404 instead. */
|
|
199
187
|
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
200
188
|
export const isUuid = (v: string | undefined): v is string =>
|
|
201
189
|
typeof v === "string" && UUID_RE.test(v);
|
|
@@ -232,26 +220,16 @@ export function decodeLogCursor(
|
|
|
232
220
|
return null;
|
|
233
221
|
}
|
|
234
222
|
|
|
235
|
-
/**
|
|
236
|
-
* A LIKE pattern matching `term` anywhere, with the term's own `%`, `_` and
|
|
237
|
-
* `\` taken literally rather than as wildcards.
|
|
238
|
-
*/
|
|
223
|
+
/** A LIKE pattern matching `term` anywhere, its own wildcards escaped. */
|
|
239
224
|
export function contains(term: string): string {
|
|
240
225
|
return `%${term.replace(/[\\%_]/g, "\\$&")}%`;
|
|
241
226
|
}
|
|
242
227
|
|
|
243
228
|
export function getQueryParam(ctx: RouteContext, key: string): string | undefined {
|
|
244
|
-
|
|
245
|
-
if (typeof ctx.query.get === "function") {
|
|
246
|
-
return ctx.query.get(key) ?? undefined;
|
|
247
|
-
}
|
|
248
|
-
return (ctx.query as any)[key] ?? undefined;
|
|
229
|
+
return ctx.query.get(key) ?? undefined;
|
|
249
230
|
}
|
|
250
231
|
|
|
251
|
-
/**
|
|
252
|
-
* Page size from `?limit=`, clamped to [1, max]. Missing or non-numeric
|
|
253
|
-
* values fall back to `fallback`; `undefined` there means "no limit".
|
|
254
|
-
*/
|
|
232
|
+
/** `?limit=` clamped to [1, max]; missing or junk gives `fallback` (`undefined` = no limit). */
|
|
255
233
|
export function parseLimit<F extends number | undefined>(
|
|
256
234
|
ctx: RouteContext,
|
|
257
235
|
fallback: F,
|
|
@@ -67,7 +67,7 @@ export function createSystemHandlers(deps: Deps) {
|
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
//
|
|
70
|
+
// Unconsumed backlog: consumer group lag plus pending entries.
|
|
71
71
|
async function getStreamBacklog(
|
|
72
72
|
redis: any,
|
|
73
73
|
streamKey: string,
|
|
@@ -10,9 +10,7 @@ import { type Deps } from "./shared.js";
|
|
|
10
10
|
export function createUnsubscribeHandlers(deps: Deps) {
|
|
11
11
|
const { logger } = deps;
|
|
12
12
|
|
|
13
|
-
// Both routes are
|
|
14
|
-
// has no API key. The signed token in the URL is the credential and it
|
|
15
|
-
// authorises exactly one thing for one address.
|
|
13
|
+
// Both routes are public; the signed token in the URL is the credential.
|
|
16
14
|
|
|
17
15
|
function sendHtml(res: ServerResponse, status: number, body: string): void {
|
|
18
16
|
res.writeHead(status, {
|
|
@@ -44,14 +42,7 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
|
|
|
44
42
|
</head><body><h1>${escapeHtml(title)}</h1>${message}</body></html>`;
|
|
45
43
|
}
|
|
46
44
|
|
|
47
|
-
|
|
48
|
-
* GET — the page a human lands on after clicking the link in the mail body.
|
|
49
|
-
*
|
|
50
|
-
* Deliberately does not unsubscribe anything. Corporate mail scanners and
|
|
51
|
-
* link prescanners issue a GET against every URL in a message; performing the
|
|
52
|
-
* opt-out here would unsubscribe people who never clicked. The mutation lives
|
|
53
|
-
* behind the POST.
|
|
54
|
-
*/
|
|
45
|
+
// GET only shows a confirm page: mail scanners GET every link in a message.
|
|
55
46
|
async function unsubscribePage(
|
|
56
47
|
_req: IncomingMessage,
|
|
57
48
|
res: ServerResponse,
|
|
@@ -96,23 +87,14 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
|
|
|
96
87
|
);
|
|
97
88
|
}
|
|
98
89
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
*
|
|
102
|
-
* Serves two callers with one handler: a mail client doing RFC 8058 one-click
|
|
103
|
-
* (body `List-Unsubscribe=One-Click`, token in the query string, no
|
|
104
|
-
* confirmation possible), and the form on the page above. Neither can carry a
|
|
105
|
-
* CSRF token, and the one-click spec forbids an interstitial, so the signed
|
|
106
|
-
* token is the whole of the authorisation.
|
|
107
|
-
*/
|
|
90
|
+
// POST serves RFC 8058 one-click and the form above. Neither can carry a CSRF
|
|
91
|
+
// token, so the signed token is the whole authorisation.
|
|
108
92
|
async function unsubscribe(
|
|
109
93
|
req: IncomingMessage,
|
|
110
94
|
res: ServerResponse,
|
|
111
95
|
ctx: RouteContext,
|
|
112
96
|
): Promise<void> {
|
|
113
|
-
//
|
|
114
|
-
// the token in the query string, so there is nothing here worth reading —
|
|
115
|
-
// but an unread body leaves bytes on a keep-alive socket. Discard it.
|
|
97
|
+
// Nothing in the body is needed, but unread bytes would stay on a keep-alive socket.
|
|
116
98
|
req.resume();
|
|
117
99
|
|
|
118
100
|
const cfg = readBaseConfig();
|
|
@@ -148,8 +130,7 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
|
|
|
148
130
|
const internalId = userRows[0]?.id;
|
|
149
131
|
|
|
150
132
|
if (claim.topics.length > 0 && internalId) {
|
|
151
|
-
// Scoped opt-out:
|
|
152
|
-
// including anything transactional, working.
|
|
133
|
+
// Scoped opt-out: transactional mail keeps working.
|
|
153
134
|
await deps.db
|
|
154
135
|
.insert(userTopicPreferences)
|
|
155
136
|
.values(claim.topics.map((topic) => ({ userId: internalId, topic, enabled: false })))
|
|
@@ -158,9 +139,7 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
|
|
|
158
139
|
set: { enabled: sql`excluded.enabled` },
|
|
159
140
|
});
|
|
160
141
|
} else {
|
|
161
|
-
// No topic
|
|
162
|
-
// request has to be honoured, so fall back to suppressing the address
|
|
163
|
-
// outright — over-honouring an unsubscribe is the safe direction.
|
|
142
|
+
// No topic, or the user is gone: suppress the address outright.
|
|
164
143
|
await deps.db
|
|
165
144
|
.insert(suppressions)
|
|
166
145
|
.values({
|
|
@@ -183,8 +162,6 @@ code{background:#f4f4f5;padding:.1rem .35rem;border-radius:4px}</style>
|
|
|
183
162
|
"unsubscribe honoured",
|
|
184
163
|
);
|
|
185
164
|
} catch (err) {
|
|
186
|
-
// A 5xx here means the mail client reports failure and, worse, the person
|
|
187
|
-
// believes they have opted out when they have not.
|
|
188
165
|
logger.error({ err, projectId: claim.projectId }, "unsubscribe failed to apply");
|
|
189
166
|
return sendHtml(
|
|
190
167
|
res,
|
|
@@ -228,8 +228,7 @@ export function createUsersHandlers(deps: Deps) {
|
|
|
228
228
|
const contactTargets = contacts.map((c) => c.target);
|
|
229
229
|
let logs: any[] = [];
|
|
230
230
|
if (contactTargets.length > 0) {
|
|
231
|
-
//
|
|
232
|
-
// passing them back as parameters failed outright past 65,535 sends.
|
|
231
|
+
// A subquery, not a list of ids: Postgres caps bind parameters at 65,535.
|
|
233
232
|
const sentTo = deps.db
|
|
234
233
|
.select({ taskId: deliveryOutbox.taskId })
|
|
235
234
|
.from(deliveryOutbox)
|
package/src/services/api/http.ts
CHANGED
|
@@ -2,11 +2,13 @@ import type { IncomingMessage, ServerResponse } from "node:http";
|
|
|
2
2
|
import { StringDecoder } from "node:string_decoder";
|
|
3
3
|
import type { ZodError } from "zod";
|
|
4
4
|
|
|
5
|
+
const MAX_PAYLOAD_SIZE = 5 * 1024 * 1024;
|
|
6
|
+
|
|
7
|
+
/** The body as text, for webhook signatures that are computed over the raw bytes. */
|
|
5
8
|
export async function readRawBody(req: IncomingMessage): Promise<string> {
|
|
6
9
|
const raw = await new Promise<string>((resolve, reject) => {
|
|
7
10
|
const chunks: Buffer[] = [];
|
|
8
11
|
let totalLength = 0;
|
|
9
|
-
const MAX_PAYLOAD_SIZE = 5 * 1024 * 1024; // 5MB limit
|
|
10
12
|
req.on("data", (chunk: Buffer) => {
|
|
11
13
|
totalLength += chunk.length;
|
|
12
14
|
if (totalLength > MAX_PAYLOAD_SIZE) {
|
|
@@ -24,16 +26,11 @@ export async function readRawBody(req: IncomingMessage): Promise<string> {
|
|
|
24
26
|
|
|
25
27
|
export async function readJsonBody(req: IncomingMessage): Promise<unknown> {
|
|
26
28
|
return new Promise<unknown>((resolve, reject) => {
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
// bytes for HMAC verification. The limit counts bytes as they arrive;
|
|
30
|
-
// counting decoded characters let multibyte text through at up to three
|
|
31
|
-
// times the limit. StringDecoder keeps a character split across two
|
|
32
|
-
// chunks intact.
|
|
29
|
+
// The limit counts bytes, not decoded characters; StringDecoder keeps a
|
|
30
|
+
// character split across two chunks intact.
|
|
33
31
|
const decoder = new StringDecoder("utf8");
|
|
34
32
|
let body = "";
|
|
35
33
|
let totalLength = 0;
|
|
36
|
-
const MAX_PAYLOAD_SIZE = 5 * 1024 * 1024;
|
|
37
34
|
req.on("data", (chunk: string | Buffer) => {
|
|
38
35
|
const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
39
36
|
totalLength += bytes.length;
|
|
@@ -86,7 +83,6 @@ export function sendNoContent(res: ServerResponse): void {
|
|
|
86
83
|
res.writeHead(204).end();
|
|
87
84
|
}
|
|
88
85
|
|
|
89
|
-
/** Turn a Zod error into a 400 response with field-level issues. */
|
|
90
86
|
export function sendValidationError(res: ServerResponse, error: ZodError): void {
|
|
91
87
|
sendJson(res, 400, {
|
|
92
88
|
error: "validation_error",
|
|
@@ -94,7 +90,6 @@ export function sendValidationError(res: ServerResponse, error: ZodError): void
|
|
|
94
90
|
});
|
|
95
91
|
}
|
|
96
92
|
|
|
97
|
-
/** A thrown HttpError short-circuits a handler with a specific status/body. */
|
|
98
93
|
export class HttpError extends Error {
|
|
99
94
|
constructor(
|
|
100
95
|
readonly status: number,
|
package/src/services/api/main.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { loadEnv, readBaseConfig } from "@/config/index.js";
|
|
|
3
3
|
import { createLogger } from "@/logger/index.js";
|
|
4
4
|
import { RedisClient } from "@/redis/index.js";
|
|
5
5
|
import { StreamProducer } from "@/queue/index.js";
|
|
6
|
-
import { createDatabase } from "@/db/index.js";
|
|
6
|
+
import { createDatabase, type Db, type Sql } from "@/db/index.js";
|
|
7
7
|
import {
|
|
8
8
|
UserRepository,
|
|
9
9
|
ContactRepository,
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
import { STREAMS } from "@/contracts/index.js";
|
|
17
17
|
import { readJsonBody, sendJson, HttpError, extractAuthToken } from "./http.js";
|
|
18
18
|
import { Router } from "./router.js";
|
|
19
|
-
import { createHandlers } from "./handlers.js";
|
|
19
|
+
import { createHandlers, type Deps } from "./handlers.js";
|
|
20
20
|
import { handleAdminRequest } from "./admin-static.js";
|
|
21
21
|
import { RedisLiveDeliveryEvents } from "./live-events.js";
|
|
22
22
|
import { createHealthHandlers } from "./health.js";
|
|
@@ -33,26 +33,21 @@ import { ApiRateLimiter } from "@/rate-limiter/index.js";
|
|
|
33
33
|
|
|
34
34
|
export { extractAuthToken };
|
|
35
35
|
|
|
36
|
-
/** Pub/sub channel used to drop a cached API key across every API process. */
|
|
37
36
|
export const API_KEY_INVALIDATION_CHANNEL = "apikey.invalidated";
|
|
38
37
|
|
|
39
|
-
/** Default window length in milliseconds for API rate limiting (1 minute). */
|
|
40
38
|
export const API_RATE_LIMIT_WINDOW_MS = 60_000;
|
|
41
39
|
|
|
42
|
-
// ─── Bootstrap ─────────────────────────────────────────────────────────────
|
|
43
|
-
|
|
44
40
|
loadEnv();
|
|
45
41
|
const config = readBaseConfig();
|
|
46
42
|
|
|
47
43
|
let logger: ReturnType<typeof createLogger>;
|
|
48
44
|
let redis: RedisClient;
|
|
49
|
-
let sql:
|
|
50
|
-
let db:
|
|
45
|
+
let sql: Sql;
|
|
46
|
+
let db: Db;
|
|
51
47
|
let producers: Record<string, StreamProducer>;
|
|
52
|
-
let deps:
|
|
53
|
-
let h:
|
|
54
|
-
//
|
|
55
|
-
// and invalidated on key change so a revoked key stops working promptly.
|
|
48
|
+
let deps: Deps;
|
|
49
|
+
let h: ReturnType<typeof createHandlers>;
|
|
50
|
+
// Revoked keys are also dropped via pub/sub, so the TTL only bounds a missed message.
|
|
56
51
|
interface CachedAuthKey {
|
|
57
52
|
projectId: string;
|
|
58
53
|
projectRateLimitRpm: number;
|
|
@@ -68,7 +63,6 @@ let router: Router;
|
|
|
68
63
|
let liveEvents: RedisLiveDeliveryEvents | null = null;
|
|
69
64
|
let apiRateLimiter: ApiRateLimiter;
|
|
70
65
|
|
|
71
|
-
/** Whether `token` is ADMIN_API_KEY, compared in constant time. */
|
|
72
66
|
function isAdminApiKey(token: string): boolean {
|
|
73
67
|
if (!config.ADMIN_API_KEY) return false;
|
|
74
68
|
const expectedBuffer = Buffer.from(config.ADMIN_API_KEY);
|
|
@@ -79,21 +73,15 @@ function isAdminApiKey(token: string): boolean {
|
|
|
79
73
|
);
|
|
80
74
|
}
|
|
81
75
|
|
|
82
|
-
/**
|
|
83
|
-
* What a dashboard user's role may do. `admin_users.role` is free text (the
|
|
84
|
-
* create-admin script takes any value), so only the roles known to grant
|
|
85
|
-
* writes do; anything else, `read_only` included, is read-only.
|
|
86
|
-
*/
|
|
76
|
+
/** `admin_users.role` is free text, so anything but a known writer role is read-only. */
|
|
87
77
|
export function sessionKeyRole(role: string): "admin" | "read_only" {
|
|
88
78
|
return role === "superadmin" || role === "admin" ? "admin" : "read_only";
|
|
89
79
|
}
|
|
90
80
|
|
|
91
81
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* entry is the one a client can write itself, so it is used only when the
|
|
96
|
-
* chain is shorter than the configured hops.
|
|
82
|
+
* Each trusted proxy appends the address it saw to X-Forwarded-For, so the
|
|
83
|
+
* client is `trustedHops` back from the peer. Entries further left are
|
|
84
|
+
* client-written.
|
|
97
85
|
*/
|
|
98
86
|
export function getClientIp(req: IncomingMessage, trustedHops: number): string {
|
|
99
87
|
const peer = req.socket.remoteAddress || "unknown";
|
|
@@ -124,8 +112,6 @@ async function handleCreateProject(req: IncomingMessage, res: ServerResponse): P
|
|
|
124
112
|
sendJson(res, 201, { id: projectId, apiKey });
|
|
125
113
|
}
|
|
126
114
|
|
|
127
|
-
// ─── HTTP server ────────────────────────────────────────────────────────────
|
|
128
|
-
|
|
129
115
|
let server: ReturnType<typeof createServer>;
|
|
130
116
|
|
|
131
117
|
export async function startApiServer() {
|
|
@@ -270,10 +256,7 @@ export async function startApiServer() {
|
|
|
270
256
|
server.keepAliveTimeout = 5_000;
|
|
271
257
|
|
|
272
258
|
async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
273
|
-
//
|
|
274
|
-
// request. `new URL()` parses scheme, host, port, path, and query — most of
|
|
275
|
-
// which we never use on the hot path. URLSearchParams is built lazily and only
|
|
276
|
-
// materialises when a handler or the auth block actually needs it.
|
|
259
|
+
// URL is parsed only if something reads the query.
|
|
277
260
|
const rawUrl = req.url ?? "/";
|
|
278
261
|
const qIdx = rawUrl.indexOf("?");
|
|
279
262
|
const pathname = qIdx === -1 ? rawUrl : rawUrl.slice(0, qIdx);
|
|
@@ -325,9 +308,7 @@ export async function startApiServer() {
|
|
|
325
308
|
|
|
326
309
|
const isProjectManagement = pathname === "/v1/projects" || pathname.startsWith("/v1/projects/");
|
|
327
310
|
|
|
328
|
-
//
|
|
329
|
-
// will. The signed token in the URL is the credential, and it authorises
|
|
330
|
-
// exactly one action for one address.
|
|
311
|
+
// Reached from a mail client; the signed token in the URL is the credential.
|
|
331
312
|
const isPublicUnsubscribe = pathname === "/v1/unsubscribe";
|
|
332
313
|
const isPublicAuth = pathname === "/v1/auth/login";
|
|
333
314
|
|
|
@@ -412,16 +393,15 @@ export async function startApiServer() {
|
|
|
412
393
|
});
|
|
413
394
|
tokenHashToToken.set(tokenHash, token);
|
|
414
395
|
if (tokenHashToToken.size > 2000) {
|
|
415
|
-
//
|
|
416
|
-
// cannot evict it, so the two are dropped together.
|
|
396
|
+
// Cleared together so every cached token stays revocable by hash.
|
|
417
397
|
tokenHashToToken.clear();
|
|
418
398
|
authCache.clear();
|
|
419
399
|
}
|
|
420
400
|
}
|
|
421
401
|
}
|
|
422
402
|
|
|
423
|
-
//
|
|
424
|
-
if (keyRole === "read_only") {
|
|
403
|
+
// Logging out only ends the caller's own session, so read-only users may.
|
|
404
|
+
if (keyRole === "read_only" && pathname !== "/v1/auth/logout") {
|
|
425
405
|
const isMutating = req.method !== "GET" && req.method !== "OPTIONS";
|
|
426
406
|
if (isMutating) {
|
|
427
407
|
sendJson(res, 403, { error: "forbidden", message: "API key is read-only" });
|
|
@@ -429,19 +409,12 @@ export async function startApiServer() {
|
|
|
429
409
|
}
|
|
430
410
|
}
|
|
431
411
|
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
if (!res.headersSent) {
|
|
439
|
-
res.setHeader("Retry-After", "60");
|
|
440
|
-
sendJson(res, 429, {
|
|
441
|
-
error: "too_many_requests",
|
|
442
|
-
message: `Project rate limit exceeded (max ${projectRateLimitRpm} req/min)`,
|
|
443
|
-
});
|
|
444
|
-
}
|
|
412
|
+
if (!(await apiRateLimiter.check(projectId || "global", projectRateLimitRpm))) {
|
|
413
|
+
res.setHeader("Retry-After", "60");
|
|
414
|
+
sendJson(res, 429, {
|
|
415
|
+
error: "too_many_requests",
|
|
416
|
+
message: `Project rate limit exceeded (max ${projectRateLimitRpm} req/min)`,
|
|
417
|
+
});
|
|
445
418
|
return;
|
|
446
419
|
}
|
|
447
420
|
} else if (isProjectManagement) {
|
|
@@ -456,8 +429,7 @@ export async function startApiServer() {
|
|
|
456
429
|
if (token.startsWith("nk_sess_")) {
|
|
457
430
|
const session = await getAdminSession(redis.native, token);
|
|
458
431
|
if (session) {
|
|
459
|
-
//
|
|
460
|
-
// a read-only dashboard user gets to look and nothing more.
|
|
432
|
+
// Minting keys hands out admin credentials, so read-only sessions may only look.
|
|
461
433
|
if (sessionKeyRole(session.role) === "read_only" && req.method !== "GET") {
|
|
462
434
|
sendJson(res, 403, { error: "forbidden", message: "Session is read-only" });
|
|
463
435
|
return;
|
|
@@ -475,10 +447,7 @@ export async function startApiServer() {
|
|
|
475
447
|
isAdminToken = true;
|
|
476
448
|
}
|
|
477
449
|
|
|
478
|
-
//
|
|
479
|
-
// rate limit — they are public by necessity. Cap them per client instead so
|
|
480
|
-
// an open endpoint cannot be used to hammer the process. Generous, because
|
|
481
|
-
// a shared corporate egress IP can legitimately produce a burst.
|
|
450
|
+
// Public, so capped per client instead of per project; generous for shared corporate IPs.
|
|
482
451
|
if (isPublicUnsubscribe) {
|
|
483
452
|
const clientIp = getClientIp(req, config.TRUST_PROXY);
|
|
484
453
|
try {
|
|
@@ -490,14 +459,12 @@ export async function startApiServer() {
|
|
|
490
459
|
return;
|
|
491
460
|
}
|
|
492
461
|
} catch (err) {
|
|
493
|
-
//
|
|
494
|
-
// here is the lesser evil against a recipient who cannot opt out.
|
|
462
|
+
// Fail open: a recipient who cannot opt out is worse than an unthrottled endpoint.
|
|
495
463
|
logger.warn({ err }, "unsubscribe rate limit unavailable — allowing request");
|
|
496
464
|
}
|
|
497
465
|
}
|
|
498
466
|
|
|
499
|
-
//
|
|
500
|
-
// Throttle login attempts per IP to prevent credential brute-forcing and CPU exhaustion.
|
|
467
|
+
// Per-IP cap against brute force; each attempt also costs a scrypt hash.
|
|
501
468
|
if (isPublicAuth) {
|
|
502
469
|
const clientIp = getClientIp(req, config.TRUST_PROXY);
|
|
503
470
|
try {
|
|
@@ -601,8 +568,6 @@ export async function startApiServer() {
|
|
|
601
568
|
});
|
|
602
569
|
}
|
|
603
570
|
|
|
604
|
-
// ─── Shutdown ──────────────────────────────────────────────────────────────
|
|
605
|
-
|
|
606
571
|
export async function stopApiServer(): Promise<void> {
|
|
607
572
|
logger?.info("api shutdown initiated");
|
|
608
573
|
if (server) {
|
|
@@ -618,8 +583,7 @@ export async function stopApiServer(): Promise<void> {
|
|
|
618
583
|
}
|
|
619
584
|
await liveEvents?.close();
|
|
620
585
|
liveEvents = null;
|
|
621
|
-
//
|
|
622
|
-
// the previous instance's cached auth.
|
|
586
|
+
// Module-scoped, so a restart in the same process must not inherit cached auth.
|
|
623
587
|
authCache = new LRUCache<string, CachedAuthKey>(1000, AUTH_CACHE_TTL_MS);
|
|
624
588
|
tokenHashToToken.clear();
|
|
625
589
|
if (sql) await sql.end();
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
2
2
|
import { inArray } from "drizzle-orm";
|
|
3
|
+
import type { Db } from "@/db/index.js";
|
|
3
4
|
import type { Logger } from "@/logger/index.js";
|
|
4
5
|
import type { Transport } from "@/transport/index.js";
|
|
5
6
|
import { messageLogs, suppressions } from "@/db/schema.js";
|
|
6
7
|
import { normaliseTarget } from "@/shared/index.js";
|
|
7
8
|
import { readRawBody, sendJson } from "./http.js";
|
|
8
9
|
|
|
9
|
-
/** The POST route a provider delivers webhook events to. */
|
|
10
10
|
export function createWebhookHandler(
|
|
11
11
|
transport: Transport,
|
|
12
12
|
channel: string,
|
|
13
|
-
deps: { db:
|
|
13
|
+
deps: { db: Db; logger: Logger },
|
|
14
14
|
) {
|
|
15
15
|
const { db, logger } = deps;
|
|
16
16
|
return async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
|
|
@@ -64,17 +64,13 @@ export function createWebhookHandler(
|
|
|
64
64
|
if (row.providerMessageId) {
|
|
65
65
|
logsToProject.set(row.providerMessageId, row.projectId);
|
|
66
66
|
logsToTask.set(row.providerMessageId, row.taskId);
|
|
67
|
-
//
|
|
68
|
-
// about, or an open never counts toward the send that
|
|
69
|
-
// earned it.
|
|
67
|
+
// So an open counts toward the campaign of the send it is about.
|
|
70
68
|
logsToCampaign.set(row.providerMessageId, row.campaignId);
|
|
71
69
|
}
|
|
72
70
|
}
|
|
73
71
|
}
|
|
74
72
|
|
|
75
|
-
//
|
|
76
|
-
// unresolvable providerMessageId used to be written against a
|
|
77
|
-
// zero-UUID project, which is unjoinable and leaks across tenants.
|
|
73
|
+
// Unattributable events are dropped: there is no project to store them under.
|
|
78
74
|
const attributable = events.filter(
|
|
79
75
|
(e) => e.providerMessageId && logsToProject.has(e.providerMessageId),
|
|
80
76
|
);
|
|
@@ -101,14 +97,11 @@ export function createWebhookHandler(
|
|
|
101
97
|
}
|
|
102
98
|
|
|
103
99
|
if (rows.length > 0) {
|
|
104
|
-
// Providers
|
|
105
|
-
// already recorded must be a no-op rather than a constraint error.
|
|
100
|
+
// Providers redeliver on any non-2xx; a repeat must be a no-op.
|
|
106
101
|
await db.insert(messageLogs).values(rows).onConflictDoNothing();
|
|
107
102
|
}
|
|
108
103
|
|
|
109
|
-
//
|
|
110
|
-
// address must stop future sends — logging it is not enough.
|
|
111
|
-
// Soft bounces are excluded: the address is still good.
|
|
104
|
+
// Stop future sends on unsubscribes, complaints and hard bounces (not soft ones).
|
|
112
105
|
const suppressionRows = attributable
|
|
113
106
|
.filter(
|
|
114
107
|
(e) =>
|
|
@@ -127,8 +120,7 @@ export function createWebhookHandler(
|
|
|
127
120
|
}));
|
|
128
121
|
|
|
129
122
|
if (suppressionRows.length > 0) {
|
|
130
|
-
// First reason wins: a later bounce
|
|
131
|
-
// record that the person actively unsubscribed.
|
|
123
|
+
// First reason wins: a later bounce must not hide an unsubscribe.
|
|
132
124
|
await db.insert(suppressions).values(suppressionRows).onConflictDoNothing();
|
|
133
125
|
logger.info(
|
|
134
126
|
{ channel, count: suppressionRows.length },
|
|
@@ -144,8 +136,6 @@ export function createWebhookHandler(
|
|
|
144
136
|
e.status === "bounced"),
|
|
145
137
|
).length;
|
|
146
138
|
if (unsuppressable > 0) {
|
|
147
|
-
// Worth shouting about: the event arrived, we logged it, and
|
|
148
|
-
// we still cannot stop mailing the person.
|
|
149
139
|
logger.warn(
|
|
150
140
|
{ channel, count: unsuppressable },
|
|
151
141
|
"suppression events carried no recipient address — transport must set WebhookEvent.recipient",
|
|
@@ -159,8 +149,7 @@ export function createWebhookHandler(
|
|
|
159
149
|
}),
|
|
160
150
|
]);
|
|
161
151
|
} catch (err) {
|
|
162
|
-
//
|
|
163
|
-
// response bodies in their delivery logs.
|
|
152
|
+
// No detail in the body: providers keep response bodies in their logs.
|
|
164
153
|
logger.error({ err, channel }, "webhook processing error or timeout");
|
|
165
154
|
if (!res.headersSent) {
|
|
166
155
|
sendJson(res, 500, { error: "internal_error" });
|