@hogsend/engine 0.37.2 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/index.ts +8 -0
- package/src/lib/preferences.ts +28 -1
- package/src/lib/tracked.ts +20 -7
- package/src/routes/admin/bulk.ts +37 -7
- package/src/routes/admin/suppressions.ts +158 -6
- package/src/worker.ts +2 -0
- package/src/workflows/import-suppressions.ts +256 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hogsend/engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.38.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -42,14 +42,14 @@
|
|
|
42
42
|
"svix": "^1.95.1",
|
|
43
43
|
"winston": "^3.19.0",
|
|
44
44
|
"zod": "^4.4.3",
|
|
45
|
-
"@hogsend/core": "^0.
|
|
46
|
-
"@hogsend/db": "^0.
|
|
47
|
-
"@hogsend/email": "^0.
|
|
48
|
-
"@hogsend/plugin-posthog": "^0.
|
|
49
|
-
"@hogsend/plugin-resend": "^0.
|
|
45
|
+
"@hogsend/core": "^0.38.0",
|
|
46
|
+
"@hogsend/db": "^0.38.0",
|
|
47
|
+
"@hogsend/email": "^0.38.0",
|
|
48
|
+
"@hogsend/plugin-posthog": "^0.38.0",
|
|
49
|
+
"@hogsend/plugin-resend": "^0.38.0"
|
|
50
50
|
},
|
|
51
51
|
"optionalDependencies": {
|
|
52
|
-
"@hogsend/plugin-postmark": "^0.
|
|
52
|
+
"@hogsend/plugin-postmark": "^0.38.0"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
55
|
"@types/node": "^22.15.3",
|
package/src/index.ts
CHANGED
|
@@ -564,6 +564,14 @@ export {
|
|
|
564
564
|
reapDueWebhookDeliveriesTask,
|
|
565
565
|
} from "./workflows/deliver-webhook.js";
|
|
566
566
|
export { importContactsTask } from "./workflows/import-contacts.js";
|
|
567
|
+
export {
|
|
568
|
+
importSuppressionsTask,
|
|
569
|
+
type MappedSuppressionRow,
|
|
570
|
+
mapSuppressionRow,
|
|
571
|
+
SUPPRESSION_REASONS,
|
|
572
|
+
type SuppressionImportRow,
|
|
573
|
+
type SuppressionReason,
|
|
574
|
+
} from "./workflows/import-suppressions.js";
|
|
567
575
|
export { sendCampaignTask } from "./workflows/send-campaign.js";
|
|
568
576
|
// --- Built-in Hatchet workflow tasks ---
|
|
569
577
|
export { sendEmailTask } from "./workflows/send-email.js";
|
package/src/lib/preferences.ts
CHANGED
|
@@ -18,6 +18,11 @@ const logger = createLogger(process.env.LOG_LEVEL);
|
|
|
18
18
|
* `externalId` is the `user_id` column value: the contact's `external_id` when it
|
|
19
19
|
* has one, else the contact `id` (uuid) fallback for an email-only contact (risk
|
|
20
20
|
* 10). `email` is REQUIRED — both columns are NOT NULL and form the PK.
|
|
21
|
+
*
|
|
22
|
+
* `emitOutbound` (default true) gates the `contact.unsubscribed` outbound emit.
|
|
23
|
+
* Bulk historical imports (import-suppressions) pass false: a 50k-row import
|
|
24
|
+
* must not fan out 50k opt-out events for opt-outs that happened on another
|
|
25
|
+
* platform months ago. Every interactive caller keeps the default.
|
|
21
26
|
*/
|
|
22
27
|
export async function upsertEmailPreference(opts: {
|
|
23
28
|
db: Database;
|
|
@@ -26,9 +31,18 @@ export async function upsertEmailPreference(opts: {
|
|
|
26
31
|
update: {
|
|
27
32
|
unsubscribedAll?: boolean;
|
|
28
33
|
suppressed?: boolean;
|
|
34
|
+
/** Set `suppressed_at` explicitly (used alongside `suppressed: true`). */
|
|
35
|
+
suppressedAt?: Date;
|
|
36
|
+
/**
|
|
37
|
+
* Record an imported hard bounce: `bounce_count = GREATEST(bounce_count, 1)`
|
|
38
|
+
* + `last_bounce_at = now`. GREATEST (not increment) so re-running an import
|
|
39
|
+
* is idempotent and never inflates a genuine bounce history.
|
|
40
|
+
*/
|
|
41
|
+
recordBounce?: boolean;
|
|
29
42
|
categoryKey?: string;
|
|
30
43
|
categoryValue?: boolean;
|
|
31
44
|
};
|
|
45
|
+
emitOutbound?: boolean;
|
|
32
46
|
}): Promise<void> {
|
|
33
47
|
const { db, externalId, email, update } = opts;
|
|
34
48
|
|
|
@@ -40,6 +54,13 @@ export async function upsertEmailPreference(opts: {
|
|
|
40
54
|
if (update.suppressed !== undefined) {
|
|
41
55
|
setClause.suppressed = update.suppressed;
|
|
42
56
|
}
|
|
57
|
+
if (update.suppressedAt !== undefined) {
|
|
58
|
+
setClause.suppressedAt = update.suppressedAt;
|
|
59
|
+
}
|
|
60
|
+
if (update.recordBounce) {
|
|
61
|
+
setClause.bounceCount = sql`GREATEST(${emailPreferences.bounceCount}, 1)`;
|
|
62
|
+
setClause.lastBounceAt = new Date();
|
|
63
|
+
}
|
|
43
64
|
if (update.categoryKey !== undefined) {
|
|
44
65
|
const jsonValue = update.categoryValue ? "true" : "false";
|
|
45
66
|
setClause.categories = sql`jsonb_set(COALESCE(${emailPreferences.categories}, '{}'::jsonb), ${`{${update.categoryKey}}`}, ${jsonValue}::jsonb)`;
|
|
@@ -56,6 +77,12 @@ export async function upsertEmailPreference(opts: {
|
|
|
56
77
|
...(update.suppressed !== undefined
|
|
57
78
|
? { suppressed: update.suppressed }
|
|
58
79
|
: {}),
|
|
80
|
+
...(update.suppressedAt !== undefined
|
|
81
|
+
? { suppressedAt: update.suppressedAt }
|
|
82
|
+
: {}),
|
|
83
|
+
...(update.recordBounce
|
|
84
|
+
? { bounceCount: 1, lastBounceAt: new Date() }
|
|
85
|
+
: {}),
|
|
59
86
|
...(update.categoryKey !== undefined
|
|
60
87
|
? {
|
|
61
88
|
categories: { [update.categoryKey]: update.categoryValue ?? false },
|
|
@@ -76,7 +103,7 @@ export async function upsertEmailPreference(opts: {
|
|
|
76
103
|
// fire-and-forget so a transient outbound error never fails the pref write.
|
|
77
104
|
const isUnsubscribe =
|
|
78
105
|
update.unsubscribedAll === true || update.categoryValue === false;
|
|
79
|
-
if (isUnsubscribe) {
|
|
106
|
+
if (isUnsubscribe && (opts.emitOutbound ?? true)) {
|
|
80
107
|
const scope: "all" | "category" =
|
|
81
108
|
update.unsubscribedAll === true ? "all" : "category";
|
|
82
109
|
void emitOutbound({
|
package/src/lib/tracked.ts
CHANGED
|
@@ -521,16 +521,19 @@ async function checkSuppression(
|
|
|
521
521
|
email: string,
|
|
522
522
|
category?: string,
|
|
523
523
|
): Promise<SuppressionReason> {
|
|
524
|
+
// An address can legitimately have MORE THAN ONE `email_preferences` row:
|
|
525
|
+
// the PK is (user_id, email), and a suppression imported before the contact
|
|
526
|
+
// existed is keyed (email, email) while later interactive writes key
|
|
527
|
+
// (external_id, email). Aggregate across ALL rows — any suppression signal
|
|
528
|
+
// on any row blocks the send — so an imported unsubscribe/bounce can never
|
|
529
|
+
// be shadowed by a newer clean row for the same address.
|
|
524
530
|
const rows = await db
|
|
525
531
|
.select()
|
|
526
532
|
.from(emailPreferences)
|
|
527
|
-
.where(eq(emailPreferences.email, email))
|
|
528
|
-
.limit(1);
|
|
533
|
+
.where(eq(emailPreferences.email, email));
|
|
529
534
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
if (prefs?.suppressed) return "suppressed";
|
|
533
|
-
if (prefs?.unsubscribedAll) return "unsubscribed";
|
|
535
|
+
if (rows.some((prefs) => prefs.suppressed)) return "suppressed";
|
|
536
|
+
if (rows.some((prefs) => prefs.unsubscribedAll)) return "unsubscribed";
|
|
534
537
|
|
|
535
538
|
// Registry-aware polarity (§2.6, D3) — applied through the SINGLE source of
|
|
536
539
|
// truth `ListRegistry.isSubscribed` so it matches the preference center EXACTLY
|
|
@@ -540,8 +543,18 @@ async function checkSuppression(
|
|
|
540
543
|
// so absence-of-true (the common "never opted in" case) MUST block — otherwise
|
|
541
544
|
// a contact the preference center shows as "Unsubscribed" would still receive
|
|
542
545
|
// the mail (the two surfaces would disagree, which §2.6 forbids).
|
|
546
|
+
// Category maps are merged across rows with explicit FALSE winning: an
|
|
547
|
+
// opt-out recorded on any row blocks, matching the conservative aggregation
|
|
548
|
+
// above. (In the common single-row case this is identical to reading the
|
|
549
|
+
// row's map directly.)
|
|
543
550
|
if (category) {
|
|
544
|
-
const categories
|
|
551
|
+
const categories: Record<string, boolean> = {};
|
|
552
|
+
for (const prefs of rows) {
|
|
553
|
+
const map = (prefs.categories ?? {}) as Record<string, boolean>;
|
|
554
|
+
for (const [key, value] of Object.entries(map)) {
|
|
555
|
+
categories[key] = (categories[key] ?? true) && value;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
545
558
|
if (!getListRegistry().isSubscribed(categories, category)) {
|
|
546
559
|
return "category_unsubscribed";
|
|
547
560
|
}
|
package/src/routes/admin/bulk.ts
CHANGED
|
@@ -20,7 +20,13 @@ const importRoute = createRoute({
|
|
|
20
20
|
"application/json": {
|
|
21
21
|
schema: z.object({
|
|
22
22
|
format: z.enum(["csv", "json"]),
|
|
23
|
-
|
|
23
|
+
// Cap matches Hatchet's ~4MB gRPC message ceiling — a bigger
|
|
24
|
+
// payload could never be enqueued anyway. Split large lists into
|
|
25
|
+
// multiple jobs (the CLI chunks at 5,000 rows per job).
|
|
26
|
+
data: z
|
|
27
|
+
.string()
|
|
28
|
+
.min(1)
|
|
29
|
+
.max(4_000_000, "data exceeds 4MB — split into smaller jobs"),
|
|
24
30
|
fileName: z.string().optional(),
|
|
25
31
|
}),
|
|
26
32
|
},
|
|
@@ -241,7 +247,7 @@ const batchEnrollRoute = createRoute({
|
|
|
241
247
|
|
|
242
248
|
export const bulkRouter = new OpenAPIHono<AppEnv>()
|
|
243
249
|
.openapi(importRoute, async (c) => {
|
|
244
|
-
const { db } = c.get("container");
|
|
250
|
+
const { db, logger } = c.get("container");
|
|
245
251
|
const body = c.req.valid("json");
|
|
246
252
|
|
|
247
253
|
const [job] = await db
|
|
@@ -254,11 +260,35 @@ export const bulkRouter = new OpenAPIHono<AppEnv>()
|
|
|
254
260
|
|
|
255
261
|
if (!job) throw new Error("Failed to create import job");
|
|
256
262
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
263
|
+
// Fire-and-forget (`runNoWait`, not the old awaited `run`): the route's
|
|
264
|
+
// contract is a 202 "queued" — awaiting the task to completion before
|
|
265
|
+
// responding defeated the async job + status-poll design on large imports.
|
|
266
|
+
// A failed enqueue marks the durable job row `failed` so pollers get a
|
|
267
|
+
// terminal state instead of `pending` forever.
|
|
268
|
+
void importContactsTask
|
|
269
|
+
.runNoWait({ jobId: job.id, data: body.data, format: body.format })
|
|
270
|
+
.catch(async (error: unknown) => {
|
|
271
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
272
|
+
logger.warn("contacts/import: task enqueue failed", {
|
|
273
|
+
jobId: job.id,
|
|
274
|
+
error: message,
|
|
275
|
+
});
|
|
276
|
+
try {
|
|
277
|
+
await db
|
|
278
|
+
.update(importJobs)
|
|
279
|
+
.set({
|
|
280
|
+
status: "failed",
|
|
281
|
+
errors: [{ row: 0, error: `Task enqueue failed: ${message}` }],
|
|
282
|
+
updatedAt: new Date(),
|
|
283
|
+
})
|
|
284
|
+
.where(eq(importJobs.id, job.id));
|
|
285
|
+
} catch (dbError: unknown) {
|
|
286
|
+
logger.warn("contacts/import: could not mark job failed", {
|
|
287
|
+
jobId: job.id,
|
|
288
|
+
error: dbError instanceof Error ? dbError.message : String(dbError),
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
});
|
|
262
292
|
|
|
263
293
|
return c.json({ jobId: job.id, status: "pending" }, 202);
|
|
264
294
|
})
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { emailPreferences } from "@hogsend/db";
|
|
1
|
+
import { emailPreferences, importJobs } from "@hogsend/db";
|
|
2
2
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
3
|
import { and, count, desc, eq, gt, or, type SQL } from "drizzle-orm";
|
|
4
4
|
import type { AppEnv } from "../../app.js";
|
|
5
5
|
import { serializePrefs } from "../../lib/contacts.js";
|
|
6
|
+
import { errorSchema } from "../../lib/schemas.js";
|
|
7
|
+
import { importSuppressionsTask } from "../../workflows/import-suppressions.js";
|
|
6
8
|
|
|
7
9
|
// Maps the requested suppression type to a predicate over email_preferences.
|
|
8
10
|
// `complained` has no dedicated column — a complaint sets `suppressed` without
|
|
@@ -77,9 +79,86 @@ const listSuppressionsRoute = createRoute({
|
|
|
77
79
|
},
|
|
78
80
|
});
|
|
79
81
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
82
|
+
const importSuppressionsRoute = createRoute({
|
|
83
|
+
method: "post",
|
|
84
|
+
path: "/import",
|
|
85
|
+
tags: ["Admin — Suppressions"],
|
|
86
|
+
summary: "Bulk import a suppression list",
|
|
87
|
+
description:
|
|
88
|
+
"Queues an async import of unsubscribes / bounces / complaints into email_preferences. " +
|
|
89
|
+
"Rows: email (required), reason (unsubscribed | bounced | complained, default unsubscribed), externalId (optional). " +
|
|
90
|
+
"Historical imports do NOT emit per-row contact.unsubscribed events.",
|
|
91
|
+
request: {
|
|
92
|
+
body: {
|
|
93
|
+
content: {
|
|
94
|
+
"application/json": {
|
|
95
|
+
schema: z.object({
|
|
96
|
+
format: z.enum(["csv", "json"]),
|
|
97
|
+
// Cap matches Hatchet's ~4MB gRPC message ceiling — a bigger
|
|
98
|
+
// payload could never be enqueued anyway. Split large lists into
|
|
99
|
+
// multiple jobs (the CLI chunks at 5,000 rows per job).
|
|
100
|
+
data: z
|
|
101
|
+
.string()
|
|
102
|
+
.min(1)
|
|
103
|
+
.max(4_000_000, "data exceeds 4MB — split into smaller jobs"),
|
|
104
|
+
fileName: z.string().optional(),
|
|
105
|
+
}),
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
responses: {
|
|
111
|
+
202: {
|
|
112
|
+
content: {
|
|
113
|
+
"application/json": {
|
|
114
|
+
schema: z.object({
|
|
115
|
+
jobId: z.string(),
|
|
116
|
+
status: z.string(),
|
|
117
|
+
}),
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
description: "Import job queued",
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const importStatusRoute = createRoute({
|
|
126
|
+
method: "get",
|
|
127
|
+
path: "/import/{jobId}",
|
|
128
|
+
tags: ["Admin — Suppressions"],
|
|
129
|
+
summary: "Get suppression import job status",
|
|
130
|
+
request: {
|
|
131
|
+
params: z.object({ jobId: z.string().uuid() }),
|
|
132
|
+
},
|
|
133
|
+
responses: {
|
|
134
|
+
200: {
|
|
135
|
+
content: {
|
|
136
|
+
"application/json": {
|
|
137
|
+
schema: z.object({
|
|
138
|
+
id: z.string(),
|
|
139
|
+
status: z.string(),
|
|
140
|
+
totalRows: z.number().nullable(),
|
|
141
|
+
processedRows: z.number(),
|
|
142
|
+
failedRows: z.number(),
|
|
143
|
+
errors: z
|
|
144
|
+
.array(z.object({ row: z.number(), error: z.string() }))
|
|
145
|
+
.nullable(),
|
|
146
|
+
createdAt: z.string(),
|
|
147
|
+
updatedAt: z.string(),
|
|
148
|
+
}),
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
description: "Import job details",
|
|
152
|
+
},
|
|
153
|
+
404: {
|
|
154
|
+
content: { "application/json": { schema: errorSchema } },
|
|
155
|
+
description: "Import job not found",
|
|
156
|
+
},
|
|
157
|
+
},
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
export const suppressionsRouter = new OpenAPIHono<AppEnv>()
|
|
161
|
+
.openapi(listSuppressionsRoute, async (c) => {
|
|
83
162
|
const { db } = c.get("container");
|
|
84
163
|
const { type, limit, offset } = c.req.valid("query");
|
|
85
164
|
|
|
@@ -105,5 +184,78 @@ export const suppressionsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
105
184
|
},
|
|
106
185
|
200,
|
|
107
186
|
);
|
|
108
|
-
}
|
|
109
|
-
)
|
|
187
|
+
})
|
|
188
|
+
.openapi(importSuppressionsRoute, async (c) => {
|
|
189
|
+
const { db, logger } = c.get("container");
|
|
190
|
+
const body = c.req.valid("json");
|
|
191
|
+
|
|
192
|
+
const [job] = await db
|
|
193
|
+
.insert(importJobs)
|
|
194
|
+
.values({
|
|
195
|
+
format: body.format,
|
|
196
|
+
fileName: body.fileName ?? null,
|
|
197
|
+
})
|
|
198
|
+
.returning();
|
|
199
|
+
|
|
200
|
+
if (!job) throw new Error("Failed to create import job");
|
|
201
|
+
|
|
202
|
+
// Enqueue fire-and-forget: the 202 means "queued", so a slow/unreachable
|
|
203
|
+
// broker never blocks the response. Unlike the outbound emit spine, a
|
|
204
|
+
// failed enqueue here has a durable job row to carry the failure — mark it
|
|
205
|
+
// `failed` so pollers get a terminal state instead of `pending` forever.
|
|
206
|
+
void importSuppressionsTask
|
|
207
|
+
.runNoWait({ jobId: job.id, data: body.data, format: body.format })
|
|
208
|
+
.catch(async (error: unknown) => {
|
|
209
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
210
|
+
logger.warn("suppressions/import: task enqueue failed", {
|
|
211
|
+
jobId: job.id,
|
|
212
|
+
error: message,
|
|
213
|
+
});
|
|
214
|
+
try {
|
|
215
|
+
await db
|
|
216
|
+
.update(importJobs)
|
|
217
|
+
.set({
|
|
218
|
+
status: "failed",
|
|
219
|
+
errors: [{ row: 0, error: `Task enqueue failed: ${message}` }],
|
|
220
|
+
updatedAt: new Date(),
|
|
221
|
+
})
|
|
222
|
+
.where(eq(importJobs.id, job.id));
|
|
223
|
+
} catch (dbError: unknown) {
|
|
224
|
+
logger.warn("suppressions/import: could not mark job failed", {
|
|
225
|
+
jobId: job.id,
|
|
226
|
+
error: dbError instanceof Error ? dbError.message : String(dbError),
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
return c.json({ jobId: job.id, status: "pending" }, 202);
|
|
232
|
+
})
|
|
233
|
+
.openapi(importStatusRoute, async (c) => {
|
|
234
|
+
const { db } = c.get("container");
|
|
235
|
+
const { jobId } = c.req.valid("param");
|
|
236
|
+
|
|
237
|
+
const rows = await db
|
|
238
|
+
.select()
|
|
239
|
+
.from(importJobs)
|
|
240
|
+
.where(eq(importJobs.id, jobId))
|
|
241
|
+
.limit(1);
|
|
242
|
+
|
|
243
|
+
const job = rows[0];
|
|
244
|
+
if (!job) {
|
|
245
|
+
return c.json({ error: "Import job not found" }, 404);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
return c.json(
|
|
249
|
+
{
|
|
250
|
+
id: job.id,
|
|
251
|
+
status: job.status,
|
|
252
|
+
totalRows: job.totalRows,
|
|
253
|
+
processedRows: job.processedRows,
|
|
254
|
+
failedRows: job.failedRows,
|
|
255
|
+
errors: job.errors as Array<{ row: number; error: string }> | null,
|
|
256
|
+
createdAt: job.createdAt.toISOString(),
|
|
257
|
+
updatedAt: job.updatedAt.toISOString(),
|
|
258
|
+
},
|
|
259
|
+
200,
|
|
260
|
+
);
|
|
261
|
+
});
|
package/src/worker.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
reapDueWebhookDeliveriesTask,
|
|
27
27
|
} from "./workflows/deliver-webhook.js";
|
|
28
28
|
import { importContactsTask } from "./workflows/import-contacts.js";
|
|
29
|
+
import { importSuppressionsTask } from "./workflows/import-suppressions.js";
|
|
29
30
|
import {
|
|
30
31
|
reapStuckCampaignsTask,
|
|
31
32
|
sendCampaignTask,
|
|
@@ -92,6 +93,7 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
92
93
|
sendEmailTask,
|
|
93
94
|
sendFeedTask,
|
|
94
95
|
importContactsTask,
|
|
96
|
+
importSuppressionsTask,
|
|
95
97
|
sendCampaignTask,
|
|
96
98
|
reapStuckCampaignsTask,
|
|
97
99
|
deliverWebhookTask,
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
import { createDatabase, importJobs } from "@hogsend/db";
|
|
2
|
+
import { eq } from "drizzle-orm";
|
|
3
|
+
import Papa from "papaparse";
|
|
4
|
+
import { resolveRecipient } from "../lib/contacts.js";
|
|
5
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
6
|
+
import { upsertEmailPreference } from "../lib/preferences.js";
|
|
7
|
+
|
|
8
|
+
const BATCH_SIZE = 500;
|
|
9
|
+
|
|
10
|
+
export const SUPPRESSION_REASONS = [
|
|
11
|
+
"unsubscribed",
|
|
12
|
+
"bounced",
|
|
13
|
+
"complained",
|
|
14
|
+
] as const;
|
|
15
|
+
|
|
16
|
+
export type SuppressionReason = (typeof SUPPRESSION_REASONS)[number];
|
|
17
|
+
|
|
18
|
+
/** One raw input row (a parsed CSV record or a JSON array element). */
|
|
19
|
+
export interface SuppressionImportRow {
|
|
20
|
+
email?: string;
|
|
21
|
+
reason?: string;
|
|
22
|
+
externalId?: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The `email_preferences` update a validated row maps to. Timestamp columns
|
|
27
|
+
* are expressed as flags so the mapping stays pure (the task applies `now`):
|
|
28
|
+
* `recordBounce` = `bounce_count = GREATEST(bounce_count, 1)` + `last_bounce_at`,
|
|
29
|
+
* `setSuppressedAt` = `suppressed_at = now`.
|
|
30
|
+
*/
|
|
31
|
+
export interface MappedSuppressionRow {
|
|
32
|
+
email: string;
|
|
33
|
+
externalId?: string;
|
|
34
|
+
reason: SuppressionReason;
|
|
35
|
+
update: {
|
|
36
|
+
unsubscribedAll?: boolean;
|
|
37
|
+
suppressed?: boolean;
|
|
38
|
+
setSuppressedAt?: boolean;
|
|
39
|
+
recordBounce?: boolean;
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Deliberately loose: enough to reject rows that can't be an address at all
|
|
44
|
+
// (no `@`, no domain dot) without bouncing real-world addresses. The engine
|
|
45
|
+
// stores normalized-raw emails (see `normalizeEmail`), it does not verify them.
|
|
46
|
+
const EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Pure row → preference-update mapping for the suppression import. Validates
|
|
50
|
+
* and lowercases `email`, defaults `reason` to `unsubscribed`, and maps each
|
|
51
|
+
* reason onto the existing derived `email_preferences` semantics (NO schema
|
|
52
|
+
* change — see `routes/admin/suppressions.ts` typeFilter):
|
|
53
|
+
*
|
|
54
|
+
* - `unsubscribed` → `unsubscribed_all = true`
|
|
55
|
+
* - `bounced` → `suppressed = true` + bounce slate (`bounce_count >= 1`)
|
|
56
|
+
* - `complained` → `suppressed = true` with `bounce_count` untouched, so the
|
|
57
|
+
* derived "complained" view (suppressed AND bounceCount = 0)
|
|
58
|
+
* still identifies it
|
|
59
|
+
*
|
|
60
|
+
* Throws on an invalid row (missing/malformed email, unknown reason); the task
|
|
61
|
+
* records the error against the row index.
|
|
62
|
+
*/
|
|
63
|
+
export function mapSuppressionRow(
|
|
64
|
+
row: SuppressionImportRow,
|
|
65
|
+
): MappedSuppressionRow {
|
|
66
|
+
const email = row.email?.trim().toLowerCase();
|
|
67
|
+
if (!email) {
|
|
68
|
+
throw new Error("Row has no email");
|
|
69
|
+
}
|
|
70
|
+
if (!EMAIL_REGEX.test(email)) {
|
|
71
|
+
throw new Error(`Invalid email: ${email}`);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const reason = (row.reason?.trim().toLowerCase() ||
|
|
75
|
+
"unsubscribed") as SuppressionReason;
|
|
76
|
+
if (!SUPPRESSION_REASONS.includes(reason)) {
|
|
77
|
+
throw new Error(
|
|
78
|
+
`Invalid reason "${row.reason}" — expected one of: ${SUPPRESSION_REASONS.join(", ")}`,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const externalId = row.externalId?.trim() || undefined;
|
|
83
|
+
|
|
84
|
+
switch (reason) {
|
|
85
|
+
case "unsubscribed":
|
|
86
|
+
return { email, externalId, reason, update: { unsubscribedAll: true } };
|
|
87
|
+
case "bounced":
|
|
88
|
+
return {
|
|
89
|
+
email,
|
|
90
|
+
externalId,
|
|
91
|
+
reason,
|
|
92
|
+
update: { suppressed: true, setSuppressedAt: true, recordBounce: true },
|
|
93
|
+
};
|
|
94
|
+
case "complained":
|
|
95
|
+
return {
|
|
96
|
+
email,
|
|
97
|
+
externalId,
|
|
98
|
+
reason,
|
|
99
|
+
update: { suppressed: true, setSuppressedAt: true },
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export const importSuppressionsTask = hatchet.task({
|
|
105
|
+
name: "import-suppressions",
|
|
106
|
+
retries: 0,
|
|
107
|
+
executionTimeout: "600s",
|
|
108
|
+
fn: async (input: { jobId: string; data: string; format: string }) => {
|
|
109
|
+
const { db } = createDatabase({
|
|
110
|
+
url: process.env.DATABASE_URL ?? "",
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
await db
|
|
114
|
+
.update(importJobs)
|
|
115
|
+
.set({ status: "processing", updatedAt: new Date() })
|
|
116
|
+
.where(eq(importJobs.id, input.jobId));
|
|
117
|
+
|
|
118
|
+
let rows: SuppressionImportRow[];
|
|
119
|
+
|
|
120
|
+
try {
|
|
121
|
+
if (input.format === "json") {
|
|
122
|
+
rows = JSON.parse(input.data);
|
|
123
|
+
} else {
|
|
124
|
+
rows = parseCsv(input.data);
|
|
125
|
+
}
|
|
126
|
+
} catch (err) {
|
|
127
|
+
await db
|
|
128
|
+
.update(importJobs)
|
|
129
|
+
.set({
|
|
130
|
+
status: "failed",
|
|
131
|
+
errors: [
|
|
132
|
+
{
|
|
133
|
+
row: 0,
|
|
134
|
+
error: `Parse error: ${err instanceof Error ? err.message : String(err)}`,
|
|
135
|
+
},
|
|
136
|
+
],
|
|
137
|
+
updatedAt: new Date(),
|
|
138
|
+
})
|
|
139
|
+
.where(eq(importJobs.id, input.jobId));
|
|
140
|
+
return { status: "failed" };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
await db
|
|
144
|
+
.update(importJobs)
|
|
145
|
+
.set({ totalRows: rows.length, updatedAt: new Date() })
|
|
146
|
+
.where(eq(importJobs.id, input.jobId));
|
|
147
|
+
|
|
148
|
+
let processed = 0;
|
|
149
|
+
let failed = 0;
|
|
150
|
+
const errors: Array<{ row: number; error: string }> = [];
|
|
151
|
+
|
|
152
|
+
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
|
153
|
+
const batch = rows.slice(i, i + BATCH_SIZE);
|
|
154
|
+
const results = await Promise.allSettled(
|
|
155
|
+
batch.map((row) => applySuppressionRow(db, row)),
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
results.forEach((result, batchIdx) => {
|
|
159
|
+
if (result.status === "fulfilled") {
|
|
160
|
+
processed++;
|
|
161
|
+
} else {
|
|
162
|
+
failed++;
|
|
163
|
+
errors.push({
|
|
164
|
+
row: i + batchIdx,
|
|
165
|
+
error: result.reason?.message ?? "Unknown error",
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
await db
|
|
171
|
+
.update(importJobs)
|
|
172
|
+
.set({
|
|
173
|
+
processedRows: processed,
|
|
174
|
+
failedRows: failed,
|
|
175
|
+
updatedAt: new Date(),
|
|
176
|
+
})
|
|
177
|
+
.where(eq(importJobs.id, input.jobId));
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
await db
|
|
181
|
+
.update(importJobs)
|
|
182
|
+
.set({
|
|
183
|
+
status: failed === rows.length ? "failed" : "completed",
|
|
184
|
+
processedRows: processed,
|
|
185
|
+
failedRows: failed,
|
|
186
|
+
errors: errors.length > 0 ? errors.slice(0, 100) : null,
|
|
187
|
+
updatedAt: new Date(),
|
|
188
|
+
})
|
|
189
|
+
.where(eq(importJobs.id, input.jobId));
|
|
190
|
+
|
|
191
|
+
return { status: "completed", processed, failed };
|
|
192
|
+
},
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
type Db = ReturnType<typeof createDatabase>["db"];
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Map one row and write it through the single preference choke point.
|
|
199
|
+
*
|
|
200
|
+
* `user_id` resolution follows the existing convention (lib/preferences.ts):
|
|
201
|
+
* resolve the contact by email — its `external_id` when present, else the
|
|
202
|
+
* contact uuid. When NO contact exists the row is still written
|
|
203
|
+
* (`checkSuppression` keys on email alone at send time) using the row's
|
|
204
|
+
* `externalId` if given, else the email itself — the same synthesized fallback
|
|
205
|
+
* `resolveRecipient` uses for a never-seen address.
|
|
206
|
+
*
|
|
207
|
+
* `emitOutbound: false` — a historical import must not fan out per-row
|
|
208
|
+
* `contact.unsubscribed` events.
|
|
209
|
+
*/
|
|
210
|
+
async function applySuppressionRow(
|
|
211
|
+
db: Db,
|
|
212
|
+
row: SuppressionImportRow,
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
const mapped = mapSuppressionRow(row);
|
|
215
|
+
|
|
216
|
+
const recipient = await resolveRecipient({ db, email: mapped.email });
|
|
217
|
+
// resolveRecipient with an email arg always returns (synthesizing
|
|
218
|
+
// `contactId: email` when no contact exists); the null-check keeps types
|
|
219
|
+
// honest.
|
|
220
|
+
const hasContact = recipient !== null && recipient.contactId !== mapped.email;
|
|
221
|
+
const userId = hasContact
|
|
222
|
+
? (recipient.externalId ?? recipient.contactId)
|
|
223
|
+
: (mapped.externalId ?? mapped.email);
|
|
224
|
+
|
|
225
|
+
const { setSuppressedAt, recordBounce, ...update } = mapped.update;
|
|
226
|
+
|
|
227
|
+
await upsertEmailPreference({
|
|
228
|
+
db,
|
|
229
|
+
externalId: userId,
|
|
230
|
+
email: mapped.email,
|
|
231
|
+
update: {
|
|
232
|
+
...update,
|
|
233
|
+
...(setSuppressedAt ? { suppressedAt: new Date() } : {}),
|
|
234
|
+
...(recordBounce ? { recordBounce: true } : {}),
|
|
235
|
+
},
|
|
236
|
+
emitOutbound: false,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function parseCsv(data: string): SuppressionImportRow[] {
|
|
241
|
+
const result = Papa.parse<Record<string, string>>(data, {
|
|
242
|
+
header: true,
|
|
243
|
+
skipEmptyLines: true,
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
const fields = result.meta.fields ?? [];
|
|
247
|
+
if (!fields.includes("email")) {
|
|
248
|
+
throw new Error("CSV must have an email column");
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
return result.data.map((row) => ({
|
|
252
|
+
email: row.email || undefined,
|
|
253
|
+
reason: row.reason || undefined,
|
|
254
|
+
externalId: row.externalId || undefined,
|
|
255
|
+
}));
|
|
256
|
+
}
|