@hogsend/engine 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/package.json +6 -6
  2. package/src/buckets/bucket-access.ts +213 -0
  3. package/src/buckets/bucket-reactions.ts +225 -0
  4. package/src/buckets/check-membership.ts +35 -15
  5. package/src/buckets/define-bucket.ts +79 -8
  6. package/src/buckets/registry.ts +81 -0
  7. package/src/container.ts +69 -4
  8. package/src/env.ts +4 -0
  9. package/src/index.ts +27 -0
  10. package/src/journeys/journey-context.ts +5 -1
  11. package/src/lib/boot.ts +12 -2
  12. package/src/lib/bucket-emit.ts +49 -7
  13. package/src/lib/contacts.ts +1083 -18
  14. package/src/lib/email-service-types.ts +8 -0
  15. package/src/lib/ingestion.ts +63 -33
  16. package/src/lib/mailer.ts +1 -0
  17. package/src/lib/preferences.ts +106 -0
  18. package/src/lib/tracked.ts +159 -34
  19. package/src/lib/tracking-events.ts +1 -1
  20. package/src/lists/define-list.ts +81 -0
  21. package/src/lists/registry-singleton.ts +39 -0
  22. package/src/lists/registry.ts +95 -0
  23. package/src/middleware/api-key.ts +33 -7
  24. package/src/middleware/rate-limit.ts +73 -49
  25. package/src/routes/_shared.ts +30 -0
  26. package/src/routes/admin/api-keys.ts +1 -1
  27. package/src/routes/admin/buckets.ts +39 -9
  28. package/src/routes/admin/bulk.ts +7 -3
  29. package/src/routes/admin/contacts.ts +66 -57
  30. package/src/routes/admin/events.ts +65 -0
  31. package/src/routes/admin/journeys.ts +3 -1
  32. package/src/routes/admin/preferences.ts +2 -2
  33. package/src/routes/admin/reporting.ts +3 -3
  34. package/src/routes/admin/timeline.ts +5 -2
  35. package/src/routes/campaigns/index.ts +252 -0
  36. package/src/routes/contacts/index.ts +188 -0
  37. package/src/routes/email/preferences.ts +27 -3
  38. package/src/routes/email/unsubscribe.ts +7 -49
  39. package/src/routes/emails/index.ts +133 -0
  40. package/src/routes/events/index.ts +119 -0
  41. package/src/routes/index.ts +52 -2
  42. package/src/routes/lists/index.ts +222 -0
  43. package/src/worker.ts +25 -2
  44. package/src/workflows/bucket-backfill.ts +122 -22
  45. package/src/workflows/bucket-reconcile.ts +225 -12
  46. package/src/workflows/import-contacts.ts +28 -20
  47. package/src/workflows/send-campaign.ts +589 -0
  48. package/src/routes/ingest.ts +0 -71
@@ -53,6 +53,12 @@ export interface SendTrackedEmailOptions<
53
53
  replyTo?: string | string[];
54
54
  skipPreferenceCheck?: boolean;
55
55
  baseUrl?: string;
56
+ /**
57
+ * Caller-supplied idempotency key (POST /v1/emails). A retry with the same key
58
+ * short-circuits to the prior `email_sends` row instead of dispatching a
59
+ * duplicate provider send.
60
+ */
61
+ idempotencyKey?: string;
56
62
  }
57
63
 
58
64
  export interface TrackedSendResult {
@@ -128,6 +134,8 @@ export interface EmailServiceSendOptions<
128
134
  headers?: Record<string, string>;
129
135
  replyTo?: string | string[];
130
136
  skipPreferenceCheck?: boolean;
137
+ /** Caller-supplied idempotency key (POST /v1/emails) — dedups duplicate sends. */
138
+ idempotencyKey?: string;
131
139
  }
132
140
 
133
141
  export interface EmailServiceWebhookOptions {
@@ -4,15 +4,27 @@ import type { JourneyRegistry } from "@hogsend/core/registry";
4
4
  import { type Database, journeyStates, userEvents } from "@hogsend/db";
5
5
  import { and, eq, inArray, isNull } from "drizzle-orm";
6
6
  import { checkBucketMembership } from "../buckets/check-membership.js";
7
- import { upsertContact } from "./contacts.js";
7
+ import { resolveOrCreateContact } from "./contacts.js";
8
8
  import type { Logger } from "./logger.js";
9
9
 
10
10
  export interface IngestEvent {
11
11
  event: string;
12
- userId: string;
13
- userEmail: string;
14
- properties: Record<string, unknown>;
12
+ /** D1: optional — email-only / anonymous events resolve a key downstream. */
13
+ userId?: string;
14
+ userEmail?: string;
15
+ /** D1: future anonymous→identified path. Threaded into the resolver. */
16
+ anonymousId?: string;
17
+ /** D2: → `user_events` + Hatchet `trigger.where`/`exitOn` ONLY. */
18
+ eventProperties: Record<string, unknown>;
19
+ /** D2: → `contacts.properties` merge ONLY. */
20
+ contactProperties?: Record<string, unknown>;
15
21
  idempotencyKey?: string;
22
+ /**
23
+ * Caller-supplied event time (§2.5 `timestamp`). When set, `user_events`
24
+ * `occurred_at` is stamped from it (backfill/replay) instead of defaulting to
25
+ * the ingest instant. Accepts a `Date` or an ISO-8601 string.
26
+ */
27
+ occurredAt?: Date | string;
16
28
  }
17
29
 
18
30
  export interface ExitResult {
@@ -35,14 +47,36 @@ export async function ingestEvent(opts: {
35
47
  }): Promise<IngestResult> {
36
48
  const { db, registry, hatchet, logger, event } = opts;
37
49
 
50
+ // (1) Resolve identity FIRST (awaited — no longer fire-and-forget). The
51
+ // contact-referencing tables join on a NOT NULL text key, so an email-only /
52
+ // anonymous event (D1 optional userId) needs a canonical key resolved before
53
+ // any insert (risk 2). The resolver applies ONLY contactProperties to
54
+ // `contacts.properties` (D2 split) and returns BOTH the canonical contact id
55
+ // AND its resolved string key (external_id ?? anonymous_id ?? contact.id —
56
+ // risk 1/6), so no second read-back of the contact row is needed.
57
+ const { resolvedKey } = await resolveOrCreateContact({
58
+ db,
59
+ userId: event.userId,
60
+ email: event.userEmail || undefined,
61
+ anonymousId: event.anonymousId,
62
+ contactProperties: event.contactProperties,
63
+ });
64
+
65
+ // Caller-supplied event time (backfill/replay). Coerced to a Date; undefined
66
+ // falls back to the `occurred_at` DB default (ingest instant).
67
+ const occurredAt = event.occurredAt ? new Date(event.occurredAt) : undefined;
68
+
69
+ // (2) Idempotency dedup + `user_events` insert keyed on the resolved key, with
70
+ // ONLY eventProperties in the properties bag (D2).
38
71
  if (event.idempotencyKey) {
39
72
  const result = await db
40
73
  .insert(userEvents)
41
74
  .values({
42
- userId: event.userId,
75
+ userId: resolvedKey,
43
76
  event: event.event,
44
- properties: event.properties,
77
+ properties: event.eventProperties,
45
78
  idempotencyKey: event.idempotencyKey,
79
+ ...(occurredAt ? { occurredAt } : {}),
46
80
  })
47
81
  .onConflictDoNothing({
48
82
  target: userEvents.idempotencyKey,
@@ -54,14 +88,17 @@ export async function ingestEvent(opts: {
54
88
  }
55
89
  } else {
56
90
  await db.insert(userEvents).values({
57
- userId: event.userId,
91
+ userId: resolvedKey,
58
92
  event: event.event,
59
- properties: event.properties,
93
+ properties: event.eventProperties,
94
+ ...(occurredAt ? { occurredAt } : {}),
60
95
  });
61
96
  }
62
97
 
98
+ // (3) Build the JSON-serializable subset of eventProperties for the Hatchet
99
+ // push payload (scalars only — the SDK serializes the envelope).
63
100
  const serializableProperties = Object.fromEntries(
64
- Object.entries(event.properties).filter(
101
+ Object.entries(event.eventProperties).filter(
65
102
  ([, v]) =>
66
103
  typeof v === "string" ||
67
104
  typeof v === "number" ||
@@ -70,57 +107,50 @@ export async function ingestEvent(opts: {
70
107
  ),
71
108
  ) as Record<string, string | number | boolean | null>;
72
109
 
110
+ // (4) Hatchet push + (5) checkExits, both keyed on the resolved key. The push
111
+ // payload wire key STAYS `properties` (bucket tests assert on it — risk 9).
73
112
  const [, exits] = await Promise.all([
74
113
  hatchet.events.push(event.event, {
75
- userId: event.userId,
76
- userEmail: event.userEmail,
114
+ userId: resolvedKey,
115
+ userEmail: event.userEmail ?? "",
77
116
  properties: serializableProperties,
78
117
  }),
79
118
  checkExits(db, registry, hatchet, logger, {
80
- userId: event.userId,
119
+ userId: resolvedKey,
81
120
  eventName: event.event,
82
- properties: event.properties,
83
- }),
84
- upsertContact({
85
- db,
86
- externalId: event.userId,
87
- email: event.userEmail || undefined,
88
- properties: event.properties,
89
- }).catch((err) => {
90
- logger.warn("Contact upsert failed", {
91
- userId: event.userId,
92
- error: err instanceof Error ? err.message : String(err),
93
- });
121
+ properties: event.eventProperties,
94
122
  }),
95
123
  ]);
96
124
 
97
- // Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
98
- // Promise.all above: its property eval reads MERGED contact state, and its
99
- // bucket:entered/left emissions recurse back into ingestEvent (the recursion
100
- // guard in checkBucketMembership bounds them). Best-effort: a bucket failure
101
- // must not fail the ingest of the originating event.
125
+ // (6) Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
126
+ // Promise.all above: its property eval reads contact state this-ingest
127
+ // contactProperties patch, and its bucket:entered/left emissions recurse back
128
+ // into ingestEvent (the recursion guard in checkBucketMembership bounds them).
129
+ // Best-effort: a bucket failure must not fail the ingest of the originating
130
+ // event.
102
131
  try {
103
132
  await checkBucketMembership({
104
133
  db,
105
134
  registry,
106
135
  hatchet,
107
136
  logger,
108
- userId: event.userId,
137
+ userId: resolvedKey,
109
138
  userEmail: event.userEmail || null,
110
139
  event: event.event,
111
- properties: event.properties,
140
+ eventProperties: event.eventProperties,
141
+ contactProperties: event.contactProperties ?? {},
112
142
  });
113
143
  } catch (err) {
114
144
  logger.warn("Bucket membership check failed", {
115
145
  event: event.event,
116
- userId: event.userId,
146
+ userId: resolvedKey,
117
147
  error: err instanceof Error ? err.message : String(err),
118
148
  });
119
149
  }
120
150
 
121
151
  logger.info("Event ingested", {
122
152
  event: event.event,
123
- userId: event.userId,
153
+ userId: resolvedKey,
124
154
  exits: exits.filter((e) => e.exited).length,
125
155
  });
126
156
 
package/src/lib/mailer.ts CHANGED
@@ -98,6 +98,7 @@ export function createTrackedMailer(
98
98
  headers: options.headers,
99
99
  replyTo: options.replyTo,
100
100
  skipPreferenceCheck: options.skipPreferenceCheck,
101
+ idempotencyKey: options.idempotencyKey,
101
102
  baseUrl: config.baseUrl,
102
103
  },
103
104
  });
@@ -0,0 +1,106 @@
1
+ import type { Database } from "@hogsend/db";
2
+ import { emailPreferences } from "@hogsend/db";
3
+ import { sql } from "drizzle-orm";
4
+ import { resolveRecipient } from "./contacts.js";
5
+
6
+ /**
7
+ * Single source of truth for an `email_preferences` upsert: the `(user_id, email)`
8
+ * onConflict + the jsonb category-flip. Extracted from the private
9
+ * `upsertPreference` that used to live in `routes/email/unsubscribe.ts` (decision
10
+ * #9) so subscribe/unsubscribe routes, the preference center, list membership, and
11
+ * the unsubscribe-token flow all share ONE write.
12
+ *
13
+ * `externalId` is the `user_id` column value: the contact's `external_id` when it
14
+ * has one, else the contact `id` (uuid) fallback for an email-only contact (risk
15
+ * 10). `email` is REQUIRED — both columns are NOT NULL and form the PK.
16
+ */
17
+ export async function upsertEmailPreference(opts: {
18
+ db: Database;
19
+ externalId: string;
20
+ email: string;
21
+ update: {
22
+ unsubscribedAll?: boolean;
23
+ suppressed?: boolean;
24
+ categoryKey?: string;
25
+ categoryValue?: boolean;
26
+ };
27
+ }): Promise<void> {
28
+ const { db, externalId, email, update } = opts;
29
+
30
+ const setClause: Record<string, unknown> = { updatedAt: new Date() };
31
+
32
+ if (update.unsubscribedAll !== undefined) {
33
+ setClause.unsubscribedAll = update.unsubscribedAll;
34
+ }
35
+ if (update.suppressed !== undefined) {
36
+ setClause.suppressed = update.suppressed;
37
+ }
38
+ if (update.categoryKey !== undefined) {
39
+ const jsonValue = update.categoryValue ? "true" : "false";
40
+ setClause.categories = sql`jsonb_set(COALESCE(${emailPreferences.categories}, '{}'::jsonb), ${`{${update.categoryKey}}`}, ${jsonValue}::jsonb)`;
41
+ }
42
+
43
+ await db
44
+ .insert(emailPreferences)
45
+ .values({
46
+ userId: externalId,
47
+ email,
48
+ ...(update.unsubscribedAll !== undefined
49
+ ? { unsubscribedAll: update.unsubscribedAll }
50
+ : {}),
51
+ ...(update.suppressed !== undefined
52
+ ? { suppressed: update.suppressed }
53
+ : {}),
54
+ ...(update.categoryKey !== undefined
55
+ ? {
56
+ categories: { [update.categoryKey]: update.categoryValue ?? false },
57
+ }
58
+ : {}),
59
+ })
60
+ .onConflictDoUpdate({
61
+ target: [emailPreferences.userId, emailPreferences.email],
62
+ set: setClause,
63
+ });
64
+ }
65
+
66
+ /**
67
+ * D3 list-membership write. Resolves the caller's identity to the deterministic
68
+ * `(externalId | contactId fallback, email)` pair via `resolveRecipient`, then
69
+ * writes one category flip per list key through `upsertEmailPreference`.
70
+ *
71
+ * Requires a resolvable email — `email_preferences.email` is NOT NULL and the
72
+ * preference center / unsubscribe-token flow key on it (risk 10). The caller is
73
+ * expected to have already run `resolveOrCreateContact` (so the contact exists);
74
+ * this reads identity back. Throws if no email can be resolved — the route maps
75
+ * that to a 400 ("Contact has no email; cannot manage list membership").
76
+ */
77
+ export async function applyListMembership(opts: {
78
+ db: Database;
79
+ userId?: string;
80
+ email?: string;
81
+ lists: Record<string, boolean>;
82
+ }): Promise<void> {
83
+ const { db, userId, email, lists } = opts;
84
+
85
+ const entries = Object.entries(lists);
86
+ if (entries.length === 0) return;
87
+
88
+ const recipient = await resolveRecipient({ db, userId, email });
89
+ if (!recipient) {
90
+ throw new Error("Contact has no email; cannot manage list membership");
91
+ }
92
+
93
+ // `user_id` column = external_id when present, else the contact id (uuid)
94
+ // fallback — the SAME deterministic key used by subscribe writes,
95
+ // preference-center reads, and unsubscribe-token issuance (risk 10).
96
+ const externalId = recipient.externalId ?? recipient.contactId;
97
+
98
+ for (const [categoryKey, categoryValue] of entries) {
99
+ await upsertEmailPreference({
100
+ db,
101
+ externalId,
102
+ email: recipient.email,
103
+ update: { categoryKey, categoryValue },
104
+ });
105
+ }
106
+ }
@@ -7,8 +7,13 @@ import type {
7
7
  TemplateName,
8
8
  TemplateRegistry,
9
9
  } from "@hogsend/email";
10
- import { getTemplate, renderToHtml } from "@hogsend/email";
10
+ import {
11
+ generateUnsubscribeUrl,
12
+ getTemplate,
13
+ renderToHtml,
14
+ } from "@hogsend/email";
11
15
  import { eq } from "drizzle-orm";
16
+ import { getListRegistry } from "../lists/registry-singleton.js";
12
17
  import type {
13
18
  FrequencyCapConfig,
14
19
  SendTrackedEmailOptions,
@@ -50,11 +55,55 @@ export async function sendTrackedEmail<K extends TemplateName>(
50
55
  options,
51
56
  } = opts;
52
57
 
58
+ // The idempotency-collision result, built identically whether the prior row is
59
+ // found by the up-front short-circuit select OR the concurrent-insert loser
60
+ // path below: surface the winner's send id, mapping "sent" → sent and anything
61
+ // else → a skipped/"frequency_capped" placeholder.
62
+ const idempotentResult = (prior: {
63
+ id: string;
64
+ status: string;
65
+ }): TrackedSendResult =>
66
+ ({
67
+ emailSendId: prior.id,
68
+ resendId: "",
69
+ status: prior.status === "sent" ? "sent" : "skipped",
70
+ ...(prior.status === "sent" ? {} : { reason: "frequency_capped" }),
71
+ }) as TrackedSendResult;
72
+
73
+ // Idempotency short-circuit (POST /v1/emails): a retry with the same key
74
+ // returns the prior send instead of dispatching a duplicate provider call /
75
+ // tracking artifacts (mirrors the user_events idempotency pattern).
76
+ if (options.idempotencyKey) {
77
+ const existing = await db
78
+ .select({ id: emailSends.id, status: emailSends.status })
79
+ .from(emailSends)
80
+ .where(eq(emailSends.idempotencyKey, options.idempotencyKey))
81
+ .limit(1);
82
+ const prior = existing[0];
83
+ if (prior) {
84
+ return idempotentResult(prior);
85
+ }
86
+ }
87
+
88
+ // Resolve the template ONCE up front so its default category is available to
89
+ // the suppression check (a public /v1/emails send may omit `category`, in
90
+ // which case the per-category suppression must still consult the template's
91
+ // own category — otherwise an unsubscribed recipient leaks the mail while the
92
+ // row is stamped with that very category — risk 6 / §2.6).
93
+ const {
94
+ element,
95
+ subject: defaultSubject,
96
+ category: templateCategory,
97
+ } = getTemplate({ key: options.templateKey, props: options.props, registry });
98
+
99
+ const effectiveCategory = options.category ?? templateCategory;
100
+ const subject = options.subject ?? defaultSubject;
101
+
53
102
  if (!options.skipPreferenceCheck) {
54
103
  const suppression = await checkSuppression(
55
104
  db,
56
105
  options.to,
57
- options.category,
106
+ effectiveCategory,
58
107
  );
59
108
  if (suppression) {
60
109
  const rows = await db
@@ -64,11 +113,14 @@ export async function sendTrackedEmail<K extends TemplateName>(
64
113
  fromEmail: options.from,
65
114
  toEmail: options.to,
66
115
  subject: options.subject ?? "",
67
- category: options.category,
116
+ category: effectiveCategory,
68
117
  journeyStateId: options.journeyStateId,
69
118
  userId: options.userId,
70
119
  userEmail: options.userEmail ?? options.to,
71
120
  status: "failed",
121
+ // A suppressed send does NOT consume the idempotency key — leaving it
122
+ // unset lets a later retry (e.g. after the recipient re-subscribes)
123
+ // actually attempt the send rather than dedup to the suppressed row.
72
124
  })
73
125
  .returning({ id: emailSends.id });
74
126
 
@@ -89,6 +141,10 @@ export async function sendTrackedEmail<K extends TemplateName>(
89
141
  // Frequency cap — consulted only for non-system sends (system mail sets
90
142
  // skipPreferenceCheck and bypasses both suppression and the cap). On a cap
91
143
  // hit: no provider call, no row inserted, no throw — the journey continues.
144
+ // Keyed on the caller-supplied `options.category` (NOT the template default)
145
+ // so the cap's byCategory/exempt rules apply exactly to what the caller
146
+ // asked to cap — distinct from suppression, which needs the template default
147
+ // to honor a per-category unsubscribe even when the caller omits `category`.
92
148
  if (frequencyCap) {
93
149
  const capped = await isFrequencyCapped({
94
150
  db,
@@ -111,37 +167,91 @@ export async function sendTrackedEmail<K extends TemplateName>(
111
167
  }
112
168
  }
113
169
 
114
- const {
115
- element,
116
- subject: defaultSubject,
117
- category,
118
- } = getTemplate({ key: options.templateKey, props: options.props, registry });
170
+ // Unsubscribe surface (RFC 8058 / CAN-SPAM): generate the per-recipient
171
+ // unsubscribe URL ONCE and inject it both as the in-body template prop AND the
172
+ // List-Unsubscribe / List-Unsubscribe-Post: One-Click headers, so EVERY send
173
+ // through the tracked mailer — journey AND public /v1/emails — carries it
174
+ // uniformly. Suppressed only for true system mail (skipPreferenceCheck). Built
175
+ // from the SAME user_id fallback (externalId ?? contactId) the email_sends row
176
+ // uses, keeping the token externalId consistent with the preference-center key.
177
+ const secret = process.env.BETTER_AUTH_SECRET;
178
+ let unsubscribeUrl: string | undefined;
179
+ if (!options.skipPreferenceCheck && options.baseUrl && secret) {
180
+ unsubscribeUrl = generateUnsubscribeUrl({
181
+ baseUrl: options.baseUrl,
182
+ secret,
183
+ externalId: options.userId ?? options.to,
184
+ email: options.to,
185
+ category: effectiveCategory,
186
+ });
187
+ }
119
188
 
120
- const subject = options.subject ?? defaultSubject;
189
+ const sendHeaders: Record<string, string> = { ...(options.headers ?? {}) };
190
+ if (unsubscribeUrl && !("List-Unsubscribe" in sendHeaders)) {
191
+ sendHeaders["List-Unsubscribe"] = `<${unsubscribeUrl}>`;
192
+ sendHeaders["List-Unsubscribe-Post"] = "List-Unsubscribe=One-Click";
193
+ }
121
194
 
122
- const insertRows = await db
123
- .insert(emailSends)
124
- .values({
125
- templateKey: options.templateKey,
126
- fromEmail: options.from,
127
- toEmail: options.to,
128
- subject,
129
- category: options.category ?? category,
130
- journeyStateId: options.journeyStateId,
131
- userId: options.userId,
132
- userEmail: options.userEmail ?? options.to,
133
- status: "queued",
134
- })
135
- .returning({ id: emailSends.id });
195
+ // Re-render the template element with the unsubscribe URL merged into props so
196
+ // the in-body footer link renders for journeyless public sends too. Journey
197
+ // sends already pass `unsubscribeUrl` in props (lib/email.ts); only set it when
198
+ // the caller didn't, so we never clobber an explicitly-passed value.
199
+ const propsRecord = options.props as unknown as
200
+ | Record<string, unknown>
201
+ | undefined;
202
+ const sendElement =
203
+ unsubscribeUrl && propsRecord?.unsubscribeUrl == null
204
+ ? getTemplate({
205
+ key: options.templateKey,
206
+ props: {
207
+ ...(propsRecord ?? {}),
208
+ unsubscribeUrl,
209
+ } as unknown as typeof options.props,
210
+ registry,
211
+ }).element
212
+ : element;
213
+
214
+ const baseInsert = db.insert(emailSends).values({
215
+ templateKey: options.templateKey,
216
+ fromEmail: options.from,
217
+ toEmail: options.to,
218
+ subject,
219
+ category: effectiveCategory,
220
+ journeyStateId: options.journeyStateId,
221
+ userId: options.userId,
222
+ userEmail: options.userEmail ?? options.to,
223
+ status: "queued",
224
+ idempotencyKey: options.idempotencyKey,
225
+ });
226
+
227
+ // With an idempotency key, swallow a concurrent-insert collision on the unique
228
+ // index (the select-then-insert above is not atomic) and return the winner.
229
+ const insertRows = options.idempotencyKey
230
+ ? await baseInsert
231
+ .onConflictDoNothing({ target: emailSends.idempotencyKey })
232
+ .returning({ id: emailSends.id })
233
+ : await baseInsert.returning({ id: emailSends.id });
136
234
 
137
235
  const insertedRow = insertRows[0];
236
+ if (!insertedRow && options.idempotencyKey) {
237
+ // A concurrent send claimed the key first — return its row.
238
+ const winner = await db
239
+ .select({ id: emailSends.id, status: emailSends.status })
240
+ .from(emailSends)
241
+ .where(eq(emailSends.idempotencyKey, options.idempotencyKey))
242
+ .limit(1);
243
+ const won = winner[0];
244
+ if (won) {
245
+ return idempotentResult(won);
246
+ }
247
+ }
138
248
  if (!insertedRow) throw new Error("Failed to insert email_sends row");
139
249
  const emailSendId = insertedRow.id;
140
250
 
141
251
  try {
142
252
  let html: string | undefined;
143
253
  if (options.baseUrl && prepareTrackedHtml) {
144
- const rawHtml = await renderToHtml(element);
254
+ const rawHtml = await renderToHtml(sendElement);
145
255
  html = await prepareTrackedHtml({
146
256
  html: rawHtml,
147
257
  emailSendId,
@@ -154,9 +264,9 @@ export async function sendTrackedEmail<K extends TemplateName>(
154
264
  from: options.from,
155
265
  to: options.to,
156
266
  subject,
157
- ...(html ? { html } : { react: element }),
267
+ ...(html ? { html } : { react: sendElement }),
158
268
  tags: options.tags,
159
- headers: options.headers,
269
+ headers: sendHeaders,
160
270
  replyTo: options.replyTo,
161
271
  });
162
272
 
@@ -176,10 +286,18 @@ export async function sendTrackedEmail<K extends TemplateName>(
176
286
  status: "sent",
177
287
  };
178
288
  } catch (error) {
289
+ // A provider send failed (transient SMTP/network/429). Stamp `failed` AND
290
+ // RELEASE the idempotency key (set it null), exactly like the suppression
291
+ // path deliberately never consumes it: this lets a retry genuinely
292
+ // RE-ATTEMPT the send rather than dedup to this failed row. Without the
293
+ // release, the up-front short-circuit would return this `failed` row mapped
294
+ // to `skipped`, so a real delivery failure would (a) never be re-sent and
295
+ // (b) silently vanish from the campaign's failedCount into skippedCount.
179
296
  await db
180
297
  .update(emailSends)
181
298
  .set({
182
299
  status: "failed",
300
+ idempotencyKey: null,
183
301
  updatedAt: new Date(),
184
302
  })
185
303
  .where(eq(emailSends.id, emailSendId));
@@ -201,17 +319,24 @@ async function checkSuppression(
201
319
  .where(eq(emailPreferences.email, email))
202
320
  .limit(1);
203
321
 
204
- if (rows.length === 0) return null;
205
-
206
322
  const prefs = rows[0];
207
- if (!prefs) return null;
208
323
 
209
- if (prefs.suppressed) return "suppressed";
210
- if (prefs.unsubscribedAll) return "unsubscribed";
324
+ if (prefs?.suppressed) return "suppressed";
325
+ if (prefs?.unsubscribedAll) return "unsubscribed";
211
326
 
212
- if (category && prefs.categories) {
213
- const categories = prefs.categories as Record<string, boolean>;
214
- if (categories[category] === false) return "category_unsubscribed";
327
+ // Registry-aware polarity (§2.6, D3) — applied through the SINGLE source of
328
+ // truth `ListRegistry.isSubscribed` so it matches the preference center EXACTLY
329
+ // (categories default to `{}` when there is NO prefs row or NO categories map).
330
+ // This MUST run even when the row is absent/empty: an opt-out list
331
+ // (`defaultOptIn:false`) requires `categories[id] === true` to be subscribed,
332
+ // so absence-of-true (the common "never opted in" case) MUST block — otherwise
333
+ // a contact the preference center shows as "Unsubscribed" would still receive
334
+ // the mail (the two surfaces would disagree, which §2.6 forbids).
335
+ if (category) {
336
+ const categories = (prefs?.categories ?? {}) as Record<string, boolean>;
337
+ if (!getListRegistry().isSubscribed(categories, category)) {
338
+ return "category_unsubscribed";
339
+ }
215
340
  }
216
341
 
217
342
  return null;
@@ -78,7 +78,7 @@ export async function pushTrackingEvent(
78
78
  event,
79
79
  userId: ctx.userId,
80
80
  userEmail: ctx.userEmail,
81
- properties,
81
+ eventProperties: properties,
82
82
  },
83
83
  });
84
84
  }
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Email lists (D3) — code-defined subscription categories layered on top of the
3
+ * existing `email_preferences.categories` JSONB. A list is just a named category
4
+ * with a declared default polarity (`defaultOptIn`); there is NO new table.
5
+ *
6
+ * `defineList()` is the authoring entry point, mirroring `defineBucket()` /
7
+ * `defineJourney()`: a synchronous, definition-time call that validates the id
8
+ * and returns a `DefinedList`. The id shares the `email_preferences.categories`
9
+ * key namespace, so two ids are RESERVED for the engine's own non-list
10
+ * categories (`transactional`, `journey`) and rejected here (open risk #11).
11
+ */
12
+
13
+ /** A list id can collide with these built-in non-list categories — reject. */
14
+ const RESERVED_LIST_IDS = new Set(["transactional", "journey"]);
15
+
16
+ /** Allowed list-id shape: alphanumerics, dash, underscore (case-insensitive). */
17
+ const LIST_ID_PATTERN = /^[a-z0-9_-]+$/i;
18
+
19
+ /**
20
+ * The validated, fully-defaulted list metadata. `enabled` is always present
21
+ * after `defineList` (defaults to `true`); authoring input leaves it optional.
22
+ */
23
+ export interface ListMeta<Id extends string = string> {
24
+ id: Id;
25
+ name: string;
26
+ description?: string;
27
+ defaultOptIn: boolean;
28
+ enabled: boolean;
29
+ }
30
+
31
+ /**
32
+ * A defined list. `meta` carries the canonical, defaulted metadata; `id` is
33
+ * surfaced directly for literal-typed consumption (mirrors `DefinedBucket`).
34
+ */
35
+ export interface DefinedList<Id extends string = string> {
36
+ readonly meta: ListMeta<Id>;
37
+ readonly id: Id;
38
+ }
39
+
40
+ /**
41
+ * Define an email list. Validates the id against {@link LIST_ID_PATTERN} and the
42
+ * {@link RESERVED_LIST_IDS} blocklist (since list ids share the
43
+ * `email_preferences.categories` namespace), then returns a `DefinedList` with
44
+ * `enabled` defaulted to `true`.
45
+ *
46
+ * @throws if `id` is empty, malformed, or a reserved category id.
47
+ */
48
+ export function defineList<const Id extends string>(meta: {
49
+ id: Id;
50
+ name: string;
51
+ description?: string;
52
+ defaultOptIn: boolean;
53
+ enabled?: boolean;
54
+ }): DefinedList<Id> {
55
+ const { id, name, description, defaultOptIn, enabled } = meta;
56
+
57
+ if (!id || !LIST_ID_PATTERN.test(id)) {
58
+ throw new Error(
59
+ `Invalid list id "${id}": must match /^[a-z0-9_-]+$/i (letters, digits, "-", "_").`,
60
+ );
61
+ }
62
+
63
+ if (RESERVED_LIST_IDS.has(id.toLowerCase())) {
64
+ throw new Error(
65
+ `Reserved list id "${id}": "transactional" and "journey" are built-in email-preference categories and cannot be used as list ids.`,
66
+ );
67
+ }
68
+
69
+ const resolvedMeta: ListMeta<Id> = {
70
+ id,
71
+ name,
72
+ ...(description !== undefined ? { description } : {}),
73
+ defaultOptIn,
74
+ enabled: enabled ?? true,
75
+ };
76
+
77
+ return {
78
+ meta: resolvedMeta,
79
+ id,
80
+ };
81
+ }