@hogsend/engine 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +19 -10
  3. package/src/buckets/check-membership.ts +55 -8
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +3 -0
  6. package/src/index.ts +10 -6
  7. package/src/journeys/execute-journey-run.ts +119 -50
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +27 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +4 -1
  13. package/src/lib/contacts.ts +408 -152
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +14 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +78 -29
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/revenue.ts +20 -5
  23. package/src/lib/sms-tracked.ts +18 -1
  24. package/src/lib/tracked.ts +19 -8
  25. package/src/lib/tracking-events.ts +28 -0
  26. package/src/lib/unique-violation.ts +44 -0
  27. package/src/routes/admin/buckets.ts +20 -2
  28. package/src/routes/admin/bulk.ts +14 -2
  29. package/src/routes/admin/contacts.ts +4 -3
  30. package/src/routes/admin/conversions.ts +20 -4
  31. package/src/routes/admin/emails.ts +17 -2
  32. package/src/routes/admin/events.ts +32 -41
  33. package/src/routes/admin/funnels.ts +6 -1
  34. package/src/routes/admin/groups.ts +4 -0
  35. package/src/routes/admin/impact.ts +30 -10
  36. package/src/routes/admin/journey-impact.ts +18 -9
  37. package/src/routes/admin/journeys.ts +8 -1
  38. package/src/routes/admin/maintenance.ts +7 -1
  39. package/src/routes/admin/preferences.ts +85 -46
  40. package/src/routes/admin/reporting.ts +25 -5
  41. package/src/routes/admin/targeting.ts +1 -0
  42. package/src/routes/admin/timeline.ts +16 -10
  43. package/src/routes/email/preferences.ts +18 -1
  44. package/src/routes/lists/index.ts +6 -6
  45. package/src/workflows/backfill-contact-id.ts +210 -30
  46. package/src/workflows/bucket-backfill.ts +145 -67
  47. package/src/workflows/bucket-reconcile.ts +150 -50
  48. package/src/workflows/check-alerts.ts +4 -0
  49. package/src/workflows/impact-digest.ts +10 -3
  50. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  51. package/src/workflows/send-campaign.ts +18 -11
@@ -1,4 +1,5 @@
1
1
  import {
2
+ bySubject,
2
3
  type JourneyGraph,
3
4
  journeyGraphSchema,
4
5
  resolveTemplateKeyFromConst,
@@ -24,6 +25,7 @@ import {
24
25
  } from "drizzle-orm";
25
26
  import type { AppEnv } from "../../app.js";
26
27
  import { buildJourneyGraph } from "../../journeys/graph/build-graph.js";
28
+ import { lookupContactIdByKey } from "../../lib/contacts.js";
27
29
  import { ingestEvent } from "../../lib/ingestion.js";
28
30
  import {
29
31
  computeJourneyLift,
@@ -744,8 +746,13 @@ export const journeysRouter = new OpenAPIHono<AppEnv>()
744
746
  if (status) {
745
747
  conditions.push(eq(journeyStates.status, status));
746
748
  }
749
+ // Resolve the operator-supplied text key to its owning contact so the
750
+ // filter finds enrollments stamped under keys that person has since moved
751
+ // off. Falls back to the literal string match when no contact owns the
752
+ // key (pre-flip behaviour).
747
753
  if (userId) {
748
- conditions.push(eq(journeyStates.userId, userId));
754
+ const contactId = await lookupContactIdByKey(db, userId);
755
+ conditions.push(bySubject(journeyStates, { contactId, userKey: userId }));
749
756
  }
750
757
 
751
758
  const where = and(...conditions);
@@ -60,6 +60,7 @@ const backfillContactIdRoute = createRoute({
60
60
 
61
61
  const verifyCountsSchema = z.object({
62
62
  missing: z.number(),
63
+ duplicates: z.number(),
63
64
  mismatched: z.number(),
64
65
  orphaned: z.number(),
65
66
  });
@@ -77,7 +78,12 @@ const verifyContactIdRoute = createRoute({
77
78
  "all) — the corruption detector an FK could never give, since an FK " +
78
79
  "proves the uuid exists while this proves it is the RIGHT uuid; " +
79
80
  "`orphaned` = contact_id is NULL and no live contact owns the key, which " +
80
- "is EXPECTED and permitted forever (PRD 04 D5) and is never a failure. " +
81
+ "is EXPECTED and permitted forever (PRD 04 D5) and is never a failure; " +
82
+ "`duplicates` = a live contact owns the key but stamping the row would " +
83
+ "violate a contact-scoped unique index (PRD 05 T3) because that contact " +
84
+ "already holds a row for the same live journey / live bucket / address — " +
85
+ "the backfill skips those on purpose rather than cancelling a live " +
86
+ "enrollment, so they are a triage list, not a failure. " +
81
87
  "`flipReady` is true iff every table reports missing = mismatched = 0 — " +
82
88
  "that is the entry gate for the release that flips reads onto the " +
83
89
  "column. Judge it only once `lastSweepAt` is non-null: before a sweep " +
@@ -1,8 +1,12 @@
1
1
  import { emailPreferences } from "@hogsend/db";
2
2
  import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
- import { eq, sql } from "drizzle-orm";
3
+ import { and, eq, sql } from "drizzle-orm";
4
4
  import type { AppEnv } from "../../app.js";
5
5
  import { resolveContact, serializePrefs } from "../../lib/contacts.js";
6
+ import {
7
+ isUniqueViolationOn,
8
+ UQ_CONTACT_EMAIL_PREFERENCES,
9
+ } from "../../lib/unique-violation.js";
6
10
 
7
11
  const preferencesResponseSchema = z.object({
8
12
  id: z.string(),
@@ -101,10 +105,12 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
101
105
  return c.json({ error: "Contact not found" }, 404);
102
106
  }
103
107
 
108
+ // PRD 05 T6 — the contact is in hand: read by ownership stamp, never by
109
+ // the mutable string key (which goes stale on adoption).
104
110
  const rows = await db
105
111
  .select()
106
112
  .from(emailPreferences)
107
- .where(eq(emailPreferences.userId, contact.externalId ?? contact.id))
113
+ .where(eq(emailPreferences.contactId, contact.id))
108
114
  .limit(1);
109
115
 
110
116
  if (rows.length === 0) {
@@ -127,51 +133,84 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
127
133
  if (!contact.email) {
128
134
  return c.json({ error: "Contact has no email address" }, 400);
129
135
  }
136
+ // Bound once: the narrowing from the guard above does not survive into the
137
+ // `upsertOnStringKey` closure below.
138
+ const email = contact.email;
130
139
 
131
- const [upserted] = await db
132
- .insert(emailPreferences)
133
- .values({
134
- userId: contact.externalId ?? contact.id,
135
- email: contact.email,
136
- // PRD 04 dual-write: this handler resolved the contact one statement
137
- // ago, so the owning id is in hand — zero queries, no D6 wrapper needed
138
- // (there is no new call that could throw).
139
- contactId: contact.id,
140
- unsubscribedAll: body.unsubscribedAll ?? false,
141
- suppressed: body.suppressed ?? false,
142
- categories: body.categories ?? {},
143
- })
144
- .onConflictDoUpdate({
145
- target: [emailPreferences.userId, emailPreferences.email],
146
- set: {
147
- // Fill-if-known, never null-out — the same conflict arm
148
- // `upsertEmailPreference` uses, so the two writers to this table
149
- // cannot disagree about whether a stamp may be erased.
150
- contactId: sql`coalesce(excluded.contact_id, ${emailPreferences.contactId})`,
151
- ...(body.unsubscribedAll !== undefined
152
- ? { unsubscribedAll: body.unsubscribedAll }
153
- : {}),
154
- ...(body.suppressed !== undefined
155
- ? {
156
- suppressed: body.suppressed,
157
- suppressedAt: body.suppressed ? new Date() : null,
158
- // Un-suppressing clears the bounce slate. `bounceCount` only
159
- // drives the auto-suppress threshold (the send-gate keys off
160
- // `suppressed`/`unsubscribedAll`), so a leftover count would
161
- // otherwise keep a bounced recipient pinned to the suppression
162
- // list with no way to remove them.
163
- ...(body.suppressed
164
- ? {}
165
- : { bounceCount: 0, lastBounceAt: null }),
166
- }
167
- : {}),
168
- ...(body.categories !== undefined
169
- ? { categories: body.categories }
170
- : {}),
171
- updatedAt: new Date(),
172
- },
173
- })
174
- .returning();
140
+ const setClause = {
141
+ // Fill-if-known, never null-out — the same conflict arm
142
+ // `upsertEmailPreference` uses, so the two writers to this table
143
+ // cannot disagree about whether a stamp may be erased.
144
+ contactId: sql`coalesce(excluded.contact_id, ${emailPreferences.contactId})`,
145
+ ...(body.unsubscribedAll !== undefined
146
+ ? { unsubscribedAll: body.unsubscribedAll }
147
+ : {}),
148
+ ...(body.suppressed !== undefined
149
+ ? {
150
+ suppressed: body.suppressed,
151
+ suppressedAt: body.suppressed ? new Date() : null,
152
+ // Un-suppressing clears the bounce slate. `bounceCount` only
153
+ // drives the auto-suppress threshold (the send-gate keys off
154
+ // `suppressed`/`unsubscribedAll`), so a leftover count would
155
+ // otherwise keep a bounced recipient pinned to the suppression
156
+ // list with no way to remove them.
157
+ ...(body.suppressed ? {} : { bounceCount: 0, lastBounceAt: null }),
158
+ }
159
+ : {}),
160
+ ...(body.categories !== undefined ? { categories: body.categories } : {}),
161
+ updatedAt: new Date(),
162
+ };
163
+
164
+ const upsertOnStringKey = () =>
165
+ db
166
+ .insert(emailPreferences)
167
+ .values({
168
+ userId: contact.externalId ?? contact.id,
169
+ email,
170
+ // PRD 04 dual-write: this handler resolved the contact one statement
171
+ // ago, so the owning id is in hand — zero queries, no D6 wrapper
172
+ // needed (there is no new call that could throw).
173
+ contactId: contact.id,
174
+ unsubscribedAll: body.unsubscribedAll ?? false,
175
+ suppressed: body.suppressed ?? false,
176
+ categories: body.categories ?? {},
177
+ })
178
+ .onConflictDoUpdate({
179
+ target: [emailPreferences.userId, emailPreferences.email],
180
+ set: setClause,
181
+ })
182
+ .returning();
183
+
184
+ let upserted: typeof emailPreferences.$inferSelect | undefined;
185
+ try {
186
+ [upserted] = await upsertOnStringKey();
187
+ } catch (err) {
188
+ // PRD 05 T3 — the twin of the conversion in `upsertEmailPreference`; see
189
+ // its comment for why the arbiter stays on the string key. This route
190
+ // reaches the same row from the OTHER side (it keys the insert off
191
+ // `external_id ?? id` while an adopted row may still carry the old anon
192
+ // string), so it needs the same fallback or an admin preference edit
193
+ // would 500 on exactly the contacts adoption has touched.
194
+ if (!isUniqueViolationOn(err, UQ_CONTACT_EMAIL_PREFERENCES)) throw err;
195
+ const { contactId: _excludedCoalesce, ...contactScopedSet } = setClause;
196
+ [upserted] = await db
197
+ .update(emailPreferences)
198
+ .set(contactScopedSet)
199
+ .where(
200
+ and(
201
+ eq(emailPreferences.contactId, contact.id),
202
+ eq(emailPreferences.email, email),
203
+ ),
204
+ )
205
+ .returning();
206
+ // The conflicting row vanished between the INSERT and the UPDATE (a
207
+ // concurrent merge folded it away): zero rows back, `upserted` undefined,
208
+ // and the throw below would turn an ordinary race into a 500 the old
209
+ // single-statement path could not produce. The row is gone, so the
210
+ // arbiter INSERT now lands — retry it ONCE. See the twin in
211
+ // `lib/preferences.ts`.
212
+ if (!upserted) [upserted] = await upsertOnStringKey();
213
+ }
175
214
 
176
215
  if (!upserted) {
177
216
  throw new Error("Failed to upsert preferences");
@@ -1,3 +1,4 @@
1
+ import { bySubject } from "@hogsend/core";
1
2
  import { emailSends, userEvents } from "@hogsend/db";
2
3
  import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
4
  import {
@@ -12,7 +13,7 @@ import {
12
13
  sql,
13
14
  } from "drizzle-orm";
14
15
  import type { AppEnv } from "../../app.js";
15
- import { contactKey, resolveContact } from "../../lib/contacts.js";
16
+ import { lookupContactIdByKey, resolveContact } from "../../lib/contacts.js";
16
17
  import { rate, TRUNC_SQL } from "../../lib/metrics-sql.js";
17
18
  import { errorSchema } from "../../lib/schemas.js";
18
19
 
@@ -283,9 +284,11 @@ export const reportingRouter = new OpenAPIHono<AppEnv>()
283
284
  return c.json({ error: "Contact not found" }, 404);
284
285
  }
285
286
 
286
- // Denormalized identity makes this single-table; fall back to the contact's
287
- // email so journeyless sends still surface.
288
- const idConds = [eq(emailSends.userId, contactKey(contact))];
287
+ // PRD 05 T6 — sends are matched by ownership stamp (the send-time string
288
+ // key is frozen and goes stale on adoption). The email leg stays: a
289
+ // journeyless/raw send carries no userId, so its stamp is NULL by design
290
+ // (D7) and only the address can surface it.
291
+ const idConds = [eq(emailSends.contactId, contact.id)];
289
292
  if (contact.email) idConds.push(eq(emailSends.userEmail, contact.email));
290
293
  const where = or(...idConds);
291
294
 
@@ -417,7 +420,24 @@ reportingRouter.get("/sends/export", async (c) => {
417
420
  eq(emailSends.status, q.status as typeof emailSends.$inferSelect.status),
418
421
  );
419
422
  if (q.category) conditions.push(eq(emailSends.category, q.category));
420
- if (q.userId) conditions.push(eq(emailSends.userId, q.userId));
423
+ if (q.userId) {
424
+ // PRD 05 T6 — resolve the caller's key to its contact so the export
425
+ // follows the subject across adoption; an unresolvable key degrades to
426
+ // the frozen string leg (bySubject's else-arm). D6-wrapped: a bookkeeping
427
+ // probe may never fail the export.
428
+ let exportContactId: string | null = null;
429
+ try {
430
+ exportContactId = await lookupContactIdByKey(db, q.userId);
431
+ } catch {
432
+ exportContactId = null;
433
+ }
434
+ conditions.push(
435
+ bySubject(emailSends, {
436
+ contactId: exportContactId,
437
+ userKey: q.userId,
438
+ }),
439
+ );
440
+ }
421
441
  if (q.engagement && q.engagement in engagementColumn)
422
442
  conditions.push(
423
443
  isNotNull(
@@ -239,6 +239,7 @@ export const targetingRouter = new OpenAPIHono<AppEnv>()
239
239
  mode: "server",
240
240
  db,
241
241
  userId: contactKey,
242
+ contactId: row.id,
242
243
  });
243
244
  };
244
245
 
@@ -1,3 +1,4 @@
1
+ import { bySubject } from "@hogsend/core";
1
2
  import { emailSends, journeyStates, userEvents } from "@hogsend/db";
2
3
  import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
4
  import { and, count, desc, eq, isNull } from "drizzle-orm";
@@ -64,10 +65,15 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
64
65
  return c.json({ error: "Contact not found" }, 404);
65
66
  }
66
67
 
67
- // Resolved string key the history tables (user_events/journey_states/
68
- // email_sends) were written under: external_id, else anonymous_id, else the
69
- // contact uuid (matches ingestEvent's resolvedKey).
70
- const externalId = contactKey(contact);
68
+ // The timeline reads history BY SUBJECT. `contact.id` always exists here
69
+ // (the contact was just resolved), so every leg below scopes on the
70
+ // `contact_id` FK — which is what makes the timeline show activity a
71
+ // person accumulated under keys they have since moved off (an anonymous
72
+ // id before they registered). `userKey` is the resolved string key those
73
+ // rows were originally written under (external_id, else anonymous_id,
74
+ // else the contact uuid — matches ingestEvent's resolvedKey); it is the
75
+ // `bySubject` fallback arm and is never reached from here.
76
+ const subject = { contactId: contact.id, userKey: contactKey(contact) };
71
77
  const entries: TimelineEntry[] = [];
72
78
 
73
79
  const shouldFetch = (t: string) => !type || type === t;
@@ -85,7 +91,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
85
91
  ? db
86
92
  .select()
87
93
  .from(userEvents)
88
- .where(eq(userEvents.userId, externalId))
94
+ .where(bySubject(userEvents, subject))
89
95
  .orderBy(desc(userEvents.occurredAt))
90
96
  .limit(fetchLimit)
91
97
  : Promise.resolve([]),
@@ -95,7 +101,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
95
101
  .from(journeyStates)
96
102
  .where(
97
103
  and(
98
- eq(journeyStates.userId, externalId),
104
+ bySubject(journeyStates, subject),
99
105
  isNull(journeyStates.deletedAt),
100
106
  ),
101
107
  )
@@ -123,7 +129,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
123
129
  )
124
130
  .where(
125
131
  and(
126
- eq(journeyStates.userId, externalId),
132
+ bySubject(journeyStates, subject),
127
133
  isNull(journeyStates.deletedAt),
128
134
  ),
129
135
  )
@@ -134,7 +140,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
134
140
  ? db
135
141
  .select({ count: count() })
136
142
  .from(userEvents)
137
- .where(eq(userEvents.userId, externalId))
143
+ .where(bySubject(userEvents, subject))
138
144
  .then((r) => r[0]?.count ?? 0)
139
145
  : Promise.resolve(0),
140
146
  shouldFetch("journey")
@@ -143,7 +149,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
143
149
  .from(journeyStates)
144
150
  .where(
145
151
  and(
146
- eq(journeyStates.userId, externalId),
152
+ bySubject(journeyStates, subject),
147
153
  isNull(journeyStates.deletedAt),
148
154
  ),
149
155
  )
@@ -159,7 +165,7 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
159
165
  )
160
166
  .where(
161
167
  and(
162
- eq(journeyStates.userId, externalId),
168
+ bySubject(journeyStates, subject),
163
169
  isNull(journeyStates.deletedAt),
164
170
  ),
165
171
  )
@@ -1,3 +1,4 @@
1
+ import { bySubject } from "@hogsend/core";
1
2
  import { emailPreferences } from "@hogsend/db";
2
3
  import type { UnsubscribeTokenPayload } from "@hogsend/email";
3
4
  import {
@@ -8,6 +9,7 @@ import {
8
9
  import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
9
10
  import { and, eq } from "drizzle-orm";
10
11
  import type { AppEnv } from "../../app.js";
12
+ import { lookupContactIdByKey } from "../../lib/contacts.js";
11
13
  import { htmlPage } from "../../lib/html.js";
12
14
  import { getListRegistry } from "../../lists/registry-singleton.js";
13
15
 
@@ -66,12 +68,27 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>().openapi(
66
68
 
67
69
  const { externalId, email } = payload;
68
70
 
71
+ // PRD 05 T6 — the token freezes whatever key the subject had when the mail
72
+ // left. Resolve it to the contact so a key adopted SINCE issuance still
73
+ // reaches the row; a stale/unresolvable key degrades to the string leg
74
+ // (bySubject's else-arm), which matches the old read exactly. D6-wrapped:
75
+ // a bookkeeping probe may never 500 the preference center.
76
+ let tokenContactId: string | null = null;
77
+ try {
78
+ tokenContactId = await lookupContactIdByKey(db, externalId);
79
+ } catch {
80
+ tokenContactId = null;
81
+ }
82
+
69
83
  const rows = await db
70
84
  .select()
71
85
  .from(emailPreferences)
72
86
  .where(
73
87
  and(
74
- eq(emailPreferences.userId, externalId),
88
+ bySubject(emailPreferences, {
89
+ contactId: tokenContactId,
90
+ userKey: externalId,
91
+ }),
75
92
  eq(emailPreferences.email, email),
76
93
  ),
77
94
  )
@@ -410,13 +410,14 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
410
410
  if ((userId || email) && !c.get("publishable")) {
411
411
  const recipient = await resolveRecipient({ db, userId, email });
412
412
  if (recipient) {
413
- const extId = recipient.externalId ?? recipient.contactId;
413
+ // PRD 05 T6 — contact-scoped read; the `externalId ?? contactId`
414
+ // string derivation is retired.
414
415
  const rows = await db
415
416
  .select()
416
417
  .from(emailPreferences)
417
418
  .where(
418
419
  and(
419
- eq(emailPreferences.userId, extId),
420
+ eq(emailPreferences.contactId, recipient.contactId),
420
421
  eq(emailPreferences.email, recipient.email),
421
422
  ),
422
423
  )
@@ -463,15 +464,14 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
463
464
  if (!recipient) {
464
465
  return c.json({ categories: {}, unsubscribedAll: false }, 200);
465
466
  }
466
- // `email_preferences.user_id` is keyed on `external_id ?? contact.id`
467
- // (risk 10) — mirror `resolveRecipient`'s contactId fallback.
468
- const extId = recipient.externalId ?? recipient.contactId;
467
+ // PRD 05 T6 — contact-scoped read; the `externalId ?? contactId` string
468
+ // derivation is retired.
469
469
  const rows = await db
470
470
  .select()
471
471
  .from(emailPreferences)
472
472
  .where(
473
473
  and(
474
- eq(emailPreferences.userId, extId),
474
+ eq(emailPreferences.contactId, recipient.contactId),
475
475
  eq(emailPreferences.email, recipient.email),
476
476
  ),
477
477
  )