@hogsend/engine 0.59.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +34 -11
  3. package/src/buckets/check-membership.ts +74 -10
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +10 -0
  6. package/src/index.ts +10 -6
  7. package/src/journeys/execute-journey-run.ts +139 -70
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +35 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +17 -17
  13. package/src/lib/contacts.ts +541 -175
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +14 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +78 -29
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/refine.ts +16 -10
  23. package/src/lib/revenue.ts +20 -5
  24. package/src/lib/sms-inbound.ts +6 -1
  25. package/src/lib/sms-tracked.ts +18 -1
  26. package/src/lib/timezone.ts +11 -1
  27. package/src/lib/tracked.ts +19 -8
  28. package/src/lib/tracking-events.ts +28 -0
  29. package/src/lib/unique-violation.ts +44 -0
  30. package/src/routes/admin/buckets.ts +21 -1
  31. package/src/routes/admin/bulk.ts +14 -2
  32. package/src/routes/admin/contacts.ts +20 -7
  33. package/src/routes/admin/conversions.ts +20 -4
  34. package/src/routes/admin/emails.ts +17 -2
  35. package/src/routes/admin/events.ts +32 -41
  36. package/src/routes/admin/funnels.ts +6 -1
  37. package/src/routes/admin/groups.ts +4 -0
  38. package/src/routes/admin/impact.ts +30 -10
  39. package/src/routes/admin/journey-impact.ts +18 -9
  40. package/src/routes/admin/journeys.ts +8 -1
  41. package/src/routes/admin/maintenance.ts +7 -1
  42. package/src/routes/admin/preferences.ts +85 -46
  43. package/src/routes/admin/reporting.ts +25 -5
  44. package/src/routes/admin/targeting.ts +1 -0
  45. package/src/routes/admin/timeline.ts +16 -10
  46. package/src/routes/email/preferences.ts +18 -1
  47. package/src/routes/feed/recipient.ts +5 -0
  48. package/src/routes/lists/index.ts +6 -6
  49. package/src/workflows/backfill-contact-id.ts +210 -30
  50. package/src/workflows/bucket-backfill.ts +145 -67
  51. package/src/workflows/bucket-reconcile.ts +164 -51
  52. package/src/workflows/check-alerts.ts +4 -0
  53. package/src/workflows/impact-digest.ts +10 -3
  54. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  55. package/src/workflows/send-campaign.ts +18 -11
@@ -5,7 +5,7 @@ import {
5
5
  type Database,
6
6
  importJobs,
7
7
  } from "@hogsend/db";
8
- import { and, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
8
+ import { and, desc, eq, gt, inArray, isNull, type SQL, sql } from "drizzle-orm";
9
9
  import { contactKeySql } from "../lib/contacts.js";
10
10
  import { hatchet } from "../lib/hatchet.js";
11
11
  import type { Logger } from "../lib/logger.js";
@@ -106,6 +106,10 @@ export interface ContactIdBackfillResult extends JsonObject {
106
106
  /** Alias VALUES owned by more than one live contact across the two permitted
107
107
  * kinds. Skipped, never guessed (D4); a non-zero count wants a human. */
108
108
  ambiguousAliases: number;
109
+ /** PRD 05 T3 — stale `email_preferences` rows whose opt-out state was folded
110
+ * into the contact's already-stamped row for the same address, and which were
111
+ * then deleted. The only rows this job ever removes. */
112
+ preferencesFolded: number;
109
113
  /** UPDATE statements issued. Observability for the pacing rule. */
110
114
  statements: number;
111
115
  reason?: string;
@@ -129,6 +133,118 @@ function positiveInt(value: number | undefined, fallback: number): number {
129
133
  return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
130
134
  }
131
135
 
136
+ /** The three tables migration 0071 gives a CONTACT-scoped partial unique index
137
+ * (PRD 05 T3). Everything below that special-cases a table keys off this set. */
138
+ const CONTACT_SCOPED_TABLES = new Set<ContactIdBackfillTable>([
139
+ "journey_states",
140
+ "bucket_memberships",
141
+ "email_preferences",
142
+ ]);
143
+
144
+ /**
145
+ * PRD 05 T3 — "the owning contact ALREADY holds a row that this row's stamp
146
+ * would duplicate", correlated to the row `t` and to an owner-id expression.
147
+ *
148
+ * The sweep is a FOURTH writer to the three contact-scoped unique indexes, and
149
+ * the only one whose rows start OUTSIDE them: `contact_id IS NULL` is outside
150
+ * every one of the three predicates, so it is the STAMP that moves a row in.
151
+ * When the contact already owns a stamped twin (same live journey, same live
152
+ * bucket, same address — exactly the population adoption creates by stamping
153
+ * `contact_id` without rewriting `user_id`) the stamp raises 23505, and this
154
+ * job's blanket catch would turn ONE such row into a failed job that abandons
155
+ * the whole sweep — which the boot / 24h re-enqueue then re-hits forever.
156
+ *
157
+ * Each arm reproduces its index's WHERE clause EXACTLY. Keep in sync with:
158
+ * uq_contact_journey_active packages/db/src/schema/journey-states.ts
159
+ * uq_contact_bucket_active .../bucket-memberships.ts
160
+ * email_preferences_contact_email_idx .../email-preferences.ts
161
+ */
162
+ function contactScopedTwinExists(
163
+ table: ContactIdBackfillTable,
164
+ owner: SQL,
165
+ ): SQL {
166
+ switch (table) {
167
+ case "journey_states":
168
+ return sql`t.status IN ('active', 'waiting') AND EXISTS (
169
+ SELECT 1 FROM journey_states x
170
+ WHERE x.contact_id = ${owner}
171
+ AND x.journey_id = t.journey_id
172
+ AND x.status IN ('active', 'waiting'))`;
173
+ case "bucket_memberships":
174
+ return sql`t.status = 'active' AND t.deleted_at IS NULL AND EXISTS (
175
+ SELECT 1 FROM bucket_memberships x
176
+ WHERE x.contact_id = ${owner}
177
+ AND x.bucket_id = t.bucket_id
178
+ AND x.status = 'active'
179
+ AND x.deleted_at IS NULL)`;
180
+ case "email_preferences":
181
+ return sql`EXISTS (
182
+ SELECT 1 FROM email_preferences x
183
+ WHERE x.contact_id = ${owner} AND x.email = t.email)`;
184
+ default:
185
+ return sql`false`;
186
+ }
187
+ }
188
+
189
+ /**
190
+ * PRD 05 T3 — the ONE fold this sweep performs. Everywhere else it only ever
191
+ * fills a NULL column; `email_preferences` earns the exception because a stale
192
+ * row left NULL is an OPT-OUT the flipped (contact-scoped) read will never see.
193
+ *
194
+ * Per stale-keyed NULL row whose contact already holds a row for the same
195
+ * address: fold the stale state into the TWIN under the merge path's
196
+ * `foldEmailPreferences` rule — OR the opt-outs (`unsubscribed_all`,
197
+ * `suppressed`), union the categories with the twin winning a conflict EXCEPT
198
+ * that FALSE always wins (an opt-out is never lost), which also means a grant
199
+ * the twin never heard of (a stale `sms: true` consent) survives — then DELETE
200
+ * the stale row so nothing is left outside the index. Never the other
201
+ * direction: the twin is the row the flipped read resolves.
202
+ *
203
+ * Postgres runs every data-modifying CTE exactly once and to completion whether
204
+ * or not the primary query reads it, so the fold lands before the delete; the
205
+ * two touch disjoint rows (the twin is stamped, the stale row is not).
206
+ */
207
+ async function foldStalePreferences(opts: {
208
+ db: Database;
209
+ key: string;
210
+ contactId: string;
211
+ }): Promise<number> {
212
+ const { db, key, contactId } = opts;
213
+ const rows = await db.execute<{ id: string }>(sql`
214
+ WITH stale AS (
215
+ SELECT p.id, p.email, p.unsubscribed_all, p.suppressed, p.suppressed_at,
216
+ p.bounce_count, p.last_bounce_at, p.categories
217
+ FROM email_preferences p
218
+ WHERE p.user_id = ${key}
219
+ AND p.contact_id IS NULL
220
+ AND EXISTS (
221
+ SELECT 1 FROM email_preferences t
222
+ WHERE t.contact_id = ${contactId}::uuid AND t.email = p.email
223
+ )
224
+ ), folded AS (
225
+ UPDATE email_preferences t
226
+ SET unsubscribed_all = t.unsubscribed_all OR s.unsubscribed_all,
227
+ suppressed = t.suppressed OR s.suppressed,
228
+ suppressed_at = least(t.suppressed_at, s.suppressed_at),
229
+ bounce_count = greatest(t.bounce_count, s.bounce_count),
230
+ last_bounce_at = least(t.last_bounce_at, s.last_bounce_at),
231
+ categories = coalesce(s.categories, '{}'::jsonb)
232
+ || coalesce(t.categories, '{}'::jsonb)
233
+ || coalesce((
234
+ SELECT jsonb_object_agg(k.key, 'false'::jsonb)
235
+ FROM jsonb_each(coalesce(s.categories, '{}'::jsonb)) k
236
+ WHERE k.value = 'false'::jsonb
237
+ ), '{}'::jsonb),
238
+ updated_at = now()
239
+ FROM stale s
240
+ WHERE t.contact_id = ${contactId}::uuid AND t.email = s.email
241
+ )
242
+ DELETE FROM email_preferences WHERE id IN (SELECT id FROM stale)
243
+ RETURNING id
244
+ `);
245
+ return rows.length;
246
+ }
247
+
132
248
  /**
133
249
  * PASS 1, one table, one contact. The PRD's verbatim statement (D3), looped
134
250
  * until it affects zero rows. Returns rows stamped and statements issued.
@@ -137,6 +253,15 @@ function positiveInt(value: number | undefined, fallback: number): number {
137
253
  * what bounds the row-lock count: the fat tail of this system is a bot anon id
138
254
  * with tens of thousands of events under ONE key, and an unbounded statement
139
255
  * there is the lock/WAL spike D3 exists to avoid.
256
+ *
257
+ * PRD 05 T3: on the three contact-scoped tables the inner SELECT also carries
258
+ * {@link contactScopedTwinExists} as a NOT guard, so a row whose stamp would
259
+ * collide is never selected — it is SKIPPED, not stamped and not repaired. The
260
+ * sweep does not get to cancel a live enrollment or membership to make room;
261
+ * `verifyContactIdBackfill` reports what it skipped as `duplicates` (not
262
+ * `missing`) so the gate can still open and the triage list stays visible.
263
+ * Skipping in the SELECT (rather than swallowing a 23505) also keeps the loop
264
+ * terminating: a skipped row simply stops being a candidate.
140
265
  */
141
266
  async function fillCanonicalKey(opts: {
142
267
  db: Database;
@@ -145,24 +270,35 @@ async function fillCanonicalKey(opts: {
145
270
  contactId: string;
146
271
  rowsPerStatement: number;
147
272
  pauseMs: number;
148
- }): Promise<{ updated: number; statements: number }> {
273
+ }): Promise<{ updated: number; statements: number; folded: number }> {
149
274
  const { db, table, key, contactId, rowsPerStatement, pauseMs } = opts;
150
275
  const name = sql.identifier(table);
151
276
  let updated = 0;
152
277
  let statements = 0;
278
+ let folded = 0;
279
+
280
+ // Opt-outs first: a folded row is deleted, so it never reaches the guard.
281
+ if (table === "email_preferences") {
282
+ folded = await foldStalePreferences({ db, key, contactId });
283
+ statements += 1;
284
+ }
285
+
286
+ const guard = CONTACT_SCOPED_TABLES.has(table)
287
+ ? sql` AND NOT (${contactScopedTwinExists(table, sql`${contactId}::uuid`)})`
288
+ : sql.empty();
153
289
 
154
290
  for (;;) {
155
291
  const rows = await db.execute<{ id: string }>(sql`
156
292
  UPDATE ${name} SET contact_id = ${contactId}::uuid
157
293
  WHERE id IN (
158
- SELECT id FROM ${name}
159
- WHERE user_id = ${key} AND contact_id IS NULL
294
+ SELECT t.id FROM ${name} t
295
+ WHERE t.user_id = ${key} AND t.contact_id IS NULL${guard}
160
296
  LIMIT ${rowsPerStatement}
161
297
  )
162
298
  RETURNING id
163
299
  `);
164
300
  statements += 1;
165
- if (rows.length === 0) return { updated, statements };
301
+ if (rows.length === 0) return { updated, statements, folded };
166
302
  updated += rows.length;
167
303
 
168
304
  if (statements >= MAX_STATEMENTS_PER_LOOP) {
@@ -296,6 +432,7 @@ export async function runContactIdBackfill(opts: {
296
432
  let contactsScanned = 0;
297
433
  let statements = 0;
298
434
  let ambiguousAliases = 0;
435
+ let preferencesFolded = 0;
299
436
 
300
437
  const markJob = async (
301
438
  patch: Partial<typeof importJobs.$inferInsert>,
@@ -345,6 +482,7 @@ export async function runContactIdBackfill(opts: {
345
482
  });
346
483
  canonical[table] += filled.updated;
347
484
  statements += filled.statements;
485
+ preferencesFolded += filled.folded;
348
486
  }
349
487
  }
350
488
 
@@ -385,6 +523,7 @@ export async function runContactIdBackfill(opts: {
385
523
  });
386
524
  alias[table] += filled.updated;
387
525
  statements += filled.statements;
526
+ preferencesFolded += filled.folded;
388
527
  }
389
528
  }
390
529
 
@@ -407,6 +546,7 @@ export async function runContactIdBackfill(opts: {
407
546
  alias,
408
547
  updated,
409
548
  ambiguousAliases,
549
+ preferencesFolded,
410
550
  statements,
411
551
  });
412
552
  return {
@@ -416,6 +556,7 @@ export async function runContactIdBackfill(opts: {
416
556
  alias,
417
557
  updated,
418
558
  ambiguousAliases,
559
+ preferencesFolded,
419
560
  statements,
420
561
  };
421
562
  } catch (err) {
@@ -434,6 +575,7 @@ export async function runContactIdBackfill(opts: {
434
575
  TABLES.reduce((sum, t) => sum + canonical[t], 0) +
435
576
  TABLES.reduce((sum, t) => sum + alias[t], 0),
436
577
  ambiguousAliases,
578
+ preferencesFolded,
437
579
  statements,
438
580
  reason: message,
439
581
  };
@@ -444,13 +586,24 @@ export async function runContactIdBackfill(opts: {
444
586
  // T6 — the invariant probe
445
587
  // ---------------------------------------------------------------------------
446
588
 
447
- /** One table's verdict. `missing` and `mismatched` are failures; `orphaned` is
448
- * information (D5) and is expected to be non-zero forever. */
589
+ /** One table's verdict. `missing` and `mismatched` are failures; `orphaned`
590
+ * (D5) and `duplicates` (PRD 05 T3) are information and may be non-zero. */
449
591
  export interface ContactIdVerifyCounts {
450
592
  /** A live contact owns the row's `user_id` — canonically OR by alias — but
451
- * `contact_id` is NULL. After a completed sweep this is a HOLE: either the
452
- * backfill missed the row or a dual-write site dropped it (D6). MUST be 0. */
593
+ * `contact_id` is NULL and the stamp WOULD land. After a completed sweep this
594
+ * is a HOLE: either the backfill missed the row or a dual-write site dropped
595
+ * it (D6). MUST be 0. */
453
596
  missing: number;
597
+ /**
598
+ * PRD 05 T3 — a live contact owns the row's `user_id`, `contact_id` is NULL,
599
+ * and stamping it would violate one of the contact-scoped partial unique
600
+ * indexes because the contact ALREADY holds a row for that live journey / live
601
+ * bucket / address. The sweep skips these on purpose (cancelling someone's
602
+ * live enrollment is not a background job's call), so counting them as
603
+ * `missing` would pin the gate shut forever on a population no sweep can
604
+ * drain. Reported separately so operators can triage it; NOT part of the gate.
605
+ */
606
+ duplicates: number;
454
607
  /** `contact_id` points at a contact that owns the row's `user_id` NEITHER
455
608
  * canonically NOR by alias (including a pointer at no contact at all). The
456
609
  * corruption detector, and the one count an FK could never give: an FK proves
@@ -472,8 +625,10 @@ export interface ContactIdVerifyResult {
472
625
  * **THE PRD 05 ENTRY GATE.** True iff EVERY table reports `missing === 0` AND
473
626
  * `mismatched === 0` — i.e. every row a live contact owns is stamped, and
474
627
  * every stamp points at a contact that really owns that row's key. `orphaned`
475
- * is deliberately NOT part of the gate (D5 makes it legitimately non-zero
476
- * forever). PRD 05 flips reads from `user_id` onto `contact_id`; flipping
628
+ * and `duplicates` are deliberately NOT part of the gate (D5 makes the first
629
+ * legitimately non-zero forever; the second is a population the sweep skips on
630
+ * purpose — see {@link ContactIdVerifyCounts.duplicates} — so gating on it
631
+ * would shut the door on a number no sweep can drain). PRD 05 flips reads from `user_id` onto `contact_id`; flipping
477
632
  * while this is false means silently losing history (`missing`) or attributing
478
633
  * it to the wrong person (`mismatched`). Judge it AFTER a completed sweep — a
479
634
  * false reading with no sweep on record is just a pending backfill.
@@ -500,10 +655,13 @@ export interface ContactIdVerifyResult {
500
655
  *
501
656
  * Three properties worth naming:
502
657
  *
503
- * - `missing` and `orphaned` PARTITION the NULL-stamped rows: every row with
504
- * `contact_id IS NULL` is one or the other, never both, never neither. So
505
- * `missing + orphaned = count(*) WHERE contact_id IS NULL`, which is what
506
- * makes "zero NULLs" the wrong completion criterion and this the right one.
658
+ * - `missing`, `duplicates` and `orphaned` PARTITION the NULL-stamped rows:
659
+ * every row with `contact_id IS NULL` is exactly one of the three, so
660
+ * `missing + duplicates + orphaned = count(*) WHERE contact_id IS NULL` —
661
+ * which is what makes "zero NULLs" the wrong completion criterion and this
662
+ * the right one. `duplicates` (PRD 05 T3) splits off the owned rows the sweep
663
+ * deliberately leaves NULL because stamping them would violate a
664
+ * contact-scoped unique index.
507
665
  * - `mismatched` does NOT require the pointed-at contact to be LIVE. A
508
666
  * soft-deleted contact whose key still matches is consistent data (someone
509
667
  * was deleted), not corruption; demanding liveness would fail the gate on
@@ -545,25 +703,44 @@ async function verifyTable(
545
703
  const name = sql.identifier(table);
546
704
  const scope = scopeToKeys(userIds);
547
705
 
548
- const [nulls] = await db.execute<{ missing: number; orphaned: number }>(sql`
549
- SELECT count(*) FILTER (WHERE s.owned)::int AS missing,
550
- count(*) FILTER (WHERE NOT s.owned)::int AS orphaned
706
+ // PRD 05 T3 — the owner is now RESOLVED (a LATERAL) rather than merely
707
+ // asserted (two EXISTS), because `duplicates` needs the owner's id to ask
708
+ // whether that contact already holds the row this stamp would duplicate. Same
709
+ // ownership definition, same two legs, same cost profile (the canonical leg is
710
+ // a `coalesce(...)` comparison either way, so neither shape is index-driven).
711
+ const collides = CONTACT_SCOPED_TABLES.has(table)
712
+ ? contactScopedTwinExists(table, sql`owner.id`)
713
+ : sql`false`;
714
+
715
+ const [nulls] = await db.execute<{
716
+ missing: number;
717
+ duplicates: number;
718
+ orphaned: number;
719
+ }>(sql`
720
+ SELECT count(*) FILTER (WHERE s.owner IS NOT NULL AND NOT s.collides)::int
721
+ AS missing,
722
+ count(*) FILTER (WHERE s.owner IS NOT NULL AND s.collides)::int
723
+ AS duplicates,
724
+ count(*) FILTER (WHERE s.owner IS NULL)::int
725
+ AS orphaned
551
726
  FROM (
552
- SELECT (
553
- EXISTS (
554
- SELECT 1 FROM contacts
555
- WHERE contacts.deleted_at IS NULL
556
- AND ${contactKeySql()} = t.user_id
557
- )
558
- OR EXISTS (
727
+ SELECT owner.id AS owner, (${collides}) AS collides
728
+ FROM ${name} t
729
+ LEFT JOIN LATERAL (
730
+ SELECT contacts.id
731
+ FROM contacts
732
+ WHERE contacts.deleted_at IS NULL
733
+ AND (
734
+ ${contactKeySql()} = t.user_id
735
+ OR EXISTS (
559
736
  SELECT 1 FROM contact_aliases a
560
- JOIN contacts ON contacts.id = a.contact_id
561
- AND contacts.deleted_at IS NULL
562
- WHERE a.alias_value = t.user_id
737
+ WHERE a.contact_id = contacts.id
738
+ AND a.alias_value = t.user_id
563
739
  AND a.alias_kind IN ('external', 'anonymous')
564
740
  )
565
- ) AS owned
566
- FROM ${name} t
741
+ )
742
+ LIMIT 1
743
+ ) owner ON true
567
744
  WHERE t.contact_id IS NULL${scope}
568
745
  ) s
569
746
  `);
@@ -585,6 +762,7 @@ async function verifyTable(
585
762
 
586
763
  return {
587
764
  missing: Number(nulls?.missing ?? 0),
765
+ duplicates: Number(nulls?.duplicates ?? 0),
588
766
  mismatched: Number(stamped?.mismatched ?? 0),
589
767
  orphaned: Number(nulls?.orphaned ?? 0),
590
768
  };
@@ -629,6 +807,7 @@ export async function verifyContactIdBackfill(opts: {
629
807
  const tables = {} as Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
630
808
  const totals: ContactIdVerifyCounts = {
631
809
  missing: 0,
810
+ duplicates: 0,
632
811
  mismatched: 0,
633
812
  orphaned: 0,
634
813
  };
@@ -637,6 +816,7 @@ export async function verifyContactIdBackfill(opts: {
637
816
  const counts = await verifyTable(opts.db, table, opts.userIds);
638
817
  tables[table] = counts;
639
818
  totals.missing += counts.missing;
819
+ totals.duplicates += counts.duplicates;
640
820
  totals.mismatched += counts.mismatched;
641
821
  totals.orphaned += counts.orphaned;
642
822
  }