@hogsend/engine 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +19 -10
  3. package/src/buckets/check-membership.ts +55 -8
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +3 -0
  6. package/src/index.ts +10 -6
  7. package/src/journeys/execute-journey-run.ts +119 -50
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +27 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +4 -1
  13. package/src/lib/contacts.ts +408 -152
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +14 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +78 -29
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/revenue.ts +20 -5
  23. package/src/lib/sms-tracked.ts +18 -1
  24. package/src/lib/tracked.ts +19 -8
  25. package/src/lib/tracking-events.ts +28 -0
  26. package/src/lib/unique-violation.ts +44 -0
  27. package/src/routes/admin/buckets.ts +20 -2
  28. package/src/routes/admin/bulk.ts +14 -2
  29. package/src/routes/admin/contacts.ts +4 -3
  30. package/src/routes/admin/conversions.ts +20 -4
  31. package/src/routes/admin/emails.ts +17 -2
  32. package/src/routes/admin/events.ts +32 -41
  33. package/src/routes/admin/funnels.ts +6 -1
  34. package/src/routes/admin/groups.ts +4 -0
  35. package/src/routes/admin/impact.ts +30 -10
  36. package/src/routes/admin/journey-impact.ts +18 -9
  37. package/src/routes/admin/journeys.ts +8 -1
  38. package/src/routes/admin/maintenance.ts +7 -1
  39. package/src/routes/admin/preferences.ts +85 -46
  40. package/src/routes/admin/reporting.ts +25 -5
  41. package/src/routes/admin/targeting.ts +1 -0
  42. package/src/routes/admin/timeline.ts +16 -10
  43. package/src/routes/email/preferences.ts +18 -1
  44. package/src/routes/lists/index.ts +6 -6
  45. package/src/workflows/backfill-contact-id.ts +210 -30
  46. package/src/workflows/bucket-backfill.ts +145 -67
  47. package/src/workflows/bucket-reconcile.ts +150 -50
  48. package/src/workflows/check-alerts.ts +4 -0
  49. package/src/workflows/impact-digest.ts +10 -3
  50. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  51. package/src/workflows/send-campaign.ts +18 -11
@@ -13,11 +13,13 @@ import {
13
13
  } from "@hogsend/db";
14
14
  import {
15
15
  and,
16
+ type Column,
16
17
  eq,
17
18
  ilike,
18
19
  inArray,
19
20
  isNotNull,
20
21
  isNull,
22
+ not,
21
23
  or,
22
24
  type SQL,
23
25
  sql,
@@ -804,6 +806,111 @@ export function contactKeySql() {
804
806
  return sql<string>`coalesce(${contacts.externalId}, ${contacts.anonymousId}, ${contacts.id}::text)`;
805
807
  }
806
808
 
809
+ /**
810
+ * The ADOPTION STAMP fragment: `coalesce(<table>.contact_id, :contactId)`.
811
+ *
812
+ * Every statement below that moves a history row's mutable `user_id` string
813
+ * onto an absorbing contact's key ALSO stamps that contact's uuid — so the
814
+ * later read-flip onto `contact_id` leaves no adopted row behind. Two
815
+ * properties make this fragment (rather than a bare `contact_id = :id`) the
816
+ * only correct shape:
817
+ *
818
+ * - SECURITY, not optimization. The `coalesce` is a NULL guard: it stamps only
819
+ * rows nobody owns yet (written while the contact did not exist — the whole
820
+ * point of refusing to mint on observation). A row already carrying ANOTHER
821
+ * contact's id is left alone, so a mis-gated adoption can never re-parent a
822
+ * second person's history even if it does move the string key.
823
+ * - It must ride IN the same UPDATE that rewrites `user_id`. A separate stamp
824
+ * statement afterwards matches zero rows — the key it would filter on has
825
+ * already moved — and one before it would need its own scan of the old key.
826
+ *
827
+ * Idempotent by construction: re-running an adoption finds `contact_id` already
828
+ * non-NULL and changes nothing.
829
+ *
830
+ * The `::uuid` cast pins the bound parameter's type the way {@link
831
+ * contactKeySql}'s `::text` does; without it the param arrives untyped and
832
+ * Postgres has to infer it from the coalesce.
833
+ */
834
+ function adoptedContactId(column: Column, contactId: string): SQL<string> {
835
+ return sql<string>`coalesce(${column}, ${contactId}::uuid)`;
836
+ }
837
+
838
+ /**
839
+ * How a fold writes its routed rows (PRD 05 T9).
840
+ *
841
+ * - Merge mode (default): the historical shape — rewrite `user_id` onto the
842
+ * survivor key, stamp `contact_id` under the {@link adoptedContactId} NULL
843
+ * guard, denormalize the survivor's email.
844
+ * - `stampOnly` (the adoption path): one column. The loser scope is
845
+ * `orphansOnly` (see {@link foldScopes}), so every matched row carries
846
+ * `contact_id IS NULL` and a plain stamp cannot re-parent another person's
847
+ * history. `user_id` is frozen at its write-time value — reads no longer
848
+ * consult it, and D9 keeps writing it on inserts.
849
+ */
850
+ interface FoldWriteOpts {
851
+ stampOnly?: boolean;
852
+ }
853
+
854
+ /**
855
+ * PRD 05 T3 — "which rows belong to the loser, and which to the survivor", for
856
+ * the three folds below.
857
+ *
858
+ * Migration 0071 adds a CONTACT-scoped partial unique index to `journey_states`,
859
+ * `bucket_memberships` and `email_preferences`. That makes a string-key-only
860
+ * fold unsafe: adoption stamps `contact_id` WITHOUT rewriting `user_id`, so a
861
+ * row the SURVIVOR already owns can sit under a stale key. A fold that looked
862
+ * only at `user_id = survivorKey` would not see it, would call the slot free,
863
+ * and would re-point (or NULL-stamp) a loser row into a second row satisfying
864
+ * the new index — a 23505 raised inside `resolveContact`'s transaction with no
865
+ * handler, which wedges identity resolution for that person permanently (every
866
+ * retry fails identically). So ownership is asked BOTH ways, `user_id` and
867
+ * `contact_id`.
868
+ *
869
+ * `loserId` is the mirror on the other side and is supplied only by the merge
870
+ * path: the adoption path ({@link adoptOrphanHistory}) folds history onto the
871
+ * contact that already owns its key, where the "loser" rows would carry that
872
+ * same `contact_id` and matching on it would swallow the whole contact. The
873
+ * loser test wins the tie — a row is either being adopted or being folded
874
+ * into, never both.
875
+ *
876
+ * `orphansOnly` (the adoption path, PRD 05 T9): the loser side additionally
877
+ * requires `contact_id IS NULL`. This predicate is the anti-theft guard that
878
+ * replaced `keysAnotherContact`'s adoption-gating role — a row another contact
879
+ * owns already carries that owner's id and can never match, so adoption
880
+ * cannot re-parent a second person's history. Never combined with `loserId`
881
+ * (only the merge path supplies one).
882
+ */
883
+ function foldScopes(opts: {
884
+ userIdCol: Column;
885
+ contactIdCol: Column;
886
+ survivorKey: string;
887
+ survivorId: string;
888
+ loserKeys: string[];
889
+ loserId?: string;
890
+ orphansOnly?: boolean;
891
+ }): { isLoser: SQL; isSurvivor: SQL } {
892
+ // `IS NOT DISTINCT FROM`, never `=`. `contact_id` is nullable, and a plain
893
+ // equality against a NULL column is UNKNOWN, not false — which `NOT (… OR
894
+ // UNKNOWN)` then swallows, silently dropping EVERY contactless row out of the
895
+ // survivor set. That is the occupancy the folds dedupe against, so the folds
896
+ // would stop seeing the rows they exist to protect.
897
+ const sameContact = (id: string): SQL =>
898
+ sql`${opts.contactIdCol} IS NOT DISTINCT FROM ${id}::uuid`;
899
+
900
+ const byKey = opts.orphansOnly
901
+ ? (and(
902
+ inArray(opts.userIdCol, opts.loserKeys),
903
+ isNull(opts.contactIdCol),
904
+ ) as SQL)
905
+ : inArray(opts.userIdCol, opts.loserKeys);
906
+ const isLoser = opts.loserId ? or(byKey, sameContact(opts.loserId)) : byKey;
907
+ const isSurvivor = and(
908
+ or(eq(opts.userIdCol, opts.survivorKey), sameContact(opts.survivorId)),
909
+ not(isLoser as SQL),
910
+ );
911
+ return { isLoser: isLoser as SQL, isSurvivor: isSurvivor as SQL };
912
+ }
913
+
807
914
  /**
808
915
  * The identity keys + options EVERY resolve entry point accepts. Named (rather
809
916
  * than inline) so {@link resolveOrCreateContact} and {@link
@@ -1106,35 +1213,36 @@ async function resolveContactShared(
1106
1213
 
1107
1214
  // HISTORY ADOPTION (D2). Once observation stops minting a row, a later
1108
1215
  // identify no longer finds an anon contact to fill-in-link — it lands
1109
- // HERE, in the create arm, which historically had no repoint. Everything
1110
- // already keyed on the anon id (user_events, journey_states,
1111
- // bucket_memberships, email_sends, email_preferences — incl. an
1112
- // unsubscribe) would be silently orphaned under a key no contact owns.
1216
+ // HERE, in the create arm. Everything already keyed on the anon id
1217
+ // (user_events, journey_states, bucket_memberships, email_sends,
1218
+ // email_preferences — incl. an unsubscribe) would otherwise be silently
1219
+ // orphaned under a key no contact owns.
1113
1220
  //
1114
- // So: when the brand-new row's canonical key (`external_id ??
1115
- // anonymous_id ?? id`) is NOT the supplied `anonymousId`, the anon key was
1116
- // just superseded — re-point its history exactly as `fillInLink` does on a
1117
- // key flip. Only ever FROM `anonymousId`: an email/discord-shaped key is
1118
- // never canonical, so it never keyed history and must never be repointed.
1119
- //
1120
- // Gated on `keysAnotherContact`: resolution missed this value in the
1121
- // ANONYMOUS namespace, which does not mean nobody owns it. If it is some
1122
- // other live contact's canonical key, those rows are theirs and adopting
1123
- // them here would hand one person's history to another.
1221
+ // Adoption is self-gating (T9): `adoptOrphanHistory` touches only
1222
+ // `contact_id IS NULL` rows, so a value that is some other live
1223
+ // contact's key cannot surrender their history — those rows carry their
1224
+ // owner's id. `keysAnotherContact` therefore no longer gates adoption;
1225
+ // it survives below ONLY to gate the `mergedKeys` REPORT — a foreign
1226
+ // anon id must never enter the analytics stitch.
1124
1227
  const createdKey = contactKey(createdRow);
1228
+ // History written under the row's OWN key BEFORE it existed. Not a
1229
+ // merge: no key is absorbed, so it reports nothing in `mergedKeys` and
1230
+ // fires no analytics stitch.
1231
+ await adoptOrphanHistory(tx, createdKey, createdRow);
1125
1232
  let mergedKeys: string[] | undefined;
1126
- if (
1127
- anonymousId &&
1128
- anonymousId !== createdKey &&
1129
- !(await keysAnotherContact(tx, anonymousId, createdRow.id))
1130
- ) {
1131
- await repointOwnHistory(tx, anonymousId, createdKey, createdRow);
1132
- // §5.3 emission point 2 (canonical-key flip): report the absorbed anon
1133
- // key so `ingestEvent` still fires `mergeAnalyticsIdentities({ reason:
1134
- // "key_flip" })` — otherwise the DB history is adopted but the PostHog
1135
- // anon→known stitch stays broken. Safe to alias by construction (an
1136
- // anonymous key never identified a person — MF-2).
1137
- mergedKeys = [anonymousId];
1233
+ if (anonymousId && anonymousId !== createdKey) {
1234
+ // The anon key was just superseded as canonical — stamp its orphans.
1235
+ // Only ever FROM `anonymousId`: an email/discord-shaped key is never
1236
+ // canonical, so it never keyed history.
1237
+ await adoptOrphanHistory(tx, anonymousId, createdRow);
1238
+ if (!(await keysAnotherContact(tx, anonymousId, createdRow.id))) {
1239
+ // §5.3 emission point 2 (canonical-key flip): report the absorbed
1240
+ // anon key so `ingestEvent` still fires `mergeAnalyticsIdentities({
1241
+ // reason: "key_flip" })` — otherwise the DB history is adopted but
1242
+ // the PostHog anon→known stitch stays broken. Safe to alias by
1243
+ // construction (an anonymous key never identified a person — MF-2).
1244
+ mergedKeys = [anonymousId];
1245
+ }
1138
1246
  }
1139
1247
 
1140
1248
  return {
@@ -1495,9 +1603,9 @@ async function fillInLink(
1495
1603
 
1496
1604
  await tx.update(contacts).set(set).where(eq(contacts.id, row.id));
1497
1605
 
1498
- // Re-point the contact's own history if the canonical key flipped. The
1499
- // updated row (with its new email/keys) is what foldJourneyStates/email_sends
1500
- // denormalize into.
1606
+ // Adopt the contact's own orphan history (stamp-only since T9; nothing
1607
+ // rewrites the string key anymore). The updated row carries the post-fill
1608
+ // keys the fold scopes resolve ownership with.
1501
1609
  const newKey = nextExternalId ?? nextAnonymousId ?? row.id;
1502
1610
  // §5.3 emission point 2 (canonical-key flip): when the key flips, the OLD key
1503
1611
  // is folded into the NEW one. MF-3 gate — only emit a merge when `oldKey` was
@@ -1518,9 +1626,12 @@ async function fillInLink(
1518
1626
 
1519
1627
  let mergedKeys: string[] | undefined;
1520
1628
  let mergedIdentifiedKeys: string[] | undefined;
1629
+ // T9: adoption is unconditional — with nothing keyed on the canonical key,
1630
+ // "did the key flip" no longer decides whether history is stamped. Orphans
1631
+ // under the pre-fill key are adopted either way (idempotent when there are
1632
+ // none); the flip test now gates only the analytics REPORT.
1633
+ await adoptOrphanHistory(tx, oldKey, updatedRow);
1521
1634
  if (newKey !== oldKey) {
1522
- await repointOwnHistory(tx, oldKey, newKey, updatedRow);
1523
-
1524
1635
  const oldKeyWasExternalId =
1525
1636
  row.externalId != null && oldKey === row.externalId;
1526
1637
  if (oldKeyWasExternalId) {
@@ -1568,7 +1679,7 @@ async function fillInLink(
1568
1679
  for (const key of claimed) {
1569
1680
  if (key.kind !== "anonymous") continue;
1570
1681
  if (key.value === newKey || key.value === oldKey) continue;
1571
- await repointOwnHistory(tx, key.value, newKey, updatedRow);
1682
+ await adoptOrphanHistory(tx, key.value, updatedRow);
1572
1683
  // Reported so `mergeAnalyticsIdentities` still fires the anon→known stitch;
1573
1684
  // appended, since a key flip above may already have folded a uuid/anon key.
1574
1685
  mergedKeys = [...(mergedKeys ?? []), key.value];
@@ -1630,22 +1741,38 @@ async function mergeContacts(
1630
1741
  safeLoserKeys.push(loser.anonymousId ?? loser.id);
1631
1742
  }
1632
1743
 
1633
- // (ii) user_events.user_id rewrite.
1744
+ // (ii) user_events.user_id rewrite. The paired `contact_id` stamp is the
1745
+ // ADOPTION half only (NULL rows — history the loser's keys accumulated
1746
+ // before any contact existed). Rows already owned by the LOSER keep
1747
+ // `loser.id` here and are re-pointed wholesale by (vi-b-hist) below, which
1748
+ // is the one statement that may overwrite a non-NULL owner. Splitting it
1749
+ // that way keeps a single rule in the fragment (never take a row from
1750
+ // another contact) and one explicit, auditable place where a merge does.
1634
1751
  await tx
1635
1752
  .update(userEvents)
1636
- .set({ userId: survivorKey })
1753
+ .set({
1754
+ userId: survivorKey,
1755
+ contactId: adoptedContactId(userEvents.contactId, survivor.id),
1756
+ })
1637
1757
  .where(inArray(userEvents.userId, loserKeysToRewrite));
1638
1758
 
1639
1759
  // (iii) journey_states — exit the loser's duplicate active/waiting row when
1640
1760
  // the survivor already holds an active/waiting row in the same journey
1641
1761
  // (respect uq_user_journey_active), THEN rewrite user_id/user_email.
1642
- await foldJourneyStates(tx, survivorKey, loserKeysToRewrite, survivor);
1762
+ await foldJourneyStates(
1763
+ tx,
1764
+ survivorKey,
1765
+ loserKeysToRewrite,
1766
+ survivor,
1767
+ loser.id,
1768
+ );
1643
1769
 
1644
1770
  // (iv) email_sends rewrite user_id + userEmail to survivor's.
1645
1771
  await tx
1646
1772
  .update(emailSends)
1647
1773
  .set({
1648
1774
  userId: survivorKey,
1775
+ contactId: adoptedContactId(emailSends.contactId, survivor.id),
1649
1776
  ...(survivor.email ? { userEmail: survivor.email } : {}),
1650
1777
  })
1651
1778
  .where(inArray(emailSends.userId, loserKeysToRewrite));
@@ -1653,17 +1780,29 @@ async function mergeContacts(
1653
1780
  // (v) bucket_memberships — soft-leave the loser's duplicate active
1654
1781
  // membership when the survivor already holds one in the same bucket (respect
1655
1782
  // uq_user_bucket_active, preserve survivor's dwell clock), THEN rewrite.
1656
- await foldBucketMemberships(tx, survivorKey, loserKeysToRewrite);
1783
+ await foldBucketMemberships(
1784
+ tx,
1785
+ survivorKey,
1786
+ loserKeysToRewrite,
1787
+ survivor.id,
1788
+ loser.id,
1789
+ );
1657
1790
 
1658
1791
  // (vi) email_preferences FOLD (never blind-rewrite — risk 6).
1659
- await foldEmailPreferences(tx, loserKeysToRewrite, survivorKey);
1792
+ await foldEmailPreferences(
1793
+ tx,
1794
+ loserKeysToRewrite,
1795
+ survivorKey,
1796
+ survivor.id,
1797
+ loser.id,
1798
+ );
1660
1799
 
1661
1800
  // (vi-b) deals + crm_links re-point: these carry contact_id uuid FKs
1662
1801
  // (not user keys), which the key rewrites above never touch. Without
1663
1802
  // this, the loser's open deal is orphaned on a soft-deleted row — the
1664
1803
  // survivor's next stage event/trigger would mint a SECOND deal (and a
1665
- // duplicate deal.sold). No unique index involves contact_id, so plain
1666
- // UPDATEs suffice.
1804
+ // duplicate deal.sold). Neither table has a unique index on contact_id,
1805
+ // so plain UPDATEs suffice.
1667
1806
  await tx
1668
1807
  .update(deals)
1669
1808
  .set({ contactId: survivor.id })
@@ -1674,15 +1813,32 @@ async function mergeContacts(
1674
1813
  .where(eq(crmLinks.contactId, loser.id));
1675
1814
 
1676
1815
  // (vi-b-hist) the five HISTORY tables' `contact_id` re-point (PRD 04 T3).
1677
- // Same shape and same reasoning as the deals/crm_links repoints above: a
1678
- // uuid column the string-key rewrites never touch, and no unique index
1679
- // involves `contact_id`, so plain UPDATEs suffice (no fold needed — unlike
1680
- // the (vi)/(vi-c) folds, which exist only to respect a unique index).
1816
+ // Same shape as the deals/crm_links repoints above: a uuid column the
1817
+ // string-key rewrites never touch.
1818
+ //
1819
+ // PRD 05 T3: `contact_id` IS now a unique-index column on three of these
1820
+ // five (migration 0071), so "plain UPDATEs suffice" is no longer true on its
1821
+ // own — it holds only because the folds above ran FIRST and already know
1822
+ // about `contact_id`. {@link foldScopes} makes each fold read the survivor's
1823
+ // occupancy AND the loser's rows by `user_id` OR `contact_id`, so by the
1824
+ // time these statements run, every loser row that would have duplicated a
1825
+ // survivor row inside a contact-scoped index has been exited, left or
1826
+ // deleted. That is the invariant these three statements depend on; widen a
1827
+ // fold's scope, never this.
1681
1828
  // Deliberately lands AHEAD of the dual-write that populates the column
1682
1829
  // (PRD 04 D9/D10): while every row is still NULL this is a pure no-op, and
1683
1830
  // the moment the dual-write starts, merges are already correct. Shipped in
1684
1831
  // the other order, every merge in between would strand history rows
1685
1832
  // pointing at a soft-deleted loser contact.
1833
+ //
1834
+ // It is also the OVERWRITE half of the key rewrites above: those stamp only
1835
+ // rows nobody owned (the {@link adoptedContactId} NULL guard), so the
1836
+ // loser-OWNED rows arrive here still carrying `loser.id` and this is what
1837
+ // moves them. Between the two halves, every row the merge leaves keyed to
1838
+ // the survivor also carries the survivor's uuid. Runs after the rewrites
1839
+ // rather than before because it filters on `contact_id`, not `user_id` —
1840
+ // the order the key rewrites move is irrelevant to it — and re-running the
1841
+ // pair changes nothing (`contact_id = loser.id` matches zero rows twice).
1686
1842
  await tx
1687
1843
  .update(userEvents)
1688
1844
  .set({ contactId: survivor.id })
@@ -1867,11 +2023,10 @@ async function mergeContacts(
1867
2023
  await ensureIdentityAliases(tx, postSurvivor);
1868
2024
 
1869
2025
  // If the survivor's canonical key flipped (it had no external_id/anonymous_id
1870
- // and the merge promoted one from the call/loser), re-point the survivor's OWN
1871
- // history — including everything the loser rewrites just pointed at the old
1872
- // survivorKey — onto the new key (risk 1). Without this, a survivor whose
1873
- // history was keyed on its uuid/anonymous_id is orphaned the moment it gains
1874
- // an external_id mid-merge.
2026
+ // and the merge promoted one from the call/loser), stamp any orphans still
2027
+ // sitting under the OLD survivor key (risk 1). The loser folds above already
2028
+ // stamped everything they routed, so this reaches only rows that predate the
2029
+ // survivor's own contact_id era.
1875
2030
  const newSurvivorKey =
1876
2031
  (survivorSet.externalId as string | undefined) ??
1877
2032
  survivor.externalId ??
@@ -1887,7 +2042,7 @@ async function mergeContacts(
1887
2042
  (survivorSet.anonymousId as string | undefined) ?? survivor.anonymousId,
1888
2043
  email: (survivorSet.email as string | undefined) ?? survivor.email,
1889
2044
  };
1890
- await repointOwnHistory(tx, survivorKey, newSurvivorKey, updatedSurvivor);
2045
+ await adoptOrphanHistory(tx, survivorKey, updatedSurvivor);
1891
2046
  }
1892
2047
 
1893
2048
  // `newSurvivorKey` IS the post-merge canonical key of the survivor — the same
@@ -1911,43 +2066,56 @@ async function mergeContacts(
1911
2066
  * the survivor key still collides whenever the survivor already holds a live row
1912
2067
  * for that journey.
1913
2068
  *
1914
- * Fix: build the survivor's occupied (journey_id|status) set over ALL statuses.
1915
- * For active/waiting collisions, EXIT the loser's row first (preserve the
2069
+ * Fix: build the survivor's occupancy and route each loser row to exit / delete /
2070
+ * rewrite. For a LIVE collision, EXIT the loser's row first (preserve the
1916
2071
  * survivor's live run) so the rewrite lands an 'exited' (out-of-predicate) row.
1917
- * For any OTHER collision (terminal), DELETE the loser's duplicate — no longer
1918
- * REQUIRED by the constraint (terminal rows are outside the predicate), but kept
1919
- * as hygiene so the survivor doesn't carry two identical terminal rows (which
1920
- * would inflate the count()-based ctx.history.journey.entryCount). Rewrite only
1921
- * the non-colliding remainder onto the survivor key (+ survivor email). Re-check
2072
+ * For a TERMINAL collision, DELETE the loser's duplicate — no longer REQUIRED by
2073
+ * the constraint (terminal rows are outside the predicate), but kept as hygiene
2074
+ * so the survivor doesn't carry two identical terminal rows (which would inflate
2075
+ * the count()-based ctx.history.journey.entryCount). Rewrite only the
2076
+ * non-colliding remainder onto the survivor key (+ survivor email). Re-check
1922
2077
  * 'exited' occupancy after exiting so a just-exited loser row that would now
1923
2078
  * duplicate a pre-existing survivor 'exited' row is dropped rather than rewritten.
2079
+ *
2080
+ * Occupancy is asked BOTH ways (see {@link foldScopes}), and LIVE occupancy is
2081
+ * keyed on the journey ALONE: both unique indexes say `status IN ('active',
2082
+ * 'waiting')`, i.e. the two live statuses share ONE slot, so a survivor 'active'
2083
+ * row and a loser 'waiting' row in the same journey collide even though their
2084
+ * (journey|status) pairs differ.
1924
2085
  */
1925
2086
  async function foldJourneyStates(
1926
2087
  tx: Tx,
1927
2088
  survivorKey: string,
1928
2089
  loserKeys: string[],
1929
2090
  survivor: ContactRow,
2091
+ loserId?: string,
2092
+ opts?: FoldWriteOpts,
1930
2093
  ): Promise<void> {
1931
2094
  const ACTIVE = new Set<string>(["active", "waiting"]);
2095
+ const { isLoser, isSurvivor } = foldScopes({
2096
+ userIdCol: journeyStates.userId,
2097
+ contactIdCol: journeyStates.contactId,
2098
+ survivorKey,
2099
+ survivorId: survivor.id,
2100
+ loserKeys,
2101
+ loserId,
2102
+ orphansOnly: opts?.stampOnly,
2103
+ });
1932
2104
 
1933
- // Every (journey_id|status) pair the survivor already holds (ALL statuses).
1934
2105
  const survivorRows = await tx
1935
2106
  .select({
1936
2107
  journeyId: journeyStates.journeyId,
1937
2108
  status: journeyStates.status,
1938
2109
  })
1939
2110
  .from(journeyStates)
1940
- .where(
1941
- and(
1942
- eq(journeyStates.userId, survivorKey),
1943
- isNull(journeyStates.deletedAt),
1944
- ),
1945
- );
1946
- // Running occupied set — mutated as we exit/rewrite loser rows so two loser
1947
- // rows in the same journey/status (3-way merge) can't collide with each other.
1948
- const occupied = new Set(
1949
- survivorRows.map((s) => `${s.journeyId}|${s.status}`),
2111
+ .where(and(isSurvivor, isNull(journeyStates.deletedAt)));
2112
+
2113
+ // Both sets are mutated as loser rows are routed, so two loser rows from
2114
+ // different keys (3-way merge) can't collide with each other either.
2115
+ const live = new Set(
2116
+ survivorRows.filter((s) => ACTIVE.has(s.status)).map((s) => s.journeyId),
1950
2117
  );
2118
+ const slots = new Set(survivorRows.map((s) => `${s.journeyId}|${s.status}`));
1951
2119
 
1952
2120
  const loserRows = await tx
1953
2121
  .select({
@@ -1956,41 +2124,39 @@ async function foldJourneyStates(
1956
2124
  status: journeyStates.status,
1957
2125
  })
1958
2126
  .from(journeyStates)
1959
- .where(
1960
- and(
1961
- inArray(journeyStates.userId, loserKeys),
1962
- isNull(journeyStates.deletedAt),
1963
- ),
1964
- );
2127
+ .where(and(isLoser, isNull(journeyStates.deletedAt)));
1965
2128
 
1966
2129
  const idsToExit: string[] = [];
1967
2130
  const idsToDelete: string[] = [];
1968
2131
  const idsToRewrite: string[] = [];
1969
2132
 
1970
2133
  for (const l of loserRows) {
1971
- const key = `${l.journeyId}|${l.status}`;
1972
- if (occupied.has(key)) {
1973
- if (ACTIVE.has(l.status)) {
1974
- // Survivor (or a prior loser) already holds a live row in this
1975
- // journey/status — exit the loser's so the live run continues. Only do
1976
- // so if the resulting 'exited' slot is itself free; otherwise drop it.
1977
- const exitedKey = `${l.journeyId}|exited`;
1978
- if (occupied.has(exitedKey)) {
1979
- idsToDelete.push(l.id);
1980
- } else {
1981
- idsToExit.push(l.id);
1982
- occupied.add(exitedKey);
1983
- }
1984
- } else {
1985
- // Terminal collision (both completed/failed/exited the same journey) —
1986
- // the survivor already records this state; drop the loser duplicate.
2134
+ const slot = `${l.journeyId}|${l.status}`;
2135
+ if (ACTIVE.has(l.status)) {
2136
+ if (!live.has(l.journeyId)) {
2137
+ // The one live slot for this journey is free — claim it.
2138
+ idsToRewrite.push(l.id);
2139
+ live.add(l.journeyId);
2140
+ slots.add(slot);
2141
+ continue;
2142
+ }
2143
+ // Survivor (or a prior loser) already holds the live slot — exit the
2144
+ // loser's row so the live run continues. Only do so if the resulting
2145
+ // 'exited' slot is itself free; otherwise drop it.
2146
+ const exitedSlot = `${l.journeyId}|exited`;
2147
+ if (slots.has(exitedSlot)) {
1987
2148
  idsToDelete.push(l.id);
2149
+ } else {
2150
+ idsToExit.push(l.id);
2151
+ slots.add(exitedSlot);
1988
2152
  }
2153
+ } else if (slots.has(slot)) {
2154
+ // Terminal collision (both completed/failed/exited the same journey) —
2155
+ // the survivor already records this state; drop the loser duplicate.
2156
+ idsToDelete.push(l.id);
1989
2157
  } else {
1990
- // Free slot — rewrite onto the survivor key. Claim it so a sibling loser
1991
- // row in the same journey/status routes to exit/delete instead.
1992
2158
  idsToRewrite.push(l.id);
1993
- occupied.add(key);
2159
+ slots.add(slot);
1994
2160
  }
1995
2161
  }
1996
2162
 
@@ -2007,17 +2173,25 @@ async function foldJourneyStates(
2007
2173
  .where(inArray(journeyStates.id, idsToExit));
2008
2174
  }
2009
2175
 
2010
- // Rewrite both the originally non-colliding rows AND the just-exited rows onto
2011
- // the survivor key (the exited rows now sit in claimed-free 'exited' slots).
2176
+ // Route both the originally non-colliding rows AND the just-exited rows to
2177
+ // the survivor (the exited rows now sit in claimed-free 'exited' slots).
2178
+ // Stamp mode (adoption, PRD 05 T9): one column — every matched row is an
2179
+ // orphan (`orphansOnly` scope), so a plain stamp cannot re-parent anything.
2180
+ // The string key and denormalized email stay frozen at write-time values.
2012
2181
  const rewriteIds = [...idsToRewrite, ...idsToExit];
2013
2182
  if (rewriteIds.length > 0) {
2014
2183
  await tx
2015
2184
  .update(journeyStates)
2016
- .set({
2017
- userId: survivorKey,
2018
- ...(survivor.email ? { userEmail: survivor.email } : {}),
2019
- updatedAt: new Date(),
2020
- })
2185
+ .set(
2186
+ opts?.stampOnly
2187
+ ? { contactId: survivor.id, updatedAt: new Date() }
2188
+ : {
2189
+ userId: survivorKey,
2190
+ contactId: adoptedContactId(journeyStates.contactId, survivor.id),
2191
+ ...(survivor.email ? { userEmail: survivor.email } : {}),
2192
+ updatedAt: new Date(),
2193
+ },
2194
+ )
2021
2195
  .where(inArray(journeyStates.id, rewriteIds));
2022
2196
  }
2023
2197
  }
@@ -2027,18 +2201,35 @@ async function foldJourneyStates(
2027
2201
  * a bucket where a loser key also holds one, soft-LEAVE the loser's row first
2028
2202
  * (uq_user_bucket_active forbids two active rows for the same (user, bucket);
2029
2203
  * preserve the survivor's dwell clock), then rewrite the rest onto the survivor.
2204
+ *
2205
+ * `survivorId` is the absorbing contact's uuid — the soft-left rows are rewritten
2206
+ * by the same statement as the rest, so both halves of a folded pair leave here
2207
+ * carrying it (see {@link adoptedContactId} for the NULL guard).
2030
2208
  */
2031
2209
  async function foldBucketMemberships(
2032
2210
  tx: Tx,
2033
2211
  survivorKey: string,
2034
2212
  loserKeys: string[],
2213
+ survivorId: string,
2214
+ loserId?: string,
2215
+ opts?: FoldWriteOpts,
2035
2216
  ): Promise<void> {
2217
+ const { isLoser, isSurvivor } = foldScopes({
2218
+ userIdCol: bucketMemberships.userId,
2219
+ contactIdCol: bucketMemberships.contactId,
2220
+ survivorKey,
2221
+ survivorId,
2222
+ loserKeys,
2223
+ loserId,
2224
+ orphansOnly: opts?.stampOnly,
2225
+ });
2226
+
2036
2227
  const survivorActive = await tx
2037
2228
  .select({ bucketId: bucketMemberships.bucketId })
2038
2229
  .from(bucketMemberships)
2039
2230
  .where(
2040
2231
  and(
2041
- eq(bucketMemberships.userId, survivorKey),
2232
+ isSurvivor,
2042
2233
  eq(bucketMemberships.status, "active"),
2043
2234
  isNull(bucketMemberships.deletedAt),
2044
2235
  ),
@@ -2053,15 +2244,19 @@ async function foldBucketMemberships(
2053
2244
  .from(bucketMemberships)
2054
2245
  .where(
2055
2246
  and(
2056
- inArray(bucketMemberships.userId, loserKeys),
2247
+ isLoser,
2057
2248
  eq(bucketMemberships.status, "active"),
2058
2249
  isNull(bucketMemberships.deletedAt),
2059
2250
  ),
2060
2251
  );
2061
2252
 
2062
- const idsToLeave = loserActive
2063
- .filter((l) => occupied.has(l.bucketId))
2064
- .map((l) => l.id);
2253
+ // Claim as we go, so two loser rows active in the SAME bucket (3-way merge)
2254
+ // route the second to 'left' instead of both rewriting onto the survivor.
2255
+ const idsToLeave: string[] = [];
2256
+ for (const l of loserActive) {
2257
+ if (occupied.has(l.bucketId)) idsToLeave.push(l.id);
2258
+ else occupied.add(l.bucketId);
2259
+ }
2065
2260
 
2066
2261
  if (idsToLeave.length > 0) {
2067
2262
  await tx
@@ -2072,8 +2267,19 @@ async function foldBucketMemberships(
2072
2267
 
2073
2268
  await tx
2074
2269
  .update(bucketMemberships)
2075
- .set({ userId: survivorKey, updatedAt: new Date() })
2076
- .where(inArray(bucketMemberships.userId, loserKeys));
2270
+ .set(
2271
+ opts?.stampOnly
2272
+ ? { contactId: survivorId, updatedAt: new Date() }
2273
+ : {
2274
+ userId: survivorKey,
2275
+ contactId: adoptedContactId(
2276
+ bucketMemberships.contactId,
2277
+ survivorId,
2278
+ ),
2279
+ updatedAt: new Date(),
2280
+ },
2281
+ )
2282
+ .where(isLoser);
2077
2283
  }
2078
2284
 
2079
2285
  /**
@@ -2127,18 +2333,35 @@ async function foldGroupMemberships(
2127
2333
  * 3-way merge where two losers each carry a pref for the SAME email folds
2128
2334
  * loser2 into loser1's already-folded result instead of colliding on
2129
2335
  * `uq(user_id, email)` (risk 3). The loser row is deleted after folding.
2336
+ *
2337
+ * This is the one fold whose SURVIVING row is not the row that moved — a
2338
+ * collision keeps the TARGET and deletes the loser — so `survivorId` is stamped
2339
+ * on both branches: onto the re-pointed row when the slot is free, and onto the
2340
+ * target as part of the fold write when it is not. Same NULL guard either way
2341
+ * (see {@link adoptedContactId}); the fold branch expresses it in TS because the
2342
+ * target row is already in hand.
2130
2343
  */
2131
2344
  async function foldEmailPreferences(
2132
2345
  tx: Tx,
2133
2346
  loserKeys: string[],
2134
2347
  survivorKey: string,
2348
+ survivorId: string,
2349
+ loserId?: string,
2350
+ opts?: FoldWriteOpts,
2135
2351
  ): Promise<void> {
2136
2352
  if (loserKeys.length === 0) return;
2137
2353
 
2138
- const loserPrefs = await tx
2139
- .select()
2140
- .from(emailPreferences)
2141
- .where(inArray(emailPreferences.userId, loserKeys));
2354
+ const { isLoser, isSurvivor } = foldScopes({
2355
+ userIdCol: emailPreferences.userId,
2356
+ contactIdCol: emailPreferences.contactId,
2357
+ survivorKey,
2358
+ survivorId,
2359
+ loserKeys,
2360
+ loserId,
2361
+ orphansOnly: opts?.stampOnly,
2362
+ });
2363
+
2364
+ const loserPrefs = await tx.select().from(emailPreferences).where(isLoser);
2142
2365
 
2143
2366
  if (loserPrefs.length === 0) return;
2144
2367
 
@@ -2149,25 +2372,37 @@ async function foldEmailPreferences(
2149
2372
  };
2150
2373
 
2151
2374
  for (const lp of loserPrefs) {
2152
- // Re-read the CURRENT target row for (survivorKey, lp.email) — it may be the
2153
- // original survivor pref, a prior loser's just-folded pref, or absent.
2375
+ // Re-read the CURRENT target row for this address — it may be the original
2376
+ // survivor pref, a survivor row adoption stamped under a STALE key, a prior
2377
+ // loser's just-folded pref, or absent. A stamped row is preferred when both
2378
+ // exist: it is the row the contact-scoped read resolves, and the row whose
2379
+ // uniqueness a re-point would violate.
2154
2380
  const targetRows = await tx
2155
2381
  .select()
2156
2382
  .from(emailPreferences)
2157
- .where(
2158
- and(
2159
- eq(emailPreferences.userId, survivorKey),
2160
- eq(emailPreferences.email, lp.email),
2161
- ),
2162
- )
2383
+ .where(and(isSurvivor, eq(emailPreferences.email, lp.email)))
2384
+ .orderBy(sql`${emailPreferences.contactId} IS NULL`)
2163
2385
  .limit(1);
2164
2386
  const target = targetRows[0];
2165
2387
 
2166
2388
  if (!target) {
2167
- // The (survivorKey, lp.email) slot is free — re-point the loser row.
2389
+ // The slot is free on BOTH keys. Merge mode re-points the loser row;
2390
+ // stamp mode (adoption) writes ownership only — the row is an orphan by
2391
+ // scope, so a plain stamp cannot re-parent anything.
2168
2392
  await tx
2169
2393
  .update(emailPreferences)
2170
- .set({ userId: survivorKey, updatedAt: new Date() })
2394
+ .set(
2395
+ opts?.stampOnly
2396
+ ? { contactId: survivorId, updatedAt: new Date() }
2397
+ : {
2398
+ userId: survivorKey,
2399
+ contactId: adoptedContactId(
2400
+ emailPreferences.contactId,
2401
+ survivorId,
2402
+ ),
2403
+ updatedAt: new Date(),
2404
+ },
2405
+ )
2171
2406
  .where(eq(emailPreferences.id, lp.id));
2172
2407
  continue;
2173
2408
  }
@@ -2193,6 +2428,7 @@ async function foldEmailPreferences(
2193
2428
  categories: foldedCategories,
2194
2429
  suppressedAt: earliest(target.suppressedAt, lp.suppressedAt),
2195
2430
  lastBounceAt: earliest(target.lastBounceAt, lp.lastBounceAt),
2431
+ contactId: target.contactId ?? survivorId,
2196
2432
  updatedAt: new Date(),
2197
2433
  })
2198
2434
  .where(eq(emailPreferences.id, target.id));
@@ -2204,48 +2440,68 @@ async function foldEmailPreferences(
2204
2440
  }
2205
2441
 
2206
2442
  /**
2207
- * Re-point a contact's OWN string-keyed history when its canonical key flips
2208
- * (risk 1). resolvedKey downstream is `external_id ?? anonymous_id ?? id`, so
2209
- * when fill-in-link attaches an external_id to a previously email-only/anon
2210
- * contact (or merge promotes the survivor's external_id), the contact's
2211
- * canonical key changes and its existing user_events/journey_states/email_sends/
2212
- * bucket_memberships/email_preferences rows (keyed on the OLD key) would be
2213
- * silently orphaned. Rewrite them from `oldKey` to `newKey`, applying the SAME
2214
- * active/terminal dedupe as the merge fold so the rewrite can't violate
2215
- * uq_user_journey_active / uq_user_bucket_active / uq(user_id,email).
2443
+ * Adopt ORPHAN history sitting under `fromKey` onto its owning contact — the
2444
+ * D4 statement, made permanent by PRD 05 T9: `UPDATE <t> SET contact_id = :id
2445
+ * WHERE user_id = :fromKey AND contact_id IS NULL`, on five tables. One
2446
+ * column. The string key is never rewritten again; reads follow `contact_id`
2447
+ * (T4-T8), and D9 keeps writing `user_id` on every insert, so history stays a
2448
+ * frozen record of the key it happened under.
2216
2449
  *
2217
- * No-op when oldKey === newKey (the canonical key did not change).
2450
+ * This replaced two functions:
2451
+ * - `repointOwnHistory`, which rewrote `user_id` from the old canonical key
2452
+ * to the new one on a key flip (with three dedupe folds against the
2453
+ * string-scoped unique indexes), and
2454
+ * - `adoptOwnKeyHistory`, which stamped history already sitting under a
2455
+ * brand-new contact's OWN key.
2456
+ * The "did the canonical key flip" test that distinguished them is meaningless
2457
+ * when nothing is keyed on the canonical key, so both call shapes are one
2458
+ * function now: name the key, stamp its orphans.
2459
+ *
2460
+ * The `contact_id IS NULL` predicate is the anti-theft guard (D6): a row
2461
+ * another contact owns already carries that owner's id and can never match.
2462
+ * `keysAnotherContact` no longer gates adoption — it survives only to gate
2463
+ * the `mergedKeys` REPORT (the analytics stitch must never alias a foreign
2464
+ * key) and the attach path inside `claimIdentityKey`.
2465
+ *
2466
+ * The folds still run, re-expressed on the contact scope (D4/D5): the
2467
+ * adopting contact may ALREADY own a live row for the same journey/bucket (or
2468
+ * a pref row for the same address), and a bare stamp would mint a second row
2469
+ * satisfying `uq_contact_journey_active` / `uq_contact_bucket_active` /
2470
+ * `email_preferences_contact_email_idx` — a 23505 inside the resolve
2471
+ * transaction. Same routing as the merge fold (exit / soft-leave / dedupe /
2472
+ * pref-fold), stamp-only writes.
2473
+ *
2474
+ * Idempotent: a second run finds `contact_id` already set and matches nothing.
2218
2475
  */
2219
- async function repointOwnHistory(
2476
+ async function adoptOrphanHistory(
2220
2477
  tx: Tx,
2221
- oldKey: string,
2222
- newKey: string,
2478
+ fromKey: string,
2223
2479
  row: ContactRow,
2224
2480
  ): Promise<void> {
2225
- if (oldKey === newKey) return;
2481
+ const ownKey = contactKey(row);
2226
2482
 
2227
- // user_events: no unique constraint on user_id — blind rewrite.
2483
+ // user_events + email_sends: no unique constraint involved — plain stamps.
2228
2484
  await tx
2229
2485
  .update(userEvents)
2230
- .set({ userId: newKey })
2231
- .where(eq(userEvents.userId, oldKey));
2232
-
2233
- // journey_states + bucket_memberships: dedupe against the survivor/new key's
2234
- // existing rows (the folds already handle the collision logic).
2235
- await foldJourneyStates(tx, newKey, [oldKey], row);
2236
- await foldBucketMemberships(tx, newKey, [oldKey]);
2237
-
2238
- // email_sends: no unique constraint on user_id — blind rewrite.
2486
+ .set({ contactId: row.id })
2487
+ .where(and(eq(userEvents.userId, fromKey), isNull(userEvents.contactId)));
2239
2488
  await tx
2240
2489
  .update(emailSends)
2241
- .set({
2242
- userId: newKey,
2243
- ...(row.email ? { userEmail: row.email } : {}),
2244
- })
2245
- .where(eq(emailSends.userId, oldKey));
2246
-
2247
- // email_preferences: FOLD into the new key's rows (uq(user_id, email)).
2248
- await foldEmailPreferences(tx, [oldKey], newKey);
2490
+ .set({ contactId: row.id })
2491
+ .where(and(eq(emailSends.userId, fromKey), isNull(emailSends.contactId)));
2492
+
2493
+ // journey_states / bucket_memberships / email_preferences: stamp through
2494
+ // the folds so a collision with a row the contact already owns is routed
2495
+ // (exited / left / folded) instead of violating the contact-scoped index.
2496
+ await foldJourneyStates(tx, ownKey, [fromKey], row, undefined, {
2497
+ stampOnly: true,
2498
+ });
2499
+ await foldBucketMemberships(tx, ownKey, [fromKey], row.id, undefined, {
2500
+ stampOnly: true,
2501
+ });
2502
+ await foldEmailPreferences(tx, [fromKey], ownKey, row.id, undefined, {
2503
+ stampOnly: true,
2504
+ });
2249
2505
  }
2250
2506
 
2251
2507
  /** RECORD a contact_aliases row per loser key → survivor (reason 'merge'). */