@hogsend/engine 0.59.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +34 -11
  3. package/src/buckets/check-membership.ts +74 -10
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +10 -0
  6. package/src/index.ts +10 -6
  7. package/src/journeys/execute-journey-run.ts +139 -70
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +35 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +17 -17
  13. package/src/lib/contacts.ts +541 -175
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +14 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +78 -29
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/refine.ts +16 -10
  23. package/src/lib/revenue.ts +20 -5
  24. package/src/lib/sms-inbound.ts +6 -1
  25. package/src/lib/sms-tracked.ts +18 -1
  26. package/src/lib/timezone.ts +11 -1
  27. package/src/lib/tracked.ts +19 -8
  28. package/src/lib/tracking-events.ts +28 -0
  29. package/src/lib/unique-violation.ts +44 -0
  30. package/src/routes/admin/buckets.ts +21 -1
  31. package/src/routes/admin/bulk.ts +14 -2
  32. package/src/routes/admin/contacts.ts +20 -7
  33. package/src/routes/admin/conversions.ts +20 -4
  34. package/src/routes/admin/emails.ts +17 -2
  35. package/src/routes/admin/events.ts +32 -41
  36. package/src/routes/admin/funnels.ts +6 -1
  37. package/src/routes/admin/groups.ts +4 -0
  38. package/src/routes/admin/impact.ts +30 -10
  39. package/src/routes/admin/journey-impact.ts +18 -9
  40. package/src/routes/admin/journeys.ts +8 -1
  41. package/src/routes/admin/maintenance.ts +7 -1
  42. package/src/routes/admin/preferences.ts +85 -46
  43. package/src/routes/admin/reporting.ts +25 -5
  44. package/src/routes/admin/targeting.ts +1 -0
  45. package/src/routes/admin/timeline.ts +16 -10
  46. package/src/routes/email/preferences.ts +18 -1
  47. package/src/routes/feed/recipient.ts +5 -0
  48. package/src/routes/lists/index.ts +6 -6
  49. package/src/workflows/backfill-contact-id.ts +210 -30
  50. package/src/workflows/bucket-backfill.ts +145 -67
  51. package/src/workflows/bucket-reconcile.ts +164 -51
  52. package/src/workflows/check-alerts.ts +4 -0
  53. package/src/workflows/impact-digest.ts +10 -3
  54. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  55. package/src/workflows/send-campaign.ts +18 -11
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hogsend/engine",
3
- "version": "0.59.0",
3
+ "version": "0.61.0",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -53,18 +53,18 @@
53
53
  "svix": "^1.95.1",
54
54
  "winston": "^3.19.0",
55
55
  "zod": "^4.4.3",
56
- "@hogsend/attribution": "^0.59.0",
57
- "@hogsend/core": "^0.59.0",
58
- "@hogsend/db": "^0.59.0",
59
- "@hogsend/email": "^0.59.0",
60
- "@hogsend/plugin-posthog": "^0.59.0",
61
- "@hogsend/plugin-resend": "^0.59.0",
62
- "@hogsend/sms": "^0.59.0"
56
+ "@hogsend/attribution": "^0.61.0",
57
+ "@hogsend/core": "^0.61.0",
58
+ "@hogsend/db": "^0.61.0",
59
+ "@hogsend/email": "^0.61.0",
60
+ "@hogsend/plugin-posthog": "^0.61.0",
61
+ "@hogsend/plugin-resend": "^0.61.0",
62
+ "@hogsend/sms": "^0.61.0"
63
63
  },
64
64
  "optionalDependencies": {
65
- "@hogsend/plugin-apollo": "^0.59.0",
66
- "@hogsend/plugin-postmark": "^0.59.0",
67
- "@hogsend/plugin-twilio": "^0.59.0"
65
+ "@hogsend/plugin-apollo": "^0.61.0",
66
+ "@hogsend/plugin-postmark": "^0.61.0",
67
+ "@hogsend/plugin-twilio": "^0.61.0"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/node": "^22.15.3",
@@ -1,6 +1,6 @@
1
1
  import { bucketMemberships, contacts, type Database } from "@hogsend/db";
2
2
  import { and, count as countFn, eq, gt, isNull } from "drizzle-orm";
3
- import { contactKeySql } from "../lib/contacts.js";
3
+ import { lookupContactIdByKey } from "../lib/contacts.js";
4
4
  import { getDb } from "../lib/db.js";
5
5
 
6
6
  /**
@@ -57,10 +57,14 @@ function toError(err: unknown): Error {
57
57
  * container re-binds the accessors with a `dbResolver` that returns its own
58
58
  * `db`; the `getDb()` default bypasses the container.
59
59
  *
60
- * Every query `innerJoin`s `contacts` on `externalId` and filters
61
- * `isNull(deletedAt)` on both tables — GDPR parity with every reconcile/admin
62
- * query. No method throws (except the iterator on a page error); failures are
63
- * carried in the result's `error`.
60
+ * Every query `innerJoin`s `contacts` on the membership's OWNING CONTACT
61
+ * (`bucket_memberships.contact_id`) and filters `isNull(deletedAt)` on both
62
+ * tables — GDPR parity with every reconcile/admin query. The join used to run
63
+ * on the mutable text key (`coalesce(external_id, anonymous_id, id) =
64
+ * user_id`); it now runs on the FK, so a membership adopted onto a contact
65
+ * under a since-stale key stays visible instead of dropping out of every count
66
+ * and page. No method throws (except the iterator on a page error); failures
67
+ * are carried in the result's `error`.
64
68
  */
65
69
  export function createBucketAccessor(
66
70
  bucketId: string,
@@ -75,7 +79,7 @@ export function createBucketAccessor(
75
79
  const rows = await db
76
80
  .select({ value: countFn() })
77
81
  .from(bucketMemberships)
78
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
82
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
79
83
  .where(
80
84
  and(
81
85
  eq(bucketMemberships.bucketId, bucketId),
@@ -95,15 +99,34 @@ export function createBucketAccessor(
95
99
  ): Promise<{ data: boolean; error: Error | null }> {
96
100
  try {
97
101
  const db = dbResolver();
98
- // O(1) probe on the partial active unique index (uq_user_bucket_active).
102
+ // The subject predicate moves to the CONTACT side: `userId` is the caller's
103
+ // canonical key, so we name the contact it resolves to and let the FK join
104
+ // find that contact's membership. Probing `bucket_memberships.user_id`
105
+ // instead answers "false" for a member whose row was adopted under an
106
+ // anon-era key — the person is in the bucket, `has()` says they are not,
107
+ // and every gate built on it silently opens.
108
+ //
109
+ // PRD 07 T7 — the PRESENTED key goes through the alias-aware primitive
110
+ // rather than a `coalesce(external_id, anonymous_id, id) = :key` probe.
111
+ // That probe is blind to a merged loser's STALE key (its row is
112
+ // soft-deleted), so the survivor's membership read "false" for a key the
113
+ // survivor still owns through the identity table.
114
+ //
115
+ // The no-resolve arm is UNCHANGED: on live contacts a coalesce match
116
+ // implies one of the three identity columns equals the key, so the
117
+ // primitive's column leg is strictly wider — a `null` here is a key the
118
+ // old probe missed too, and the honest answer stays `false`.
119
+ const resolvedId = await lookupContactIdByKey(db, userId);
120
+ if (!resolvedId) return { data: false, error: null };
121
+
99
122
  const rows = await db
100
123
  .select({ id: bucketMemberships.id })
101
124
  .from(bucketMemberships)
102
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
125
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
103
126
  .where(
104
127
  and(
105
128
  eq(bucketMemberships.bucketId, bucketId),
106
- eq(bucketMemberships.userId, userId),
129
+ eq(contacts.id, resolvedId),
107
130
  eq(bucketMemberships.status, "active"),
108
131
  isNull(bucketMemberships.deletedAt),
109
132
  isNull(contacts.deletedAt),
@@ -145,7 +168,7 @@ export function createBucketAccessor(
145
168
  entryCount: bucketMemberships.entryCount,
146
169
  })
147
170
  .from(bucketMemberships)
148
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
171
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
149
172
  .where(and(...conditions))
150
173
  .orderBy(bucketMemberships.id)
151
174
  // +1 peek to detect a next page.
@@ -153,7 +176,7 @@ export function createBucketAccessor(
153
176
  db
154
177
  .select({ value: countFn() })
155
178
  .from(bucketMemberships)
156
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
179
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
157
180
  .where(
158
181
  and(
159
182
  eq(bucketMemberships.bucketId, bucketId),
@@ -1,6 +1,7 @@
1
1
  import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
2
2
  import {
3
3
  type BucketMeta,
4
+ bySubject,
4
5
  collectPropertyNames,
5
6
  durationToMs,
6
7
  evaluateCondition,
@@ -9,7 +10,11 @@ import type { JourneyRegistry } from "@hogsend/core/registry";
9
10
  import { bucketMemberships, contacts, type Database } from "@hogsend/db";
10
11
  import { and, desc, eq, isNotNull, isNull } from "drizzle-orm";
11
12
  import { emitBucketTransition } from "../lib/bucket-emit.js";
12
- import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
13
+ import {
14
+ contactKeySql,
15
+ lookupContactIdByKey,
16
+ normalizeEmailOrNull,
17
+ } from "../lib/contacts.js";
13
18
  import type { Logger } from "../lib/logger.js";
14
19
  import {
15
20
  BUCKET_EVENT_PREFIX,
@@ -201,13 +206,26 @@ export async function checkBucketMembership(opts: {
201
206
  // the GDPR guard below — "never (re-)evaluate or emit for a
202
207
  // soft-deleted contact" — did not fire for them. A soft-deleted
203
208
  // email-only contact could still transition buckets and emit.
209
+ //
210
+ // PRD 07 T7 — the presented key resolves through the alias-aware primitive
211
+ // FIRST, so a merged-away key reads the SURVIVOR's properties instead of
212
+ // the nothing-found arm the coalesce probe returned for it.
213
+ //
214
+ // The coalesce probe SURVIVES as the fallback, and it is load-bearing, not
215
+ // belt-and-braces: `lookupContactIdByKey` filters `deleted_at IS NULL`, so
216
+ // it returns null for exactly the soft-deleted contact that failure (2)
217
+ // above is about. Stopping at the null would leave `contactDeleted` false
218
+ // and silently re-open the GDPR guard for the rows it exists to protect.
219
+ const resolvedId = await lookupContactIdByKey(db, userId);
204
220
  const [contact] = await db
205
221
  .select({
206
222
  properties: contacts.properties,
207
223
  deletedAt: contacts.deletedAt,
208
224
  })
209
225
  .from(contacts)
210
- .where(eq(contactKeySql(), userId))
226
+ .where(
227
+ resolvedId ? eq(contacts.id, resolvedId) : eq(contactKeySql(), userId),
228
+ )
211
229
  .limit(1);
212
230
  if (contact) {
213
231
  storedContactProps =
@@ -235,10 +253,17 @@ export async function checkBucketMembership(opts: {
235
253
  if (!bucket.criteria) continue;
236
254
 
237
255
  // wasMember — current active, non-deleted membership row (cheap pre-filter;
238
- // the authoritative guard is the RETURNING-gated mutation below).
256
+ // the authoritative guard is the RETURNING-gated mutation below). Read by
257
+ // SUBJECT: a membership adopted onto this contact under a since-stale key
258
+ // is invisible to a `user_id` probe, and missing it re-runs the JOIN path,
259
+ // which then loses the arbiter-less race and no-ops — a member who silently
260
+ // never re-emits and never leaves.
239
261
  const active = await db.query.bucketMemberships.findFirst({
240
262
  where: and(
241
- eq(bucketMemberships.userId, userId),
263
+ bySubject(bucketMemberships, {
264
+ contactId: contactId ?? null,
265
+ userKey: userId,
266
+ }),
242
267
  eq(bucketMemberships.bucketId, bucket.id),
243
268
  eq(bucketMemberships.status, "active"),
244
269
  isNull(bucketMemberships.deletedAt),
@@ -252,7 +277,16 @@ export async function checkBucketMembership(opts: {
252
277
  // sub-conditions read the merged journeyContext.
253
278
  const isMember = await evaluateCondition({
254
279
  condition: bucket.criteria,
255
- ctx: { db, userId, journeyContext },
280
+ ctx: {
281
+ db,
282
+ userId,
283
+ // `ingestEvent` resolved the subject before calling us, so the event /
284
+ // count legs read this person's whole history — including the rows
285
+ // adopted from an anon-era key — instead of only what the current
286
+ // string key happens to name.
287
+ contactId: contactId ?? null,
288
+ journeyContext,
289
+ },
256
290
  });
257
291
 
258
292
  if (!wasMember && isMember) {
@@ -324,13 +358,27 @@ async function handleJoin(opts: {
324
358
  // this (user, bucket) (Section 6.3 / 8.2). priorCount also drives the entryLimit
325
359
  // gate. Shared with the reconcile-discovered join path so the ordinal can
326
360
  // never drift between the two writers.
327
- const priorCount = await countPriorMemberships(db, bucket.id, userId);
361
+ const priorCount = await countPriorMemberships(
362
+ db,
363
+ bucket.id,
364
+ userId,
365
+ contactId ?? null,
366
+ );
328
367
  const epoch = priorCount + 1;
329
368
 
330
- // INSERT a FRESH active row. ON CONFLICT DO NOTHING targets the partial active
369
+ // INSERT a FRESH active row. ON CONFLICT DO NOTHING covers the partial active
331
370
  // unique index (uq_user_bucket_active): a concurrent emitter that already
332
371
  // inserted the active row makes THIS insert return zero rows → we do NOT emit
333
372
  // (the loser mutates nothing — Section 6.3 governing rule).
373
+ //
374
+ // PRD 05 T3 — the DO NOTHING is deliberately ARBITER-LESS. Two partial unique
375
+ // indexes now guard this table (uq_user_bucket_active and its contact-scoped
376
+ // twin uq_contact_bucket_active), Postgres allows only ONE conflict target per
377
+ // statement, and both must land in the same "already a member" branch: the
378
+ // contact one is what catches an adopted row whose `user_id` still differs.
379
+ // Narrowing this to a column target would let that collision escape as a raw
380
+ // 23505 — and a target naming `contact_id` would additionally miss every
381
+ // contactless (NULL) member.
334
382
  const expiresAt = computeExpiresAt(bucket);
335
383
  // Unconditional TTL deadline — set once on join, swept by the reconcile cron.
336
384
  const maxDwellAt = computeMaxDwellAt(bucket);
@@ -386,7 +434,15 @@ async function handleJoin(opts: {
386
434
  // The active row is always written (Studio size must reflect reality) and the
387
435
  // epoch always advances via the real insert; only the bucket:entered emission
388
436
  // is gated by the entryLimit policy (Section 6.3).
389
- if (await shouldEmitJoin({ db, bucket, userId, priorCount })) {
437
+ if (
438
+ await shouldEmitJoin({
439
+ db,
440
+ bucket,
441
+ userId,
442
+ contactId: contactId ?? null,
443
+ priorCount,
444
+ })
445
+ ) {
390
446
  await emitBucketTransition({
391
447
  db,
392
448
  registry,
@@ -532,9 +588,17 @@ export async function shouldEmitJoin(opts: {
532
588
  db: Database;
533
589
  bucket: BucketMeta;
534
590
  userId: string;
591
+ /**
592
+ * The owning contact's `contacts.id`, or `null` when the caller has none.
593
+ * REQUIRED (not optional) on purpose: the prior-cycle lookup below is the
594
+ * cooldown gate, and an optional field a caller forgets silently drops it
595
+ * back onto the mutable text key — which misses the adopted prior cycle and
596
+ * re-emits a `once` join. Pass `null` explicitly for a contactless subject.
597
+ */
598
+ contactId: string | null;
535
599
  priorCount: number;
536
600
  }): Promise<boolean> {
537
- const { db, bucket, userId, priorCount } = opts;
601
+ const { db, bucket, userId, contactId, priorCount } = opts;
538
602
  // First-ever join always emits.
539
603
  if (priorCount === 0) return true;
540
604
  switch (bucket.entryLimit ?? "unlimited") {
@@ -556,7 +620,7 @@ export async function shouldEmitJoin(opts: {
556
620
  .from(bucketMemberships)
557
621
  .where(
558
622
  and(
559
- eq(bucketMemberships.userId, userId),
623
+ bySubject(bucketMemberships, { contactId, userKey: userId }),
560
624
  eq(bucketMemberships.bucketId, bucket.id),
561
625
  eq(bucketMemberships.status, "left"),
562
626
  isNotNull(bucketMemberships.leftAt),
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  type BucketMeta,
3
+ bySubject,
3
4
  type ConditionEval,
4
5
  type DurationObject,
5
6
  durationToMs,
@@ -25,18 +26,27 @@ export const BUCKET_EVENT_PREFIX = "bucket:";
25
26
  *
26
27
  * The predicate MUST stay status-agnostic — narrowing it (e.g. to active-only)
27
28
  * would corrupt entryCount and the entryLimit cooldown.
29
+ *
30
+ * `contactId` is REQUIRED (not optional) for the same reason
31
+ * `checkEntryLimit`'s is: an optional field silently defaults a new caller onto
32
+ * the mutable text key, and here that under-counts the epoch — a person whose
33
+ * anon-era memberships were adopted would re-enter at ordinal 1 and slip the
34
+ * `once`/`once_per_period` gate. Pass `null` explicitly for a subject with no
35
+ * contact; that is a permanent, supported state (memberships are text-keyed
36
+ * with no contact FK).
28
37
  */
29
38
  export async function countPriorMemberships(
30
39
  db: Database,
31
40
  bucketId: string,
32
41
  userId: string,
42
+ contactId: string | null,
33
43
  ): Promise<number> {
34
44
  const [counted] = await db
35
45
  .select({ priorCount: sql<number>`count(*)::int` })
36
46
  .from(bucketMemberships)
37
47
  .where(
38
48
  and(
39
- eq(bucketMemberships.userId, userId),
49
+ bySubject(bucketMemberships, { contactId, userKey: userId }),
40
50
  eq(bucketMemberships.bucketId, bucketId),
41
51
  ),
42
52
  );
@@ -62,6 +62,10 @@ const CONNECTOR_IDENTITY_COLUMNS: Record<string, PgColumn | undefined> = {
62
62
  * members. Failure direction here is under-delivery, never over-delivery.
63
63
  */
64
64
  export function cohortSuppressionSql(): SQL {
65
+ // PRD 07 T7/D9 — the erasure leg is NOT flipped onto the identity table:
66
+ // `campaign_recipients.user_id` is a SEND-TIME ECHO of the key the anchor
67
+ // wrote (the table has no `contact_id`), so joining it back to `contacts` is
68
+ // deliberate. Re-keying the recipient table is out of PRD 07's scope.
65
69
  return sql`not exists (
66
70
  select 1 from ${emailPreferences}
67
71
  where lower(${emailPreferences.email}) = ${campaignRecipients.email}
@@ -161,6 +165,9 @@ function eventSql(condition: EventCondition, startedAt: Date): SQL {
161
165
  Date.now() - durationToMs(condition.within),
162
166
  ).toISOString()}::timestamptz`
163
167
  : sql.empty();
168
+ // PRD 05 T8 — this correlation STAYS string-keyed: `campaign_recipients`
169
+ // has no `contact_id` column, so the text key is the only subject available
170
+ // on the outer side of the join.
164
171
  const positive = sql`exists (
165
172
  select 1 from ${userEvents}
166
173
  where ${userEvents.userId} = ${campaignRecipients.userId}
@@ -186,6 +193,9 @@ function channelIdentitySql(condition: ChannelIdentityCondition): SQL {
186
193
  `Campaign wave \`where\`: channel_identity connector "${condition.connector}" has no linked-identity source in v1 — only "discord" is supported.`,
187
194
  );
188
195
  }
196
+ // PRD 07 T7/D9 — same stance as `cohortSuppressionSql`: the recipient
197
+ // `user_id` is a send-time echo key, not a live identity key, so this join
198
+ // stays string-matched; re-keying `campaign_recipients` is out of scope.
189
199
  const positive = sql`exists (
190
200
  select 1 from ${contacts}
191
201
  where (${contacts.externalId} = ${campaignRecipients.userId}
package/src/index.ts CHANGED
@@ -37,14 +37,18 @@ export type {
37
37
  // types) so content can import everything from `@hogsend/engine`.
38
38
  export * from "@hogsend/core";
39
39
  // --- Capability-provider contracts (canonical origin: @hogsend/core) ---
40
- // Email provider contract + analytics contract, re-exported so consumers can
41
- // import them from `@hogsend/engine`. (`SendEmailOptions` is intentionally
42
- // omitted here: the engine's public `SendEmailOptions` is the high-level
43
- // journey-facing send options from `./lib/email.js`; the provider-contract
44
- // `SendEmailOptions` remains available via `@hogsend/core`.)
45
- export {
40
+ // Email provider contract + analytics contract, plus `bySubject` (the
41
+ // either/or history scope every engine-side history read is expected to move
42
+ // onto), re-exported so consumers can import them from `@hogsend/engine`.
43
+ // (`SendEmailOptions` is intentionally omitted here: the engine's public
44
+ // `SendEmailOptions` is the high-level journey-facing send options from
45
+ // `./lib/email.js`; the provider-contract `SendEmailOptions` remains available
46
+ // via `@hogsend/core`.)
47
+ export {
48
+ bySubject,
46
49
  defineAnalyticsProvider,
47
50
  defineEmailProvider,
51
+ type Subject,
48
52
  WebhookHandshakeSignal,
49
53
  } from "@hogsend/core";
50
54
  export {