@hogsend/engine 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +19 -10
  3. package/src/buckets/check-membership.ts +55 -8
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +3 -0
  6. package/src/index.ts +10 -6
  7. package/src/journeys/execute-journey-run.ts +119 -50
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +27 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +4 -1
  13. package/src/lib/contacts.ts +408 -152
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +14 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +78 -29
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/revenue.ts +20 -5
  23. package/src/lib/sms-tracked.ts +18 -1
  24. package/src/lib/tracked.ts +19 -8
  25. package/src/lib/tracking-events.ts +28 -0
  26. package/src/lib/unique-violation.ts +44 -0
  27. package/src/routes/admin/buckets.ts +20 -2
  28. package/src/routes/admin/bulk.ts +14 -2
  29. package/src/routes/admin/contacts.ts +4 -3
  30. package/src/routes/admin/conversions.ts +20 -4
  31. package/src/routes/admin/emails.ts +17 -2
  32. package/src/routes/admin/events.ts +32 -41
  33. package/src/routes/admin/funnels.ts +6 -1
  34. package/src/routes/admin/groups.ts +4 -0
  35. package/src/routes/admin/impact.ts +30 -10
  36. package/src/routes/admin/journey-impact.ts +18 -9
  37. package/src/routes/admin/journeys.ts +8 -1
  38. package/src/routes/admin/maintenance.ts +7 -1
  39. package/src/routes/admin/preferences.ts +85 -46
  40. package/src/routes/admin/reporting.ts +25 -5
  41. package/src/routes/admin/targeting.ts +1 -0
  42. package/src/routes/admin/timeline.ts +16 -10
  43. package/src/routes/email/preferences.ts +18 -1
  44. package/src/routes/lists/index.ts +6 -6
  45. package/src/workflows/backfill-contact-id.ts +210 -30
  46. package/src/workflows/bucket-backfill.ts +145 -67
  47. package/src/workflows/bucket-reconcile.ts +150 -50
  48. package/src/workflows/check-alerts.ts +4 -0
  49. package/src/workflows/impact-digest.ts +10 -3
  50. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  51. package/src/workflows/send-campaign.ts +18 -11
@@ -15,7 +15,17 @@ import {
15
15
  importJobs,
16
16
  userEvents,
17
17
  } from "@hogsend/db";
18
- import { and, eq, gt, gte, inArray, isNull, max, sql } from "drizzle-orm";
18
+ import {
19
+ and,
20
+ eq,
21
+ gt,
22
+ gte,
23
+ inArray,
24
+ isNotNull,
25
+ isNull,
26
+ max,
27
+ sql,
28
+ } from "drizzle-orm";
19
29
  import {
20
30
  computeExpiresAt,
21
31
  computeMaxDwellAt,
@@ -33,6 +43,23 @@ import { stableStringify } from "../lib/stable-stringify.js";
33
43
  /** Insert chunk size, reusing the import-contacts precedent (Section 6.6). */
34
44
  const BATCH_SIZE = 500;
35
45
 
46
+ /**
47
+ * One matcher this backfill will materialize as a membership.
48
+ *
49
+ * The set-based selectors used to return a bare `string[]` of canonical keys,
50
+ * and every downstream lookup (prior-membership counts, dwell anchors, the
51
+ * email/contact-id fill-in) re-keyed off that string. Carrying the resolved
52
+ * `contactId` instead makes the SUBJECT the join key end to end: the counts and
53
+ * anchors below read the whole person rather than whatever string a row happens
54
+ * to name today. `userKey` still rides along because
55
+ * `bucket_memberships.user_id` is still written (PRD 05 D9).
56
+ */
57
+ interface BackfillMatcher {
58
+ contactId: string;
59
+ userKey: string;
60
+ email: string | null;
61
+ }
62
+
36
63
  /** import_jobs.format discriminator for the reused status record (Section 6.6). */
37
64
  export const FIRST_TIME_FORMAT = "bucket-backfill";
38
65
  const REEVAL_FORMAT = "bucket-reeval";
@@ -181,14 +208,14 @@ async function backfillJoins(opts: {
181
208
  const { db, bucket, jobId } = opts;
182
209
  const criteria = bucket.criteria as ConditionEval;
183
210
 
184
- const matcherIds =
211
+ const matchers =
185
212
  criteria.type === "event"
186
213
  ? await selectEventMatchers(db, criteria)
187
214
  : await selectCompositeMatchers(db, criteria);
188
215
 
189
216
  await db
190
217
  .update(importJobs)
191
- .set({ totalRows: matcherIds.length, updatedAt: new Date() })
218
+ .set({ totalRows: matchers.length, updatedAt: new Date() })
192
219
  .where(eq(importJobs.id, jobId));
193
220
 
194
221
  // Unconditional max-dwell TTL deadline, stamped once at insert (mirrors the
@@ -216,30 +243,17 @@ async function backfillJoins(opts: {
216
243
  // omission: arming at backfill would fan out one durable task per inserted row.
217
244
 
218
245
  let inserted = 0;
219
- for (let i = 0; i < matcherIds.length; i += BATCH_SIZE) {
220
- const chunk = matcherIds.slice(i, i + BATCH_SIZE);
221
-
222
- // userEmail backfilled from the contacts row where available. The chunk
223
- // holds the RESOLVED key (coalesce(external_id, anonymous_id, id)) — for an
224
- // email-only / anonymous contact that is the anonymous_id or the uuid id, NOT
225
- // the (null) external_id. Looking up by `contacts.externalId` would miss
226
- // those rows and write a NULL userEmail despite the contact having an email,
227
- // so we key the lookup + the map by the SAME coalesce expression the chunk
228
- // carries (matches reconcileBucketJoins, which reads userId + email off one
229
- // contacts row).
230
- const resolvedKey = contactKeySql();
231
- const chunkContacts = await db
232
- .select({ userKey: resolvedKey, email: contacts.email, id: contacts.id })
233
- .from(contacts)
234
- .where(and(inArray(resolvedKey, chunk), isNull(contacts.deletedAt)));
235
- const emailByUser = new Map(chunkContacts.map((c) => [c.userKey, c.email]));
236
- // PRD 04 dual-write: the SAME already-issued per-chunk read carries the row
237
- // id, so stamping `contact_id` costs one extra selected column and zero
238
- // extra queries. A chunk key with no live contacts row (a membership keyed
239
- // on a since-deleted / never-minted subject) simply stamps NULL.
240
- const contactIdByUser = new Map(
241
- chunkContacts.map((c) => [c.userKey, c.id]),
242
- );
246
+ for (let i = 0; i < matchers.length; i += BATCH_SIZE) {
247
+ const chunk = matchers.slice(i, i + BATCH_SIZE);
248
+ // Every chunked lookup below is keyed on the OWNING CONTACT, so the chunk
249
+ // boundary and each aggregate's GROUP BY name the same subject. Keying them
250
+ // on the canonical string while the selectors resolve contacts is exactly
251
+ // the drift that makes a set-based job miss silently: an adopted row would
252
+ // land in no bucket of the map and read back as "no prior membership".
253
+ const chunkContactIds = chunk.map((m) => m.contactId);
254
+ // The selectors already read email + contact id off the same `contacts` row
255
+ // they matched on, so the per-chunk contacts lookup this loop used to issue
256
+ // is gone (one fewer query per chunk, and no second chance to key it wrong).
243
257
 
244
258
  // Fix A: entryCount = 1 + prior memberships for each (user, bucket), the
245
259
  // same monotonic ordinal the live join computes (check-membership.ts). On a
@@ -249,19 +263,23 @@ async function backfillJoins(opts: {
249
263
  // based path must not reintroduce the O(P) serial-query trap).
250
264
  const priorCounts = await db
251
265
  .select({
252
- userId: bucketMemberships.userId,
266
+ contactId: bucketMemberships.contactId,
253
267
  cnt: sql<number>`count(*)::int`,
254
268
  })
255
269
  .from(bucketMemberships)
256
270
  .where(
257
271
  and(
258
272
  eq(bucketMemberships.bucketId, bucket.id),
259
- inArray(bucketMemberships.userId, chunk),
273
+ inArray(bucketMemberships.contactId, chunkContactIds),
260
274
  ),
261
275
  )
262
- .groupBy(bucketMemberships.userId);
263
- const priorByUser = new Map(
264
- priorCounts.map((r) => [r.userId, Number(r.cnt)]),
276
+ .groupBy(bucketMemberships.contactId);
277
+ const priorByContact = new Map(
278
+ priorCounts
279
+ .filter(
280
+ (r): r is { contactId: string; cnt: number } => r.contactId != null,
281
+ )
282
+ .map((r) => [r.contactId, Number(r.cnt)]),
265
283
  );
266
284
 
267
285
  // Batched dwell-anchor derivation (LOCKED DECISION 1): one GROUP BY
@@ -269,50 +287,57 @@ async function backfillJoins(opts: {
269
287
  // priorCounts GROUP BY above (never per-user serial queries). Only computed
270
288
  // when the criteria shape exposes a cheap per-matcher anchor event; an empty
271
289
  // map leaves dwellAnchorAt NULL → the dwell gate falls back to enteredAt.
272
- let anchorByUser = new Map<string, Date>();
290
+ let anchorByContact = new Map<string, Date>();
273
291
  if (anchorEvent != null) {
274
292
  const anchors = await db
275
293
  .select({
276
- userId: userEvents.userId,
294
+ contactId: userEvents.contactId,
277
295
  lastAt: max(userEvents.occurredAt),
278
296
  })
279
297
  .from(userEvents)
280
298
  .where(
281
299
  and(
282
300
  eq(userEvents.event, anchorEvent),
283
- inArray(userEvents.userId, chunk),
301
+ inArray(userEvents.contactId, chunkContactIds),
284
302
  ),
285
303
  )
286
- .groupBy(userEvents.userId);
287
- anchorByUser = new Map(
304
+ .groupBy(userEvents.contactId);
305
+ anchorByContact = new Map(
288
306
  anchors
289
307
  .filter(
290
- (r): r is { userId: string; lastAt: Date } => r.lastAt != null,
308
+ (r): r is { contactId: string; lastAt: Date } =>
309
+ r.contactId != null && r.lastAt != null,
291
310
  )
292
- .map((r) => [r.userId, r.lastAt]),
311
+ .map((r) => [r.contactId, r.lastAt]),
293
312
  );
294
313
  }
295
314
 
296
- const rows = chunk.map((userId) => ({
297
- userId,
315
+ const rows = chunk.map((matcher) => ({
316
+ userId: matcher.userKey,
298
317
  // Normalized at the write site (audience-model.md wart #1) — belt and
299
318
  // braces on top of the contacts row already being normalized.
300
- userEmail: normalizeEmailOrNull(emailByUser.get(userId)),
319
+ userEmail: normalizeEmailOrNull(matcher.email),
301
320
  bucketId: bucket.id,
302
321
  status: "active" as const,
303
322
  source: "backfill" as const,
304
- entryCount: 1 + (priorByUser.get(userId) ?? 0),
323
+ entryCount: 1 + (priorByContact.get(matcher.contactId) ?? 0),
305
324
  expiresAt: computeExpiresAt(bucket),
306
325
  maxDwellAt,
307
326
  // Historical dwell anchor where derivable; NULL otherwise (→ enteredAt).
308
- dwellAnchorAt: anchorByUser.get(userId) ?? null,
327
+ dwellAnchorAt: anchorByContact.get(matcher.contactId) ?? null,
309
328
  lastEvaluatedAt: new Date(),
310
- contactId: contactIdByUser.get(userId) ?? null,
329
+ contactId: matcher.contactId,
311
330
  }));
312
331
 
313
332
  const result = await db
314
333
  .insert(bucketMemberships)
315
334
  .values(rows)
335
+ // PRD 05 T3 — ARBITER-LESS by design: it must absorb BOTH partial unique
336
+ // indexes on this table (uq_user_bucket_active and the contact-scoped
337
+ // uq_contact_bucket_active). On a BATCH that matters twice over: two
338
+ // distinct `user_id`s in one chunk can resolve to the SAME contact, and
339
+ // DO NOTHING (unlike DO UPDATE) also collapses that intra-statement
340
+ // conflict instead of raising. See `checkBucketMembership`'s insert.
316
341
  .onConflictDoNothing()
317
342
  .returning({ id: bucketMemberships.id });
318
343
 
@@ -343,11 +368,15 @@ async function reevalLeaves(opts: {
343
368
  const criteria = bucket.criteria as ConditionEval;
344
369
 
345
370
  // The set of users who STILL match (so non-matching active members = leavers).
346
- const matcherIds =
371
+ // Membership and matcher are compared by CONTACT: comparing canonical strings
372
+ // marks an adopted member as "no longer matching" purely because their key
373
+ // moved, and a criteria-change re-eval would then evict them and emit
374
+ // `bucket:left` into live journeys.
375
+ const matchers =
347
376
  criteria.type === "event"
348
377
  ? await selectEventMatchers(db, criteria)
349
378
  : await selectCompositeMatchers(db, criteria);
350
- const matcherSet = new Set(matcherIds);
379
+ const matcherSet = new Set(matchers.map((m) => m.contactId));
351
380
 
352
381
  const activeMembers = await db
353
382
  .select({
@@ -361,7 +390,7 @@ async function reevalLeaves(opts: {
361
390
  entryCount: bucketMemberships.entryCount,
362
391
  })
363
392
  .from(bucketMemberships)
364
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
393
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
365
394
  .where(
366
395
  and(
367
396
  eq(bucketMemberships.bucketId, bucket.id),
@@ -371,7 +400,7 @@ async function reevalLeaves(opts: {
371
400
  ),
372
401
  );
373
402
 
374
- const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
403
+ const leavers = activeMembers.filter((m) => !matcherSet.has(m.contactId));
375
404
  if (leavers.length === 0) return 0;
376
405
 
377
406
  // Membership key → contact row id for the emit's provenance pin. Keying by
@@ -433,11 +462,26 @@ async function reevalLeaves(opts: {
433
462
  return leftCount;
434
463
  }
435
464
 
436
- /** Set-based matcher user-ids for a single-event criterion (Section 6.6). */
465
+ /**
466
+ * Set-based matchers for a single-event criterion (Section 6.6).
467
+ *
468
+ * D8 (cohort preservation): BOTH shapes below matched `contacts.external_id`
469
+ * against the event key, so a contact with a NULL `external_id` — email-only or
470
+ * anonymous — has never been materialized by this selector. Flipping the join
471
+ * to `user_events.contact_id` would silently admit that whole cohort and move
472
+ * every bucket's count on a release billed as a read flip, so the cohort is
473
+ * held EXACTLY where it was with an explicit `isNotNull(contacts.externalId)`.
474
+ * Widening it is a real fix (the cron's join path already reaches them) but it
475
+ * is a deliberate, separately-observable change — see OPEN ITEMS in the PR.
476
+ *
477
+ * What DOES change is per-contact: the events counted are now the ones the
478
+ * contact OWNS, so an anon-era event adopted onto them counts toward the
479
+ * window. That is the flip.
480
+ */
437
481
  async function selectEventMatchers(
438
482
  db: Database,
439
483
  criteria: Extract<ConditionEval, { type: "event" }>,
440
- ): Promise<string[]> {
484
+ ): Promise<BackfillMatcher[]> {
441
485
  const cutoff = criteria.within
442
486
  ? new Date(Date.now() - durationToMs(criteria.within))
443
487
  : null;
@@ -454,32 +498,47 @@ async function selectEventMatchers(
454
498
  // mirroring the cron's `ever_fired` semi-join. Excludes never-active
455
499
  // contacts so the two writers select the same lapsed-only cohort.
456
500
  const everFired = db
457
- .selectDistinct({ userId: userEvents.userId })
501
+ .selectDistinct({ contactId: userEvents.contactId })
458
502
  .from(userEvents)
459
- .where(eq(userEvents.event, criteria.eventName))
503
+ .where(
504
+ and(
505
+ eq(userEvents.event, criteria.eventName),
506
+ isNotNull(userEvents.contactId),
507
+ ),
508
+ )
460
509
  .as("ever_fired");
461
510
 
462
511
  const present = db
463
- .select({ userId: userEvents.userId })
512
+ .select({ contactId: userEvents.contactId })
464
513
  .from(userEvents)
465
514
  .where(
466
515
  and(
467
516
  eq(userEvents.event, criteria.eventName),
517
+ isNotNull(userEvents.contactId),
468
518
  cutoff ? gte(userEvents.occurredAt, cutoff) : undefined,
469
519
  ),
470
520
  )
471
- .groupBy(userEvents.userId)
521
+ .groupBy(userEvents.contactId)
472
522
  .as("present");
473
523
 
474
524
  const rows = await db
475
525
  .select({
476
- userId: contactKeySql(),
526
+ contactId: contacts.id,
527
+ userKey: contactKeySql(),
528
+ email: contacts.email,
477
529
  })
478
530
  .from(contacts)
479
- .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
480
- .leftJoin(present, eq(present.userId, contacts.externalId))
481
- .where(and(isNull(contacts.deletedAt), isNull(present.userId)));
482
- return rows.map((r) => r.userId);
531
+ .innerJoin(everFired, eq(everFired.contactId, contacts.id))
532
+ .leftJoin(present, eq(present.contactId, contacts.id))
533
+ .where(
534
+ and(
535
+ isNull(contacts.deletedAt),
536
+ isNull(present.contactId),
537
+ // D8: hold the cohort where the `external_id` join left it.
538
+ isNotNull(contacts.externalId),
539
+ ),
540
+ );
541
+ return rows;
483
542
  }
484
543
 
485
544
  // exists / count: group counts then filter by the operator. Fix B: innerJoin
@@ -490,23 +549,33 @@ async function selectEventMatchers(
490
549
  // or orphan-event userIds, diverging from the live/reconcile paths.
491
550
  const rows = await db
492
551
  .select({
493
- userId: userEvents.userId,
552
+ contactId: contacts.id,
553
+ userKey: contactKeySql(),
554
+ email: contacts.email,
494
555
  cnt: sql<number>`count(*)::int`,
495
556
  })
496
557
  .from(userEvents)
497
- .innerJoin(contacts, eq(contacts.externalId, userEvents.userId))
558
+ .innerJoin(contacts, eq(contacts.id, userEvents.contactId))
498
559
  .where(
499
560
  and(
500
561
  eq(userEvents.event, criteria.eventName),
501
562
  isNull(contacts.deletedAt),
563
+ // D8: hold the cohort where the `external_id` join left it.
564
+ isNotNull(contacts.externalId),
502
565
  cutoff ? gte(userEvents.occurredAt, cutoff) : undefined,
503
566
  ),
504
567
  )
505
- .groupBy(userEvents.userId);
568
+ // Grouping on the contacts PK functionally determines the other projected
569
+ // contacts columns, so the canonical key and email need no aggregate.
570
+ .groupBy(contacts.id);
506
571
 
507
572
  return rows
508
573
  .filter((r) => matchesEventCount(criteria, Number(r.cnt)))
509
- .map((r) => r.userId);
574
+ .map((r) => ({
575
+ contactId: r.contactId,
576
+ userKey: r.userKey,
577
+ email: r.email,
578
+ }));
510
579
  }
511
580
 
512
581
  /**
@@ -527,8 +596,8 @@ async function selectEventMatchers(
527
596
  async function selectCompositeMatchers(
528
597
  db: Database,
529
598
  criteria: ConditionEval,
530
- ): Promise<string[]> {
531
- const matchers: string[] = [];
599
+ ): Promise<BackfillMatcher[]> {
600
+ const matchers: BackfillMatcher[] = [];
532
601
  let cursor: string | null = null;
533
602
 
534
603
  for (;;) {
@@ -536,6 +605,7 @@ async function selectCompositeMatchers(
536
605
  .select({
537
606
  id: contacts.id,
538
607
  userId: contactKeySql(),
608
+ email: contacts.email,
539
609
  properties: contacts.properties,
540
610
  })
541
611
  .from(contacts)
@@ -554,11 +624,19 @@ async function selectCompositeMatchers(
554
624
  ctx: {
555
625
  db,
556
626
  userId: contact.userId,
627
+ // The scan IS a contacts scan, so the subject is exact.
628
+ contactId: contact.id,
557
629
  journeyContext:
558
630
  (contact.properties as Record<string, unknown> | null) ?? {},
559
631
  },
560
632
  });
561
- if (isMember) matchers.push(contact.userId);
633
+ if (isMember) {
634
+ matchers.push({
635
+ contactId: contact.id,
636
+ userKey: contact.userId,
637
+ email: contact.email,
638
+ });
639
+ }
562
640
  }
563
641
 
564
642
  // A short page (fewer than a full batch) means the scan is exhausted.