@hogsend/engine 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +19 -10
  3. package/src/buckets/check-membership.ts +61 -8
  4. package/src/buckets/membership-epoch.ts +11 -1
  5. package/src/campaigns/cohort-sql.ts +3 -0
  6. package/src/index.ts +26 -6
  7. package/src/journeys/execute-journey-run.ts +180 -29
  8. package/src/journeys/journey-context.ts +43 -7
  9. package/src/lib/agent/tools.ts +29 -4
  10. package/src/lib/attribution-backfill.ts +27 -4
  11. package/src/lib/attribution.ts +22 -3
  12. package/src/lib/connector-actions.ts +4 -1
  13. package/src/lib/contacts.ts +518 -148
  14. package/src/lib/conversion-dispatch.ts +16 -2
  15. package/src/lib/enrollment-guards.ts +19 -12
  16. package/src/lib/feed.ts +5 -5
  17. package/src/lib/flags.ts +26 -9
  18. package/src/lib/ingestion.ts +23 -2
  19. package/src/lib/journey-lift.ts +17 -5
  20. package/src/lib/preferences.ts +116 -30
  21. package/src/lib/recipient-preferences.ts +16 -2
  22. package/src/lib/revenue.ts +20 -5
  23. package/src/lib/sms-tracked.ts +18 -1
  24. package/src/lib/tracked.ts +50 -8
  25. package/src/lib/tracking-events.ts +28 -0
  26. package/src/lib/unique-violation.ts +44 -0
  27. package/src/routes/admin/buckets.ts +20 -2
  28. package/src/routes/admin/bulk.ts +18 -2
  29. package/src/routes/admin/contacts.ts +4 -3
  30. package/src/routes/admin/conversions.ts +20 -4
  31. package/src/routes/admin/emails.ts +17 -2
  32. package/src/routes/admin/events.ts +32 -41
  33. package/src/routes/admin/funnels.ts +6 -1
  34. package/src/routes/admin/groups.ts +4 -0
  35. package/src/routes/admin/impact.ts +30 -10
  36. package/src/routes/admin/index.ts +2 -0
  37. package/src/routes/admin/journey-impact.ts +18 -9
  38. package/src/routes/admin/journeys.ts +8 -1
  39. package/src/routes/admin/maintenance.ts +216 -0
  40. package/src/routes/admin/preferences.ts +85 -38
  41. package/src/routes/admin/reporting.ts +25 -5
  42. package/src/routes/admin/targeting.ts +1 -0
  43. package/src/routes/admin/timeline.ts +16 -10
  44. package/src/routes/email/preferences.ts +18 -1
  45. package/src/routes/lists/index.ts +11 -6
  46. package/src/worker.ts +20 -0
  47. package/src/workflows/backfill-contact-id.ts +951 -0
  48. package/src/workflows/bucket-backfill.ts +145 -59
  49. package/src/workflows/bucket-reconcile.ts +154 -50
  50. package/src/workflows/check-alerts.ts +4 -0
  51. package/src/workflows/impact-digest.ts +10 -3
  52. package/src/workflows/journey-blueprint-interpreter.ts +8 -1
  53. package/src/workflows/send-campaign.ts +18 -11
@@ -15,7 +15,17 @@ import {
15
15
  importJobs,
16
16
  userEvents,
17
17
  } from "@hogsend/db";
18
- import { and, eq, gt, gte, inArray, isNull, max, sql } from "drizzle-orm";
18
+ import {
19
+ and,
20
+ eq,
21
+ gt,
22
+ gte,
23
+ inArray,
24
+ isNotNull,
25
+ isNull,
26
+ max,
27
+ sql,
28
+ } from "drizzle-orm";
19
29
  import {
20
30
  computeExpiresAt,
21
31
  computeMaxDwellAt,
@@ -33,6 +43,23 @@ import { stableStringify } from "../lib/stable-stringify.js";
33
43
  /** Insert chunk size, reusing the import-contacts precedent (Section 6.6). */
34
44
  const BATCH_SIZE = 500;
35
45
 
46
+ /**
47
+ * One matcher this backfill will materialize as a membership.
48
+ *
49
+ * The set-based selectors used to return a bare `string[]` of canonical keys,
50
+ * and every downstream lookup (prior-membership counts, dwell anchors, the
51
+ * email/contact-id fill-in) re-keyed off that string. Carrying the resolved
52
+ * `contactId` instead makes the SUBJECT the join key end to end: the counts and
53
+ * anchors below read the whole person rather than whatever string a row happens
54
+ * to name today. `userKey` still rides along because
55
+ * `bucket_memberships.user_id` is still written (PRD 05 D9).
56
+ */
57
+ interface BackfillMatcher {
58
+ contactId: string;
59
+ userKey: string;
60
+ email: string | null;
61
+ }
62
+
36
63
  /** import_jobs.format discriminator for the reused status record (Section 6.6). */
37
64
  export const FIRST_TIME_FORMAT = "bucket-backfill";
38
65
  const REEVAL_FORMAT = "bucket-reeval";
@@ -181,14 +208,14 @@ async function backfillJoins(opts: {
181
208
  const { db, bucket, jobId } = opts;
182
209
  const criteria = bucket.criteria as ConditionEval;
183
210
 
184
- const matcherIds =
211
+ const matchers =
185
212
  criteria.type === "event"
186
213
  ? await selectEventMatchers(db, criteria)
187
214
  : await selectCompositeMatchers(db, criteria);
188
215
 
189
216
  await db
190
217
  .update(importJobs)
191
- .set({ totalRows: matcherIds.length, updatedAt: new Date() })
218
+ .set({ totalRows: matchers.length, updatedAt: new Date() })
192
219
  .where(eq(importJobs.id, jobId));
193
220
 
194
221
  // Unconditional max-dwell TTL deadline, stamped once at insert (mirrors the
@@ -216,23 +243,17 @@ async function backfillJoins(opts: {
216
243
  // omission: arming at backfill would fan out one durable task per inserted row.
217
244
 
218
245
  let inserted = 0;
219
- for (let i = 0; i < matcherIds.length; i += BATCH_SIZE) {
220
- const chunk = matcherIds.slice(i, i + BATCH_SIZE);
221
-
222
- // userEmail backfilled from the contacts row where available. The chunk
223
- // holds the RESOLVED key (coalesce(external_id, anonymous_id, id)) — for an
224
- // email-only / anonymous contact that is the anonymous_id or the uuid id, NOT
225
- // the (null) external_id. Looking up by `contacts.externalId` would miss
226
- // those rows and write a NULL userEmail despite the contact having an email,
227
- // so we key the lookup + the map by the SAME coalesce expression the chunk
228
- // carries (matches reconcileBucketJoins, which reads userId + email off one
229
- // contacts row).
230
- const resolvedKey = contactKeySql();
231
- const chunkContacts = await db
232
- .select({ userKey: resolvedKey, email: contacts.email })
233
- .from(contacts)
234
- .where(and(inArray(resolvedKey, chunk), isNull(contacts.deletedAt)));
235
- const emailByUser = new Map(chunkContacts.map((c) => [c.userKey, c.email]));
246
+ for (let i = 0; i < matchers.length; i += BATCH_SIZE) {
247
+ const chunk = matchers.slice(i, i + BATCH_SIZE);
248
+ // Every chunked lookup below is keyed on the OWNING CONTACT, so the chunk
249
+ // boundary and each aggregate's GROUP BY name the same subject. Keying them
250
+ // on the canonical string while the selectors resolve contacts is exactly
251
+ // the drift that makes a set-based job miss silently: an adopted row would
252
+ // land in no bucket of the map and read back as "no prior membership".
253
+ const chunkContactIds = chunk.map((m) => m.contactId);
254
+ // The selectors already read email + contact id off the same `contacts` row
255
+ // they matched on, so the per-chunk contacts lookup this loop used to issue
256
+ // is gone (one fewer query per chunk, and no second chance to key it wrong).
236
257
 
237
258
  // Fix A: entryCount = 1 + prior memberships for each (user, bucket), the
238
259
  // same monotonic ordinal the live join computes (check-membership.ts). On a
@@ -242,19 +263,23 @@ async function backfillJoins(opts: {
242
263
  // based path must not reintroduce the O(P) serial-query trap).
243
264
  const priorCounts = await db
244
265
  .select({
245
- userId: bucketMemberships.userId,
266
+ contactId: bucketMemberships.contactId,
246
267
  cnt: sql<number>`count(*)::int`,
247
268
  })
248
269
  .from(bucketMemberships)
249
270
  .where(
250
271
  and(
251
272
  eq(bucketMemberships.bucketId, bucket.id),
252
- inArray(bucketMemberships.userId, chunk),
273
+ inArray(bucketMemberships.contactId, chunkContactIds),
253
274
  ),
254
275
  )
255
- .groupBy(bucketMemberships.userId);
256
- const priorByUser = new Map(
257
- priorCounts.map((r) => [r.userId, Number(r.cnt)]),
276
+ .groupBy(bucketMemberships.contactId);
277
+ const priorByContact = new Map(
278
+ priorCounts
279
+ .filter(
280
+ (r): r is { contactId: string; cnt: number } => r.contactId != null,
281
+ )
282
+ .map((r) => [r.contactId, Number(r.cnt)]),
258
283
  );
259
284
 
260
285
  // Batched dwell-anchor derivation (LOCKED DECISION 1): one GROUP BY
@@ -262,49 +287,57 @@ async function backfillJoins(opts: {
262
287
  // priorCounts GROUP BY above (never per-user serial queries). Only computed
263
288
  // when the criteria shape exposes a cheap per-matcher anchor event; an empty
264
289
  // map leaves dwellAnchorAt NULL → the dwell gate falls back to enteredAt.
265
- let anchorByUser = new Map<string, Date>();
290
+ let anchorByContact = new Map<string, Date>();
266
291
  if (anchorEvent != null) {
267
292
  const anchors = await db
268
293
  .select({
269
- userId: userEvents.userId,
294
+ contactId: userEvents.contactId,
270
295
  lastAt: max(userEvents.occurredAt),
271
296
  })
272
297
  .from(userEvents)
273
298
  .where(
274
299
  and(
275
300
  eq(userEvents.event, anchorEvent),
276
- inArray(userEvents.userId, chunk),
301
+ inArray(userEvents.contactId, chunkContactIds),
277
302
  ),
278
303
  )
279
- .groupBy(userEvents.userId);
280
- anchorByUser = new Map(
304
+ .groupBy(userEvents.contactId);
305
+ anchorByContact = new Map(
281
306
  anchors
282
307
  .filter(
283
- (r): r is { userId: string; lastAt: Date } => r.lastAt != null,
308
+ (r): r is { contactId: string; lastAt: Date } =>
309
+ r.contactId != null && r.lastAt != null,
284
310
  )
285
- .map((r) => [r.userId, r.lastAt]),
311
+ .map((r) => [r.contactId, r.lastAt]),
286
312
  );
287
313
  }
288
314
 
289
- const rows = chunk.map((userId) => ({
290
- userId,
315
+ const rows = chunk.map((matcher) => ({
316
+ userId: matcher.userKey,
291
317
  // Normalized at the write site (audience-model.md wart #1) — belt and
292
318
  // braces on top of the contacts row already being normalized.
293
- userEmail: normalizeEmailOrNull(emailByUser.get(userId)),
319
+ userEmail: normalizeEmailOrNull(matcher.email),
294
320
  bucketId: bucket.id,
295
321
  status: "active" as const,
296
322
  source: "backfill" as const,
297
- entryCount: 1 + (priorByUser.get(userId) ?? 0),
323
+ entryCount: 1 + (priorByContact.get(matcher.contactId) ?? 0),
298
324
  expiresAt: computeExpiresAt(bucket),
299
325
  maxDwellAt,
300
326
  // Historical dwell anchor where derivable; NULL otherwise (→ enteredAt).
301
- dwellAnchorAt: anchorByUser.get(userId) ?? null,
327
+ dwellAnchorAt: anchorByContact.get(matcher.contactId) ?? null,
302
328
  lastEvaluatedAt: new Date(),
329
+ contactId: matcher.contactId,
303
330
  }));
304
331
 
305
332
  const result = await db
306
333
  .insert(bucketMemberships)
307
334
  .values(rows)
335
+ // PRD 05 T3 — ARBITER-LESS by design: it must absorb BOTH partial unique
336
+ // indexes on this table (uq_user_bucket_active and the contact-scoped
337
+ // uq_contact_bucket_active). On a BATCH that matters twice over: two
338
+ // distinct `user_id`s in one chunk can resolve to the SAME contact, and
339
+ // DO NOTHING (unlike DO UPDATE) also collapses that intra-statement
340
+ // conflict instead of raising. See `checkBucketMembership`'s insert.
308
341
  .onConflictDoNothing()
309
342
  .returning({ id: bucketMemberships.id });
310
343
 
@@ -335,11 +368,15 @@ async function reevalLeaves(opts: {
335
368
  const criteria = bucket.criteria as ConditionEval;
336
369
 
337
370
  // The set of users who STILL match (so non-matching active members = leavers).
338
- const matcherIds =
371
+ // Membership and matcher are compared by CONTACT: comparing canonical strings
372
+ // marks an adopted member as "no longer matching" purely because their key
373
+ // moved, and a criteria-change re-eval would then evict them and emit
374
+ // `bucket:left` into live journeys.
375
+ const matchers =
339
376
  criteria.type === "event"
340
377
  ? await selectEventMatchers(db, criteria)
341
378
  : await selectCompositeMatchers(db, criteria);
342
- const matcherSet = new Set(matcherIds);
379
+ const matcherSet = new Set(matchers.map((m) => m.contactId));
343
380
 
344
381
  const activeMembers = await db
345
382
  .select({
@@ -353,7 +390,7 @@ async function reevalLeaves(opts: {
353
390
  entryCount: bucketMemberships.entryCount,
354
391
  })
355
392
  .from(bucketMemberships)
356
- .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
393
+ .innerJoin(contacts, eq(contacts.id, bucketMemberships.contactId))
357
394
  .where(
358
395
  and(
359
396
  eq(bucketMemberships.bucketId, bucket.id),
@@ -363,7 +400,7 @@ async function reevalLeaves(opts: {
363
400
  ),
364
401
  );
365
402
 
366
- const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
403
+ const leavers = activeMembers.filter((m) => !matcherSet.has(m.contactId));
367
404
  if (leavers.length === 0) return 0;
368
405
 
369
406
  // Membership key → contact row id for the emit's provenance pin. Keying by
@@ -425,11 +462,26 @@ async function reevalLeaves(opts: {
425
462
  return leftCount;
426
463
  }
427
464
 
428
- /** Set-based matcher user-ids for a single-event criterion (Section 6.6). */
465
+ /**
466
+ * Set-based matchers for a single-event criterion (Section 6.6).
467
+ *
468
+ * D8 (cohort preservation): BOTH shapes below matched `contacts.external_id`
469
+ * against the event key, so a contact with a NULL `external_id` — email-only or
470
+ * anonymous — has never been materialized by this selector. Flipping the join
471
+ * to `user_events.contact_id` would silently admit that whole cohort and move
472
+ * every bucket's count on a release billed as a read flip, so the cohort is
473
+ * held EXACTLY where it was with an explicit `isNotNull(contacts.externalId)`.
474
+ * Widening it is a real fix (the cron's join path already reaches them) but it
475
+ * is a deliberate, separately-observable change — see OPEN ITEMS in the PR.
476
+ *
477
+ * What DOES change is per-contact: the events counted are now the ones the
478
+ * contact OWNS, so an anon-era event adopted onto them counts toward the
479
+ * window. That is the flip.
480
+ */
429
481
  async function selectEventMatchers(
430
482
  db: Database,
431
483
  criteria: Extract<ConditionEval, { type: "event" }>,
432
- ): Promise<string[]> {
484
+ ): Promise<BackfillMatcher[]> {
433
485
  const cutoff = criteria.within
434
486
  ? new Date(Date.now() - durationToMs(criteria.within))
435
487
  : null;
@@ -446,32 +498,47 @@ async function selectEventMatchers(
446
498
  // mirroring the cron's `ever_fired` semi-join. Excludes never-active
447
499
  // contacts so the two writers select the same lapsed-only cohort.
448
500
  const everFired = db
449
- .selectDistinct({ userId: userEvents.userId })
501
+ .selectDistinct({ contactId: userEvents.contactId })
450
502
  .from(userEvents)
451
- .where(eq(userEvents.event, criteria.eventName))
503
+ .where(
504
+ and(
505
+ eq(userEvents.event, criteria.eventName),
506
+ isNotNull(userEvents.contactId),
507
+ ),
508
+ )
452
509
  .as("ever_fired");
453
510
 
454
511
  const present = db
455
- .select({ userId: userEvents.userId })
512
+ .select({ contactId: userEvents.contactId })
456
513
  .from(userEvents)
457
514
  .where(
458
515
  and(
459
516
  eq(userEvents.event, criteria.eventName),
517
+ isNotNull(userEvents.contactId),
460
518
  cutoff ? gte(userEvents.occurredAt, cutoff) : undefined,
461
519
  ),
462
520
  )
463
- .groupBy(userEvents.userId)
521
+ .groupBy(userEvents.contactId)
464
522
  .as("present");
465
523
 
466
524
  const rows = await db
467
525
  .select({
468
- userId: contactKeySql(),
526
+ contactId: contacts.id,
527
+ userKey: contactKeySql(),
528
+ email: contacts.email,
469
529
  })
470
530
  .from(contacts)
471
- .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
472
- .leftJoin(present, eq(present.userId, contacts.externalId))
473
- .where(and(isNull(contacts.deletedAt), isNull(present.userId)));
474
- return rows.map((r) => r.userId);
531
+ .innerJoin(everFired, eq(everFired.contactId, contacts.id))
532
+ .leftJoin(present, eq(present.contactId, contacts.id))
533
+ .where(
534
+ and(
535
+ isNull(contacts.deletedAt),
536
+ isNull(present.contactId),
537
+ // D8: hold the cohort where the `external_id` join left it.
538
+ isNotNull(contacts.externalId),
539
+ ),
540
+ );
541
+ return rows;
475
542
  }
476
543
 
477
544
  // exists / count: group counts then filter by the operator. Fix B: innerJoin
@@ -482,23 +549,33 @@ async function selectEventMatchers(
482
549
  // or orphan-event userIds, diverging from the live/reconcile paths.
483
550
  const rows = await db
484
551
  .select({
485
- userId: userEvents.userId,
552
+ contactId: contacts.id,
553
+ userKey: contactKeySql(),
554
+ email: contacts.email,
486
555
  cnt: sql<number>`count(*)::int`,
487
556
  })
488
557
  .from(userEvents)
489
- .innerJoin(contacts, eq(contacts.externalId, userEvents.userId))
558
+ .innerJoin(contacts, eq(contacts.id, userEvents.contactId))
490
559
  .where(
491
560
  and(
492
561
  eq(userEvents.event, criteria.eventName),
493
562
  isNull(contacts.deletedAt),
563
+ // D8: hold the cohort where the `external_id` join left it.
564
+ isNotNull(contacts.externalId),
494
565
  cutoff ? gte(userEvents.occurredAt, cutoff) : undefined,
495
566
  ),
496
567
  )
497
- .groupBy(userEvents.userId);
568
+ // Grouping on the contacts PK functionally determines the other projected
569
+ // contacts columns, so the canonical key and email need no aggregate.
570
+ .groupBy(contacts.id);
498
571
 
499
572
  return rows
500
573
  .filter((r) => matchesEventCount(criteria, Number(r.cnt)))
501
- .map((r) => r.userId);
574
+ .map((r) => ({
575
+ contactId: r.contactId,
576
+ userKey: r.userKey,
577
+ email: r.email,
578
+ }));
502
579
  }
503
580
 
504
581
  /**
@@ -519,8 +596,8 @@ async function selectEventMatchers(
519
596
  async function selectCompositeMatchers(
520
597
  db: Database,
521
598
  criteria: ConditionEval,
522
- ): Promise<string[]> {
523
- const matchers: string[] = [];
599
+ ): Promise<BackfillMatcher[]> {
600
+ const matchers: BackfillMatcher[] = [];
524
601
  let cursor: string | null = null;
525
602
 
526
603
  for (;;) {
@@ -528,6 +605,7 @@ async function selectCompositeMatchers(
528
605
  .select({
529
606
  id: contacts.id,
530
607
  userId: contactKeySql(),
608
+ email: contacts.email,
531
609
  properties: contacts.properties,
532
610
  })
533
611
  .from(contacts)
@@ -546,11 +624,19 @@ async function selectCompositeMatchers(
546
624
  ctx: {
547
625
  db,
548
626
  userId: contact.userId,
627
+ // The scan IS a contacts scan, so the subject is exact.
628
+ contactId: contact.id,
549
629
  journeyContext:
550
630
  (contact.properties as Record<string, unknown> | null) ?? {},
551
631
  },
552
632
  });
553
- if (isMember) matchers.push(contact.userId);
633
+ if (isMember) {
634
+ matchers.push({
635
+ contactId: contact.id,
636
+ userKey: contact.userId,
637
+ email: contact.email,
638
+ });
639
+ }
554
640
  }
555
641
 
556
642
  // A short page (fewer than a full batch) means the scan is exhausted.