@hogsend/engine 0.54.0 → 0.56.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/lib/refine.ts CHANGED
@@ -25,7 +25,6 @@ import {
25
25
  type RefineTarget,
26
26
  runRefineChain,
27
27
  } from "./refine-chain.js";
28
- import { pickRefinedTraits } from "./refine-traits.js";
29
28
 
30
29
  export type {
31
30
  RefineContactOptions,
@@ -40,16 +39,17 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
40
39
 
41
40
  /**
42
41
  * Refinement — the one new public function of this release. Resolves a contact,
43
- * spends AT MOST one provider lookup, and lands the vendor's answer as flat
44
- * `refined_*` contact properties THROUGH `ingestEvent`, so bucket membership
45
- * re-evaluates synchronously and the GTM loop closes:
42
+ * spends AT MOST one provider lookup, and lands the vendor's answer on the
43
+ * CANONICAL contact-property fields (fill-if-absent, so first-party data is
44
+ * never overwritten) THROUGH `ingestEvent`, so bucket membership re-evaluates
45
+ * synchronously and the GTM loop closes:
46
46
  *
47
47
  * ```
48
48
  * behaviour → bucket("gtm-high-intent").on("enter")
49
49
  * → refineContact()
50
- * → contacts.properties.refined_* (via ingestEvent)
50
+ * → contacts.properties.{company_employees, seniority, …} (via ingestEvent)
51
51
  * → checkBucketMembership re-runs
52
- * → bucket("gtm-qualified", b => b.prop("refined_company_employees").gte(100))
52
+ * → bucket("gtm-qualified", b => b.prop("company_employees").gte(100))
53
53
  * ```
54
54
  *
55
55
  * A STANDALONE import, never a `ctx` method — the journey context stays
@@ -129,6 +129,9 @@ export async function refineContact(
129
129
  ...(input.contactId ? { contactId: input.contactId } : {}),
130
130
  eventProperties: input.eventProperties,
131
131
  contactProperties: input.contactProperties,
132
+ ...(input.touchProperties
133
+ ? { touchProperties: input.touchProperties }
134
+ : {}),
132
135
  source: "enrichment",
133
136
  ...(input.idempotencyKey
134
137
  ? { idempotencyKey: input.idempotencyKey }
@@ -145,8 +148,8 @@ export async function refineContact(
145
148
  * Resolve the refinement subject and everything the lookup needs from it: the
146
149
  * canonical key the ingest writes back under, the lookup key (email first, then
147
150
  * the contact's company domain), the vendor query's name/company hints, and the
148
- * `refined_*` traits already on the row (what a cache hit compares against to
149
- * decide whether this contact still needs the stored answer landed).
151
+ * row's existing properties (which fill-if-absent reads at landing time to
152
+ * decide which vendor facts to keep).
150
153
  *
151
154
  * An email with no contact row is still refinable — the ingest at the end of the
152
155
  * chain creates the contact — so a brand-new address is not a `no_lookup_key`.
@@ -171,24 +174,28 @@ async function resolveRefineTarget(
171
174
 
172
175
  const properties = row.properties ?? {};
173
176
  const resolvedEmail = row.email ?? email;
174
- // REPLAY SAFETY: every key here must be one this function does NOT write.
175
- // The domain becomes the `lookupKey`, which is the `enrichment_lookups`
177
+ // REPLAY SAFETY: the domain this picks must be STABLE across a run and its
178
+ // replay. The domain becomes the `lookupKey`, which is the `enrichment_lookups`
176
179
  // arbiter — Layer 2, the version-independent exactly-once backstop. (Layer 1,
177
180
  // the memo key, is derived from the caller's arguments alone and is immune.)
178
- // A self-referential source would make a replay derive a DIFFERENT ledger key,
179
- // miss the row it just wrote, and charge the vendor a second time.
181
+ // A source that refinement itself changes between the run and the replay would
182
+ // derive a DIFFERENT ledger key, miss the row it just wrote, and charge the
183
+ // vendor a second time.
180
184
  //
181
- // `refined_company_domain` is deliberately EXCLUDED: `flattenTraits` writes it
182
- // (refine-traits.ts) from the vendor's canonical domain, which need not equal
183
- // the one we looked up by. A contact carrying only `domain` would resolve via
184
- // `domain` on the original run, then — once `refined_company_domain` exists and
185
- // outranks it — resolve differently on replay, deriving a different memo key.
186
- // The three keys below are externally supplied and therefore replay-stable.
185
+ // `company_domain` is the key `flattenTraits` WRITES (the vendor's canonical
186
+ // domain, which need not equal the one we looked up by), so it is listed LAST.
187
+ // That ordering — externally-supplied `companyDomain`/`domain` before the
188
+ // refinement-written `company_domain` — is what keeps the pick stable: the only
189
+ // way refinement can FILL `company_domain` is fill-if-absent (it was absent),
190
+ // and whenever it is the pick it was already present (so fill-if-absent leaves
191
+ // it untouched). A domain-only contact looked up by `domain` keeps resolving
192
+ // via `domain` even after the vendor fills `company_domain`, because `domain`
193
+ // outranks it.
187
194
  const domain = readString(
188
195
  properties,
189
- "company_domain",
190
196
  "companyDomain",
191
197
  "domain",
198
+ "company_domain",
192
199
  );
193
200
 
194
201
  return {
@@ -202,15 +209,9 @@ async function resolveRefineTarget(
202
209
  ...spread("lastName", readString(properties, "lastName", "last_name")),
203
210
  ...spread(
204
211
  "company",
205
- readString(
206
- properties,
207
- "company",
208
- "company_name",
209
- "companyName",
210
- "refined_company_name",
211
- ),
212
+ readString(properties, "company", "company_name", "companyName"),
212
213
  ),
213
- refinedProperties: pickRefinedTraits(properties),
214
+ existingProperties: properties,
214
215
  };
215
216
  }
216
217
 
@@ -1,5 +1,6 @@
1
1
  import type { SmsProvider } from "@hogsend/core";
2
2
  import type { env as envSchema } from "../env.js";
3
+ import { loadOptionalPlugin } from "./load-optional-plugin.js";
3
4
 
4
5
  /**
5
6
  * `@hogsend/plugin-twilio` is an OPT-IN, deferred-publish package — an engine
@@ -25,17 +26,16 @@ const TWILIO_PACKAGE = ["@hogsend", "plugin-twilio"].join("/");
25
26
 
26
27
  let createTwilioProvider: CreateTwilioProvider | null = null;
27
28
  if (process.env.TWILIO_ACCOUNT_SID && process.env.TWILIO_AUTH_TOKEN) {
28
- try {
29
- ({ createTwilioProvider } = (await import(TWILIO_PACKAGE)) as {
30
- createTwilioProvider: CreateTwilioProvider;
31
- });
32
- } catch {
33
- // Credentials set but the opt-in package isn't installed. Leave the factory
34
- // null — `smsProvidersFromEnv` skips the preset, and if Twilio was the
35
- // resolved active provider the container throws a clear "not registered"
36
- // error directing the operator to install `@hogsend/plugin-twilio`.
37
- createTwilioProvider = null;
38
- }
29
+ // Previously a bare `catch {}` that swallowed the failure with no log at all:
30
+ // a Twilio-configured deploy registered no provider, the container installed
31
+ // the inert throwing stub, and the first symptom was `sendSms` throwing at
32
+ // send time with nothing at boot to explain why. Report it where it happens.
33
+ createTwilioProvider = await loadOptionalPlugin<CreateTwilioProvider>({
34
+ specifier: TWILIO_PACKAGE,
35
+ exportName: "createTwilioProvider",
36
+ enabledBy: "TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN are set",
37
+ onFailure: (message) => console.warn(message),
38
+ });
39
39
  }
40
40
 
41
41
  /**
@@ -346,6 +346,10 @@ async function reevalLeaves(opts: {
346
346
  id: bucketMemberships.id,
347
347
  userId: bucketMemberships.userId,
348
348
  userEmail: bucketMemberships.userEmail,
349
+ // Joined contact row → its id rides along as the ENGINE-INTERNAL
350
+ // provenance pin for the leave emit, so the re-ingest folds into the
351
+ // known row instead of minting a phantom `external_id` twin (issue #608).
352
+ contactId: contacts.id,
349
353
  entryCount: bucketMemberships.entryCount,
350
354
  })
351
355
  .from(bucketMemberships)
@@ -362,6 +366,13 @@ async function reevalLeaves(opts: {
362
366
  const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
363
367
  if (leavers.length === 0) return 0;
364
368
 
369
+ // Membership key → contact row id for the emit's provenance pin. Keying by
370
+ // userId is safe: the partial-active unique index guarantees one active row
371
+ // per (user, bucket), so a flipped row maps to exactly one leaver.
372
+ const contactIdByUser = new Map(
373
+ leavers.map((m) => [m.userId, m.contactId] as const),
374
+ );
375
+
365
376
  let leftCount = 0;
366
377
  for (let i = 0; i < leavers.length; i += BATCH_SIZE) {
367
378
  const chunk = leavers.slice(i, i + BATCH_SIZE);
@@ -400,6 +411,9 @@ async function reevalLeaves(opts: {
400
411
  bucket,
401
412
  userId: row.userId,
402
413
  userEmail: row.userEmail,
414
+ // Provenance pin from the leaver's already-joined contact row, so the
415
+ // leave re-ingest folds into it instead of minting a twin (issue #608).
416
+ contactId: contactIdByUser.get(row.userId),
403
417
  epoch: row.entryCount,
404
418
  source: "backfill",
405
419
  reason: "criteria",
@@ -283,6 +283,13 @@ export const bucketExpiryTask = hatchet.durableTask({
283
283
  bucket,
284
284
  userId: input.userId,
285
285
  userEmail: input.userEmail,
286
+ // NO `contactId` provenance pin here (issue #608): this task has no
287
+ // contact row in scope, and the only channel that could carry one — the
288
+ // `bucket:arm-expiry` Hatchet event payload — is reachable from public
289
+ // ingest pushes, so an id smuggled through it would not be the
290
+ // unforgeable engine-internal provenance the pin requires. A value
291
+ // lookup here would re-implement exactly the resolution the pin exists
292
+ // to bypass. Degrades to the pre-pin resolve.
286
293
  epoch: flipped.entryCount,
287
294
  source: "reconcile",
288
295
  // Fast-expiry is a criteria re-confirm leave (Section 6.7).
@@ -293,6 +300,19 @@ export const bucketExpiryTask = hatchet.durableTask({
293
300
  },
294
301
  });
295
302
 
303
+ /**
304
+ * A member selected to leave: the membership's canonical string key plus the
305
+ * joined contact row's id — the ENGINE-INTERNAL provenance pin threaded into
306
+ * `emitBucketTransition` so the leave re-ingest folds into the known row
307
+ * instead of minting a phantom `external_id` twin (issue #608). Every leaver
308
+ * source query already joins `contacts` (the GDPR live-contact filter), so the
309
+ * id is free.
310
+ */
311
+ interface BucketLeaver {
312
+ userId: string;
313
+ contactId: string;
314
+ }
315
+
296
316
  /**
297
317
  * Set-based SHOULD-LEAVE for one time-based bucket → bulk CAS → RETURNING-gated
298
318
  * emit. For single-event `not_exists`/`exists`/`count within` criteria the
@@ -311,14 +331,14 @@ async function reconcileBucketLeaves(opts: {
311
331
 
312
332
  // A single-event/within/count criterion → set-based SHOULD-LEAVE query.
313
333
  if (criteria.type === "event") {
314
- const leaverIds = await selectEventLeavers(db, bucket, criteria);
315
- if (leaverIds.length === 0) return 0;
334
+ const leavers = await selectEventLeavers(db, bucket, criteria);
335
+ if (leavers.length === 0) return 0;
316
336
  return bulkLeave({
317
337
  db,
318
338
  logger,
319
339
  journeyRegistry,
320
340
  bucket,
321
- userIds: leaverIds,
341
+ leavers,
322
342
  reason: "criteria",
323
343
  });
324
344
  }
@@ -338,7 +358,7 @@ async function selectEventLeavers(
338
358
  db: Database,
339
359
  bucket: BucketMeta,
340
360
  criteria: Extract<ConditionEval, { type: "event" }>,
341
- ): Promise<string[]> {
361
+ ): Promise<BucketLeaver[]> {
342
362
  const cutoff = criteria.within
343
363
  ? new Date(Date.now() - durationToMs(criteria.within))
344
364
  : null;
@@ -372,10 +392,13 @@ async function selectEventLeavers(
372
392
  .groupBy(userEvents.userId)
373
393
  .as("counted");
374
394
 
375
- // LEFT JOIN members → windowed counts. A missing/zero count is a 0.
395
+ // LEFT JOIN members → windowed counts. A missing/zero count is a 0. The
396
+ // contact row is already joined (GDPR filter), so its id rides along as the
397
+ // provenance pin for the leave emit (issue #608).
376
398
  const rows = await db
377
399
  .select({
378
400
  userId: members.userId,
401
+ contactId: contacts.id,
379
402
  cnt: sql<number>`coalesce(${counted.cnt}, 0)`,
380
403
  })
381
404
  .from(members)
@@ -385,7 +408,7 @@ async function selectEventLeavers(
385
408
 
386
409
  return rows
387
410
  .filter((r) => shouldLeaveByCount(criteria, Number(r.cnt)))
388
- .map((r) => r.userId);
411
+ .map((r) => ({ userId: r.userId, contactId: r.contactId }));
389
412
  }
390
413
 
391
414
  /**
@@ -409,6 +432,9 @@ async function reconcileCompositeLeaves(opts: {
409
432
  const members = await db
410
433
  .select({
411
434
  userId: bucketMemberships.userId,
435
+ // Joined contact row → its id is the provenance pin for the leave emit
436
+ // (issue #608).
437
+ contactId: contacts.id,
412
438
  properties: contacts.properties,
413
439
  })
414
440
  .from(bucketMemberships)
@@ -424,7 +450,7 @@ async function reconcileCompositeLeaves(opts: {
424
450
  .orderBy(sql`${bucketMemberships.lastEvaluatedAt} asc nulls first`)
425
451
  .limit(BATCH_SIZE);
426
452
 
427
- const leaverIds: string[] = [];
453
+ const leavers: BucketLeaver[] = [];
428
454
  const evaluatedIds: string[] = [];
429
455
  for (const member of members) {
430
456
  evaluatedIds.push(member.userId);
@@ -435,7 +461,9 @@ async function reconcileCompositeLeaves(opts: {
435
461
  condition: criteria,
436
462
  ctx: { db, userId: member.userId, journeyContext },
437
463
  });
438
- if (!isMember) leaverIds.push(member.userId);
464
+ if (!isMember) {
465
+ leavers.push({ userId: member.userId, contactId: member.contactId });
466
+ }
439
467
  }
440
468
 
441
469
  // Bump lastEvaluatedAt for the whole chunk so the next tick advances the cursor
@@ -453,13 +481,13 @@ async function reconcileCompositeLeaves(opts: {
453
481
  );
454
482
  }
455
483
 
456
- if (leaverIds.length === 0) return 0;
484
+ if (leavers.length === 0) return 0;
457
485
  return bulkLeave({
458
486
  db,
459
487
  logger,
460
488
  journeyRegistry,
461
489
  bucket,
462
- userIds: leaverIds,
490
+ leavers,
463
491
  reason: "criteria",
464
492
  });
465
493
  }
@@ -481,7 +509,12 @@ async function reconcileBucketTtlLeaves(opts: {
481
509
  const { db, logger, journeyRegistry, bucket } = opts;
482
510
 
483
511
  const expired = await db
484
- .select({ userId: bucketMemberships.userId })
512
+ .select({
513
+ userId: bucketMemberships.userId,
514
+ // Joined contact row → its id is the provenance pin for the leave emit
515
+ // (issue #608).
516
+ contactId: contacts.id,
517
+ })
485
518
  .from(bucketMemberships)
486
519
  .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
487
520
  .where(
@@ -501,7 +534,7 @@ async function reconcileBucketTtlLeaves(opts: {
501
534
  logger,
502
535
  journeyRegistry,
503
536
  bucket,
504
- userIds: expired.map((r) => r.userId),
537
+ leavers: expired,
505
538
  reason: "maxDwell",
506
539
  });
507
540
  }
@@ -518,11 +551,18 @@ async function bulkLeave(opts: {
518
551
  logger: Logger;
519
552
  journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
520
553
  bucket: BucketMeta;
521
- userIds: string[];
554
+ leavers: BucketLeaver[];
522
555
  /** Why these members leave — TTL passes "maxDwell", criteria passes "criteria". */
523
556
  reason: BucketLeaveReason;
524
557
  }): Promise<number> {
525
- const { db, logger, journeyRegistry, bucket, userIds, reason } = opts;
558
+ const { db, logger, journeyRegistry, bucket, leavers, reason } = opts;
559
+ const userIds = leavers.map((l) => l.userId);
560
+ // Membership key → resolved contact row id, for the emit's provenance pin.
561
+ // Safe to key by userId: the partial-active unique index guarantees one
562
+ // active row per (user, bucket), so a flipped row maps to exactly one leaver.
563
+ const contactIdByUser = new Map(
564
+ leavers.map((l) => [l.userId, l.contactId] as const),
565
+ );
526
566
 
527
567
  const dwellMs = bucket.minDwell ? durationToMs(bucket.minDwell) : 0;
528
568
  const dwellCutoff = dwellMs > 0 ? new Date(Date.now() - dwellMs) : null;
@@ -565,6 +605,9 @@ async function bulkLeave(opts: {
565
605
  bucket,
566
606
  userId: row.userId,
567
607
  userEmail: row.userEmail,
608
+ // Provenance pin from the leaver's already-joined contact row, so the
609
+ // leave re-ingest folds into it instead of minting a twin (issue #608).
610
+ contactId: contactIdByUser.get(row.userId),
568
611
  epoch: row.entryCount,
569
612
  source: "reconcile",
570
613
  reason,
@@ -643,6 +686,9 @@ async function reconcileBucketDwell(opts: {
643
686
  id: bucketMemberships.id,
644
687
  userId: bucketMemberships.userId,
645
688
  userEmail: bucketMemberships.userEmail,
689
+ // Joined contact row → its id is the provenance pin for the dwell emit
690
+ // (issue #608).
691
+ contactId: contacts.id,
646
692
  entryCount: bucketMemberships.entryCount,
647
693
  anchor: sql<Date>`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt})`,
648
694
  dwellState: bucketMemberships.dwellState,
@@ -704,6 +750,9 @@ async function reconcileBucketDwell(opts: {
704
750
  bucket,
705
751
  userId: m.userId,
706
752
  userEmail: m.userEmail,
753
+ // Provenance pin from the candidate's already-joined contact row
754
+ // (issue #608).
755
+ contactId: m.contactId,
707
756
  epoch: m.entryCount,
708
757
  source: "reconcile",
709
758
  dwellLabel: label,
@@ -857,6 +906,11 @@ async function reconcileBucketJoins(opts: {
857
906
  const baseQuery = db
858
907
  .select({
859
908
  userId: contactKey,
909
+ // The candidate scan reads straight off `contacts`, so the row id rides
910
+ // along as the provenance pin for the join emit (issue #608) —
911
+ // load-bearing for exactly the null-external_id contacts this coalesce
912
+ // join exists to include.
913
+ contactId: contacts.id,
860
914
  email: contacts.email,
861
915
  })
862
916
  .from(contacts)
@@ -934,6 +988,7 @@ async function reconcileBucketJoins(opts: {
934
988
  bucket,
935
989
  userId: candidate.userId,
936
990
  userEmail: candidate.email ?? null,
991
+ contactId: candidate.contactId,
937
992
  });
938
993
  if (transitioned) joined += 1;
939
994
  }
@@ -1035,8 +1090,10 @@ async function reconcileJoinOne(opts: {
1035
1090
  bucket: BucketMeta;
1036
1091
  userId: string;
1037
1092
  userEmail: string | null;
1093
+ /** Provenance pin for the join emit — see {@link BucketLeaver} (issue #608). */
1094
+ contactId?: string;
1038
1095
  }): Promise<boolean> {
1039
- const { db, logger, journeyRegistry, bucket, userId } = opts;
1096
+ const { db, logger, journeyRegistry, bucket, userId, contactId } = opts;
1040
1097
  // Normalized at the write site (audience-model.md wart #1) — the contact row
1041
1098
  // should already be normalized, but the membership keyspace must never
1042
1099
  // depend on that.
@@ -1081,6 +1138,8 @@ async function reconcileJoinOne(opts: {
1081
1138
  bucket,
1082
1139
  userId,
1083
1140
  userEmail,
1141
+ // Provenance pin from the candidate scan's contact row (issue #608).
1142
+ contactId,
1084
1143
  epoch,
1085
1144
  source: "reconcile",
1086
1145
  });