@hogsend/engine 0.53.0 → 0.55.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/package.json +12 -11
  2. package/src/buckets/check-membership.ts +45 -3
  3. package/src/connectors/define-action.ts +15 -2
  4. package/src/container.ts +85 -0
  5. package/src/env.ts +21 -0
  6. package/src/index.ts +19 -0
  7. package/src/journeys/durable-law-harness.test.ts +107 -0
  8. package/src/journeys/durable-law-harness.ts +139 -0
  9. package/src/journeys/journey-boundary.test.ts +70 -0
  10. package/src/journeys/journey-boundary.ts +10 -1
  11. package/src/journeys/with-durable-gate.test.ts +258 -0
  12. package/src/journeys/with-durable-gate.ts +141 -0
  13. package/src/lib/bucket-emit.test.ts +145 -0
  14. package/src/lib/bucket-emit.ts +29 -3
  15. package/src/lib/connector-actions.test.ts +292 -1
  16. package/src/lib/connector-actions.ts +65 -6
  17. package/src/lib/email-providers-from-env.ts +15 -16
  18. package/src/lib/enrichment-ledger.ts +207 -0
  19. package/src/lib/enrichment-provider-registry.test.ts +64 -0
  20. package/src/lib/enrichment-provider-registry.ts +35 -0
  21. package/src/lib/enrichment-providers-from-env.apollo.test.ts +30 -0
  22. package/src/lib/enrichment-providers-from-env.test.ts +28 -0
  23. package/src/lib/enrichment-providers-from-env.ts +52 -0
  24. package/src/lib/enrichment-registry-singleton.ts +45 -0
  25. package/src/lib/ingestion.ts +5 -0
  26. package/src/lib/load-optional-plugin.test.ts +133 -0
  27. package/src/lib/load-optional-plugin.ts +142 -0
  28. package/src/lib/outbound.ts +27 -0
  29. package/src/lib/refine-chain.test.ts +764 -0
  30. package/src/lib/refine-chain.ts +583 -0
  31. package/src/lib/refine-traits.test.ts +141 -0
  32. package/src/lib/refine-traits.ts +128 -0
  33. package/src/lib/refine.ts +287 -0
  34. package/src/lib/sms-providers-from-env.ts +11 -11
  35. package/src/lib/webhook-catalog-sync.test.ts +112 -0
  36. package/src/lib/webhook-signing.ts +5 -1
  37. package/src/routes/admin/contacts.ts +113 -13
  38. package/src/workflows/bucket-backfill.ts +14 -0
  39. package/src/workflows/bucket-reconcile.ts +74 -15
@@ -0,0 +1,112 @@
1
+ import assert from "node:assert/strict";
2
+ import { readFileSync } from "node:fs";
3
+ import test from "node:test";
4
+ import { fileURLToPath } from "node:url";
5
+ import { WEBHOOK_EVENT_TYPES } from "./webhook-signing.js";
6
+
7
+ /**
8
+ * THE DRIFT CHECK the outbound catalog never had.
9
+ *
10
+ * `WEBHOOK_EVENT_TYPES` is vendored into two copies that cannot import the
11
+ * engine — the CLI's own tuple and the client's `OutboundEventType` union — and
12
+ * until this file the only thing keeping them in sync was a comment asking
13
+ * nicely. Every event added since has been a coin flip.
14
+ *
15
+ * So: read the two vendored files off disk and compare the FULL sets, not just
16
+ * the newest member. Whoever adds event 32 and forgets a copy gets a failing
17
+ * test naming the file they missed, rather than a subscriber who can never
18
+ * register for it.
19
+ *
20
+ * This is a monorepo-source check by construction — it reads sibling packages'
21
+ * `src/`, which is exactly the coupling it is asserting.
22
+ */
23
+
24
+ const CLI_CATALOG = fileURLToPath(
25
+ new URL("../../../cli/src/commands/webhooks.ts", import.meta.url),
26
+ );
27
+ const CLIENT_CATALOG = fileURLToPath(
28
+ new URL("../../../client/src/types.ts", import.meta.url),
29
+ );
30
+
31
+ /** The source between `start` and the next `end`, exclusive. */
32
+ function between(file: string, start: string, end: string): string {
33
+ const source = readFileSync(file, "utf8");
34
+ const from = source.indexOf(start);
35
+ assert.notEqual(from, -1, `${file}: could not find \`${start}\``);
36
+ const to = source.indexOf(end, from + start.length);
37
+ assert.notEqual(
38
+ to,
39
+ -1,
40
+ `${file}: could not find \`${end}\` after \`${start}\``,
41
+ );
42
+ return source.slice(from + start.length, to);
43
+ }
44
+
45
+ /** Every `"<namespace>.<name>"` literal in a block of source. */
46
+ function eventLiterals(block: string): string[] {
47
+ return [...block.matchAll(/"([a-z]+\.[a-z_]+)"/g)].map(
48
+ (match) => match[1] as string,
49
+ );
50
+ }
51
+
52
+ function readCliCatalog(): string[] {
53
+ return eventLiterals(
54
+ between(CLI_CATALOG, "const WEBHOOK_EVENT_TYPES = [", "] as const;"),
55
+ );
56
+ }
57
+
58
+ function readClientCatalog(): string[] {
59
+ return eventLiterals(
60
+ between(CLIENT_CATALOG, "export type OutboundEventType =", ";"),
61
+ );
62
+ }
63
+
64
+ /** Set-wise, because the three copies are not in the same ORDER (they never
65
+ * have been) — only membership is load-bearing. */
66
+ function assertSameSet(actual: string[], expected: string[], what: string) {
67
+ const missing = expected.filter((event) => !actual.includes(event));
68
+ const extra = actual.filter(
69
+ (event) => !(expected as string[]).includes(event),
70
+ );
71
+ assert.deepEqual(
72
+ { missing, extra },
73
+ { missing: [], extra: [] },
74
+ `${what} has drifted from the engine's WEBHOOK_EVENT_TYPES`,
75
+ );
76
+ assert.equal(
77
+ new Set(actual).size,
78
+ actual.length,
79
+ `${what} contains a duplicate entry`,
80
+ );
81
+ }
82
+
83
+ test("AC 4: contact.refined is present in ALL THREE hand-synced catalogs", () => {
84
+ assert.ok(
85
+ (WEBHOOK_EVENT_TYPES as readonly string[]).includes("contact.refined"),
86
+ "engine WEBHOOK_EVENT_TYPES is missing contact.refined",
87
+ );
88
+ assert.ok(
89
+ readCliCatalog().includes("contact.refined"),
90
+ "packages/cli/src/commands/webhooks.ts is missing contact.refined",
91
+ );
92
+ assert.ok(
93
+ readClientCatalog().includes("contact.refined"),
94
+ "packages/client/src/types.ts is missing contact.refined",
95
+ );
96
+ });
97
+
98
+ test("AC 4: the CLI's vendored catalog is the engine catalog, entry for entry", () => {
99
+ assertSameSet(
100
+ readCliCatalog(),
101
+ [...WEBHOOK_EVENT_TYPES],
102
+ "packages/cli/src/commands/webhooks.ts",
103
+ );
104
+ });
105
+
106
+ test("AC 4: the client's OutboundEventType union is the engine catalog, entry for entry", () => {
107
+ assertSameSet(
108
+ readClientCatalog(),
109
+ [...WEBHOOK_EVENT_TYPES],
110
+ "packages/client/src/types.ts",
111
+ );
112
+ });
@@ -25,7 +25,7 @@ import { Webhook } from "svix";
25
25
  */
26
26
 
27
27
  /**
28
- * The 30-event catalog — the SINGLE source of truth (schema, routes, client,
28
+ * The 31-event catalog — the SINGLE source of truth (schema, routes, client,
29
29
  * CLI all derive from this). The `webhook.test` sentinel is intentionally NOT a
30
30
  * member (it is delivered out-of-band regardless of an endpoint's `eventTypes`).
31
31
  *
@@ -63,6 +63,10 @@ export const WEBHOOK_EVENT_TYPES = [
63
63
  // Global control group membership (impact plan §4.3) — emitted once per
64
64
  // contact key on the first withheld send.
65
65
  "contact.control_group",
66
+ // A refinement landed vendor traits on a contact (`refineContact`). Emitted
67
+ // ONLY on a genuine `status: "refined"` — never on `cached`, `not_found` or
68
+ // `skipped`. A cache hit spends nothing and is not an event.
69
+ "contact.refined",
66
70
  "email.sent",
67
71
  "email.delivered",
68
72
  "email.opened",
@@ -65,22 +65,55 @@ const contactGroupSchema = z.object({
65
65
  joinedAt: z.string(),
66
66
  });
67
67
 
68
+ /**
69
+ * PRD 06 — validation gate for a jsonb property key used in ordering or
70
+ * filtering. This is a REJECTION layer only: the key is additionally ALWAYS
71
+ * bound as a SQL parameter in the query builder (never interpolated), so the
72
+ * two layers are independent defences, not substitutes for each other.
73
+ */
74
+ const PROPERTY_KEY_RE = /^[A-Za-z0-9_.-]{1,64}$/;
75
+
68
76
  const listRoute = createRoute({
69
77
  method: "get",
70
78
  path: "/",
71
79
  tags: ["Admin"],
72
80
  summary: "List contacts",
73
81
  request: {
74
- query: z.object({
75
- limit: z.coerce.number().min(1).max(100).default(50),
76
- offset: z.coerce.number().min(0).default(0),
77
- search: z.string().optional(),
78
- // Long-tail value filters (plan §4b.3): the "find my value customers"
79
- // query surface.
80
- minRevenue: z.coerce.number().optional(),
81
- // Plain string: valid stages are the deployment's configured ladder.
82
- dealStage: z.string().optional(),
83
- }),
82
+ query: z
83
+ .object({
84
+ limit: z.coerce.number().min(1).max(100).default(50),
85
+ offset: z.coerce.number().min(0).default(0),
86
+ search: z.string().optional(),
87
+ // Long-tail value filters (plan §4b.3): the "find my value customers"
88
+ // query surface.
89
+ minRevenue: z.coerce.number().optional(),
90
+ // Plain string: valid stages are the deployment's configured ladder.
91
+ dealStage: z.string().optional(),
92
+ // Leaderboard surface (PRD 06): rank contacts by the NUMERIC value of
93
+ // a jsonb property ("who do I call today"). Defaults reproduce the
94
+ // pre-existing behaviour exactly (ORDER BY last_seen_at DESC).
95
+ orderBy: z
96
+ .enum(["lastSeenAt", "firstSeenAt", "property"])
97
+ .default("lastSeenAt"),
98
+ orderProperty: z.string().regex(PROPERTY_KEY_RE).optional(),
99
+ orderDir: z.enum(["asc", "desc"]).default("desc"),
100
+ // Numeric property filter: only contacts whose value at `propertyKey`
101
+ // is a real JSON number ≥ `propertyGte`. Composable with ordering.
102
+ propertyKey: z.string().regex(PROPERTY_KEY_RE).optional(),
103
+ propertyGte: z.coerce.number().optional(),
104
+ })
105
+ .refine(
106
+ (q) => q.orderBy !== "property" || q.orderProperty !== undefined,
107
+ {
108
+ message: "orderProperty is required when orderBy=property",
109
+ },
110
+ )
111
+ .refine(
112
+ (q) => q.propertyGte === undefined || q.propertyKey !== undefined,
113
+ {
114
+ message: "propertyKey is required when propertyGte is set",
115
+ },
116
+ ),
84
117
  },
85
118
  responses: {
86
119
  200: {
@@ -96,6 +129,16 @@ const listRoute = createRoute({
96
129
  },
97
130
  description: "Paginated contact list",
98
131
  },
132
+ 400: {
133
+ content: {
134
+ "application/json": {
135
+ // zod-openapi's default validation hook shape (ZodError passthrough).
136
+ schema: z.object({ success: z.boolean() }).passthrough(),
137
+ },
138
+ },
139
+ description:
140
+ "Invalid query (bad property key, or orderBy=property without orderProperty)",
141
+ },
99
142
  },
100
143
  });
101
144
 
@@ -246,11 +289,67 @@ const serializeContact = (row: typeof contacts.$inferSelect) =>
246
289
  export const contactsRouter = new OpenAPIHono<AppEnv>()
247
290
  .openapi(listRoute, async (c) => {
248
291
  const { db } = c.get("container");
249
- const { limit, offset, search, minRevenue, dealStage } =
250
- c.req.valid("query");
292
+ const {
293
+ limit,
294
+ offset,
295
+ search,
296
+ minRevenue,
297
+ dealStage,
298
+ orderBy,
299
+ orderProperty,
300
+ orderDir,
301
+ propertyKey,
302
+ propertyGte,
303
+ } = c.req.valid("query");
251
304
 
252
305
  const searchFilter = search ? contactSearchFilter(search) : undefined;
253
306
 
307
+ // Type-guarded numeric read of a jsonb property (PRD 06 / DECISIONS §3.4).
308
+ // `contacts.properties` is untyped jsonb and `/v1/events` accepts arbitrary
309
+ // values, so a bare `(properties->>key)::numeric` raises Postgres 22P02 —
310
+ // and 500s the whole request — the first time ONE contact holds e.g. "n/a"
311
+ // at the key. Only real JSON numbers count; anything else yields NULL
312
+ // (sorted last, excluded by the ≥ filter). The key rides as a BOUND
313
+ // PARAMETER — never interpolated — with the regex validation above as a
314
+ // second, independent layer.
315
+ const numericProperty = (key: string) =>
316
+ sql`CASE WHEN jsonb_typeof(${contacts.properties} -> ${key}) = 'number' THEN (${contacts.properties} ->> ${key})::numeric END`;
317
+
318
+ // Filter applies only when both halves are present (the schema 400s a
319
+ // dangling propertyGte). A NULL from the guard fails the ≥ comparison, so
320
+ // non-numeric holders are excluded rather than erroring.
321
+ const propertyFilter =
322
+ propertyKey !== undefined && propertyGte !== undefined
323
+ ? sql`${numericProperty(propertyKey)} >= ${propertyGte}`
324
+ : undefined;
325
+
326
+ // NULLS LAST in BOTH directions so unscored (or non-numeric) contacts
327
+ // never top the list; last_seen_at breaks ties deterministically. The
328
+ // default path (`orderBy=lastSeenAt`, `orderDir=desc`) is byte-identical
329
+ // to the pre-PRD-06 hardcoded ORDER BY — existing callers see no change.
330
+ // NOTE: the GIN index on `properties` accelerates containment filters
331
+ // only; this ORDER BY is a sequential scan — fine at GTM volume
332
+ // (thousands of contacts), not at analytics volume.
333
+ const orderClauses =
334
+ orderBy === "property" && orderProperty !== undefined
335
+ ? [
336
+ orderDir === "asc"
337
+ ? sql`${numericProperty(orderProperty)} asc nulls last`
338
+ : sql`${numericProperty(orderProperty)} desc nulls last`,
339
+ desc(contacts.lastSeenAt),
340
+ ]
341
+ : orderBy === "firstSeenAt"
342
+ ? [
343
+ orderDir === "asc"
344
+ ? asc(contacts.firstSeenAt)
345
+ : desc(contacts.firstSeenAt),
346
+ ]
347
+ : [
348
+ orderDir === "asc"
349
+ ? asc(contacts.lastSeenAt)
350
+ : desc(contacts.lastSeenAt),
351
+ ];
352
+
254
353
  // Valued events are keyed by the contact's canonical event key
255
354
  // (external_id ?? anonymous_id ?? id) — same precedence ingestEvent
256
355
  // resolves. Served by the partial user_events_valued_user_idx.
@@ -284,6 +383,7 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
284
383
  ...(searchFilter ? [searchFilter] : []),
285
384
  ...(revenueFilter ? [revenueFilter] : []),
286
385
  ...(dealStageFilter ? [dealStageFilter] : []),
386
+ ...(propertyFilter ? [propertyFilter] : []),
287
387
  );
288
388
 
289
389
  const [rows, totalRows] = await Promise.all([
@@ -291,7 +391,7 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
291
391
  .select()
292
392
  .from(contacts)
293
393
  .where(where)
294
- .orderBy(desc(contacts.lastSeenAt))
394
+ .orderBy(...orderClauses)
295
395
  .limit(limit)
296
396
  .offset(offset),
297
397
  db.select({ count: count() }).from(contacts).where(where),
@@ -346,6 +346,10 @@ async function reevalLeaves(opts: {
346
346
  id: bucketMemberships.id,
347
347
  userId: bucketMemberships.userId,
348
348
  userEmail: bucketMemberships.userEmail,
349
+ // Joined contact row → its id rides along as the ENGINE-INTERNAL
350
+ // provenance pin for the leave emit, so the re-ingest folds into the
351
+ // known row instead of minting a phantom `external_id` twin (issue #608).
352
+ contactId: contacts.id,
349
353
  entryCount: bucketMemberships.entryCount,
350
354
  })
351
355
  .from(bucketMemberships)
@@ -362,6 +366,13 @@ async function reevalLeaves(opts: {
362
366
  const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
363
367
  if (leavers.length === 0) return 0;
364
368
 
369
+ // Membership key → contact row id for the emit's provenance pin. Keying by
370
+ // userId is safe: the partial-active unique index guarantees one active row
371
+ // per (user, bucket), so a flipped row maps to exactly one leaver.
372
+ const contactIdByUser = new Map(
373
+ leavers.map((m) => [m.userId, m.contactId] as const),
374
+ );
375
+
365
376
  let leftCount = 0;
366
377
  for (let i = 0; i < leavers.length; i += BATCH_SIZE) {
367
378
  const chunk = leavers.slice(i, i + BATCH_SIZE);
@@ -400,6 +411,9 @@ async function reevalLeaves(opts: {
400
411
  bucket,
401
412
  userId: row.userId,
402
413
  userEmail: row.userEmail,
414
+ // Provenance pin from the leaver's already-joined contact row, so the
415
+ // leave re-ingest folds into it instead of minting a twin (issue #608).
416
+ contactId: contactIdByUser.get(row.userId),
403
417
  epoch: row.entryCount,
404
418
  source: "backfill",
405
419
  reason: "criteria",
@@ -283,6 +283,13 @@ export const bucketExpiryTask = hatchet.durableTask({
283
283
  bucket,
284
284
  userId: input.userId,
285
285
  userEmail: input.userEmail,
286
+ // NO `contactId` provenance pin here (issue #608): this task has no
287
+ // contact row in scope, and the only channel that could carry one — the
288
+ // `bucket:arm-expiry` Hatchet event payload — is reachable from public
289
+ // ingest pushes, so an id smuggled through it would not be the
290
+ // unforgeable engine-internal provenance the pin requires. A value
291
+ // lookup here would re-implement exactly the resolution the pin exists
292
+ // to bypass. Degrades to the pre-pin resolve.
286
293
  epoch: flipped.entryCount,
287
294
  source: "reconcile",
288
295
  // Fast-expiry is a criteria re-confirm leave (Section 6.7).
@@ -293,6 +300,19 @@ export const bucketExpiryTask = hatchet.durableTask({
293
300
  },
294
301
  });
295
302
 
303
+ /**
304
+ * A member selected to leave: the membership's canonical string key plus the
305
+ * joined contact row's id — the ENGINE-INTERNAL provenance pin threaded into
306
+ * `emitBucketTransition` so the leave re-ingest folds into the known row
307
+ * instead of minting a phantom `external_id` twin (issue #608). Every leaver
308
+ * source query already joins `contacts` (the GDPR live-contact filter), so the
309
+ * id is free.
310
+ */
311
+ interface BucketLeaver {
312
+ userId: string;
313
+ contactId: string;
314
+ }
315
+
296
316
  /**
297
317
  * Set-based SHOULD-LEAVE for one time-based bucket → bulk CAS → RETURNING-gated
298
318
  * emit. For single-event `not_exists`/`exists`/`count within` criteria the
@@ -311,14 +331,14 @@ async function reconcileBucketLeaves(opts: {
311
331
 
312
332
  // A single-event/within/count criterion → set-based SHOULD-LEAVE query.
313
333
  if (criteria.type === "event") {
314
- const leaverIds = await selectEventLeavers(db, bucket, criteria);
315
- if (leaverIds.length === 0) return 0;
334
+ const leavers = await selectEventLeavers(db, bucket, criteria);
335
+ if (leavers.length === 0) return 0;
316
336
  return bulkLeave({
317
337
  db,
318
338
  logger,
319
339
  journeyRegistry,
320
340
  bucket,
321
- userIds: leaverIds,
341
+ leavers,
322
342
  reason: "criteria",
323
343
  });
324
344
  }
@@ -338,7 +358,7 @@ async function selectEventLeavers(
338
358
  db: Database,
339
359
  bucket: BucketMeta,
340
360
  criteria: Extract<ConditionEval, { type: "event" }>,
341
- ): Promise<string[]> {
361
+ ): Promise<BucketLeaver[]> {
342
362
  const cutoff = criteria.within
343
363
  ? new Date(Date.now() - durationToMs(criteria.within))
344
364
  : null;
@@ -372,10 +392,13 @@ async function selectEventLeavers(
372
392
  .groupBy(userEvents.userId)
373
393
  .as("counted");
374
394
 
375
- // LEFT JOIN members → windowed counts. A missing/zero count is a 0.
395
+ // LEFT JOIN members → windowed counts. A missing/zero count is a 0. The
396
+ // contact row is already joined (GDPR filter), so its id rides along as the
397
+ // provenance pin for the leave emit (issue #608).
376
398
  const rows = await db
377
399
  .select({
378
400
  userId: members.userId,
401
+ contactId: contacts.id,
379
402
  cnt: sql<number>`coalesce(${counted.cnt}, 0)`,
380
403
  })
381
404
  .from(members)
@@ -385,7 +408,7 @@ async function selectEventLeavers(
385
408
 
386
409
  return rows
387
410
  .filter((r) => shouldLeaveByCount(criteria, Number(r.cnt)))
388
- .map((r) => r.userId);
411
+ .map((r) => ({ userId: r.userId, contactId: r.contactId }));
389
412
  }
390
413
 
391
414
  /**
@@ -409,6 +432,9 @@ async function reconcileCompositeLeaves(opts: {
409
432
  const members = await db
410
433
  .select({
411
434
  userId: bucketMemberships.userId,
435
+ // Joined contact row → its id is the provenance pin for the leave emit
436
+ // (issue #608).
437
+ contactId: contacts.id,
412
438
  properties: contacts.properties,
413
439
  })
414
440
  .from(bucketMemberships)
@@ -424,7 +450,7 @@ async function reconcileCompositeLeaves(opts: {
424
450
  .orderBy(sql`${bucketMemberships.lastEvaluatedAt} asc nulls first`)
425
451
  .limit(BATCH_SIZE);
426
452
 
427
- const leaverIds: string[] = [];
453
+ const leavers: BucketLeaver[] = [];
428
454
  const evaluatedIds: string[] = [];
429
455
  for (const member of members) {
430
456
  evaluatedIds.push(member.userId);
@@ -435,7 +461,9 @@ async function reconcileCompositeLeaves(opts: {
435
461
  condition: criteria,
436
462
  ctx: { db, userId: member.userId, journeyContext },
437
463
  });
438
- if (!isMember) leaverIds.push(member.userId);
464
+ if (!isMember) {
465
+ leavers.push({ userId: member.userId, contactId: member.contactId });
466
+ }
439
467
  }
440
468
 
441
469
  // Bump lastEvaluatedAt for the whole chunk so the next tick advances the cursor
@@ -453,13 +481,13 @@ async function reconcileCompositeLeaves(opts: {
453
481
  );
454
482
  }
455
483
 
456
- if (leaverIds.length === 0) return 0;
484
+ if (leavers.length === 0) return 0;
457
485
  return bulkLeave({
458
486
  db,
459
487
  logger,
460
488
  journeyRegistry,
461
489
  bucket,
462
- userIds: leaverIds,
490
+ leavers,
463
491
  reason: "criteria",
464
492
  });
465
493
  }
@@ -481,7 +509,12 @@ async function reconcileBucketTtlLeaves(opts: {
481
509
  const { db, logger, journeyRegistry, bucket } = opts;
482
510
 
483
511
  const expired = await db
484
- .select({ userId: bucketMemberships.userId })
512
+ .select({
513
+ userId: bucketMemberships.userId,
514
+ // Joined contact row → its id is the provenance pin for the leave emit
515
+ // (issue #608).
516
+ contactId: contacts.id,
517
+ })
485
518
  .from(bucketMemberships)
486
519
  .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
487
520
  .where(
@@ -501,7 +534,7 @@ async function reconcileBucketTtlLeaves(opts: {
501
534
  logger,
502
535
  journeyRegistry,
503
536
  bucket,
504
- userIds: expired.map((r) => r.userId),
537
+ leavers: expired,
505
538
  reason: "maxDwell",
506
539
  });
507
540
  }
@@ -518,11 +551,18 @@ async function bulkLeave(opts: {
518
551
  logger: Logger;
519
552
  journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
520
553
  bucket: BucketMeta;
521
- userIds: string[];
554
+ leavers: BucketLeaver[];
522
555
  /** Why these members leave — TTL passes "maxDwell", criteria passes "criteria". */
523
556
  reason: BucketLeaveReason;
524
557
  }): Promise<number> {
525
- const { db, logger, journeyRegistry, bucket, userIds, reason } = opts;
558
+ const { db, logger, journeyRegistry, bucket, leavers, reason } = opts;
559
+ const userIds = leavers.map((l) => l.userId);
560
+ // Membership key → resolved contact row id, for the emit's provenance pin.
561
+ // Safe to key by userId: the partial-active unique index guarantees one
562
+ // active row per (user, bucket), so a flipped row maps to exactly one leaver.
563
+ const contactIdByUser = new Map(
564
+ leavers.map((l) => [l.userId, l.contactId] as const),
565
+ );
526
566
 
527
567
  const dwellMs = bucket.minDwell ? durationToMs(bucket.minDwell) : 0;
528
568
  const dwellCutoff = dwellMs > 0 ? new Date(Date.now() - dwellMs) : null;
@@ -565,6 +605,9 @@ async function bulkLeave(opts: {
565
605
  bucket,
566
606
  userId: row.userId,
567
607
  userEmail: row.userEmail,
608
+ // Provenance pin from the leaver's already-joined contact row, so the
609
+ // leave re-ingest folds into it instead of minting a twin (issue #608).
610
+ contactId: contactIdByUser.get(row.userId),
568
611
  epoch: row.entryCount,
569
612
  source: "reconcile",
570
613
  reason,
@@ -643,6 +686,9 @@ async function reconcileBucketDwell(opts: {
643
686
  id: bucketMemberships.id,
644
687
  userId: bucketMemberships.userId,
645
688
  userEmail: bucketMemberships.userEmail,
689
+ // Joined contact row → its id is the provenance pin for the dwell emit
690
+ // (issue #608).
691
+ contactId: contacts.id,
646
692
  entryCount: bucketMemberships.entryCount,
647
693
  anchor: sql<Date>`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt})`,
648
694
  dwellState: bucketMemberships.dwellState,
@@ -704,6 +750,9 @@ async function reconcileBucketDwell(opts: {
704
750
  bucket,
705
751
  userId: m.userId,
706
752
  userEmail: m.userEmail,
753
+ // Provenance pin from the candidate's already-joined contact row
754
+ // (issue #608).
755
+ contactId: m.contactId,
707
756
  epoch: m.entryCount,
708
757
  source: "reconcile",
709
758
  dwellLabel: label,
@@ -857,6 +906,11 @@ async function reconcileBucketJoins(opts: {
857
906
  const baseQuery = db
858
907
  .select({
859
908
  userId: contactKey,
909
+ // The candidate scan reads straight off `contacts`, so the row id rides
910
+ // along as the provenance pin for the join emit (issue #608) —
911
+ // load-bearing for exactly the null-external_id contacts this coalesce
912
+ // join exists to include.
913
+ contactId: contacts.id,
860
914
  email: contacts.email,
861
915
  })
862
916
  .from(contacts)
@@ -934,6 +988,7 @@ async function reconcileBucketJoins(opts: {
934
988
  bucket,
935
989
  userId: candidate.userId,
936
990
  userEmail: candidate.email ?? null,
991
+ contactId: candidate.contactId,
937
992
  });
938
993
  if (transitioned) joined += 1;
939
994
  }
@@ -1035,8 +1090,10 @@ async function reconcileJoinOne(opts: {
1035
1090
  bucket: BucketMeta;
1036
1091
  userId: string;
1037
1092
  userEmail: string | null;
1093
+ /** Provenance pin for the join emit — see {@link BucketLeaver} (issue #608). */
1094
+ contactId?: string;
1038
1095
  }): Promise<boolean> {
1039
- const { db, logger, journeyRegistry, bucket, userId } = opts;
1096
+ const { db, logger, journeyRegistry, bucket, userId, contactId } = opts;
1040
1097
  // Normalized at the write site (audience-model.md wart #1) — the contact row
1041
1098
  // should already be normalized, but the membership keyspace must never
1042
1099
  // depend on that.
@@ -1081,6 +1138,8 @@ async function reconcileJoinOne(opts: {
1081
1138
  bucket,
1082
1139
  userId,
1083
1140
  userEmail,
1141
+ // Provenance pin from the candidate scan's contact row (issue #608).
1142
+ contactId,
1084
1143
  epoch,
1085
1144
  source: "reconcile",
1086
1145
  });