@hogsend/engine 0.56.0 → 0.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,7 @@ import {
7
7
  contactKey,
8
8
  normalizeEmail,
9
9
  resolveContact,
10
+ resolveViaAlias,
10
11
  } from "../../lib/contacts.js";
11
12
  import {
12
13
  InvalidUserTokenError,
@@ -14,7 +15,31 @@ import {
14
15
  } from "../../lib/user-token.js";
15
16
 
16
17
  export type FeedRecipient =
17
- | { ok: true; recipientKey: string; contactId?: string }
18
+ | {
19
+ ok: true;
20
+ recipientKey: string;
21
+ contactId?: string;
22
+ /**
23
+ * D1 REFUSAL verdict for the feed's OWN re-ingests (`emitMarkEvents`,
24
+ * `inapp.feed_cleared`), threaded into their ingest policy's `create`
25
+ * leg (`false` ⇒ `create: "refuse-on-miss"` — PRD 06 L5 row 8). It is
26
+ * the ONLY thing those engine-internal re-emits inherit from the
27
+ * originating request: their `allowMerge`/`trustedKinds` stay full
28
+ * server trust, because the resolver's caller is the engine with a
29
+ * server-derived subject, not the pk_ HTTP caller (L4).
30
+ * `false` ⇒ this recipient is an UNSEEN anonymous visitor: no live contact
31
+ * owns their anon id, so a re-ingest keyed on it is pure OBSERVATION and
32
+ * must not mint. Omitted (⇒ create as before) on every identity-asserting
33
+ * arm — a token/secret `userId`/`email`, or an anon id that already owns a
34
+ * row (`contactId` is then set and the resolve pins to it).
35
+ *
36
+ * Set ONLY on the anon arm, where D8 holds: `recipientKey` IS the raw
37
+ * `anonymousId`, which is exactly the key the create arm would have made
38
+ * canonical, so a refused re-ingest and a creating one agree byte-for-byte
39
+ * and no history is stranded.
40
+ */
41
+ allowCreate?: boolean;
42
+ }
18
43
  | { ok: false; status: 400 | 403; error: string };
19
44
 
20
45
  export interface FeedRecipientParams {
@@ -116,10 +141,27 @@ export async function resolveFeedRecipient(
116
141
  ),
117
142
  )
118
143
  .limit(1);
144
+ // PRD 03 T5: a SECOND device's anon id lives ONLY as an identity row —
145
+ // the column holds the first device's. Without this fallback the resolver
146
+ // read the person as having no contact, forced `allowCreate: false`, and
147
+ // the device's mark/clear re-ingests were refused: the events stranded
148
+ // under the raw device id instead of folding into the contact. Trust-wise
149
+ // this is the same class as the column hit: the id is the caller's OWN
150
+ // browser id held by the contact, never an identified canonical key (those
151
+ // were 403'd by `collidesWithIdentified` above).
152
+ const contactId =
153
+ anonRow[0]?.id ??
154
+ (await resolveViaAlias(db, "anonymous", params.anonymousId))?.id;
119
155
  return {
120
156
  ok: true,
121
157
  recipientKey: params.anonymousId,
122
- contactId: anonRow[0]?.id,
158
+ contactId,
159
+ // No row to pin to ⇒ the provenance pin cannot save us: an unpinned
160
+ // re-ingest keyed on this anon id would MINT `external_id = <anonId>`,
161
+ // the very ghost `sendFeedItem` just refused (D1/PRD 02 site 3), and the
162
+ // next poll would 403 on it. Refuse instead — legal here and only here,
163
+ // because `recipientKey` is the raw `anonymousId` (D8).
164
+ ...(contactId ? {} : { allowCreate: false }),
123
165
  };
124
166
  }
125
167
 
@@ -317,6 +317,17 @@ async function applyListSubscription(opts: {
317
317
  db,
318
318
  userId,
319
319
  email,
320
+ // PRD 06 T3 (L5 row 12): a subscribe/unsubscribe reaches here only
321
+ // server-trusted or token-proven (`gatePublishableIdentity` at both call
322
+ // sites) and the body carries no `anonymousId`, so the clamp would be
323
+ // inert — `allowMerge: "any"` is behaviour-identical, and the narrow
324
+ // `trustedKinds` is the honest statement of what this caller may
325
+ // assert. Declared now, enforced by T5.
326
+ policy: {
327
+ create: "on-miss",
328
+ allowMerge: "any",
329
+ trustedKinds: ["external", "email"],
330
+ },
320
331
  });
321
332
 
322
333
  emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
@@ -499,6 +510,16 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
499
510
  db,
500
511
  userId,
501
512
  email,
513
+ // PRD 06 T3 (L5 row 13): same trust statement as the subscribe path —
514
+ // server or token-proven caller, no `anonymousId` in the body (the
515
+ // handler 400s without an email/userId), so `allowMerge: "any"` is
516
+ // behaviour-identical and the narrow `trustedKinds` is honest. Enforced
517
+ // by T5.
518
+ policy: {
519
+ create: "on-miss",
520
+ allowMerge: "any",
521
+ trustedKinds: ["external", "email"],
522
+ },
502
523
  });
503
524
 
504
525
  emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
@@ -28,8 +28,9 @@ import {
28
28
  * a concrete userId. `visitor_kind = 'token'`.
29
29
  * - a raw `anonymousId` is PROVENANCE-ONLY: rejected if it collides with an
30
30
  * identified contact's canonical key (`collidesWithIdentified`), and the
31
- * bus ingest runs under `restrictToAnonymous` — it can never attach to /
32
- * merge into an identified contact. `visitor_kind = 'anon'`.
31
+ * bus ingest declares the anon-clamped policy (`allowMerge:
32
+ * "anonymous-only"`) — it can never attach to / merge into an identified
33
+ * contact. `visitor_kind = 'anon'`.
33
34
  * - a bare asserted email/userId is not even in the schema.
34
35
  * - INVARIANT: nothing the ref resolves to (above all `links.distinct_id`)
35
36
  * ever enters the contact resolver as a subject — identity inputs come
@@ -198,7 +199,45 @@ export const arriveRouter = new OpenAPIHono<AppEnv>().openapi(
198
199
  hatchet,
199
200
  logger,
200
201
  analytics,
201
- restrictToAnonymous: !isToken,
202
+ // PRD 06 T3 (L5 rows 6-7): trust is DECLARED by this caller, not
203
+ // re-inferred inside the resolver.
204
+ //
205
+ // `create` — the D1 creation guard. An anon arrival is pure
206
+ // OBSERVATION — a visitor landing from a tracked hit with nothing but
207
+ // a browser id is not grounds for a CRM row. The stamp, the
208
+ // `link.arrived` event and the outbound emit all still happen (D2);
209
+ // only the `contacts` INSERT is refused. A token leg is a
210
+ // server-minted identity assertion and creates exactly as before. D8
211
+ // holds on both legs: the ingest key below is `userId` on the token
212
+ // leg and `anonymousId` on the anon one, and `stamp.visitorDistinctId`
213
+ // is guaranteed non-empty by the guard above — both are canonical-key
214
+ // shapes, so the refusal keys history on the same string the create
215
+ // arm would have.
216
+ //
217
+ // POLARITY — the one place in this stack where the two legs are
218
+ // OPPOSITES rather than coinciding: the anon leg clamps merges
219
+ // (`allowMerge: "anonymous-only"`, no token) while REFUSING creation;
220
+ // the token leg permits creation while never clamping. They read as
221
+ // complements here only because the token is simultaneously what makes
222
+ // a merge safe and what makes creation warranted; collapsing the two
223
+ // legs into one is forbidden (they answer different questions and
224
+ // diverge at every other call site).
225
+ //
226
+ // `trustedKinds` — derived from the first-write-wins STAMP, never the
227
+ // current request: exactly one key per leg, matching the declared
228
+ // kinds (`userId`/external on the token leg, `anonymousId` on the anon
229
+ // one). Declared now, enforced by T5.
230
+ policy: isToken
231
+ ? {
232
+ create: "on-miss",
233
+ allowMerge: "any",
234
+ trustedKinds: ["anonymous", "external"],
235
+ }
236
+ : {
237
+ create: "refuse-on-miss",
238
+ allowMerge: "anonymous-only",
239
+ trustedKinds: ["anonymous"],
240
+ },
202
241
  event: {
203
242
  event: LINK_ARRIVED,
204
243
  ...(isToken
package/src/worker.ts CHANGED
@@ -31,6 +31,10 @@ import {
31
31
  reapDueWebhookDeliveriesTask,
32
32
  } from "./workflows/deliver-webhook.js";
33
33
  import { dispatchConversionTask } from "./workflows/dispatch-conversion.js";
34
+ import {
35
+ enqueueIdentityAliasBackfill,
36
+ identityAliasBackfillTask,
37
+ } from "./workflows/identity-alias-backfill.js";
34
38
  import { impactDigestTask } from "./workflows/impact-digest.js";
35
39
  import { importContactsTask } from "./workflows/import-contacts.js";
36
40
  import { importSuppressionsTask } from "./workflows/import-suppressions.js";
@@ -132,6 +136,7 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
132
136
  impactDigestTask,
133
137
  bucketReconcileTask,
134
138
  bucketBackfillTask,
139
+ identityAliasBackfillTask,
135
140
  crmReconcileTask,
136
141
  dispatchConversionTask,
137
142
  ...journeyTasks,
@@ -228,6 +233,19 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
228
233
  });
229
234
  });
230
235
 
236
+ // PRD 02: fill `contact_aliases` once per deployment (skipped when a
237
+ // non-failed job record exists). Same fire-and-forget, best-effort stance
238
+ // as the bucket backfills above; internally try/caught, `.catch` is the
239
+ // belt-and-suspenders.
240
+ enqueueIdentityAliasBackfill({
241
+ db: container.db,
242
+ logger: container.logger,
243
+ }).catch((err) => {
244
+ container.logger.warn("Identity alias backfill enqueue (boot) failed", {
245
+ error: err instanceof Error ? err.message : String(err),
246
+ });
247
+ });
248
+
231
249
  // Reconcile code-defined campaigns into scheduled rows (create / sync /
232
250
  // expire — see `reconcileDefinedCampaigns`). Same fire-and-forget stance
233
251
  // as the bucket backfills above: best-effort, never blocks the listener.
@@ -353,7 +353,7 @@ async function reevalLeaves(opts: {
353
353
  entryCount: bucketMemberships.entryCount,
354
354
  })
355
355
  .from(bucketMemberships)
356
- .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
356
+ .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
357
357
  .where(
358
358
  and(
359
359
  eq(bucketMemberships.bucketId, bucket.id),
@@ -118,6 +118,14 @@ export const bucketReconcileTask = hatchet.task({
118
118
  if (!timeBased && !bucket.maxDwell && !hasDwell) continue;
119
119
 
120
120
  try {
121
+ // ONE-SHOT COHORT CLAIM — the first-tick guard for the join-key fix.
122
+ // Until this bucket has been claimed, its previously-invisible members
123
+ // (canonical key != external_id) carry age clocks that ran unwatched,
124
+ // so every age-driven emission is due AT ONCE. Reset those clocks
125
+ // instead of firing the backlog, then skip this bucket for this tick so
126
+ // no pass can act on a cohort mid-reset. The next tick is normal.
127
+ if (await claimCoalesceCohort({ db, logger, bucket })) continue;
128
+
121
129
  if (timeBased) {
122
130
  reconciled += await reconcileBucketLeaves({
123
131
  db,
@@ -209,6 +217,15 @@ export interface BucketArmExpiryInput extends JsonObject {
209
217
  armedExpiresAt: string;
210
218
  /** ms from arming to the deadline (the durable sleep). */
211
219
  msUntilExpiry: number;
220
+ /**
221
+ * D1 creation guard inherited from the ingest that armed this timer, present
222
+ * ONLY when that ingest REFUSED (`armExpiryTimer` omits the key otherwise, so
223
+ * an unset value keeps today's create-by-default semantics and no JSON null
224
+ * crosses the wire). Forwarded into the leave `emitBucketTransition` below —
225
+ * the one thing that keeps the woken timer from minting the phantom the
226
+ * synchronous emits already refuse (issue #608, D11 category (iii)).
227
+ */
228
+ allowCreate?: boolean;
212
229
  }
213
230
 
214
231
  export const bucketExpiryTask = hatchet.durableTask({
@@ -290,6 +307,15 @@ export const bucketExpiryTask = hatchet.durableTask({
290
307
  // unforgeable engine-internal provenance the pin requires. A value
291
308
  // lookup here would re-implement exactly the resolution the pin exists
292
309
  // to bypass. Degrades to the pre-pin resolve.
310
+ //
311
+ // The inherited `allowCreate` IS admissible over that same forgeable
312
+ // channel, because its two values are not symmetric in blast radius: a
313
+ // forged `false` can only ever SUPPRESS a write, costing at most a
314
+ // contact the refused ingest would not have created anyway, while a
315
+ // forged `true` is byte-identical to the default this task already had.
316
+ // Neither can fold a transition into a victim's row the way a forged id
317
+ // can — so the boolean rides, and the id stays off the wire.
318
+ allowCreate: input.allowCreate,
293
319
  epoch: flipped.entryCount,
294
320
  source: "reconcile",
295
321
  // Fast-expiry is a criteria re-confirm leave (Section 6.7).
@@ -403,7 +429,7 @@ async function selectEventLeavers(
403
429
  })
404
430
  .from(members)
405
431
  .leftJoin(counted, eq(members.userId, counted.userId))
406
- .innerJoin(contacts, eq(contacts.externalId, members.userId))
432
+ .innerJoin(contacts, eq(contactKeySql(), members.userId))
407
433
  .where(isNull(contacts.deletedAt));
408
434
 
409
435
  return rows
@@ -438,7 +464,7 @@ async function reconcileCompositeLeaves(opts: {
438
464
  properties: contacts.properties,
439
465
  })
440
466
  .from(bucketMemberships)
441
- .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
467
+ .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
442
468
  .where(
443
469
  and(
444
470
  eq(bucketMemberships.bucketId, bucket.id),
@@ -516,7 +542,7 @@ async function reconcileBucketTtlLeaves(opts: {
516
542
  contactId: contacts.id,
517
543
  })
518
544
  .from(bucketMemberships)
519
- .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
545
+ .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
520
546
  .where(
521
547
  and(
522
548
  eq(bucketMemberships.bucketId, bucket.id),
@@ -646,6 +672,105 @@ async function bulkLeave(opts: {
646
672
  * `floor((sweepInstant - anchor) / offsetMs)` (gap-stable, NOT a fire count). For
647
673
  * `after` the ordinal is always 1 (one-shot).
648
674
  */
675
+ /**
676
+ * ONE-SHOT COHORT CLAIM — the first-tick guard for the join-key correction.
677
+ *
678
+ * The leave/dwell passes used to join `contacts.external_id`, while memberships
679
+ * key on the canonical `external_id ?? anonymous_id ?? id`. Every member whose
680
+ * contact carries a NULL `external_id` (email-only, keyed on its uuid;
681
+ * anonymous, keyed on its `anonymous_id`) was therefore enrollable — the join
682
+ * path has always scanned on the coalesce key — but never left, never
683
+ * dwell-fired, never re-evaluated. Their membership-age clocks ran unwatched,
684
+ * frequently for months.
685
+ *
686
+ * Correcting the join makes that whole cohort due AT ONCE. A dwell reaction is
687
+ * a full journey (same `(user, ctx)` shape as a journey `run`), so it can
688
+ * `sendEmail` — the first tick would otherwise deliver a backlog of months-old
689
+ * lifecycle mail to real recipients, which is worse than the bug being fixed.
690
+ *
691
+ * So the first tick RESETS the cohort's age clocks to now instead: the dwell
692
+ * anchor moves to this instant, dwell stamps clear, and a `maxDwell` TTL is
693
+ * re-armed a full window out. Nothing is emitted and nothing is silently
694
+ * swallowed — every age-driven emission still happens, just on an honest
695
+ * schedule measured from the moment the cron could first see the member.
696
+ *
697
+ * Deliberately NOT deferred: criteria-driven leaves. Those evaluate against
698
+ * present-day events and properties, so they are timely rather than stale, and
699
+ * suppressing them would hold members in a bucket they no longer match.
700
+ *
701
+ * Returns `true` ONLY when rows were actually reset — the caller then SKIPS
702
+ * this bucket's passes for this tick so nothing acts on a cohort mid-reset.
703
+ * An EMPTY cohort (the overwhelmingly common case: no email-only or anonymous
704
+ * members, and every deployment that never had any) returns `false` and the
705
+ * tick proceeds normally, so this guard costs nothing where it protects
706
+ * nothing. Either way the claim is recorded on
707
+ * `bucket_configs.coalesceClaimedAt` and never runs again for this bucket:
708
+ * once the join is correct, no member can become stranded, so a member
709
+ * enrolled later starts its age clocks under a cron that can already see it.
710
+ */
711
+ async function claimCoalesceCohort(opts: {
712
+ db: Database;
713
+ logger: Logger;
714
+ bucket: BucketMeta;
715
+ }): Promise<boolean> {
716
+ const { db, logger, bucket } = opts;
717
+
718
+ const config = await db.query.bucketConfigs.findFirst({
719
+ where: eq(bucketConfigs.bucketId, bucket.id),
720
+ });
721
+ if (config?.coalesceClaimedAt != null) return false;
722
+
723
+ const now = new Date();
724
+ const maxDwellMs = bucket.maxDwell ? durationToMs(bucket.maxDwell) : null;
725
+
726
+ // The cohort is exactly "resolves by the canonical key, but NOT by
727
+ // external_id" — the set the old join could not see. A correlated EXISTS
728
+ // keeps this a single UPDATE; the columns mirror the age clocks the
729
+ // dwell/TTL passes read (`coalesce(dwellAnchorAt, enteredAt)` and
730
+ // `maxDwellAt`).
731
+ const claimed = await db
732
+ .update(bucketMemberships)
733
+ .set({
734
+ dwellAnchorAt: now,
735
+ dwellState: {},
736
+ ...(maxDwellMs != null
737
+ ? { maxDwellAt: new Date(now.getTime() + maxDwellMs) }
738
+ : {}),
739
+ lastEvaluatedAt: now,
740
+ updatedAt: now,
741
+ })
742
+ .where(
743
+ and(
744
+ eq(bucketMemberships.bucketId, bucket.id),
745
+ eq(bucketMemberships.status, "active"),
746
+ isNull(bucketMemberships.deletedAt),
747
+ sql`exists (
748
+ select 1 from contacts c
749
+ where coalesce(c.external_id, c.anonymous_id, c.id::text) = ${bucketMemberships.userId}
750
+ and c.deleted_at is null
751
+ and c.external_id is null
752
+ )`,
753
+ ),
754
+ )
755
+ .returning({ id: bucketMemberships.id });
756
+
757
+ await db
758
+ .insert(bucketConfigs)
759
+ .values({ bucketId: bucket.id, coalesceClaimedAt: now })
760
+ .onConflictDoUpdate({
761
+ target: bucketConfigs.bucketId,
762
+ set: { coalesceClaimedAt: now, updatedAt: now },
763
+ });
764
+
765
+ if (claimed.length === 0) return false;
766
+
767
+ logger.info("Bucket coalesce-key cohort claimed — age clocks reset", {
768
+ bucketId: bucket.id,
769
+ memberships: claimed.length,
770
+ });
771
+ return true;
772
+ }
773
+
649
774
  async function reconcileBucketDwell(opts: {
650
775
  db: Database;
651
776
  logger: Logger;
@@ -694,7 +819,7 @@ async function reconcileBucketDwell(opts: {
694
819
  dwellState: bucketMemberships.dwellState,
695
820
  })
696
821
  .from(bucketMemberships)
697
- .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
822
+ .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
698
823
  .where(
699
824
  and(
700
825
  eq(bucketMemberships.bucketId, bucket.id),