@hogsend/engine 0.56.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +5 -10
- package/src/buckets/check-membership.ts +78 -6
- package/src/index.ts +31 -1
- package/src/journeys/execute-journey-run.ts +56 -1
- package/src/journeys/journey-context.ts +59 -0
- package/src/lib/bucket-emit.test.ts +83 -2
- package/src/lib/bucket-emit.ts +16 -0
- package/src/lib/contacts.ts +1031 -141
- package/src/lib/crm-ingest.ts +9 -0
- package/src/lib/feed.ts +64 -3
- package/src/lib/identity-service.ts +15 -2
- package/src/lib/ingestion.ts +180 -14
- package/src/lib/tracking-events.ts +33 -6
- package/src/routes/admin/agent.ts +26 -0
- package/src/routes/admin/contacts.ts +65 -5
- package/src/routes/admin/identity.ts +197 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/contacts/index.ts +50 -7
- package/src/routes/events/index.ts +94 -6
- package/src/routes/feed/index.ts +36 -0
- package/src/routes/feed/recipient.ts +44 -2
- package/src/routes/lists/index.ts +21 -0
- package/src/routes/tracking/arrive.ts +42 -3
- package/src/worker.ts +18 -0
- package/src/workflows/bucket-backfill.ts +1 -1
- package/src/workflows/bucket-reconcile.ts +129 -4
- package/src/workflows/identity-alias-backfill.ts +448 -0
- package/src/workflows/import-contacts.ts +10 -0
- package/src/workflows/send-campaign.ts +31 -13
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hogsend/engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.57.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -53,18 +53,18 @@
|
|
|
53
53
|
"svix": "^1.95.1",
|
|
54
54
|
"winston": "^3.19.0",
|
|
55
55
|
"zod": "^4.4.3",
|
|
56
|
-
"@hogsend/attribution": "^0.
|
|
57
|
-
"@hogsend/core": "^0.
|
|
58
|
-
"@hogsend/db": "^0.
|
|
59
|
-
"@hogsend/email": "^0.
|
|
60
|
-
"@hogsend/plugin-posthog": "^0.
|
|
61
|
-
"@hogsend/plugin-resend": "^0.
|
|
62
|
-
"@hogsend/sms": "^0.
|
|
56
|
+
"@hogsend/attribution": "^0.57.0",
|
|
57
|
+
"@hogsend/core": "^0.57.0",
|
|
58
|
+
"@hogsend/db": "^0.57.0",
|
|
59
|
+
"@hogsend/email": "^0.57.0",
|
|
60
|
+
"@hogsend/plugin-posthog": "^0.57.0",
|
|
61
|
+
"@hogsend/plugin-resend": "^0.57.0",
|
|
62
|
+
"@hogsend/sms": "^0.57.0"
|
|
63
63
|
},
|
|
64
64
|
"optionalDependencies": {
|
|
65
|
-
"@hogsend/plugin-apollo": "^0.
|
|
66
|
-
"@hogsend/plugin-postmark": "^0.
|
|
67
|
-
"@hogsend/plugin-twilio": "^0.
|
|
65
|
+
"@hogsend/plugin-apollo": "^0.57.0",
|
|
66
|
+
"@hogsend/plugin-postmark": "^0.57.0",
|
|
67
|
+
"@hogsend/plugin-twilio": "^0.57.0"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/node": "^22.15.3",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { bucketMemberships, contacts, type Database } from "@hogsend/db";
|
|
2
2
|
import { and, count as countFn, eq, gt, isNull } from "drizzle-orm";
|
|
3
|
+
import { contactKeySql } from "../lib/contacts.js";
|
|
3
4
|
import { getDb } from "../lib/db.js";
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -74,7 +75,7 @@ export function createBucketAccessor(
|
|
|
74
75
|
const rows = await db
|
|
75
76
|
.select({ value: countFn() })
|
|
76
77
|
.from(bucketMemberships)
|
|
77
|
-
.innerJoin(contacts, eq(
|
|
78
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
78
79
|
.where(
|
|
79
80
|
and(
|
|
80
81
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -98,7 +99,7 @@ export function createBucketAccessor(
|
|
|
98
99
|
const rows = await db
|
|
99
100
|
.select({ id: bucketMemberships.id })
|
|
100
101
|
.from(bucketMemberships)
|
|
101
|
-
.innerJoin(contacts, eq(
|
|
102
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
102
103
|
.where(
|
|
103
104
|
and(
|
|
104
105
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -144,10 +145,7 @@ export function createBucketAccessor(
|
|
|
144
145
|
entryCount: bucketMemberships.entryCount,
|
|
145
146
|
})
|
|
146
147
|
.from(bucketMemberships)
|
|
147
|
-
.innerJoin(
|
|
148
|
-
contacts,
|
|
149
|
-
eq(contacts.externalId, bucketMemberships.userId),
|
|
150
|
-
)
|
|
148
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
151
149
|
.where(and(...conditions))
|
|
152
150
|
.orderBy(bucketMemberships.id)
|
|
153
151
|
// +1 peek to detect a next page.
|
|
@@ -155,10 +153,7 @@ export function createBucketAccessor(
|
|
|
155
153
|
db
|
|
156
154
|
.select({ value: countFn() })
|
|
157
155
|
.from(bucketMemberships)
|
|
158
|
-
.innerJoin(
|
|
159
|
-
contacts,
|
|
160
|
-
eq(contacts.externalId, bucketMemberships.userId),
|
|
161
|
-
)
|
|
156
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
162
157
|
.where(
|
|
163
158
|
and(
|
|
164
159
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -9,7 +9,7 @@ import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
|
9
9
|
import { bucketMemberships, contacts, type Database } from "@hogsend/db";
|
|
10
10
|
import { and, desc, eq, isNotNull, isNull } from "drizzle-orm";
|
|
11
11
|
import { emitBucketTransition } from "../lib/bucket-emit.js";
|
|
12
|
-
import { normalizeEmailOrNull } from "../lib/contacts.js";
|
|
12
|
+
import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
|
|
13
13
|
import type { Logger } from "../lib/logger.js";
|
|
14
14
|
import {
|
|
15
15
|
BUCKET_EVENT_PREFIX,
|
|
@@ -61,6 +61,18 @@ export async function checkBucketMembership(opts: {
|
|
|
61
61
|
* contact degrades to the pin-less emit.
|
|
62
62
|
*/
|
|
63
63
|
contactId?: string;
|
|
64
|
+
/**
|
|
65
|
+
* D1 creation guard, INHERITED from the originating ingest and forwarded into
|
|
66
|
+
* every `emitBucketTransition` (and from there into both of its re-ingests).
|
|
67
|
+
* `ingestEvent` passes `false` exactly when its own resolve REFUSED, i.e.
|
|
68
|
+
* when there is no `contactId` to pin with — because the alternative,
|
|
69
|
+
* degrading the pin to `contactId ?? undefined`, makes the transition
|
|
70
|
+
* re-ingest treat the anon canonical key as an EXTERNAL key and mint an
|
|
71
|
+
* `external_id = <anonId>` phantom twin (issue #608 from the other side).
|
|
72
|
+
* Bucket evaluation itself is UNAFFECTED: `bucket_memberships` is text-keyed
|
|
73
|
+
* with no contact FK, so a contactless subject is still a first-class member.
|
|
74
|
+
*/
|
|
75
|
+
allowCreate?: boolean;
|
|
64
76
|
event: string;
|
|
65
77
|
/**
|
|
66
78
|
* D2: the event payload — candidate-narrowing ONLY. It NO LONGER participates
|
|
@@ -90,6 +102,7 @@ export async function checkBucketMembership(opts: {
|
|
|
90
102
|
logger,
|
|
91
103
|
userId,
|
|
92
104
|
contactId,
|
|
105
|
+
allowCreate,
|
|
93
106
|
event,
|
|
94
107
|
eventProperties,
|
|
95
108
|
contactProperties: contactPropertiesPatch,
|
|
@@ -170,13 +183,31 @@ export async function checkBucketMembership(opts: {
|
|
|
170
183
|
let storedContactProps: Record<string, unknown> = {};
|
|
171
184
|
let contactDeleted = false;
|
|
172
185
|
if (needsContactState) {
|
|
186
|
+
// Match on the CANONICAL key, not `external_id` alone. Memberships and
|
|
187
|
+
// events are keyed on `coalesce(external_id, anonymous_id, id)`
|
|
188
|
+
// (`contactKeySql`), so an email-only contact is keyed on its uuid and an
|
|
189
|
+
// anonymous one on its `anonymous_id` — both have a NULL `external_id` and
|
|
190
|
+
// neither was ever found here. The cron's join scan already made exactly
|
|
191
|
+
// this correction for the same reason (`bucket-reconcile.ts`, "joining on
|
|
192
|
+
// contacts.externalId would … silently drop exactly the dormant email-only
|
|
193
|
+
// contacts this cron exists to reconcile"); this is that fix applied to the
|
|
194
|
+
// real-time path, which was missed.
|
|
195
|
+
//
|
|
196
|
+
// Two things were wrong, and the second is the serious one:
|
|
197
|
+
// 1. property criteria evaluated against `{}` instead of the contact's
|
|
198
|
+
// real state, so a property leg silently answered "absent" for every
|
|
199
|
+
// such person;
|
|
200
|
+
// 2. `contactDeleted` stayed false because the row was never found, so
|
|
201
|
+
// the GDPR guard below — "never (re-)evaluate or emit for a
|
|
202
|
+
// soft-deleted contact" — did not fire for them. A soft-deleted
|
|
203
|
+
// email-only contact could still transition buckets and emit.
|
|
173
204
|
const [contact] = await db
|
|
174
205
|
.select({
|
|
175
206
|
properties: contacts.properties,
|
|
176
207
|
deletedAt: contacts.deletedAt,
|
|
177
208
|
})
|
|
178
209
|
.from(contacts)
|
|
179
|
-
.where(eq(
|
|
210
|
+
.where(eq(contactKeySql(), userId))
|
|
180
211
|
.limit(1);
|
|
181
212
|
if (contact) {
|
|
182
213
|
storedContactProps =
|
|
@@ -234,6 +265,7 @@ export async function checkBucketMembership(opts: {
|
|
|
234
265
|
userId,
|
|
235
266
|
userEmail,
|
|
236
267
|
contactId,
|
|
268
|
+
allowCreate,
|
|
237
269
|
});
|
|
238
270
|
if (transition) transitions.push(transition);
|
|
239
271
|
} else if (wasMember && isMember) {
|
|
@@ -253,6 +285,7 @@ export async function checkBucketMembership(opts: {
|
|
|
253
285
|
userId,
|
|
254
286
|
userEmail,
|
|
255
287
|
contactId,
|
|
288
|
+
allowCreate,
|
|
256
289
|
});
|
|
257
290
|
if (transition) transitions.push(transition);
|
|
258
291
|
}
|
|
@@ -272,6 +305,8 @@ async function handleJoin(opts: {
|
|
|
272
305
|
userEmail: string | null;
|
|
273
306
|
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
274
307
|
contactId?: string;
|
|
308
|
+
/** Inherited creation guard for the emit — see `checkBucketMembership`. */
|
|
309
|
+
allowCreate?: boolean;
|
|
275
310
|
}): Promise<BucketTransition | null> {
|
|
276
311
|
const {
|
|
277
312
|
db,
|
|
@@ -282,6 +317,7 @@ async function handleJoin(opts: {
|
|
|
282
317
|
userId,
|
|
283
318
|
userEmail,
|
|
284
319
|
contactId,
|
|
320
|
+
allowCreate,
|
|
285
321
|
} = opts;
|
|
286
322
|
|
|
287
323
|
// entryCount ordinal = 1 + count of ALL prior memberships (active + left) for
|
|
@@ -324,6 +360,10 @@ async function handleJoin(opts: {
|
|
|
324
360
|
// is written. The cron remains the authoritative backstop, so a push failure
|
|
325
361
|
// is best-effort. We arm against the persisted expiresAt so the timer's CAS on
|
|
326
362
|
// wake matches the row (or no-ops if a later event re-armed the window).
|
|
363
|
+
// `allowCreate` rides along because the timer's eventual leave emit is a
|
|
364
|
+
// re-ingest DERIVED from this (possibly refused) event — D11 applies to the
|
|
365
|
+
// ASYNCHRONOUS producer exactly as it does to the two synchronous ones. The
|
|
366
|
+
// `contactId` pin deliberately does NOT (see `bucketExpiryTask`).
|
|
327
367
|
if (bucket.fastExpiry && expiresAt) {
|
|
328
368
|
await armExpiryTimer({
|
|
329
369
|
hatchet,
|
|
@@ -333,6 +373,7 @@ async function handleJoin(opts: {
|
|
|
333
373
|
userId,
|
|
334
374
|
userEmail,
|
|
335
375
|
expiresAt,
|
|
376
|
+
allowCreate,
|
|
336
377
|
});
|
|
337
378
|
}
|
|
338
379
|
|
|
@@ -351,8 +392,10 @@ async function handleJoin(opts: {
|
|
|
351
392
|
userEmail,
|
|
352
393
|
// Pin the re-ingest to the already-resolved contact row so an
|
|
353
394
|
// anonymous-only contact's transition folds in instead of minting a
|
|
354
|
-
// phantom twin (issue #608).
|
|
395
|
+
// phantom twin (issue #608). With no row to pin to, the originating
|
|
396
|
+
// ingest's refusal is inherited instead — never a degraded pin.
|
|
355
397
|
contactId,
|
|
398
|
+
allowCreate,
|
|
356
399
|
epoch,
|
|
357
400
|
source: "event",
|
|
358
401
|
});
|
|
@@ -378,6 +421,8 @@ async function handleLeave(opts: {
|
|
|
378
421
|
userEmail: string | null;
|
|
379
422
|
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
380
423
|
contactId?: string;
|
|
424
|
+
/** Inherited creation guard for the emit — see `checkBucketMembership`. */
|
|
425
|
+
allowCreate?: boolean;
|
|
381
426
|
}): Promise<BucketTransition | null> {
|
|
382
427
|
const {
|
|
383
428
|
db,
|
|
@@ -389,6 +434,7 @@ async function handleLeave(opts: {
|
|
|
389
434
|
userId,
|
|
390
435
|
userEmail,
|
|
391
436
|
contactId,
|
|
437
|
+
allowCreate,
|
|
392
438
|
} = opts;
|
|
393
439
|
|
|
394
440
|
// minDwell DEFERS (never silently drops) the leave (Section 6.3). We set a
|
|
@@ -453,9 +499,11 @@ async function handleLeave(opts: {
|
|
|
453
499
|
bucket,
|
|
454
500
|
userId,
|
|
455
501
|
userEmail,
|
|
456
|
-
// Same provenance pin as the join emit — a
|
|
457
|
-
// identical path and would mint the same
|
|
502
|
+
// Same provenance pin (and same inherited refusal) as the join emit — a
|
|
503
|
+
// leave re-ingests through the identical path and would mint the same
|
|
504
|
+
// phantom twin (issue #608).
|
|
458
505
|
contactId,
|
|
506
|
+
allowCreate,
|
|
459
507
|
epoch: flipped.entryCount,
|
|
460
508
|
source: "event",
|
|
461
509
|
reason: "criteria",
|
|
@@ -539,6 +587,12 @@ function withinMinDwell(
|
|
|
539
587
|
* by `checkBucketMembership`, so arming does NOT re-enter bucket evaluation.
|
|
540
588
|
* Best-effort: the cron is the authoritative backstop, so a push failure is logged
|
|
541
589
|
* and swallowed.
|
|
590
|
+
*
|
|
591
|
+
* The arm payload is the ONLY channel between this join and the leave the woken
|
|
592
|
+
* task emits, so the inherited D1 refusal has to ride it — otherwise the timer's
|
|
593
|
+
* `emitBucketTransition` re-resolves the anon canonical key create-on-miss and
|
|
594
|
+
* mints the `external_id = <anonId>` phantom twin the synchronous emits were
|
|
595
|
+
* already taught to refuse (issue #608, D11 category (iii)).
|
|
542
596
|
*/
|
|
543
597
|
async function armExpiryTimer(opts: {
|
|
544
598
|
hatchet: HatchetClient;
|
|
@@ -548,8 +602,25 @@ async function armExpiryTimer(opts: {
|
|
|
548
602
|
userId: string;
|
|
549
603
|
userEmail: string | null;
|
|
550
604
|
expiresAt: Date;
|
|
605
|
+
/**
|
|
606
|
+
* D1 creation guard inherited from the originating ingest via `handleJoin`.
|
|
607
|
+
* Carried on the wire ONLY when `false`: an omitted key keeps today's
|
|
608
|
+
* create-by-default semantics for every producer that legitimately creates
|
|
609
|
+
* and keeps a JSON null off a Hatchet payload (the
|
|
610
|
+
* `...(contactId !== null ? { contactId } : {})` idiom at `ingestion.ts`).
|
|
611
|
+
*/
|
|
612
|
+
allowCreate?: boolean;
|
|
551
613
|
}): Promise<void> {
|
|
552
|
-
const {
|
|
614
|
+
const {
|
|
615
|
+
hatchet,
|
|
616
|
+
logger,
|
|
617
|
+
bucket,
|
|
618
|
+
rowId,
|
|
619
|
+
userId,
|
|
620
|
+
userEmail,
|
|
621
|
+
expiresAt,
|
|
622
|
+
allowCreate,
|
|
623
|
+
} = opts;
|
|
553
624
|
const msUntilExpiry = Math.max(0, expiresAt.getTime() - Date.now());
|
|
554
625
|
try {
|
|
555
626
|
await hatchet.events.push("bucket:arm-expiry", {
|
|
@@ -559,6 +630,7 @@ async function armExpiryTimer(opts: {
|
|
|
559
630
|
userEmail,
|
|
560
631
|
armedExpiresAt: expiresAt.toISOString(),
|
|
561
632
|
msUntilExpiry,
|
|
633
|
+
...(allowCreate === false ? { allowCreate: false } : {}),
|
|
562
634
|
});
|
|
563
635
|
} catch (err) {
|
|
564
636
|
logger.warn("Bucket fast-expiry arm failed (cron backstop covers it)", {
|
package/src/index.ts
CHANGED
|
@@ -394,7 +394,25 @@ export {
|
|
|
394
394
|
verifyConnectorState,
|
|
395
395
|
} from "./lib/connector-state.js";
|
|
396
396
|
// --- Contacts identity (resolve/create — used by connector member-link) ---
|
|
397
|
-
|
|
397
|
+
// `resolveContactNoCreate` is the refuse-on-miss sibling (D1): same resolution,
|
|
398
|
+
// but pure observation never mints a `contacts` row.
|
|
399
|
+
// `identifiedContactFilter` is the read-side counterpart (PRD 01): the single
|
|
400
|
+
// "has this person ever identified?" predicate behind `?identity=`.
|
|
401
|
+
// `deleteIdentityAliasesForContact` is the erasure hook (PRD 02 T1): a
|
|
402
|
+
// consumer-built deletion flow that soft-deletes `contacts` rows directly must
|
|
403
|
+
// call it too, or the erased person's identity keys survive in
|
|
404
|
+
// `contact_aliases`.
|
|
405
|
+
// `ResolvePolicy`/`IdentityKind` (PRD 06): the explicit caller-declared trust
|
|
406
|
+
// shape both resolver entry points accept via `policy` — the additive
|
|
407
|
+
// replacement for the deprecated `restrictToAnonymous`/`allowCreate` booleans.
|
|
408
|
+
export {
|
|
409
|
+
deleteIdentityAliasesForContact,
|
|
410
|
+
type IdentityKind,
|
|
411
|
+
identifiedContactFilter,
|
|
412
|
+
type ResolvePolicy,
|
|
413
|
+
resolveContactNoCreate,
|
|
414
|
+
resolveOrCreateContact,
|
|
415
|
+
} from "./lib/contacts.js";
|
|
398
416
|
// --- Conversion dispatch (plan §5.2): destinations registry + delivery ---
|
|
399
417
|
export {
|
|
400
418
|
ConversionDestinationRegistry,
|
|
@@ -564,6 +582,7 @@ export {
|
|
|
564
582
|
type IngestEvent,
|
|
565
583
|
type IngestResult,
|
|
566
584
|
ingestEvent,
|
|
585
|
+
ingestTransformResult,
|
|
567
586
|
} from "./lib/ingestion.js";
|
|
568
587
|
// --- Reconciled journey-lift helper (impact experiments D4.1) — the ONE
|
|
569
588
|
// implementation of the causal cohort math, shared by /lift (2a), /impact
|
|
@@ -921,6 +940,17 @@ export {
|
|
|
921
940
|
deliverWebhookTask,
|
|
922
941
|
reapDueWebhookDeliveriesTask,
|
|
923
942
|
} from "./workflows/deliver-webhook.js";
|
|
943
|
+
// --- Identity alias backfill (PRD 02): task + boot enqueue + parity probe ---
|
|
944
|
+
export {
|
|
945
|
+
type AliasParityRow,
|
|
946
|
+
enqueueIdentityAliasBackfill,
|
|
947
|
+
IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
948
|
+
type IdentityAliasBackfillInput,
|
|
949
|
+
type IdentityAliasBackfillResult,
|
|
950
|
+
identityAliasBackfillTask,
|
|
951
|
+
identityAliasParity,
|
|
952
|
+
runIdentityAliasBackfill,
|
|
953
|
+
} from "./workflows/identity-alias-backfill.js";
|
|
924
954
|
export {
|
|
925
955
|
buildImpactDigest,
|
|
926
956
|
detectLiftCrossings,
|
|
@@ -12,9 +12,10 @@ import {
|
|
|
12
12
|
journeyConfigs,
|
|
13
13
|
journeyStates,
|
|
14
14
|
} from "@hogsend/db";
|
|
15
|
-
import { and, eq, inArray, notInArray, sql } from "drizzle-orm";
|
|
15
|
+
import { and, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
|
16
16
|
import { getAnalytics } from "../lib/analytics-singleton.js";
|
|
17
17
|
import { blueprintGraphLock } from "../lib/blueprint-lock.js";
|
|
18
|
+
import { ALL_IDENTITY_KINDS } from "../lib/contacts.js";
|
|
18
19
|
import { getDb } from "../lib/db.js";
|
|
19
20
|
import {
|
|
20
21
|
checkEmailPreferences,
|
|
@@ -384,15 +385,69 @@ export async function executeJourneyRun(
|
|
|
384
385
|
// concurrent double-diversion dedups here even if it raced the
|
|
385
386
|
// row existence check above.
|
|
386
387
|
try {
|
|
388
|
+
// D11 — this re-ingest is DERIVED from the entry event, so it
|
|
389
|
+
// must inherit that event's creation verdict instead of
|
|
390
|
+
// re-resolving the key cold. Since PRD 02 site 1 a journey's
|
|
391
|
+
// `userId` is routinely a browser anon id owning NO contact row
|
|
392
|
+
// (`ingestEvent` pushes `userId: resolvedKey` and OMITS
|
|
393
|
+
// `contactId` on a refusal), and `findByKey` reads a bare
|
|
394
|
+
// `userId` as kind "external" — so a create-on-miss re-ingest
|
|
395
|
+
// here writes `external_id = <anonId>`, which is strictly WORSE
|
|
396
|
+
// than the ghost being removed: `collidesWithIdentified` returns
|
|
397
|
+
// true for it, so `resolveFeedRecipient` starts 403-ing the
|
|
398
|
+
// visitor out of their own bell.
|
|
399
|
+
//
|
|
400
|
+
// The verdict is DERIVED, never hardcoded — a held-out run for a
|
|
401
|
+
// real contact, or one asserting an email, must keep creating.
|
|
402
|
+
// This leg runs BEFORE the timezone `contact` fetch below, so
|
|
403
|
+
// provenance is resolved locally: the pushed `input.contactId`
|
|
404
|
+
// when the entry ingest resolved a row, else ONE indexed lookup
|
|
405
|
+
// for the live contact owning this canonical key on either
|
|
406
|
+
// identity column (`contactKey = external_id ?? anonymous_id`;
|
|
407
|
+
// the tz fetch reads `external_id` only, which misses an
|
|
408
|
+
// anon-keyed contact).
|
|
409
|
+
const subjectContactId =
|
|
410
|
+
input.contactId ??
|
|
411
|
+
(
|
|
412
|
+
await db.query.contacts.findFirst({
|
|
413
|
+
where: and(
|
|
414
|
+
isNull(contacts.deletedAt),
|
|
415
|
+
or(
|
|
416
|
+
eq(contacts.externalId, userId),
|
|
417
|
+
eq(contacts.anonymousId, userId),
|
|
418
|
+
),
|
|
419
|
+
),
|
|
420
|
+
columns: { id: true },
|
|
421
|
+
})
|
|
422
|
+
)?.id;
|
|
387
423
|
await ingestEvent({
|
|
388
424
|
db,
|
|
389
425
|
registry: getJourneyRegistrySingleton(),
|
|
390
426
|
hatchet,
|
|
391
427
|
logger,
|
|
428
|
+
// PRD 06 T4 (L5 row 26): an engine-internal lifecycle emit is
|
|
429
|
+
// a FULLY trusted caller (L4) — full `trustedKinds`, never a
|
|
430
|
+
// clamp. It inherits exactly ONE thing from the verdict: the
|
|
431
|
+
// `create` leg. Refuse ONLY when nothing was resolved AND
|
|
432
|
+
// nothing is asserted. A `userEmail` is a durable identity the
|
|
433
|
+
// caller asserted (D1), so refusing it would change behavior
|
|
434
|
+
// for identified users; anonymous runs carry `""` here
|
|
435
|
+
// (`ingestEvent` pushes `userEmail ?? ""`), correctly falsy.
|
|
436
|
+
policy: {
|
|
437
|
+
create:
|
|
438
|
+
subjectContactId || userEmail
|
|
439
|
+
? "on-miss"
|
|
440
|
+
: "refuse-on-miss",
|
|
441
|
+
allowMerge: "any",
|
|
442
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
443
|
+
},
|
|
392
444
|
event: {
|
|
393
445
|
event: "journey.heldout",
|
|
394
446
|
userId,
|
|
395
447
|
userEmail,
|
|
448
|
+
// Pin to the exact subject row so the re-emit folds in
|
|
449
|
+
// instead of value-probing a key that could mint a twin.
|
|
450
|
+
...(subjectContactId ? { contactId: subjectContactId } : {}),
|
|
396
451
|
eventProperties: {
|
|
397
452
|
journeyId: meta.id,
|
|
398
453
|
journeyName: meta.name,
|
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
notInArray,
|
|
48
48
|
sql,
|
|
49
49
|
} from "drizzle-orm";
|
|
50
|
+
import { ALL_IDENTITY_KINDS } from "../lib/contacts.js";
|
|
50
51
|
import { checkEmailPreferences } from "../lib/enrollment-guards.js";
|
|
51
52
|
import { countRecentSends } from "../lib/frequency-cap.js";
|
|
52
53
|
import { toSleepDuration } from "../lib/hatchet-duration.js";
|
|
@@ -1262,16 +1263,74 @@ export function createJourneyContext(
|
|
|
1262
1263
|
registerKey(boundary, idempotencyKey);
|
|
1263
1264
|
}
|
|
1264
1265
|
|
|
1266
|
+
// D11 — a `ctx.trigger` re-ingest is DERIVED from the run's own entry
|
|
1267
|
+
// event, so it must inherit that event's creation verdict rather than
|
|
1268
|
+
// re-resolve the key cold. Since PRD 02 site 1 a journey's `userId` is
|
|
1269
|
+
// routinely a browser anon id owning NO contact row (`ingestEvent` pushes
|
|
1270
|
+
// `userId: resolvedKey` and OMITS `contactId` on a refusal), and
|
|
1271
|
+
// `findByKey` reads a bare `userId` as kind "external" — so a
|
|
1272
|
+
// create-on-miss re-ingest here writes `external_id = <anonId>`, strictly
|
|
1273
|
+
// WORSE than the ghost being removed: `collidesWithIdentified` returns
|
|
1274
|
+
// true for it, so `resolveFeedRecipient` starts 403-ing the visitor out of
|
|
1275
|
+
// their own bell.
|
|
1276
|
+
//
|
|
1277
|
+
// The verdict is per-ARM, because the two arms have different subjects.
|
|
1278
|
+
//
|
|
1279
|
+
// SELF arm (`targetUserId === userId`) — the re-ingest really is derived
|
|
1280
|
+
// from this run's own entry event, so it inherits that event's verdict,
|
|
1281
|
+
// DERIVED (never hardcoded) from two signals already in scope:
|
|
1282
|
+
// - `config.contactId` — the run's resolved subject row id (pushed by
|
|
1283
|
+
// the entry ingest, else the enrollment's own contact lookup).
|
|
1284
|
+
// Present ⇒ the subject HAS a row: pin to it so the re-emit folds
|
|
1285
|
+
// into that exact row instead of value-probing a canonical key that
|
|
1286
|
+
// could mint a phantom twin.
|
|
1287
|
+
// - an asserted email (`targetEmail ?? userEmail`) — a durable identity
|
|
1288
|
+
// the caller is asserting (D1), so it keeps creating. Anonymous runs
|
|
1289
|
+
// carry `""` here (`ingestEvent` pushes `userEmail ?? ""`), correctly
|
|
1290
|
+
// falsy.
|
|
1291
|
+
// Neither ⇒ nothing resolved and nothing asserted: refuse, so re-ingesting
|
|
1292
|
+
// the run's own anon key cold cannot re-resolve it as an EXTERNAL key and
|
|
1293
|
+
// mint the `external_id = <anonId>` twin. The refusal only ever bites a
|
|
1294
|
+
// total MISS — a subject that already owns a contact still links/merges
|
|
1295
|
+
// exactly as before.
|
|
1296
|
+
//
|
|
1297
|
+
// CROSS-USER arm — the run's identity says NOTHING about the person the
|
|
1298
|
+
// author named, so neither signal is in scope for the target: the pin is
|
|
1299
|
+
// `undefined` BY CONSTRUCTION (pinning an author-overridden `userId` to
|
|
1300
|
+
// the run's subject row would misattribute the event), and the run's own
|
|
1301
|
+
// `userEmail` belongs to somebody else. Reading the verdict off them would
|
|
1302
|
+
// make an anonymous run refuse to mint a NAMED target — inverting D1,
|
|
1303
|
+
// under which `ctx.trigger({ userId })` is exactly the "server-side caller
|
|
1304
|
+
// explicitly asks" case — and would strip that target of everything gated
|
|
1305
|
+
// on a non-null `contact_id` (conversions, attribution credits, funnel
|
|
1306
|
+
// progress, deals). So this arm keeps creating, unconditionally.
|
|
1307
|
+
const isSelfTrigger = targetUserId === userId;
|
|
1308
|
+
const pinnedContactId = isSelfTrigger ? config.contactId : undefined;
|
|
1309
|
+
const assertsIdentity = isSelfTrigger
|
|
1310
|
+
? !!pinnedContactId || !!(targetEmail ?? userEmail)
|
|
1311
|
+
: true;
|
|
1312
|
+
|
|
1265
1313
|
const runIngest = () =>
|
|
1266
1314
|
ingestEvent({
|
|
1267
1315
|
db,
|
|
1268
1316
|
registry,
|
|
1269
1317
|
hatchet,
|
|
1270
1318
|
logger,
|
|
1319
|
+
// PRD 06 T4 (L5 row 25): an engine-internal re-emit is a FULLY
|
|
1320
|
+
// trusted caller regardless of what authenticated the originating
|
|
1321
|
+
// request (L4) — full `trustedKinds`, never a clamp. The ONE thing
|
|
1322
|
+
// it inherits from the verdict above is the `create` leg, keyed off
|
|
1323
|
+
// the same `assertsIdentity` derivation as before.
|
|
1324
|
+
policy: {
|
|
1325
|
+
create: assertsIdentity ? "on-miss" : "refuse-on-miss",
|
|
1326
|
+
allowMerge: "any",
|
|
1327
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
1328
|
+
},
|
|
1271
1329
|
event: {
|
|
1272
1330
|
event,
|
|
1273
1331
|
userId: targetUserId,
|
|
1274
1332
|
userEmail: targetEmail ?? userEmail,
|
|
1333
|
+
...(pinnedContactId ? { contactId: pinnedContactId } : {}),
|
|
1275
1334
|
eventProperties: properties ?? {},
|
|
1276
1335
|
value,
|
|
1277
1336
|
currency,
|
|
@@ -51,14 +51,20 @@ function genericBoundRegistry(): JourneyRegistry {
|
|
|
51
51
|
} as unknown as JourneyRegistry;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Recording ingest stub — captures each re-ingested event, plus the
|
|
56
|
+
* `allowCreate` guard, which is a SIBLING of `event` in the ingest opts (not a
|
|
57
|
+
* field on it) and so needs its own recorder.
|
|
58
|
+
*/
|
|
55
59
|
function recordingIngest() {
|
|
56
60
|
const events: IngestEvent[] = [];
|
|
61
|
+
const allowCreates: (boolean | undefined)[] = [];
|
|
57
62
|
const ingest: typeof ingestEvent = async (opts) => {
|
|
58
63
|
events.push(opts.event);
|
|
64
|
+
allowCreates.push(opts.allowCreate);
|
|
59
65
|
return { stored: true, exits: [], contactKey: opts.event.userId ?? "" };
|
|
60
66
|
};
|
|
61
|
-
return { events, ingest };
|
|
67
|
+
return { events, allowCreates, ingest };
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
// db/hatchet are never touched by the stubbed ingest; emitOutbound is
|
|
@@ -143,3 +149,78 @@ test("no contactId: degrades to exactly the pin-less re-ingest", async () => {
|
|
|
143
149
|
assert.equal(event.contactId, undefined);
|
|
144
150
|
}
|
|
145
151
|
});
|
|
152
|
+
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
// D1 — the INHERITED creation guard. When the originating ingest refused to
|
|
155
|
+
// mint, there is no `contactId` to pin with, and degrading the pin here is
|
|
156
|
+
// exactly what would mint an `external_id = <anonId>` phantom twin. So the
|
|
157
|
+
// refusal itself rides the re-ingests instead — and, like the pin, it must ride
|
|
158
|
+
// BOTH legs: forwarding it only on the alias would leave the ghost half-alive
|
|
159
|
+
// for any journey bound to the generic `bucket:<kind>` form.
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
test("allowCreate: false rides BOTH re-ingests (alias + generic)", async () => {
|
|
163
|
+
const { events, allowCreates, ingest } = recordingIngest();
|
|
164
|
+
|
|
165
|
+
await emitBucketTransition({
|
|
166
|
+
db,
|
|
167
|
+
registry: genericBoundRegistry(),
|
|
168
|
+
hatchet,
|
|
169
|
+
logger,
|
|
170
|
+
kind: "entered",
|
|
171
|
+
bucket,
|
|
172
|
+
userId: "anon-608",
|
|
173
|
+
userEmail: null,
|
|
174
|
+
allowCreate: false,
|
|
175
|
+
epoch: 1,
|
|
176
|
+
ingest,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
assert.equal(events.length, 2);
|
|
180
|
+
assert.deepEqual(allowCreates, [false, false]);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("allowCreate: false rides the leave re-ingests too", async () => {
|
|
184
|
+
const { events, allowCreates, ingest } = recordingIngest();
|
|
185
|
+
|
|
186
|
+
await emitBucketTransition({
|
|
187
|
+
db,
|
|
188
|
+
registry: genericBoundRegistry(),
|
|
189
|
+
hatchet,
|
|
190
|
+
logger,
|
|
191
|
+
kind: "left",
|
|
192
|
+
bucket,
|
|
193
|
+
userId: "anon-608",
|
|
194
|
+
userEmail: null,
|
|
195
|
+
allowCreate: false,
|
|
196
|
+
epoch: 2,
|
|
197
|
+
reason: "criteria",
|
|
198
|
+
ingest,
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
assert.equal(events.length, 2);
|
|
202
|
+
assert.deepEqual(allowCreates, [false, false]);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("no allowCreate: the creating producers are untouched", async () => {
|
|
206
|
+
const { allowCreates, ingest } = recordingIngest();
|
|
207
|
+
|
|
208
|
+
await emitBucketTransition({
|
|
209
|
+
db,
|
|
210
|
+
registry: genericBoundRegistry(),
|
|
211
|
+
hatchet,
|
|
212
|
+
logger,
|
|
213
|
+
kind: "entered",
|
|
214
|
+
bucket,
|
|
215
|
+
userId: "anon-608",
|
|
216
|
+
userEmail: null,
|
|
217
|
+
contactId: CONTACT_ID,
|
|
218
|
+
epoch: 1,
|
|
219
|
+
ingest,
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
// The reconcile cron and the backfill legitimately create; leaving the option
|
|
223
|
+
// unset must reach `ingestEvent` as `undefined` (its default-true path), not
|
|
224
|
+
// as a `false` someone helpfully filled in.
|
|
225
|
+
assert.deepEqual(allowCreates, [undefined, undefined]);
|
|
226
|
+
});
|
package/src/lib/bucket-emit.ts
CHANGED
|
@@ -51,6 +51,17 @@ export async function emitBucketTransition(opts: {
|
|
|
51
51
|
* the pin-less resolve (exactly the pre-pin behaviour).
|
|
52
52
|
*/
|
|
53
53
|
contactId?: string;
|
|
54
|
+
/**
|
|
55
|
+
* D1 creation guard for BOTH re-ingests below, inherited from the originating
|
|
56
|
+
* ingest via `checkBucketMembership`. `false` ⇒ the transition event stores
|
|
57
|
+
* and routes exactly as always, but mints no `contacts` row. This is the
|
|
58
|
+
* ONLY correct behavior when the producer has no `contactId` to pin with:
|
|
59
|
+
* degrading the pin instead would make the resolver read this transition's
|
|
60
|
+
* `userId` (the subject's canonical key, often its `anonymous_id`) as an
|
|
61
|
+
* EXTERNAL key and mint a phantom `external_id` twin (issue #608). Producers
|
|
62
|
+
* that legitimately create (the reconcile cron, backfill) leave it unset.
|
|
63
|
+
*/
|
|
64
|
+
allowCreate?: boolean;
|
|
54
65
|
epoch: number;
|
|
55
66
|
source?: BucketTransitionSource;
|
|
56
67
|
/** Carried on a `left` transition's properties → `ctx.reason`. */
|
|
@@ -80,6 +91,7 @@ export async function emitBucketTransition(opts: {
|
|
|
80
91
|
userId,
|
|
81
92
|
userEmail,
|
|
82
93
|
contactId,
|
|
94
|
+
allowCreate,
|
|
83
95
|
epoch,
|
|
84
96
|
source = "event",
|
|
85
97
|
reason,
|
|
@@ -141,6 +153,7 @@ export async function emitBucketTransition(opts: {
|
|
|
141
153
|
registry,
|
|
142
154
|
hatchet,
|
|
143
155
|
logger,
|
|
156
|
+
allowCreate,
|
|
144
157
|
event: {
|
|
145
158
|
event: eventName,
|
|
146
159
|
userId,
|
|
@@ -161,6 +174,9 @@ export async function emitBucketTransition(opts: {
|
|
|
161
174
|
registry,
|
|
162
175
|
hatchet,
|
|
163
176
|
logger,
|
|
177
|
+
// Same inherited refusal as the alias emit above — forwarding it on ONLY
|
|
178
|
+
// one leg would leave the ghost half-alive for generic-bound journeys.
|
|
179
|
+
allowCreate,
|
|
164
180
|
event: {
|
|
165
181
|
event: genericEvent,
|
|
166
182
|
userId,
|