@hogsend/engine 0.55.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +5 -10
- package/src/buckets/check-membership.ts +91 -6
- package/src/index.ts +31 -1
- package/src/journeys/execute-journey-run.ts +56 -1
- package/src/journeys/journey-context.ts +59 -0
- package/src/lib/bucket-emit.test.ts +83 -2
- package/src/lib/bucket-emit.ts +16 -0
- package/src/lib/contacts.ts +1031 -141
- package/src/lib/crm-ingest.ts +9 -0
- package/src/lib/enrichment-ledger.ts +1 -1
- package/src/lib/feed.ts +64 -3
- package/src/lib/identity-service.ts +15 -2
- package/src/lib/ingestion.ts +191 -14
- package/src/lib/outbound.ts +3 -3
- package/src/lib/refine-chain.test.ts +83 -25
- package/src/lib/refine-chain.ts +124 -23
- package/src/lib/refine-traits.test.ts +74 -41
- package/src/lib/refine-traits.ts +107 -52
- package/src/lib/refine.ts +28 -27
- package/src/lib/tracking-events.ts +33 -6
- package/src/routes/admin/agent.ts +26 -0
- package/src/routes/admin/contacts.ts +65 -5
- package/src/routes/admin/identity.ts +197 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/contacts/index.ts +50 -7
- package/src/routes/events/index.ts +94 -6
- package/src/routes/feed/index.ts +36 -0
- package/src/routes/feed/recipient.ts +44 -2
- package/src/routes/lists/index.ts +21 -0
- package/src/routes/tracking/arrive.ts +42 -3
- package/src/worker.ts +18 -0
- package/src/workflows/bucket-backfill.ts +1 -1
- package/src/workflows/bucket-reconcile.ts +129 -4
- package/src/workflows/identity-alias-backfill.ts +448 -0
- package/src/workflows/import-contacts.ts +10 -0
- package/src/workflows/send-campaign.ts +31 -13
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hogsend/engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.57.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -53,18 +53,18 @@
|
|
|
53
53
|
"svix": "^1.95.1",
|
|
54
54
|
"winston": "^3.19.0",
|
|
55
55
|
"zod": "^4.4.3",
|
|
56
|
-
"@hogsend/attribution": "^0.
|
|
57
|
-
"@hogsend/core": "^0.
|
|
58
|
-
"@hogsend/db": "^0.
|
|
59
|
-
"@hogsend/email": "^0.
|
|
60
|
-
"@hogsend/plugin-posthog": "^0.
|
|
61
|
-
"@hogsend/plugin-resend": "^0.
|
|
62
|
-
"@hogsend/sms": "^0.
|
|
56
|
+
"@hogsend/attribution": "^0.57.0",
|
|
57
|
+
"@hogsend/core": "^0.57.0",
|
|
58
|
+
"@hogsend/db": "^0.57.0",
|
|
59
|
+
"@hogsend/email": "^0.57.0",
|
|
60
|
+
"@hogsend/plugin-posthog": "^0.57.0",
|
|
61
|
+
"@hogsend/plugin-resend": "^0.57.0",
|
|
62
|
+
"@hogsend/sms": "^0.57.0"
|
|
63
63
|
},
|
|
64
64
|
"optionalDependencies": {
|
|
65
|
-
"@hogsend/plugin-apollo": "^0.
|
|
66
|
-
"@hogsend/plugin-postmark": "^0.
|
|
67
|
-
"@hogsend/plugin-twilio": "^0.
|
|
65
|
+
"@hogsend/plugin-apollo": "^0.57.0",
|
|
66
|
+
"@hogsend/plugin-postmark": "^0.57.0",
|
|
67
|
+
"@hogsend/plugin-twilio": "^0.57.0"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/node": "^22.15.3",
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { bucketMemberships, contacts, type Database } from "@hogsend/db";
|
|
2
2
|
import { and, count as countFn, eq, gt, isNull } from "drizzle-orm";
|
|
3
|
+
import { contactKeySql } from "../lib/contacts.js";
|
|
3
4
|
import { getDb } from "../lib/db.js";
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -74,7 +75,7 @@ export function createBucketAccessor(
|
|
|
74
75
|
const rows = await db
|
|
75
76
|
.select({ value: countFn() })
|
|
76
77
|
.from(bucketMemberships)
|
|
77
|
-
.innerJoin(contacts, eq(
|
|
78
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
78
79
|
.where(
|
|
79
80
|
and(
|
|
80
81
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -98,7 +99,7 @@ export function createBucketAccessor(
|
|
|
98
99
|
const rows = await db
|
|
99
100
|
.select({ id: bucketMemberships.id })
|
|
100
101
|
.from(bucketMemberships)
|
|
101
|
-
.innerJoin(contacts, eq(
|
|
102
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
102
103
|
.where(
|
|
103
104
|
and(
|
|
104
105
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -144,10 +145,7 @@ export function createBucketAccessor(
|
|
|
144
145
|
entryCount: bucketMemberships.entryCount,
|
|
145
146
|
})
|
|
146
147
|
.from(bucketMemberships)
|
|
147
|
-
.innerJoin(
|
|
148
|
-
contacts,
|
|
149
|
-
eq(contacts.externalId, bucketMemberships.userId),
|
|
150
|
-
)
|
|
148
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
151
149
|
.where(and(...conditions))
|
|
152
150
|
.orderBy(bucketMemberships.id)
|
|
153
151
|
// +1 peek to detect a next page.
|
|
@@ -155,10 +153,7 @@ export function createBucketAccessor(
|
|
|
155
153
|
db
|
|
156
154
|
.select({ value: countFn() })
|
|
157
155
|
.from(bucketMemberships)
|
|
158
|
-
.innerJoin(
|
|
159
|
-
contacts,
|
|
160
|
-
eq(contacts.externalId, bucketMemberships.userId),
|
|
161
|
-
)
|
|
156
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
162
157
|
.where(
|
|
163
158
|
and(
|
|
164
159
|
eq(bucketMemberships.bucketId, bucketId),
|
|
@@ -9,7 +9,7 @@ import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
|
9
9
|
import { bucketMemberships, contacts, type Database } from "@hogsend/db";
|
|
10
10
|
import { and, desc, eq, isNotNull, isNull } from "drizzle-orm";
|
|
11
11
|
import { emitBucketTransition } from "../lib/bucket-emit.js";
|
|
12
|
-
import { normalizeEmailOrNull } from "../lib/contacts.js";
|
|
12
|
+
import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
|
|
13
13
|
import type { Logger } from "../lib/logger.js";
|
|
14
14
|
import {
|
|
15
15
|
BUCKET_EVENT_PREFIX,
|
|
@@ -61,6 +61,18 @@ export async function checkBucketMembership(opts: {
|
|
|
61
61
|
* contact degrades to the pin-less emit.
|
|
62
62
|
*/
|
|
63
63
|
contactId?: string;
|
|
64
|
+
/**
|
|
65
|
+
* D1 creation guard, INHERITED from the originating ingest and forwarded into
|
|
66
|
+
* every `emitBucketTransition` (and from there into both of its re-ingests).
|
|
67
|
+
* `ingestEvent` passes `false` exactly when its own resolve REFUSED, i.e.
|
|
68
|
+
* when there is no `contactId` to pin with — because the alternative,
|
|
69
|
+
* degrading the pin to `contactId ?? undefined`, makes the transition
|
|
70
|
+
* re-ingest treat the anon canonical key as an EXTERNAL key and mint an
|
|
71
|
+
* `external_id = <anonId>` phantom twin (issue #608 from the other side).
|
|
72
|
+
* Bucket evaluation itself is UNAFFECTED: `bucket_memberships` is text-keyed
|
|
73
|
+
* with no contact FK, so a contactless subject is still a first-class member.
|
|
74
|
+
*/
|
|
75
|
+
allowCreate?: boolean;
|
|
64
76
|
event: string;
|
|
65
77
|
/**
|
|
66
78
|
* D2: the event payload — candidate-narrowing ONLY. It NO LONGER participates
|
|
@@ -72,6 +84,14 @@ export async function checkBucketMembership(opts: {
|
|
|
72
84
|
* the very first event after a property change evaluates correctly (risk 7).
|
|
73
85
|
*/
|
|
74
86
|
contactProperties?: Record<string, unknown>;
|
|
87
|
+
/**
|
|
88
|
+
* Extra property NAMES for candidate-narrowing ONLY — never overlaid on the
|
|
89
|
+
* contact row for value eval. A producer whose write patch omits a key it
|
|
90
|
+
* nonetheless changed (refinement's fill-if-absent drops already-held facts to
|
|
91
|
+
* avoid clobbering first-party data) lists those names here so their fit
|
|
92
|
+
* buckets are still re-checked against live contact state.
|
|
93
|
+
*/
|
|
94
|
+
touchProperties?: string[];
|
|
75
95
|
/** Optional override; defaults to the process bucket-registry singleton. */
|
|
76
96
|
bucketRegistry?: ReturnType<typeof getBucketRegistrySingleton>;
|
|
77
97
|
}): Promise<BucketTransition[]> {
|
|
@@ -82,9 +102,11 @@ export async function checkBucketMembership(opts: {
|
|
|
82
102
|
logger,
|
|
83
103
|
userId,
|
|
84
104
|
contactId,
|
|
105
|
+
allowCreate,
|
|
85
106
|
event,
|
|
86
107
|
eventProperties,
|
|
87
108
|
contactProperties: contactPropertiesPatch,
|
|
109
|
+
touchProperties,
|
|
88
110
|
} = opts;
|
|
89
111
|
// The caller's email comes verbatim from the raw event payload — normalize
|
|
90
112
|
// ONCE here so the membership row, the emitted bucket:* transition events,
|
|
@@ -118,6 +140,10 @@ export async function checkBucketMembership(opts: {
|
|
|
118
140
|
for (const key of [
|
|
119
141
|
...Object.keys(eventProperties ?? {}),
|
|
120
142
|
...Object.keys(contactPropertiesPatch ?? {}),
|
|
143
|
+
// Names-only touches (e.g. refinement's fill-if-absent dropped facts) surface
|
|
144
|
+
// a fit bucket here so it is re-checked against live contact state below,
|
|
145
|
+
// even though the value is not in this ingest's write patch.
|
|
146
|
+
...(touchProperties ?? []),
|
|
121
147
|
]) {
|
|
122
148
|
for (const bucket of bucketRegistry.getByReferencedProperty(key)) {
|
|
123
149
|
candidateMap.set(bucket.id, bucket);
|
|
@@ -157,13 +183,31 @@ export async function checkBucketMembership(opts: {
|
|
|
157
183
|
let storedContactProps: Record<string, unknown> = {};
|
|
158
184
|
let contactDeleted = false;
|
|
159
185
|
if (needsContactState) {
|
|
186
|
+
// Match on the CANONICAL key, not `external_id` alone. Memberships and
|
|
187
|
+
// events are keyed on `coalesce(external_id, anonymous_id, id)`
|
|
188
|
+
// (`contactKeySql`), so an email-only contact is keyed on its uuid and an
|
|
189
|
+
// anonymous one on its `anonymous_id` — both have a NULL `external_id` and
|
|
190
|
+
// neither was ever found here. The cron's join scan already made exactly
|
|
191
|
+
// this correction for the same reason (`bucket-reconcile.ts`, "joining on
|
|
192
|
+
// contacts.externalId would … silently drop exactly the dormant email-only
|
|
193
|
+
// contacts this cron exists to reconcile"); this is that fix applied to the
|
|
194
|
+
// real-time path, which was missed.
|
|
195
|
+
//
|
|
196
|
+
// Two things were wrong, and the second is the serious one:
|
|
197
|
+
// 1. property criteria evaluated against `{}` instead of the contact's
|
|
198
|
+
// real state, so a property leg silently answered "absent" for every
|
|
199
|
+
// such person;
|
|
200
|
+
// 2. `contactDeleted` stayed false because the row was never found, so
|
|
201
|
+
// the GDPR guard below — "never (re-)evaluate or emit for a
|
|
202
|
+
// soft-deleted contact" — did not fire for them. A soft-deleted
|
|
203
|
+
// email-only contact could still transition buckets and emit.
|
|
160
204
|
const [contact] = await db
|
|
161
205
|
.select({
|
|
162
206
|
properties: contacts.properties,
|
|
163
207
|
deletedAt: contacts.deletedAt,
|
|
164
208
|
})
|
|
165
209
|
.from(contacts)
|
|
166
|
-
.where(eq(
|
|
210
|
+
.where(eq(contactKeySql(), userId))
|
|
167
211
|
.limit(1);
|
|
168
212
|
if (contact) {
|
|
169
213
|
storedContactProps =
|
|
@@ -221,6 +265,7 @@ export async function checkBucketMembership(opts: {
|
|
|
221
265
|
userId,
|
|
222
266
|
userEmail,
|
|
223
267
|
contactId,
|
|
268
|
+
allowCreate,
|
|
224
269
|
});
|
|
225
270
|
if (transition) transitions.push(transition);
|
|
226
271
|
} else if (wasMember && isMember) {
|
|
@@ -240,6 +285,7 @@ export async function checkBucketMembership(opts: {
|
|
|
240
285
|
userId,
|
|
241
286
|
userEmail,
|
|
242
287
|
contactId,
|
|
288
|
+
allowCreate,
|
|
243
289
|
});
|
|
244
290
|
if (transition) transitions.push(transition);
|
|
245
291
|
}
|
|
@@ -259,6 +305,8 @@ async function handleJoin(opts: {
|
|
|
259
305
|
userEmail: string | null;
|
|
260
306
|
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
261
307
|
contactId?: string;
|
|
308
|
+
/** Inherited creation guard for the emit — see `checkBucketMembership`. */
|
|
309
|
+
allowCreate?: boolean;
|
|
262
310
|
}): Promise<BucketTransition | null> {
|
|
263
311
|
const {
|
|
264
312
|
db,
|
|
@@ -269,6 +317,7 @@ async function handleJoin(opts: {
|
|
|
269
317
|
userId,
|
|
270
318
|
userEmail,
|
|
271
319
|
contactId,
|
|
320
|
+
allowCreate,
|
|
272
321
|
} = opts;
|
|
273
322
|
|
|
274
323
|
// entryCount ordinal = 1 + count of ALL prior memberships (active + left) for
|
|
@@ -311,6 +360,10 @@ async function handleJoin(opts: {
|
|
|
311
360
|
// is written. The cron remains the authoritative backstop, so a push failure
|
|
312
361
|
// is best-effort. We arm against the persisted expiresAt so the timer's CAS on
|
|
313
362
|
// wake matches the row (or no-ops if a later event re-armed the window).
|
|
363
|
+
// `allowCreate` rides along because the timer's eventual leave emit is a
|
|
364
|
+
// re-ingest DERIVED from this (possibly refused) event — D11 applies to the
|
|
365
|
+
// ASYNCHRONOUS producer exactly as it does to the two synchronous ones. The
|
|
366
|
+
// `contactId` pin deliberately does NOT (see `bucketExpiryTask`).
|
|
314
367
|
if (bucket.fastExpiry && expiresAt) {
|
|
315
368
|
await armExpiryTimer({
|
|
316
369
|
hatchet,
|
|
@@ -320,6 +373,7 @@ async function handleJoin(opts: {
|
|
|
320
373
|
userId,
|
|
321
374
|
userEmail,
|
|
322
375
|
expiresAt,
|
|
376
|
+
allowCreate,
|
|
323
377
|
});
|
|
324
378
|
}
|
|
325
379
|
|
|
@@ -338,8 +392,10 @@ async function handleJoin(opts: {
|
|
|
338
392
|
userEmail,
|
|
339
393
|
// Pin the re-ingest to the already-resolved contact row so an
|
|
340
394
|
// anonymous-only contact's transition folds in instead of minting a
|
|
341
|
-
// phantom twin (issue #608).
|
|
395
|
+
// phantom twin (issue #608). With no row to pin to, the originating
|
|
396
|
+
// ingest's refusal is inherited instead — never a degraded pin.
|
|
342
397
|
contactId,
|
|
398
|
+
allowCreate,
|
|
343
399
|
epoch,
|
|
344
400
|
source: "event",
|
|
345
401
|
});
|
|
@@ -365,6 +421,8 @@ async function handleLeave(opts: {
|
|
|
365
421
|
userEmail: string | null;
|
|
366
422
|
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
367
423
|
contactId?: string;
|
|
424
|
+
/** Inherited creation guard for the emit — see `checkBucketMembership`. */
|
|
425
|
+
allowCreate?: boolean;
|
|
368
426
|
}): Promise<BucketTransition | null> {
|
|
369
427
|
const {
|
|
370
428
|
db,
|
|
@@ -376,6 +434,7 @@ async function handleLeave(opts: {
|
|
|
376
434
|
userId,
|
|
377
435
|
userEmail,
|
|
378
436
|
contactId,
|
|
437
|
+
allowCreate,
|
|
379
438
|
} = opts;
|
|
380
439
|
|
|
381
440
|
// minDwell DEFERS (never silently drops) the leave (Section 6.3). We set a
|
|
@@ -440,9 +499,11 @@ async function handleLeave(opts: {
|
|
|
440
499
|
bucket,
|
|
441
500
|
userId,
|
|
442
501
|
userEmail,
|
|
443
|
-
// Same provenance pin as the join emit — a
|
|
444
|
-
// identical path and would mint the same
|
|
502
|
+
// Same provenance pin (and same inherited refusal) as the join emit — a
|
|
503
|
+
// leave re-ingests through the identical path and would mint the same
|
|
504
|
+
// phantom twin (issue #608).
|
|
445
505
|
contactId,
|
|
506
|
+
allowCreate,
|
|
446
507
|
epoch: flipped.entryCount,
|
|
447
508
|
source: "event",
|
|
448
509
|
reason: "criteria",
|
|
@@ -526,6 +587,12 @@ function withinMinDwell(
|
|
|
526
587
|
* by `checkBucketMembership`, so arming does NOT re-enter bucket evaluation.
|
|
527
588
|
* Best-effort: the cron is the authoritative backstop, so a push failure is logged
|
|
528
589
|
* and swallowed.
|
|
590
|
+
*
|
|
591
|
+
* The arm payload is the ONLY channel between this join and the leave the woken
|
|
592
|
+
* task emits, so the inherited D1 refusal has to ride it — otherwise the timer's
|
|
593
|
+
* `emitBucketTransition` re-resolves the anon canonical key create-on-miss and
|
|
594
|
+
* mints the `external_id = <anonId>` phantom twin the synchronous emits were
|
|
595
|
+
* already taught to refuse (issue #608, D11 category (iii)).
|
|
529
596
|
*/
|
|
530
597
|
async function armExpiryTimer(opts: {
|
|
531
598
|
hatchet: HatchetClient;
|
|
@@ -535,8 +602,25 @@ async function armExpiryTimer(opts: {
|
|
|
535
602
|
userId: string;
|
|
536
603
|
userEmail: string | null;
|
|
537
604
|
expiresAt: Date;
|
|
605
|
+
/**
|
|
606
|
+
* D1 creation guard inherited from the originating ingest via `handleJoin`.
|
|
607
|
+
* Carried on the wire ONLY when `false`: an omitted key keeps today's
|
|
608
|
+
* create-by-default semantics for every producer that legitimately creates
|
|
609
|
+
* and keeps a JSON null off a Hatchet payload (the
|
|
610
|
+
* `...(contactId !== null ? { contactId } : {})` idiom at `ingestion.ts`).
|
|
611
|
+
*/
|
|
612
|
+
allowCreate?: boolean;
|
|
538
613
|
}): Promise<void> {
|
|
539
|
-
const {
|
|
614
|
+
const {
|
|
615
|
+
hatchet,
|
|
616
|
+
logger,
|
|
617
|
+
bucket,
|
|
618
|
+
rowId,
|
|
619
|
+
userId,
|
|
620
|
+
userEmail,
|
|
621
|
+
expiresAt,
|
|
622
|
+
allowCreate,
|
|
623
|
+
} = opts;
|
|
540
624
|
const msUntilExpiry = Math.max(0, expiresAt.getTime() - Date.now());
|
|
541
625
|
try {
|
|
542
626
|
await hatchet.events.push("bucket:arm-expiry", {
|
|
@@ -546,6 +630,7 @@ async function armExpiryTimer(opts: {
|
|
|
546
630
|
userEmail,
|
|
547
631
|
armedExpiresAt: expiresAt.toISOString(),
|
|
548
632
|
msUntilExpiry,
|
|
633
|
+
...(allowCreate === false ? { allowCreate: false } : {}),
|
|
549
634
|
});
|
|
550
635
|
} catch (err) {
|
|
551
636
|
logger.warn("Bucket fast-expiry arm failed (cron backstop covers it)", {
|
package/src/index.ts
CHANGED
|
@@ -394,7 +394,25 @@ export {
|
|
|
394
394
|
verifyConnectorState,
|
|
395
395
|
} from "./lib/connector-state.js";
|
|
396
396
|
// --- Contacts identity (resolve/create — used by connector member-link) ---
|
|
397
|
-
|
|
397
|
+
// `resolveContactNoCreate` is the refuse-on-miss sibling (D1): same resolution,
|
|
398
|
+
// but pure observation never mints a `contacts` row.
|
|
399
|
+
// `identifiedContactFilter` is the read-side counterpart (PRD 01): the single
|
|
400
|
+
// "has this person ever identified?" predicate behind `?identity=`.
|
|
401
|
+
// `deleteIdentityAliasesForContact` is the erasure hook (PRD 02 T1): a
|
|
402
|
+
// consumer-built deletion flow that soft-deletes `contacts` rows directly must
|
|
403
|
+
// call it too, or the erased person's identity keys survive in
|
|
404
|
+
// `contact_aliases`.
|
|
405
|
+
// `ResolvePolicy`/`IdentityKind` (PRD 06): the explicit caller-declared trust
|
|
406
|
+
// shape both resolver entry points accept via `policy` — the additive
|
|
407
|
+
// replacement for the deprecated `restrictToAnonymous`/`allowCreate` booleans.
|
|
408
|
+
export {
|
|
409
|
+
deleteIdentityAliasesForContact,
|
|
410
|
+
type IdentityKind,
|
|
411
|
+
identifiedContactFilter,
|
|
412
|
+
type ResolvePolicy,
|
|
413
|
+
resolveContactNoCreate,
|
|
414
|
+
resolveOrCreateContact,
|
|
415
|
+
} from "./lib/contacts.js";
|
|
398
416
|
// --- Conversion dispatch (plan §5.2): destinations registry + delivery ---
|
|
399
417
|
export {
|
|
400
418
|
ConversionDestinationRegistry,
|
|
@@ -564,6 +582,7 @@ export {
|
|
|
564
582
|
type IngestEvent,
|
|
565
583
|
type IngestResult,
|
|
566
584
|
ingestEvent,
|
|
585
|
+
ingestTransformResult,
|
|
567
586
|
} from "./lib/ingestion.js";
|
|
568
587
|
// --- Reconciled journey-lift helper (impact experiments D4.1) — the ONE
|
|
569
588
|
// implementation of the causal cohort math, shared by /lift (2a), /impact
|
|
@@ -921,6 +940,17 @@ export {
|
|
|
921
940
|
deliverWebhookTask,
|
|
922
941
|
reapDueWebhookDeliveriesTask,
|
|
923
942
|
} from "./workflows/deliver-webhook.js";
|
|
943
|
+
// --- Identity alias backfill (PRD 02): task + boot enqueue + parity probe ---
|
|
944
|
+
export {
|
|
945
|
+
type AliasParityRow,
|
|
946
|
+
enqueueIdentityAliasBackfill,
|
|
947
|
+
IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
948
|
+
type IdentityAliasBackfillInput,
|
|
949
|
+
type IdentityAliasBackfillResult,
|
|
950
|
+
identityAliasBackfillTask,
|
|
951
|
+
identityAliasParity,
|
|
952
|
+
runIdentityAliasBackfill,
|
|
953
|
+
} from "./workflows/identity-alias-backfill.js";
|
|
924
954
|
export {
|
|
925
955
|
buildImpactDigest,
|
|
926
956
|
detectLiftCrossings,
|
|
@@ -12,9 +12,10 @@ import {
|
|
|
12
12
|
journeyConfigs,
|
|
13
13
|
journeyStates,
|
|
14
14
|
} from "@hogsend/db";
|
|
15
|
-
import { and, eq, inArray, notInArray, sql } from "drizzle-orm";
|
|
15
|
+
import { and, eq, inArray, isNull, notInArray, or, sql } from "drizzle-orm";
|
|
16
16
|
import { getAnalytics } from "../lib/analytics-singleton.js";
|
|
17
17
|
import { blueprintGraphLock } from "../lib/blueprint-lock.js";
|
|
18
|
+
import { ALL_IDENTITY_KINDS } from "../lib/contacts.js";
|
|
18
19
|
import { getDb } from "../lib/db.js";
|
|
19
20
|
import {
|
|
20
21
|
checkEmailPreferences,
|
|
@@ -384,15 +385,69 @@ export async function executeJourneyRun(
|
|
|
384
385
|
// concurrent double-diversion dedups here even if it raced the
|
|
385
386
|
// row existence check above.
|
|
386
387
|
try {
|
|
388
|
+
// D11 — this re-ingest is DERIVED from the entry event, so it
|
|
389
|
+
// must inherit that event's creation verdict instead of
|
|
390
|
+
// re-resolving the key cold. Since PRD 02 site 1 a journey's
|
|
391
|
+
// `userId` is routinely a browser anon id owning NO contact row
|
|
392
|
+
// (`ingestEvent` pushes `userId: resolvedKey` and OMITS
|
|
393
|
+
// `contactId` on a refusal), and `findByKey` reads a bare
|
|
394
|
+
// `userId` as kind "external" — so a create-on-miss re-ingest
|
|
395
|
+
// here writes `external_id = <anonId>`, which is strictly WORSE
|
|
396
|
+
// than the ghost being removed: `collidesWithIdentified` returns
|
|
397
|
+
// true for it, so `resolveFeedRecipient` starts 403-ing the
|
|
398
|
+
// visitor out of their own bell.
|
|
399
|
+
//
|
|
400
|
+
// The verdict is DERIVED, never hardcoded — a held-out run for a
|
|
401
|
+
// real contact, or one asserting an email, must keep creating.
|
|
402
|
+
// This leg runs BEFORE the timezone `contact` fetch below, so
|
|
403
|
+
// provenance is resolved locally: the pushed `input.contactId`
|
|
404
|
+
// when the entry ingest resolved a row, else ONE indexed lookup
|
|
405
|
+
// for the live contact owning this canonical key on either
|
|
406
|
+
// identity column (`contactKey = external_id ?? anonymous_id`;
|
|
407
|
+
// the tz fetch reads `external_id` only, which misses an
|
|
408
|
+
// anon-keyed contact).
|
|
409
|
+
const subjectContactId =
|
|
410
|
+
input.contactId ??
|
|
411
|
+
(
|
|
412
|
+
await db.query.contacts.findFirst({
|
|
413
|
+
where: and(
|
|
414
|
+
isNull(contacts.deletedAt),
|
|
415
|
+
or(
|
|
416
|
+
eq(contacts.externalId, userId),
|
|
417
|
+
eq(contacts.anonymousId, userId),
|
|
418
|
+
),
|
|
419
|
+
),
|
|
420
|
+
columns: { id: true },
|
|
421
|
+
})
|
|
422
|
+
)?.id;
|
|
387
423
|
await ingestEvent({
|
|
388
424
|
db,
|
|
389
425
|
registry: getJourneyRegistrySingleton(),
|
|
390
426
|
hatchet,
|
|
391
427
|
logger,
|
|
428
|
+
// PRD 06 T4 (L5 row 26): an engine-internal lifecycle emit is
|
|
429
|
+
// a FULLY trusted caller (L4) — full `trustedKinds`, never a
|
|
430
|
+
// clamp. It inherits exactly ONE thing from the verdict: the
|
|
431
|
+
// `create` leg. Refuse ONLY when nothing was resolved AND
|
|
432
|
+
// nothing is asserted. A `userEmail` is a durable identity the
|
|
433
|
+
// caller asserted (D1), so refusing it would change behavior
|
|
434
|
+
// for identified users; anonymous runs carry `""` here
|
|
435
|
+
// (`ingestEvent` pushes `userEmail ?? ""`), correctly falsy.
|
|
436
|
+
policy: {
|
|
437
|
+
create:
|
|
438
|
+
subjectContactId || userEmail
|
|
439
|
+
? "on-miss"
|
|
440
|
+
: "refuse-on-miss",
|
|
441
|
+
allowMerge: "any",
|
|
442
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
443
|
+
},
|
|
392
444
|
event: {
|
|
393
445
|
event: "journey.heldout",
|
|
394
446
|
userId,
|
|
395
447
|
userEmail,
|
|
448
|
+
// Pin to the exact subject row so the re-emit folds in
|
|
449
|
+
// instead of value-probing a key that could mint a twin.
|
|
450
|
+
...(subjectContactId ? { contactId: subjectContactId } : {}),
|
|
396
451
|
eventProperties: {
|
|
397
452
|
journeyId: meta.id,
|
|
398
453
|
journeyName: meta.name,
|
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
notInArray,
|
|
48
48
|
sql,
|
|
49
49
|
} from "drizzle-orm";
|
|
50
|
+
import { ALL_IDENTITY_KINDS } from "../lib/contacts.js";
|
|
50
51
|
import { checkEmailPreferences } from "../lib/enrollment-guards.js";
|
|
51
52
|
import { countRecentSends } from "../lib/frequency-cap.js";
|
|
52
53
|
import { toSleepDuration } from "../lib/hatchet-duration.js";
|
|
@@ -1262,16 +1263,74 @@ export function createJourneyContext(
|
|
|
1262
1263
|
registerKey(boundary, idempotencyKey);
|
|
1263
1264
|
}
|
|
1264
1265
|
|
|
1266
|
+
// D11 — a `ctx.trigger` re-ingest is DERIVED from the run's own entry
|
|
1267
|
+
// event, so it must inherit that event's creation verdict rather than
|
|
1268
|
+
// re-resolve the key cold. Since PRD 02 site 1 a journey's `userId` is
|
|
1269
|
+
// routinely a browser anon id owning NO contact row (`ingestEvent` pushes
|
|
1270
|
+
// `userId: resolvedKey` and OMITS `contactId` on a refusal), and
|
|
1271
|
+
// `findByKey` reads a bare `userId` as kind "external" — so a
|
|
1272
|
+
// create-on-miss re-ingest here writes `external_id = <anonId>`, strictly
|
|
1273
|
+
// WORSE than the ghost being removed: `collidesWithIdentified` returns
|
|
1274
|
+
// true for it, so `resolveFeedRecipient` starts 403-ing the visitor out of
|
|
1275
|
+
// their own bell.
|
|
1276
|
+
//
|
|
1277
|
+
// The verdict is per-ARM, because the two arms have different subjects.
|
|
1278
|
+
//
|
|
1279
|
+
// SELF arm (`targetUserId === userId`) — the re-ingest really is derived
|
|
1280
|
+
// from this run's own entry event, so it inherits that event's verdict,
|
|
1281
|
+
// DERIVED (never hardcoded) from two signals already in scope:
|
|
1282
|
+
// - `config.contactId` — the run's resolved subject row id (pushed by
|
|
1283
|
+
// the entry ingest, else the enrollment's own contact lookup).
|
|
1284
|
+
// Present ⇒ the subject HAS a row: pin to it so the re-emit folds
|
|
1285
|
+
// into that exact row instead of value-probing a canonical key that
|
|
1286
|
+
// could mint a phantom twin.
|
|
1287
|
+
// - an asserted email (`targetEmail ?? userEmail`) — a durable identity
|
|
1288
|
+
// the caller is asserting (D1), so it keeps creating. Anonymous runs
|
|
1289
|
+
// carry `""` here (`ingestEvent` pushes `userEmail ?? ""`), correctly
|
|
1290
|
+
// falsy.
|
|
1291
|
+
// Neither ⇒ nothing resolved and nothing asserted: refuse, so re-ingesting
|
|
1292
|
+
// the run's own anon key cold cannot re-resolve it as an EXTERNAL key and
|
|
1293
|
+
// mint the `external_id = <anonId>` twin. The refusal only ever bites a
|
|
1294
|
+
// total MISS — a subject that already owns a contact still links/merges
|
|
1295
|
+
// exactly as before.
|
|
1296
|
+
//
|
|
1297
|
+
// CROSS-USER arm — the run's identity says NOTHING about the person the
|
|
1298
|
+
// author named, so neither signal is in scope for the target: the pin is
|
|
1299
|
+
// `undefined` BY CONSTRUCTION (pinning an author-overridden `userId` to
|
|
1300
|
+
// the run's subject row would misattribute the event), and the run's own
|
|
1301
|
+
// `userEmail` belongs to somebody else. Reading the verdict off them would
|
|
1302
|
+
// make an anonymous run refuse to mint a NAMED target — inverting D1,
|
|
1303
|
+
// under which `ctx.trigger({ userId })` is exactly the "server-side caller
|
|
1304
|
+
// explicitly asks" case — and would strip that target of everything gated
|
|
1305
|
+
// on a non-null `contact_id` (conversions, attribution credits, funnel
|
|
1306
|
+
// progress, deals). So this arm keeps creating, unconditionally.
|
|
1307
|
+
const isSelfTrigger = targetUserId === userId;
|
|
1308
|
+
const pinnedContactId = isSelfTrigger ? config.contactId : undefined;
|
|
1309
|
+
const assertsIdentity = isSelfTrigger
|
|
1310
|
+
? !!pinnedContactId || !!(targetEmail ?? userEmail)
|
|
1311
|
+
: true;
|
|
1312
|
+
|
|
1265
1313
|
const runIngest = () =>
|
|
1266
1314
|
ingestEvent({
|
|
1267
1315
|
db,
|
|
1268
1316
|
registry,
|
|
1269
1317
|
hatchet,
|
|
1270
1318
|
logger,
|
|
1319
|
+
// PRD 06 T4 (L5 row 25): an engine-internal re-emit is a FULLY
|
|
1320
|
+
// trusted caller regardless of what authenticated the originating
|
|
1321
|
+
// request (L4) — full `trustedKinds`, never a clamp. The ONE thing
|
|
1322
|
+
// it inherits from the verdict above is the `create` leg, keyed off
|
|
1323
|
+
// the same `assertsIdentity` derivation as before.
|
|
1324
|
+
policy: {
|
|
1325
|
+
create: assertsIdentity ? "on-miss" : "refuse-on-miss",
|
|
1326
|
+
allowMerge: "any",
|
|
1327
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
1328
|
+
},
|
|
1271
1329
|
event: {
|
|
1272
1330
|
event,
|
|
1273
1331
|
userId: targetUserId,
|
|
1274
1332
|
userEmail: targetEmail ?? userEmail,
|
|
1333
|
+
...(pinnedContactId ? { contactId: pinnedContactId } : {}),
|
|
1275
1334
|
eventProperties: properties ?? {},
|
|
1276
1335
|
value,
|
|
1277
1336
|
currency,
|
|
@@ -51,14 +51,20 @@ function genericBoundRegistry(): JourneyRegistry {
|
|
|
51
51
|
} as unknown as JourneyRegistry;
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
/**
|
|
54
|
+
/**
|
|
55
|
+
* Recording ingest stub — captures each re-ingested event, plus the
|
|
56
|
+
* `allowCreate` guard, which is a SIBLING of `event` in the ingest opts (not a
|
|
57
|
+
* field on it) and so needs its own recorder.
|
|
58
|
+
*/
|
|
55
59
|
function recordingIngest() {
|
|
56
60
|
const events: IngestEvent[] = [];
|
|
61
|
+
const allowCreates: (boolean | undefined)[] = [];
|
|
57
62
|
const ingest: typeof ingestEvent = async (opts) => {
|
|
58
63
|
events.push(opts.event);
|
|
64
|
+
allowCreates.push(opts.allowCreate);
|
|
59
65
|
return { stored: true, exits: [], contactKey: opts.event.userId ?? "" };
|
|
60
66
|
};
|
|
61
|
-
return { events, ingest };
|
|
67
|
+
return { events, allowCreates, ingest };
|
|
62
68
|
}
|
|
63
69
|
|
|
64
70
|
// db/hatchet are never touched by the stubbed ingest; emitOutbound is
|
|
@@ -143,3 +149,78 @@ test("no contactId: degrades to exactly the pin-less re-ingest", async () => {
|
|
|
143
149
|
assert.equal(event.contactId, undefined);
|
|
144
150
|
}
|
|
145
151
|
});
|
|
152
|
+
|
|
153
|
+
// ---------------------------------------------------------------------------
|
|
154
|
+
// D1 — the INHERITED creation guard. When the originating ingest refused to
|
|
155
|
+
// mint, there is no `contactId` to pin with, and degrading the pin here is
|
|
156
|
+
// exactly what would mint an `external_id = <anonId>` phantom twin. So the
|
|
157
|
+
// refusal itself rides the re-ingests instead — and, like the pin, it must ride
|
|
158
|
+
// BOTH legs: forwarding it only on the alias would leave the ghost half-alive
|
|
159
|
+
// for any journey bound to the generic `bucket:<kind>` form.
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
|
|
162
|
+
test("allowCreate: false rides BOTH re-ingests (alias + generic)", async () => {
|
|
163
|
+
const { events, allowCreates, ingest } = recordingIngest();
|
|
164
|
+
|
|
165
|
+
await emitBucketTransition({
|
|
166
|
+
db,
|
|
167
|
+
registry: genericBoundRegistry(),
|
|
168
|
+
hatchet,
|
|
169
|
+
logger,
|
|
170
|
+
kind: "entered",
|
|
171
|
+
bucket,
|
|
172
|
+
userId: "anon-608",
|
|
173
|
+
userEmail: null,
|
|
174
|
+
allowCreate: false,
|
|
175
|
+
epoch: 1,
|
|
176
|
+
ingest,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
assert.equal(events.length, 2);
|
|
180
|
+
assert.deepEqual(allowCreates, [false, false]);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test("allowCreate: false rides the leave re-ingests too", async () => {
|
|
184
|
+
const { events, allowCreates, ingest } = recordingIngest();
|
|
185
|
+
|
|
186
|
+
await emitBucketTransition({
|
|
187
|
+
db,
|
|
188
|
+
registry: genericBoundRegistry(),
|
|
189
|
+
hatchet,
|
|
190
|
+
logger,
|
|
191
|
+
kind: "left",
|
|
192
|
+
bucket,
|
|
193
|
+
userId: "anon-608",
|
|
194
|
+
userEmail: null,
|
|
195
|
+
allowCreate: false,
|
|
196
|
+
epoch: 2,
|
|
197
|
+
reason: "criteria",
|
|
198
|
+
ingest,
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
assert.equal(events.length, 2);
|
|
202
|
+
assert.deepEqual(allowCreates, [false, false]);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("no allowCreate: the creating producers are untouched", async () => {
|
|
206
|
+
const { allowCreates, ingest } = recordingIngest();
|
|
207
|
+
|
|
208
|
+
await emitBucketTransition({
|
|
209
|
+
db,
|
|
210
|
+
registry: genericBoundRegistry(),
|
|
211
|
+
hatchet,
|
|
212
|
+
logger,
|
|
213
|
+
kind: "entered",
|
|
214
|
+
bucket,
|
|
215
|
+
userId: "anon-608",
|
|
216
|
+
userEmail: null,
|
|
217
|
+
contactId: CONTACT_ID,
|
|
218
|
+
epoch: 1,
|
|
219
|
+
ingest,
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
// The reconcile cron and the backfill legitimately create; leaving the option
|
|
223
|
+
// unset must reach `ingestEvent` as `undefined` (its default-true path), not
|
|
224
|
+
// as a `false` someone helpfully filled in.
|
|
225
|
+
assert.deepEqual(allowCreates, [undefined, undefined]);
|
|
226
|
+
});
|