@hogsend/engine 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -11
- package/src/buckets/check-membership.ts +45 -3
- package/src/connectors/define-action.ts +15 -2
- package/src/container.ts +85 -0
- package/src/env.ts +21 -0
- package/src/index.ts +19 -0
- package/src/journeys/durable-law-harness.test.ts +107 -0
- package/src/journeys/durable-law-harness.ts +139 -0
- package/src/journeys/journey-boundary.test.ts +70 -0
- package/src/journeys/journey-boundary.ts +10 -1
- package/src/journeys/with-durable-gate.test.ts +258 -0
- package/src/journeys/with-durable-gate.ts +141 -0
- package/src/lib/bucket-emit.test.ts +145 -0
- package/src/lib/bucket-emit.ts +29 -3
- package/src/lib/connector-actions.test.ts +292 -1
- package/src/lib/connector-actions.ts +65 -6
- package/src/lib/email-providers-from-env.ts +15 -16
- package/src/lib/enrichment-ledger.ts +207 -0
- package/src/lib/enrichment-provider-registry.test.ts +64 -0
- package/src/lib/enrichment-provider-registry.ts +35 -0
- package/src/lib/enrichment-providers-from-env.apollo.test.ts +30 -0
- package/src/lib/enrichment-providers-from-env.test.ts +28 -0
- package/src/lib/enrichment-providers-from-env.ts +52 -0
- package/src/lib/enrichment-registry-singleton.ts +45 -0
- package/src/lib/ingestion.ts +5 -0
- package/src/lib/load-optional-plugin.test.ts +133 -0
- package/src/lib/load-optional-plugin.ts +142 -0
- package/src/lib/outbound.ts +27 -0
- package/src/lib/refine-chain.test.ts +764 -0
- package/src/lib/refine-chain.ts +583 -0
- package/src/lib/refine-traits.test.ts +141 -0
- package/src/lib/refine-traits.ts +128 -0
- package/src/lib/refine.ts +287 -0
- package/src/lib/sms-providers-from-env.ts +11 -11
- package/src/lib/webhook-catalog-sync.test.ts +112 -0
- package/src/lib/webhook-signing.ts +5 -1
- package/src/routes/admin/contacts.ts +113 -13
- package/src/workflows/bucket-backfill.ts +14 -0
- package/src/workflows/bucket-reconcile.ts +74 -15
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import test from "node:test";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { WEBHOOK_EVENT_TYPES } from "./webhook-signing.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* THE DRIFT CHECK the outbound catalog never had.
|
|
9
|
+
*
|
|
10
|
+
* `WEBHOOK_EVENT_TYPES` is vendored into two copies that cannot import the
|
|
11
|
+
* engine — the CLI's own tuple and the client's `OutboundEventType` union — and
|
|
12
|
+
* until this file the only thing keeping them in sync was a comment asking
|
|
13
|
+
* nicely. Every event added since has been a coin flip.
|
|
14
|
+
*
|
|
15
|
+
* So: read the two vendored files off disk and compare the FULL sets, not just
|
|
16
|
+
* the newest member. Whoever adds event 32 and forgets a copy gets a failing
|
|
17
|
+
* test naming the file they missed, rather than a subscriber who can never
|
|
18
|
+
* register for it.
|
|
19
|
+
*
|
|
20
|
+
* This is a monorepo-source check by construction — it reads sibling packages'
|
|
21
|
+
* `src/`, which is exactly the coupling it is asserting.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
const CLI_CATALOG = fileURLToPath(
|
|
25
|
+
new URL("../../../cli/src/commands/webhooks.ts", import.meta.url),
|
|
26
|
+
);
|
|
27
|
+
const CLIENT_CATALOG = fileURLToPath(
|
|
28
|
+
new URL("../../../client/src/types.ts", import.meta.url),
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
/** The source between `start` and the next `end`, exclusive. */
|
|
32
|
+
function between(file: string, start: string, end: string): string {
|
|
33
|
+
const source = readFileSync(file, "utf8");
|
|
34
|
+
const from = source.indexOf(start);
|
|
35
|
+
assert.notEqual(from, -1, `${file}: could not find \`${start}\``);
|
|
36
|
+
const to = source.indexOf(end, from + start.length);
|
|
37
|
+
assert.notEqual(
|
|
38
|
+
to,
|
|
39
|
+
-1,
|
|
40
|
+
`${file}: could not find \`${end}\` after \`${start}\``,
|
|
41
|
+
);
|
|
42
|
+
return source.slice(from + start.length, to);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Every `"<namespace>.<name>"` literal in a block of source. */
|
|
46
|
+
function eventLiterals(block: string): string[] {
|
|
47
|
+
return [...block.matchAll(/"([a-z]+\.[a-z_]+)"/g)].map(
|
|
48
|
+
(match) => match[1] as string,
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readCliCatalog(): string[] {
|
|
53
|
+
return eventLiterals(
|
|
54
|
+
between(CLI_CATALOG, "const WEBHOOK_EVENT_TYPES = [", "] as const;"),
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function readClientCatalog(): string[] {
|
|
59
|
+
return eventLiterals(
|
|
60
|
+
between(CLIENT_CATALOG, "export type OutboundEventType =", ";"),
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Set-wise, because the three copies are not in the same ORDER (they never
|
|
65
|
+
* have been) — only membership is load-bearing. */
|
|
66
|
+
function assertSameSet(actual: string[], expected: string[], what: string) {
|
|
67
|
+
const missing = expected.filter((event) => !actual.includes(event));
|
|
68
|
+
const extra = actual.filter(
|
|
69
|
+
(event) => !(expected as string[]).includes(event),
|
|
70
|
+
);
|
|
71
|
+
assert.deepEqual(
|
|
72
|
+
{ missing, extra },
|
|
73
|
+
{ missing: [], extra: [] },
|
|
74
|
+
`${what} has drifted from the engine's WEBHOOK_EVENT_TYPES`,
|
|
75
|
+
);
|
|
76
|
+
assert.equal(
|
|
77
|
+
new Set(actual).size,
|
|
78
|
+
actual.length,
|
|
79
|
+
`${what} contains a duplicate entry`,
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
test("AC 4: contact.refined is present in ALL THREE hand-synced catalogs", () => {
|
|
84
|
+
assert.ok(
|
|
85
|
+
(WEBHOOK_EVENT_TYPES as readonly string[]).includes("contact.refined"),
|
|
86
|
+
"engine WEBHOOK_EVENT_TYPES is missing contact.refined",
|
|
87
|
+
);
|
|
88
|
+
assert.ok(
|
|
89
|
+
readCliCatalog().includes("contact.refined"),
|
|
90
|
+
"packages/cli/src/commands/webhooks.ts is missing contact.refined",
|
|
91
|
+
);
|
|
92
|
+
assert.ok(
|
|
93
|
+
readClientCatalog().includes("contact.refined"),
|
|
94
|
+
"packages/client/src/types.ts is missing contact.refined",
|
|
95
|
+
);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test("AC 4: the CLI's vendored catalog is the engine catalog, entry for entry", () => {
|
|
99
|
+
assertSameSet(
|
|
100
|
+
readCliCatalog(),
|
|
101
|
+
[...WEBHOOK_EVENT_TYPES],
|
|
102
|
+
"packages/cli/src/commands/webhooks.ts",
|
|
103
|
+
);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("AC 4: the client's OutboundEventType union is the engine catalog, entry for entry", () => {
|
|
107
|
+
assertSameSet(
|
|
108
|
+
readClientCatalog(),
|
|
109
|
+
[...WEBHOOK_EVENT_TYPES],
|
|
110
|
+
"packages/client/src/types.ts",
|
|
111
|
+
);
|
|
112
|
+
});
|
|
@@ -25,7 +25,7 @@ import { Webhook } from "svix";
|
|
|
25
25
|
*/
|
|
26
26
|
|
|
27
27
|
/**
|
|
28
|
-
* The
|
|
28
|
+
* The 31-event catalog — the SINGLE source of truth (schema, routes, client,
|
|
29
29
|
* CLI all derive from this). The `webhook.test` sentinel is intentionally NOT a
|
|
30
30
|
* member (it is delivered out-of-band regardless of an endpoint's `eventTypes`).
|
|
31
31
|
*
|
|
@@ -63,6 +63,10 @@ export const WEBHOOK_EVENT_TYPES = [
|
|
|
63
63
|
// Global control group membership (impact plan §4.3) — emitted once per
|
|
64
64
|
// contact key on the first withheld send.
|
|
65
65
|
"contact.control_group",
|
|
66
|
+
// A refinement landed vendor traits on a contact (`refineContact`). Emitted
|
|
67
|
+
// ONLY on a genuine `status: "refined"` — never on `cached`, `not_found` or
|
|
68
|
+
// `skipped`. A cache hit spends nothing and is not an event.
|
|
69
|
+
"contact.refined",
|
|
66
70
|
"email.sent",
|
|
67
71
|
"email.delivered",
|
|
68
72
|
"email.opened",
|
|
@@ -65,22 +65,55 @@ const contactGroupSchema = z.object({
|
|
|
65
65
|
joinedAt: z.string(),
|
|
66
66
|
});
|
|
67
67
|
|
|
68
|
+
/**
|
|
69
|
+
* PRD 06 — validation gate for a jsonb property key used in ordering or
|
|
70
|
+
* filtering. This is a REJECTION layer only: the key is additionally ALWAYS
|
|
71
|
+
* bound as a SQL parameter in the query builder (never interpolated), so the
|
|
72
|
+
* two layers are independent defences, not substitutes for each other.
|
|
73
|
+
*/
|
|
74
|
+
const PROPERTY_KEY_RE = /^[A-Za-z0-9_.-]{1,64}$/;
|
|
75
|
+
|
|
68
76
|
const listRoute = createRoute({
|
|
69
77
|
method: "get",
|
|
70
78
|
path: "/",
|
|
71
79
|
tags: ["Admin"],
|
|
72
80
|
summary: "List contacts",
|
|
73
81
|
request: {
|
|
74
|
-
query: z
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
82
|
+
query: z
|
|
83
|
+
.object({
|
|
84
|
+
limit: z.coerce.number().min(1).max(100).default(50),
|
|
85
|
+
offset: z.coerce.number().min(0).default(0),
|
|
86
|
+
search: z.string().optional(),
|
|
87
|
+
// Long-tail value filters (plan §4b.3): the "find my value customers"
|
|
88
|
+
// query surface.
|
|
89
|
+
minRevenue: z.coerce.number().optional(),
|
|
90
|
+
// Plain string: valid stages are the deployment's configured ladder.
|
|
91
|
+
dealStage: z.string().optional(),
|
|
92
|
+
// Leaderboard surface (PRD 06): rank contacts by the NUMERIC value of
|
|
93
|
+
// a jsonb property ("who do I call today"). Defaults reproduce the
|
|
94
|
+
// pre-existing behaviour exactly (ORDER BY last_seen_at DESC).
|
|
95
|
+
orderBy: z
|
|
96
|
+
.enum(["lastSeenAt", "firstSeenAt", "property"])
|
|
97
|
+
.default("lastSeenAt"),
|
|
98
|
+
orderProperty: z.string().regex(PROPERTY_KEY_RE).optional(),
|
|
99
|
+
orderDir: z.enum(["asc", "desc"]).default("desc"),
|
|
100
|
+
// Numeric property filter: only contacts whose value at `propertyKey`
|
|
101
|
+
// is a real JSON number ≥ `propertyGte`. Composable with ordering.
|
|
102
|
+
propertyKey: z.string().regex(PROPERTY_KEY_RE).optional(),
|
|
103
|
+
propertyGte: z.coerce.number().optional(),
|
|
104
|
+
})
|
|
105
|
+
.refine(
|
|
106
|
+
(q) => q.orderBy !== "property" || q.orderProperty !== undefined,
|
|
107
|
+
{
|
|
108
|
+
message: "orderProperty is required when orderBy=property",
|
|
109
|
+
},
|
|
110
|
+
)
|
|
111
|
+
.refine(
|
|
112
|
+
(q) => q.propertyGte === undefined || q.propertyKey !== undefined,
|
|
113
|
+
{
|
|
114
|
+
message: "propertyKey is required when propertyGte is set",
|
|
115
|
+
},
|
|
116
|
+
),
|
|
84
117
|
},
|
|
85
118
|
responses: {
|
|
86
119
|
200: {
|
|
@@ -96,6 +129,16 @@ const listRoute = createRoute({
|
|
|
96
129
|
},
|
|
97
130
|
description: "Paginated contact list",
|
|
98
131
|
},
|
|
132
|
+
400: {
|
|
133
|
+
content: {
|
|
134
|
+
"application/json": {
|
|
135
|
+
// zod-openapi's default validation hook shape (ZodError passthrough).
|
|
136
|
+
schema: z.object({ success: z.boolean() }).passthrough(),
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
description:
|
|
140
|
+
"Invalid query (bad property key, or orderBy=property without orderProperty)",
|
|
141
|
+
},
|
|
99
142
|
},
|
|
100
143
|
});
|
|
101
144
|
|
|
@@ -246,11 +289,67 @@ const serializeContact = (row: typeof contacts.$inferSelect) =>
|
|
|
246
289
|
export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
247
290
|
.openapi(listRoute, async (c) => {
|
|
248
291
|
const { db } = c.get("container");
|
|
249
|
-
const {
|
|
250
|
-
|
|
292
|
+
const {
|
|
293
|
+
limit,
|
|
294
|
+
offset,
|
|
295
|
+
search,
|
|
296
|
+
minRevenue,
|
|
297
|
+
dealStage,
|
|
298
|
+
orderBy,
|
|
299
|
+
orderProperty,
|
|
300
|
+
orderDir,
|
|
301
|
+
propertyKey,
|
|
302
|
+
propertyGte,
|
|
303
|
+
} = c.req.valid("query");
|
|
251
304
|
|
|
252
305
|
const searchFilter = search ? contactSearchFilter(search) : undefined;
|
|
253
306
|
|
|
307
|
+
// Type-guarded numeric read of a jsonb property (PRD 06 / DECISIONS §3.4).
|
|
308
|
+
// `contacts.properties` is untyped jsonb and `/v1/events` accepts arbitrary
|
|
309
|
+
// values, so a bare `(properties->>key)::numeric` raises Postgres 22P02 —
|
|
310
|
+
// and 500s the whole request — the first time ONE contact holds e.g. "n/a"
|
|
311
|
+
// at the key. Only real JSON numbers count; anything else yields NULL
|
|
312
|
+
// (sorted last, excluded by the ≥ filter). The key rides as a BOUND
|
|
313
|
+
// PARAMETER — never interpolated — with the regex validation above as a
|
|
314
|
+
// second, independent layer.
|
|
315
|
+
const numericProperty = (key: string) =>
|
|
316
|
+
sql`CASE WHEN jsonb_typeof(${contacts.properties} -> ${key}) = 'number' THEN (${contacts.properties} ->> ${key})::numeric END`;
|
|
317
|
+
|
|
318
|
+
// Filter applies only when both halves are present (the schema 400s a
|
|
319
|
+
// dangling propertyGte). A NULL from the guard fails the ≥ comparison, so
|
|
320
|
+
// non-numeric holders are excluded rather than erroring.
|
|
321
|
+
const propertyFilter =
|
|
322
|
+
propertyKey !== undefined && propertyGte !== undefined
|
|
323
|
+
? sql`${numericProperty(propertyKey)} >= ${propertyGte}`
|
|
324
|
+
: undefined;
|
|
325
|
+
|
|
326
|
+
// NULLS LAST in BOTH directions so unscored (or non-numeric) contacts
|
|
327
|
+
// never top the list; last_seen_at breaks ties deterministically. The
|
|
328
|
+
// default path (`orderBy=lastSeenAt`, `orderDir=desc`) is byte-identical
|
|
329
|
+
// to the pre-PRD-06 hardcoded ORDER BY — existing callers see no change.
|
|
330
|
+
// NOTE: the GIN index on `properties` accelerates containment filters
|
|
331
|
+
// only; this ORDER BY is a sequential scan — fine at GTM volume
|
|
332
|
+
// (thousands of contacts), not at analytics volume.
|
|
333
|
+
const orderClauses =
|
|
334
|
+
orderBy === "property" && orderProperty !== undefined
|
|
335
|
+
? [
|
|
336
|
+
orderDir === "asc"
|
|
337
|
+
? sql`${numericProperty(orderProperty)} asc nulls last`
|
|
338
|
+
: sql`${numericProperty(orderProperty)} desc nulls last`,
|
|
339
|
+
desc(contacts.lastSeenAt),
|
|
340
|
+
]
|
|
341
|
+
: orderBy === "firstSeenAt"
|
|
342
|
+
? [
|
|
343
|
+
orderDir === "asc"
|
|
344
|
+
? asc(contacts.firstSeenAt)
|
|
345
|
+
: desc(contacts.firstSeenAt),
|
|
346
|
+
]
|
|
347
|
+
: [
|
|
348
|
+
orderDir === "asc"
|
|
349
|
+
? asc(contacts.lastSeenAt)
|
|
350
|
+
: desc(contacts.lastSeenAt),
|
|
351
|
+
];
|
|
352
|
+
|
|
254
353
|
// Valued events are keyed by the contact's canonical event key
|
|
255
354
|
// (external_id ?? anonymous_id ?? id) — same precedence ingestEvent
|
|
256
355
|
// resolves. Served by the partial user_events_valued_user_idx.
|
|
@@ -284,6 +383,7 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
|
284
383
|
...(searchFilter ? [searchFilter] : []),
|
|
285
384
|
...(revenueFilter ? [revenueFilter] : []),
|
|
286
385
|
...(dealStageFilter ? [dealStageFilter] : []),
|
|
386
|
+
...(propertyFilter ? [propertyFilter] : []),
|
|
287
387
|
);
|
|
288
388
|
|
|
289
389
|
const [rows, totalRows] = await Promise.all([
|
|
@@ -291,7 +391,7 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
|
291
391
|
.select()
|
|
292
392
|
.from(contacts)
|
|
293
393
|
.where(where)
|
|
294
|
-
.orderBy(
|
|
394
|
+
.orderBy(...orderClauses)
|
|
295
395
|
.limit(limit)
|
|
296
396
|
.offset(offset),
|
|
297
397
|
db.select({ count: count() }).from(contacts).where(where),
|
|
@@ -346,6 +346,10 @@ async function reevalLeaves(opts: {
|
|
|
346
346
|
id: bucketMemberships.id,
|
|
347
347
|
userId: bucketMemberships.userId,
|
|
348
348
|
userEmail: bucketMemberships.userEmail,
|
|
349
|
+
// Joined contact row → its id rides along as the ENGINE-INTERNAL
|
|
350
|
+
// provenance pin for the leave emit, so the re-ingest folds into the
|
|
351
|
+
// known row instead of minting a phantom `external_id` twin (issue #608).
|
|
352
|
+
contactId: contacts.id,
|
|
349
353
|
entryCount: bucketMemberships.entryCount,
|
|
350
354
|
})
|
|
351
355
|
.from(bucketMemberships)
|
|
@@ -362,6 +366,13 @@ async function reevalLeaves(opts: {
|
|
|
362
366
|
const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
|
|
363
367
|
if (leavers.length === 0) return 0;
|
|
364
368
|
|
|
369
|
+
// Membership key → contact row id for the emit's provenance pin. Keying by
|
|
370
|
+
// userId is safe: the partial-active unique index guarantees one active row
|
|
371
|
+
// per (user, bucket), so a flipped row maps to exactly one leaver.
|
|
372
|
+
const contactIdByUser = new Map(
|
|
373
|
+
leavers.map((m) => [m.userId, m.contactId] as const),
|
|
374
|
+
);
|
|
375
|
+
|
|
365
376
|
let leftCount = 0;
|
|
366
377
|
for (let i = 0; i < leavers.length; i += BATCH_SIZE) {
|
|
367
378
|
const chunk = leavers.slice(i, i + BATCH_SIZE);
|
|
@@ -400,6 +411,9 @@ async function reevalLeaves(opts: {
|
|
|
400
411
|
bucket,
|
|
401
412
|
userId: row.userId,
|
|
402
413
|
userEmail: row.userEmail,
|
|
414
|
+
// Provenance pin from the leaver's already-joined contact row, so the
|
|
415
|
+
// leave re-ingest folds into it instead of minting a twin (issue #608).
|
|
416
|
+
contactId: contactIdByUser.get(row.userId),
|
|
403
417
|
epoch: row.entryCount,
|
|
404
418
|
source: "backfill",
|
|
405
419
|
reason: "criteria",
|
|
@@ -283,6 +283,13 @@ export const bucketExpiryTask = hatchet.durableTask({
|
|
|
283
283
|
bucket,
|
|
284
284
|
userId: input.userId,
|
|
285
285
|
userEmail: input.userEmail,
|
|
286
|
+
// NO `contactId` provenance pin here (issue #608): this task has no
|
|
287
|
+
// contact row in scope, and the only channel that could carry one — the
|
|
288
|
+
// `bucket:arm-expiry` Hatchet event payload — is reachable from public
|
|
289
|
+
// ingest pushes, so an id smuggled through it would not be the
|
|
290
|
+
// unforgeable engine-internal provenance the pin requires. A value
|
|
291
|
+
// lookup here would re-implement exactly the resolution the pin exists
|
|
292
|
+
// to bypass. Degrades to the pre-pin resolve.
|
|
286
293
|
epoch: flipped.entryCount,
|
|
287
294
|
source: "reconcile",
|
|
288
295
|
// Fast-expiry is a criteria re-confirm leave (Section 6.7).
|
|
@@ -293,6 +300,19 @@ export const bucketExpiryTask = hatchet.durableTask({
|
|
|
293
300
|
},
|
|
294
301
|
});
|
|
295
302
|
|
|
303
|
+
/**
|
|
304
|
+
* A member selected to leave: the membership's canonical string key plus the
|
|
305
|
+
* joined contact row's id — the ENGINE-INTERNAL provenance pin threaded into
|
|
306
|
+
* `emitBucketTransition` so the leave re-ingest folds into the known row
|
|
307
|
+
* instead of minting a phantom `external_id` twin (issue #608). Every leaver
|
|
308
|
+
* source query already joins `contacts` (the GDPR live-contact filter), so the
|
|
309
|
+
* id is free.
|
|
310
|
+
*/
|
|
311
|
+
interface BucketLeaver {
|
|
312
|
+
userId: string;
|
|
313
|
+
contactId: string;
|
|
314
|
+
}
|
|
315
|
+
|
|
296
316
|
/**
|
|
297
317
|
* Set-based SHOULD-LEAVE for one time-based bucket → bulk CAS → RETURNING-gated
|
|
298
318
|
* emit. For single-event `not_exists`/`exists`/`count within` criteria the
|
|
@@ -311,14 +331,14 @@ async function reconcileBucketLeaves(opts: {
|
|
|
311
331
|
|
|
312
332
|
// A single-event/within/count criterion → set-based SHOULD-LEAVE query.
|
|
313
333
|
if (criteria.type === "event") {
|
|
314
|
-
const
|
|
315
|
-
if (
|
|
334
|
+
const leavers = await selectEventLeavers(db, bucket, criteria);
|
|
335
|
+
if (leavers.length === 0) return 0;
|
|
316
336
|
return bulkLeave({
|
|
317
337
|
db,
|
|
318
338
|
logger,
|
|
319
339
|
journeyRegistry,
|
|
320
340
|
bucket,
|
|
321
|
-
|
|
341
|
+
leavers,
|
|
322
342
|
reason: "criteria",
|
|
323
343
|
});
|
|
324
344
|
}
|
|
@@ -338,7 +358,7 @@ async function selectEventLeavers(
|
|
|
338
358
|
db: Database,
|
|
339
359
|
bucket: BucketMeta,
|
|
340
360
|
criteria: Extract<ConditionEval, { type: "event" }>,
|
|
341
|
-
): Promise<
|
|
361
|
+
): Promise<BucketLeaver[]> {
|
|
342
362
|
const cutoff = criteria.within
|
|
343
363
|
? new Date(Date.now() - durationToMs(criteria.within))
|
|
344
364
|
: null;
|
|
@@ -372,10 +392,13 @@ async function selectEventLeavers(
|
|
|
372
392
|
.groupBy(userEvents.userId)
|
|
373
393
|
.as("counted");
|
|
374
394
|
|
|
375
|
-
// LEFT JOIN members → windowed counts. A missing/zero count is a 0.
|
|
395
|
+
// LEFT JOIN members → windowed counts. A missing/zero count is a 0. The
|
|
396
|
+
// contact row is already joined (GDPR filter), so its id rides along as the
|
|
397
|
+
// provenance pin for the leave emit (issue #608).
|
|
376
398
|
const rows = await db
|
|
377
399
|
.select({
|
|
378
400
|
userId: members.userId,
|
|
401
|
+
contactId: contacts.id,
|
|
379
402
|
cnt: sql<number>`coalesce(${counted.cnt}, 0)`,
|
|
380
403
|
})
|
|
381
404
|
.from(members)
|
|
@@ -385,7 +408,7 @@ async function selectEventLeavers(
|
|
|
385
408
|
|
|
386
409
|
return rows
|
|
387
410
|
.filter((r) => shouldLeaveByCount(criteria, Number(r.cnt)))
|
|
388
|
-
.map((r) => r.userId);
|
|
411
|
+
.map((r) => ({ userId: r.userId, contactId: r.contactId }));
|
|
389
412
|
}
|
|
390
413
|
|
|
391
414
|
/**
|
|
@@ -409,6 +432,9 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
409
432
|
const members = await db
|
|
410
433
|
.select({
|
|
411
434
|
userId: bucketMemberships.userId,
|
|
435
|
+
// Joined contact row → its id is the provenance pin for the leave emit
|
|
436
|
+
// (issue #608).
|
|
437
|
+
contactId: contacts.id,
|
|
412
438
|
properties: contacts.properties,
|
|
413
439
|
})
|
|
414
440
|
.from(bucketMemberships)
|
|
@@ -424,7 +450,7 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
424
450
|
.orderBy(sql`${bucketMemberships.lastEvaluatedAt} asc nulls first`)
|
|
425
451
|
.limit(BATCH_SIZE);
|
|
426
452
|
|
|
427
|
-
const
|
|
453
|
+
const leavers: BucketLeaver[] = [];
|
|
428
454
|
const evaluatedIds: string[] = [];
|
|
429
455
|
for (const member of members) {
|
|
430
456
|
evaluatedIds.push(member.userId);
|
|
@@ -435,7 +461,9 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
435
461
|
condition: criteria,
|
|
436
462
|
ctx: { db, userId: member.userId, journeyContext },
|
|
437
463
|
});
|
|
438
|
-
if (!isMember)
|
|
464
|
+
if (!isMember) {
|
|
465
|
+
leavers.push({ userId: member.userId, contactId: member.contactId });
|
|
466
|
+
}
|
|
439
467
|
}
|
|
440
468
|
|
|
441
469
|
// Bump lastEvaluatedAt for the whole chunk so the next tick advances the cursor
|
|
@@ -453,13 +481,13 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
453
481
|
);
|
|
454
482
|
}
|
|
455
483
|
|
|
456
|
-
if (
|
|
484
|
+
if (leavers.length === 0) return 0;
|
|
457
485
|
return bulkLeave({
|
|
458
486
|
db,
|
|
459
487
|
logger,
|
|
460
488
|
journeyRegistry,
|
|
461
489
|
bucket,
|
|
462
|
-
|
|
490
|
+
leavers,
|
|
463
491
|
reason: "criteria",
|
|
464
492
|
});
|
|
465
493
|
}
|
|
@@ -481,7 +509,12 @@ async function reconcileBucketTtlLeaves(opts: {
|
|
|
481
509
|
const { db, logger, journeyRegistry, bucket } = opts;
|
|
482
510
|
|
|
483
511
|
const expired = await db
|
|
484
|
-
.select({
|
|
512
|
+
.select({
|
|
513
|
+
userId: bucketMemberships.userId,
|
|
514
|
+
// Joined contact row → its id is the provenance pin for the leave emit
|
|
515
|
+
// (issue #608).
|
|
516
|
+
contactId: contacts.id,
|
|
517
|
+
})
|
|
485
518
|
.from(bucketMemberships)
|
|
486
519
|
.innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
|
|
487
520
|
.where(
|
|
@@ -501,7 +534,7 @@ async function reconcileBucketTtlLeaves(opts: {
|
|
|
501
534
|
logger,
|
|
502
535
|
journeyRegistry,
|
|
503
536
|
bucket,
|
|
504
|
-
|
|
537
|
+
leavers: expired,
|
|
505
538
|
reason: "maxDwell",
|
|
506
539
|
});
|
|
507
540
|
}
|
|
@@ -518,11 +551,18 @@ async function bulkLeave(opts: {
|
|
|
518
551
|
logger: Logger;
|
|
519
552
|
journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
|
|
520
553
|
bucket: BucketMeta;
|
|
521
|
-
|
|
554
|
+
leavers: BucketLeaver[];
|
|
522
555
|
/** Why these members leave — TTL passes "maxDwell", criteria passes "criteria". */
|
|
523
556
|
reason: BucketLeaveReason;
|
|
524
557
|
}): Promise<number> {
|
|
525
|
-
const { db, logger, journeyRegistry, bucket,
|
|
558
|
+
const { db, logger, journeyRegistry, bucket, leavers, reason } = opts;
|
|
559
|
+
const userIds = leavers.map((l) => l.userId);
|
|
560
|
+
// Membership key → resolved contact row id, for the emit's provenance pin.
|
|
561
|
+
// Safe to key by userId: the partial-active unique index guarantees one
|
|
562
|
+
// active row per (user, bucket), so a flipped row maps to exactly one leaver.
|
|
563
|
+
const contactIdByUser = new Map(
|
|
564
|
+
leavers.map((l) => [l.userId, l.contactId] as const),
|
|
565
|
+
);
|
|
526
566
|
|
|
527
567
|
const dwellMs = bucket.minDwell ? durationToMs(bucket.minDwell) : 0;
|
|
528
568
|
const dwellCutoff = dwellMs > 0 ? new Date(Date.now() - dwellMs) : null;
|
|
@@ -565,6 +605,9 @@ async function bulkLeave(opts: {
|
|
|
565
605
|
bucket,
|
|
566
606
|
userId: row.userId,
|
|
567
607
|
userEmail: row.userEmail,
|
|
608
|
+
// Provenance pin from the leaver's already-joined contact row, so the
|
|
609
|
+
// leave re-ingest folds into it instead of minting a twin (issue #608).
|
|
610
|
+
contactId: contactIdByUser.get(row.userId),
|
|
568
611
|
epoch: row.entryCount,
|
|
569
612
|
source: "reconcile",
|
|
570
613
|
reason,
|
|
@@ -643,6 +686,9 @@ async function reconcileBucketDwell(opts: {
|
|
|
643
686
|
id: bucketMemberships.id,
|
|
644
687
|
userId: bucketMemberships.userId,
|
|
645
688
|
userEmail: bucketMemberships.userEmail,
|
|
689
|
+
// Joined contact row → its id is the provenance pin for the dwell emit
|
|
690
|
+
// (issue #608).
|
|
691
|
+
contactId: contacts.id,
|
|
646
692
|
entryCount: bucketMemberships.entryCount,
|
|
647
693
|
anchor: sql<Date>`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt})`,
|
|
648
694
|
dwellState: bucketMemberships.dwellState,
|
|
@@ -704,6 +750,9 @@ async function reconcileBucketDwell(opts: {
|
|
|
704
750
|
bucket,
|
|
705
751
|
userId: m.userId,
|
|
706
752
|
userEmail: m.userEmail,
|
|
753
|
+
// Provenance pin from the candidate's already-joined contact row
|
|
754
|
+
// (issue #608).
|
|
755
|
+
contactId: m.contactId,
|
|
707
756
|
epoch: m.entryCount,
|
|
708
757
|
source: "reconcile",
|
|
709
758
|
dwellLabel: label,
|
|
@@ -857,6 +906,11 @@ async function reconcileBucketJoins(opts: {
|
|
|
857
906
|
const baseQuery = db
|
|
858
907
|
.select({
|
|
859
908
|
userId: contactKey,
|
|
909
|
+
// The candidate scan reads straight off `contacts`, so the row id rides
|
|
910
|
+
// along as the provenance pin for the join emit (issue #608) —
|
|
911
|
+
// load-bearing for exactly the null-external_id contacts this coalesce
|
|
912
|
+
// join exists to include.
|
|
913
|
+
contactId: contacts.id,
|
|
860
914
|
email: contacts.email,
|
|
861
915
|
})
|
|
862
916
|
.from(contacts)
|
|
@@ -934,6 +988,7 @@ async function reconcileBucketJoins(opts: {
|
|
|
934
988
|
bucket,
|
|
935
989
|
userId: candidate.userId,
|
|
936
990
|
userEmail: candidate.email ?? null,
|
|
991
|
+
contactId: candidate.contactId,
|
|
937
992
|
});
|
|
938
993
|
if (transitioned) joined += 1;
|
|
939
994
|
}
|
|
@@ -1035,8 +1090,10 @@ async function reconcileJoinOne(opts: {
|
|
|
1035
1090
|
bucket: BucketMeta;
|
|
1036
1091
|
userId: string;
|
|
1037
1092
|
userEmail: string | null;
|
|
1093
|
+
/** Provenance pin for the join emit — see {@link BucketLeaver} (issue #608). */
|
|
1094
|
+
contactId?: string;
|
|
1038
1095
|
}): Promise<boolean> {
|
|
1039
|
-
const { db, logger, journeyRegistry, bucket, userId } = opts;
|
|
1096
|
+
const { db, logger, journeyRegistry, bucket, userId, contactId } = opts;
|
|
1040
1097
|
// Normalized at the write site (audience-model.md wart #1) — the contact row
|
|
1041
1098
|
// should already be normalized, but the membership keyspace must never
|
|
1042
1099
|
// depend on that.
|
|
@@ -1081,6 +1138,8 @@ async function reconcileJoinOne(opts: {
|
|
|
1081
1138
|
bucket,
|
|
1082
1139
|
userId,
|
|
1083
1140
|
userEmail,
|
|
1141
|
+
// Provenance pin from the candidate scan's contact row (issue #608).
|
|
1142
|
+
contactId,
|
|
1084
1143
|
epoch,
|
|
1085
1144
|
source: "reconcile",
|
|
1086
1145
|
});
|