@hogsend/engine 0.54.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/check-membership.ts +45 -3
- package/src/lib/bucket-emit.test.ts +145 -0
- package/src/lib/bucket-emit.ts +29 -3
- package/src/lib/email-providers-from-env.ts +15 -16
- package/src/lib/enrichment-providers-from-env.test.ts +5 -44
- package/src/lib/enrichment-providers-from-env.ts +8 -41
- package/src/lib/ingestion.ts +5 -0
- package/src/lib/load-optional-plugin.test.ts +133 -0
- package/src/lib/load-optional-plugin.ts +142 -0
- package/src/lib/sms-providers-from-env.ts +11 -11
- package/src/workflows/bucket-backfill.ts +14 -0
- package/src/workflows/bucket-reconcile.ts +74 -15
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hogsend/engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.55.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -53,18 +53,18 @@
|
|
|
53
53
|
"svix": "^1.95.1",
|
|
54
54
|
"winston": "^3.19.0",
|
|
55
55
|
"zod": "^4.4.3",
|
|
56
|
-
"@hogsend/attribution": "^0.
|
|
57
|
-
"@hogsend/core": "^0.
|
|
58
|
-
"@hogsend/db": "^0.
|
|
59
|
-
"@hogsend/email": "^0.
|
|
60
|
-
"@hogsend/plugin-posthog": "^0.
|
|
61
|
-
"@hogsend/plugin-resend": "^0.
|
|
62
|
-
"@hogsend/sms": "^0.
|
|
56
|
+
"@hogsend/attribution": "^0.55.0",
|
|
57
|
+
"@hogsend/core": "^0.55.0",
|
|
58
|
+
"@hogsend/db": "^0.55.0",
|
|
59
|
+
"@hogsend/email": "^0.55.0",
|
|
60
|
+
"@hogsend/plugin-posthog": "^0.55.0",
|
|
61
|
+
"@hogsend/plugin-resend": "^0.55.0",
|
|
62
|
+
"@hogsend/sms": "^0.55.0"
|
|
63
63
|
},
|
|
64
64
|
"optionalDependencies": {
|
|
65
|
-
"@hogsend/plugin-apollo": "^0.
|
|
66
|
-
"@hogsend/plugin-postmark": "^0.
|
|
67
|
-
"@hogsend/plugin-twilio": "^0.
|
|
65
|
+
"@hogsend/plugin-apollo": "^0.55.0",
|
|
66
|
+
"@hogsend/plugin-postmark": "^0.55.0",
|
|
67
|
+
"@hogsend/plugin-twilio": "^0.55.0"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/node": "^22.15.3",
|
|
@@ -51,6 +51,16 @@ export async function checkBucketMembership(opts: {
|
|
|
51
51
|
logger: Logger;
|
|
52
52
|
userId: string;
|
|
53
53
|
userEmail: string | null;
|
|
54
|
+
/**
|
|
55
|
+
* The subject contact's resolved row id (`contacts.id`) — `ingestEvent` has
|
|
56
|
+
* already resolved the contact before calling us, so it threads the id here
|
|
57
|
+
* and we carry it into every `emitBucketTransition` as the ENGINE-INTERNAL
|
|
58
|
+
* provenance pin. Without it, a transition for a contact whose canonical key
|
|
59
|
+
* is its `anonymous_id` re-ingests a bare `userId` and mints a phantom
|
|
60
|
+
* `external_id` twin (issue #608). Optional: a caller without a resolved
|
|
61
|
+
* contact degrades to the pin-less emit.
|
|
62
|
+
*/
|
|
63
|
+
contactId?: string;
|
|
54
64
|
event: string;
|
|
55
65
|
/**
|
|
56
66
|
* D2: the event payload — candidate-narrowing ONLY. It NO LONGER participates
|
|
@@ -71,6 +81,7 @@ export async function checkBucketMembership(opts: {
|
|
|
71
81
|
hatchet,
|
|
72
82
|
logger,
|
|
73
83
|
userId,
|
|
84
|
+
contactId,
|
|
74
85
|
event,
|
|
75
86
|
eventProperties,
|
|
76
87
|
contactProperties: contactPropertiesPatch,
|
|
@@ -209,6 +220,7 @@ export async function checkBucketMembership(opts: {
|
|
|
209
220
|
bucket,
|
|
210
221
|
userId,
|
|
211
222
|
userEmail,
|
|
223
|
+
contactId,
|
|
212
224
|
});
|
|
213
225
|
if (transition) transitions.push(transition);
|
|
214
226
|
} else if (wasMember && isMember) {
|
|
@@ -227,6 +239,7 @@ export async function checkBucketMembership(opts: {
|
|
|
227
239
|
active,
|
|
228
240
|
userId,
|
|
229
241
|
userEmail,
|
|
242
|
+
contactId,
|
|
230
243
|
});
|
|
231
244
|
if (transition) transitions.push(transition);
|
|
232
245
|
}
|
|
@@ -244,8 +257,19 @@ async function handleJoin(opts: {
|
|
|
244
257
|
bucket: BucketMeta;
|
|
245
258
|
userId: string;
|
|
246
259
|
userEmail: string | null;
|
|
260
|
+
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
261
|
+
contactId?: string;
|
|
247
262
|
}): Promise<BucketTransition | null> {
|
|
248
|
-
const {
|
|
263
|
+
const {
|
|
264
|
+
db,
|
|
265
|
+
registry,
|
|
266
|
+
hatchet,
|
|
267
|
+
logger,
|
|
268
|
+
bucket,
|
|
269
|
+
userId,
|
|
270
|
+
userEmail,
|
|
271
|
+
contactId,
|
|
272
|
+
} = opts;
|
|
249
273
|
|
|
250
274
|
// entryCount ordinal = 1 + count of ALL prior memberships (active + left) for
|
|
251
275
|
// this (user, bucket) (Section 6.3 / 8.2). priorCount also drives the entryLimit
|
|
@@ -312,6 +336,10 @@ async function handleJoin(opts: {
|
|
|
312
336
|
bucket,
|
|
313
337
|
userId,
|
|
314
338
|
userEmail,
|
|
339
|
+
// Pin the re-ingest to the already-resolved contact row so an
|
|
340
|
+
// anonymous-only contact's transition folds in instead of minting a
|
|
341
|
+
// phantom twin (issue #608).
|
|
342
|
+
contactId,
|
|
315
343
|
epoch,
|
|
316
344
|
source: "event",
|
|
317
345
|
});
|
|
@@ -335,9 +363,20 @@ async function handleLeave(opts: {
|
|
|
335
363
|
active: typeof bucketMemberships.$inferSelect;
|
|
336
364
|
userId: string;
|
|
337
365
|
userEmail: string | null;
|
|
366
|
+
/** Provenance pin for the emit — see `checkBucketMembership` (issue #608). */
|
|
367
|
+
contactId?: string;
|
|
338
368
|
}): Promise<BucketTransition | null> {
|
|
339
|
-
const {
|
|
340
|
-
|
|
369
|
+
const {
|
|
370
|
+
db,
|
|
371
|
+
registry,
|
|
372
|
+
hatchet,
|
|
373
|
+
logger,
|
|
374
|
+
bucket,
|
|
375
|
+
active,
|
|
376
|
+
userId,
|
|
377
|
+
userEmail,
|
|
378
|
+
contactId,
|
|
379
|
+
} = opts;
|
|
341
380
|
|
|
342
381
|
// minDwell DEFERS (never silently drops) the leave (Section 6.3). We set a
|
|
343
382
|
// pending-leave deadline on expiresAt = enteredAt + minDwell so the reconcile
|
|
@@ -401,6 +440,9 @@ async function handleLeave(opts: {
|
|
|
401
440
|
bucket,
|
|
402
441
|
userId,
|
|
403
442
|
userEmail,
|
|
443
|
+
// Same provenance pin as the join emit — a leave re-ingests through the
|
|
444
|
+
// identical path and would mint the same phantom twin (issue #608).
|
|
445
|
+
contactId,
|
|
404
446
|
epoch: flipped.entryCount,
|
|
405
447
|
source: "event",
|
|
406
448
|
reason: "criteria",
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
4
|
+
import type { BucketMeta } from "@hogsend/core";
|
|
5
|
+
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
6
|
+
import type { Database } from "@hogsend/db";
|
|
7
|
+
import type { IngestEvent, ingestEvent } from "./ingestion.js";
|
|
8
|
+
import type { Logger } from "./logger.js";
|
|
9
|
+
|
|
10
|
+
// bucket-emit's import chain reaches env.ts (validates required env vars at
|
|
11
|
+
// import time) and lib/hatchet.ts (parses HATCHET_CLIENT_TOKEN as a JWT at
|
|
12
|
+
// import time) — stub both BEFORE the dynamic import. Nothing is ever
|
|
13
|
+
// contacted: the ingest seam intercepts everything. The token is the same
|
|
14
|
+
// well-formed dummy JWT apps/api's vitest config injects.
|
|
15
|
+
process.env.DATABASE_URL ??= "postgresql://test:test@localhost:5432/test";
|
|
16
|
+
process.env.BETTER_AUTH_SECRET ??= "test-secret";
|
|
17
|
+
process.env.HATCHET_CLIENT_TOKEN ??=
|
|
18
|
+
"eyJhbGciOiJFUzI1NiIsImtpZCI6InRlc3QifQ.eyJhdWQiOiJsb2NhbGhvc3QiLCJleHAiOjQ5MzMyNDA5ODMsImdycGNfYnJvYWRjYXN0X2FkZHJlc3MiOiJsb2NhbGhvc3Q6NzA3NyIsImlhdCI6MTc3OTY0MDk4MywiaXNzIjoibG9jYWxob3N0Iiwic2VydmVyX3VybCI6ImxvY2FsaG9zdCIsInN1YiI6InRlc3QtdGVuYW50LWlkIiwidG9rZW5faWQiOiJ0ZXN0LXRva2VuLWlkIn0.test";
|
|
19
|
+
const { emitBucketTransition } = await import("./bucket-emit.js");
|
|
20
|
+
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Issue #608 — the `contactId` provenance pin. `emitBucketTransition`
|
|
23
|
+
// re-ingests every transition; `userId` is the contact's CANONICAL key
|
|
24
|
+
// (`external_id ?? anonymous_id ?? id`), but the resolver treats a bare
|
|
25
|
+
// `userId` as an EXTERNAL key, so an anonymous-only contact gets a phantom
|
|
26
|
+
// `external_id` twin minted per transition unless the pin rides the re-ingest.
|
|
27
|
+
// These tests assert at the `ingest` seam (no live DB — the Section 14
|
|
28
|
+
// pattern): the pin must ride BOTH re-ingests (per-bucket alias AND generic),
|
|
29
|
+
// and its absence must degrade to exactly the pin-less call shape.
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
const noop = () => {};
|
|
33
|
+
const logger: Logger = {
|
|
34
|
+
debug: noop,
|
|
35
|
+
info: noop,
|
|
36
|
+
warn: noop,
|
|
37
|
+
error: noop,
|
|
38
|
+
} as unknown as Logger;
|
|
39
|
+
|
|
40
|
+
const bucket: BucketMeta = {
|
|
41
|
+
id: "power-users",
|
|
42
|
+
name: "Power users",
|
|
43
|
+
} as BucketMeta;
|
|
44
|
+
|
|
45
|
+
/** A registry stub with a journey bound to the generic `bucket:<kind>` form, so
|
|
46
|
+
* the SECOND (generic) re-ingest actually fires and can be asserted on. */
|
|
47
|
+
function genericBoundRegistry(): JourneyRegistry {
|
|
48
|
+
return {
|
|
49
|
+
getByTriggerEvent: (event: string) =>
|
|
50
|
+
event === "bucket:entered" || event === "bucket:left" ? [{}] : [],
|
|
51
|
+
} as unknown as JourneyRegistry;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Recording ingest stub — captures each re-ingested event. */
|
|
55
|
+
function recordingIngest() {
|
|
56
|
+
const events: IngestEvent[] = [];
|
|
57
|
+
const ingest: typeof ingestEvent = async (opts) => {
|
|
58
|
+
events.push(opts.event);
|
|
59
|
+
return { stored: true, exits: [], contactKey: opts.event.userId ?? "" };
|
|
60
|
+
};
|
|
61
|
+
return { events, ingest };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// db/hatchet are never touched by the stubbed ingest; emitOutbound is
|
|
65
|
+
// fire-and-forget (`void ...catch(logger.warn)`) so its rejection on the empty
|
|
66
|
+
// db stub is swallowed by design.
|
|
67
|
+
const db = {} as Database;
|
|
68
|
+
const hatchet = {} as HatchetClient;
|
|
69
|
+
|
|
70
|
+
const CONTACT_ID = "9f3a1c2e-0000-4000-8000-000000000608";
|
|
71
|
+
|
|
72
|
+
test("entered: the contactId pin rides BOTH re-ingests (alias + generic)", async () => {
|
|
73
|
+
const { events, ingest } = recordingIngest();
|
|
74
|
+
|
|
75
|
+
await emitBucketTransition({
|
|
76
|
+
db,
|
|
77
|
+
registry: genericBoundRegistry(),
|
|
78
|
+
hatchet,
|
|
79
|
+
logger,
|
|
80
|
+
kind: "entered",
|
|
81
|
+
bucket,
|
|
82
|
+
userId: "anon-608",
|
|
83
|
+
userEmail: null,
|
|
84
|
+
contactId: CONTACT_ID,
|
|
85
|
+
epoch: 1,
|
|
86
|
+
ingest,
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
assert.equal(events.length, 2);
|
|
90
|
+
const [alias, generic] = events;
|
|
91
|
+
assert.equal(alias?.event, "bucket:entered:power-users");
|
|
92
|
+
assert.equal(generic?.event, "bucket:entered");
|
|
93
|
+
// The pin on EACH leg — dropping it on either one leaves the phantom-twin
|
|
94
|
+
// mint half-alive (issue #608).
|
|
95
|
+
assert.equal(alias?.contactId, CONTACT_ID);
|
|
96
|
+
assert.equal(generic?.contactId, CONTACT_ID);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("left: the contactId pin rides the leave re-ingest too", async () => {
|
|
100
|
+
const { events, ingest } = recordingIngest();
|
|
101
|
+
|
|
102
|
+
await emitBucketTransition({
|
|
103
|
+
db,
|
|
104
|
+
registry: genericBoundRegistry(),
|
|
105
|
+
hatchet,
|
|
106
|
+
logger,
|
|
107
|
+
kind: "left",
|
|
108
|
+
bucket,
|
|
109
|
+
userId: "anon-608",
|
|
110
|
+
userEmail: null,
|
|
111
|
+
contactId: CONTACT_ID,
|
|
112
|
+
epoch: 2,
|
|
113
|
+
reason: "criteria",
|
|
114
|
+
ingest,
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
assert.equal(events.length, 2);
|
|
118
|
+
for (const event of events) {
|
|
119
|
+
assert.equal(event.contactId, CONTACT_ID);
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("no contactId: degrades to exactly the pin-less re-ingest", async () => {
|
|
124
|
+
const { events, ingest } = recordingIngest();
|
|
125
|
+
|
|
126
|
+
await emitBucketTransition({
|
|
127
|
+
db,
|
|
128
|
+
registry: genericBoundRegistry(),
|
|
129
|
+
hatchet,
|
|
130
|
+
logger,
|
|
131
|
+
kind: "entered",
|
|
132
|
+
bucket,
|
|
133
|
+
userId: "anon-608",
|
|
134
|
+
userEmail: null,
|
|
135
|
+
epoch: 1,
|
|
136
|
+
ingest,
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// A producer with no resolved contact row in scope (e.g. the fast-expiry
|
|
140
|
+
// timer) must NOT invent a pin — the resolver falls back to value keys.
|
|
141
|
+
assert.equal(events.length, 2);
|
|
142
|
+
for (const event of events) {
|
|
143
|
+
assert.equal(event.contactId, undefined);
|
|
144
|
+
}
|
|
145
|
+
});
|
package/src/lib/bucket-emit.ts
CHANGED
|
@@ -40,6 +40,17 @@ export async function emitBucketTransition(opts: {
|
|
|
40
40
|
bucket: BucketMeta;
|
|
41
41
|
userId: string;
|
|
42
42
|
userEmail: string | null;
|
|
43
|
+
/**
|
|
44
|
+
* The subject contact's resolved row id (`contacts.id`) — the ENGINE-INTERNAL
|
|
45
|
+
* provenance pin, threaded into BOTH re-ingests below. `userId` here is the
|
|
46
|
+
* contact's CANONICAL key (`external_id ?? anonymous_id ?? id`), but the
|
|
47
|
+
* resolver treats a bare `userId` as an EXTERNAL key and never probes
|
|
48
|
+
* `anonymous_id` — so without the pin an anonymous-only contact gets a
|
|
49
|
+
* phantom `external_id` twin minted on every transition (issue #608).
|
|
50
|
+
* OPTIONAL: a producer that has no resolved contact row in scope degrades to
|
|
51
|
+
* the pin-less resolve (exactly the pre-pin behaviour).
|
|
52
|
+
*/
|
|
53
|
+
contactId?: string;
|
|
43
54
|
epoch: number;
|
|
44
55
|
source?: BucketTransitionSource;
|
|
45
56
|
/** Carried on a `left` transition's properties → `ctx.reason`. */
|
|
@@ -52,6 +63,12 @@ export async function emitBucketTransition(opts: {
|
|
|
52
63
|
* absorbed by the `userEvents` dedup. Surfaced as `dwellCount`.
|
|
53
64
|
*/
|
|
54
65
|
dwellOrdinal?: number;
|
|
66
|
+
/**
|
|
67
|
+
* Testing seam — the re-ingest fn (defaults to the real `ingestEvent`), so a
|
|
68
|
+
* unit test can assert the provenance pin rides BOTH re-ingests without a
|
|
69
|
+
* live database (the Section 14 pattern `checkBucketMembership` uses).
|
|
70
|
+
*/
|
|
71
|
+
ingest?: typeof ingestEvent;
|
|
55
72
|
}): Promise<void> {
|
|
56
73
|
const {
|
|
57
74
|
db,
|
|
@@ -62,11 +79,13 @@ export async function emitBucketTransition(opts: {
|
|
|
62
79
|
bucket,
|
|
63
80
|
userId,
|
|
64
81
|
userEmail,
|
|
82
|
+
contactId,
|
|
65
83
|
epoch,
|
|
66
84
|
source = "event",
|
|
67
85
|
reason,
|
|
68
86
|
dwellLabel,
|
|
69
87
|
dwellOrdinal,
|
|
88
|
+
ingest = ingestEvent,
|
|
70
89
|
} = opts;
|
|
71
90
|
|
|
72
91
|
// The dwell transition emits a labelled event so two dwell reactions on one
|
|
@@ -114,8 +133,10 @@ export async function emitBucketTransition(opts: {
|
|
|
114
133
|
|
|
115
134
|
// Per-bucket alias — the recommended, narrowly-routed binding. The
|
|
116
135
|
// deterministic idempotencyKey rides the userEvents dedup short-circuit as
|
|
117
|
-
// defense-in-depth (Section 6.3).
|
|
118
|
-
|
|
136
|
+
// defense-in-depth (Section 6.3). `contactId` pins the resolver to the known
|
|
137
|
+
// row so an anonymous canonical key folds back in instead of minting a
|
|
138
|
+
// phantom `external_id` twin (issue #608).
|
|
139
|
+
await ingest({
|
|
119
140
|
db,
|
|
120
141
|
registry,
|
|
121
142
|
hatchet,
|
|
@@ -124,6 +145,7 @@ export async function emitBucketTransition(opts: {
|
|
|
124
145
|
event: eventName,
|
|
125
146
|
userId,
|
|
126
147
|
userEmail: userEmail ?? "",
|
|
148
|
+
contactId,
|
|
127
149
|
eventProperties: properties,
|
|
128
150
|
source: "bucket",
|
|
129
151
|
idempotencyKey,
|
|
@@ -134,7 +156,7 @@ export async function emitBucketTransition(opts: {
|
|
|
134
156
|
// recursion-guarded generic event is not written for nothing (Section 8.5).
|
|
135
157
|
const genericEvent = `bucket:${kind}`;
|
|
136
158
|
if (registry.getByTriggerEvent(genericEvent).length > 0) {
|
|
137
|
-
await
|
|
159
|
+
await ingest({
|
|
138
160
|
db,
|
|
139
161
|
registry,
|
|
140
162
|
hatchet,
|
|
@@ -143,6 +165,10 @@ export async function emitBucketTransition(opts: {
|
|
|
143
165
|
event: genericEvent,
|
|
144
166
|
userId,
|
|
145
167
|
userEmail: userEmail ?? "",
|
|
168
|
+
// Same provenance pin as the alias emit above — dropping it on ONLY
|
|
169
|
+
// this leg would leave the twin-minting half-alive for generic-bound
|
|
170
|
+
// journeys (issue #608).
|
|
171
|
+
contactId,
|
|
146
172
|
eventProperties: properties,
|
|
147
173
|
source: "bucket",
|
|
148
174
|
idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}:generic`,
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { EmailProvider } from "@hogsend/core";
|
|
2
2
|
import { createResendProvider } from "@hogsend/plugin-resend";
|
|
3
3
|
import type { env as envSchema } from "../env.js";
|
|
4
|
+
import { loadOptionalPlugin } from "./load-optional-plugin.js";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
|
-
* `@hogsend/plugin-postmark` is an OPT-IN
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* with E404 on plugin-postmark for every consumer that doesn't have it.
|
|
7
|
+
* `@hogsend/plugin-postmark` is an OPT-IN package: an engine
|
|
8
|
+
* `optionalDependency`, NOT a hard one. So we MUST NOT statically import it — a
|
|
9
|
+
* static `import` would make the package mandatory at engine load for every
|
|
10
|
+
* consumer, including the ones that never send through Postmark.
|
|
11
11
|
*
|
|
12
12
|
* Instead we load it lazily, ONCE, behind a top-level guarded dynamic import:
|
|
13
13
|
* the `import()` only fires when `POSTMARK_SERVER_TOKEN` is present (the same
|
|
@@ -34,17 +34,16 @@ const POSTMARK_PACKAGE = ["@hogsend", "plugin-postmark"].join("/");
|
|
|
34
34
|
|
|
35
35
|
let createPostmarkProvider: CreatePostmarkProvider | null = null;
|
|
36
36
|
if (process.env.POSTMARK_SERVER_TOKEN) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
}
|
|
37
|
+
// Previously a bare `catch {}` that swallowed the failure silently. If
|
|
38
|
+
// Postmark was the resolved active provider the container still threw its
|
|
39
|
+
// "not registered" boot error, but a deploy that merely set the token fell
|
|
40
|
+
// back to Resend with nothing in the logs to say why.
|
|
41
|
+
createPostmarkProvider = await loadOptionalPlugin<CreatePostmarkProvider>({
|
|
42
|
+
specifier: POSTMARK_PACKAGE,
|
|
43
|
+
exportName: "createPostmarkProvider",
|
|
44
|
+
enabledBy: "POSTMARK_SERVER_TOKEN is set",
|
|
45
|
+
onFailure: (message) => console.warn(message),
|
|
46
|
+
});
|
|
48
47
|
}
|
|
49
48
|
|
|
50
49
|
/**
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
3
|
import type { env as envSchema } from "../env.js";
|
|
4
|
-
import {
|
|
5
|
-
enrichmentProvidersFromEnv,
|
|
6
|
-
loadEnrichmentPluginFactory,
|
|
7
|
-
} from "./enrichment-providers-from-env.js";
|
|
4
|
+
import { enrichmentProvidersFromEnv } from "./enrichment-providers-from-env.js";
|
|
8
5
|
|
|
9
6
|
// This test process never sets APOLLO_API_KEY, so the module-scope guarded
|
|
10
7
|
// dynamic import in enrichment-providers-from-env.ts must not have fired —
|
|
@@ -25,43 +22,7 @@ test("AC 7 (preset half): key set but factory unresolved → empty list, no thro
|
|
|
25
22
|
assert.deepEqual(enrichmentProvidersFromEnv(envWithKey), []);
|
|
26
23
|
});
|
|
27
24
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
const messages: string[] = [];
|
|
33
|
-
const factory = await loadEnrichmentPluginFactory({
|
|
34
|
-
specifier: "@hogsend/plugin-test-does-not-exist",
|
|
35
|
-
exportName: "createTestProvider",
|
|
36
|
-
onMissing: (message) => messages.push(message),
|
|
37
|
-
});
|
|
38
|
-
assert.equal(factory, null);
|
|
39
|
-
assert.equal(messages.length, 1);
|
|
40
|
-
assert.match(messages[0] ?? "", /@hogsend\/plugin-test-does-not-exist/);
|
|
41
|
-
});
|
|
42
|
-
|
|
43
|
-
test("AC 7 control: resolvable specifier → returns the named export", async () => {
|
|
44
|
-
// Proves the null above comes from the unresolvable specifier, not a broken
|
|
45
|
-
// loader — keeping the guard-half assertion honest.
|
|
46
|
-
const messages: string[] = [];
|
|
47
|
-
const factory = await loadEnrichmentPluginFactory<
|
|
48
|
-
typeof import("node:path")["join"]
|
|
49
|
-
>({
|
|
50
|
-
specifier: "node:path",
|
|
51
|
-
exportName: "join",
|
|
52
|
-
onMissing: (message) => messages.push(message),
|
|
53
|
-
});
|
|
54
|
-
assert.equal(typeof factory, "function");
|
|
55
|
-
assert.equal(messages.length, 0);
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
test("a resolvable specifier MISSING the named export → null + logged, no throw", async () => {
|
|
59
|
-
const messages: string[] = [];
|
|
60
|
-
const factory = await loadEnrichmentPluginFactory({
|
|
61
|
-
specifier: "node:path",
|
|
62
|
-
exportName: "createApolloProvider",
|
|
63
|
-
onMissing: (message) => messages.push(message),
|
|
64
|
-
});
|
|
65
|
-
assert.equal(factory, null);
|
|
66
|
-
assert.equal(messages.length, 1);
|
|
67
|
-
});
|
|
25
|
+
// The guard itself now lives in `load-optional-plugin.ts` and is covered by
|
|
26
|
+
// `load-optional-plugin.test.ts` — including the classification this preset
|
|
27
|
+
// depends on (absent package vs installed-but-broken). Only the preset's own
|
|
28
|
+
// behaviour is asserted here.
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { EnrichmentProvider } from "@hogsend/core";
|
|
2
2
|
import type { env as envSchema } from "../env.js";
|
|
3
|
+
import { loadOptionalPlugin } from "./load-optional-plugin.js";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* `@hogsend/plugin-apollo` is an OPT-IN, deferred-publish package — an engine
|
|
@@ -18,49 +19,15 @@ type CreateApolloProvider = (cfg: { apiKey: string }) => EnrichmentProvider;
|
|
|
18
19
|
|
|
19
20
|
const APOLLO_PACKAGE = ["@hogsend", "plugin-apollo"].join("/");
|
|
20
21
|
|
|
21
|
-
/**
|
|
22
|
-
* The guard around the dynamic import, extracted so it is testable against an
|
|
23
|
-
* INJECTED unresolvable specifier (never against `@hogsend/plugin-apollo`
|
|
24
|
-
* genuinely being absent — once that package lands in `optionalDependencies`,
|
|
25
|
-
* an absence-based test would silently assert the opposite of what it means).
|
|
26
|
-
*
|
|
27
|
-
* Resolves to the named factory export, or null when the package cannot be
|
|
28
|
-
* resolved / the export is missing — in which case `onMissing` fires (once,
|
|
29
|
-
* since the module-scope call site runs once per process) and nothing throws:
|
|
30
|
-
* a key set without the plugin installed degrades gracefully, and if the
|
|
31
|
-
* provider was EXPLICITLY requested the container still throws its clear
|
|
32
|
-
* "not registered" boot error.
|
|
33
|
-
*/
|
|
34
|
-
export async function loadEnrichmentPluginFactory<T>(opts: {
|
|
35
|
-
specifier: string;
|
|
36
|
-
exportName: string;
|
|
37
|
-
onMissing?: (message: string) => void;
|
|
38
|
-
}): Promise<T | null> {
|
|
39
|
-
try {
|
|
40
|
-
const mod = (await import(opts.specifier)) as Record<string, unknown>;
|
|
41
|
-
const factory = mod[opts.exportName];
|
|
42
|
-
if (typeof factory !== "function") {
|
|
43
|
-
throw new Error(`missing export "${opts.exportName}"`);
|
|
44
|
-
}
|
|
45
|
-
return factory as T;
|
|
46
|
-
} catch {
|
|
47
|
-
opts.onMissing?.(
|
|
48
|
-
`enrichment plugin "${opts.specifier}" is not installed — its env preset is skipped. ` +
|
|
49
|
-
`Install the package (pnpm add ${opts.specifier}) to enable it.`,
|
|
50
|
-
);
|
|
51
|
-
return null;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
22
|
let createApolloProvider: CreateApolloProvider | null = null;
|
|
56
23
|
if (process.env.APOLLO_API_KEY) {
|
|
57
|
-
createApolloProvider =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
24
|
+
createApolloProvider = await loadOptionalPlugin<CreateApolloProvider>({
|
|
25
|
+
specifier: APOLLO_PACKAGE,
|
|
26
|
+
exportName: "createApolloProvider",
|
|
27
|
+
enabledBy: "APOLLO_API_KEY is set",
|
|
28
|
+
// Module scope ⇒ logs ONCE per process, and only when the key is set.
|
|
29
|
+
onFailure: (message) => console.warn(message),
|
|
30
|
+
});
|
|
64
31
|
}
|
|
65
32
|
|
|
66
33
|
/**
|
package/src/lib/ingestion.ts
CHANGED
|
@@ -778,6 +778,11 @@ export async function ingestEvent(opts: {
|
|
|
778
778
|
hatchet,
|
|
779
779
|
logger,
|
|
780
780
|
userId: resolvedKey,
|
|
781
|
+
// The resolved row id rides along as the ENGINE-INTERNAL provenance pin
|
|
782
|
+
// for the bucket transition re-ingests: `resolvedKey` may be the
|
|
783
|
+
// contact's `anonymous_id`, which a bare re-ingest would treat as an
|
|
784
|
+
// external key and mint a phantom twin for (issue #608).
|
|
785
|
+
contactId,
|
|
781
786
|
userEmail: event.userEmail || null,
|
|
782
787
|
event: event.event,
|
|
783
788
|
eventProperties: event.eventProperties,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import test from "node:test";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import {
|
|
8
|
+
loadOptionalPlugin,
|
|
9
|
+
type PluginLoadOutcome,
|
|
10
|
+
} from "./load-optional-plugin.js";
|
|
11
|
+
|
|
12
|
+
function collect() {
|
|
13
|
+
const seen: Array<{ message: string; outcome: PluginLoadOutcome }> = [];
|
|
14
|
+
return {
|
|
15
|
+
seen,
|
|
16
|
+
onFailure: (message: string, outcome: PluginLoadOutcome) =>
|
|
17
|
+
seen.push({ message, outcome }),
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
test("resolvable specifier → returns the named export, logs nothing", async () => {
|
|
22
|
+
// The control. Proves a null below comes from the specifier, not a broken
|
|
23
|
+
// loader.
|
|
24
|
+
const { seen, onFailure } = collect();
|
|
25
|
+
const factory = await loadOptionalPlugin<typeof import("node:path")["join"]>({
|
|
26
|
+
specifier: "node:path",
|
|
27
|
+
exportName: "join",
|
|
28
|
+
enabledBy: "TEST is set",
|
|
29
|
+
onFailure,
|
|
30
|
+
});
|
|
31
|
+
assert.equal(typeof factory, "function");
|
|
32
|
+
assert.equal(seen.length, 0);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("absent package → not-installed, names the install command", async () => {
|
|
36
|
+
// An INJECTED unresolvable specifier, never a real @hogsend plugin: those are
|
|
37
|
+
// engine optionalDependencies, so an absence-based test would silently invert
|
|
38
|
+
// its own meaning the moment the package resolves.
|
|
39
|
+
const { seen, onFailure } = collect();
|
|
40
|
+
const factory = await loadOptionalPlugin({
|
|
41
|
+
specifier: "@hogsend/plugin-test-does-not-exist",
|
|
42
|
+
exportName: "createTestProvider",
|
|
43
|
+
enabledBy: "TEST_TOKEN is set",
|
|
44
|
+
onFailure,
|
|
45
|
+
});
|
|
46
|
+
assert.equal(factory, null);
|
|
47
|
+
assert.equal(seen.length, 1);
|
|
48
|
+
assert.equal(seen[0]?.outcome, "not-installed");
|
|
49
|
+
assert.match(
|
|
50
|
+
seen[0]?.message ?? "",
|
|
51
|
+
/pnpm add @hogsend\/plugin-test-does-not-exist/,
|
|
52
|
+
);
|
|
53
|
+
// The reason it was attempted at all must be in the message — otherwise the
|
|
54
|
+
// operator cannot tell which credential switched this on.
|
|
55
|
+
assert.match(seen[0]?.message ?? "", /TEST_TOKEN is set/);
|
|
56
|
+
// The direct-dependency nuance is the entire point: an engine
|
|
57
|
+
// optionalDependency is installed but never linked at the consumer's top
|
|
58
|
+
// level, so a bundled app cannot resolve it.
|
|
59
|
+
assert.match(seen[0]?.message ?? "", /DIRECT dependency/);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("present but broken → load-failed, and does NOT say 'not installed'", async () => {
|
|
63
|
+
// The regression that motivated this module. The old loader reported every
|
|
64
|
+
// failure as "is not installed", so an operator whose package WAS installed
|
|
65
|
+
// (but shipped raw .ts that Node cannot type-strip under node_modules) was
|
|
66
|
+
// told to reinstall it — advice that could never work. A load failure must
|
|
67
|
+
// never be described as an absence.
|
|
68
|
+
const { seen, onFailure } = collect();
|
|
69
|
+
const factory = await loadOptionalPlugin({
|
|
70
|
+
// Resolves, then throws on evaluation.
|
|
71
|
+
specifier: "data:text/javascript,throw new Error('boom on evaluate')",
|
|
72
|
+
exportName: "createTestProvider",
|
|
73
|
+
enabledBy: "TEST_TOKEN is set",
|
|
74
|
+
onFailure,
|
|
75
|
+
});
|
|
76
|
+
assert.equal(factory, null);
|
|
77
|
+
assert.equal(seen[0]?.outcome, "load-failed");
|
|
78
|
+
assert.doesNotMatch(seen[0]?.message ?? "", /is not installed/);
|
|
79
|
+
assert.match(seen[0]?.message ?? "", /boom on evaluate/);
|
|
80
|
+
assert.match(seen[0]?.message ?? "", /packaging bug/);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("a MISSING TRANSITIVE dep is not reported as the plugin being absent", async () => {
|
|
84
|
+
// The case that forces classification to inspect the message, not just the
|
|
85
|
+
// code. "plugin-twilio is installed but `twilio` is not" throws the SAME
|
|
86
|
+
// ERR_MODULE_NOT_FOUND as "plugin-twilio is not installed"; keying on the
|
|
87
|
+
// code alone would tell the operator to reinstall a package that is already
|
|
88
|
+
// there while the real gap goes unnamed.
|
|
89
|
+
//
|
|
90
|
+
// This needs a real file on disk. A `data:` URL cannot resolve bare
|
|
91
|
+
// specifiers at all (ERR_UNSUPPORTED_RESOLVE_REQUEST), and a `node:`-prefixed
|
|
92
|
+
// miss throws ERR_UNKNOWN_BUILTIN_MODULE — neither reaches this branch, so
|
|
93
|
+
// either would pass while testing nothing.
|
|
94
|
+
const dir = await mkdtemp(join(tmpdir(), "hogsend-plugin-load-"));
|
|
95
|
+
try {
|
|
96
|
+
const file = join(dir, "broken.mjs");
|
|
97
|
+
await writeFile(
|
|
98
|
+
file,
|
|
99
|
+
'import "totally-not-a-real-package-xyz";\nexport const createTestProvider = () => {};\n',
|
|
100
|
+
);
|
|
101
|
+
const { seen, onFailure } = collect();
|
|
102
|
+
const factory = await loadOptionalPlugin({
|
|
103
|
+
specifier: pathToFileURL(file).href,
|
|
104
|
+
exportName: "createTestProvider",
|
|
105
|
+
enabledBy: "TEST_TOKEN is set",
|
|
106
|
+
onFailure,
|
|
107
|
+
});
|
|
108
|
+
assert.equal(factory, null);
|
|
109
|
+
assert.equal(seen[0]?.outcome, "load-failed");
|
|
110
|
+
assert.doesNotMatch(seen[0]?.message ?? "", /is not installed/);
|
|
111
|
+
// The missing package must be named, or the operator learns nothing.
|
|
112
|
+
assert.match(seen[0]?.message ?? "", /totally-not-a-real-package-xyz/);
|
|
113
|
+
} finally {
|
|
114
|
+
await rm(dir, { recursive: true, force: true });
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
test("resolvable but missing the named export → missing-export, flags version skew", async () => {
|
|
119
|
+
const { seen, onFailure } = collect();
|
|
120
|
+
const factory = await loadOptionalPlugin({
|
|
121
|
+
specifier: "node:path",
|
|
122
|
+
exportName: "createApolloProvider",
|
|
123
|
+
enabledBy: "APOLLO_API_KEY is set",
|
|
124
|
+
onFailure,
|
|
125
|
+
});
|
|
126
|
+
assert.equal(factory, null);
|
|
127
|
+
assert.equal(seen[0]?.outcome, "missing-export");
|
|
128
|
+
assert.match(
|
|
129
|
+
seen[0]?.message ?? "",
|
|
130
|
+
/does not export "createApolloProvider"/,
|
|
131
|
+
);
|
|
132
|
+
assert.match(seen[0]?.message ?? "", /version/);
|
|
133
|
+
});
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared loader for the engine's OPT-IN plugin packages (`@hogsend/plugin-apollo`,
|
|
3
|
+
* `-postmark`, `-twilio`). Each is an engine `optionalDependency`, never a hard
|
|
4
|
+
* one: a static import would make the package mandatory at engine load and break
|
|
5
|
+
* `npm install @hogsend/engine` for every consumer without it.
|
|
6
|
+
*
|
|
7
|
+
* So each preset loads its package lazily, ONCE, behind a guarded dynamic import
|
|
8
|
+
* gated on the relevant credential. The specifier is assembled at runtime (not a
|
|
9
|
+
* literal) so `tsc` never resolves the module's types for a consumer that does
|
|
10
|
+
* not have the opt-in package.
|
|
11
|
+
*
|
|
12
|
+
* ## Why the failure message matters so much
|
|
13
|
+
*
|
|
14
|
+
* All three presets previously swallowed the failure — SMS silently, enrichment
|
|
15
|
+
* behind a message that said "is not installed" for EVERY failure mode. That
|
|
16
|
+
* message was actively wrong in the most common real case (installed, but the
|
|
17
|
+
* package shipped raw `.ts` that Node cannot type-strip inside node_modules), so
|
|
18
|
+
* an operator following it reinstalled a package they already had. The whole
|
|
19
|
+
* point of this module is that the two failure modes are DIFFERENT problems with
|
|
20
|
+
* different fixes, and conflating them sends people to fix the wrong thing.
|
|
21
|
+
*
|
|
22
|
+
* `not-installed` is a configuration problem the operator can fix.
|
|
23
|
+
* `load-failed` is OUR bug and should be reported, not worked around.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
export type PluginLoadOutcome =
|
|
27
|
+
| "not-installed"
|
|
28
|
+
| "load-failed"
|
|
29
|
+
| "missing-export";
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Distinguish "the package itself is absent" from "the package is present but
|
|
33
|
+
* blew up while loading".
|
|
34
|
+
*
|
|
35
|
+
* Both surface as `ERR_MODULE_NOT_FOUND` in the general case — a plugin whose
|
|
36
|
+
* own npm dependency is missing throws exactly the same code. So a bare code
|
|
37
|
+
* check would mislabel "plugin-twilio is installed but `twilio` isn't" as
|
|
38
|
+
* "plugin-twilio is not installed" and send the operator to reinstall the wrong
|
|
39
|
+
* package. Requiring the message to name the specifier keeps the two apart.
|
|
40
|
+
*/
|
|
41
|
+
function classify(error: unknown, specifier: string): PluginLoadOutcome {
|
|
42
|
+
const err = error as { code?: string; message?: string } | undefined;
|
|
43
|
+
const notFound =
|
|
44
|
+
err?.code === "ERR_MODULE_NOT_FOUND" || err?.code === "MODULE_NOT_FOUND";
|
|
45
|
+
return notFound && (err?.message ?? "").includes(specifier)
|
|
46
|
+
? "not-installed"
|
|
47
|
+
: "load-failed";
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function describe(error: unknown): string {
|
|
51
|
+
const err = error as { code?: string; message?: string } | undefined;
|
|
52
|
+
const code = err?.code ? `${err.code}: ` : "";
|
|
53
|
+
return `${code}${(err?.message ?? String(error)).split("\n")[0]}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function messageFor(opts: {
|
|
57
|
+
specifier: string;
|
|
58
|
+
exportName: string;
|
|
59
|
+
enabledBy: string;
|
|
60
|
+
outcome: PluginLoadOutcome;
|
|
61
|
+
error?: unknown;
|
|
62
|
+
}): string {
|
|
63
|
+
const { specifier, exportName, enabledBy, outcome, error } = opts;
|
|
64
|
+
const skipped = `${enabledBy}, but its env preset is skipped`;
|
|
65
|
+
|
|
66
|
+
if (outcome === "not-installed") {
|
|
67
|
+
return (
|
|
68
|
+
`${skipped}: "${specifier}" is not installed. ` +
|
|
69
|
+
`Add it as a DIRECT dependency of your app: pnpm add ${specifier}. ` +
|
|
70
|
+
`Note that the engine listing it under optionalDependencies is NOT enough — ` +
|
|
71
|
+
`your app bundles the engine, so this import resolves against YOUR ` +
|
|
72
|
+
`node_modules, where an engine-only optional dependency is never linked.`
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (outcome === "missing-export") {
|
|
77
|
+
return (
|
|
78
|
+
`${skipped}: "${specifier}" loaded but does not export "${exportName}". ` +
|
|
79
|
+
`This usually means the package version does not match @hogsend/engine — ` +
|
|
80
|
+
`align them on the same engine line (pnpm add ${specifier}@<engine version>).`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return (
|
|
85
|
+
`${skipped}: "${specifier}" IS installed but failed to load — ` +
|
|
86
|
+
`${describe(error)}. This is a packaging bug in Hogsend rather than ` +
|
|
87
|
+
`anything wrong with your configuration; please report it. ` +
|
|
88
|
+
`Reinstalling the package will not help.`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Resolve a named factory export from an optional plugin package.
|
|
94
|
+
*
|
|
95
|
+
* Returns the factory, or `null` when the package cannot be loaded — in which
|
|
96
|
+
* case `onFailure` fires (once, since every call site runs at module scope, once
|
|
97
|
+
* per process) and NOTHING throws. A credential set without a working plugin
|
|
98
|
+
* degrades gracefully; if the provider was EXPLICITLY selected the container
|
|
99
|
+
* still throws its own clear "not registered" error at boot.
|
|
100
|
+
*
|
|
101
|
+
* Deliberately testable against an INJECTED specifier so the guard can be
|
|
102
|
+
* exercised without depending on a real package being absent — once these
|
|
103
|
+
* packages are engine optionalDependencies, an absence-based test would silently
|
|
104
|
+
* assert the opposite of what it means.
|
|
105
|
+
*/
|
|
106
|
+
export async function loadOptionalPlugin<T>(opts: {
|
|
107
|
+
specifier: string;
|
|
108
|
+
exportName: string;
|
|
109
|
+
/** What switched this on, e.g. `"APOLLO_API_KEY is set"`. Named in the message. */
|
|
110
|
+
enabledBy: string;
|
|
111
|
+
onFailure?: (message: string, outcome: PluginLoadOutcome) => void;
|
|
112
|
+
}): Promise<T | null> {
|
|
113
|
+
const { specifier, exportName, enabledBy, onFailure } = opts;
|
|
114
|
+
|
|
115
|
+
let mod: Record<string, unknown>;
|
|
116
|
+
try {
|
|
117
|
+
mod = (await import(specifier)) as Record<string, unknown>;
|
|
118
|
+
} catch (error) {
|
|
119
|
+
const outcome = classify(error, specifier);
|
|
120
|
+
onFailure?.(
|
|
121
|
+
messageFor({ specifier, exportName, enabledBy, outcome, error }),
|
|
122
|
+
outcome,
|
|
123
|
+
);
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const factory = mod[exportName];
|
|
128
|
+
if (typeof factory !== "function") {
|
|
129
|
+
onFailure?.(
|
|
130
|
+
messageFor({
|
|
131
|
+
specifier,
|
|
132
|
+
exportName,
|
|
133
|
+
enabledBy,
|
|
134
|
+
outcome: "missing-export",
|
|
135
|
+
}),
|
|
136
|
+
"missing-export",
|
|
137
|
+
);
|
|
138
|
+
return null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return factory as T;
|
|
142
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { SmsProvider } from "@hogsend/core";
|
|
2
2
|
import type { env as envSchema } from "../env.js";
|
|
3
|
+
import { loadOptionalPlugin } from "./load-optional-plugin.js";
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* `@hogsend/plugin-twilio` is an OPT-IN, deferred-publish package — an engine
|
|
@@ -25,17 +26,16 @@ const TWILIO_PACKAGE = ["@hogsend", "plugin-twilio"].join("/");
|
|
|
25
26
|
|
|
26
27
|
let createTwilioProvider: CreateTwilioProvider | null = null;
|
|
27
28
|
if (process.env.TWILIO_ACCOUNT_SID && process.env.TWILIO_AUTH_TOKEN) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
29
|
+
// Previously a bare `catch {}` that swallowed the failure with no log at all:
|
|
30
|
+
// a Twilio-configured deploy registered no provider, the container installed
|
|
31
|
+
// the inert throwing stub, and the first symptom was `sendSms` throwing at
|
|
32
|
+
// send time with nothing at boot to explain why. Report it where it happens.
|
|
33
|
+
createTwilioProvider = await loadOptionalPlugin<CreateTwilioProvider>({
|
|
34
|
+
specifier: TWILIO_PACKAGE,
|
|
35
|
+
exportName: "createTwilioProvider",
|
|
36
|
+
enabledBy: "TWILIO_ACCOUNT_SID and TWILIO_AUTH_TOKEN are set",
|
|
37
|
+
onFailure: (message) => console.warn(message),
|
|
38
|
+
});
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
/**
|
|
@@ -346,6 +346,10 @@ async function reevalLeaves(opts: {
|
|
|
346
346
|
id: bucketMemberships.id,
|
|
347
347
|
userId: bucketMemberships.userId,
|
|
348
348
|
userEmail: bucketMemberships.userEmail,
|
|
349
|
+
// Joined contact row → its id rides along as the ENGINE-INTERNAL
|
|
350
|
+
// provenance pin for the leave emit, so the re-ingest folds into the
|
|
351
|
+
// known row instead of minting a phantom `external_id` twin (issue #608).
|
|
352
|
+
contactId: contacts.id,
|
|
349
353
|
entryCount: bucketMemberships.entryCount,
|
|
350
354
|
})
|
|
351
355
|
.from(bucketMemberships)
|
|
@@ -362,6 +366,13 @@ async function reevalLeaves(opts: {
|
|
|
362
366
|
const leavers = activeMembers.filter((m) => !matcherSet.has(m.userId));
|
|
363
367
|
if (leavers.length === 0) return 0;
|
|
364
368
|
|
|
369
|
+
// Membership key → contact row id for the emit's provenance pin. Keying by
|
|
370
|
+
// userId is safe: the partial-active unique index guarantees one active row
|
|
371
|
+
// per (user, bucket), so a flipped row maps to exactly one leaver.
|
|
372
|
+
const contactIdByUser = new Map(
|
|
373
|
+
leavers.map((m) => [m.userId, m.contactId] as const),
|
|
374
|
+
);
|
|
375
|
+
|
|
365
376
|
let leftCount = 0;
|
|
366
377
|
for (let i = 0; i < leavers.length; i += BATCH_SIZE) {
|
|
367
378
|
const chunk = leavers.slice(i, i + BATCH_SIZE);
|
|
@@ -400,6 +411,9 @@ async function reevalLeaves(opts: {
|
|
|
400
411
|
bucket,
|
|
401
412
|
userId: row.userId,
|
|
402
413
|
userEmail: row.userEmail,
|
|
414
|
+
// Provenance pin from the leaver's already-joined contact row, so the
|
|
415
|
+
// leave re-ingest folds into it instead of minting a twin (issue #608).
|
|
416
|
+
contactId: contactIdByUser.get(row.userId),
|
|
403
417
|
epoch: row.entryCount,
|
|
404
418
|
source: "backfill",
|
|
405
419
|
reason: "criteria",
|
|
@@ -283,6 +283,13 @@ export const bucketExpiryTask = hatchet.durableTask({
|
|
|
283
283
|
bucket,
|
|
284
284
|
userId: input.userId,
|
|
285
285
|
userEmail: input.userEmail,
|
|
286
|
+
// NO `contactId` provenance pin here (issue #608): this task has no
|
|
287
|
+
// contact row in scope, and the only channel that could carry one — the
|
|
288
|
+
// `bucket:arm-expiry` Hatchet event payload — is reachable from public
|
|
289
|
+
// ingest pushes, so an id smuggled through it would not be the
|
|
290
|
+
// unforgeable engine-internal provenance the pin requires. A value
|
|
291
|
+
// lookup here would re-implement exactly the resolution the pin exists
|
|
292
|
+
// to bypass. Degrades to the pre-pin resolve.
|
|
286
293
|
epoch: flipped.entryCount,
|
|
287
294
|
source: "reconcile",
|
|
288
295
|
// Fast-expiry is a criteria re-confirm leave (Section 6.7).
|
|
@@ -293,6 +300,19 @@ export const bucketExpiryTask = hatchet.durableTask({
|
|
|
293
300
|
},
|
|
294
301
|
});
|
|
295
302
|
|
|
303
|
+
/**
|
|
304
|
+
* A member selected to leave: the membership's canonical string key plus the
|
|
305
|
+
* joined contact row's id — the ENGINE-INTERNAL provenance pin threaded into
|
|
306
|
+
* `emitBucketTransition` so the leave re-ingest folds into the known row
|
|
307
|
+
* instead of minting a phantom `external_id` twin (issue #608). Every leaver
|
|
308
|
+
* source query already joins `contacts` (the GDPR live-contact filter), so the
|
|
309
|
+
* id is free.
|
|
310
|
+
*/
|
|
311
|
+
interface BucketLeaver {
|
|
312
|
+
userId: string;
|
|
313
|
+
contactId: string;
|
|
314
|
+
}
|
|
315
|
+
|
|
296
316
|
/**
|
|
297
317
|
* Set-based SHOULD-LEAVE for one time-based bucket → bulk CAS → RETURNING-gated
|
|
298
318
|
* emit. For single-event `not_exists`/`exists`/`count within` criteria the
|
|
@@ -311,14 +331,14 @@ async function reconcileBucketLeaves(opts: {
|
|
|
311
331
|
|
|
312
332
|
// A single-event/within/count criterion → set-based SHOULD-LEAVE query.
|
|
313
333
|
if (criteria.type === "event") {
|
|
314
|
-
const
|
|
315
|
-
if (
|
|
334
|
+
const leavers = await selectEventLeavers(db, bucket, criteria);
|
|
335
|
+
if (leavers.length === 0) return 0;
|
|
316
336
|
return bulkLeave({
|
|
317
337
|
db,
|
|
318
338
|
logger,
|
|
319
339
|
journeyRegistry,
|
|
320
340
|
bucket,
|
|
321
|
-
|
|
341
|
+
leavers,
|
|
322
342
|
reason: "criteria",
|
|
323
343
|
});
|
|
324
344
|
}
|
|
@@ -338,7 +358,7 @@ async function selectEventLeavers(
|
|
|
338
358
|
db: Database,
|
|
339
359
|
bucket: BucketMeta,
|
|
340
360
|
criteria: Extract<ConditionEval, { type: "event" }>,
|
|
341
|
-
): Promise<
|
|
361
|
+
): Promise<BucketLeaver[]> {
|
|
342
362
|
const cutoff = criteria.within
|
|
343
363
|
? new Date(Date.now() - durationToMs(criteria.within))
|
|
344
364
|
: null;
|
|
@@ -372,10 +392,13 @@ async function selectEventLeavers(
|
|
|
372
392
|
.groupBy(userEvents.userId)
|
|
373
393
|
.as("counted");
|
|
374
394
|
|
|
375
|
-
// LEFT JOIN members → windowed counts. A missing/zero count is a 0.
|
|
395
|
+
// LEFT JOIN members → windowed counts. A missing/zero count is a 0. The
|
|
396
|
+
// contact row is already joined (GDPR filter), so its id rides along as the
|
|
397
|
+
// provenance pin for the leave emit (issue #608).
|
|
376
398
|
const rows = await db
|
|
377
399
|
.select({
|
|
378
400
|
userId: members.userId,
|
|
401
|
+
contactId: contacts.id,
|
|
379
402
|
cnt: sql<number>`coalesce(${counted.cnt}, 0)`,
|
|
380
403
|
})
|
|
381
404
|
.from(members)
|
|
@@ -385,7 +408,7 @@ async function selectEventLeavers(
|
|
|
385
408
|
|
|
386
409
|
return rows
|
|
387
410
|
.filter((r) => shouldLeaveByCount(criteria, Number(r.cnt)))
|
|
388
|
-
.map((r) => r.userId);
|
|
411
|
+
.map((r) => ({ userId: r.userId, contactId: r.contactId }));
|
|
389
412
|
}
|
|
390
413
|
|
|
391
414
|
/**
|
|
@@ -409,6 +432,9 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
409
432
|
const members = await db
|
|
410
433
|
.select({
|
|
411
434
|
userId: bucketMemberships.userId,
|
|
435
|
+
// Joined contact row → its id is the provenance pin for the leave emit
|
|
436
|
+
// (issue #608).
|
|
437
|
+
contactId: contacts.id,
|
|
412
438
|
properties: contacts.properties,
|
|
413
439
|
})
|
|
414
440
|
.from(bucketMemberships)
|
|
@@ -424,7 +450,7 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
424
450
|
.orderBy(sql`${bucketMemberships.lastEvaluatedAt} asc nulls first`)
|
|
425
451
|
.limit(BATCH_SIZE);
|
|
426
452
|
|
|
427
|
-
const
|
|
453
|
+
const leavers: BucketLeaver[] = [];
|
|
428
454
|
const evaluatedIds: string[] = [];
|
|
429
455
|
for (const member of members) {
|
|
430
456
|
evaluatedIds.push(member.userId);
|
|
@@ -435,7 +461,9 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
435
461
|
condition: criteria,
|
|
436
462
|
ctx: { db, userId: member.userId, journeyContext },
|
|
437
463
|
});
|
|
438
|
-
if (!isMember)
|
|
464
|
+
if (!isMember) {
|
|
465
|
+
leavers.push({ userId: member.userId, contactId: member.contactId });
|
|
466
|
+
}
|
|
439
467
|
}
|
|
440
468
|
|
|
441
469
|
// Bump lastEvaluatedAt for the whole chunk so the next tick advances the cursor
|
|
@@ -453,13 +481,13 @@ async function reconcileCompositeLeaves(opts: {
|
|
|
453
481
|
);
|
|
454
482
|
}
|
|
455
483
|
|
|
456
|
-
if (
|
|
484
|
+
if (leavers.length === 0) return 0;
|
|
457
485
|
return bulkLeave({
|
|
458
486
|
db,
|
|
459
487
|
logger,
|
|
460
488
|
journeyRegistry,
|
|
461
489
|
bucket,
|
|
462
|
-
|
|
490
|
+
leavers,
|
|
463
491
|
reason: "criteria",
|
|
464
492
|
});
|
|
465
493
|
}
|
|
@@ -481,7 +509,12 @@ async function reconcileBucketTtlLeaves(opts: {
|
|
|
481
509
|
const { db, logger, journeyRegistry, bucket } = opts;
|
|
482
510
|
|
|
483
511
|
const expired = await db
|
|
484
|
-
.select({
|
|
512
|
+
.select({
|
|
513
|
+
userId: bucketMemberships.userId,
|
|
514
|
+
// Joined contact row → its id is the provenance pin for the leave emit
|
|
515
|
+
// (issue #608).
|
|
516
|
+
contactId: contacts.id,
|
|
517
|
+
})
|
|
485
518
|
.from(bucketMemberships)
|
|
486
519
|
.innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
|
|
487
520
|
.where(
|
|
@@ -501,7 +534,7 @@ async function reconcileBucketTtlLeaves(opts: {
|
|
|
501
534
|
logger,
|
|
502
535
|
journeyRegistry,
|
|
503
536
|
bucket,
|
|
504
|
-
|
|
537
|
+
leavers: expired,
|
|
505
538
|
reason: "maxDwell",
|
|
506
539
|
});
|
|
507
540
|
}
|
|
@@ -518,11 +551,18 @@ async function bulkLeave(opts: {
|
|
|
518
551
|
logger: Logger;
|
|
519
552
|
journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
|
|
520
553
|
bucket: BucketMeta;
|
|
521
|
-
|
|
554
|
+
leavers: BucketLeaver[];
|
|
522
555
|
/** Why these members leave — TTL passes "maxDwell", criteria passes "criteria". */
|
|
523
556
|
reason: BucketLeaveReason;
|
|
524
557
|
}): Promise<number> {
|
|
525
|
-
const { db, logger, journeyRegistry, bucket,
|
|
558
|
+
const { db, logger, journeyRegistry, bucket, leavers, reason } = opts;
|
|
559
|
+
const userIds = leavers.map((l) => l.userId);
|
|
560
|
+
// Membership key → resolved contact row id, for the emit's provenance pin.
|
|
561
|
+
// Safe to key by userId: the partial-active unique index guarantees one
|
|
562
|
+
// active row per (user, bucket), so a flipped row maps to exactly one leaver.
|
|
563
|
+
const contactIdByUser = new Map(
|
|
564
|
+
leavers.map((l) => [l.userId, l.contactId] as const),
|
|
565
|
+
);
|
|
526
566
|
|
|
527
567
|
const dwellMs = bucket.minDwell ? durationToMs(bucket.minDwell) : 0;
|
|
528
568
|
const dwellCutoff = dwellMs > 0 ? new Date(Date.now() - dwellMs) : null;
|
|
@@ -565,6 +605,9 @@ async function bulkLeave(opts: {
|
|
|
565
605
|
bucket,
|
|
566
606
|
userId: row.userId,
|
|
567
607
|
userEmail: row.userEmail,
|
|
608
|
+
// Provenance pin from the leaver's already-joined contact row, so the
|
|
609
|
+
// leave re-ingest folds into it instead of minting a twin (issue #608).
|
|
610
|
+
contactId: contactIdByUser.get(row.userId),
|
|
568
611
|
epoch: row.entryCount,
|
|
569
612
|
source: "reconcile",
|
|
570
613
|
reason,
|
|
@@ -643,6 +686,9 @@ async function reconcileBucketDwell(opts: {
|
|
|
643
686
|
id: bucketMemberships.id,
|
|
644
687
|
userId: bucketMemberships.userId,
|
|
645
688
|
userEmail: bucketMemberships.userEmail,
|
|
689
|
+
// Joined contact row → its id is the provenance pin for the dwell emit
|
|
690
|
+
// (issue #608).
|
|
691
|
+
contactId: contacts.id,
|
|
646
692
|
entryCount: bucketMemberships.entryCount,
|
|
647
693
|
anchor: sql<Date>`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt})`,
|
|
648
694
|
dwellState: bucketMemberships.dwellState,
|
|
@@ -704,6 +750,9 @@ async function reconcileBucketDwell(opts: {
|
|
|
704
750
|
bucket,
|
|
705
751
|
userId: m.userId,
|
|
706
752
|
userEmail: m.userEmail,
|
|
753
|
+
// Provenance pin from the candidate's already-joined contact row
|
|
754
|
+
// (issue #608).
|
|
755
|
+
contactId: m.contactId,
|
|
707
756
|
epoch: m.entryCount,
|
|
708
757
|
source: "reconcile",
|
|
709
758
|
dwellLabel: label,
|
|
@@ -857,6 +906,11 @@ async function reconcileBucketJoins(opts: {
|
|
|
857
906
|
const baseQuery = db
|
|
858
907
|
.select({
|
|
859
908
|
userId: contactKey,
|
|
909
|
+
// The candidate scan reads straight off `contacts`, so the row id rides
|
|
910
|
+
// along as the provenance pin for the join emit (issue #608) —
|
|
911
|
+
// load-bearing for exactly the null-external_id contacts this coalesce
|
|
912
|
+
// join exists to include.
|
|
913
|
+
contactId: contacts.id,
|
|
860
914
|
email: contacts.email,
|
|
861
915
|
})
|
|
862
916
|
.from(contacts)
|
|
@@ -934,6 +988,7 @@ async function reconcileBucketJoins(opts: {
|
|
|
934
988
|
bucket,
|
|
935
989
|
userId: candidate.userId,
|
|
936
990
|
userEmail: candidate.email ?? null,
|
|
991
|
+
contactId: candidate.contactId,
|
|
937
992
|
});
|
|
938
993
|
if (transitioned) joined += 1;
|
|
939
994
|
}
|
|
@@ -1035,8 +1090,10 @@ async function reconcileJoinOne(opts: {
|
|
|
1035
1090
|
bucket: BucketMeta;
|
|
1036
1091
|
userId: string;
|
|
1037
1092
|
userEmail: string | null;
|
|
1093
|
+
/** Provenance pin for the join emit — see {@link BucketLeaver} (issue #608). */
|
|
1094
|
+
contactId?: string;
|
|
1038
1095
|
}): Promise<boolean> {
|
|
1039
|
-
const { db, logger, journeyRegistry, bucket, userId } = opts;
|
|
1096
|
+
const { db, logger, journeyRegistry, bucket, userId, contactId } = opts;
|
|
1040
1097
|
// Normalized at the write site (audience-model.md wart #1) — the contact row
|
|
1041
1098
|
// should already be normalized, but the membership keyspace must never
|
|
1042
1099
|
// depend on that.
|
|
@@ -1081,6 +1138,8 @@ async function reconcileJoinOne(opts: {
|
|
|
1081
1138
|
bucket,
|
|
1082
1139
|
userId,
|
|
1083
1140
|
userEmail,
|
|
1141
|
+
// Provenance pin from the candidate scan's contact row (issue #608).
|
|
1142
|
+
contactId,
|
|
1084
1143
|
epoch,
|
|
1085
1144
|
source: "reconcile",
|
|
1086
1145
|
});
|