@hogsend/engine 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -11
- package/src/buckets/check-membership.ts +45 -3
- package/src/connectors/define-action.ts +15 -2
- package/src/container.ts +85 -0
- package/src/env.ts +21 -0
- package/src/index.ts +19 -0
- package/src/journeys/durable-law-harness.test.ts +107 -0
- package/src/journeys/durable-law-harness.ts +139 -0
- package/src/journeys/journey-boundary.test.ts +70 -0
- package/src/journeys/journey-boundary.ts +10 -1
- package/src/journeys/with-durable-gate.test.ts +258 -0
- package/src/journeys/with-durable-gate.ts +141 -0
- package/src/lib/bucket-emit.test.ts +145 -0
- package/src/lib/bucket-emit.ts +29 -3
- package/src/lib/connector-actions.test.ts +292 -1
- package/src/lib/connector-actions.ts +65 -6
- package/src/lib/email-providers-from-env.ts +15 -16
- package/src/lib/enrichment-ledger.ts +207 -0
- package/src/lib/enrichment-provider-registry.test.ts +64 -0
- package/src/lib/enrichment-provider-registry.ts +35 -0
- package/src/lib/enrichment-providers-from-env.apollo.test.ts +30 -0
- package/src/lib/enrichment-providers-from-env.test.ts +28 -0
- package/src/lib/enrichment-providers-from-env.ts +52 -0
- package/src/lib/enrichment-registry-singleton.ts +45 -0
- package/src/lib/ingestion.ts +5 -0
- package/src/lib/load-optional-plugin.test.ts +133 -0
- package/src/lib/load-optional-plugin.ts +142 -0
- package/src/lib/outbound.ts +27 -0
- package/src/lib/refine-chain.test.ts +764 -0
- package/src/lib/refine-chain.ts +583 -0
- package/src/lib/refine-traits.test.ts +141 -0
- package/src/lib/refine-traits.ts +128 -0
- package/src/lib/refine.ts +287 -0
- package/src/lib/sms-providers-from-env.ts +11 -11
- package/src/lib/webhook-catalog-sync.test.ts +112 -0
- package/src/lib/webhook-signing.ts +5 -1
- package/src/routes/admin/contacts.ts +113 -13
- package/src/workflows/bucket-backfill.ts +14 -0
- package/src/workflows/bucket-reconcile.ts +74 -15
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
EnrichmentProvider,
|
|
3
|
+
EnrichmentQuery,
|
|
4
|
+
EnrichmentResult,
|
|
5
|
+
} from "@hogsend/core";
|
|
6
|
+
import {
|
|
7
|
+
type CallerRef,
|
|
8
|
+
callerRefFromArgs,
|
|
9
|
+
withDurableGate,
|
|
10
|
+
} from "../journeys/with-durable-gate.js";
|
|
11
|
+
import type {
|
|
12
|
+
EnrichmentLookupKind,
|
|
13
|
+
EnrichmentLookupStatus,
|
|
14
|
+
} from "./enrichment-ledger.js";
|
|
15
|
+
import { flattenTraits, REFINED_META_KEYS } from "./refine-traits.js";
|
|
16
|
+
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// Public surface
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
export interface RefineContactOptions {
|
|
22
|
+
/** The contact's canonical key (`external_id ?? anonymous_id ?? id`). */
|
|
23
|
+
userId?: string;
|
|
24
|
+
email?: string;
|
|
25
|
+
/** The contact row id — the unforgeable, engine-internal pin. */
|
|
26
|
+
contactId?: string;
|
|
27
|
+
/** Override the container's active enrichment provider by registry id. */
|
|
28
|
+
provider?: string;
|
|
29
|
+
/** Bypass the TTL + negative cache. Does NOT bypass the budget cap. */
|
|
30
|
+
force?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Disambiguates the replay key when two refine sites in one journey share a
|
|
33
|
+
* nearest wait label. Mirrors `sendEmail`/`sendSms`/`sendConnectorAction`.
|
|
34
|
+
*/
|
|
35
|
+
idempotencyLabel?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface RefineContactResult {
|
|
39
|
+
status: "refined" | "cached" | "not_found" | "skipped";
|
|
40
|
+
/**
|
|
41
|
+
* Present on `skipped` — `no_lookup_key` | `no_provider` | `budget_exceeded`
|
|
42
|
+
* | `provider_error` | `ingest_failed`.
|
|
43
|
+
*/
|
|
44
|
+
reason?: string;
|
|
45
|
+
/** The `refined_*` patch — on `refined` AND on `cached`. */
|
|
46
|
+
properties?: Record<string, unknown>;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The refinement subject, resolved INSIDE the durable memo. */
|
|
50
|
+
export interface RefineTarget {
|
|
51
|
+
/** `contacts.id`, when a row exists. */
|
|
52
|
+
contactId?: string;
|
|
53
|
+
/** Canonical contact key, used as the ingest `userId`. */
|
|
54
|
+
userId?: string;
|
|
55
|
+
email?: string;
|
|
56
|
+
/** Company domain, when the contact carries one and has no email. */
|
|
57
|
+
domain?: string;
|
|
58
|
+
firstName?: string;
|
|
59
|
+
lastName?: string;
|
|
60
|
+
company?: string;
|
|
61
|
+
/** The `refined_*` traits already on THIS contact. */
|
|
62
|
+
refinedProperties?: Record<string, unknown>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The event the refinement pipeline emits through `ingestEvent`. */
|
|
66
|
+
export const REFINE_EVENT = "contact.refined";
|
|
67
|
+
|
|
68
|
+
export interface RefineIngestInput {
|
|
69
|
+
userId?: string;
|
|
70
|
+
email?: string;
|
|
71
|
+
contactId?: string;
|
|
72
|
+
eventProperties: Record<string, unknown>;
|
|
73
|
+
contactProperties: Record<string, unknown>;
|
|
74
|
+
idempotencyKey?: string;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* What the outbound `contact.refined` emit needs. Handed to
|
|
79
|
+
* {@link RefineChainDeps.emitRefined} on a GENUINE refinement and nothing else.
|
|
80
|
+
*
|
|
81
|
+
* `traitKeys` are the mapped trait NAMES, never their values — the payload says
|
|
82
|
+
* what changed and a subscriber that wants the values reads the contact.
|
|
83
|
+
*/
|
|
84
|
+
export interface RefineEmitInput {
|
|
85
|
+
/** The canonical contact key the traits landed under. */
|
|
86
|
+
userId?: string;
|
|
87
|
+
email?: string;
|
|
88
|
+
contactId?: string;
|
|
89
|
+
/** The enrichment provider id that answered. */
|
|
90
|
+
provider: string;
|
|
91
|
+
/** Sorted key NAMES of the `refined_*` patch. */
|
|
92
|
+
traitKeys: string[];
|
|
93
|
+
/** The lookup instant — the same value the patch's `refined_at` carries. */
|
|
94
|
+
refinedAt: Date;
|
|
95
|
+
/** Per-endpoint delivery dedupe — see {@link refineOutboundDedupeKey}. */
|
|
96
|
+
dedupeKey: string;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** What the ledger gate needs to know about a prior lookup. */
|
|
100
|
+
export interface RefineLedgerRow {
|
|
101
|
+
status: EnrichmentLookupStatus;
|
|
102
|
+
expiresAt: Date;
|
|
103
|
+
/** The normalized `refined_*` patch the paid answer produced, if stored. */
|
|
104
|
+
traits?: Record<string, unknown> | null;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Everything stateful the chain touches, injected. Two reasons this is a seam
|
|
109
|
+
* rather than direct imports: `refineContact` must reach `ingestEvent` (which
|
|
110
|
+
* needs the db/registry/hatchet/logger singletons) from a STANDALONE call site,
|
|
111
|
+
* and the positional-journal law (AC 11) has to be provable by a pure test with
|
|
112
|
+
* no database and no Hatchet client.
|
|
113
|
+
*/
|
|
114
|
+
export interface RefineChainDeps {
|
|
115
|
+
/** The resolved ACTIVE provider, or undefined when none is configured. */
|
|
116
|
+
provider?: EnrichmentProvider;
|
|
117
|
+
/** The active provider's `meta.id` — what the ledger rows are keyed by. */
|
|
118
|
+
providerId?: string;
|
|
119
|
+
/** `ENRICHMENT_TTL_DAYS`. */
|
|
120
|
+
ttlDays: number;
|
|
121
|
+
/** `ENRICHMENT_MONTHLY_LOOKUPS`; 0 = uncapped. */
|
|
122
|
+
monthlyCap: number;
|
|
123
|
+
now: () => Date;
|
|
124
|
+
logger: {
|
|
125
|
+
info(message: string, meta?: Record<string, unknown>): void;
|
|
126
|
+
warn(message: string, meta?: Record<string, unknown>): void;
|
|
127
|
+
error(message: string, meta?: Record<string, unknown>): void;
|
|
128
|
+
};
|
|
129
|
+
resolveTarget(opts: RefineContactOptions): Promise<RefineTarget | null>;
|
|
130
|
+
findLedgerRow(input: {
|
|
131
|
+
provider: string;
|
|
132
|
+
lookupKind: EnrichmentLookupKind;
|
|
133
|
+
lookupKey: string;
|
|
134
|
+
}): Promise<RefineLedgerRow | null>;
|
|
135
|
+
/** Provider CALLS charged to the window starting at `since`. */
|
|
136
|
+
countLookupsSince(input: { since: Date }): Promise<number>;
|
|
137
|
+
writeLedgerRow(input: {
|
|
138
|
+
provider: string;
|
|
139
|
+
lookupKind: EnrichmentLookupKind;
|
|
140
|
+
lookupKey: string;
|
|
141
|
+
status: EnrichmentLookupStatus;
|
|
142
|
+
contactId?: string | null;
|
|
143
|
+
refinedAt: Date;
|
|
144
|
+
expiresAt: Date;
|
|
145
|
+
raw?: unknown;
|
|
146
|
+
traits?: Record<string, unknown> | null;
|
|
147
|
+
spendWindow: Date;
|
|
148
|
+
}): Promise<void>;
|
|
149
|
+
ingest(input: RefineIngestInput): Promise<void>;
|
|
150
|
+
/** First instant of the budget window containing `at` (UTC calendar month). */
|
|
151
|
+
budgetWindowStart(at: Date): Date;
|
|
152
|
+
/**
|
|
153
|
+
* Fire-and-forget outbound `contact.refined` emit. SYNCHRONOUS and
|
|
154
|
+
* `void`-returning by contract: the chain must not await it, so an outbound
|
|
155
|
+
* problem can never slow or fail a refinement. Optional — a caller with no
|
|
156
|
+
* db/hatchet (a test, a pure driver) omits it and the chain is unchanged.
|
|
157
|
+
*/
|
|
158
|
+
emitRefined?(input: RefineEmitInput): void;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
162
|
+
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
// The gate chain
|
|
165
|
+
// ---------------------------------------------------------------------------
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* The refinement gate chain, in its fixed order.
|
|
169
|
+
*
|
|
170
|
+
* THE LAW (the single thing this file exists to get right): the Hatchet journal
|
|
171
|
+
* is POSITIONAL, and `boundary.memoize` is a durable call, so its ISSUANCE must
|
|
172
|
+
* never be conditional on a live DB read, and its KEY must never contain one.
|
|
173
|
+
* The law is written out at `lib/feed.ts:152-165`; the corrected shape is
|
|
174
|
+
* `lib/connector-actions.ts:320-381` and this function mirrors it exactly.
|
|
175
|
+
*
|
|
176
|
+
* Refinement is the WORST case for the law, which is why the structure below is
|
|
177
|
+
* not negotiable. TWO live reads want to sit in front of the memo and neither
|
|
178
|
+
* may:
|
|
179
|
+
*
|
|
180
|
+
* - the ledger gate reads the very row this function's own final step writes,
|
|
181
|
+
* so a naive "check the ledger, early-return `cached`, otherwise memoize"
|
|
182
|
+
* would issue the durable call on the first run and NOT on the replay — not
|
|
183
|
+
* racily, but on EVERY replay;
|
|
184
|
+
* - resolving the SUBJECT is a live `contacts` read, and refinement is the one
|
|
185
|
+
* function in the engine whose whole purpose is to be called after a wait —
|
|
186
|
+
* exactly when the row has had time to gain an email (a form fill, an
|
|
187
|
+
* identify, a merge). With eviction live, replay-from-top is the NORMAL
|
|
188
|
+
* resume after every long sleep, so the divergence window is the whole
|
|
189
|
+
* sleep, and an anonymous-then-identified contact would issue ZERO memos on
|
|
190
|
+
* the run and ONE on the replay.
|
|
191
|
+
*
|
|
192
|
+
* So:
|
|
193
|
+
*
|
|
194
|
+
* - Step 0 is a PURE ARGUMENT check: with no `contactId`/`email`/`userId` at
|
|
195
|
+
* all there is nothing to refine and nothing to key from. It reads no
|
|
196
|
+
* database, so it cannot diverge, and returning `no_lookup_key` before the
|
|
197
|
+
* memo is safe.
|
|
198
|
+
* - Step 1 (no active provider) is a boot-time config read — also genuinely
|
|
199
|
+
* replay-stable, also safe outside.
|
|
200
|
+
* - Then `withDurableGate` OWNS the rest of the shape structurally (it is this
|
|
201
|
+
* function's former hand-rolled block, extracted): whenever a journey
|
|
202
|
+
* boundary exists it UNCONDITIONALLY derives the key from the CALLER'S
|
|
203
|
+
* ARGUMENTS ONLY (the branded `CallerRef` makes anything else a type
|
|
204
|
+
* error), `registerKey`s it, and issues `boundary.memoize`. No condition
|
|
205
|
+
* guards that call.
|
|
206
|
+
* - Steps 2-6 (resolve, ledger gate, budget cap, provider call, ledger write +
|
|
207
|
+
* ingest) ALL live INSIDE the memo closure, so EVERY verdict — including
|
|
208
|
+
* `no_lookup_key` — is recorded by the durable memo and replayed verbatim.
|
|
209
|
+
* - With NO boundary (a webhook, a cron, a test) the chain runs directly with
|
|
210
|
+
* no memo at all, exactly as `connector-actions.ts:342` does. Layer 2 — the
|
|
211
|
+
* `enrichment_lookups` unique index — carries exactly-once on its own there.
|
|
212
|
+
*/
|
|
213
|
+
export async function runRefineChain(
|
|
214
|
+
deps: RefineChainDeps,
|
|
215
|
+
opts: RefineContactOptions,
|
|
216
|
+
): Promise<RefineContactResult> {
|
|
217
|
+
// ---- (0) Pure ARGUMENT check. No DB. Safe outside the memo. ---------------
|
|
218
|
+
// `callerRef` is also the memo discriminant: authored input, never a resolved
|
|
219
|
+
// one. `deriveJourneyKey`'s contract is that the key contains no wall-clock
|
|
220
|
+
// and no live-DB-read result (journey-boundary.ts) — a lookup key resolved
|
|
221
|
+
// off `contacts` would break that, and a domain→email upgrade mid-sleep would
|
|
222
|
+
// silently re-key the memo AND the ledger row, double-charging the vendor.
|
|
223
|
+
const callerRef = refineCallerRef(opts);
|
|
224
|
+
if (!callerRef) return { status: "skipped", reason: "no_lookup_key" };
|
|
225
|
+
|
|
226
|
+
// ---- (1) No active provider. OUTSIDE the memo (a config read). -----------
|
|
227
|
+
const provider = deps.provider;
|
|
228
|
+
const providerId = deps.providerId ?? provider?.meta.id;
|
|
229
|
+
if (!provider || !providerId) {
|
|
230
|
+
return { status: "skipped", reason: "no_provider" };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// boundary resolution → key derivation → registerKey → memoize is owned by
|
|
234
|
+
// `withDurableGate`, which is the extraction of the block that used to live
|
|
235
|
+
// right here — the ordering is structural now, not this function's to write.
|
|
236
|
+
return withDurableGate(
|
|
237
|
+
{ kind: "refine", callerRef, idempotencyLabel: opts.idempotencyLabel },
|
|
238
|
+
(idempotencyKey?: string) =>
|
|
239
|
+
runGates({ deps, opts, provider, providerId, idempotencyKey }),
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The caller's own reference to the subject, normalized — the ONLY input the
|
|
245
|
+
* replay key is allowed to be derived from. Precedence mirrors
|
|
246
|
+
* `resolveRefineTarget`'s specificity order, but every leg is an argument the
|
|
247
|
+
* journey authored, so it is identical on a run and its replay (Hatchet
|
|
248
|
+
* re-derives task input from the recorded payload).
|
|
249
|
+
*/
|
|
250
|
+
function refineCallerRef(opts: RefineContactOptions): CallerRef | undefined {
|
|
251
|
+
const contactId = callerRefFromArgs(opts.contactId);
|
|
252
|
+
if (contactId) return contactId;
|
|
253
|
+
// Mirrors `normalizeEmail` (contacts.ts) without importing the db layer into
|
|
254
|
+
// what must stay a pure, database-free module.
|
|
255
|
+
const email = callerRefFromArgs(opts.email?.toLowerCase());
|
|
256
|
+
if (email) return email;
|
|
257
|
+
return callerRefFromArgs(opts.userId);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Steps 2-6 — everything stateful, and therefore everything that must be
|
|
262
|
+
* RECORDED by the memo rather than re-derived on a replay. Cheap-before-spend
|
|
263
|
+
* ordering still holds; it just holds inside the closure.
|
|
264
|
+
*/
|
|
265
|
+
async function runGates(args: {
|
|
266
|
+
deps: RefineChainDeps;
|
|
267
|
+
opts: RefineContactOptions;
|
|
268
|
+
provider: EnrichmentProvider;
|
|
269
|
+
providerId: string;
|
|
270
|
+
idempotencyKey?: string;
|
|
271
|
+
}): Promise<RefineContactResult> {
|
|
272
|
+
const { deps, opts, provider, providerId } = args;
|
|
273
|
+
|
|
274
|
+
// ---- (2) Resolve the subject + the lookup key. INSIDE the memo. ----------
|
|
275
|
+
// A live `contacts` read, so the verdict it produces — including the
|
|
276
|
+
// `no_lookup_key` skip — is memo-recorded and replayed verbatim.
|
|
277
|
+
const target = await deps.resolveTarget(opts);
|
|
278
|
+
const email = target?.email?.trim() || undefined;
|
|
279
|
+
const domain = target?.domain?.trim() || undefined;
|
|
280
|
+
const lookupKey = email ?? domain;
|
|
281
|
+
if (!target || !lookupKey) {
|
|
282
|
+
return { status: "skipped", reason: "no_lookup_key" };
|
|
283
|
+
}
|
|
284
|
+
const lookupKind: EnrichmentLookupKind = email ? "email" : "domain";
|
|
285
|
+
|
|
286
|
+
// ---- (3) Ledger gate — TTL cache + negative cache. Zero spend. -----------
|
|
287
|
+
if (!opts.force) {
|
|
288
|
+
const row = await deps.findLedgerRow({
|
|
289
|
+
provider: providerId,
|
|
290
|
+
lookupKind,
|
|
291
|
+
lookupKey,
|
|
292
|
+
});
|
|
293
|
+
if (row && row.expiresAt.getTime() > deps.now().getTime()) {
|
|
294
|
+
// A `found` row is a PAID positive answer. The ledger is keyed by
|
|
295
|
+
// (provider, kind, key) with NO contact dimension, and a `domain` key is
|
|
296
|
+
// shared by every contact at that company — so "already paid for" and
|
|
297
|
+
// "this contact already has the traits" are different questions. Land the
|
|
298
|
+
// stored patch on WHOEVER is being asked about, then return `cached`.
|
|
299
|
+
if (row.status === "found") {
|
|
300
|
+
return landCachedTraits({
|
|
301
|
+
deps,
|
|
302
|
+
providerId,
|
|
303
|
+
target,
|
|
304
|
+
row,
|
|
305
|
+
...(args.idempotencyKey
|
|
306
|
+
? { idempotencyKey: args.idempotencyKey }
|
|
307
|
+
: {}),
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
// A `not_found` row is a PAID negative answer — a miss costs money too.
|
|
311
|
+
if (row.status === "not_found") return { status: "not_found" };
|
|
312
|
+
// `error` deliberately falls through: it is not a paid result, so it must
|
|
313
|
+
// never suppress a retry (AC 7).
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ---- (4) Budget cap. Fails CLOSED; `force` does not bypass it. -----------
|
|
318
|
+
// Counts provider CALLS in the window, not ledger rows: the row is one per
|
|
319
|
+
// subject and `force` updates it in place, so a row count would let a force
|
|
320
|
+
// loop on one key spend without limit.
|
|
321
|
+
if (deps.monthlyCap > 0) {
|
|
322
|
+
const used = await deps.countLookupsSince({
|
|
323
|
+
since: deps.budgetWindowStart(deps.now()),
|
|
324
|
+
});
|
|
325
|
+
if (used >= deps.monthlyCap) {
|
|
326
|
+
deps.logger.warn("refineContact: monthly enrichment budget exhausted", {
|
|
327
|
+
provider: providerId,
|
|
328
|
+
used,
|
|
329
|
+
cap: deps.monthlyCap,
|
|
330
|
+
});
|
|
331
|
+
return { status: "skipped", reason: "budget_exceeded" };
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
// ---- (5) The one call that costs money. ----------------------------------
|
|
336
|
+
const query: EnrichmentQuery = {
|
|
337
|
+
...(target.email ? { email: target.email } : {}),
|
|
338
|
+
...(target.domain ? { domain: target.domain } : {}),
|
|
339
|
+
...(target.firstName ? { firstName: target.firstName } : {}),
|
|
340
|
+
...(target.lastName ? { lastName: target.lastName } : {}),
|
|
341
|
+
...(target.company ? { company: target.company } : {}),
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
let result: EnrichmentResult;
|
|
345
|
+
try {
|
|
346
|
+
result = await provider.enrichPerson(query);
|
|
347
|
+
} catch (error) {
|
|
348
|
+
// A vendor failure must never fail the journey run or the ingest that
|
|
349
|
+
// triggered it. Record the SPEND (the request left the building, so the cap
|
|
350
|
+
// must see it) and return a skip.
|
|
351
|
+
deps.logger.error("refineContact: enrichment provider failed", {
|
|
352
|
+
provider: providerId,
|
|
353
|
+
lookupKind,
|
|
354
|
+
error: error instanceof Error ? error.message : String(error),
|
|
355
|
+
});
|
|
356
|
+
const failedAt = deps.now();
|
|
357
|
+
try {
|
|
358
|
+
await deps.writeLedgerRow({
|
|
359
|
+
provider: providerId,
|
|
360
|
+
lookupKind,
|
|
361
|
+
lookupKey,
|
|
362
|
+
status: "error",
|
|
363
|
+
contactId: target.contactId ?? null,
|
|
364
|
+
refinedAt: failedAt,
|
|
365
|
+
// Already expired on arrival — belt-and-braces behind the status check
|
|
366
|
+
// in step 3, so an `error` row can never suppress a later retry.
|
|
367
|
+
expiresAt: failedAt,
|
|
368
|
+
raw: null,
|
|
369
|
+
traits: null,
|
|
370
|
+
spendWindow: deps.budgetWindowStart(failedAt),
|
|
371
|
+
});
|
|
372
|
+
} catch (writeError) {
|
|
373
|
+
deps.logger.error("refineContact: could not record the error lookup", {
|
|
374
|
+
provider: providerId,
|
|
375
|
+
error:
|
|
376
|
+
writeError instanceof Error ? writeError.message : String(writeError),
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
return { status: "skipped", reason: "provider_error" };
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
// ---- (6) Ledger row FIRST, then the ingest that closes the loop. ---------
|
|
383
|
+
const refinedAt = deps.now();
|
|
384
|
+
const properties = flattenTraits(result, {
|
|
385
|
+
provider: providerId,
|
|
386
|
+
refinedAt,
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
// The ledger row carries the NORMALIZED patch, not just the vendor's `raw`:
|
|
390
|
+
// `raw` is vendor-shaped and the engine cannot re-flatten it without the
|
|
391
|
+
// provider, so without this a later cache hit for a DIFFERENT contact sharing
|
|
392
|
+
// the key would have nothing to land.
|
|
393
|
+
await deps.writeLedgerRow({
|
|
394
|
+
provider: providerId,
|
|
395
|
+
lookupKind,
|
|
396
|
+
lookupKey,
|
|
397
|
+
status: result.found ? "found" : "not_found",
|
|
398
|
+
contactId: target.contactId ?? null,
|
|
399
|
+
refinedAt,
|
|
400
|
+
expiresAt: new Date(refinedAt.getTime() + deps.ttlDays * MS_PER_DAY),
|
|
401
|
+
raw: result.raw,
|
|
402
|
+
traits: result.found ? properties : null,
|
|
403
|
+
spendWindow: deps.budgetWindowStart(refinedAt),
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
const landed = await landTraits({
|
|
407
|
+
deps,
|
|
408
|
+
target,
|
|
409
|
+
properties,
|
|
410
|
+
providerId,
|
|
411
|
+
found: result.found,
|
|
412
|
+
cached: false,
|
|
413
|
+
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
414
|
+
});
|
|
415
|
+
if (!landed) return { status: "skipped", reason: "ingest_failed" };
|
|
416
|
+
if (!result.found) return { status: "not_found" };
|
|
417
|
+
|
|
418
|
+
// ---- (7) OUTBOUND `contact.refined`. The ONLY emission site. -------------
|
|
419
|
+
// Placed here on purpose: on the already-decided `refined` verdict, AFTER
|
|
420
|
+
// every stateful step, INSIDE the gate closure. It issues no durable call and
|
|
421
|
+
// sits nowhere near the gate's own ordering, so the journal is byte-identical
|
|
422
|
+
// with and without it — and on an eviction-capable replay the closure is not
|
|
423
|
+
// re-entered at all, so the event is not re-emitted either.
|
|
424
|
+
//
|
|
425
|
+
// Deliberately NOT emitted for `cached` / `not_found` / `skipped`: a cache hit
|
|
426
|
+
// spends nothing and changes nothing a subscriber did not already hear about.
|
|
427
|
+
//
|
|
428
|
+
// The dep is void-returning, so nothing here is awaited and an outbound
|
|
429
|
+
// failure cannot touch this result.
|
|
430
|
+
deps.emitRefined?.({
|
|
431
|
+
...(target.userId ? { userId: target.userId } : {}),
|
|
432
|
+
...(target.email ? { email: target.email } : {}),
|
|
433
|
+
...(target.contactId ? { contactId: target.contactId } : {}),
|
|
434
|
+
provider: providerId,
|
|
435
|
+
// VENDOR facts only — `refined_at` / `refined_provider` are provenance and
|
|
436
|
+
// are dropped. Two reasons, both deliberate: the envelope already carries
|
|
437
|
+
// both as top-level `at` / `provider`, so keeping them here would say the
|
|
438
|
+
// same thing twice; and `flattenTraits` stamps them UNCONDITIONALLY, so a
|
|
439
|
+
// found-but-empty vendor answer would otherwise report two changed traits
|
|
440
|
+
// when nothing about the person changed. `traits: []` now says that
|
|
441
|
+
// honestly.
|
|
442
|
+
traitKeys: Object.keys(properties)
|
|
443
|
+
.filter((key) => !REFINED_META_KEYS.includes(key))
|
|
444
|
+
.sort(),
|
|
445
|
+
refinedAt,
|
|
446
|
+
dedupeKey: refineOutboundDedupeKey({
|
|
447
|
+
provider: providerId,
|
|
448
|
+
lookupKind,
|
|
449
|
+
lookupKey,
|
|
450
|
+
refinedAt,
|
|
451
|
+
}),
|
|
452
|
+
});
|
|
453
|
+
|
|
454
|
+
return { status: "refined", properties };
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
/**
|
|
458
|
+
* A cache HIT on a key that may well have been paid for by a DIFFERENT contact.
|
|
459
|
+
*
|
|
460
|
+
* `cached` means exactly one thing: THIS LOOKUP KEY WAS ALREADY PAID FOR. The
|
|
461
|
+
* spend contract (AC 2) is untouched — zero provider calls, zero new ledger
|
|
462
|
+
* rows — but the OUTCOME is no longer suppressed along with the spend. The
|
|
463
|
+
* stored patch is landed on the contact being asked about, so the GTM loop
|
|
464
|
+
* closes for the second, third and hundredth contact at the same company, not
|
|
465
|
+
* just the one that paid.
|
|
466
|
+
*
|
|
467
|
+
* The ingest runs even when the contact already carries every trait, because
|
|
468
|
+
* the property merge is not the point: `checkBucketMembership` re-runs ONLY on
|
|
469
|
+
* the ingest path (DECISIONS §3.3), and a contact can hold the traits while its
|
|
470
|
+
* membership was never evaluated — which is precisely the state a failed ingest
|
|
471
|
+
* leaves behind. That makes the retry after an `ingest_failed` free.
|
|
472
|
+
*
|
|
473
|
+
* Rows written before the `traits` column existed have nothing stored; those
|
|
474
|
+
* fall back to the caller's own traits, which is exactly the old behaviour.
|
|
475
|
+
*/
|
|
476
|
+
async function landCachedTraits(args: {
|
|
477
|
+
deps: RefineChainDeps;
|
|
478
|
+
providerId: string;
|
|
479
|
+
target: RefineTarget;
|
|
480
|
+
row: RefineLedgerRow;
|
|
481
|
+
idempotencyKey?: string;
|
|
482
|
+
}): Promise<RefineContactResult> {
|
|
483
|
+
const { deps, target, row } = args;
|
|
484
|
+
const stored = row.traits ?? undefined;
|
|
485
|
+
if (!stored) {
|
|
486
|
+
return { status: "cached", properties: target.refinedProperties ?? {} };
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
const landed = await landTraits({
|
|
490
|
+
deps,
|
|
491
|
+
target,
|
|
492
|
+
properties: stored,
|
|
493
|
+
providerId: args.providerId,
|
|
494
|
+
found: true,
|
|
495
|
+
cached: true,
|
|
496
|
+
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
497
|
+
});
|
|
498
|
+
// A failed ingest on a cache hit costs nothing to retry: the ledger row (and
|
|
499
|
+
// its stored patch) stays exactly as it is, so the NEXT call hits this same
|
|
500
|
+
// branch and lands the traits for free.
|
|
501
|
+
if (!landed) return { status: "skipped", reason: "ingest_failed" };
|
|
502
|
+
|
|
503
|
+
return { status: "cached", properties: stored };
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* THROUGH `ingestEvent`, never `resolveOrCreateContact`: only the ingest path
|
|
508
|
+
* re-runs `checkBucketMembership`, and that synchronous re-evaluation is the
|
|
509
|
+
* entire point — it is what turns a fit trait into a bucket transition.
|
|
510
|
+
*
|
|
511
|
+
* Returns false when the ingest failed. The throw is swallowed deliberately:
|
|
512
|
+
* refinement must never fail the journey run or the ingest that triggered it,
|
|
513
|
+
* and by this point a paid ledger row is already committed — letting the
|
|
514
|
+
* exception escape would spend the money AND kill the run.
|
|
515
|
+
*/
|
|
516
|
+
async function landTraits(args: {
|
|
517
|
+
deps: RefineChainDeps;
|
|
518
|
+
target: RefineTarget;
|
|
519
|
+
properties: Record<string, unknown>;
|
|
520
|
+
providerId: string;
|
|
521
|
+
found: boolean;
|
|
522
|
+
cached: boolean;
|
|
523
|
+
idempotencyKey?: string;
|
|
524
|
+
}): Promise<boolean> {
|
|
525
|
+
const { deps, target } = args;
|
|
526
|
+
try {
|
|
527
|
+
await deps.ingest({
|
|
528
|
+
...(target.userId ? { userId: target.userId } : {}),
|
|
529
|
+
...(target.email ? { email: target.email } : {}),
|
|
530
|
+
...(target.contactId ? { contactId: target.contactId } : {}),
|
|
531
|
+
eventProperties: {
|
|
532
|
+
provider: args.providerId,
|
|
533
|
+
found: args.found,
|
|
534
|
+
cached: args.cached,
|
|
535
|
+
},
|
|
536
|
+
contactProperties: args.properties,
|
|
537
|
+
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
538
|
+
});
|
|
539
|
+
return true;
|
|
540
|
+
} catch (error) {
|
|
541
|
+
deps.logger.error("refineContact: could not land the refined traits", {
|
|
542
|
+
provider: args.providerId,
|
|
543
|
+
cached: args.cached,
|
|
544
|
+
error: error instanceof Error ? error.message : String(error),
|
|
545
|
+
});
|
|
546
|
+
return false;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/**
|
|
551
|
+
* The per-endpoint delivery dedupe key for `contact.refined`.
|
|
552
|
+
*
|
|
553
|
+
* The LEDGER ROW'S IDENTITY — `(provider, lookupKind, lookupKey)`, the
|
|
554
|
+
* `enrichment_lookups` unique triple — PLUS the lookup instant of the answer
|
|
555
|
+
* being announced.
|
|
556
|
+
*
|
|
557
|
+
* The instant is load-bearing, not decoration. `webhook_deliveries` carries a
|
|
558
|
+
* unique `(endpointId, dedupeKey)` index (schema/webhook-deliveries.ts) and
|
|
559
|
+
* nothing in the engine ever deletes a delivery row, so a key that is only the
|
|
560
|
+
* triple is PERMANENT: a contact refined today, re-refined in 90 days when
|
|
561
|
+
* `ENRICHMENT_TTL_DAYS` expires and the vendor reports a new job title, would
|
|
562
|
+
* recompute a byte-identical key and the subscriber would never be told —
|
|
563
|
+
* which is precisely the event a CRM sync exists to receive. Same for any
|
|
564
|
+
* `force: true` refresh.
|
|
565
|
+
*
|
|
566
|
+
* That costs nothing on the re-drive side, because the instant is not what
|
|
567
|
+
* defends a re-drive: the LEDGER GATE is. A retry of the same refinement finds
|
|
568
|
+
* the row the first attempt wrote, returns `cached`, and never reaches the
|
|
569
|
+
* emit at all. The dedupe is the backstop for the narrow race where two
|
|
570
|
+
* concurrent lookups of one subject both pass that gate — and for the truly
|
|
571
|
+
* identical answer (same `refinedAt`) it still holds exactly.
|
|
572
|
+
*
|
|
573
|
+
* `toISOString()` because the key must be byte-stable: a locale string is not.
|
|
574
|
+
*/
|
|
575
|
+
export function refineOutboundDedupeKey(input: {
|
|
576
|
+
provider: string;
|
|
577
|
+
lookupKind: EnrichmentLookupKind;
|
|
578
|
+
lookupKey: string;
|
|
579
|
+
refinedAt: Date;
|
|
580
|
+
}): string {
|
|
581
|
+
const at = input.refinedAt.toISOString();
|
|
582
|
+
return `refine:${input.provider}:${input.lookupKind}:${input.lookupKey}:${at}`;
|
|
583
|
+
}
|