@hogsend/engine 0.55.0 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/check-membership.ts +13 -0
- package/src/lib/enrichment-ledger.ts +1 -1
- package/src/lib/ingestion.ts +11 -0
- package/src/lib/outbound.ts +3 -3
- package/src/lib/refine-chain.test.ts +83 -25
- package/src/lib/refine-chain.ts +124 -23
- package/src/lib/refine-traits.test.ts +74 -41
- package/src/lib/refine-traits.ts +107 -52
- package/src/lib/refine.ts +28 -27
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hogsend/engine",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.56.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -53,18 +53,18 @@
|
|
|
53
53
|
"svix": "^1.95.1",
|
|
54
54
|
"winston": "^3.19.0",
|
|
55
55
|
"zod": "^4.4.3",
|
|
56
|
-
"@hogsend/attribution": "^0.
|
|
57
|
-
"@hogsend/core": "^0.
|
|
58
|
-
"@hogsend/db": "^0.
|
|
59
|
-
"@hogsend/email": "^0.
|
|
60
|
-
"@hogsend/plugin-posthog": "^0.
|
|
61
|
-
"@hogsend/plugin-resend": "^0.
|
|
62
|
-
"@hogsend/sms": "^0.
|
|
56
|
+
"@hogsend/attribution": "^0.56.0",
|
|
57
|
+
"@hogsend/core": "^0.56.0",
|
|
58
|
+
"@hogsend/db": "^0.56.0",
|
|
59
|
+
"@hogsend/email": "^0.56.0",
|
|
60
|
+
"@hogsend/plugin-posthog": "^0.56.0",
|
|
61
|
+
"@hogsend/plugin-resend": "^0.56.0",
|
|
62
|
+
"@hogsend/sms": "^0.56.0"
|
|
63
63
|
},
|
|
64
64
|
"optionalDependencies": {
|
|
65
|
-
"@hogsend/plugin-apollo": "^0.
|
|
66
|
-
"@hogsend/plugin-postmark": "^0.
|
|
67
|
-
"@hogsend/plugin-twilio": "^0.
|
|
65
|
+
"@hogsend/plugin-apollo": "^0.56.0",
|
|
66
|
+
"@hogsend/plugin-postmark": "^0.56.0",
|
|
67
|
+
"@hogsend/plugin-twilio": "^0.56.0"
|
|
68
68
|
},
|
|
69
69
|
"devDependencies": {
|
|
70
70
|
"@types/node": "^22.15.3",
|
|
@@ -72,6 +72,14 @@ export async function checkBucketMembership(opts: {
|
|
|
72
72
|
* the very first event after a property change evaluates correctly (risk 7).
|
|
73
73
|
*/
|
|
74
74
|
contactProperties?: Record<string, unknown>;
|
|
75
|
+
/**
|
|
76
|
+
* Extra property NAMES for candidate-narrowing ONLY — never overlaid on the
|
|
77
|
+
* contact row for value eval. A producer whose write patch omits a key it
|
|
78
|
+
* nonetheless changed (refinement's fill-if-absent drops already-held facts to
|
|
79
|
+
* avoid clobbering first-party data) lists those names here so their fit
|
|
80
|
+
* buckets are still re-checked against live contact state.
|
|
81
|
+
*/
|
|
82
|
+
touchProperties?: string[];
|
|
75
83
|
/** Optional override; defaults to the process bucket-registry singleton. */
|
|
76
84
|
bucketRegistry?: ReturnType<typeof getBucketRegistrySingleton>;
|
|
77
85
|
}): Promise<BucketTransition[]> {
|
|
@@ -85,6 +93,7 @@ export async function checkBucketMembership(opts: {
|
|
|
85
93
|
event,
|
|
86
94
|
eventProperties,
|
|
87
95
|
contactProperties: contactPropertiesPatch,
|
|
96
|
+
touchProperties,
|
|
88
97
|
} = opts;
|
|
89
98
|
// The caller's email comes verbatim from the raw event payload — normalize
|
|
90
99
|
// ONCE here so the membership row, the emitted bucket:* transition events,
|
|
@@ -118,6 +127,10 @@ export async function checkBucketMembership(opts: {
|
|
|
118
127
|
for (const key of [
|
|
119
128
|
...Object.keys(eventProperties ?? {}),
|
|
120
129
|
...Object.keys(contactPropertiesPatch ?? {}),
|
|
130
|
+
// Names-only touches (e.g. refinement's fill-if-absent dropped facts) surface
|
|
131
|
+
// a fit bucket here so it is re-checked against live contact state below,
|
|
132
|
+
// even though the value is not in this ingest's write patch.
|
|
133
|
+
...(touchProperties ?? []),
|
|
121
134
|
]) {
|
|
122
135
|
for (const bucket of bucketRegistry.getByReferencedProperty(key)) {
|
|
123
136
|
candidateMap.set(bucket.id, bucket);
|
|
@@ -22,7 +22,7 @@ export interface EnrichmentLookupRow {
|
|
|
22
22
|
refinedAt: Date;
|
|
23
23
|
expiresAt: Date;
|
|
24
24
|
raw: Record<string, unknown> | null;
|
|
25
|
-
/** The normalized
|
|
25
|
+
/** The normalized canonical trait patch the paid answer produced. */
|
|
26
26
|
traits: Record<string, unknown> | null;
|
|
27
27
|
spendWindow: Date | null;
|
|
28
28
|
spendCount: number;
|
package/src/lib/ingestion.ts
CHANGED
|
@@ -71,6 +71,14 @@ export interface IngestEvent {
|
|
|
71
71
|
eventProperties: Record<string, unknown>;
|
|
72
72
|
/** D2: → `contacts.properties` merge ONLY. */
|
|
73
73
|
contactProperties?: Record<string, unknown>;
|
|
74
|
+
/**
|
|
75
|
+
* Extra property NAMES to re-evaluate buckets for — candidate-narrowing ONLY,
|
|
76
|
+
* NOT written and NOT in value eval. Lets a producer whose write patch omits a
|
|
77
|
+
* key (e.g. refinement's fill-if-absent, which drops already-held facts so it
|
|
78
|
+
* never clobbers first-party data) still trigger a fit bucket that references
|
|
79
|
+
* that key. Off by default; only engine-internal callers set it.
|
|
80
|
+
*/
|
|
81
|
+
touchProperties?: string[];
|
|
74
82
|
/**
|
|
75
83
|
* The groupType→groupKey association map for this event — persisted on
|
|
76
84
|
* `user_events.groups`, drives group membership (each group row is ensured +
|
|
@@ -787,6 +795,9 @@ export async function ingestEvent(opts: {
|
|
|
787
795
|
event: event.event,
|
|
788
796
|
eventProperties: event.eventProperties,
|
|
789
797
|
contactProperties: event.contactProperties ?? {},
|
|
798
|
+
...(event.touchProperties
|
|
799
|
+
? { touchProperties: event.touchProperties }
|
|
800
|
+
: {}),
|
|
790
801
|
});
|
|
791
802
|
} catch (err) {
|
|
792
803
|
logger.warn("Bucket membership check failed", {
|
package/src/lib/outbound.ts
CHANGED
|
@@ -113,8 +113,8 @@ export interface OutboundPayloads {
|
|
|
113
113
|
* ONLY on a genuine `status: "refined"` — never on `cached`, `not_found` or
|
|
114
114
|
* `skipped`, because a cache hit is a spend decision, not a change.
|
|
115
115
|
*
|
|
116
|
-
* `traits` carries the
|
|
117
|
-
* "
|
|
116
|
+
* `traits` carries the canonical trait KEY NAMES only — `["title",
|
|
117
|
+
* "company_domain"]` — never their values. That is deliberate: the
|
|
118
118
|
* payload says WHAT CHANGED, and a subscriber that wants the values reads
|
|
119
119
|
* the contact (`GET /v1/contacts`). So vendor-licensed person data never
|
|
120
120
|
* leaves the building inside a webhook body, and the envelope stays a fixed
|
|
@@ -132,7 +132,7 @@ export interface OutboundPayloads {
|
|
|
132
132
|
provider: string;
|
|
133
133
|
/** Mapped trait key NAMES, sorted. Values are deliberately absent. */
|
|
134
134
|
traits: string[];
|
|
135
|
-
/** ISO — the lookup instant (the same value as `
|
|
135
|
+
/** ISO — the lookup instant (the same value as `enrichment.at`). */
|
|
136
136
|
at: string;
|
|
137
137
|
};
|
|
138
138
|
"email.sent": {
|
|
@@ -93,6 +93,10 @@ interface Harness {
|
|
|
93
93
|
ingests: number;
|
|
94
94
|
/** Every outbound `contact.refined` emit, in order. */
|
|
95
95
|
emits: RefineEmitInput[];
|
|
96
|
+
/** The `contactProperties` each `ingest` received, in order. */
|
|
97
|
+
ingested: Record<string, unknown>[];
|
|
98
|
+
/** The `touchProperties` each `ingest` received, in order. */
|
|
99
|
+
touched: (string[] | undefined)[];
|
|
96
100
|
}
|
|
97
101
|
|
|
98
102
|
function harness(overrides: {
|
|
@@ -116,6 +120,8 @@ function harness(overrides: {
|
|
|
116
120
|
ledgerWrites: 0,
|
|
117
121
|
ingests: 0,
|
|
118
122
|
emits: [] as RefineEmitInput[],
|
|
123
|
+
ingested: [] as Record<string, unknown>[],
|
|
124
|
+
touched: [] as (string[] | undefined)[],
|
|
119
125
|
};
|
|
120
126
|
|
|
121
127
|
const deps: RefineChainDeps = {
|
|
@@ -135,15 +141,16 @@ function harness(overrides: {
|
|
|
135
141
|
contactId: "c1",
|
|
136
142
|
userId: "u1",
|
|
137
143
|
email: "a@acme.com",
|
|
138
|
-
refinedProperties: { refined_title: "old" },
|
|
139
144
|
},
|
|
140
145
|
findLedgerRow: async () => overrides.ledgerRow ?? null,
|
|
141
146
|
countLookupsSince: async () => overrides.usedThisMonth ?? 0,
|
|
142
147
|
writeLedgerRow: async () => {
|
|
143
148
|
state.ledgerWrites += 1;
|
|
144
149
|
},
|
|
145
|
-
ingest: async () => {
|
|
150
|
+
ingest: async (input) => {
|
|
146
151
|
state.ingests += 1;
|
|
152
|
+
state.ingested.push(input.contactProperties);
|
|
153
|
+
state.touched.push(input.touchProperties);
|
|
147
154
|
if (overrides.ingest) await overrides.ingest();
|
|
148
155
|
},
|
|
149
156
|
emitRefined: (input) => {
|
|
@@ -163,6 +170,12 @@ function harness(overrides: {
|
|
|
163
170
|
get emits() {
|
|
164
171
|
return state.emits;
|
|
165
172
|
},
|
|
173
|
+
get ingested() {
|
|
174
|
+
return state.ingested;
|
|
175
|
+
},
|
|
176
|
+
get touched() {
|
|
177
|
+
return state.touched;
|
|
178
|
+
},
|
|
166
179
|
};
|
|
167
180
|
}
|
|
168
181
|
|
|
@@ -470,7 +483,7 @@ test("D2: a cache hit lands the STORED patch on a contact that never received it
|
|
|
470
483
|
// The shared-domain case: contact B hits the row contact A paid for. The
|
|
471
484
|
// ledger has no contact dimension, so "already paid for" says nothing about
|
|
472
485
|
// whether THIS contact has the traits.
|
|
473
|
-
const stored = {
|
|
486
|
+
const stored = { title: "CTO", company_employees: 250 };
|
|
474
487
|
const h = harness({
|
|
475
488
|
ledgerRow: unexpired("found", stored),
|
|
476
489
|
target: { contactId: "c2", userId: "u2", domain: "acme.com" },
|
|
@@ -491,14 +504,14 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
|
|
|
491
504
|
// buckets: only the ingest path re-runs `checkBucketMembership`, and a
|
|
492
505
|
// contact can carry the traits with no membership (exactly what a failed
|
|
493
506
|
// ingest leaves behind). So the cached verdict always re-lands.
|
|
494
|
-
const stored = {
|
|
507
|
+
const stored = { title: "CTO", company_employees: 250 };
|
|
495
508
|
const h = harness({
|
|
496
509
|
ledgerRow: unexpired("found", stored),
|
|
497
510
|
target: {
|
|
498
511
|
contactId: "c1",
|
|
499
512
|
userId: "u1",
|
|
500
513
|
domain: "acme.com",
|
|
501
|
-
|
|
514
|
+
existingProperties: { ...stored },
|
|
502
515
|
},
|
|
503
516
|
});
|
|
504
517
|
|
|
@@ -506,23 +519,74 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
|
|
|
506
519
|
|
|
507
520
|
assert.deepEqual(result, { status: "cached", properties: stored });
|
|
508
521
|
assert.equal(h.ingests, 1);
|
|
522
|
+
// The contact already held every fact, so the WRITE patch is empty (nothing
|
|
523
|
+
// first-party is clobbered) — but the ingest still runs and TOUCHES both keys,
|
|
524
|
+
// so bucket membership re-evaluates against live contact state.
|
|
525
|
+
assert.deepEqual(h.ingested, [{}]);
|
|
526
|
+
assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
|
|
527
|
+
"company_employees",
|
|
528
|
+
"title",
|
|
529
|
+
]);
|
|
509
530
|
assert.equal(h.providerCalls(), 0);
|
|
510
531
|
assert.equal(h.ledgerWrites, 0);
|
|
511
532
|
});
|
|
512
533
|
|
|
513
|
-
test("D2: a legacy row with no stored patch
|
|
534
|
+
test("D2: a legacy row with no stored patch returns cached with an empty patch", async () => {
|
|
514
535
|
const h = harness({ ledgerRow: unexpired("found", null) });
|
|
515
536
|
|
|
516
537
|
const result = await runRefineChain(h.deps, SUBJECT);
|
|
517
538
|
|
|
518
539
|
assert.deepEqual(result, {
|
|
519
540
|
status: "cached",
|
|
520
|
-
properties: {
|
|
541
|
+
properties: {},
|
|
521
542
|
});
|
|
522
543
|
assert.equal(h.ingests, 0);
|
|
523
544
|
assert.equal(h.providerCalls(), 0);
|
|
524
545
|
});
|
|
525
546
|
|
|
547
|
+
// ---------------------------------------------------------------------------
|
|
548
|
+
// Fill-if-absent — a paid answer fills gaps, never overwrites first-party data
|
|
549
|
+
// ---------------------------------------------------------------------------
|
|
550
|
+
|
|
551
|
+
test("fill-if-absent: a fresh lookup fills gaps but never overwrites a first-party value", async () => {
|
|
552
|
+
// FOUND carries title / seniority / company / company_employees. This contact
|
|
553
|
+
// already has a first-party `title` and `company`, so the paid answer must
|
|
554
|
+
// leave those two alone and only fill what was missing.
|
|
555
|
+
const h = harness({
|
|
556
|
+
target: {
|
|
557
|
+
contactId: "c1",
|
|
558
|
+
userId: "u1",
|
|
559
|
+
email: "a@acme.com",
|
|
560
|
+
existingProperties: { title: "First-party VP", company: "First Co" },
|
|
561
|
+
},
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
const result = await runRefineChain(h.deps, SUBJECT);
|
|
565
|
+
|
|
566
|
+
assert.equal(result.status, "refined");
|
|
567
|
+
// The vendor's FULL answer is still reported on the result...
|
|
568
|
+
assert.equal(result.properties?.title, "CTO");
|
|
569
|
+
// ...but the WRITE patch OMITS the already-held title + company entirely (so
|
|
570
|
+
// the additive merge leaves the live first-party value untouched — atomic, no
|
|
571
|
+
// stale write-back) and carries the vendor's value only for the gaps.
|
|
572
|
+
const landed = h.ingested[0] ?? {};
|
|
573
|
+
assert.equal("title" in landed, false);
|
|
574
|
+
assert.equal("company" in landed, false);
|
|
575
|
+
assert.equal(landed.seniority, "c_suite");
|
|
576
|
+
assert.equal(landed.company_employees, 250);
|
|
577
|
+
assert.ok(landed.enrichment, "provenance always lands");
|
|
578
|
+
// ...yet every returned fact is TOUCHED, so a fit bucket on the already-held
|
|
579
|
+
// title/company still re-evaluates against live contact state.
|
|
580
|
+
assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
|
|
581
|
+
"company",
|
|
582
|
+
"company_employees",
|
|
583
|
+
"seniority",
|
|
584
|
+
"title",
|
|
585
|
+
]);
|
|
586
|
+
// The outbound reports only the facts that actually CHANGED on this contact.
|
|
587
|
+
assert.deepEqual(h.emits[0]?.traitKeys, ["company_employees", "seniority"]);
|
|
588
|
+
});
|
|
589
|
+
|
|
526
590
|
// ---------------------------------------------------------------------------
|
|
527
591
|
// D4b — a failed ingest is a verdict, not an exception
|
|
528
592
|
// ---------------------------------------------------------------------------
|
|
@@ -543,7 +607,7 @@ test("D4b: a failed ingest does not throw out of the chain and does not report s
|
|
|
543
607
|
});
|
|
544
608
|
|
|
545
609
|
test("D4b: the retry after a failed ingest re-lands the stored patch for FREE", async () => {
|
|
546
|
-
const stored = {
|
|
610
|
+
const stored = { title: "CTO" };
|
|
547
611
|
const h = harness({ ledgerRow: unexpired("found", stored) });
|
|
548
612
|
|
|
549
613
|
const result = await runRefineChain(h.deps, SUBJECT);
|
|
@@ -558,16 +622,11 @@ test("D4b: the retry after a failed ingest re-lands the stored patch for FREE",
|
|
|
558
622
|
// ---------------------------------------------------------------------------
|
|
559
623
|
|
|
560
624
|
/**
|
|
561
|
-
* The sorted trait key names FOUND flattens to — NAMES, never values,
|
|
562
|
-
*
|
|
563
|
-
*
|
|
625
|
+
* The sorted canonical trait key names FOUND flattens to — NAMES, never values,
|
|
626
|
+
* and FACTS only: the nested `enrichment` provenance is already carried as the
|
|
627
|
+
* envelope's `at` / `provider`, so it is not a trait.
|
|
564
628
|
*/
|
|
565
|
-
const FOUND_TRAIT_KEYS = [
|
|
566
|
-
"refined_company_employees",
|
|
567
|
-
"refined_company_name",
|
|
568
|
-
"refined_seniority",
|
|
569
|
-
"refined_title",
|
|
570
|
-
];
|
|
629
|
+
const FOUND_TRAIT_KEYS = ["company", "company_employees", "seniority", "title"];
|
|
571
630
|
|
|
572
631
|
/** NOW as the dedupe key serializes it. */
|
|
573
632
|
const NOW_ISO = NOW.toISOString();
|
|
@@ -595,14 +654,13 @@ test("AC 1: a `refined` verdict emits contact.refined with the contact key, prov
|
|
|
595
654
|
assert.ok(!JSON.stringify(emitted.traitKeys).includes("CTO"));
|
|
596
655
|
assert.ok(!JSON.stringify(emitted.traitKeys).includes("Acme"));
|
|
597
656
|
// Provenance is NOT a trait — it rides as `provider` / `refinedAt` instead.
|
|
598
|
-
assert.ok(!emitted.traitKeys.includes("
|
|
599
|
-
assert.ok(!emitted.traitKeys.includes("refined_provider"));
|
|
657
|
+
assert.ok(!emitted.traitKeys.includes("enrichment"));
|
|
600
658
|
});
|
|
601
659
|
|
|
602
|
-
test("AC 1: a found-but-empty vendor answer reports NO traits rather than
|
|
603
|
-
// `flattenTraits` stamps
|
|
604
|
-
//
|
|
605
|
-
//
|
|
660
|
+
test("AC 1: a found-but-empty vendor answer reports NO traits rather than a metadata key", async () => {
|
|
661
|
+
// `flattenTraits` stamps the `enrichment` provenance unconditionally, so if it
|
|
662
|
+
// counted as a trait this payload would claim a change for an answer that told
|
|
663
|
+
// us nothing about the person.
|
|
606
664
|
const empty: EnrichmentProvider = {
|
|
607
665
|
meta: { id: "fake", name: "Fake" },
|
|
608
666
|
capabilities: { personLookup: true, companyLookup: true },
|
|
@@ -627,7 +685,7 @@ test("AC 1: a domain-keyed refinement emits with the domain lookup in its dedupe
|
|
|
627
685
|
|
|
628
686
|
test("AC 2: `cached`, `not_found` and `skipped` emit NOTHING — a cache hit is not an event", async () => {
|
|
629
687
|
const cached = harness({
|
|
630
|
-
ledgerRow: unexpired("found", {
|
|
688
|
+
ledgerRow: unexpired("found", { title: "CTO" }),
|
|
631
689
|
});
|
|
632
690
|
const notFoundCached = harness({ ledgerRow: unexpired("not_found") });
|
|
633
691
|
const notFoundLive = harness({ provider: fakeProvider("miss").provider });
|
|
@@ -728,7 +786,7 @@ test("AC 3: a re-drive never reaches the emit anyway — the LEDGER GATE returns
|
|
|
728
786
|
// wrote and returns `cached` (refine-chain.ts step 3), which has no emit at
|
|
729
787
|
// all — so a duplicate delivery needs a concurrent race, not a retry.
|
|
730
788
|
const redrive = harness({
|
|
731
|
-
ledgerRow: unexpired("found", {
|
|
789
|
+
ledgerRow: unexpired("found", { title: "CTO" }),
|
|
732
790
|
});
|
|
733
791
|
|
|
734
792
|
assert.equal((await runRefineChain(redrive.deps, SUBJECT)).status, "cached");
|
package/src/lib/refine-chain.ts
CHANGED
|
@@ -12,7 +12,11 @@ import type {
|
|
|
12
12
|
EnrichmentLookupKind,
|
|
13
13
|
EnrichmentLookupStatus,
|
|
14
14
|
} from "./enrichment-ledger.js";
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
canonicalizeStoredTraits,
|
|
17
|
+
ENRICHMENT_KEY,
|
|
18
|
+
flattenTraits,
|
|
19
|
+
} from "./refine-traits.js";
|
|
16
20
|
|
|
17
21
|
// ---------------------------------------------------------------------------
|
|
18
22
|
// Public surface
|
|
@@ -42,7 +46,12 @@ export interface RefineContactResult {
|
|
|
42
46
|
* | `provider_error` | `ingest_failed`.
|
|
43
47
|
*/
|
|
44
48
|
reason?: string;
|
|
45
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* The vendor's FULL canonical trait patch (facts + the nested `enrichment`
|
|
51
|
+
* provenance) — on `refined` AND on `cached`. This is what the vendor
|
|
52
|
+
* answered, NOT the fill-if-absent subset that actually landed on any one
|
|
53
|
+
* contact.
|
|
54
|
+
*/
|
|
46
55
|
properties?: Record<string, unknown>;
|
|
47
56
|
}
|
|
48
57
|
|
|
@@ -58,8 +67,13 @@ export interface RefineTarget {
|
|
|
58
67
|
firstName?: string;
|
|
59
68
|
lastName?: string;
|
|
60
69
|
company?: string;
|
|
61
|
-
/**
|
|
62
|
-
|
|
70
|
+
/**
|
|
71
|
+
* The properties already on THIS contact's row, used by fill-if-absent to
|
|
72
|
+
* decide which vendor facts to keep (a fact the contact already holds from a
|
|
73
|
+
* first-party source is never overwritten). Absent for a brand-new email with
|
|
74
|
+
* no row yet — nothing to preserve, so every vendor fact lands.
|
|
75
|
+
*/
|
|
76
|
+
existingProperties?: Record<string, unknown>;
|
|
63
77
|
}
|
|
64
78
|
|
|
65
79
|
/** The event the refinement pipeline emits through `ingestEvent`. */
|
|
@@ -71,6 +85,12 @@ export interface RefineIngestInput {
|
|
|
71
85
|
contactId?: string;
|
|
72
86
|
eventProperties: Record<string, unknown>;
|
|
73
87
|
contactProperties: Record<string, unknown>;
|
|
88
|
+
/**
|
|
89
|
+
* Fact key NAMES to re-evaluate buckets for even when their value is not in
|
|
90
|
+
* `contactProperties` (fill-if-absent dropped it). Candidate-narrowing only —
|
|
91
|
+
* never written, never in value eval. See {@link reconcileTraits}.
|
|
92
|
+
*/
|
|
93
|
+
touchProperties?: string[];
|
|
74
94
|
idempotencyKey?: string;
|
|
75
95
|
}
|
|
76
96
|
|
|
@@ -88,9 +108,10 @@ export interface RefineEmitInput {
|
|
|
88
108
|
contactId?: string;
|
|
89
109
|
/** The enrichment provider id that answered. */
|
|
90
110
|
provider: string;
|
|
91
|
-
/** Sorted
|
|
111
|
+
/** Sorted canonical FACT keys actually filled on this contact (never the
|
|
112
|
+
* nested `enrichment` provenance key). */
|
|
92
113
|
traitKeys: string[];
|
|
93
|
-
/** The lookup instant — the same value the patch's `
|
|
114
|
+
/** The lookup instant — the same value the patch's `enrichment.at` carries. */
|
|
94
115
|
refinedAt: Date;
|
|
95
116
|
/** Per-endpoint delivery dedupe — see {@link refineOutboundDedupeKey}. */
|
|
96
117
|
dedupeKey: string;
|
|
@@ -100,7 +121,11 @@ export interface RefineEmitInput {
|
|
|
100
121
|
export interface RefineLedgerRow {
|
|
101
122
|
status: EnrichmentLookupStatus;
|
|
102
123
|
expiresAt: Date;
|
|
103
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* The normalized canonical trait patch the paid answer produced, if stored —
|
|
126
|
+
* the vendor's FULL answer, so a cache hit can land it (fill-if-absent) on a
|
|
127
|
+
* DIFFERENT contact that shares the lookup key.
|
|
128
|
+
*/
|
|
104
129
|
traits?: Record<string, unknown> | null;
|
|
105
130
|
}
|
|
106
131
|
|
|
@@ -403,10 +428,21 @@ async function runGates(args: {
|
|
|
403
428
|
spendWindow: deps.budgetWindowStart(refinedAt),
|
|
404
429
|
});
|
|
405
430
|
|
|
431
|
+
// Fill-if-absent: the vendor's facts fill only the canonical fields THIS
|
|
432
|
+
// contact does not already hold from a first-party source — a paid lookup
|
|
433
|
+
// never clobbers truer data the app already set — while every returned fact
|
|
434
|
+
// stays in the patch so the fit bucket re-evaluates. Computed here, per
|
|
435
|
+
// contact, against the row `resolveTarget` read.
|
|
436
|
+
const { landing, filled, touch } = reconcileTraits(
|
|
437
|
+
properties,
|
|
438
|
+
target.existingProperties,
|
|
439
|
+
);
|
|
440
|
+
|
|
406
441
|
const landed = await landTraits({
|
|
407
442
|
deps,
|
|
408
443
|
target,
|
|
409
|
-
properties,
|
|
444
|
+
properties: landing,
|
|
445
|
+
touch,
|
|
410
446
|
providerId,
|
|
411
447
|
found: result.found,
|
|
412
448
|
cached: false,
|
|
@@ -432,16 +468,14 @@ async function runGates(args: {
|
|
|
432
468
|
...(target.email ? { email: target.email } : {}),
|
|
433
469
|
...(target.contactId ? { contactId: target.contactId } : {}),
|
|
434
470
|
provider: providerId,
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
//
|
|
442
|
-
traitKeys:
|
|
443
|
-
.filter((key) => !REFINED_META_KEYS.includes(key))
|
|
444
|
-
.sort(),
|
|
471
|
+
// The FACTS that actually landed on this contact — the fill-if-absent set
|
|
472
|
+
// minus the nested `enrichment` provenance key. Two deliberate consequences:
|
|
473
|
+
// the envelope already carries the provenance as top-level `at` / `provider`
|
|
474
|
+
// so keeping the `enrichment` key here would say it twice; and a fact the
|
|
475
|
+
// contact already held (not overwritten) is not reported as changed, so a
|
|
476
|
+
// subscriber hears only what genuinely changed — `traits: []` when the
|
|
477
|
+
// answer added nothing new.
|
|
478
|
+
traitKeys: [...filled].sort(),
|
|
445
479
|
refinedAt,
|
|
446
480
|
dedupeKey: refineOutboundDedupeKey({
|
|
447
481
|
provider: providerId,
|
|
@@ -470,8 +504,10 @@ async function runGates(args: {
|
|
|
470
504
|
* membership was never evaluated — which is precisely the state a failed ingest
|
|
471
505
|
* leaves behind. That makes the retry after an `ingest_failed` free.
|
|
472
506
|
*
|
|
473
|
-
*
|
|
474
|
-
*
|
|
507
|
+
* The stored patch is the vendor's FULL answer, so fill-if-absent runs here too
|
|
508
|
+
* — the second contact at a company keeps its own first-party fields and only
|
|
509
|
+
* gains what it was missing. Rows written before the `traits` column existed
|
|
510
|
+
* have nothing stored and simply return `cached` with an empty patch.
|
|
475
511
|
*/
|
|
476
512
|
async function landCachedTraits(args: {
|
|
477
513
|
deps: RefineChainDeps;
|
|
@@ -481,15 +517,19 @@ async function landCachedTraits(args: {
|
|
|
481
517
|
idempotencyKey?: string;
|
|
482
518
|
}): Promise<RefineContactResult> {
|
|
483
519
|
const { deps, target, row } = args;
|
|
484
|
-
|
|
520
|
+
// A row written by the pre-canonical release stores flat `refined_*` traits;
|
|
521
|
+
// translate them so a cache hit lands the canonical shape, not the old one.
|
|
522
|
+
const stored = row.traits ? canonicalizeStoredTraits(row.traits) : undefined;
|
|
485
523
|
if (!stored) {
|
|
486
|
-
return { status: "cached", properties:
|
|
524
|
+
return { status: "cached", properties: {} };
|
|
487
525
|
}
|
|
488
526
|
|
|
527
|
+
const { landing, touch } = reconcileTraits(stored, target.existingProperties);
|
|
489
528
|
const landed = await landTraits({
|
|
490
529
|
deps,
|
|
491
530
|
target,
|
|
492
|
-
properties:
|
|
531
|
+
properties: landing,
|
|
532
|
+
touch,
|
|
493
533
|
providerId: args.providerId,
|
|
494
534
|
found: true,
|
|
495
535
|
cached: true,
|
|
@@ -517,6 +557,7 @@ async function landTraits(args: {
|
|
|
517
557
|
deps: RefineChainDeps;
|
|
518
558
|
target: RefineTarget;
|
|
519
559
|
properties: Record<string, unknown>;
|
|
560
|
+
touch: string[];
|
|
520
561
|
providerId: string;
|
|
521
562
|
found: boolean;
|
|
522
563
|
cached: boolean;
|
|
@@ -534,6 +575,7 @@ async function landTraits(args: {
|
|
|
534
575
|
cached: args.cached,
|
|
535
576
|
},
|
|
536
577
|
contactProperties: args.properties,
|
|
578
|
+
...(args.touch.length > 0 ? { touchProperties: args.touch } : {}),
|
|
537
579
|
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
538
580
|
});
|
|
539
581
|
return true;
|
|
@@ -547,6 +589,65 @@ async function landTraits(args: {
|
|
|
547
589
|
}
|
|
548
590
|
}
|
|
549
591
|
|
|
592
|
+
/**
|
|
593
|
+
* Reconcile the vendor's patch with what the contact already holds — fill-if-
|
|
594
|
+
* absent, split into three outputs because "what to WRITE" and "what to
|
|
595
|
+
* re-evaluate buckets for" are different questions.
|
|
596
|
+
*
|
|
597
|
+
* A paid answer must never overwrite a first-party value the app set. The
|
|
598
|
+
* ATOMIC way to guarantee that is to leave the already-held keys OUT of the
|
|
599
|
+
* write patch entirely: `mergePropertiesSql` is additive (`existing || patch`),
|
|
600
|
+
* so a key absent from the patch is left exactly as the live row has it — no
|
|
601
|
+
* read-modify-write, no snapshot to go stale, no chance of reverting a value a
|
|
602
|
+
* concurrent identify wrote during the (slow) vendor call. `existingProperties`
|
|
603
|
+
* is a pre-call snapshot, so this MUST NOT write a value back from it.
|
|
604
|
+
*
|
|
605
|
+
* But dropping a key hides it from `checkBucketMembership`, which narrows its
|
|
606
|
+
* candidate buckets to the property NAMES present in the ingest — so a dropped
|
|
607
|
+
* fact would never re-evaluate its fit bucket, and a torn write (properties
|
|
608
|
+
* persisted, membership not yet formed) would never reconcile. So the dropped
|
|
609
|
+
* keys still travel, separately, as `touch`: names only, no values, fed to the
|
|
610
|
+
* bucket candidate-narrowing (never to value eval, which reads the live row).
|
|
611
|
+
*
|
|
612
|
+
* Returns:
|
|
613
|
+
* - `landing`: the write patch — the `enrichment` provenance (always) plus only
|
|
614
|
+
* the facts the contact was MISSING.
|
|
615
|
+
* - `filled`: the fact keys actually taken from the vendor (drives the outbound
|
|
616
|
+
* `traitKeys` — a subscriber hears only what genuinely changed).
|
|
617
|
+
* - `touch`: EVERY fact key the vendor returned, so its bucket re-evaluates
|
|
618
|
+
* even when the value was already present.
|
|
619
|
+
*/
|
|
620
|
+
function reconcileTraits(
|
|
621
|
+
patch: Record<string, unknown>,
|
|
622
|
+
existing: Record<string, unknown> | undefined,
|
|
623
|
+
): { landing: Record<string, unknown>; filled: string[]; touch: string[] } {
|
|
624
|
+
const landing: Record<string, unknown> = {};
|
|
625
|
+
const filled: string[] = [];
|
|
626
|
+
const touch: string[] = [];
|
|
627
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
628
|
+
if (key === ENRICHMENT_KEY) {
|
|
629
|
+
landing[key] = value;
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
touch.push(key);
|
|
633
|
+
if (hasValue(existing?.[key])) continue; // already held — preserve, atomically
|
|
634
|
+
landing[key] = value;
|
|
635
|
+
filled.push(key);
|
|
636
|
+
}
|
|
637
|
+
return { landing, filled, touch };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* A property is "already set" for fill-if-absent when it holds a real value: not
|
|
642
|
+
* undefined, not null, not an empty / whitespace-only string. A numeric 0 or a
|
|
643
|
+
* `false` COUNTS as set — it is a value the app chose, not an absence.
|
|
644
|
+
*/
|
|
645
|
+
function hasValue(value: unknown): boolean {
|
|
646
|
+
if (value === undefined || value === null) return false;
|
|
647
|
+
if (typeof value === "string") return value.trim() !== "";
|
|
648
|
+
return true;
|
|
649
|
+
}
|
|
650
|
+
|
|
550
651
|
/**
|
|
551
652
|
* The per-endpoint delivery dedupe key for `contact.refined`.
|
|
552
653
|
*
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
3
|
import type { EnrichmentResult } from "@hogsend/core";
|
|
4
|
-
import {
|
|
4
|
+
import { canonicalizeStoredTraits, flattenTraits } from "./refine-traits.js";
|
|
5
5
|
|
|
6
6
|
const REFINED_AT = new Date("2026-07-25T10:00:00.000Z");
|
|
7
7
|
const META = { provider: "fake", refinedAt: REFINED_AT };
|
|
8
|
+
const ENRICHMENT = { provider: "fake", at: "2026-07-25T10:00:00.000Z" };
|
|
8
9
|
|
|
9
|
-
test("flattenTraits maps every documented field to a
|
|
10
|
+
test("flattenTraits maps every documented field to a canonical key", () => {
|
|
10
11
|
const result: EnrichmentResult = {
|
|
11
12
|
found: true,
|
|
12
13
|
person: {
|
|
@@ -28,22 +29,52 @@ test("flattenTraits maps every documented field to a flat refined_* key", () =>
|
|
|
28
29
|
};
|
|
29
30
|
|
|
30
31
|
assert.deepEqual(flattenTraits(result, META), {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
refined_provider: "fake",
|
|
32
|
+
title: "VP of Engineering",
|
|
33
|
+
seniority: "vp",
|
|
34
|
+
department: "engineering",
|
|
35
|
+
linkedin_url: "https://linkedin.com/in/x",
|
|
36
|
+
country: "US",
|
|
37
|
+
company: "Acme",
|
|
38
|
+
company_domain: "acme.com",
|
|
39
|
+
company_industry: "software",
|
|
40
|
+
company_employees: 250,
|
|
41
|
+
company_revenue: 12_000_000,
|
|
42
|
+
company_country: "US",
|
|
43
|
+
enrichment: ENRICHMENT,
|
|
44
44
|
});
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
+
// The facts are FLAT top-level keys, never nested: `evaluatePropertyConditions`
|
|
48
|
+
// does `properties[condition.property]` with no dotted-path traversal, so a
|
|
49
|
+
// bucket must be able to author `b.prop("company_industry")` verbatim.
|
|
50
|
+
test("facts are flat top-level keys; only provenance nests under `enrichment`", () => {
|
|
51
|
+
const patch = flattenTraits(
|
|
52
|
+
{
|
|
53
|
+
found: true,
|
|
54
|
+
person: { seniority: "vp" },
|
|
55
|
+
company: { industry: "software", employeeCount: 250 },
|
|
56
|
+
raw: {},
|
|
57
|
+
},
|
|
58
|
+
META,
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
assert.equal(patch.seniority, "vp");
|
|
62
|
+
assert.equal(patch.company_industry, "software");
|
|
63
|
+
assert.equal(patch.company_employees, 250);
|
|
64
|
+
// No parallel `refined_*` vocabulary, no dotted keys.
|
|
65
|
+
for (const key of Object.keys(patch)) {
|
|
66
|
+
assert.ok(
|
|
67
|
+
!key.startsWith("refined_"),
|
|
68
|
+
`${key} must not be refined_-prefixed`,
|
|
69
|
+
);
|
|
70
|
+
assert.ok(
|
|
71
|
+
key === "enrichment" || !key.includes("."),
|
|
72
|
+
`${key} must be flat`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
assert.deepEqual(patch.enrichment, ENRICHMENT);
|
|
76
|
+
});
|
|
77
|
+
|
|
47
78
|
// AC 8 — the numeric traits must be REAL JSON numbers. `conditions/property.ts`
|
|
48
79
|
// does no coercion, so a `"250"` would silently never match a `gte(100)`.
|
|
49
80
|
test("AC 8: employee count is a real JSON number, and survives JSON round-trip", () => {
|
|
@@ -52,12 +83,12 @@ test("AC 8: employee count is a real JSON number, and survives JSON round-trip",
|
|
|
52
83
|
META,
|
|
53
84
|
);
|
|
54
85
|
|
|
55
|
-
assert.equal(typeof patch.
|
|
56
|
-
assert.equal(patch.
|
|
86
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
87
|
+
assert.equal(patch.company_employees, 250);
|
|
57
88
|
// jsonb storage is a JSON round-trip — the type must survive it.
|
|
58
89
|
const roundTripped = JSON.parse(JSON.stringify(patch));
|
|
59
|
-
assert.equal(typeof roundTripped.
|
|
60
|
-
assert.ok((roundTripped.
|
|
90
|
+
assert.equal(typeof roundTripped.company_employees, "number");
|
|
91
|
+
assert.ok((roundTripped.company_employees as number) >= 100);
|
|
61
92
|
});
|
|
62
93
|
|
|
63
94
|
test("AC 8: a numeric STRING from an untyped provider is coerced to a number", () => {
|
|
@@ -72,8 +103,8 @@ test("AC 8: a numeric STRING from an untyped provider is coerced to a number", (
|
|
|
72
103
|
META,
|
|
73
104
|
);
|
|
74
105
|
|
|
75
|
-
assert.equal(patch.
|
|
76
|
-
assert.equal(typeof patch.
|
|
106
|
+
assert.equal(patch.company_employees, 250);
|
|
107
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
77
108
|
});
|
|
78
109
|
|
|
79
110
|
// AC 9 — an absent field must be OMITTED, never written as null: the patch is
|
|
@@ -84,13 +115,9 @@ test("AC 9: absent fields are omitted from the patch entirely (never null)", ()
|
|
|
84
115
|
META,
|
|
85
116
|
);
|
|
86
117
|
|
|
87
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"refined_title",
|
|
91
|
-
]);
|
|
92
|
-
assert.equal("refined_seniority" in patch, false);
|
|
93
|
-
assert.equal("refined_company_name" in patch, false);
|
|
118
|
+
assert.deepEqual(Object.keys(patch).sort(), ["enrichment", "title"]);
|
|
119
|
+
assert.equal("seniority" in patch, false);
|
|
120
|
+
assert.equal("company" in patch, false);
|
|
94
121
|
for (const value of Object.values(patch)) {
|
|
95
122
|
assert.notEqual(value, null);
|
|
96
123
|
}
|
|
@@ -114,28 +141,34 @@ test("AC 9: null / empty-string / non-finite fields are omitted, not written", (
|
|
|
114
141
|
META,
|
|
115
142
|
);
|
|
116
143
|
|
|
117
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
118
|
-
"refined_at",
|
|
119
|
-
"refined_department",
|
|
120
|
-
"refined_provider",
|
|
121
|
-
]);
|
|
144
|
+
assert.deepEqual(Object.keys(patch).sort(), ["department", "enrichment"]);
|
|
122
145
|
});
|
|
123
146
|
|
|
124
|
-
test("a not-found result still stamps only the
|
|
147
|
+
test("a not-found result still stamps only the provenance object", () => {
|
|
125
148
|
assert.deepEqual(flattenTraits({ found: false, raw: null }, META), {
|
|
126
|
-
|
|
127
|
-
refined_provider: "fake",
|
|
149
|
+
enrichment: ENRICHMENT,
|
|
128
150
|
});
|
|
129
151
|
});
|
|
130
152
|
|
|
131
|
-
test("
|
|
153
|
+
test("canonicalizeStoredTraits translates a pre-canonical (refined_*) ledger patch", () => {
|
|
132
154
|
assert.deepEqual(
|
|
133
|
-
|
|
134
|
-
plan: "pro",
|
|
155
|
+
canonicalizeStoredTraits({
|
|
135
156
|
refined_title: "CTO",
|
|
157
|
+
refined_company_name: "Acme",
|
|
136
158
|
refined_company_employees: 250,
|
|
159
|
+
refined_at: "2026-01-01T00:00:00.000Z",
|
|
160
|
+
refined_provider: "apollo",
|
|
137
161
|
}),
|
|
138
|
-
{
|
|
162
|
+
{ title: "CTO", company: "Acme", company_employees: 250 },
|
|
139
163
|
);
|
|
140
|
-
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("canonicalizeStoredTraits leaves an already-canonical patch untouched (by reference)", () => {
|
|
167
|
+
const canonical = {
|
|
168
|
+
title: "CTO",
|
|
169
|
+
company_employees: 250,
|
|
170
|
+
enrichment: ENRICHMENT,
|
|
171
|
+
};
|
|
172
|
+
// No legacy key → returns the SAME object, no needless copy.
|
|
173
|
+
assert.equal(canonicalizeStoredTraits(canonical), canonical);
|
|
141
174
|
});
|
package/src/lib/refine-traits.ts
CHANGED
|
@@ -1,27 +1,54 @@
|
|
|
1
1
|
import type { EnrichmentResult } from "@hogsend/core";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Identity of the lookup that produced a patch — the
|
|
5
|
-
* successful refinement stamps alongside the vendor's own
|
|
4
|
+
* Identity of the lookup that produced a patch — the provenance every
|
|
5
|
+
* successful refinement stamps alongside the vendor's own facts.
|
|
6
6
|
*/
|
|
7
7
|
export interface RefinedTraitsMeta {
|
|
8
|
-
/** The enrichment provider id that answered (`
|
|
8
|
+
/** The enrichment provider id that answered (`enrichment.provider`). */
|
|
9
9
|
provider: string;
|
|
10
|
-
/** The lookup instant, written as an ISO string (`
|
|
10
|
+
/** The lookup instant, written as an ISO string (`enrichment.at`). */
|
|
11
11
|
refinedAt: Date;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* `contacts.properties`.
|
|
15
|
+
* The single nested key that carries refinement PROVENANCE — who answered and
|
|
16
|
+
* when. Split out from the vendor FACTS on purpose:
|
|
18
17
|
*
|
|
19
|
-
*
|
|
18
|
+
* - Facts land on the CANONICAL, flat property keys a contact already carries
|
|
19
|
+
* (`title`, `company`, `company_employees`, …) so a bucket segments on the
|
|
20
|
+
* field it already knows — `b.prop("company_employees").gte(100)` — instead
|
|
21
|
+
* of a parallel `refined_*` vocabulary sitting next to the real one.
|
|
22
|
+
* - Provenance is NEVER read by `evaluatePropertyConditions` (segmentation is
|
|
23
|
+
* about facts, not who told us), so it nests freely under one key with no
|
|
24
|
+
* dotted-path support required from the condition engine. That is the whole
|
|
25
|
+
* reason the facts must stay flat and the provenance is free to be an object.
|
|
26
|
+
*/
|
|
27
|
+
export const ENRICHMENT_KEY = "enrichment";
|
|
28
|
+
|
|
29
|
+
/** The provenance object stored under {@link ENRICHMENT_KEY}. */
|
|
30
|
+
export interface EnrichmentProvenance {
|
|
31
|
+
/** The enrichment provider id that answered. */
|
|
32
|
+
provider: string;
|
|
33
|
+
/** ISO — the instant the vendor answer was obtained. */
|
|
34
|
+
at: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Map a normalized {@link EnrichmentResult} onto a CANONICAL contact-property
|
|
39
|
+
* patch that `ingestEvent` merges into `contacts.properties`. The vendor facts
|
|
40
|
+
* fill the fields a contact already segments and personalizes on; a single
|
|
41
|
+
* nested `enrichment` object records the provenance.
|
|
42
|
+
*
|
|
43
|
+
* Three rules make this function load-bearing rather than cosmetic:
|
|
20
44
|
*
|
|
21
|
-
* - **
|
|
22
|
-
* `
|
|
23
|
-
*
|
|
24
|
-
*
|
|
45
|
+
* - **Facts are flat + canonical.** `evaluatePropertyConditions` reads
|
|
46
|
+
* `properties[condition.property]` — a flat lookup with no dotted-path
|
|
47
|
+
* traversal — so every fact key is a top-level string a bucket can author
|
|
48
|
+
* verbatim (`company_industry`, not `company.industry`).
|
|
49
|
+
* - **Provenance is one nested object.** The condition engine never reads it,
|
|
50
|
+
* so `enrichment: { provider, at }` replaces the old flat `refined_at` /
|
|
51
|
+
* `refined_provider` sibling keys without touching the evaluator.
|
|
25
52
|
* - **Absent means ABSENT.** An undefined vendor field is omitted from the
|
|
26
53
|
* patch entirely — never written as `null`. `mergePropertiesSql` wraps the
|
|
27
54
|
* patch in `jsonb_strip_nulls`, so a null would DELETE a previously-good key
|
|
@@ -29,6 +56,13 @@ export interface RefinedTraitsMeta {
|
|
|
29
56
|
*
|
|
30
57
|
* Numeric traits are written as real JSON numbers: `conditions/property.ts` does
|
|
31
58
|
* no coercion, so a `"42"` would silently never match a `gte`.
|
|
59
|
+
*
|
|
60
|
+
* The patch is the vendor's FULL answer, keyed canonically. Whether an
|
|
61
|
+
* individual fact actually overwrites what a contact already holds is decided at
|
|
62
|
+
* LANDING time (fill-if-absent, in `refine-chain.ts`), because that answer
|
|
63
|
+
* depends on the specific contact — the same stored patch lands on every contact
|
|
64
|
+
* that shares a company domain, and each may already carry different first-party
|
|
65
|
+
* fields.
|
|
32
66
|
*/
|
|
33
67
|
export function flattenTraits(
|
|
34
68
|
result: EnrichmentResult,
|
|
@@ -38,56 +72,77 @@ export function flattenTraits(
|
|
|
38
72
|
const person = result.person;
|
|
39
73
|
const company = result.company;
|
|
40
74
|
|
|
41
|
-
putString(patch, "
|
|
42
|
-
putString(patch, "
|
|
43
|
-
putString(patch, "
|
|
44
|
-
putString(patch, "
|
|
45
|
-
putString(patch, "
|
|
75
|
+
putString(patch, "title", person?.title);
|
|
76
|
+
putString(patch, "seniority", person?.seniority);
|
|
77
|
+
putString(patch, "department", person?.department);
|
|
78
|
+
putString(patch, "linkedin_url", person?.linkedinUrl);
|
|
79
|
+
putString(patch, "country", person?.country);
|
|
46
80
|
|
|
47
|
-
putString(patch, "
|
|
48
|
-
putString(patch, "
|
|
49
|
-
putString(patch, "
|
|
50
|
-
putNumber(patch, "
|
|
51
|
-
putNumber(patch, "
|
|
52
|
-
putString(patch, "
|
|
81
|
+
putString(patch, "company", company?.name);
|
|
82
|
+
putString(patch, "company_domain", company?.domain);
|
|
83
|
+
putString(patch, "company_industry", company?.industry);
|
|
84
|
+
putNumber(patch, "company_employees", company?.employeeCount);
|
|
85
|
+
putNumber(patch, "company_revenue", company?.estimatedRevenue);
|
|
86
|
+
putString(patch, "company_country", company?.country);
|
|
53
87
|
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
const provenance: EnrichmentProvenance = {
|
|
89
|
+
provider: meta.provider,
|
|
90
|
+
at: meta.refinedAt.toISOString(),
|
|
91
|
+
};
|
|
92
|
+
patch[ENRICHMENT_KEY] = provenance;
|
|
56
93
|
|
|
57
94
|
return patch;
|
|
58
95
|
}
|
|
59
96
|
|
|
60
|
-
/** The prefix every trait this module writes carries. */
|
|
61
|
-
export const REFINED_TRAIT_PREFIX = "refined_";
|
|
62
|
-
|
|
63
97
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
* The `refined_*` subset of an already-stored `contacts.properties` bag — what
|
|
76
|
-
* a cache hit compares the ledger row's stored patch against, to tell "this
|
|
77
|
-
* contact already has the paid answer" apart from "another contact at the same
|
|
78
|
-
* domain paid for it and this one has never seen it".
|
|
98
|
+
* Translate a ledger `traits` patch written by the PRE-canonical release (flat
|
|
99
|
+
* `refined_*` fact keys plus flat `refined_at` / `refined_provider` provenance)
|
|
100
|
+
* into the canonical shape {@link flattenTraits} now produces. A cache HIT can
|
|
101
|
+
* land a row written by an older engine, and landing it verbatim would
|
|
102
|
+
* re-introduce exactly the flat vocabulary this design removed.
|
|
103
|
+
*
|
|
104
|
+
* A patch with no `refined_*` key is already canonical and passes through
|
|
105
|
+
* untouched (the common case, once the ledger has turned over). The two legacy
|
|
106
|
+
* provenance keys are dropped: they are not facts, and the fresh ISO provenance
|
|
107
|
+
* is not recoverable from them here, so a legacy cache hit simply lands the
|
|
108
|
+
* facts without a new `enrichment` stamp.
|
|
79
109
|
*/
|
|
80
|
-
export function
|
|
81
|
-
|
|
110
|
+
export function canonicalizeStoredTraits(
|
|
111
|
+
traits: Record<string, unknown>,
|
|
82
112
|
): Record<string, unknown> {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
for (const [key, value] of Object.entries(
|
|
86
|
-
if (key
|
|
113
|
+
let sawLegacy = false;
|
|
114
|
+
const out: Record<string, unknown> = {};
|
|
115
|
+
for (const [key, value] of Object.entries(traits)) {
|
|
116
|
+
if (key === "refined_at" || key === "refined_provider") {
|
|
117
|
+
sawLegacy = true;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const canonical = LEGACY_TRAIT_KEYS[key];
|
|
121
|
+
if (canonical) {
|
|
122
|
+
sawLegacy = true;
|
|
123
|
+
out[canonical] = value;
|
|
124
|
+
} else {
|
|
125
|
+
out[key] = value;
|
|
126
|
+
}
|
|
87
127
|
}
|
|
88
|
-
return
|
|
128
|
+
return sawLegacy ? out : traits;
|
|
89
129
|
}
|
|
90
130
|
|
|
131
|
+
/** Pre-canonical `refined_*` fact key → its canonical replacement. */
|
|
132
|
+
const LEGACY_TRAIT_KEYS: Record<string, string> = {
|
|
133
|
+
refined_title: "title",
|
|
134
|
+
refined_seniority: "seniority",
|
|
135
|
+
refined_department: "department",
|
|
136
|
+
refined_linkedin_url: "linkedin_url",
|
|
137
|
+
refined_country: "country",
|
|
138
|
+
refined_company_name: "company",
|
|
139
|
+
refined_company_domain: "company_domain",
|
|
140
|
+
refined_company_industry: "company_industry",
|
|
141
|
+
refined_company_employees: "company_employees",
|
|
142
|
+
refined_company_revenue: "company_revenue",
|
|
143
|
+
refined_company_country: "company_country",
|
|
144
|
+
};
|
|
145
|
+
|
|
91
146
|
/**
|
|
92
147
|
* Write a string trait, omitting anything that carries no information. An empty
|
|
93
148
|
* (or whitespace-only) string is dropped for the same reason a null is: it is
|
|
@@ -108,8 +163,8 @@ function putString(
|
|
|
108
163
|
* Write a numeric trait as a real JSON number. A provider is contractually
|
|
109
164
|
* obliged to normalize to `number`, but a JS (untyped) provider can still hand
|
|
110
165
|
* back a numeric string — coercing here is the difference between a working
|
|
111
|
-
* `b.prop("
|
|
112
|
-
*
|
|
166
|
+
* `b.prop("company_employees").gte(100)` and one that silently never matches.
|
|
167
|
+
* Non-finite / non-numeric values are omitted entirely.
|
|
113
168
|
*/
|
|
114
169
|
function putNumber(
|
|
115
170
|
patch: Record<string, unknown>,
|
package/src/lib/refine.ts
CHANGED
|
@@ -25,7 +25,6 @@ import {
|
|
|
25
25
|
type RefineTarget,
|
|
26
26
|
runRefineChain,
|
|
27
27
|
} from "./refine-chain.js";
|
|
28
|
-
import { pickRefinedTraits } from "./refine-traits.js";
|
|
29
28
|
|
|
30
29
|
export type {
|
|
31
30
|
RefineContactOptions,
|
|
@@ -40,16 +39,17 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
|
40
39
|
|
|
41
40
|
/**
|
|
42
41
|
* Refinement — the one new public function of this release. Resolves a contact,
|
|
43
|
-
* spends AT MOST one provider lookup, and lands the vendor's answer
|
|
44
|
-
*
|
|
45
|
-
*
|
|
42
|
+
* spends AT MOST one provider lookup, and lands the vendor's answer on the
|
|
43
|
+
* CANONICAL contact-property fields (fill-if-absent, so first-party data is
|
|
44
|
+
* never overwritten) THROUGH `ingestEvent`, so bucket membership re-evaluates
|
|
45
|
+
* synchronously and the GTM loop closes:
|
|
46
46
|
*
|
|
47
47
|
* ```
|
|
48
48
|
* behaviour → bucket("gtm-high-intent").on("enter")
|
|
49
49
|
* → refineContact()
|
|
50
|
-
* → contacts.properties.
|
|
50
|
+
* → contacts.properties.{company_employees, seniority, …} (via ingestEvent)
|
|
51
51
|
* → checkBucketMembership re-runs
|
|
52
|
-
* → bucket("gtm-qualified", b => b.prop("
|
|
52
|
+
* → bucket("gtm-qualified", b => b.prop("company_employees").gte(100))
|
|
53
53
|
* ```
|
|
54
54
|
*
|
|
55
55
|
* A STANDALONE import, never a `ctx` method — the journey context stays
|
|
@@ -129,6 +129,9 @@ export async function refineContact(
|
|
|
129
129
|
...(input.contactId ? { contactId: input.contactId } : {}),
|
|
130
130
|
eventProperties: input.eventProperties,
|
|
131
131
|
contactProperties: input.contactProperties,
|
|
132
|
+
...(input.touchProperties
|
|
133
|
+
? { touchProperties: input.touchProperties }
|
|
134
|
+
: {}),
|
|
132
135
|
source: "enrichment",
|
|
133
136
|
...(input.idempotencyKey
|
|
134
137
|
? { idempotencyKey: input.idempotencyKey }
|
|
@@ -145,8 +148,8 @@ export async function refineContact(
|
|
|
145
148
|
* Resolve the refinement subject and everything the lookup needs from it: the
|
|
146
149
|
* canonical key the ingest writes back under, the lookup key (email first, then
|
|
147
150
|
* the contact's company domain), the vendor query's name/company hints, and the
|
|
148
|
-
*
|
|
149
|
-
* decide
|
|
151
|
+
* row's existing properties (which fill-if-absent reads at landing time to
|
|
152
|
+
* decide which vendor facts to keep).
|
|
150
153
|
*
|
|
151
154
|
* An email with no contact row is still refinable — the ingest at the end of the
|
|
152
155
|
* chain creates the contact — so a brand-new address is not a `no_lookup_key`.
|
|
@@ -171,24 +174,28 @@ async function resolveRefineTarget(
|
|
|
171
174
|
|
|
172
175
|
const properties = row.properties ?? {};
|
|
173
176
|
const resolvedEmail = row.email ?? email;
|
|
174
|
-
// REPLAY SAFETY:
|
|
175
|
-
// The domain becomes the `lookupKey`, which is the `enrichment_lookups`
|
|
177
|
+
// REPLAY SAFETY: the domain this picks must be STABLE across a run and its
|
|
178
|
+
// replay. The domain becomes the `lookupKey`, which is the `enrichment_lookups`
|
|
176
179
|
// arbiter — Layer 2, the version-independent exactly-once backstop. (Layer 1,
|
|
177
180
|
// the memo key, is derived from the caller's arguments alone and is immune.)
|
|
178
|
-
// A
|
|
179
|
-
// miss the row it just wrote, and charge the
|
|
181
|
+
// A source that refinement itself changes between the run and the replay would
|
|
182
|
+
// derive a DIFFERENT ledger key, miss the row it just wrote, and charge the
|
|
183
|
+
// vendor a second time.
|
|
180
184
|
//
|
|
181
|
-
// `
|
|
182
|
-
//
|
|
183
|
-
//
|
|
184
|
-
// `
|
|
185
|
-
//
|
|
186
|
-
//
|
|
185
|
+
// `company_domain` is the key `flattenTraits` WRITES (the vendor's canonical
|
|
186
|
+
// domain, which need not equal the one we looked up by), so it is listed LAST.
|
|
187
|
+
// That ordering — externally-supplied `companyDomain`/`domain` before the
|
|
188
|
+
// refinement-written `company_domain` — is what keeps the pick stable: the only
|
|
189
|
+
// way refinement can FILL `company_domain` is fill-if-absent (it was absent),
|
|
190
|
+
// and whenever it is the pick it was already present (so fill-if-absent leaves
|
|
191
|
+
// it untouched). A domain-only contact looked up by `domain` keeps resolving
|
|
192
|
+
// via `domain` even after the vendor fills `company_domain`, because `domain`
|
|
193
|
+
// outranks it.
|
|
187
194
|
const domain = readString(
|
|
188
195
|
properties,
|
|
189
|
-
"company_domain",
|
|
190
196
|
"companyDomain",
|
|
191
197
|
"domain",
|
|
198
|
+
"company_domain",
|
|
192
199
|
);
|
|
193
200
|
|
|
194
201
|
return {
|
|
@@ -202,15 +209,9 @@ async function resolveRefineTarget(
|
|
|
202
209
|
...spread("lastName", readString(properties, "lastName", "last_name")),
|
|
203
210
|
...spread(
|
|
204
211
|
"company",
|
|
205
|
-
readString(
|
|
206
|
-
properties,
|
|
207
|
-
"company",
|
|
208
|
-
"company_name",
|
|
209
|
-
"companyName",
|
|
210
|
-
"refined_company_name",
|
|
211
|
-
),
|
|
212
|
+
readString(properties, "company", "company_name", "companyName"),
|
|
212
213
|
),
|
|
213
|
-
|
|
214
|
+
existingProperties: properties,
|
|
214
215
|
};
|
|
215
216
|
}
|
|
216
217
|
|