@hogsend/engine 0.55.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +5 -10
- package/src/buckets/check-membership.ts +91 -6
- package/src/index.ts +31 -1
- package/src/journeys/execute-journey-run.ts +56 -1
- package/src/journeys/journey-context.ts +59 -0
- package/src/lib/bucket-emit.test.ts +83 -2
- package/src/lib/bucket-emit.ts +16 -0
- package/src/lib/contacts.ts +1031 -141
- package/src/lib/crm-ingest.ts +9 -0
- package/src/lib/enrichment-ledger.ts +1 -1
- package/src/lib/feed.ts +64 -3
- package/src/lib/identity-service.ts +15 -2
- package/src/lib/ingestion.ts +191 -14
- package/src/lib/outbound.ts +3 -3
- package/src/lib/refine-chain.test.ts +83 -25
- package/src/lib/refine-chain.ts +124 -23
- package/src/lib/refine-traits.test.ts +74 -41
- package/src/lib/refine-traits.ts +107 -52
- package/src/lib/refine.ts +28 -27
- package/src/lib/tracking-events.ts +33 -6
- package/src/routes/admin/agent.ts +26 -0
- package/src/routes/admin/contacts.ts +65 -5
- package/src/routes/admin/identity.ts +197 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/contacts/index.ts +50 -7
- package/src/routes/events/index.ts +94 -6
- package/src/routes/feed/index.ts +36 -0
- package/src/routes/feed/recipient.ts +44 -2
- package/src/routes/lists/index.ts +21 -0
- package/src/routes/tracking/arrive.ts +42 -3
- package/src/worker.ts +18 -0
- package/src/workflows/bucket-backfill.ts +1 -1
- package/src/workflows/bucket-reconcile.ts +129 -4
- package/src/workflows/identity-alias-backfill.ts +448 -0
- package/src/workflows/import-contacts.ts +10 -0
- package/src/workflows/send-campaign.ts +31 -13
|
@@ -93,6 +93,10 @@ interface Harness {
|
|
|
93
93
|
ingests: number;
|
|
94
94
|
/** Every outbound `contact.refined` emit, in order. */
|
|
95
95
|
emits: RefineEmitInput[];
|
|
96
|
+
/** The `contactProperties` each `ingest` received, in order. */
|
|
97
|
+
ingested: Record<string, unknown>[];
|
|
98
|
+
/** The `touchProperties` each `ingest` received, in order. */
|
|
99
|
+
touched: (string[] | undefined)[];
|
|
96
100
|
}
|
|
97
101
|
|
|
98
102
|
function harness(overrides: {
|
|
@@ -116,6 +120,8 @@ function harness(overrides: {
|
|
|
116
120
|
ledgerWrites: 0,
|
|
117
121
|
ingests: 0,
|
|
118
122
|
emits: [] as RefineEmitInput[],
|
|
123
|
+
ingested: [] as Record<string, unknown>[],
|
|
124
|
+
touched: [] as (string[] | undefined)[],
|
|
119
125
|
};
|
|
120
126
|
|
|
121
127
|
const deps: RefineChainDeps = {
|
|
@@ -135,15 +141,16 @@ function harness(overrides: {
|
|
|
135
141
|
contactId: "c1",
|
|
136
142
|
userId: "u1",
|
|
137
143
|
email: "a@acme.com",
|
|
138
|
-
refinedProperties: { refined_title: "old" },
|
|
139
144
|
},
|
|
140
145
|
findLedgerRow: async () => overrides.ledgerRow ?? null,
|
|
141
146
|
countLookupsSince: async () => overrides.usedThisMonth ?? 0,
|
|
142
147
|
writeLedgerRow: async () => {
|
|
143
148
|
state.ledgerWrites += 1;
|
|
144
149
|
},
|
|
145
|
-
ingest: async () => {
|
|
150
|
+
ingest: async (input) => {
|
|
146
151
|
state.ingests += 1;
|
|
152
|
+
state.ingested.push(input.contactProperties);
|
|
153
|
+
state.touched.push(input.touchProperties);
|
|
147
154
|
if (overrides.ingest) await overrides.ingest();
|
|
148
155
|
},
|
|
149
156
|
emitRefined: (input) => {
|
|
@@ -163,6 +170,12 @@ function harness(overrides: {
|
|
|
163
170
|
get emits() {
|
|
164
171
|
return state.emits;
|
|
165
172
|
},
|
|
173
|
+
get ingested() {
|
|
174
|
+
return state.ingested;
|
|
175
|
+
},
|
|
176
|
+
get touched() {
|
|
177
|
+
return state.touched;
|
|
178
|
+
},
|
|
166
179
|
};
|
|
167
180
|
}
|
|
168
181
|
|
|
@@ -470,7 +483,7 @@ test("D2: a cache hit lands the STORED patch on a contact that never received it
|
|
|
470
483
|
// The shared-domain case: contact B hits the row contact A paid for. The
|
|
471
484
|
// ledger has no contact dimension, so "already paid for" says nothing about
|
|
472
485
|
// whether THIS contact has the traits.
|
|
473
|
-
const stored = {
|
|
486
|
+
const stored = { title: "CTO", company_employees: 250 };
|
|
474
487
|
const h = harness({
|
|
475
488
|
ledgerRow: unexpired("found", stored),
|
|
476
489
|
target: { contactId: "c2", userId: "u2", domain: "acme.com" },
|
|
@@ -491,14 +504,14 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
|
|
|
491
504
|
// buckets: only the ingest path re-runs `checkBucketMembership`, and a
|
|
492
505
|
// contact can carry the traits with no membership (exactly what a failed
|
|
493
506
|
// ingest leaves behind). So the cached verdict always re-lands.
|
|
494
|
-
const stored = {
|
|
507
|
+
const stored = { title: "CTO", company_employees: 250 };
|
|
495
508
|
const h = harness({
|
|
496
509
|
ledgerRow: unexpired("found", stored),
|
|
497
510
|
target: {
|
|
498
511
|
contactId: "c1",
|
|
499
512
|
userId: "u1",
|
|
500
513
|
domain: "acme.com",
|
|
501
|
-
|
|
514
|
+
existingProperties: { ...stored },
|
|
502
515
|
},
|
|
503
516
|
});
|
|
504
517
|
|
|
@@ -506,23 +519,74 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
|
|
|
506
519
|
|
|
507
520
|
assert.deepEqual(result, { status: "cached", properties: stored });
|
|
508
521
|
assert.equal(h.ingests, 1);
|
|
522
|
+
// The contact already held every fact, so the WRITE patch is empty (nothing
|
|
523
|
+
// first-party is clobbered) — but the ingest still runs and TOUCHES both keys,
|
|
524
|
+
// so bucket membership re-evaluates against live contact state.
|
|
525
|
+
assert.deepEqual(h.ingested, [{}]);
|
|
526
|
+
assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
|
|
527
|
+
"company_employees",
|
|
528
|
+
"title",
|
|
529
|
+
]);
|
|
509
530
|
assert.equal(h.providerCalls(), 0);
|
|
510
531
|
assert.equal(h.ledgerWrites, 0);
|
|
511
532
|
});
|
|
512
533
|
|
|
513
|
-
test("D2: a legacy row with no stored patch
|
|
534
|
+
test("D2: a legacy row with no stored patch returns cached with an empty patch", async () => {
|
|
514
535
|
const h = harness({ ledgerRow: unexpired("found", null) });
|
|
515
536
|
|
|
516
537
|
const result = await runRefineChain(h.deps, SUBJECT);
|
|
517
538
|
|
|
518
539
|
assert.deepEqual(result, {
|
|
519
540
|
status: "cached",
|
|
520
|
-
properties: {
|
|
541
|
+
properties: {},
|
|
521
542
|
});
|
|
522
543
|
assert.equal(h.ingests, 0);
|
|
523
544
|
assert.equal(h.providerCalls(), 0);
|
|
524
545
|
});
|
|
525
546
|
|
|
547
|
+
// ---------------------------------------------------------------------------
|
|
548
|
+
// Fill-if-absent — a paid answer fills gaps, never overwrites first-party data
|
|
549
|
+
// ---------------------------------------------------------------------------
|
|
550
|
+
|
|
551
|
+
test("fill-if-absent: a fresh lookup fills gaps but never overwrites a first-party value", async () => {
|
|
552
|
+
// FOUND carries title / seniority / company / company_employees. This contact
|
|
553
|
+
// already has a first-party `title` and `company`, so the paid answer must
|
|
554
|
+
// leave those two alone and only fill what was missing.
|
|
555
|
+
const h = harness({
|
|
556
|
+
target: {
|
|
557
|
+
contactId: "c1",
|
|
558
|
+
userId: "u1",
|
|
559
|
+
email: "a@acme.com",
|
|
560
|
+
existingProperties: { title: "First-party VP", company: "First Co" },
|
|
561
|
+
},
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
const result = await runRefineChain(h.deps, SUBJECT);
|
|
565
|
+
|
|
566
|
+
assert.equal(result.status, "refined");
|
|
567
|
+
// The vendor's FULL answer is still reported on the result...
|
|
568
|
+
assert.equal(result.properties?.title, "CTO");
|
|
569
|
+
// ...but the WRITE patch OMITS the already-held title + company entirely (so
|
|
570
|
+
// the additive merge leaves the live first-party value untouched — atomic, no
|
|
571
|
+
// stale write-back) and carries the vendor's value only for the gaps.
|
|
572
|
+
const landed = h.ingested[0] ?? {};
|
|
573
|
+
assert.equal("title" in landed, false);
|
|
574
|
+
assert.equal("company" in landed, false);
|
|
575
|
+
assert.equal(landed.seniority, "c_suite");
|
|
576
|
+
assert.equal(landed.company_employees, 250);
|
|
577
|
+
assert.ok(landed.enrichment, "provenance always lands");
|
|
578
|
+
// ...yet every returned fact is TOUCHED, so a fit bucket on the already-held
|
|
579
|
+
// title/company still re-evaluates against live contact state.
|
|
580
|
+
assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
|
|
581
|
+
"company",
|
|
582
|
+
"company_employees",
|
|
583
|
+
"seniority",
|
|
584
|
+
"title",
|
|
585
|
+
]);
|
|
586
|
+
// The outbound reports only the facts that actually CHANGED on this contact.
|
|
587
|
+
assert.deepEqual(h.emits[0]?.traitKeys, ["company_employees", "seniority"]);
|
|
588
|
+
});
|
|
589
|
+
|
|
526
590
|
// ---------------------------------------------------------------------------
|
|
527
591
|
// D4b — a failed ingest is a verdict, not an exception
|
|
528
592
|
// ---------------------------------------------------------------------------
|
|
@@ -543,7 +607,7 @@ test("D4b: a failed ingest does not throw out of the chain and does not report s
|
|
|
543
607
|
});
|
|
544
608
|
|
|
545
609
|
test("D4b: the retry after a failed ingest re-lands the stored patch for FREE", async () => {
|
|
546
|
-
const stored = {
|
|
610
|
+
const stored = { title: "CTO" };
|
|
547
611
|
const h = harness({ ledgerRow: unexpired("found", stored) });
|
|
548
612
|
|
|
549
613
|
const result = await runRefineChain(h.deps, SUBJECT);
|
|
@@ -558,16 +622,11 @@ test("D4b: the retry after a failed ingest re-lands the stored patch for FREE",
|
|
|
558
622
|
// ---------------------------------------------------------------------------
|
|
559
623
|
|
|
560
624
|
/**
|
|
561
|
-
* The sorted trait key names FOUND flattens to — NAMES, never values,
|
|
562
|
-
*
|
|
563
|
-
*
|
|
625
|
+
* The sorted canonical trait key names FOUND flattens to — NAMES, never values,
|
|
626
|
+
* and FACTS only: the nested `enrichment` provenance is already carried as the
|
|
627
|
+
* envelope's `at` / `provider`, so it is not a trait.
|
|
564
628
|
*/
|
|
565
|
-
const FOUND_TRAIT_KEYS = [
|
|
566
|
-
"refined_company_employees",
|
|
567
|
-
"refined_company_name",
|
|
568
|
-
"refined_seniority",
|
|
569
|
-
"refined_title",
|
|
570
|
-
];
|
|
629
|
+
const FOUND_TRAIT_KEYS = ["company", "company_employees", "seniority", "title"];
|
|
571
630
|
|
|
572
631
|
/** NOW as the dedupe key serializes it. */
|
|
573
632
|
const NOW_ISO = NOW.toISOString();
|
|
@@ -595,14 +654,13 @@ test("AC 1: a `refined` verdict emits contact.refined with the contact key, prov
|
|
|
595
654
|
assert.ok(!JSON.stringify(emitted.traitKeys).includes("CTO"));
|
|
596
655
|
assert.ok(!JSON.stringify(emitted.traitKeys).includes("Acme"));
|
|
597
656
|
// Provenance is NOT a trait — it rides as `provider` / `refinedAt` instead.
|
|
598
|
-
assert.ok(!emitted.traitKeys.includes("
|
|
599
|
-
assert.ok(!emitted.traitKeys.includes("refined_provider"));
|
|
657
|
+
assert.ok(!emitted.traitKeys.includes("enrichment"));
|
|
600
658
|
});
|
|
601
659
|
|
|
602
|
-
test("AC 1: a found-but-empty vendor answer reports NO traits rather than
|
|
603
|
-
// `flattenTraits` stamps
|
|
604
|
-
//
|
|
605
|
-
//
|
|
660
|
+
test("AC 1: a found-but-empty vendor answer reports NO traits rather than a metadata key", async () => {
|
|
661
|
+
// `flattenTraits` stamps the `enrichment` provenance unconditionally, so if it
|
|
662
|
+
// counted as a trait this payload would claim a change for an answer that told
|
|
663
|
+
// us nothing about the person.
|
|
606
664
|
const empty: EnrichmentProvider = {
|
|
607
665
|
meta: { id: "fake", name: "Fake" },
|
|
608
666
|
capabilities: { personLookup: true, companyLookup: true },
|
|
@@ -627,7 +685,7 @@ test("AC 1: a domain-keyed refinement emits with the domain lookup in its dedupe
|
|
|
627
685
|
|
|
628
686
|
test("AC 2: `cached`, `not_found` and `skipped` emit NOTHING — a cache hit is not an event", async () => {
|
|
629
687
|
const cached = harness({
|
|
630
|
-
ledgerRow: unexpired("found", {
|
|
688
|
+
ledgerRow: unexpired("found", { title: "CTO" }),
|
|
631
689
|
});
|
|
632
690
|
const notFoundCached = harness({ ledgerRow: unexpired("not_found") });
|
|
633
691
|
const notFoundLive = harness({ provider: fakeProvider("miss").provider });
|
|
@@ -728,7 +786,7 @@ test("AC 3: a re-drive never reaches the emit anyway — the LEDGER GATE returns
|
|
|
728
786
|
// wrote and returns `cached` (refine-chain.ts step 3), which has no emit at
|
|
729
787
|
// all — so a duplicate delivery needs a concurrent race, not a retry.
|
|
730
788
|
const redrive = harness({
|
|
731
|
-
ledgerRow: unexpired("found", {
|
|
789
|
+
ledgerRow: unexpired("found", { title: "CTO" }),
|
|
732
790
|
});
|
|
733
791
|
|
|
734
792
|
assert.equal((await runRefineChain(redrive.deps, SUBJECT)).status, "cached");
|
package/src/lib/refine-chain.ts
CHANGED
|
@@ -12,7 +12,11 @@ import type {
|
|
|
12
12
|
EnrichmentLookupKind,
|
|
13
13
|
EnrichmentLookupStatus,
|
|
14
14
|
} from "./enrichment-ledger.js";
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
canonicalizeStoredTraits,
|
|
17
|
+
ENRICHMENT_KEY,
|
|
18
|
+
flattenTraits,
|
|
19
|
+
} from "./refine-traits.js";
|
|
16
20
|
|
|
17
21
|
// ---------------------------------------------------------------------------
|
|
18
22
|
// Public surface
|
|
@@ -42,7 +46,12 @@ export interface RefineContactResult {
|
|
|
42
46
|
* | `provider_error` | `ingest_failed`.
|
|
43
47
|
*/
|
|
44
48
|
reason?: string;
|
|
45
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* The vendor's FULL canonical trait patch (facts + the nested `enrichment`
|
|
51
|
+
* provenance) — on `refined` AND on `cached`. This is what the vendor
|
|
52
|
+
* answered, NOT the fill-if-absent subset that actually landed on any one
|
|
53
|
+
* contact.
|
|
54
|
+
*/
|
|
46
55
|
properties?: Record<string, unknown>;
|
|
47
56
|
}
|
|
48
57
|
|
|
@@ -58,8 +67,13 @@ export interface RefineTarget {
|
|
|
58
67
|
firstName?: string;
|
|
59
68
|
lastName?: string;
|
|
60
69
|
company?: string;
|
|
61
|
-
/**
|
|
62
|
-
|
|
70
|
+
/**
|
|
71
|
+
* The properties already on THIS contact's row, used by fill-if-absent to
|
|
72
|
+
* decide which vendor facts to keep (a fact the contact already holds from a
|
|
73
|
+
* first-party source is never overwritten). Absent for a brand-new email with
|
|
74
|
+
* no row yet — nothing to preserve, so every vendor fact lands.
|
|
75
|
+
*/
|
|
76
|
+
existingProperties?: Record<string, unknown>;
|
|
63
77
|
}
|
|
64
78
|
|
|
65
79
|
/** The event the refinement pipeline emits through `ingestEvent`. */
|
|
@@ -71,6 +85,12 @@ export interface RefineIngestInput {
|
|
|
71
85
|
contactId?: string;
|
|
72
86
|
eventProperties: Record<string, unknown>;
|
|
73
87
|
contactProperties: Record<string, unknown>;
|
|
88
|
+
/**
|
|
89
|
+
* Fact key NAMES to re-evaluate buckets for even when their value is not in
|
|
90
|
+
* `contactProperties` (fill-if-absent dropped it). Candidate-narrowing only —
|
|
91
|
+
* never written, never in value eval. See {@link reconcileTraits}.
|
|
92
|
+
*/
|
|
93
|
+
touchProperties?: string[];
|
|
74
94
|
idempotencyKey?: string;
|
|
75
95
|
}
|
|
76
96
|
|
|
@@ -88,9 +108,10 @@ export interface RefineEmitInput {
|
|
|
88
108
|
contactId?: string;
|
|
89
109
|
/** The enrichment provider id that answered. */
|
|
90
110
|
provider: string;
|
|
91
|
-
/** Sorted
|
|
111
|
+
/** Sorted canonical FACT keys actually filled on this contact (never the
|
|
112
|
+
* nested `enrichment` provenance key). */
|
|
92
113
|
traitKeys: string[];
|
|
93
|
-
/** The lookup instant — the same value the patch's `
|
|
114
|
+
/** The lookup instant — the same value the patch's `enrichment.at` carries. */
|
|
94
115
|
refinedAt: Date;
|
|
95
116
|
/** Per-endpoint delivery dedupe — see {@link refineOutboundDedupeKey}. */
|
|
96
117
|
dedupeKey: string;
|
|
@@ -100,7 +121,11 @@ export interface RefineEmitInput {
|
|
|
100
121
|
export interface RefineLedgerRow {
|
|
101
122
|
status: EnrichmentLookupStatus;
|
|
102
123
|
expiresAt: Date;
|
|
103
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* The normalized canonical trait patch the paid answer produced, if stored —
|
|
126
|
+
* the vendor's FULL answer, so a cache hit can land it (fill-if-absent) on a
|
|
127
|
+
* DIFFERENT contact that shares the lookup key.
|
|
128
|
+
*/
|
|
104
129
|
traits?: Record<string, unknown> | null;
|
|
105
130
|
}
|
|
106
131
|
|
|
@@ -403,10 +428,21 @@ async function runGates(args: {
|
|
|
403
428
|
spendWindow: deps.budgetWindowStart(refinedAt),
|
|
404
429
|
});
|
|
405
430
|
|
|
431
|
+
// Fill-if-absent: the vendor's facts fill only the canonical fields THIS
|
|
432
|
+
// contact does not already hold from a first-party source — a paid lookup
|
|
433
|
+
// never clobbers truer data the app already set — while every returned fact
|
|
434
|
+
// stays in the patch so the fit bucket re-evaluates. Computed here, per
|
|
435
|
+
// contact, against the row `resolveTarget` read.
|
|
436
|
+
const { landing, filled, touch } = reconcileTraits(
|
|
437
|
+
properties,
|
|
438
|
+
target.existingProperties,
|
|
439
|
+
);
|
|
440
|
+
|
|
406
441
|
const landed = await landTraits({
|
|
407
442
|
deps,
|
|
408
443
|
target,
|
|
409
|
-
properties,
|
|
444
|
+
properties: landing,
|
|
445
|
+
touch,
|
|
410
446
|
providerId,
|
|
411
447
|
found: result.found,
|
|
412
448
|
cached: false,
|
|
@@ -432,16 +468,14 @@ async function runGates(args: {
|
|
|
432
468
|
...(target.email ? { email: target.email } : {}),
|
|
433
469
|
...(target.contactId ? { contactId: target.contactId } : {}),
|
|
434
470
|
provider: providerId,
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
//
|
|
442
|
-
traitKeys:
|
|
443
|
-
.filter((key) => !REFINED_META_KEYS.includes(key))
|
|
444
|
-
.sort(),
|
|
471
|
+
// The FACTS that actually landed on this contact — the fill-if-absent set
|
|
472
|
+
// minus the nested `enrichment` provenance key. Two deliberate consequences:
|
|
473
|
+
// the envelope already carries the provenance as top-level `at` / `provider`
|
|
474
|
+
// so keeping the `enrichment` key here would say it twice; and a fact the
|
|
475
|
+
// contact already held (not overwritten) is not reported as changed, so a
|
|
476
|
+
// subscriber hears only what genuinely changed — `traits: []` when the
|
|
477
|
+
// answer added nothing new.
|
|
478
|
+
traitKeys: [...filled].sort(),
|
|
445
479
|
refinedAt,
|
|
446
480
|
dedupeKey: refineOutboundDedupeKey({
|
|
447
481
|
provider: providerId,
|
|
@@ -470,8 +504,10 @@ async function runGates(args: {
|
|
|
470
504
|
* membership was never evaluated — which is precisely the state a failed ingest
|
|
471
505
|
* leaves behind. That makes the retry after an `ingest_failed` free.
|
|
472
506
|
*
|
|
473
|
-
*
|
|
474
|
-
*
|
|
507
|
+
* The stored patch is the vendor's FULL answer, so fill-if-absent runs here too
|
|
508
|
+
* — the second contact at a company keeps its own first-party fields and only
|
|
509
|
+
* gains what it was missing. Rows written before the `traits` column existed
|
|
510
|
+
* have nothing stored and simply return `cached` with an empty patch.
|
|
475
511
|
*/
|
|
476
512
|
async function landCachedTraits(args: {
|
|
477
513
|
deps: RefineChainDeps;
|
|
@@ -481,15 +517,19 @@ async function landCachedTraits(args: {
|
|
|
481
517
|
idempotencyKey?: string;
|
|
482
518
|
}): Promise<RefineContactResult> {
|
|
483
519
|
const { deps, target, row } = args;
|
|
484
|
-
|
|
520
|
+
// A row written by the pre-canonical release stores flat `refined_*` traits;
|
|
521
|
+
// translate them so a cache hit lands the canonical shape, not the old one.
|
|
522
|
+
const stored = row.traits ? canonicalizeStoredTraits(row.traits) : undefined;
|
|
485
523
|
if (!stored) {
|
|
486
|
-
return { status: "cached", properties:
|
|
524
|
+
return { status: "cached", properties: {} };
|
|
487
525
|
}
|
|
488
526
|
|
|
527
|
+
const { landing, touch } = reconcileTraits(stored, target.existingProperties);
|
|
489
528
|
const landed = await landTraits({
|
|
490
529
|
deps,
|
|
491
530
|
target,
|
|
492
|
-
properties:
|
|
531
|
+
properties: landing,
|
|
532
|
+
touch,
|
|
493
533
|
providerId: args.providerId,
|
|
494
534
|
found: true,
|
|
495
535
|
cached: true,
|
|
@@ -517,6 +557,7 @@ async function landTraits(args: {
|
|
|
517
557
|
deps: RefineChainDeps;
|
|
518
558
|
target: RefineTarget;
|
|
519
559
|
properties: Record<string, unknown>;
|
|
560
|
+
touch: string[];
|
|
520
561
|
providerId: string;
|
|
521
562
|
found: boolean;
|
|
522
563
|
cached: boolean;
|
|
@@ -534,6 +575,7 @@ async function landTraits(args: {
|
|
|
534
575
|
cached: args.cached,
|
|
535
576
|
},
|
|
536
577
|
contactProperties: args.properties,
|
|
578
|
+
...(args.touch.length > 0 ? { touchProperties: args.touch } : {}),
|
|
537
579
|
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
538
580
|
});
|
|
539
581
|
return true;
|
|
@@ -547,6 +589,65 @@ async function landTraits(args: {
|
|
|
547
589
|
}
|
|
548
590
|
}
|
|
549
591
|
|
|
592
|
+
/**
|
|
593
|
+
* Reconcile the vendor's patch with what the contact already holds — fill-if-
|
|
594
|
+
* absent, split into three outputs because "what to WRITE" and "what to
|
|
595
|
+
* re-evaluate buckets for" are different questions.
|
|
596
|
+
*
|
|
597
|
+
* A paid answer must never overwrite a first-party value the app set. The
|
|
598
|
+
* ATOMIC way to guarantee that is to leave the already-held keys OUT of the
|
|
599
|
+
* write patch entirely: `mergePropertiesSql` is additive (`existing || patch`),
|
|
600
|
+
* so a key absent from the patch is left exactly as the live row has it — no
|
|
601
|
+
* read-modify-write, no snapshot to go stale, no chance of reverting a value a
|
|
602
|
+
* concurrent identify wrote during the (slow) vendor call. `existingProperties`
|
|
603
|
+
* is a pre-call snapshot, so this MUST NOT write a value back from it.
|
|
604
|
+
*
|
|
605
|
+
* But dropping a key hides it from `checkBucketMembership`, which narrows its
|
|
606
|
+
* candidate buckets to the property NAMES present in the ingest — so a dropped
|
|
607
|
+
* fact would never re-evaluate its fit bucket, and a torn write (properties
|
|
608
|
+
* persisted, membership not yet formed) would never reconcile. So the dropped
|
|
609
|
+
* keys still travel, separately, as `touch`: names only, no values, fed to the
|
|
610
|
+
* bucket candidate-narrowing (never to value eval, which reads the live row).
|
|
611
|
+
*
|
|
612
|
+
* Returns:
|
|
613
|
+
* - `landing`: the write patch — the `enrichment` provenance (always) plus only
|
|
614
|
+
* the facts the contact was MISSING.
|
|
615
|
+
* - `filled`: the fact keys actually taken from the vendor (drives the outbound
|
|
616
|
+
* `traitKeys` — a subscriber hears only what genuinely changed).
|
|
617
|
+
* - `touch`: EVERY fact key the vendor returned, so its bucket re-evaluates
|
|
618
|
+
* even when the value was already present.
|
|
619
|
+
*/
|
|
620
|
+
function reconcileTraits(
|
|
621
|
+
patch: Record<string, unknown>,
|
|
622
|
+
existing: Record<string, unknown> | undefined,
|
|
623
|
+
): { landing: Record<string, unknown>; filled: string[]; touch: string[] } {
|
|
624
|
+
const landing: Record<string, unknown> = {};
|
|
625
|
+
const filled: string[] = [];
|
|
626
|
+
const touch: string[] = [];
|
|
627
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
628
|
+
if (key === ENRICHMENT_KEY) {
|
|
629
|
+
landing[key] = value;
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
touch.push(key);
|
|
633
|
+
if (hasValue(existing?.[key])) continue; // already held — preserve, atomically
|
|
634
|
+
landing[key] = value;
|
|
635
|
+
filled.push(key);
|
|
636
|
+
}
|
|
637
|
+
return { landing, filled, touch };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* A property is "already set" for fill-if-absent when it holds a real value: not
|
|
642
|
+
* undefined, not null, not an empty / whitespace-only string. A numeric 0 or a
|
|
643
|
+
* `false` COUNTS as set — it is a value the app chose, not an absence.
|
|
644
|
+
*/
|
|
645
|
+
function hasValue(value: unknown): boolean {
|
|
646
|
+
if (value === undefined || value === null) return false;
|
|
647
|
+
if (typeof value === "string") return value.trim() !== "";
|
|
648
|
+
return true;
|
|
649
|
+
}
|
|
650
|
+
|
|
550
651
|
/**
|
|
551
652
|
* The per-endpoint delivery dedupe key for `contact.refined`.
|
|
552
653
|
*
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
3
|
import type { EnrichmentResult } from "@hogsend/core";
|
|
4
|
-
import {
|
|
4
|
+
import { canonicalizeStoredTraits, flattenTraits } from "./refine-traits.js";
|
|
5
5
|
|
|
6
6
|
const REFINED_AT = new Date("2026-07-25T10:00:00.000Z");
|
|
7
7
|
const META = { provider: "fake", refinedAt: REFINED_AT };
|
|
8
|
+
const ENRICHMENT = { provider: "fake", at: "2026-07-25T10:00:00.000Z" };
|
|
8
9
|
|
|
9
|
-
test("flattenTraits maps every documented field to a
|
|
10
|
+
test("flattenTraits maps every documented field to a canonical key", () => {
|
|
10
11
|
const result: EnrichmentResult = {
|
|
11
12
|
found: true,
|
|
12
13
|
person: {
|
|
@@ -28,22 +29,52 @@ test("flattenTraits maps every documented field to a flat refined_* key", () =>
|
|
|
28
29
|
};
|
|
29
30
|
|
|
30
31
|
assert.deepEqual(flattenTraits(result, META), {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
refined_provider: "fake",
|
|
32
|
+
title: "VP of Engineering",
|
|
33
|
+
seniority: "vp",
|
|
34
|
+
department: "engineering",
|
|
35
|
+
linkedin_url: "https://linkedin.com/in/x",
|
|
36
|
+
country: "US",
|
|
37
|
+
company: "Acme",
|
|
38
|
+
company_domain: "acme.com",
|
|
39
|
+
company_industry: "software",
|
|
40
|
+
company_employees: 250,
|
|
41
|
+
company_revenue: 12_000_000,
|
|
42
|
+
company_country: "US",
|
|
43
|
+
enrichment: ENRICHMENT,
|
|
44
44
|
});
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
+
// The facts are FLAT top-level keys, never nested: `evaluatePropertyConditions`
|
|
48
|
+
// does `properties[condition.property]` with no dotted-path traversal, so a
|
|
49
|
+
// bucket must be able to author `b.prop("company_industry")` verbatim.
|
|
50
|
+
test("facts are flat top-level keys; only provenance nests under `enrichment`", () => {
|
|
51
|
+
const patch = flattenTraits(
|
|
52
|
+
{
|
|
53
|
+
found: true,
|
|
54
|
+
person: { seniority: "vp" },
|
|
55
|
+
company: { industry: "software", employeeCount: 250 },
|
|
56
|
+
raw: {},
|
|
57
|
+
},
|
|
58
|
+
META,
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
assert.equal(patch.seniority, "vp");
|
|
62
|
+
assert.equal(patch.company_industry, "software");
|
|
63
|
+
assert.equal(patch.company_employees, 250);
|
|
64
|
+
// No parallel `refined_*` vocabulary, no dotted keys.
|
|
65
|
+
for (const key of Object.keys(patch)) {
|
|
66
|
+
assert.ok(
|
|
67
|
+
!key.startsWith("refined_"),
|
|
68
|
+
`${key} must not be refined_-prefixed`,
|
|
69
|
+
);
|
|
70
|
+
assert.ok(
|
|
71
|
+
key === "enrichment" || !key.includes("."),
|
|
72
|
+
`${key} must be flat`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
assert.deepEqual(patch.enrichment, ENRICHMENT);
|
|
76
|
+
});
|
|
77
|
+
|
|
47
78
|
// AC 8 — the numeric traits must be REAL JSON numbers. `conditions/property.ts`
|
|
48
79
|
// does no coercion, so a `"250"` would silently never match a `gte(100)`.
|
|
49
80
|
test("AC 8: employee count is a real JSON number, and survives JSON round-trip", () => {
|
|
@@ -52,12 +83,12 @@ test("AC 8: employee count is a real JSON number, and survives JSON round-trip",
|
|
|
52
83
|
META,
|
|
53
84
|
);
|
|
54
85
|
|
|
55
|
-
assert.equal(typeof patch.
|
|
56
|
-
assert.equal(patch.
|
|
86
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
87
|
+
assert.equal(patch.company_employees, 250);
|
|
57
88
|
// jsonb storage is a JSON round-trip — the type must survive it.
|
|
58
89
|
const roundTripped = JSON.parse(JSON.stringify(patch));
|
|
59
|
-
assert.equal(typeof roundTripped.
|
|
60
|
-
assert.ok((roundTripped.
|
|
90
|
+
assert.equal(typeof roundTripped.company_employees, "number");
|
|
91
|
+
assert.ok((roundTripped.company_employees as number) >= 100);
|
|
61
92
|
});
|
|
62
93
|
|
|
63
94
|
test("AC 8: a numeric STRING from an untyped provider is coerced to a number", () => {
|
|
@@ -72,8 +103,8 @@ test("AC 8: a numeric STRING from an untyped provider is coerced to a number", (
|
|
|
72
103
|
META,
|
|
73
104
|
);
|
|
74
105
|
|
|
75
|
-
assert.equal(patch.
|
|
76
|
-
assert.equal(typeof patch.
|
|
106
|
+
assert.equal(patch.company_employees, 250);
|
|
107
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
77
108
|
});
|
|
78
109
|
|
|
79
110
|
// AC 9 — an absent field must be OMITTED, never written as null: the patch is
|
|
@@ -84,13 +115,9 @@ test("AC 9: absent fields are omitted from the patch entirely (never null)", ()
|
|
|
84
115
|
META,
|
|
85
116
|
);
|
|
86
117
|
|
|
87
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"refined_title",
|
|
91
|
-
]);
|
|
92
|
-
assert.equal("refined_seniority" in patch, false);
|
|
93
|
-
assert.equal("refined_company_name" in patch, false);
|
|
118
|
+
assert.deepEqual(Object.keys(patch).sort(), ["enrichment", "title"]);
|
|
119
|
+
assert.equal("seniority" in patch, false);
|
|
120
|
+
assert.equal("company" in patch, false);
|
|
94
121
|
for (const value of Object.values(patch)) {
|
|
95
122
|
assert.notEqual(value, null);
|
|
96
123
|
}
|
|
@@ -114,28 +141,34 @@ test("AC 9: null / empty-string / non-finite fields are omitted, not written", (
|
|
|
114
141
|
META,
|
|
115
142
|
);
|
|
116
143
|
|
|
117
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
118
|
-
"refined_at",
|
|
119
|
-
"refined_department",
|
|
120
|
-
"refined_provider",
|
|
121
|
-
]);
|
|
144
|
+
assert.deepEqual(Object.keys(patch).sort(), ["department", "enrichment"]);
|
|
122
145
|
});
|
|
123
146
|
|
|
124
|
-
test("a not-found result still stamps only the
|
|
147
|
+
test("a not-found result still stamps only the provenance object", () => {
|
|
125
148
|
assert.deepEqual(flattenTraits({ found: false, raw: null }, META), {
|
|
126
|
-
|
|
127
|
-
refined_provider: "fake",
|
|
149
|
+
enrichment: ENRICHMENT,
|
|
128
150
|
});
|
|
129
151
|
});
|
|
130
152
|
|
|
131
|
-
test("
|
|
153
|
+
test("canonicalizeStoredTraits translates a pre-canonical (refined_*) ledger patch", () => {
|
|
132
154
|
assert.deepEqual(
|
|
133
|
-
|
|
134
|
-
plan: "pro",
|
|
155
|
+
canonicalizeStoredTraits({
|
|
135
156
|
refined_title: "CTO",
|
|
157
|
+
refined_company_name: "Acme",
|
|
136
158
|
refined_company_employees: 250,
|
|
159
|
+
refined_at: "2026-01-01T00:00:00.000Z",
|
|
160
|
+
refined_provider: "apollo",
|
|
137
161
|
}),
|
|
138
|
-
{
|
|
162
|
+
{ title: "CTO", company: "Acme", company_employees: 250 },
|
|
139
163
|
);
|
|
140
|
-
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("canonicalizeStoredTraits leaves an already-canonical patch untouched (by reference)", () => {
|
|
167
|
+
const canonical = {
|
|
168
|
+
title: "CTO",
|
|
169
|
+
company_employees: 250,
|
|
170
|
+
enrichment: ENRICHMENT,
|
|
171
|
+
};
|
|
172
|
+
// No legacy key → returns the SAME object, no needless copy.
|
|
173
|
+
assert.equal(canonicalizeStoredTraits(canonical), canonical);
|
|
141
174
|
});
|