@hogsend/engine 0.54.0 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/check-membership.ts +58 -3
- package/src/lib/bucket-emit.test.ts +145 -0
- package/src/lib/bucket-emit.ts +29 -3
- package/src/lib/email-providers-from-env.ts +15 -16
- package/src/lib/enrichment-ledger.ts +1 -1
- package/src/lib/enrichment-providers-from-env.test.ts +5 -44
- package/src/lib/enrichment-providers-from-env.ts +8 -41
- package/src/lib/ingestion.ts +16 -0
- package/src/lib/load-optional-plugin.test.ts +133 -0
- package/src/lib/load-optional-plugin.ts +142 -0
- package/src/lib/outbound.ts +3 -3
- package/src/lib/refine-chain.test.ts +83 -25
- package/src/lib/refine-chain.ts +124 -23
- package/src/lib/refine-traits.test.ts +74 -41
- package/src/lib/refine-traits.ts +107 -52
- package/src/lib/refine.ts +28 -27
- package/src/lib/sms-providers-from-env.ts +11 -11
- package/src/workflows/bucket-backfill.ts +14 -0
- package/src/workflows/bucket-reconcile.ts +74 -15
package/src/lib/refine-chain.ts
CHANGED
|
@@ -12,7 +12,11 @@ import type {
|
|
|
12
12
|
EnrichmentLookupKind,
|
|
13
13
|
EnrichmentLookupStatus,
|
|
14
14
|
} from "./enrichment-ledger.js";
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
canonicalizeStoredTraits,
|
|
17
|
+
ENRICHMENT_KEY,
|
|
18
|
+
flattenTraits,
|
|
19
|
+
} from "./refine-traits.js";
|
|
16
20
|
|
|
17
21
|
// ---------------------------------------------------------------------------
|
|
18
22
|
// Public surface
|
|
@@ -42,7 +46,12 @@ export interface RefineContactResult {
|
|
|
42
46
|
* | `provider_error` | `ingest_failed`.
|
|
43
47
|
*/
|
|
44
48
|
reason?: string;
|
|
45
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* The vendor's FULL canonical trait patch (facts + the nested `enrichment`
|
|
51
|
+
* provenance) — on `refined` AND on `cached`. This is what the vendor
|
|
52
|
+
* answered, NOT the fill-if-absent subset that actually landed on any one
|
|
53
|
+
* contact.
|
|
54
|
+
*/
|
|
46
55
|
properties?: Record<string, unknown>;
|
|
47
56
|
}
|
|
48
57
|
|
|
@@ -58,8 +67,13 @@ export interface RefineTarget {
|
|
|
58
67
|
firstName?: string;
|
|
59
68
|
lastName?: string;
|
|
60
69
|
company?: string;
|
|
61
|
-
/**
|
|
62
|
-
|
|
70
|
+
/**
|
|
71
|
+
* The properties already on THIS contact's row, used by fill-if-absent to
|
|
72
|
+
* decide which vendor facts to keep (a fact the contact already holds from a
|
|
73
|
+
* first-party source is never overwritten). Absent for a brand-new email with
|
|
74
|
+
* no row yet — nothing to preserve, so every vendor fact lands.
|
|
75
|
+
*/
|
|
76
|
+
existingProperties?: Record<string, unknown>;
|
|
63
77
|
}
|
|
64
78
|
|
|
65
79
|
/** The event the refinement pipeline emits through `ingestEvent`. */
|
|
@@ -71,6 +85,12 @@ export interface RefineIngestInput {
|
|
|
71
85
|
contactId?: string;
|
|
72
86
|
eventProperties: Record<string, unknown>;
|
|
73
87
|
contactProperties: Record<string, unknown>;
|
|
88
|
+
/**
|
|
89
|
+
* Fact key NAMES to re-evaluate buckets for even when their value is not in
|
|
90
|
+
* `contactProperties` (fill-if-absent dropped it). Candidate-narrowing only —
|
|
91
|
+
* never written, never in value eval. See {@link reconcileTraits}.
|
|
92
|
+
*/
|
|
93
|
+
touchProperties?: string[];
|
|
74
94
|
idempotencyKey?: string;
|
|
75
95
|
}
|
|
76
96
|
|
|
@@ -88,9 +108,10 @@ export interface RefineEmitInput {
|
|
|
88
108
|
contactId?: string;
|
|
89
109
|
/** The enrichment provider id that answered. */
|
|
90
110
|
provider: string;
|
|
91
|
-
/** Sorted
|
|
111
|
+
/** Sorted canonical FACT keys actually filled on this contact (never the
|
|
112
|
+
* nested `enrichment` provenance key). */
|
|
92
113
|
traitKeys: string[];
|
|
93
|
-
/** The lookup instant — the same value the patch's `
|
|
114
|
+
/** The lookup instant — the same value the patch's `enrichment.at` carries. */
|
|
94
115
|
refinedAt: Date;
|
|
95
116
|
/** Per-endpoint delivery dedupe — see {@link refineOutboundDedupeKey}. */
|
|
96
117
|
dedupeKey: string;
|
|
@@ -100,7 +121,11 @@ export interface RefineEmitInput {
|
|
|
100
121
|
export interface RefineLedgerRow {
|
|
101
122
|
status: EnrichmentLookupStatus;
|
|
102
123
|
expiresAt: Date;
|
|
103
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* The normalized canonical trait patch the paid answer produced, if stored —
|
|
126
|
+
* the vendor's FULL answer, so a cache hit can land it (fill-if-absent) on a
|
|
127
|
+
* DIFFERENT contact that shares the lookup key.
|
|
128
|
+
*/
|
|
104
129
|
traits?: Record<string, unknown> | null;
|
|
105
130
|
}
|
|
106
131
|
|
|
@@ -403,10 +428,21 @@ async function runGates(args: {
|
|
|
403
428
|
spendWindow: deps.budgetWindowStart(refinedAt),
|
|
404
429
|
});
|
|
405
430
|
|
|
431
|
+
// Fill-if-absent: the vendor's facts fill only the canonical fields THIS
|
|
432
|
+
// contact does not already hold from a first-party source — a paid lookup
|
|
433
|
+
// never clobbers truer data the app already set — while every returned fact
|
|
434
|
+
// stays in the patch so the fit bucket re-evaluates. Computed here, per
|
|
435
|
+
// contact, against the row `resolveTarget` read.
|
|
436
|
+
const { landing, filled, touch } = reconcileTraits(
|
|
437
|
+
properties,
|
|
438
|
+
target.existingProperties,
|
|
439
|
+
);
|
|
440
|
+
|
|
406
441
|
const landed = await landTraits({
|
|
407
442
|
deps,
|
|
408
443
|
target,
|
|
409
|
-
properties,
|
|
444
|
+
properties: landing,
|
|
445
|
+
touch,
|
|
410
446
|
providerId,
|
|
411
447
|
found: result.found,
|
|
412
448
|
cached: false,
|
|
@@ -432,16 +468,14 @@ async function runGates(args: {
|
|
|
432
468
|
...(target.email ? { email: target.email } : {}),
|
|
433
469
|
...(target.contactId ? { contactId: target.contactId } : {}),
|
|
434
470
|
provider: providerId,
|
|
435
|
-
//
|
|
436
|
-
//
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
//
|
|
442
|
-
traitKeys:
|
|
443
|
-
.filter((key) => !REFINED_META_KEYS.includes(key))
|
|
444
|
-
.sort(),
|
|
471
|
+
// The FACTS that actually landed on this contact — the fill-if-absent set
|
|
472
|
+
// minus the nested `enrichment` provenance key. Two deliberate consequences:
|
|
473
|
+
// the envelope already carries the provenance as top-level `at` / `provider`
|
|
474
|
+
// so keeping the `enrichment` key here would say it twice; and a fact the
|
|
475
|
+
// contact already held (not overwritten) is not reported as changed, so a
|
|
476
|
+
// subscriber hears only what genuinely changed — `traits: []` when the
|
|
477
|
+
// answer added nothing new.
|
|
478
|
+
traitKeys: [...filled].sort(),
|
|
445
479
|
refinedAt,
|
|
446
480
|
dedupeKey: refineOutboundDedupeKey({
|
|
447
481
|
provider: providerId,
|
|
@@ -470,8 +504,10 @@ async function runGates(args: {
|
|
|
470
504
|
* membership was never evaluated — which is precisely the state a failed ingest
|
|
471
505
|
* leaves behind. That makes the retry after an `ingest_failed` free.
|
|
472
506
|
*
|
|
473
|
-
*
|
|
474
|
-
*
|
|
507
|
+
* The stored patch is the vendor's FULL answer, so fill-if-absent runs here too
|
|
508
|
+
* — the second contact at a company keeps its own first-party fields and only
|
|
509
|
+
* gains what it was missing. Rows written before the `traits` column existed
|
|
510
|
+
* have nothing stored and simply return `cached` with an empty patch.
|
|
475
511
|
*/
|
|
476
512
|
async function landCachedTraits(args: {
|
|
477
513
|
deps: RefineChainDeps;
|
|
@@ -481,15 +517,19 @@ async function landCachedTraits(args: {
|
|
|
481
517
|
idempotencyKey?: string;
|
|
482
518
|
}): Promise<RefineContactResult> {
|
|
483
519
|
const { deps, target, row } = args;
|
|
484
|
-
|
|
520
|
+
// A row written by the pre-canonical release stores flat `refined_*` traits;
|
|
521
|
+
// translate them so a cache hit lands the canonical shape, not the old one.
|
|
522
|
+
const stored = row.traits ? canonicalizeStoredTraits(row.traits) : undefined;
|
|
485
523
|
if (!stored) {
|
|
486
|
-
return { status: "cached", properties:
|
|
524
|
+
return { status: "cached", properties: {} };
|
|
487
525
|
}
|
|
488
526
|
|
|
527
|
+
const { landing, touch } = reconcileTraits(stored, target.existingProperties);
|
|
489
528
|
const landed = await landTraits({
|
|
490
529
|
deps,
|
|
491
530
|
target,
|
|
492
|
-
properties:
|
|
531
|
+
properties: landing,
|
|
532
|
+
touch,
|
|
493
533
|
providerId: args.providerId,
|
|
494
534
|
found: true,
|
|
495
535
|
cached: true,
|
|
@@ -517,6 +557,7 @@ async function landTraits(args: {
|
|
|
517
557
|
deps: RefineChainDeps;
|
|
518
558
|
target: RefineTarget;
|
|
519
559
|
properties: Record<string, unknown>;
|
|
560
|
+
touch: string[];
|
|
520
561
|
providerId: string;
|
|
521
562
|
found: boolean;
|
|
522
563
|
cached: boolean;
|
|
@@ -534,6 +575,7 @@ async function landTraits(args: {
|
|
|
534
575
|
cached: args.cached,
|
|
535
576
|
},
|
|
536
577
|
contactProperties: args.properties,
|
|
578
|
+
...(args.touch.length > 0 ? { touchProperties: args.touch } : {}),
|
|
537
579
|
...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
|
|
538
580
|
});
|
|
539
581
|
return true;
|
|
@@ -547,6 +589,65 @@ async function landTraits(args: {
|
|
|
547
589
|
}
|
|
548
590
|
}
|
|
549
591
|
|
|
592
|
+
/**
|
|
593
|
+
* Reconcile the vendor's patch with what the contact already holds — fill-if-
|
|
594
|
+
* absent, split into three outputs because "what to WRITE" and "what to
|
|
595
|
+
* re-evaluate buckets for" are different questions.
|
|
596
|
+
*
|
|
597
|
+
* A paid answer must never overwrite a first-party value the app set. The
|
|
598
|
+
* ATOMIC way to guarantee that is to leave the already-held keys OUT of the
|
|
599
|
+
* write patch entirely: `mergePropertiesSql` is additive (`existing || patch`),
|
|
600
|
+
* so a key absent from the patch is left exactly as the live row has it — no
|
|
601
|
+
* read-modify-write, no snapshot to go stale, no chance of reverting a value a
|
|
602
|
+
* concurrent identify wrote during the (slow) vendor call. `existingProperties`
|
|
603
|
+
* is a pre-call snapshot, so this MUST NOT write a value back from it.
|
|
604
|
+
*
|
|
605
|
+
* But dropping a key hides it from `checkBucketMembership`, which narrows its
|
|
606
|
+
* candidate buckets to the property NAMES present in the ingest — so a dropped
|
|
607
|
+
* fact would never re-evaluate its fit bucket, and a torn write (properties
|
|
608
|
+
* persisted, membership not yet formed) would never reconcile. So the dropped
|
|
609
|
+
* keys still travel, separately, as `touch`: names only, no values, fed to the
|
|
610
|
+
* bucket candidate-narrowing (never to value eval, which reads the live row).
|
|
611
|
+
*
|
|
612
|
+
* Returns:
|
|
613
|
+
* - `landing`: the write patch — the `enrichment` provenance (always) plus only
|
|
614
|
+
* the facts the contact was MISSING.
|
|
615
|
+
* - `filled`: the fact keys actually taken from the vendor (drives the outbound
|
|
616
|
+
* `traitKeys` — a subscriber hears only what genuinely changed).
|
|
617
|
+
* - `touch`: EVERY fact key the vendor returned, so its bucket re-evaluates
|
|
618
|
+
* even when the value was already present.
|
|
619
|
+
*/
|
|
620
|
+
function reconcileTraits(
|
|
621
|
+
patch: Record<string, unknown>,
|
|
622
|
+
existing: Record<string, unknown> | undefined,
|
|
623
|
+
): { landing: Record<string, unknown>; filled: string[]; touch: string[] } {
|
|
624
|
+
const landing: Record<string, unknown> = {};
|
|
625
|
+
const filled: string[] = [];
|
|
626
|
+
const touch: string[] = [];
|
|
627
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
628
|
+
if (key === ENRICHMENT_KEY) {
|
|
629
|
+
landing[key] = value;
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
touch.push(key);
|
|
633
|
+
if (hasValue(existing?.[key])) continue; // already held — preserve, atomically
|
|
634
|
+
landing[key] = value;
|
|
635
|
+
filled.push(key);
|
|
636
|
+
}
|
|
637
|
+
return { landing, filled, touch };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* A property is "already set" for fill-if-absent when it holds a real value: not
|
|
642
|
+
* undefined, not null, not an empty / whitespace-only string. A numeric 0 or a
|
|
643
|
+
* `false` COUNTS as set — it is a value the app chose, not an absence.
|
|
644
|
+
*/
|
|
645
|
+
function hasValue(value: unknown): boolean {
|
|
646
|
+
if (value === undefined || value === null) return false;
|
|
647
|
+
if (typeof value === "string") return value.trim() !== "";
|
|
648
|
+
return true;
|
|
649
|
+
}
|
|
650
|
+
|
|
550
651
|
/**
|
|
551
652
|
* The per-endpoint delivery dedupe key for `contact.refined`.
|
|
552
653
|
*
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
3
|
import type { EnrichmentResult } from "@hogsend/core";
|
|
4
|
-
import {
|
|
4
|
+
import { canonicalizeStoredTraits, flattenTraits } from "./refine-traits.js";
|
|
5
5
|
|
|
6
6
|
const REFINED_AT = new Date("2026-07-25T10:00:00.000Z");
|
|
7
7
|
const META = { provider: "fake", refinedAt: REFINED_AT };
|
|
8
|
+
const ENRICHMENT = { provider: "fake", at: "2026-07-25T10:00:00.000Z" };
|
|
8
9
|
|
|
9
|
-
test("flattenTraits maps every documented field to a
|
|
10
|
+
test("flattenTraits maps every documented field to a canonical key", () => {
|
|
10
11
|
const result: EnrichmentResult = {
|
|
11
12
|
found: true,
|
|
12
13
|
person: {
|
|
@@ -28,22 +29,52 @@ test("flattenTraits maps every documented field to a flat refined_* key", () =>
|
|
|
28
29
|
};
|
|
29
30
|
|
|
30
31
|
assert.deepEqual(flattenTraits(result, META), {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
refined_provider: "fake",
|
|
32
|
+
title: "VP of Engineering",
|
|
33
|
+
seniority: "vp",
|
|
34
|
+
department: "engineering",
|
|
35
|
+
linkedin_url: "https://linkedin.com/in/x",
|
|
36
|
+
country: "US",
|
|
37
|
+
company: "Acme",
|
|
38
|
+
company_domain: "acme.com",
|
|
39
|
+
company_industry: "software",
|
|
40
|
+
company_employees: 250,
|
|
41
|
+
company_revenue: 12_000_000,
|
|
42
|
+
company_country: "US",
|
|
43
|
+
enrichment: ENRICHMENT,
|
|
44
44
|
});
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
+
// The facts are FLAT top-level keys, never nested: `evaluatePropertyConditions`
|
|
48
|
+
// does `properties[condition.property]` with no dotted-path traversal, so a
|
|
49
|
+
// bucket must be able to author `b.prop("company_industry")` verbatim.
|
|
50
|
+
test("facts are flat top-level keys; only provenance nests under `enrichment`", () => {
|
|
51
|
+
const patch = flattenTraits(
|
|
52
|
+
{
|
|
53
|
+
found: true,
|
|
54
|
+
person: { seniority: "vp" },
|
|
55
|
+
company: { industry: "software", employeeCount: 250 },
|
|
56
|
+
raw: {},
|
|
57
|
+
},
|
|
58
|
+
META,
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
assert.equal(patch.seniority, "vp");
|
|
62
|
+
assert.equal(patch.company_industry, "software");
|
|
63
|
+
assert.equal(patch.company_employees, 250);
|
|
64
|
+
// No parallel `refined_*` vocabulary, no dotted keys.
|
|
65
|
+
for (const key of Object.keys(patch)) {
|
|
66
|
+
assert.ok(
|
|
67
|
+
!key.startsWith("refined_"),
|
|
68
|
+
`${key} must not be refined_-prefixed`,
|
|
69
|
+
);
|
|
70
|
+
assert.ok(
|
|
71
|
+
key === "enrichment" || !key.includes("."),
|
|
72
|
+
`${key} must be flat`,
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
assert.deepEqual(patch.enrichment, ENRICHMENT);
|
|
76
|
+
});
|
|
77
|
+
|
|
47
78
|
// AC 8 — the numeric traits must be REAL JSON numbers. `conditions/property.ts`
|
|
48
79
|
// does no coercion, so a `"250"` would silently never match a `gte(100)`.
|
|
49
80
|
test("AC 8: employee count is a real JSON number, and survives JSON round-trip", () => {
|
|
@@ -52,12 +83,12 @@ test("AC 8: employee count is a real JSON number, and survives JSON round-trip",
|
|
|
52
83
|
META,
|
|
53
84
|
);
|
|
54
85
|
|
|
55
|
-
assert.equal(typeof patch.
|
|
56
|
-
assert.equal(patch.
|
|
86
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
87
|
+
assert.equal(patch.company_employees, 250);
|
|
57
88
|
// jsonb storage is a JSON round-trip — the type must survive it.
|
|
58
89
|
const roundTripped = JSON.parse(JSON.stringify(patch));
|
|
59
|
-
assert.equal(typeof roundTripped.
|
|
60
|
-
assert.ok((roundTripped.
|
|
90
|
+
assert.equal(typeof roundTripped.company_employees, "number");
|
|
91
|
+
assert.ok((roundTripped.company_employees as number) >= 100);
|
|
61
92
|
});
|
|
62
93
|
|
|
63
94
|
test("AC 8: a numeric STRING from an untyped provider is coerced to a number", () => {
|
|
@@ -72,8 +103,8 @@ test("AC 8: a numeric STRING from an untyped provider is coerced to a number", (
|
|
|
72
103
|
META,
|
|
73
104
|
);
|
|
74
105
|
|
|
75
|
-
assert.equal(patch.
|
|
76
|
-
assert.equal(typeof patch.
|
|
106
|
+
assert.equal(patch.company_employees, 250);
|
|
107
|
+
assert.equal(typeof patch.company_employees, "number");
|
|
77
108
|
});
|
|
78
109
|
|
|
79
110
|
// AC 9 — an absent field must be OMITTED, never written as null: the patch is
|
|
@@ -84,13 +115,9 @@ test("AC 9: absent fields are omitted from the patch entirely (never null)", ()
|
|
|
84
115
|
META,
|
|
85
116
|
);
|
|
86
117
|
|
|
87
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
"refined_title",
|
|
91
|
-
]);
|
|
92
|
-
assert.equal("refined_seniority" in patch, false);
|
|
93
|
-
assert.equal("refined_company_name" in patch, false);
|
|
118
|
+
assert.deepEqual(Object.keys(patch).sort(), ["enrichment", "title"]);
|
|
119
|
+
assert.equal("seniority" in patch, false);
|
|
120
|
+
assert.equal("company" in patch, false);
|
|
94
121
|
for (const value of Object.values(patch)) {
|
|
95
122
|
assert.notEqual(value, null);
|
|
96
123
|
}
|
|
@@ -114,28 +141,34 @@ test("AC 9: null / empty-string / non-finite fields are omitted, not written", (
|
|
|
114
141
|
META,
|
|
115
142
|
);
|
|
116
143
|
|
|
117
|
-
assert.deepEqual(Object.keys(patch).sort(), [
|
|
118
|
-
"refined_at",
|
|
119
|
-
"refined_department",
|
|
120
|
-
"refined_provider",
|
|
121
|
-
]);
|
|
144
|
+
assert.deepEqual(Object.keys(patch).sort(), ["department", "enrichment"]);
|
|
122
145
|
});
|
|
123
146
|
|
|
124
|
-
test("a not-found result still stamps only the
|
|
147
|
+
test("a not-found result still stamps only the provenance object", () => {
|
|
125
148
|
assert.deepEqual(flattenTraits({ found: false, raw: null }, META), {
|
|
126
|
-
|
|
127
|
-
refined_provider: "fake",
|
|
149
|
+
enrichment: ENRICHMENT,
|
|
128
150
|
});
|
|
129
151
|
});
|
|
130
152
|
|
|
131
|
-
test("
|
|
153
|
+
test("canonicalizeStoredTraits translates a pre-canonical (refined_*) ledger patch", () => {
|
|
132
154
|
assert.deepEqual(
|
|
133
|
-
|
|
134
|
-
plan: "pro",
|
|
155
|
+
canonicalizeStoredTraits({
|
|
135
156
|
refined_title: "CTO",
|
|
157
|
+
refined_company_name: "Acme",
|
|
136
158
|
refined_company_employees: 250,
|
|
159
|
+
refined_at: "2026-01-01T00:00:00.000Z",
|
|
160
|
+
refined_provider: "apollo",
|
|
137
161
|
}),
|
|
138
|
-
{
|
|
162
|
+
{ title: "CTO", company: "Acme", company_employees: 250 },
|
|
139
163
|
);
|
|
140
|
-
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("canonicalizeStoredTraits leaves an already-canonical patch untouched (by reference)", () => {
|
|
167
|
+
const canonical = {
|
|
168
|
+
title: "CTO",
|
|
169
|
+
company_employees: 250,
|
|
170
|
+
enrichment: ENRICHMENT,
|
|
171
|
+
};
|
|
172
|
+
// No legacy key → returns the SAME object, no needless copy.
|
|
173
|
+
assert.equal(canonicalizeStoredTraits(canonical), canonical);
|
|
141
174
|
});
|
package/src/lib/refine-traits.ts
CHANGED
|
@@ -1,27 +1,54 @@
|
|
|
1
1
|
import type { EnrichmentResult } from "@hogsend/core";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Identity of the lookup that produced a patch — the
|
|
5
|
-
* successful refinement stamps alongside the vendor's own
|
|
4
|
+
* Identity of the lookup that produced a patch — the provenance every
|
|
5
|
+
* successful refinement stamps alongside the vendor's own facts.
|
|
6
6
|
*/
|
|
7
7
|
export interface RefinedTraitsMeta {
|
|
8
|
-
/** The enrichment provider id that answered (`
|
|
8
|
+
/** The enrichment provider id that answered (`enrichment.provider`). */
|
|
9
9
|
provider: string;
|
|
10
|
-
/** The lookup instant, written as an ISO string (`
|
|
10
|
+
/** The lookup instant, written as an ISO string (`enrichment.at`). */
|
|
11
11
|
refinedAt: Date;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
/**
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* `contacts.properties`.
|
|
15
|
+
* The single nested key that carries refinement PROVENANCE — who answered and
|
|
16
|
+
* when. Split out from the vendor FACTS on purpose:
|
|
18
17
|
*
|
|
19
|
-
*
|
|
18
|
+
* - Facts land on the CANONICAL, flat property keys a contact already carries
|
|
19
|
+
* (`title`, `company`, `company_employees`, …) so a bucket segments on the
|
|
20
|
+
* field it already knows — `b.prop("company_employees").gte(100)` — instead
|
|
21
|
+
* of a parallel `refined_*` vocabulary sitting next to the real one.
|
|
22
|
+
* - Provenance is NEVER read by `evaluatePropertyConditions` (segmentation is
|
|
23
|
+
* about facts, not who told us), so it nests freely under one key with no
|
|
24
|
+
* dotted-path support required from the condition engine. That is the whole
|
|
25
|
+
* reason the facts must stay flat and the provenance is free to be an object.
|
|
26
|
+
*/
|
|
27
|
+
export const ENRICHMENT_KEY = "enrichment";
|
|
28
|
+
|
|
29
|
+
/** The provenance object stored under {@link ENRICHMENT_KEY}. */
|
|
30
|
+
export interface EnrichmentProvenance {
|
|
31
|
+
/** The enrichment provider id that answered. */
|
|
32
|
+
provider: string;
|
|
33
|
+
/** ISO — the instant the vendor answer was obtained. */
|
|
34
|
+
at: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Map a normalized {@link EnrichmentResult} onto a CANONICAL contact-property
|
|
39
|
+
* patch that `ingestEvent` merges into `contacts.properties`. The vendor facts
|
|
40
|
+
* fill the fields a contact already segments and personalizes on; a single
|
|
41
|
+
* nested `enrichment` object records the provenance.
|
|
42
|
+
*
|
|
43
|
+
* Three rules make this function load-bearing rather than cosmetic:
|
|
20
44
|
*
|
|
21
|
-
* - **
|
|
22
|
-
* `
|
|
23
|
-
*
|
|
24
|
-
*
|
|
45
|
+
* - **Facts are flat + canonical.** `evaluatePropertyConditions` reads
|
|
46
|
+
* `properties[condition.property]` — a flat lookup with no dotted-path
|
|
47
|
+
* traversal — so every fact key is a top-level string a bucket can author
|
|
48
|
+
* verbatim (`company_industry`, not `company.industry`).
|
|
49
|
+
* - **Provenance is one nested object.** The condition engine never reads it,
|
|
50
|
+
* so `enrichment: { provider, at }` replaces the old flat `refined_at` /
|
|
51
|
+
* `refined_provider` sibling keys without touching the evaluator.
|
|
25
52
|
* - **Absent means ABSENT.** An undefined vendor field is omitted from the
|
|
26
53
|
* patch entirely — never written as `null`. `mergePropertiesSql` wraps the
|
|
27
54
|
* patch in `jsonb_strip_nulls`, so a null would DELETE a previously-good key
|
|
@@ -29,6 +56,13 @@ export interface RefinedTraitsMeta {
|
|
|
29
56
|
*
|
|
30
57
|
* Numeric traits are written as real JSON numbers: `conditions/property.ts` does
|
|
31
58
|
* no coercion, so a `"42"` would silently never match a `gte`.
|
|
59
|
+
*
|
|
60
|
+
* The patch is the vendor's FULL answer, keyed canonically. Whether an
|
|
61
|
+
* individual fact actually overwrites what a contact already holds is decided at
|
|
62
|
+
* LANDING time (fill-if-absent, in `refine-chain.ts`), because that answer
|
|
63
|
+
* depends on the specific contact — the same stored patch lands on every contact
|
|
64
|
+
* that shares a company domain, and each may already carry different first-party
|
|
65
|
+
* fields.
|
|
32
66
|
*/
|
|
33
67
|
export function flattenTraits(
|
|
34
68
|
result: EnrichmentResult,
|
|
@@ -38,56 +72,77 @@ export function flattenTraits(
|
|
|
38
72
|
const person = result.person;
|
|
39
73
|
const company = result.company;
|
|
40
74
|
|
|
41
|
-
putString(patch, "
|
|
42
|
-
putString(patch, "
|
|
43
|
-
putString(patch, "
|
|
44
|
-
putString(patch, "
|
|
45
|
-
putString(patch, "
|
|
75
|
+
putString(patch, "title", person?.title);
|
|
76
|
+
putString(patch, "seniority", person?.seniority);
|
|
77
|
+
putString(patch, "department", person?.department);
|
|
78
|
+
putString(patch, "linkedin_url", person?.linkedinUrl);
|
|
79
|
+
putString(patch, "country", person?.country);
|
|
46
80
|
|
|
47
|
-
putString(patch, "
|
|
48
|
-
putString(patch, "
|
|
49
|
-
putString(patch, "
|
|
50
|
-
putNumber(patch, "
|
|
51
|
-
putNumber(patch, "
|
|
52
|
-
putString(patch, "
|
|
81
|
+
putString(patch, "company", company?.name);
|
|
82
|
+
putString(patch, "company_domain", company?.domain);
|
|
83
|
+
putString(patch, "company_industry", company?.industry);
|
|
84
|
+
putNumber(patch, "company_employees", company?.employeeCount);
|
|
85
|
+
putNumber(patch, "company_revenue", company?.estimatedRevenue);
|
|
86
|
+
putString(patch, "company_country", company?.country);
|
|
53
87
|
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
const provenance: EnrichmentProvenance = {
|
|
89
|
+
provider: meta.provider,
|
|
90
|
+
at: meta.refinedAt.toISOString(),
|
|
91
|
+
};
|
|
92
|
+
patch[ENRICHMENT_KEY] = provenance;
|
|
56
93
|
|
|
57
94
|
return patch;
|
|
58
95
|
}
|
|
59
96
|
|
|
60
|
-
/** The prefix every trait this module writes carries. */
|
|
61
|
-
export const REFINED_TRAIT_PREFIX = "refined_";
|
|
62
|
-
|
|
63
97
|
/**
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
* The `refined_*` subset of an already-stored `contacts.properties` bag — what
|
|
76
|
-
* a cache hit compares the ledger row's stored patch against, to tell "this
|
|
77
|
-
* contact already has the paid answer" apart from "another contact at the same
|
|
78
|
-
* domain paid for it and this one has never seen it".
|
|
98
|
+
* Translate a ledger `traits` patch written by the PRE-canonical release (flat
|
|
99
|
+
* `refined_*` fact keys plus flat `refined_at` / `refined_provider` provenance)
|
|
100
|
+
* into the canonical shape {@link flattenTraits} now produces. A cache HIT can
|
|
101
|
+
* land a row written by an older engine, and landing it verbatim would
|
|
102
|
+
* re-introduce exactly the flat vocabulary this design removed.
|
|
103
|
+
*
|
|
104
|
+
* A patch with no `refined_*` key is already canonical and passes through
|
|
105
|
+
* untouched (the common case, once the ledger has turned over). The two legacy
|
|
106
|
+
* provenance keys are dropped: they are not facts, and the fresh ISO provenance
|
|
107
|
+
* is not recoverable from them here, so a legacy cache hit simply lands the
|
|
108
|
+
* facts without a new `enrichment` stamp.
|
|
79
109
|
*/
|
|
80
|
-
export function
|
|
81
|
-
|
|
110
|
+
export function canonicalizeStoredTraits(
|
|
111
|
+
traits: Record<string, unknown>,
|
|
82
112
|
): Record<string, unknown> {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
for (const [key, value] of Object.entries(
|
|
86
|
-
if (key
|
|
113
|
+
let sawLegacy = false;
|
|
114
|
+
const out: Record<string, unknown> = {};
|
|
115
|
+
for (const [key, value] of Object.entries(traits)) {
|
|
116
|
+
if (key === "refined_at" || key === "refined_provider") {
|
|
117
|
+
sawLegacy = true;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const canonical = LEGACY_TRAIT_KEYS[key];
|
|
121
|
+
if (canonical) {
|
|
122
|
+
sawLegacy = true;
|
|
123
|
+
out[canonical] = value;
|
|
124
|
+
} else {
|
|
125
|
+
out[key] = value;
|
|
126
|
+
}
|
|
87
127
|
}
|
|
88
|
-
return
|
|
128
|
+
return sawLegacy ? out : traits;
|
|
89
129
|
}
|
|
90
130
|
|
|
131
|
+
/** Pre-canonical `refined_*` fact key → its canonical replacement. */
|
|
132
|
+
const LEGACY_TRAIT_KEYS: Record<string, string> = {
|
|
133
|
+
refined_title: "title",
|
|
134
|
+
refined_seniority: "seniority",
|
|
135
|
+
refined_department: "department",
|
|
136
|
+
refined_linkedin_url: "linkedin_url",
|
|
137
|
+
refined_country: "country",
|
|
138
|
+
refined_company_name: "company",
|
|
139
|
+
refined_company_domain: "company_domain",
|
|
140
|
+
refined_company_industry: "company_industry",
|
|
141
|
+
refined_company_employees: "company_employees",
|
|
142
|
+
refined_company_revenue: "company_revenue",
|
|
143
|
+
refined_company_country: "company_country",
|
|
144
|
+
};
|
|
145
|
+
|
|
91
146
|
/**
|
|
92
147
|
* Write a string trait, omitting anything that carries no information. An empty
|
|
93
148
|
* (or whitespace-only) string is dropped for the same reason a null is: it is
|
|
@@ -108,8 +163,8 @@ function putString(
|
|
|
108
163
|
* Write a numeric trait as a real JSON number. A provider is contractually
|
|
109
164
|
* obliged to normalize to `number`, but a JS (untyped) provider can still hand
|
|
110
165
|
* back a numeric string — coercing here is the difference between a working
|
|
111
|
-
* `b.prop("
|
|
112
|
-
*
|
|
166
|
+
* `b.prop("company_employees").gte(100)` and one that silently never matches.
|
|
167
|
+
* Non-finite / non-numeric values are omitted entirely.
|
|
113
168
|
*/
|
|
114
169
|
function putNumber(
|
|
115
170
|
patch: Record<string, unknown>,
|