@hogsend/engine 0.55.0 → 0.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/package.json +11 -11
  2. package/src/buckets/bucket-access.ts +5 -10
  3. package/src/buckets/check-membership.ts +91 -6
  4. package/src/index.ts +31 -1
  5. package/src/journeys/execute-journey-run.ts +56 -1
  6. package/src/journeys/journey-context.ts +59 -0
  7. package/src/lib/bucket-emit.test.ts +83 -2
  8. package/src/lib/bucket-emit.ts +16 -0
  9. package/src/lib/contacts.ts +1031 -141
  10. package/src/lib/crm-ingest.ts +9 -0
  11. package/src/lib/enrichment-ledger.ts +1 -1
  12. package/src/lib/feed.ts +64 -3
  13. package/src/lib/identity-service.ts +15 -2
  14. package/src/lib/ingestion.ts +191 -14
  15. package/src/lib/outbound.ts +3 -3
  16. package/src/lib/refine-chain.test.ts +83 -25
  17. package/src/lib/refine-chain.ts +124 -23
  18. package/src/lib/refine-traits.test.ts +74 -41
  19. package/src/lib/refine-traits.ts +107 -52
  20. package/src/lib/refine.ts +28 -27
  21. package/src/lib/tracking-events.ts +33 -6
  22. package/src/routes/admin/agent.ts +26 -0
  23. package/src/routes/admin/contacts.ts +65 -5
  24. package/src/routes/admin/identity.ts +197 -0
  25. package/src/routes/admin/index.ts +2 -0
  26. package/src/routes/contacts/index.ts +50 -7
  27. package/src/routes/events/index.ts +94 -6
  28. package/src/routes/feed/index.ts +36 -0
  29. package/src/routes/feed/recipient.ts +44 -2
  30. package/src/routes/lists/index.ts +21 -0
  31. package/src/routes/tracking/arrive.ts +42 -3
  32. package/src/worker.ts +18 -0
  33. package/src/workflows/bucket-backfill.ts +1 -1
  34. package/src/workflows/bucket-reconcile.ts +129 -4
  35. package/src/workflows/identity-alias-backfill.ts +448 -0
  36. package/src/workflows/import-contacts.ts +10 -0
  37. package/src/workflows/send-campaign.ts +31 -13
@@ -93,6 +93,10 @@ interface Harness {
93
93
  ingests: number;
94
94
  /** Every outbound `contact.refined` emit, in order. */
95
95
  emits: RefineEmitInput[];
96
+ /** The `contactProperties` each `ingest` received, in order. */
97
+ ingested: Record<string, unknown>[];
98
+ /** The `touchProperties` each `ingest` received, in order. */
99
+ touched: (string[] | undefined)[];
96
100
  }
97
101
 
98
102
  function harness(overrides: {
@@ -116,6 +120,8 @@ function harness(overrides: {
116
120
  ledgerWrites: 0,
117
121
  ingests: 0,
118
122
  emits: [] as RefineEmitInput[],
123
+ ingested: [] as Record<string, unknown>[],
124
+ touched: [] as (string[] | undefined)[],
119
125
  };
120
126
 
121
127
  const deps: RefineChainDeps = {
@@ -135,15 +141,16 @@ function harness(overrides: {
135
141
  contactId: "c1",
136
142
  userId: "u1",
137
143
  email: "a@acme.com",
138
- refinedProperties: { refined_title: "old" },
139
144
  },
140
145
  findLedgerRow: async () => overrides.ledgerRow ?? null,
141
146
  countLookupsSince: async () => overrides.usedThisMonth ?? 0,
142
147
  writeLedgerRow: async () => {
143
148
  state.ledgerWrites += 1;
144
149
  },
145
- ingest: async () => {
150
+ ingest: async (input) => {
146
151
  state.ingests += 1;
152
+ state.ingested.push(input.contactProperties);
153
+ state.touched.push(input.touchProperties);
147
154
  if (overrides.ingest) await overrides.ingest();
148
155
  },
149
156
  emitRefined: (input) => {
@@ -163,6 +170,12 @@ function harness(overrides: {
163
170
  get emits() {
164
171
  return state.emits;
165
172
  },
173
+ get ingested() {
174
+ return state.ingested;
175
+ },
176
+ get touched() {
177
+ return state.touched;
178
+ },
166
179
  };
167
180
  }
168
181
 
@@ -470,7 +483,7 @@ test("D2: a cache hit lands the STORED patch on a contact that never received it
470
483
  // The shared-domain case: contact B hits the row contact A paid for. The
471
484
  // ledger has no contact dimension, so "already paid for" says nothing about
472
485
  // whether THIS contact has the traits.
473
- const stored = { refined_title: "CTO", refined_company_employees: 250 };
486
+ const stored = { title: "CTO", company_employees: 250 };
474
487
  const h = harness({
475
488
  ledgerRow: unexpired("found", stored),
476
489
  target: { contactId: "c2", userId: "u2", domain: "acme.com" },
@@ -491,14 +504,14 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
491
504
  // buckets: only the ingest path re-runs `checkBucketMembership`, and a
492
505
  // contact can carry the traits with no membership (exactly what a failed
493
506
  // ingest leaves behind). So the cached verdict always re-lands.
494
- const stored = { refined_title: "CTO", refined_company_employees: 250 };
507
+ const stored = { title: "CTO", company_employees: 250 };
495
508
  const h = harness({
496
509
  ledgerRow: unexpired("found", stored),
497
510
  target: {
498
511
  contactId: "c1",
499
512
  userId: "u1",
500
513
  domain: "acme.com",
501
- refinedProperties: { ...stored },
514
+ existingProperties: { ...stored },
502
515
  },
503
516
  });
504
517
 
@@ -506,23 +519,74 @@ test("D2: a cache hit re-runs the ingest even when the contact already holds the
506
519
 
507
520
  assert.deepEqual(result, { status: "cached", properties: stored });
508
521
  assert.equal(h.ingests, 1);
522
+ // The contact already held every fact, so the WRITE patch is empty (nothing
523
+ // first-party is clobbered) — but the ingest still runs and TOUCHES both keys,
524
+ // so bucket membership re-evaluates against live contact state.
525
+ assert.deepEqual(h.ingested, [{}]);
526
+ assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
527
+ "company_employees",
528
+ "title",
529
+ ]);
509
530
  assert.equal(h.providerCalls(), 0);
510
531
  assert.equal(h.ledgerWrites, 0);
511
532
  });
512
533
 
513
- test("D2: a legacy row with no stored patch falls back to the caller's own traits", async () => {
534
+ test("D2: a legacy row with no stored patch returns cached with an empty patch", async () => {
514
535
  const h = harness({ ledgerRow: unexpired("found", null) });
515
536
 
516
537
  const result = await runRefineChain(h.deps, SUBJECT);
517
538
 
518
539
  assert.deepEqual(result, {
519
540
  status: "cached",
520
- properties: { refined_title: "old" },
541
+ properties: {},
521
542
  });
522
543
  assert.equal(h.ingests, 0);
523
544
  assert.equal(h.providerCalls(), 0);
524
545
  });
525
546
 
547
+ // ---------------------------------------------------------------------------
548
+ // Fill-if-absent — a paid answer fills gaps, never overwrites first-party data
549
+ // ---------------------------------------------------------------------------
550
+
551
+ test("fill-if-absent: a fresh lookup fills gaps but never overwrites a first-party value", async () => {
552
+ // FOUND carries title / seniority / company / company_employees. This contact
553
+ // already has a first-party `title` and `company`, so the paid answer must
554
+ // leave those two alone and only fill what was missing.
555
+ const h = harness({
556
+ target: {
557
+ contactId: "c1",
558
+ userId: "u1",
559
+ email: "a@acme.com",
560
+ existingProperties: { title: "First-party VP", company: "First Co" },
561
+ },
562
+ });
563
+
564
+ const result = await runRefineChain(h.deps, SUBJECT);
565
+
566
+ assert.equal(result.status, "refined");
567
+ // The vendor's FULL answer is still reported on the result...
568
+ assert.equal(result.properties?.title, "CTO");
569
+ // ...but the WRITE patch OMITS the already-held title + company entirely (so
570
+ // the additive merge leaves the live first-party value untouched — atomic, no
571
+ // stale write-back) and carries the vendor's value only for the gaps.
572
+ const landed = h.ingested[0] ?? {};
573
+ assert.equal("title" in landed, false);
574
+ assert.equal("company" in landed, false);
575
+ assert.equal(landed.seniority, "c_suite");
576
+ assert.equal(landed.company_employees, 250);
577
+ assert.ok(landed.enrichment, "provenance always lands");
578
+ // ...yet every returned fact is TOUCHED, so a fit bucket on the already-held
579
+ // title/company still re-evaluates against live contact state.
580
+ assert.deepEqual([...(h.touched[0] ?? [])].sort(), [
581
+ "company",
582
+ "company_employees",
583
+ "seniority",
584
+ "title",
585
+ ]);
586
+ // The outbound reports only the facts that actually CHANGED on this contact.
587
+ assert.deepEqual(h.emits[0]?.traitKeys, ["company_employees", "seniority"]);
588
+ });
589
+
526
590
  // ---------------------------------------------------------------------------
527
591
  // D4b — a failed ingest is a verdict, not an exception
528
592
  // ---------------------------------------------------------------------------
@@ -543,7 +607,7 @@ test("D4b: a failed ingest does not throw out of the chain and does not report s
543
607
  });
544
608
 
545
609
  test("D4b: the retry after a failed ingest re-lands the stored patch for FREE", async () => {
546
- const stored = { refined_title: "CTO" };
610
+ const stored = { title: "CTO" };
547
611
  const h = harness({ ledgerRow: unexpired("found", stored) });
548
612
 
549
613
  const result = await runRefineChain(h.deps, SUBJECT);
@@ -558,16 +622,11 @@ test("D4b: the retry after a failed ingest re-lands the stored patch for FREE",
558
622
  // ---------------------------------------------------------------------------
559
623
 
560
624
  /**
561
- * The sorted trait key names FOUND flattens to — NAMES, never values, and
562
- * VENDOR facts only: `refined_at` / `refined_provider` are provenance, already
563
- * carried as the envelope's `at` / `provider`, so they are not traits.
625
+ * The sorted canonical trait key names FOUND flattens to — NAMES, never values,
626
+ * and FACTS only: the nested `enrichment` provenance is already carried as the
627
+ * envelope's `at` / `provider`, so it is not a trait.
564
628
  */
565
- const FOUND_TRAIT_KEYS = [
566
- "refined_company_employees",
567
- "refined_company_name",
568
- "refined_seniority",
569
- "refined_title",
570
- ];
629
+ const FOUND_TRAIT_KEYS = ["company", "company_employees", "seniority", "title"];
571
630
 
572
631
  /** NOW as the dedupe key serializes it. */
573
632
  const NOW_ISO = NOW.toISOString();
@@ -595,14 +654,13 @@ test("AC 1: a `refined` verdict emits contact.refined with the contact key, prov
595
654
  assert.ok(!JSON.stringify(emitted.traitKeys).includes("CTO"));
596
655
  assert.ok(!JSON.stringify(emitted.traitKeys).includes("Acme"));
597
656
  // Provenance is NOT a trait — it rides as `provider` / `refinedAt` instead.
598
- assert.ok(!emitted.traitKeys.includes("refined_at"));
599
- assert.ok(!emitted.traitKeys.includes("refined_provider"));
657
+ assert.ok(!emitted.traitKeys.includes("enrichment"));
600
658
  });
601
659
 
602
- test("AC 1: a found-but-empty vendor answer reports NO traits rather than two metadata keys", async () => {
603
- // `flattenTraits` stamps refined_at/refined_provider unconditionally, so if
604
- // provenance counted as a trait this payload would claim two changes for an
605
- // answer that told us nothing about the person.
660
+ test("AC 1: a found-but-empty vendor answer reports NO traits rather than a metadata key", async () => {
661
+ // `flattenTraits` stamps the `enrichment` provenance unconditionally, so if it
662
+ // counted as a trait this payload would claim a change for an answer that told
663
+ // us nothing about the person.
606
664
  const empty: EnrichmentProvider = {
607
665
  meta: { id: "fake", name: "Fake" },
608
666
  capabilities: { personLookup: true, companyLookup: true },
@@ -627,7 +685,7 @@ test("AC 1: a domain-keyed refinement emits with the domain lookup in its dedupe
627
685
 
628
686
  test("AC 2: `cached`, `not_found` and `skipped` emit NOTHING — a cache hit is not an event", async () => {
629
687
  const cached = harness({
630
- ledgerRow: unexpired("found", { refined_title: "CTO" }),
688
+ ledgerRow: unexpired("found", { title: "CTO" }),
631
689
  });
632
690
  const notFoundCached = harness({ ledgerRow: unexpired("not_found") });
633
691
  const notFoundLive = harness({ provider: fakeProvider("miss").provider });
@@ -728,7 +786,7 @@ test("AC 3: a re-drive never reaches the emit anyway — the LEDGER GATE returns
728
786
  // wrote and returns `cached` (refine-chain.ts step 3), which has no emit at
729
787
  // all — so a duplicate delivery needs a concurrent race, not a retry.
730
788
  const redrive = harness({
731
- ledgerRow: unexpired("found", { refined_title: "CTO" }),
789
+ ledgerRow: unexpired("found", { title: "CTO" }),
732
790
  });
733
791
 
734
792
  assert.equal((await runRefineChain(redrive.deps, SUBJECT)).status, "cached");
@@ -12,7 +12,11 @@ import type {
12
12
  EnrichmentLookupKind,
13
13
  EnrichmentLookupStatus,
14
14
  } from "./enrichment-ledger.js";
15
- import { flattenTraits, REFINED_META_KEYS } from "./refine-traits.js";
15
+ import {
16
+ canonicalizeStoredTraits,
17
+ ENRICHMENT_KEY,
18
+ flattenTraits,
19
+ } from "./refine-traits.js";
16
20
 
17
21
  // ---------------------------------------------------------------------------
18
22
  // Public surface
@@ -42,7 +46,12 @@ export interface RefineContactResult {
42
46
  * | `provider_error` | `ingest_failed`.
43
47
  */
44
48
  reason?: string;
45
- /** The `refined_*` patch — on `refined` AND on `cached`. */
49
+ /**
50
+ * The vendor's FULL canonical trait patch (facts + the nested `enrichment`
51
+ * provenance) — on `refined` AND on `cached`. This is what the vendor
52
+ * answered, NOT the fill-if-absent subset that actually landed on any one
53
+ * contact.
54
+ */
46
55
  properties?: Record<string, unknown>;
47
56
  }
48
57
 
@@ -58,8 +67,13 @@ export interface RefineTarget {
58
67
  firstName?: string;
59
68
  lastName?: string;
60
69
  company?: string;
61
- /** The `refined_*` traits already on THIS contact. */
62
- refinedProperties?: Record<string, unknown>;
70
+ /**
71
+ * The properties already on THIS contact's row, used by fill-if-absent to
72
+ * decide which vendor facts to keep (a fact the contact already holds from a
73
+ * first-party source is never overwritten). Absent for a brand-new email with
74
+ * no row yet — nothing to preserve, so every vendor fact lands.
75
+ */
76
+ existingProperties?: Record<string, unknown>;
63
77
  }
64
78
 
65
79
  /** The event the refinement pipeline emits through `ingestEvent`. */
@@ -71,6 +85,12 @@ export interface RefineIngestInput {
71
85
  contactId?: string;
72
86
  eventProperties: Record<string, unknown>;
73
87
  contactProperties: Record<string, unknown>;
88
+ /**
89
+ * Fact key NAMES to re-evaluate buckets for even when their value is not in
90
+ * `contactProperties` (fill-if-absent dropped it). Candidate-narrowing only —
91
+ * never written, never in value eval. See {@link reconcileTraits}.
92
+ */
93
+ touchProperties?: string[];
74
94
  idempotencyKey?: string;
75
95
  }
76
96
 
@@ -88,9 +108,10 @@ export interface RefineEmitInput {
88
108
  contactId?: string;
89
109
  /** The enrichment provider id that answered. */
90
110
  provider: string;
91
- /** Sorted key NAMES of the `refined_*` patch. */
111
+ /** Sorted canonical FACT keys actually filled on this contact (never the
112
+ * nested `enrichment` provenance key). */
92
113
  traitKeys: string[];
93
- /** The lookup instant — the same value the patch's `refined_at` carries. */
114
+ /** The lookup instant — the same value the patch's `enrichment.at` carries. */
94
115
  refinedAt: Date;
95
116
  /** Per-endpoint delivery dedupe — see {@link refineOutboundDedupeKey}. */
96
117
  dedupeKey: string;
@@ -100,7 +121,11 @@ export interface RefineEmitInput {
100
121
  export interface RefineLedgerRow {
101
122
  status: EnrichmentLookupStatus;
102
123
  expiresAt: Date;
103
- /** The normalized `refined_*` patch the paid answer produced, if stored. */
124
+ /**
125
+ * The normalized canonical trait patch the paid answer produced, if stored —
126
+ * the vendor's FULL answer, so a cache hit can land it (fill-if-absent) on a
127
+ * DIFFERENT contact that shares the lookup key.
128
+ */
104
129
  traits?: Record<string, unknown> | null;
105
130
  }
106
131
 
@@ -403,10 +428,21 @@ async function runGates(args: {
403
428
  spendWindow: deps.budgetWindowStart(refinedAt),
404
429
  });
405
430
 
431
+ // Fill-if-absent: the vendor's facts fill only the canonical fields THIS
432
+ // contact does not already hold from a first-party source — a paid lookup
433
+ // never clobbers truer data the app already set — while every returned fact
434
+ // stays in the patch so the fit bucket re-evaluates. Computed here, per
435
+ // contact, against the row `resolveTarget` read.
436
+ const { landing, filled, touch } = reconcileTraits(
437
+ properties,
438
+ target.existingProperties,
439
+ );
440
+
406
441
  const landed = await landTraits({
407
442
  deps,
408
443
  target,
409
- properties,
444
+ properties: landing,
445
+ touch,
410
446
  providerId,
411
447
  found: result.found,
412
448
  cached: false,
@@ -432,16 +468,14 @@ async function runGates(args: {
432
468
  ...(target.email ? { email: target.email } : {}),
433
469
  ...(target.contactId ? { contactId: target.contactId } : {}),
434
470
  provider: providerId,
435
- // VENDOR facts only — `refined_at` / `refined_provider` are provenance and
436
- // are dropped. Two reasons, both deliberate: the envelope already carries
437
- // both as top-level `at` / `provider`, so keeping them here would say the
438
- // same thing twice; and `flattenTraits` stamps them UNCONDITIONALLY, so a
439
- // found-but-empty vendor answer would otherwise report two changed traits
440
- // when nothing about the person changed. `traits: []` now says that
441
- // honestly.
442
- traitKeys: Object.keys(properties)
443
- .filter((key) => !REFINED_META_KEYS.includes(key))
444
- .sort(),
471
+ // The FACTS that actually landed on this contact — the fill-if-absent set
472
+ // minus the nested `enrichment` provenance key. Two deliberate consequences:
473
+ // the envelope already carries the provenance as top-level `at` / `provider`
474
+ // so keeping the `enrichment` key here would say it twice; and a fact the
475
+ // contact already held (not overwritten) is not reported as changed, so a
476
+ // subscriber hears only what genuinely changed — `traits: []` when the
477
+ // answer added nothing new.
478
+ traitKeys: [...filled].sort(),
445
479
  refinedAt,
446
480
  dedupeKey: refineOutboundDedupeKey({
447
481
  provider: providerId,
@@ -470,8 +504,10 @@ async function runGates(args: {
470
504
  * membership was never evaluated — which is precisely the state a failed ingest
471
505
  * leaves behind. That makes the retry after an `ingest_failed` free.
472
506
  *
473
- * Rows written before the `traits` column existed have nothing stored; those
474
- * fall back to the caller's own traits, which is exactly the old behaviour.
507
+ * The stored patch is the vendor's FULL answer, so fill-if-absent runs here too
508
+ * — the second contact at a company keeps its own first-party fields and only
509
+ * gains what it was missing. Rows written before the `traits` column existed
510
+ * have nothing stored and simply return `cached` with an empty patch.
475
511
  */
476
512
  async function landCachedTraits(args: {
477
513
  deps: RefineChainDeps;
@@ -481,15 +517,19 @@ async function landCachedTraits(args: {
481
517
  idempotencyKey?: string;
482
518
  }): Promise<RefineContactResult> {
483
519
  const { deps, target, row } = args;
484
- const stored = row.traits ?? undefined;
520
+ // A row written by the pre-canonical release stores flat `refined_*` traits;
521
+ // translate them so a cache hit lands the canonical shape, not the old one.
522
+ const stored = row.traits ? canonicalizeStoredTraits(row.traits) : undefined;
485
523
  if (!stored) {
486
- return { status: "cached", properties: target.refinedProperties ?? {} };
524
+ return { status: "cached", properties: {} };
487
525
  }
488
526
 
527
+ const { landing, touch } = reconcileTraits(stored, target.existingProperties);
489
528
  const landed = await landTraits({
490
529
  deps,
491
530
  target,
492
- properties: stored,
531
+ properties: landing,
532
+ touch,
493
533
  providerId: args.providerId,
494
534
  found: true,
495
535
  cached: true,
@@ -517,6 +557,7 @@ async function landTraits(args: {
517
557
  deps: RefineChainDeps;
518
558
  target: RefineTarget;
519
559
  properties: Record<string, unknown>;
560
+ touch: string[];
520
561
  providerId: string;
521
562
  found: boolean;
522
563
  cached: boolean;
@@ -534,6 +575,7 @@ async function landTraits(args: {
534
575
  cached: args.cached,
535
576
  },
536
577
  contactProperties: args.properties,
578
+ ...(args.touch.length > 0 ? { touchProperties: args.touch } : {}),
537
579
  ...(args.idempotencyKey ? { idempotencyKey: args.idempotencyKey } : {}),
538
580
  });
539
581
  return true;
@@ -547,6 +589,65 @@ async function landTraits(args: {
547
589
  }
548
590
  }
549
591
 
592
+ /**
593
+ * Reconcile the vendor's patch with what the contact already holds — fill-if-
594
+ * absent, split into three outputs because "what to WRITE" and "what to
595
+ * re-evaluate buckets for" are different questions.
596
+ *
597
+ * A paid answer must never overwrite a first-party value the app set. The
598
+ * ATOMIC way to guarantee that is to leave the already-held keys OUT of the
599
+ * write patch entirely: `mergePropertiesSql` is additive (`existing || patch`),
600
+ * so a key absent from the patch is left exactly as the live row has it — no
601
+ * read-modify-write, no snapshot to go stale, no chance of reverting a value a
602
+ * concurrent identify wrote during the (slow) vendor call. `existingProperties`
603
+ * is a pre-call snapshot, so this MUST NOT write a value back from it.
604
+ *
605
+ * But dropping a key hides it from `checkBucketMembership`, which narrows its
606
+ * candidate buckets to the property NAMES present in the ingest — so a dropped
607
+ * fact would never re-evaluate its fit bucket, and a torn write (properties
608
+ * persisted, membership not yet formed) would never reconcile. So the dropped
609
+ * keys still travel, separately, as `touch`: names only, no values, fed to the
610
+ * bucket candidate-narrowing (never to value eval, which reads the live row).
611
+ *
612
+ * Returns:
613
+ * - `landing`: the write patch — the `enrichment` provenance (always) plus only
614
+ * the facts the contact was MISSING.
615
+ * - `filled`: the fact keys actually taken from the vendor (drives the outbound
616
+ * `traitKeys` — a subscriber hears only what genuinely changed).
617
+ * - `touch`: EVERY fact key the vendor returned, so its bucket re-evaluates
618
+ * even when the value was already present.
619
+ */
620
+ function reconcileTraits(
621
+ patch: Record<string, unknown>,
622
+ existing: Record<string, unknown> | undefined,
623
+ ): { landing: Record<string, unknown>; filled: string[]; touch: string[] } {
624
+ const landing: Record<string, unknown> = {};
625
+ const filled: string[] = [];
626
+ const touch: string[] = [];
627
+ for (const [key, value] of Object.entries(patch)) {
628
+ if (key === ENRICHMENT_KEY) {
629
+ landing[key] = value;
630
+ continue;
631
+ }
632
+ touch.push(key);
633
+ if (hasValue(existing?.[key])) continue; // already held — preserve, atomically
634
+ landing[key] = value;
635
+ filled.push(key);
636
+ }
637
+ return { landing, filled, touch };
638
+ }
639
+
640
+ /**
641
+ * A property is "already set" for fill-if-absent when it holds a real value: not
642
+ * undefined, not null, not an empty / whitespace-only string. A numeric 0 or a
643
+ * `false` COUNTS as set — it is a value the app chose, not an absence.
644
+ */
645
+ function hasValue(value: unknown): boolean {
646
+ if (value === undefined || value === null) return false;
647
+ if (typeof value === "string") return value.trim() !== "";
648
+ return true;
649
+ }
650
+
550
651
  /**
551
652
  * The per-endpoint delivery dedupe key for `contact.refined`.
552
653
  *
@@ -1,12 +1,13 @@
1
1
  import assert from "node:assert/strict";
2
2
  import test from "node:test";
3
3
  import type { EnrichmentResult } from "@hogsend/core";
4
- import { flattenTraits, pickRefinedTraits } from "./refine-traits.js";
4
+ import { canonicalizeStoredTraits, flattenTraits } from "./refine-traits.js";
5
5
 
6
6
  const REFINED_AT = new Date("2026-07-25T10:00:00.000Z");
7
7
  const META = { provider: "fake", refinedAt: REFINED_AT };
8
+ const ENRICHMENT = { provider: "fake", at: "2026-07-25T10:00:00.000Z" };
8
9
 
9
- test("flattenTraits maps every documented field to a flat refined_* key", () => {
10
+ test("flattenTraits maps every documented field to a canonical key", () => {
10
11
  const result: EnrichmentResult = {
11
12
  found: true,
12
13
  person: {
@@ -28,22 +29,52 @@ test("flattenTraits maps every documented field to a flat refined_* key", () =>
28
29
  };
29
30
 
30
31
  assert.deepEqual(flattenTraits(result, META), {
31
- refined_title: "VP of Engineering",
32
- refined_seniority: "vp",
33
- refined_department: "engineering",
34
- refined_linkedin_url: "https://linkedin.com/in/x",
35
- refined_country: "US",
36
- refined_company_name: "Acme",
37
- refined_company_domain: "acme.com",
38
- refined_company_industry: "software",
39
- refined_company_employees: 250,
40
- refined_company_revenue: 12_000_000,
41
- refined_company_country: "US",
42
- refined_at: "2026-07-25T10:00:00.000Z",
43
- refined_provider: "fake",
32
+ title: "VP of Engineering",
33
+ seniority: "vp",
34
+ department: "engineering",
35
+ linkedin_url: "https://linkedin.com/in/x",
36
+ country: "US",
37
+ company: "Acme",
38
+ company_domain: "acme.com",
39
+ company_industry: "software",
40
+ company_employees: 250,
41
+ company_revenue: 12_000_000,
42
+ company_country: "US",
43
+ enrichment: ENRICHMENT,
44
44
  });
45
45
  });
46
46
 
47
+ // The facts are FLAT top-level keys, never nested: `evaluatePropertyConditions`
48
+ // does `properties[condition.property]` with no dotted-path traversal, so a
49
+ // bucket must be able to author `b.prop("company_industry")` verbatim.
50
+ test("facts are flat top-level keys; only provenance nests under `enrichment`", () => {
51
+ const patch = flattenTraits(
52
+ {
53
+ found: true,
54
+ person: { seniority: "vp" },
55
+ company: { industry: "software", employeeCount: 250 },
56
+ raw: {},
57
+ },
58
+ META,
59
+ );
60
+
61
+ assert.equal(patch.seniority, "vp");
62
+ assert.equal(patch.company_industry, "software");
63
+ assert.equal(patch.company_employees, 250);
64
+ // No parallel `refined_*` vocabulary, no dotted keys.
65
+ for (const key of Object.keys(patch)) {
66
+ assert.ok(
67
+ !key.startsWith("refined_"),
68
+ `${key} must not be refined_-prefixed`,
69
+ );
70
+ assert.ok(
71
+ key === "enrichment" || !key.includes("."),
72
+ `${key} must be flat`,
73
+ );
74
+ }
75
+ assert.deepEqual(patch.enrichment, ENRICHMENT);
76
+ });
77
+
47
78
  // AC 8 — the numeric traits must be REAL JSON numbers. `conditions/property.ts`
48
79
  // does no coercion, so a `"250"` would silently never match a `gte(100)`.
49
80
  test("AC 8: employee count is a real JSON number, and survives JSON round-trip", () => {
@@ -52,12 +83,12 @@ test("AC 8: employee count is a real JSON number, and survives JSON round-trip",
52
83
  META,
53
84
  );
54
85
 
55
- assert.equal(typeof patch.refined_company_employees, "number");
56
- assert.equal(patch.refined_company_employees, 250);
86
+ assert.equal(typeof patch.company_employees, "number");
87
+ assert.equal(patch.company_employees, 250);
57
88
  // jsonb storage is a JSON round-trip — the type must survive it.
58
89
  const roundTripped = JSON.parse(JSON.stringify(patch));
59
- assert.equal(typeof roundTripped.refined_company_employees, "number");
60
- assert.ok((roundTripped.refined_company_employees as number) >= 100);
90
+ assert.equal(typeof roundTripped.company_employees, "number");
91
+ assert.ok((roundTripped.company_employees as number) >= 100);
61
92
  });
62
93
 
63
94
  test("AC 8: a numeric STRING from an untyped provider is coerced to a number", () => {
@@ -72,8 +103,8 @@ test("AC 8: a numeric STRING from an untyped provider is coerced to a number", (
72
103
  META,
73
104
  );
74
105
 
75
- assert.equal(patch.refined_company_employees, 250);
76
- assert.equal(typeof patch.refined_company_employees, "number");
106
+ assert.equal(patch.company_employees, 250);
107
+ assert.equal(typeof patch.company_employees, "number");
77
108
  });
78
109
 
79
110
  // AC 9 — an absent field must be OMITTED, never written as null: the patch is
@@ -84,13 +115,9 @@ test("AC 9: absent fields are omitted from the patch entirely (never null)", ()
84
115
  META,
85
116
  );
86
117
 
87
- assert.deepEqual(Object.keys(patch).sort(), [
88
- "refined_at",
89
- "refined_provider",
90
- "refined_title",
91
- ]);
92
- assert.equal("refined_seniority" in patch, false);
93
- assert.equal("refined_company_name" in patch, false);
118
+ assert.deepEqual(Object.keys(patch).sort(), ["enrichment", "title"]);
119
+ assert.equal("seniority" in patch, false);
120
+ assert.equal("company" in patch, false);
94
121
  for (const value of Object.values(patch)) {
95
122
  assert.notEqual(value, null);
96
123
  }
@@ -114,28 +141,34 @@ test("AC 9: null / empty-string / non-finite fields are omitted, not written", (
114
141
  META,
115
142
  );
116
143
 
117
- assert.deepEqual(Object.keys(patch).sort(), [
118
- "refined_at",
119
- "refined_department",
120
- "refined_provider",
121
- ]);
144
+ assert.deepEqual(Object.keys(patch).sort(), ["department", "enrichment"]);
122
145
  });
123
146
 
124
- test("a not-found result still stamps only the two synthetic traits", () => {
147
+ test("a not-found result still stamps only the provenance object", () => {
125
148
  assert.deepEqual(flattenTraits({ found: false, raw: null }, META), {
126
- refined_at: "2026-07-25T10:00:00.000Z",
127
- refined_provider: "fake",
149
+ enrichment: ENRICHMENT,
128
150
  });
129
151
  });
130
152
 
131
- test("pickRefinedTraits returns only the refined_* subset", () => {
153
+ test("canonicalizeStoredTraits translates a pre-canonical (refined_*) ledger patch", () => {
132
154
  assert.deepEqual(
133
- pickRefinedTraits({
134
- plan: "pro",
155
+ canonicalizeStoredTraits({
135
156
  refined_title: "CTO",
157
+ refined_company_name: "Acme",
136
158
  refined_company_employees: 250,
159
+ refined_at: "2026-01-01T00:00:00.000Z",
160
+ refined_provider: "apollo",
137
161
  }),
138
- { refined_title: "CTO", refined_company_employees: 250 },
162
+ { title: "CTO", company: "Acme", company_employees: 250 },
139
163
  );
140
- assert.deepEqual(pickRefinedTraits(undefined), {});
164
+ });
165
+
166
+ test("canonicalizeStoredTraits leaves an already-canonical patch untouched (by reference)", () => {
167
+ const canonical = {
168
+ title: "CTO",
169
+ company_employees: 250,
170
+ enrichment: ENRICHMENT,
171
+ };
172
+ // No legacy key → returns the SAME object, no needless copy.
173
+ assert.equal(canonicalizeStoredTraits(canonical), canonical);
141
174
  });