@pentatonic-ai/ai-agent-sdk 0.10.7 → 0.10.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. package/dist/index.cjs +1 -1
  2. package/dist/index.js +1 -1
  3. package/package.json +1 -1
  4. package/packages/memory-engine-v2/RFC-decay-and-fusion.md +185 -0
  5. package/packages/memory-engine-v2/RFC-fusion-drive.md +199 -0
  6. package/packages/memory-engine-v2/extractor-async/confidence.py +37 -0
  7. package/packages/memory-engine-v2/extractor-async/source_time.py +63 -0
  8. package/packages/memory-engine-v2/extractor-async/test_born_salience_parity.py +35 -0
  9. package/packages/memory-engine-v2/extractor-async/test_source_time.py +102 -0
  10. package/packages/memory-engine-v2/extractor-async/worker.py +121 -18
  11. package/packages/memory-engine-v2/extractor-sync/Dockerfile +3 -1
  12. package/packages/memory-engine-v2/extractor-sync/confidence.py +99 -0
  13. package/packages/memory-engine-v2/extractor-sync/server.py +61 -11
  14. package/packages/memory-engine-v2/extractor-sync/source_time.py +63 -0
  15. package/packages/memory-engine-v2/extractor-sync/test_confidence_parity.py +18 -0
  16. package/packages/memory-engine-v2/extractor-sync/test_paired_extraction.py +2 -2
  17. package/packages/memory-engine-v2/fusion_drive/__init__.py +0 -0
  18. package/packages/memory-engine-v2/fusion_drive/adjudicate.py +85 -0
  19. package/packages/memory-engine-v2/fusion_drive/canonical.py +94 -0
  20. package/packages/memory-engine-v2/fusion_drive/conftest.py +8 -0
  21. package/packages/memory-engine-v2/fusion_drive/merge.py +178 -0
  22. package/packages/memory-engine-v2/fusion_drive/salience.py +118 -0
  23. package/packages/memory-engine-v2/fusion_drive/test_adjudicate.py +65 -0
  24. package/packages/memory-engine-v2/fusion_drive/test_canonical.py +76 -0
  25. package/packages/memory-engine-v2/fusion_drive/test_merge.py +112 -0
  26. package/packages/memory-engine-v2/fusion_drive/test_salience.py +93 -0
  27. package/packages/memory-engine-v2/org-model/migrations/006_fusion_drive.sql +80 -0
  28. package/packages/memory-engine-v2/scripts/fusion_drive_born_salience_backfill.py +113 -0
  29. package/packages/memory-engine-v2/scripts/fusion_drive_decay.py +200 -0
  30. package/packages/memory-engine-v2/scripts/fusion_drive_fuse.py +434 -0
package/dist/index.cjs CHANGED
@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
878
878
  }
879
879
 
880
880
  // src/telemetry.js
881
- var VERSION = "0.10.7";
881
+ var VERSION = "0.10.9";
882
882
  var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
883
883
  function machineId() {
884
884
  const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";
package/dist/index.js CHANGED
@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
847
847
  }
848
848
 
849
849
  // src/telemetry.js
850
- var VERSION = "0.10.7";
850
+ var VERSION = "0.10.9";
851
851
  var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
852
852
  function machineId() {
853
853
  const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pentatonic-ai/ai-agent-sdk",
3
- "version": "0.10.7",
3
+ "version": "0.10.9",
4
4
  "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.cjs",
@@ -0,0 +1,185 @@
1
+ # RFC: the Fusion Drive — v2 memory self-healing (cross-run node fusion + decay)
2
+
3
+ > **Fusion Drive** = the continuous, arena-scoped background engine that keeps the v2
4
+ > memory graph self-healing: it *fuses* duplicate/near-duplicate nodes from different
5
+ > distillation runs into a single master node (horizontal convergence) and *decays* stale,
6
+ > low-value, and junk nodes out of existence (vertical aging). Named for the drive that
7
+ > does the fusing — the decay pass rides the same engine.
8
+
9
+ **Status:** draft / spec — 2026-06-12
10
+ **Builds on:** `RFC-entity-reconciliation.md`, `scripts/entity_resolution_v2.py` (#82),
11
+ `org-model/migrations/002_entity_merges_audit.sql`.
12
+ **Motivated by:** the v2 store is currently **pure-accretion** — three independent
13
+ properties, all verified in code, mean nothing ever leaves or improves in place:
14
+
15
+ 1. **No supersede by source_id** — event identity is `sha256(arena:content)`; re-emitting
16
+ edited content appends a new event, the old persists.
17
+ 2. **Accrete-only graph writes** — entity/fact upserts are `ON CONFLICT (id) DO UPDATE`
18
+ that only merge aliases/provenance and bump confidence; a *corrected* extraction has a
19
+ different deterministic id, so it lands **beside** the polluted node, never replacing it.
20
+ 3. **No decay/eviction** — v2 has no GC; fact confidence only moves up; recency affects
21
+ search ranking only, never retention.
22
+
23
+ Net: improving the extractor/teacher only helps **new** content. Accumulated 7B-era
24
+ pollution (hallucinated emails, numeric-ID-as-person, ungrounded entities) is immortal.
25
+ `pentatonic-team` had to be **nuked** rather than re-distilled because of this; `pip-agents`
26
+ (87k events) still carries all of it.
27
+
28
+ This RFC makes the store **self-healing** via two complementary mechanisms:
29
+ **fusion** (horizontal — converge duplicate/near-duplicate nodes from different
30
+ distillation runs into one *master* node) and **decay** (vertical — age out stale and
31
+ low-value nodes). Both are gated, arena-scoped, audited, and reversible.
32
+
33
+ ---
34
+
35
+ ## Part A — Fusion: converge near-duplicate nodes into a master
36
+
37
+ Extends the existing entity-resolution machinery along four axes.
38
+
39
+ ### A1. Online + continuous (today it's dry-run batch)
40
+ Run fusion as a scheduled per-arena pass (systemd timer on the engine box, same pattern as
41
+ the distiller autoscaler) **and** opportunistically after a distillation run touches an
42
+ arena's entities. Keep #82's invariants: dry-run default, `--apply` gate, arena scoping,
43
+ `entity_merges` rollback. Add a `fusion_runs` ledger (arena, started_at, candidates,
44
+ merged, mode) for observability.
45
+
46
+ ### A2. Cross-distillation-run detection (the actual pollution cure)
47
+ The hard case #82 misses: 7B `"1716801984"` (numeric-ID person) and Qwen3.6 `"Katie Cooper"`
48
+ are the same real entity but share **no name similarity**, so name-blocking never compares
49
+ them. New candidate signals beyond name trigrams / embedding-on-name:
50
+
51
+ - **Shared-provenance co-reference** — two entities of the same `entity_type` citing the
52
+ same `event_id` in `provenance_event_ids`, where one is low-quality (numeric / ungrounded
53
+ / single-token). The shared event's content is the adjudication context ("does this event
54
+ support these being the same person?").
55
+ - **Context embedding** — embed the *facts/statements about* an entity (not just its name),
56
+ so name-divergent dupes still cluster. Reuses the bulk-embed lane.
57
+ - **Teacher-version signal** — provenance maps to `distillation_traces.llm_model` /
58
+ `system_prompt_hash`. Prefer the newer-teacher extraction as master; an entity *only* ever
59
+ produced by the superseded teacher and never re-confirmed by the new one is both a fusion
60
+ candidate (likely a worse rendering of a node the new teacher got right) and a decay
61
+ candidate (stale-teacher orphan — see B).
62
+
63
+ ### A3. Master-node selection — replace richest-row-wins
64
+ #82 uses "richest-row-wins", which (flagged in review) would crown the typo **"Phil Mossop"**
65
+ over **"Philip Mossop"**. Replace with a **scored** canonical pick:
66
+
67
+ | Signal | Effect |
68
+ |---|---|
69
+ | **Directory/authority anchor** (name matches an org-directory / HubSpot contact / Pip `contact_email`+`contact_name`) | dominant + → canonical |
70
+ | Grounding (name appears verbatim in a provenance event's content) | + |
71
+ | Teacher recency (newer `llm_model`) | + |
72
+ | Corroboration (`cardinality(provenance_event_ids)`) | + |
73
+ | Looks-like-ID (digit-ratio > 0.5) / hallucinated-email flag / single-token bare name | − − |
74
+
75
+ Master = highest score. Losers' surface forms become **aliases** on the master (so existing
76
+ lookups still resolve), facts/relationships are repointed, losers tombstoned in
77
+ `entity_merges` with `rollback_payload`. Directory-anchored selection is the key fix: an
78
+ authoritative source, when present, beats any heuristic.
79
+
80
+ ### A4. Fact + relationship fusion (today only entities fuse)
81
+ After entity fusion (so subject/object ids are canonical):
82
+ - **Facts** — exact `(arena, subject, predicate, object)` dupes already collapse via the
83
+ content-id. **Semantic** dupes (same assertion, different surface — "joined Acme" vs "works
84
+ at Acme") need statement-embedding similarity + LLM adjudication ("same assertion?").
85
+ Master fact = max confidence + best-grounded statement; union provenance; tombstone dupes.
86
+ New `fact_merges` audit mirroring `entity_merges`.
87
+ - **Relationships** — `(from,to,type)` already collapses; a controlled rel-type vocabulary
88
+ ("works at" ≡ "employed by") is a later optional canonicalization.
89
+
90
+ ### A5. Audit, reversibility, safety rails
91
+ Reuse `entity_merges`; add `fact_merges`. Every fusion carries `rollback_payload`.
92
+ LLM-adjudicated merges store prompt+verdict. **Disclosure rail:** never send
93
+ `disclosure_class='restricted'` rows to the LLM adjudicator (data-egress; the #82 review
94
+ item). Auto-merge only above a high confidence band; everything else → human-review queue.
95
+
96
+ ---
97
+
98
+ ## Part B — Decay: age out stale and low-value nodes
99
+
100
+ ### B1. Separate `salience` from `confidence` (important)
101
+ Do **not** decay `confidence` — it means "how corroborated/true is this", and decaying it
102
+ would lie about corroboration. Add a separate **`salience`** (retention priority) to
103
+ entities/facts/relationships. Decay acts on salience; eviction keys on salience.
104
+
105
+ `salience(t) = salience₀ · exp(−ln2 · Δt / half_life[category])`, bumped on access or
106
+ re-corroboration. Per-category half-life:
107
+
108
+ | category | half-life | rationale |
109
+ |---|---|---|
110
+ | decision, commitment | very long / ∞ | durable record |
111
+ | state, preference | medium | changes but matters |
112
+ | mention, observation | short | ephemeral |
113
+
114
+ `Δt` = time since `last_seen` **or** a new `last_accessed` (bumped when a node is returned by
115
+ `/search` — cheap write, makes retrieval keep memories alive). Re-corroboration (new
116
+ provenance) resets the clock and bumps salience.
117
+
118
+ ### B2. Born-salience — the cheap partial cure
119
+ Seed `salience₀` from extraction-quality signals already computed (the trap detectors:
120
+ ungrounded, numeric-ID-person, hallucinated-email, `noise_filter` hits). **Junk is born
121
+ low**, so it decays below threshold and self-evicts fast — pollution cleans itself even
122
+ without a fusion match.
123
+
124
+ ### B3. Eviction (GC)
125
+ Node is evictable when: `salience < min_threshold` **AND** `last_seen`/`last_accessed`
126
+ older than a floor **AND** not referenced by a surviving higher-salience node (an entity
127
+ that's the subject/object of a live fact survives). Eviction = **tombstone** (soft-delete +
128
+ retention window) → hard-delete after grace, cascading to the node's Qdrant points +
129
+ `vector_provenance`. Never evict `disclosure_class='restricted'` without sign-off.
130
+
131
+ ### B4. Capacity bound (optional)
132
+ Per-arena soft cap; when exceeded, evict lowest-salience first. Backstop against unbounded
133
+ arenas.
134
+
135
+ ### B5. Cadence + safety
136
+ Background per-arena pass (timer on the engine box), dry-run → `--apply` in a quiet window,
137
+ counts logged, fully arena-scoped. Same operational shape as the distiller autoscaler /
138
+ sparse backfill.
139
+
140
+ ---
141
+
142
+ ## Part C — Ordering & how they combine
143
+
144
+ Per arena, on schedule: **(1) fusion → (2) decay.** Fusion first so a master node absorbs
145
+ its duplicates' provenance/salience *before* decay judges it (else a real node split across
146
+ two weak dupes could wrongly decay out). Then decay ages + evicts the survivors.
147
+
148
+ **This is what finally cures immortal pollution:**
149
+ - 7B polluted node *with* a correct Qwen3.6 counterpart → **fused**, correct one as master,
150
+ polluted demoted to alias / tombstoned.
151
+ - 7B pure-junk node with *no* correct counterpart (numeric-ID-person, ungrounded) → born-low
152
+ salience + no corroboration + never accessed → **decays out and is evicted**.
153
+
154
+ Together they convert the accrete-only store into a self-healing one. `pip-agents` could
155
+ then self-clean over time instead of requiring a nuke (a nuke is still faster for a one-shot
156
+ reset, but no longer the *only* path).
157
+
158
+ ---
159
+
160
+ ## Part D — Schema changes
161
+
162
+ - `entities`: `+ salience REAL DEFAULT …`, `+ last_accessed TIMESTAMPTZ`.
163
+ - `facts`: `+ salience REAL`, `+ last_accessed TIMESTAMPTZ` (keep `confidence` as-is =
164
+ corroboration truth; `asserted_at`/`expires_at` already exist).
165
+ - `relationships`: `+ salience REAL`, `+ last_accessed` (already has `weight`,
166
+ `first/last_seen`).
167
+ - new `fact_merges` audit (mirror `entity_merges` incl. `rollback_payload`).
168
+ - new `fusion_runs` + `decay_runs` ledgers for observability.
169
+ - `/search` gains a `last_accessed = NOW()` bump on returned nodes (batched).
170
+
171
+ ## Part E — Rollout (each flag-gated, arena-scoped, dry-run-first, audited)
172
+
173
+ 1. **Salience scoring only** — add columns, born-salience + decay math, NO eviction.
174
+ Observe distributions; confirm junk scores low and durable facts stay high.
175
+ 2. **Eviction** — dry-run (count what *would* evict) → `--apply` in a quiet window.
176
+ 3. **Fusion extension** — scored canonical selection (fix typo-crowning) + cross-run
177
+ detection + fact fusion, dry-run → apply.
178
+ 4. **Online/continuous** — wire fusion+decay to run after distillation per arena.
179
+
180
+ ## Open questions
181
+ - Half-life constants per category — needs a calibration pass against real arenas.
182
+ - `last_accessed` write amplification on hot search paths — batch/throttle the bump.
183
+ - Directory authority source for canonical anchoring — HubSpot contacts? a curated table?
184
+ - Interaction with the (still-open) source_id supersede mode — fusion partly subsumes it,
185
+ but explicit supersede is cheaper for known-mutable sources.
@@ -0,0 +1,199 @@
1
+ # RFC: the Fusion Drive — v2 memory self-healing (cross-run node fusion + decay)
2
+
3
+ > **Fusion Drive** = the continuous, arena-scoped background engine that keeps the v2
4
+ > memory graph self-healing: it *fuses* duplicate/near-duplicate nodes from different
5
+ > distillation runs into a single master node (horizontal convergence) and *decays* stale,
6
+ > low-value, and junk nodes out of existence (vertical aging). Named for the drive that
7
+ > does the fusing — the decay pass rides the same engine.
8
+
9
+ **Status:** spec + implementation (PR #92, then completion PR) — 2026-06-13.
10
+ **Implemented:** salience scoring + decay; **eviction** (`fusion_drive_decay.py --evict`,
11
+ reversible via `node_evictions`); **entity AND relationship decay**; **fusion** of exact +
12
+ cross-run-shared-provenance entity dupes and exact-triple fact dupes, plus an **LLM
13
+ adjudication tier via the in-VPC distiller** (Qwen3.6 — NO egress) for ambiguous cross-run
14
+ entities and semantic (same-assertion-different-words) facts; **authority signals** wired
15
+ into canonical scoring (`grounded` = name verbatim in a provenance event;
16
+ `from_current_teacher` = `distillation_traces.llm_model`); **born-salience** in BOTH the
17
+ async distiller and the sync extractor (+ backfill for existing rows); **continuous
18
+ scheduling** (the `fusion-drive-sweep` 6h timer — dry-run-default, never `--evict` from
19
+ cron). All arena-scoped, dry-run-default, transactional, reversible, audited.
20
+ **Remaining:** `in_directory` anchoring (needs an authoritative directory/contacts source —
21
+ no such table exists yet; the scorer already supports it for when one lands); and the
22
+ **half-life / threshold / salience-constant CALIBRATION pass on a real arena before
23
+ `--evict` is ever run in prod** — eviction stays a deliberate manual op until then.
24
+ **Builds on:** `RFC-entity-reconciliation.md`, `scripts/entity_resolution_v2.py` (#82),
25
+ `org-model/migrations/002_entity_merges_audit.sql`.
26
+ **Motivated by:** the v2 store is currently **pure-accretion** — three independent
27
+ properties, all verified in code, mean nothing ever leaves or improves in place:
28
+
29
+ 1. **No supersede by source_id** — event identity is `sha256(arena:content)`; re-emitting
30
+ edited content appends a new event, the old persists.
31
+ 2. **Accrete-only graph writes** — entity/fact upserts are `ON CONFLICT (id) DO UPDATE`
32
+ that only merge aliases/provenance and bump confidence; a *corrected* extraction has a
33
+ different deterministic id, so it lands **beside** the polluted node, never replacing it.
34
+ 3. **No decay/eviction** — v2 has no GC; fact confidence only moves up; recency affects
35
+ search ranking only, never retention.
36
+
37
+ Net: improving the extractor/teacher only helps **new** content. Accumulated 7B-era
38
+ pollution (hallucinated emails, numeric-ID-as-person, ungrounded entities) is immortal.
39
+ `pentatonic-team` had to be **nuked** rather than re-distilled because of this; `pip-agents`
40
+ (87k events) still carries all of it.
41
+
42
+ This RFC makes the store **self-healing** via two complementary mechanisms:
43
+ **fusion** (horizontal — converge duplicate/near-duplicate nodes from different
44
+ distillation runs into one *master* node) and **decay** (vertical — age out stale and
45
+ low-value nodes). Both are gated, arena-scoped, audited, and reversible.
46
+
47
+ ---
48
+
49
+ ## Part A — Fusion: converge near-duplicate nodes into a master
50
+
51
+ Extends the existing entity-resolution machinery along four axes.
52
+
53
+ ### A1. Online + continuous (today it's dry-run batch)
54
+ Run fusion as a scheduled per-arena pass (systemd timer on the engine box, same pattern as
55
+ the distiller autoscaler) **and** opportunistically after a distillation run touches an
56
+ arena's entities. Keep #82's invariants: dry-run default, `--apply` gate, arena scoping,
57
+ `entity_merges` rollback. Add a `fusion_runs` ledger (arena, started_at, candidates,
58
+ merged, mode) for observability.
59
+
60
+ ### A2. Cross-distillation-run detection (the actual pollution cure)
61
+ The hard case #82 misses: 7B `"1716801984"` (numeric-ID person) and Qwen3.6 `"Katie Cooper"`
62
+ are the same real entity but share **no name similarity**, so name-blocking never compares
63
+ them. New candidate signals beyond name trigrams / embedding-on-name:
64
+
65
+ - **Shared-provenance co-reference** — two entities of the same `entity_type` citing the
66
+ same `event_id` in `provenance_event_ids`, where one is low-quality (numeric / ungrounded
67
+ / single-token). The shared event's content is the adjudication context ("does this event
68
+ support these being the same person?").
69
+ - **Context embedding** — embed the *facts/statements about* an entity (not just its name),
70
+ so name-divergent dupes still cluster. Reuses the bulk-embed lane.
71
+ - **Teacher-version signal** — provenance maps to `distillation_traces.llm_model` /
72
+ `system_prompt_hash`. Prefer the newer-teacher extraction as master; an entity *only* ever
73
+ produced by the superseded teacher and never re-confirmed by the new one is both a fusion
74
+ candidate (likely a worse rendering of a node the new teacher got right) and a decay
75
+ candidate (stale-teacher orphan — see B).
76
+
77
+ ### A3. Master-node selection — replace richest-row-wins
78
+ #82 uses "richest-row-wins", which (flagged in review) would crown the typo **"Phil Mossop"**
79
+ over **"Philip Mossop"**. Replace with a **scored** canonical pick:
80
+
81
+ | Signal | Effect |
82
+ |---|---|
83
+ | **Directory/authority anchor** (name matches an org-directory / HubSpot contact / Pip `contact_email`+`contact_name`) | dominant + → canonical |
84
+ | Grounding (name appears verbatim in a provenance event's content) | + |
85
+ | Teacher recency (newer `llm_model`) | + |
86
+ | Corroboration (`cardinality(provenance_event_ids)`) | + |
87
+ | Looks-like-ID (digit-ratio > 0.5) / hallucinated-email flag / single-token bare name | − − |
88
+
89
+ Master = highest score. Losers' surface forms become **aliases** on the master (so existing
90
+ lookups still resolve), facts/relationships are repointed, losers tombstoned in
91
+ `entity_merges` with `rollback_payload`. Directory-anchored selection is the key fix: an
92
+ authoritative source, when present, beats any heuristic.
93
+
94
+ ### A4. Fact + relationship fusion (today only entities fuse)
95
+ After entity fusion (so subject/object ids are canonical):
96
+ - **Facts** — exact `(arena, subject, predicate, object)` dupes already collapse via the
97
+ content-id. **Semantic** dupes (same assertion, different surface — "joined Acme" vs "works
98
+ at Acme") need statement-embedding similarity + LLM adjudication ("same assertion?").
99
+ Master fact = max confidence + best-grounded statement; union provenance; tombstone dupes.
100
+ New `fact_merges` audit mirroring `entity_merges`.
101
+ - **Relationships** — `(from,to,type)` already collapses; a controlled rel-type vocabulary
102
+ ("works at" ≡ "employed by") is a later optional canonicalization.
103
+
104
+ ### A5. Audit, reversibility, safety rails
105
+ Reuse `entity_merges`; add `fact_merges`. Every fusion carries `rollback_payload`.
106
+ LLM-adjudicated merges store prompt+verdict. **Disclosure rail:** never send
107
+ `disclosure_class='restricted'` rows to the LLM adjudicator (data-egress; the #82 review
108
+ item). Auto-merge only above a high confidence band; everything else → human-review queue.
109
+
110
+ ---
111
+
112
+ ## Part B — Decay: age out stale and low-value nodes
113
+
114
+ ### B1. Separate `salience` from `confidence` (important)
115
+ Do **not** decay `confidence` — it means "how corroborated/true is this", and decaying it
116
+ would lie about corroboration. Add a separate **`salience`** (retention priority) to
117
+ entities/facts/relationships. Decay acts on salience; eviction keys on salience.
118
+
119
+ `salience(t) = salience₀ · exp(−ln2 · Δt / half_life[category])`, bumped on access or
120
+ re-corroboration. Per-category half-life:
121
+
122
+ | category | half-life | rationale |
123
+ |---|---|---|
124
+ | decision, commitment | very long / ∞ | durable record |
125
+ | state, preference | medium | changes but matters |
126
+ | mention, observation | short | ephemeral |
127
+
128
+ `Δt` = time since `last_seen` **or** a new `last_accessed` (bumped when a node is returned by
129
+ `/search` — cheap write, makes retrieval keep memories alive). Re-corroboration (new
130
+ provenance) resets the clock and bumps salience.
131
+
132
+ ### B2. Born-salience — the cheap partial cure
133
+ Seed `salience₀` from extraction-quality signals already computed (the trap detectors:
134
+ ungrounded, numeric-ID-person, hallucinated-email, `noise_filter` hits). **Junk is born
135
+ low**, so it decays below threshold and self-evicts fast — pollution cleans itself even
136
+ without a fusion match.
137
+
138
+ ### B3. Eviction (GC)
139
+ Node is evictable when: `salience < min_threshold` **AND** `last_seen`/`last_accessed`
140
+ older than a floor **AND** not referenced by a surviving higher-salience node (an entity
141
+ that's the subject/object of a live fact survives). Eviction = **tombstone** (soft-delete +
142
+ retention window) → hard-delete after grace, cascading to the node's Qdrant points +
143
+ `vector_provenance`. Never evict `disclosure_class='restricted'` without sign-off.
144
+
145
+ ### B4. Capacity bound (optional)
146
+ Per-arena soft cap; when exceeded, evict lowest-salience first. Backstop against unbounded
147
+ arenas.
148
+
149
+ ### B5. Cadence + safety
150
+ Background per-arena pass (timer on the engine box), dry-run → `--apply` in a quiet window,
151
+ counts logged, fully arena-scoped. Same operational shape as the distiller autoscaler /
152
+ sparse backfill.
153
+
154
+ ---
155
+
156
+ ## Part C — Ordering & how they combine
157
+
158
+ Per arena, on schedule: **(1) fusion → (2) decay.** Fusion first so a master node absorbs
159
+ its duplicates' provenance/salience *before* decay judges it (else a real node split across
160
+ two weak dupes could wrongly decay out). Then decay ages + evicts the survivors.
161
+
162
+ **This is what finally cures immortal pollution:**
163
+ - 7B polluted node *with* a correct Qwen3.6 counterpart → **fused**, correct one as master,
164
+ polluted demoted to alias / tombstoned.
165
+ - 7B pure-junk node with *no* correct counterpart (numeric-ID-person, ungrounded) → born-low
166
+ salience + no corroboration + never accessed → **decays out and is evicted**.
167
+
168
+ Together they convert the accrete-only store into a self-healing one. `pip-agents` could
169
+ then self-clean over time instead of requiring a nuke (a nuke is still faster for a one-shot
170
+ reset, but no longer the *only* path).
171
+
172
+ ---
173
+
174
+ ## Part D — Schema changes
175
+
176
+ - `entities`: `+ salience REAL DEFAULT …`, `+ last_accessed TIMESTAMPTZ`.
177
+ - `facts`: `+ salience REAL`, `+ last_accessed TIMESTAMPTZ` (keep `confidence` as-is =
178
+ corroboration truth; `asserted_at`/`expires_at` already exist).
179
+ - `relationships`: `+ salience REAL`, `+ last_accessed` (already has `weight`,
180
+ `first/last_seen`).
181
+ - new `fact_merges` audit (mirror `entity_merges` incl. `rollback_payload`).
182
+ - new `fusion_runs` + `decay_runs` ledgers for observability.
183
+ - `/search` gains a `last_accessed = NOW()` bump on returned nodes (batched).
184
+
185
+ ## Part E — Rollout (each flag-gated, arena-scoped, dry-run-first, audited)
186
+
187
+ 1. **Salience scoring only** — add columns, born-salience + decay math, NO eviction.
188
+ Observe distributions; confirm junk scores low and durable facts stay high.
189
+ 2. **Eviction** — dry-run (count what *would* evict) → `--apply` in a quiet window.
190
+ 3. **Fusion extension** — scored canonical selection (fix typo-crowning) + cross-run
191
+ detection + fact fusion, dry-run → apply.
192
+ 4. **Online/continuous** — wire fusion+decay to run after distillation per arena.
193
+
194
+ ## Open questions
195
+ - Half-life constants per category — needs a calibration pass against real arenas.
196
+ - `last_accessed` write amplification on hot search paths — batch/throttle the bump.
197
+ - Directory authority source for canonical anchoring — HubSpot contacts? a curated table?
198
+ - Interaction with the (still-open) source_id supersede mode — fusion partly subsumes it,
199
+ but explicit supersede is cheaper for known-mutable sources.
@@ -60,3 +60,40 @@ def corroborated_confidence(n_sources: int) -> float:
60
60
  if bumped > _CONF_CAP:
61
61
  return _CONF_CAP
62
62
  return round(bumped, 2)
63
+
64
+
65
+ # ── born salience (Fusion Drive) ─────────────────────────────────────
66
+ # Retention priority a node is stamped with at extraction time, SEPARATE
67
+ # from confidence (confidence = corroboration/truth; salience = how long
68
+ # it's worth keeping). Junk — flagged by the extractor's own quality
69
+ # detectors (noise name, numeric-ID-as-person, hallucinated email,
70
+ # ungrounded, etc.) — is born near the floor so the Fusion Drive decay
71
+ # pass evicts it on a short clock instead of the multi-year default.
72
+ #
73
+ # This MUST stay byte-identical to fusion_drive/salience.py:born_salience
74
+ # (the decay side uses the same scale). test_born_salience_parity.py
75
+ # guards the two against drift — same pattern as entity_id.py's parity
76
+ # test across the sync/async build contexts.
77
+ _SAL_BASE = 0.50
78
+ _SAL_CORROB_PER_SOURCE = 0.10
79
+ _SAL_CORROB_CAP = 0.30
80
+ _SAL_FLOOR = 0.01
81
+ _SAL_CEIL = 1.00
82
+ _SAL_PENALTIES = {
83
+ "noise_name": 0.45,
84
+ "numeric_id_person": 0.45,
85
+ "hallucinated_email": 0.40,
86
+ "ungrounded": 0.35,
87
+ "subject_undeclared": 0.25,
88
+ "low_signal": 0.15,
89
+ }
90
+
91
+
92
+ def born_salience(n_sources: int = 1, quality_flags: list[str] | None = None) -> float:
93
+ """Salience to stamp on a freshly extracted node. See the module note."""
94
+ s = _SAL_BASE
95
+ if n_sources > 1:
96
+ s += min(_SAL_CORROB_CAP, _SAL_CORROB_PER_SOURCE * (n_sources - 1))
97
+ for flag in quality_flags or []:
98
+ s -= _SAL_PENALTIES.get(flag, 0.0)
99
+ return round(max(_SAL_FLOOR, min(_SAL_CEIL, s)), 4)
@@ -0,0 +1,63 @@
1
+ """source_time — robust ISO-8601 source-time parsing for graph stamping.
2
+
3
+ The memory graph must stamp `events.emitted_at` and the graph rows'
4
+ `first_seen` / `last_seen` / `asserted_at` from the SOURCE time of the
5
+ content (when the email/meeting/message actually happened), NOT the
6
+ ingest wall-clock (`NOW()`). The source time is carried on the event as
7
+ `attributes.timestamp` (ISO-8601). This helper promotes it.
8
+
9
+ Mirrors `compat/server.py:_parse_ts` (handles the bare `Z` suffix that
10
+ `datetime.fromisoformat` only learned in 3.11) but returns a tz-aware
11
+ `datetime` rather than a unix float, because the destination columns are
12
+ `TIMESTAMPTZ` and we want psycopg to bind a datetime, not an epoch.
13
+
14
+ CONTRACT (load-bearing): callers MUST fall back to the existing default
15
+ (received / NOW) when the source time is absent or unparseable. This
16
+ helper NEVER raises and returns `None` on anything it can't parse — the
17
+ caller is responsible for the `or NOW()` fallback so we never NULL a
18
+ NOT NULL column or crash the ingest/distill path.
19
+
20
+ NOTE: keep this byte-identical with the copy in extractor-sync/. Same
21
+ convention as entity_id.py — two services, one parsing rule.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ from datetime import datetime, timezone
27
+ from typing import Any
28
+
29
+
30
+ def parse_source_time(value: Any) -> datetime | None:
31
+ """Best-effort ISO-8601 -> tz-aware datetime. Returns None on
32
+ anything we can't parse (caller falls back to NOW()).
33
+
34
+ Accepts both the bare `Z` suffix and explicit offsets. A parsed
35
+ value with no offset is assumed UTC (the producers emit UTC ISO
36
+ strings; a naive datetime would break TIMESTAMPTZ comparisons)."""
37
+ if not isinstance(value, str) or not value:
38
+ return None
39
+ try:
40
+ # `fromisoformat` handles `+00:00` but not the bare `Z` suffix
41
+ # until Python 3.11; normalise to be safe across runtime
42
+ # versions on the engine box.
43
+ dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
44
+ except Exception:
45
+ return None
46
+ if dt.tzinfo is None:
47
+ # Producer emitted a naive ISO string; treat as UTC rather than
48
+ # letting psycopg interpret it in the server's local zone.
49
+ dt = dt.replace(tzinfo=timezone.utc)
50
+ return dt
51
+
52
+
53
+ def event_source_time(event: dict[str, Any]) -> datetime | None:
54
+ """Pull the source time off an event dict's attributes.
55
+
56
+ Precedence: `attributes.timestamp` (the source/content time) wins
57
+ over `attributes.emitted_at` (a producer-supplied emit-now, which is
58
+ closer to ingest time). Returns None if neither parses — caller
59
+ falls back to NOW()."""
60
+ attrs = event.get("attributes") or {}
61
+ return parse_source_time(attrs.get("timestamp")) or parse_source_time(
62
+ attrs.get("emitted_at")
63
+ )
@@ -0,0 +1,35 @@
1
+ """Parity guard: confidence.born_salience (worker, copied into the container)
2
+ must stay byte-equivalent to fusion_drive/salience.born_salience (the decay
3
+ side). Same pattern as test_entity_id_parity.py — the two live across a Docker
4
+ build-context boundary and would silently drift otherwise."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import os
9
+ import sys
10
+
11
+ import confidence as worker
12
+
13
+ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "fusion_drive"))
14
+ import salience as drive # noqa: E402
15
+
16
+
17
+ def test_constants_match():
18
+ assert worker._SAL_BASE == drive.BASE_SALIENCE
19
+ assert worker._SAL_CORROB_PER_SOURCE == drive.CORROB_PER_SOURCE
20
+ assert worker._SAL_CORROB_CAP == drive.CORROB_CAP
21
+ assert worker._SAL_FLOOR == drive.SALIENCE_FLOOR
22
+ assert worker._SAL_CEIL == drive.SALIENCE_CEIL
23
+ assert worker._SAL_PENALTIES == drive.QUALITY_PENALTIES
24
+
25
+
26
+ def test_output_matches_across_input_matrix():
27
+ flagsets = [
28
+ None, [], ["noise_name"], ["numeric_id_person"], ["hallucinated_email"],
29
+ ["ungrounded"], ["subject_undeclared"], ["low_signal"],
30
+ ["numeric_id_person", "hallucinated_email", "ungrounded"],
31
+ ["noise_name"] * 5,
32
+ ]
33
+ for n in (1, 2, 3, 5, 100):
34
+ for flags in flagsets:
35
+ assert worker.born_salience(n, flags) == drive.born_salience(n_sources=n, quality_flags=flags), (n, flags)
@@ -0,0 +1,102 @@
1
+ """Tests for source_time — promoting source event time onto graph rows.
2
+
3
+ The contract under test: source time present and parseable → used;
4
+ absent, empty, or garbage → returns None so the caller falls back to
5
+ NOW() (never crashes, never NULLs a NOT NULL column).
6
+
7
+ Run: pytest packages/memory-engine-v2/extractor-async/test_source_time.py
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from datetime import datetime, timezone
13
+
14
+ import pytest
15
+
16
+ from source_time import event_source_time, parse_source_time
17
+
18
+
19
+ class TestParseSourceTime:
20
+ def test_iso_with_z_suffix(self):
21
+ dt = parse_source_time("2025-03-14T09:30:00Z")
22
+ assert dt == datetime(2025, 3, 14, 9, 30, tzinfo=timezone.utc)
23
+
24
+ def test_iso_with_explicit_offset(self):
25
+ dt = parse_source_time("2025-03-14T09:30:00+00:00")
26
+ assert dt == datetime(2025, 3, 14, 9, 30, tzinfo=timezone.utc)
27
+
28
+ def test_iso_with_nonzero_offset_preserved(self):
29
+ dt = parse_source_time("2025-03-14T12:30:00+03:00")
30
+ # 12:30+03:00 == 09:30 UTC
31
+ assert dt.utcoffset().total_seconds() == 3 * 3600
32
+ assert dt.astimezone(timezone.utc) == datetime(
33
+ 2025, 3, 14, 9, 30, tzinfo=timezone.utc
34
+ )
35
+
36
+ def test_naive_iso_assumed_utc(self):
37
+ # No offset → must NOT come back naive (would break TIMESTAMPTZ
38
+ # comparisons); we assume UTC.
39
+ dt = parse_source_time("2025-03-14T09:30:00")
40
+ assert dt is not None
41
+ assert dt.tzinfo is not None
42
+ assert dt == datetime(2025, 3, 14, 9, 30, tzinfo=timezone.utc)
43
+
44
+ # --- fallback cases: must return None, never raise ---
45
+
46
+ @pytest.mark.parametrize(
47
+ "bad",
48
+ [
49
+ None,
50
+ "",
51
+ "not-a-date",
52
+ "2025-13-99T99:99:99Z", # structurally ISO-ish but invalid
53
+ "14/03/2025", # wrong format
54
+ 12345, # not a string
55
+ [], # not a string
56
+ {"timestamp": "x"}, # not a string
57
+ ],
58
+ )
59
+ def test_garbage_or_absent_returns_none(self, bad):
60
+ assert parse_source_time(bad) is None
61
+
62
+
63
+ class TestEventSourceTime:
64
+ def test_prefers_timestamp_over_emitted_at(self):
65
+ ev = {
66
+ "attributes": {
67
+ "timestamp": "2025-01-01T00:00:00Z", # source time
68
+ "emitted_at": "2025-06-01T00:00:00Z", # producer emit-now
69
+ }
70
+ }
71
+ assert event_source_time(ev) == datetime(
72
+ 2025, 1, 1, 0, 0, tzinfo=timezone.utc
73
+ )
74
+
75
+ def test_falls_back_to_emitted_at_when_no_timestamp(self):
76
+ ev = {"attributes": {"emitted_at": "2025-06-01T00:00:00Z"}}
77
+ assert event_source_time(ev) == datetime(
78
+ 2025, 6, 1, 0, 0, tzinfo=timezone.utc
79
+ )
80
+
81
+ def test_none_when_neither_present(self):
82
+ assert event_source_time({"attributes": {}}) is None
83
+
84
+ def test_none_when_no_attributes(self):
85
+ # Must not crash on an event with a missing/None attributes bag.
86
+ assert event_source_time({}) is None
87
+ assert event_source_time({"attributes": None}) is None
88
+
89
+ def test_garbage_timestamp_falls_back_to_emitted_at(self):
90
+ ev = {
91
+ "attributes": {
92
+ "timestamp": "garbage",
93
+ "emitted_at": "2025-06-01T00:00:00Z",
94
+ }
95
+ }
96
+ assert event_source_time(ev) == datetime(
97
+ 2025, 6, 1, 0, 0, tzinfo=timezone.utc
98
+ )
99
+
100
+ def test_all_garbage_returns_none(self):
101
+ ev = {"attributes": {"timestamp": "nope", "emitted_at": "also-nope"}}
102
+ assert event_source_time(ev) is None