@pentatonic-ai/ai-agent-sdk 0.10.7 → 0.10.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/packages/memory-engine-v2/RFC-decay-and-fusion.md +185 -0
- package/packages/memory-engine-v2/RFC-fusion-drive.md +193 -0
- package/packages/memory-engine-v2/extractor-async/confidence.py +37 -0
- package/packages/memory-engine-v2/extractor-async/test_born_salience_parity.py +35 -0
- package/packages/memory-engine-v2/extractor-async/worker.py +36 -6
- package/packages/memory-engine-v2/fusion_drive/__init__.py +0 -0
- package/packages/memory-engine-v2/fusion_drive/canonical.py +94 -0
- package/packages/memory-engine-v2/fusion_drive/conftest.py +8 -0
- package/packages/memory-engine-v2/fusion_drive/merge.py +178 -0
- package/packages/memory-engine-v2/fusion_drive/salience.py +118 -0
- package/packages/memory-engine-v2/fusion_drive/test_canonical.py +76 -0
- package/packages/memory-engine-v2/fusion_drive/test_merge.py +112 -0
- package/packages/memory-engine-v2/fusion_drive/test_salience.py +93 -0
- package/packages/memory-engine-v2/org-model/migrations/006_fusion_drive.sql +80 -0
- package/packages/memory-engine-v2/scripts/fusion_drive_born_salience_backfill.py +113 -0
- package/packages/memory-engine-v2/scripts/fusion_drive_decay.py +181 -0
- package/packages/memory-engine-v2/scripts/fusion_drive_fuse.py +264 -0
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Fusion Drive — scored canonical-node selection (pure functions).
|
|
2
|
+
|
|
3
|
+
When fusion decides a set of entities are the same real thing, ONE becomes
|
|
4
|
+
the master (canonical) and the rest become its aliases. entity_resolution_v2
|
|
5
|
+
(#82) currently picks "richest-row-wins", which crowns the typo "Phil
|
|
6
|
+
Mossop" over "Philip Mossop" if the typo's row happens to be richer. This
|
|
7
|
+
replaces that with a scored pick whose dominant signal is an authoritative
|
|
8
|
+
directory match — so when an org directory / CRM knows the real name, it
|
|
9
|
+
wins regardless of row richness. See RFC-fusion-drive.md A3.
|
|
10
|
+
|
|
11
|
+
Pure: all external signals (directory membership, grounding, teacher
|
|
12
|
+
recency) are passed in, so this is fully unit-testable without DB/LLM.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
|
|
20
|
+
# Scoring weights. Directory anchoring dominates everything (a known-good
|
|
21
|
+
# authoritative name beats any heuristic); penalties for ID-like / bare /
|
|
22
|
+
# hallucinated names are large enough to sink an otherwise-rich row.
|
|
23
|
+
W_DIRECTORY = 100.0 # name matches an org-directory / CRM contact
|
|
24
|
+
W_GROUNDED = 10.0 # name appears verbatim in a provenance event
|
|
25
|
+
W_TEACHER_RECENCY = 8.0 # extracted by the current (not superseded) teacher
|
|
26
|
+
W_PER_CORROBORATION = 1.0
|
|
27
|
+
CORROBORATION_CAP = 10.0
|
|
28
|
+
P_LOOKS_LIKE_ID = 60.0 # name is mostly digits (numeric-ID-as-person)
|
|
29
|
+
P_HALLUCINATED_EMAIL = 25.0
|
|
30
|
+
P_BARE_SINGLE_TOKEN = 5.0
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class CanonicalCandidate:
|
|
35
|
+
"""One entity in a fuse-set, plus the resolved external signals."""
|
|
36
|
+
entity_id: str
|
|
37
|
+
canonical_name: str
|
|
38
|
+
n_provenance: int = 1
|
|
39
|
+
in_directory: bool = False # authoritative match (HubSpot contact, etc.)
|
|
40
|
+
grounded: bool = False # name verbatim in a provenance event
|
|
41
|
+
from_current_teacher: bool = False
|
|
42
|
+
hallucinated_email: bool = False
|
|
43
|
+
aliases: list[str] = field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _digit_ratio(s: str) -> float:
|
|
47
|
+
stripped = re.sub(r"\s+", "", s)
|
|
48
|
+
if not stripped:
|
|
49
|
+
return 1.0
|
|
50
|
+
return sum(c.isdigit() for c in stripped) / len(stripped)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def looks_like_id(name: str) -> bool:
|
|
54
|
+
"""Mostly-digit names are extractor noise (numeric IDs mistyped as
|
|
55
|
+
people), not real canonical names."""
|
|
56
|
+
return _digit_ratio(name) > 0.5
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def is_bare_single_token(name: str) -> bool:
|
|
60
|
+
return len(name.split()) == 1
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def canonical_score(c: CanonicalCandidate) -> float:
|
|
64
|
+
score = 0.0
|
|
65
|
+
if c.in_directory:
|
|
66
|
+
score += W_DIRECTORY
|
|
67
|
+
if c.grounded:
|
|
68
|
+
score += W_GROUNDED
|
|
69
|
+
if c.from_current_teacher:
|
|
70
|
+
score += W_TEACHER_RECENCY
|
|
71
|
+
score += min(CORROBORATION_CAP, W_PER_CORROBORATION * max(0, c.n_provenance))
|
|
72
|
+
if looks_like_id(c.canonical_name):
|
|
73
|
+
score -= P_LOOKS_LIKE_ID
|
|
74
|
+
if c.hallucinated_email:
|
|
75
|
+
score -= P_HALLUCINATED_EMAIL
|
|
76
|
+
if is_bare_single_token(c.canonical_name):
|
|
77
|
+
score -= P_BARE_SINGLE_TOKEN
|
|
78
|
+
return round(score, 4)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def pick_master(candidates: list[CanonicalCandidate]) -> tuple[CanonicalCandidate, list[CanonicalCandidate]]:
|
|
82
|
+
"""Return (master, losers). Master = highest canonical_score; ties
|
|
83
|
+
break toward more provenance, then longer name (stable, deterministic).
|
|
84
|
+
Losers' surface forms become aliases on the master downstream."""
|
|
85
|
+
if not candidates:
|
|
86
|
+
raise ValueError("pick_master requires at least one candidate")
|
|
87
|
+
ranked = sorted(
|
|
88
|
+
candidates,
|
|
89
|
+
# Final key is entity_id so a total tie is resolved deterministically
|
|
90
|
+
# regardless of input order (stable sort alone would leak order).
|
|
91
|
+
key=lambda c: (canonical_score(c), c.n_provenance, len(c.canonical_name), c.entity_id),
|
|
92
|
+
reverse=True,
|
|
93
|
+
)
|
|
94
|
+
return ranked[0], ranked[1:]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Put this package dir on sys.path so the flat sibling imports in the
|
|
2
|
+
test modules (`import salience`, `from canonical import ...`) resolve no
|
|
3
|
+
matter which directory pytest is invoked from."""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
sys.path.insert(0, os.path.dirname(__file__))
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Fusion Drive — merge & eviction PLAN builders (pure functions).
|
|
2
|
+
|
|
3
|
+
The risky part of fusion/eviction is mutating prod rows: repointing facts
|
|
4
|
+
and relationships off a deprecated entity, summing relationship weights on
|
|
5
|
+
collision, unioning aliases/provenance, and deleting the right rows with a
|
|
6
|
+
recoverable receipt. We isolate ALL of that decision-making into pure plan
|
|
7
|
+
builders here (no DB), so it's exhaustively unit-testable; the scripts then
|
|
8
|
+
execute the returned plan inside a single transaction.
|
|
9
|
+
|
|
10
|
+
A plan is a dict of explicit operations. The executor performs them in order
|
|
11
|
+
and is otherwise dumb. Nothing here touches a database or a clock.
|
|
12
|
+
See RFC-fusion-drive.md Parts A4/A5 + B3.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# ── entity fusion plan ───────────────────────────────────────────────
|
|
22
|
+
@dataclass
|
|
23
|
+
class EntityMergePlan:
|
|
24
|
+
arena: str
|
|
25
|
+
master_id: str
|
|
26
|
+
# master row mutations
|
|
27
|
+
master_aliases: list[str]
|
|
28
|
+
master_provenance: list[str]
|
|
29
|
+
# repoints: (table, column, from_id) -> master_id
|
|
30
|
+
fact_subject_repoints: list[str] = field(default_factory=list) # fact ids
|
|
31
|
+
fact_object_repoints: list[str] = field(default_factory=list)
|
|
32
|
+
rel_endpoint_repoints: list[str] = field(default_factory=list) # rel ids simply repointed
|
|
33
|
+
rel_collisions: list[dict] = field(default_factory=list) # {keep, drop, summed_weight, provenance}
|
|
34
|
+
deprecated_entity_ids: list[str] = field(default_factory=list)
|
|
35
|
+
audit_rows: list[dict] = field(default_factory=list) # entity_merges rows
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _union(*lists: list[str]) -> list[str]:
|
|
39
|
+
seen: dict[str, None] = {}
|
|
40
|
+
for lst in lists:
|
|
41
|
+
for x in lst or []:
|
|
42
|
+
if x not in seen:
|
|
43
|
+
seen[x] = None
|
|
44
|
+
return list(seen.keys())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_entity_merge_plan(
|
|
48
|
+
*,
|
|
49
|
+
arena: str,
|
|
50
|
+
master: dict,
|
|
51
|
+
losers: list[dict],
|
|
52
|
+
facts: list[dict],
|
|
53
|
+
relationships: list[dict],
|
|
54
|
+
merge_signal: str = "online_resolver",
|
|
55
|
+
) -> EntityMergePlan:
|
|
56
|
+
"""Compute every mutation to fold `losers` into `master`.
|
|
57
|
+
|
|
58
|
+
master/losers: {id, canonical_name, aliases, provenance_event_ids}
|
|
59
|
+
facts: {id, subject_entity_id, object_entity_id} touching any loser
|
|
60
|
+
relationships: {id, from_entity_id, to_entity_id, relationship_type,
|
|
61
|
+
weight, provenance_event_ids} touching any loser
|
|
62
|
+
"""
|
|
63
|
+
loser_ids = {l["id"] for l in losers}
|
|
64
|
+
if master["id"] in loser_ids:
|
|
65
|
+
raise ValueError("master cannot also be a loser")
|
|
66
|
+
|
|
67
|
+
# master accretes every loser's surface form + provenance
|
|
68
|
+
aliases = _union(
|
|
69
|
+
master.get("aliases", []),
|
|
70
|
+
[l["canonical_name"] for l in losers],
|
|
71
|
+
*[l.get("aliases", []) for l in losers],
|
|
72
|
+
)
|
|
73
|
+
# don't list the master's own canonical_name as an alias of itself
|
|
74
|
+
aliases = [a for a in aliases if a != master["canonical_name"]]
|
|
75
|
+
provenance = _union(
|
|
76
|
+
master.get("provenance_event_ids", []),
|
|
77
|
+
*[l.get("provenance_event_ids", []) for l in losers],
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
plan = EntityMergePlan(
|
|
81
|
+
arena=arena,
|
|
82
|
+
master_id=master["id"],
|
|
83
|
+
master_aliases=aliases,
|
|
84
|
+
master_provenance=provenance,
|
|
85
|
+
deprecated_entity_ids=sorted(loser_ids),
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# facts: repoint subject/object off losers onto master
|
|
89
|
+
for f in facts:
|
|
90
|
+
if f.get("subject_entity_id") in loser_ids:
|
|
91
|
+
plan.fact_subject_repoints.append(f["id"])
|
|
92
|
+
if f.get("object_entity_id") in loser_ids:
|
|
93
|
+
plan.fact_object_repoints.append(f["id"])
|
|
94
|
+
|
|
95
|
+
# relationships: repoint endpoints; a repoint can collide with an
|
|
96
|
+
# existing rel of the same (from,to,type) → keep one, sum weights,
|
|
97
|
+
# union provenance, drop the other. Detect collisions on the
|
|
98
|
+
# post-repoint key.
|
|
99
|
+
def repointed_key(r: dict) -> tuple:
|
|
100
|
+
frm = master["id"] if r["from_entity_id"] in loser_ids else r["from_entity_id"]
|
|
101
|
+
to = master["id"] if r["to_entity_id"] in loser_ids else r["to_entity_id"]
|
|
102
|
+
return (frm, to, r["relationship_type"])
|
|
103
|
+
|
|
104
|
+
by_key: dict[tuple, dict] = {}
|
|
105
|
+
for r in relationships:
|
|
106
|
+
touches = r["from_entity_id"] in loser_ids or r["to_entity_id"] in loser_ids
|
|
107
|
+
key = repointed_key(r)
|
|
108
|
+
if key in by_key:
|
|
109
|
+
keep = by_key[key]
|
|
110
|
+
plan.rel_collisions.append({
|
|
111
|
+
"keep": keep["id"],
|
|
112
|
+
"drop": r["id"],
|
|
113
|
+
"summed_weight": round(keep.get("weight", 1.0) + r.get("weight", 1.0), 4),
|
|
114
|
+
"provenance": _union(keep.get("provenance_event_ids", []),
|
|
115
|
+
r.get("provenance_event_ids", [])),
|
|
116
|
+
})
|
|
117
|
+
else:
|
|
118
|
+
by_key[key] = r
|
|
119
|
+
if touches:
|
|
120
|
+
plan.rel_endpoint_repoints.append(r["id"])
|
|
121
|
+
|
|
122
|
+
# audit + rollback receipt, one per deprecated entity
|
|
123
|
+
for l in losers:
|
|
124
|
+
plan.audit_rows.append({
|
|
125
|
+
"arena": arena,
|
|
126
|
+
"canonical_id": master["id"],
|
|
127
|
+
"deprecated_id": l["id"],
|
|
128
|
+
"deprecated_canonical_name": l["canonical_name"],
|
|
129
|
+
"deprecated_aliases": l.get("aliases", []),
|
|
130
|
+
"merge_signal": merge_signal,
|
|
131
|
+
"rollback_payload": l, # full row, sufficient to recreate
|
|
132
|
+
})
|
|
133
|
+
return plan
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ── fact fusion plan (exact-triple dupes) ────────────────────────────
|
|
137
|
+
def build_fact_merge_plan(*, arena: str, dup_facts: list[dict]) -> dict | None:
|
|
138
|
+
"""`dup_facts` all share (arena, subject, predicate, object). Master =
|
|
139
|
+
highest confidence, then longest statement (most informative), then id.
|
|
140
|
+
Others' provenance is unioned into the master; they are deleted."""
|
|
141
|
+
if len(dup_facts) < 2:
|
|
142
|
+
return None
|
|
143
|
+
ranked = sorted(
|
|
144
|
+
dup_facts,
|
|
145
|
+
key=lambda f: (f.get("confidence", 0.0), len(f.get("statement", "")), f["id"]),
|
|
146
|
+
reverse=True,
|
|
147
|
+
)
|
|
148
|
+
master, losers = ranked[0], ranked[1:]
|
|
149
|
+
provenance = _union(master.get("provenance_event_ids", []),
|
|
150
|
+
*[l.get("provenance_event_ids", []) for l in losers])
|
|
151
|
+
return {
|
|
152
|
+
"arena": arena,
|
|
153
|
+
"master_id": master["id"],
|
|
154
|
+
"master_provenance": provenance,
|
|
155
|
+
"deprecated_ids": [l["id"] for l in losers],
|
|
156
|
+
"audit_rows": [{
|
|
157
|
+
"arena": arena,
|
|
158
|
+
"canonical_id": master["id"],
|
|
159
|
+
"deprecated_id": l["id"],
|
|
160
|
+
"deprecated_statement": l.get("statement", ""),
|
|
161
|
+
"merge_signal": "exact_triple",
|
|
162
|
+
"provenance_unioned": len(l.get("provenance_event_ids", [])),
|
|
163
|
+
"rollback_payload": l,
|
|
164
|
+
} for l in losers],
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ── eviction plan ────────────────────────────────────────────────────
|
|
169
|
+
def build_eviction_receipt(node_kind: str, row: dict) -> dict:
|
|
170
|
+
"""A node_evictions audit row carrying enough to recreate the deleted
|
|
171
|
+
node. The executor only deletes nodes the salience pass already
|
|
172
|
+
classified evictable; this just packages the rollback receipt."""
|
|
173
|
+
return {
|
|
174
|
+
"node_kind": node_kind, # entity | fact | relationship
|
|
175
|
+
"node_id": row["id"],
|
|
176
|
+
"arena": row["arena"],
|
|
177
|
+
"rollback_payload": row,
|
|
178
|
+
}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Fusion Drive — salience scoring + time decay (pure functions).
|
|
2
|
+
|
|
3
|
+
Salience is a node's RETENTION PRIORITY (0..1), distinct from a fact's
|
|
4
|
+
`confidence` (which means corroboration/truth and only moves up). Salience
|
|
5
|
+
is seeded at birth from extraction-quality signals, decays with time since
|
|
6
|
+
last activity on a per-category half-life, and is reset/raised by
|
|
7
|
+
re-corroboration or retrieval. Eviction (a later phase) keys on salience.
|
|
8
|
+
|
|
9
|
+
Everything here is a pure function — no DB, no clock — so the decay pass
|
|
10
|
+
just supplies `now` and the stored fields. See RFC-fusion-drive.md Part B.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
# ── born salience ────────────────────────────────────────────────────
|
|
16
|
+
# A node is born at BASE, nudged up by corroboration and down by each
|
|
17
|
+
# extraction-quality red flag. Junk (noise name, numeric-ID person,
|
|
18
|
+
# hallucinated email, ungrounded) is born near the floor so it decays
|
|
19
|
+
# below the eviction threshold fast even with no fusion match — this is
|
|
20
|
+
# the "born-low" mechanism that lets decay target pollution rather than
|
|
21
|
+
# just age everything on the same clock.
|
|
22
|
+
BASE_SALIENCE = 0.50
|
|
23
|
+
CORROB_PER_SOURCE = 0.10 # each extra corroborating event
|
|
24
|
+
CORROB_CAP = 0.30 # max uplift from corroboration
|
|
25
|
+
SALIENCE_FLOOR = 0.01
|
|
26
|
+
SALIENCE_CEIL = 1.00
|
|
27
|
+
|
|
28
|
+
# Quality penalties (subtracted from born salience). Tuned so any single
|
|
29
|
+
# hard-junk signal alone lands a node below the nominal 0.3 decay-sweep
|
|
30
|
+
# threshold; combined signals drive it to the floor.
|
|
31
|
+
QUALITY_PENALTIES = {
|
|
32
|
+
"noise_name": 0.45, # noise_filter.is_noise_entity_name hit
|
|
33
|
+
"numeric_id_person": 0.45, # person whose name is mostly digits (ID-as-person)
|
|
34
|
+
"hallucinated_email": 0.40, # email not present in any provenance event
|
|
35
|
+
"ungrounded": 0.35, # name/statement not substring of any source
|
|
36
|
+
"subject_undeclared": 0.25, # fact subject not among the event's entities
|
|
37
|
+
"low_signal": 0.15, # extracted from <60 chars of content
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _clamp(x: float) -> float:
|
|
42
|
+
return max(SALIENCE_FLOOR, min(SALIENCE_CEIL, x))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def born_salience(*, n_sources: int = 1, quality_flags: list[str] | None = None) -> float:
|
|
46
|
+
"""Salience to stamp on a freshly extracted node.
|
|
47
|
+
|
|
48
|
+
n_sources: corroborating events (cardinality of provenance).
|
|
49
|
+
quality_flags: subset of QUALITY_PENALTIES keys that fired for this node.
|
|
50
|
+
"""
|
|
51
|
+
s = BASE_SALIENCE
|
|
52
|
+
if n_sources > 1:
|
|
53
|
+
s += min(CORROB_CAP, CORROB_PER_SOURCE * (n_sources - 1))
|
|
54
|
+
for flag in quality_flags or []:
|
|
55
|
+
s -= QUALITY_PENALTIES.get(flag, 0.0)
|
|
56
|
+
return round(_clamp(s), 4)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ── time decay ───────────────────────────────────────────────────────
|
|
60
|
+
# Half-lives in DAYS, by fact category (entities/relationships use the
|
|
61
|
+
# kind-level defaults). Durable categories (decisions, commitments) are
|
|
62
|
+
# effectively non-decaying; ephemeral ones (mentions, observations) fade
|
|
63
|
+
# in weeks. These are starting constants — Part B open question flags a
|
|
64
|
+
# calibration pass against real arenas.
|
|
65
|
+
FACT_HALF_LIFE_DAYS = {
|
|
66
|
+
"decision": 3650,
|
|
67
|
+
"commitment": 3650,
|
|
68
|
+
"state": 180,
|
|
69
|
+
"preference": 180,
|
|
70
|
+
"mention": 30,
|
|
71
|
+
"observation": 30,
|
|
72
|
+
}
|
|
73
|
+
FACT_HALF_LIFE_DEFAULT = 90
|
|
74
|
+
ENTITY_HALF_LIFE_DAYS = 365
|
|
75
|
+
RELATIONSHIP_HALF_LIFE_DAYS = 180
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def half_life_days(kind: str, category: str | None = None) -> float:
|
|
79
|
+
"""kind ∈ {'fact','entity','relationship'}. category only used for facts."""
|
|
80
|
+
if kind == "fact":
|
|
81
|
+
return FACT_HALF_LIFE_DAYS.get((category or "").lower(), FACT_HALF_LIFE_DEFAULT)
|
|
82
|
+
if kind == "entity":
|
|
83
|
+
return ENTITY_HALF_LIFE_DAYS
|
|
84
|
+
if kind == "relationship":
|
|
85
|
+
return RELATIONSHIP_HALF_LIFE_DAYS
|
|
86
|
+
return FACT_HALF_LIFE_DEFAULT
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def decayed_salience(salience0: float, age_days: float, hl_days: float) -> float:
|
|
90
|
+
"""Exponential half-life decay. age_days is time since the most recent
|
|
91
|
+
of (last_accessed, last_seen/asserted) — i.e. the clock resets on
|
|
92
|
+
access or re-corroboration, so used/reconfirmed memories don't fade."""
|
|
93
|
+
if age_days <= 0 or hl_days <= 0:
|
|
94
|
+
return round(_clamp(salience0), 4)
|
|
95
|
+
return round(_clamp(salience0 * (0.5 ** (age_days / hl_days))), 4)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ── eviction predicate (computed in Phase 1, ACTED ON in a later phase) ─
|
|
99
|
+
EVICT_THRESHOLD = 0.05 # salience below this → eviction candidate
|
|
100
|
+
EVICT_MIN_AGE_DAYS = 30 # ...and untouched at least this long
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def is_evictable(
|
|
104
|
+
*,
|
|
105
|
+
current_salience: float,
|
|
106
|
+
age_days: float,
|
|
107
|
+
referenced_by_live_node: bool,
|
|
108
|
+
disclosure_class: str = "private",
|
|
109
|
+
) -> bool:
|
|
110
|
+
"""An entity that is the subject/object of a surviving higher-salience
|
|
111
|
+
fact is NOT evictable (would orphan the fact). Restricted disclosure is
|
|
112
|
+
never auto-evicted (needs sign-off). Phase 1 only REPORTS this; the
|
|
113
|
+
decay pass does not delete until a later flagged phase."""
|
|
114
|
+
if disclosure_class == "restricted":
|
|
115
|
+
return False
|
|
116
|
+
if referenced_by_live_node:
|
|
117
|
+
return False
|
|
118
|
+
return current_salience < EVICT_THRESHOLD and age_days >= EVICT_MIN_AGE_DAYS
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Unit tests for Fusion Drive scored canonical selection (pure, no DB)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from canonical import CanonicalCandidate, canonical_score, pick_master, looks_like_id
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TestLooksLikeId:
|
|
9
|
+
def test_pure_digits(self):
|
|
10
|
+
assert looks_like_id("1716801984")
|
|
11
|
+
|
|
12
|
+
def test_real_name(self):
|
|
13
|
+
assert not looks_like_id("Katie Cooper")
|
|
14
|
+
|
|
15
|
+
def test_mostly_digits(self):
|
|
16
|
+
assert looks_like_id("user 90210 33")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TestCanonicalScore:
|
|
20
|
+
def test_directory_match_dominates(self):
|
|
21
|
+
directoried = CanonicalCandidate("e1", "Philip Mossop", in_directory=True)
|
|
22
|
+
rich_typo = CanonicalCandidate("e2", "Phil Mossop", n_provenance=50, grounded=True)
|
|
23
|
+
assert canonical_score(directoried) > canonical_score(rich_typo)
|
|
24
|
+
|
|
25
|
+
def test_numeric_id_person_heavily_penalised(self):
|
|
26
|
+
idp = CanonicalCandidate("e1", "1716801984", n_provenance=20)
|
|
27
|
+
real = CanonicalCandidate("e2", "Katie Cooper", n_provenance=1)
|
|
28
|
+
assert canonical_score(real) > canonical_score(idp)
|
|
29
|
+
|
|
30
|
+
def test_current_teacher_beats_superseded_when_otherwise_equal(self):
|
|
31
|
+
new = CanonicalCandidate("e1", "Acme Corp", from_current_teacher=True)
|
|
32
|
+
old = CanonicalCandidate("e2", "Acme Corp", from_current_teacher=False)
|
|
33
|
+
assert canonical_score(new) > canonical_score(old)
|
|
34
|
+
|
|
35
|
+
def test_hallucinated_email_penalised(self):
|
|
36
|
+
clean = CanonicalCandidate("e1", "Sam Patel")
|
|
37
|
+
halluc = CanonicalCandidate("e2", "Sam Patel", hallucinated_email=True)
|
|
38
|
+
assert canonical_score(clean) > canonical_score(halluc)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class TestPickMaster:
|
|
42
|
+
def test_the_phil_mossop_typo_case(self):
|
|
43
|
+
# The exact regression the RFC calls out: directory-known correct
|
|
44
|
+
# spelling must win over a richer typo row.
|
|
45
|
+
cands = [
|
|
46
|
+
CanonicalCandidate("typo", "Phil Mossop", n_provenance=40, grounded=True),
|
|
47
|
+
CanonicalCandidate("real", "Philip Mossop", n_provenance=3, in_directory=True),
|
|
48
|
+
]
|
|
49
|
+
master, losers = pick_master(cands)
|
|
50
|
+
assert master.entity_id == "real"
|
|
51
|
+
assert [l.entity_id for l in losers] == ["typo"]
|
|
52
|
+
|
|
53
|
+
def test_numeric_id_loses_to_real_name(self):
|
|
54
|
+
cands = [
|
|
55
|
+
CanonicalCandidate("idp", "1716801984", n_provenance=30),
|
|
56
|
+
CanonicalCandidate("named", "Katie Cooper", n_provenance=2, grounded=True),
|
|
57
|
+
]
|
|
58
|
+
master, _ = pick_master(cands)
|
|
59
|
+
assert master.entity_id == "named"
|
|
60
|
+
|
|
61
|
+
def test_single_candidate_is_its_own_master(self):
|
|
62
|
+
c = CanonicalCandidate("solo", "Solo Entity")
|
|
63
|
+
master, losers = pick_master([c])
|
|
64
|
+
assert master is c and losers == []
|
|
65
|
+
|
|
66
|
+
def test_deterministic_tie_break(self):
|
|
67
|
+
a = CanonicalCandidate("a", "Acme", n_provenance=5)
|
|
68
|
+
b = CanonicalCandidate("b", "Acme", n_provenance=5)
|
|
69
|
+
m1, _ = pick_master([a, b])
|
|
70
|
+
m2, _ = pick_master([b, a])
|
|
71
|
+
assert m1.entity_id == m2.entity_id # order-independent
|
|
72
|
+
|
|
73
|
+
def test_empty_raises(self):
|
|
74
|
+
import pytest
|
|
75
|
+
with pytest.raises(ValueError):
|
|
76
|
+
pick_master([])
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"""Unit tests for Fusion Drive merge & eviction plan builders (pure, no DB)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
from merge import build_entity_merge_plan, build_fact_merge_plan, build_eviction_receipt
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _ent(id, name, aliases=None, prov=None):
|
|
10
|
+
return {"id": id, "canonical_name": name, "aliases": aliases or [], "provenance_event_ids": prov or []}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class TestEntityMergePlan:
|
|
14
|
+
def test_aliases_and_provenance_union(self):
|
|
15
|
+
master = _ent("m", "Philip Mossop", aliases=["P. Mossop"], prov=["e1"])
|
|
16
|
+
loser = _ent("l", "Phil Mossop", aliases=["phil"], prov=["e2", "e1"])
|
|
17
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=[loser], facts=[], relationships=[])
|
|
18
|
+
# loser's canonical + aliases fold into master aliases; master's own name excluded
|
|
19
|
+
assert "Phil Mossop" in plan.master_aliases
|
|
20
|
+
assert "phil" in plan.master_aliases
|
|
21
|
+
assert "Philip Mossop" not in plan.master_aliases
|
|
22
|
+
# provenance deduped union
|
|
23
|
+
assert sorted(plan.master_provenance) == ["e1", "e2"]
|
|
24
|
+
assert plan.deprecated_entity_ids == ["l"]
|
|
25
|
+
|
|
26
|
+
def test_fact_subject_and_object_repoints(self):
|
|
27
|
+
master, loser = _ent("m", "Acme"), _ent("l", "ACME Inc")
|
|
28
|
+
facts = [
|
|
29
|
+
{"id": "f1", "subject_entity_id": "l", "object_entity_id": None},
|
|
30
|
+
{"id": "f2", "subject_entity_id": "x", "object_entity_id": "l"},
|
|
31
|
+
{"id": "f3", "subject_entity_id": "x", "object_entity_id": "y"}, # untouched
|
|
32
|
+
]
|
|
33
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=[loser], facts=facts, relationships=[])
|
|
34
|
+
assert plan.fact_subject_repoints == ["f1"]
|
|
35
|
+
assert plan.fact_object_repoints == ["f2"]
|
|
36
|
+
|
|
37
|
+
def test_relationship_repoint_without_collision(self):
|
|
38
|
+
master, loser = _ent("m", "Bob"), _ent("l", "Bobby")
|
|
39
|
+
rels = [{"id": "r1", "from_entity_id": "l", "to_entity_id": "z",
|
|
40
|
+
"relationship_type": "works_for", "weight": 2.0, "provenance_event_ids": ["e1"]}]
|
|
41
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=[loser], facts=[], relationships=rels)
|
|
42
|
+
assert plan.rel_endpoint_repoints == ["r1"]
|
|
43
|
+
assert plan.rel_collisions == []
|
|
44
|
+
|
|
45
|
+
def test_relationship_collision_sums_weight_and_unions_provenance(self):
|
|
46
|
+
# master already has (m -> z, works_for); loser has (l -> z, works_for).
|
|
47
|
+
# Repointing l->m collides; keep one, sum weights, union provenance, drop other.
|
|
48
|
+
master, loser = _ent("m", "Bob"), _ent("l", "Bobby")
|
|
49
|
+
rels = [
|
|
50
|
+
{"id": "r_master", "from_entity_id": "m", "to_entity_id": "z",
|
|
51
|
+
"relationship_type": "works_for", "weight": 3.0, "provenance_event_ids": ["e1"]},
|
|
52
|
+
{"id": "r_loser", "from_entity_id": "l", "to_entity_id": "z",
|
|
53
|
+
"relationship_type": "works_for", "weight": 2.0, "provenance_event_ids": ["e2"]},
|
|
54
|
+
]
|
|
55
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=[loser], facts=[], relationships=rels)
|
|
56
|
+
assert len(plan.rel_collisions) == 1
|
|
57
|
+
c = plan.rel_collisions[0]
|
|
58
|
+
assert c["keep"] == "r_master" and c["drop"] == "r_loser"
|
|
59
|
+
assert c["summed_weight"] == 5.0
|
|
60
|
+
assert sorted(c["provenance"]) == ["e1", "e2"]
|
|
61
|
+
|
|
62
|
+
def test_audit_rows_carry_rollback_payload(self):
|
|
63
|
+
master, loser = _ent("m", "Real"), _ent("l", "Dupe", prov=["e9"])
|
|
64
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=[loser], facts=[], relationships=[])
|
|
65
|
+
assert len(plan.audit_rows) == 1
|
|
66
|
+
row = plan.audit_rows[0]
|
|
67
|
+
assert row["canonical_id"] == "m" and row["deprecated_id"] == "l"
|
|
68
|
+
assert row["rollback_payload"] == loser # full row preserved
|
|
69
|
+
|
|
70
|
+
def test_master_cannot_be_loser(self):
|
|
71
|
+
m = _ent("m", "X")
|
|
72
|
+
with pytest.raises(ValueError):
|
|
73
|
+
build_entity_merge_plan(arena="a", master=m, losers=[m], facts=[], relationships=[])
|
|
74
|
+
|
|
75
|
+
def test_multi_loser_merge(self):
|
|
76
|
+
master = _ent("m", "Katie Cooper", prov=["e1"])
|
|
77
|
+
losers = [_ent("l1", "1716801984", prov=["e2"]), _ent("l2", "K. Cooper", prov=["e3"])]
|
|
78
|
+
plan = build_entity_merge_plan(arena="a", master=master, losers=losers, facts=[], relationships=[])
|
|
79
|
+
assert plan.deprecated_entity_ids == ["l1", "l2"]
|
|
80
|
+
assert sorted(plan.master_provenance) == ["e1", "e2", "e3"]
|
|
81
|
+
assert len(plan.audit_rows) == 2
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class TestFactMergePlan:
|
|
85
|
+
def test_picks_highest_confidence_master(self):
|
|
86
|
+
dups = [
|
|
87
|
+
{"id": "f1", "confidence": 0.5, "statement": "short", "provenance_event_ids": ["e1"]},
|
|
88
|
+
{"id": "f2", "confidence": 0.9, "statement": "the better one", "provenance_event_ids": ["e2"]},
|
|
89
|
+
]
|
|
90
|
+
plan = build_fact_merge_plan(arena="a", dup_facts=dups)
|
|
91
|
+
assert plan["master_id"] == "f2"
|
|
92
|
+
assert plan["deprecated_ids"] == ["f1"]
|
|
93
|
+
assert sorted(plan["master_provenance"]) == ["e1", "e2"]
|
|
94
|
+
|
|
95
|
+
def test_single_fact_no_merge(self):
|
|
96
|
+
assert build_fact_merge_plan(arena="a", dup_facts=[{"id": "f1"}]) is None
|
|
97
|
+
|
|
98
|
+
def test_tie_breaks_on_statement_length(self):
|
|
99
|
+
dups = [
|
|
100
|
+
{"id": "f1", "confidence": 0.7, "statement": "x", "provenance_event_ids": []},
|
|
101
|
+
{"id": "f2", "confidence": 0.7, "statement": "longer statement", "provenance_event_ids": []},
|
|
102
|
+
]
|
|
103
|
+
plan = build_fact_merge_plan(arena="a", dup_facts=dups)
|
|
104
|
+
assert plan["master_id"] == "f2"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class TestEvictionReceipt:
|
|
108
|
+
def test_carries_full_row(self):
|
|
109
|
+
row = {"id": "e1", "arena": "a", "canonical_name": "Ghost"}
|
|
110
|
+
r = build_eviction_receipt("entity", row)
|
|
111
|
+
assert r["node_kind"] == "entity" and r["node_id"] == "e1"
|
|
112
|
+
assert r["rollback_payload"] == row
|