@pentatonic-ai/ai-agent-sdk 0.10.19 → 0.10.21

package/packages/memory-engine-v2/scripts/build_retrain_corpus.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""Build a student retrain corpus from CLEAN teacher gold.
+
+The student (NuExtract-2.0-4B FT) was originally trained on teacher traces
+produced under the *old* distiller prompts (bbdaba / f1e0ff), which had no
+email-discipline or modality rules — so the student learned to (a) promote
+bystander emails into a person's aliases (the Johann/hotel over-merge), (b)
+collapse future/invited roles to established `state` facts, (c) conflate
+"X & Y" into one entity, and (d) mint email-named / generic-infra entities.
+
+This builder draws ONLY from traces produced under the clean prompt (#126
+modality/attribution + #129 email-discipline & entity-separation), whose
+`system_prompt_hash` is the deployed clean hash. Building from old-prompt
+traces would just re-teach the defects, so that is explicitly NOT the default
+(you must pass --allow-dirty-hash to override, and you shouldn't).
+
+A defect filter runs as a SECOND line of defence: even clean-prompt output is
+screened for the known defect signatures and dropped if any survive. Every
+drop is counted by reason so the corpus's cleanliness is auditable (not a
+black box — see the printed report).
+
+INPUT — an NDJSON stream of trace rows, one object per line:
+    {"event_id": "...", "user_prompt": "...", "raw_response": "...",
+     "system_prompt_hash": "..."}
+Produce it from the engine box's org_model DB with row_to_json (the escaping
+that bit us before — \\copy double-escapes, $-quoting gets eaten by the shell —
+is avoided by -At + row_to_json):
+
+    sudo docker exec -i pme2-org-model psql -U pme -d org_model -At -c \\
+      "SELECT row_to_json(t) FROM (
+         SELECT event_id, user_prompt, raw_response, system_prompt_hash
+         FROM distillation_traces
+         WHERE system_prompt_hash = '6ccfe70f1286a131'
+       ) t" > traces.ndjson
+
+OUTPUT — {"input": <per-event block>, "output": <extraction JSON string>}
+JSONL(.gz), the exact shape train_lora.py's load() consumes (it keeps rows
+where both `input` and `output` are truthy, then trains user=input ->
+assistant=output via the NuExtract chat template; no system prompt in the
+pair). The corpus is PER-EVENT while a trace is a 3-event chunk, so each
+trace's user_prompt is split on the `[event K]` markers and matched to
+raw_response[index == K].
+
+Usage:
+    python build_retrain_corpus.py --traces traces.ndjson --out retrain_clean.jsonl.gz
+    zcat traces.ndjson.gz | python build_retrain_corpus.py --traces - --out c.jsonl.gz
+"""
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import json
+import re
+import sys
+from collections import Counter
+
+# The clean prompt deployed as SDK 0.10.19 (#126 + #129). Verify against the
+# running extractor-async (worker.SYSTEM_PROMPT_HASH) before a real corpus cut —
+# a prompt edit advances this and old-hash traces must not silently leak in.
+CLEAN_PROMPT_HASH = "6ccfe70f1286a131"
+
+# Generic infra / environment tokens that must never be standalone entities
+# (mirrors the #129 DISTINCT ENTITIES rule — kept in sync by hand).
+INFRA_TOKENS = {
+    "prod", "production", "staging", "stage", "uat", "qa", "dev", "test",
+    "warehouse", "datalake", "data lake", "cluster", "backend", "frontend",
+    "the system", "the platform", "the api", "the database", "the server",
+}
+
+EVENT_BLOCK_RE = re.compile(r"(?=^\[event \d+\])", re.MULTILINE)
+EVENT_IDX_RE = re.compile(r"^\[event (\d+)\]")
+EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+def _email_plausibly_belongs(person_name: str, email: str) -> bool:
+    """Keep an email on a person only if it plausibly is theirs: a name token
+    appears in the local-part, or the initials match. Same heuristic as the
+    write-side guard (#128) so corpus filtering and runtime agree."""
+    local = email.split("@", 1)[0].lower()
+    local_alnum = re.sub(r"[^a-z0-9]", "", local)
+    tokens = [t for t in re.split(r"\s+", person_name.lower()) if t]
+    if not tokens:
+        return False
+    for t in tokens:
+        t_alnum = re.sub(r"[^a-z0-9]", "", t)
+        if len(t_alnum) >= 3 and t_alnum in local_alnum:
+            return True
+    initials = "".join(t[0] for t in tokens if t)
+    if len(initials) >= 2 and initials in local_alnum:
+        return True
+    return False
+
+
+def _entity_defect(ent: dict) -> str | None:
+    """Return a drop-reason if this entity carries a known defect, else None."""
+    name = (ent.get("name") or "").strip()
+    etype = (ent.get("type") or "").lower()
+    if not name:
+        return "empty_entity_name"
+    if EMAIL_RE.match(name):
+        return "email_as_entity"
+    if name.lower() in INFRA_TOKENS:
+        return "generic_infra_entity"
+    # Conflation: "Acme & Globex" / "Alice and Bob" smuggled into one node.
+    if re.search(r"\s&\s", name) or re.search(r"\b and \b", name.lower()):
+        return "conflated_entity"
+    if etype == "person":
+        for a in ent.get("aliases") or []:
+            if isinstance(a, str) and "@" in a and " " not in a \
+                    and not _email_plausibly_belongs(name, a):
+                return "bystander_email_alias"
+    return None
+
+
+def _output_is_clean(obj: dict) -> str | None:
+    """Screen one per-event extraction object; return a drop-reason or None."""
+    if not isinstance(obj, dict):
+        return "output_not_object"
+    for ent in obj.get("entities") or []:
+        r = _entity_defect(ent)
+        if r:
+            return r
+    return None
+
+
+def split_events(user_prompt: str) -> dict[int, str]:
+    """Split a chunk prompt into {event_index: block_text}."""
+    blocks: dict[int, str] = {}
+    for block in EVENT_BLOCK_RE.split(user_prompt):
+        block = block.rstrip()
+        m = EVENT_IDX_RE.match(block)
+        if m:
+            blocks[int(m.group(1))] = block
+    return blocks
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--traces", required=True,
+                    help="NDJSON trace rows, or '-' for stdin")
+    ap.add_argument("--out", required=True, help="output .jsonl.gz")
+    ap.add_argument("--hash", default=CLEAN_PROMPT_HASH,
+                    help=f"keep only this system_prompt_hash (default {CLEAN_PROMPT_HASH})")
+    ap.add_argument("--allow-dirty-hash", action="store_true",
+                    help="do NOT filter by hash — DANGER: re-teaches old-prompt defects")
+    ap.add_argument("--report", help="optional path for a JSON stats report")
+    args = ap.parse_args()
+
+    fh = sys.stdin if args.traces == "-" else open(args.traces, encoding="utf-8")
+    stats = Counter()
+    seen: set[str] = set()
+    examples: list[dict] = []
+
+    for line in fh:
+        line = line.strip()
+        if not line:
+            continue
+        stats["trace_rows"] += 1
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError:
+            stats["drop_trace_unparseable"] += 1
+            continue
+
+        if not args.allow_dirty_hash and row.get("system_prompt_hash") != args.hash:
+            stats["drop_wrong_hash"] += 1
+            continue
+
+        raw = row.get("raw_response") or ""
+        try:
+            parsed = json.loads(raw)
+        except json.JSONDecodeError:
+            stats["drop_response_unparseable"] += 1
+            continue
+        # raw_response is either a single per-event object (current trace
+        # format — one row per event) or, for legacy chunked traces, a JSON
+        # array / {"events": [...]} of per-event objects.
+        if isinstance(parsed, dict):
+            objs = parsed.get("events") if isinstance(parsed.get("events"), list) else [parsed]
+        elif isinstance(parsed, list):
+            objs = parsed
+        else:
+            stats["drop_response_shape"] += 1
+            continue
+
+        blocks = split_events(row.get("user_prompt") or "")
+        for obj in objs:
+            if not isinstance(obj, dict):
+                stats["drop_obj_not_object"] += 1
+                continue
+            idx = obj.get("index")
+            block = blocks.get(idx) if idx is not None else None
+            # Single-event trace: one block, one object — match by position
+            # even if the stored index doesn't line up with the marker.
+            if block is None and len(objs) == 1 and len(blocks) == 1:
+                block = next(iter(blocks.values()))
+            if not block:
+                stats["drop_no_matching_block"] += 1
+                continue
+
+            key = hashlib.sha1(block.encode("utf-8")).hexdigest()
+            if key in seen:
+                stats["drop_dup"] += 1
+                continue
+
+            reason = _output_is_clean(obj)
+            if reason:
+                stats[f"drop_{reason}"] += 1
+                continue
+
+            seen.add(key)
+            examples.append({"input": block, "output": json.dumps(obj, ensure_ascii=False)})
+            stats["kept"] += 1
+
+    if args.traces != "-":
+        fh.close()
+
+    with gzip.open(args.out, "wt", encoding="utf-8") as out:
+        for ex in examples:
+            out.write(json.dumps(ex, ensure_ascii=False) + "\n")
+
+    report = {"out": args.out, "hash": (None if args.allow_dirty_hash else args.hash),
+              "stats": dict(sorted(stats.items()))}
+    print(json.dumps(report, indent=2))
+    if args.report:
+        with open(args.report, "w", encoding="utf-8") as rf:
+            json.dump(report, rf, indent=2)
+
+    if stats["kept"] == 0:
+        print("\nWARNING: 0 examples kept. If you targeted the clean hash, the "
+              "clean-prompt teacher has not produced enough gold yet — let it "
+              "accumulate (or run a teacher-only re-distill of a curated event "
+              "slice through the clean prompt), then re-run.", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/packages/memory-engine-v2/scripts/fusion_defrag.py

@@ -0,0 +1,440 @@
+#!/usr/bin/env python3
+"""Fusion de-fragmentation — cluster same-surname PERSON fragments and propose
+merges (RFC-decay-and-fusion A2/A3). DRY-RUN by default.
+
+The deterministic upsert resolver (worker.py) converges same-form / shared-alias
+entities, but it CANNOT safely merge surface-form/nickname variants of one real
+person (e.g. "Will Vickers" / "William Vickers" / "William F. Vickers" / bare
+"Vickers") — they have different normalized names and no shared alias, so they
+fragment (209 "Vickers" nodes observed). Merging them is Fusion's job, and it is
+DESTRUCTIVE (repoints facts/relationships, tombstones losers), so the over-merge
+failure mode (folding two DIFFERENT people, or a person into an org) must be
+designed out. This tool is conservative + dry-run-first; --apply is double-gated.
+
+CLUSTERING (anti-over-merge by construction):
+  - PERSON entities only; never crosses entity_type (so "Vickers Oils" the org is
+    never pulled in).
+  - Same surname token (the --surname scope).
+  - First-name compatibility for the NON-surname tokens: equal, or one an initial
+    of the other (W ↔ William), or one a prefix of the other (Will ⊂ William).
+    Two DISTINCT full first names (Will vs Jane) are INCOMPATIBLE → never merged.
+  - Union-find over compatible NON-bare names → each cluster = one real person.
+  - Bare "<surname>" nodes (no first name) are folded in ONLY when there is
+    exactly ONE non-bare cluster for the surname (unambiguous); otherwise they
+    are left for human review (never used to bridge two distinct people).
+
+CANONICAL (A3 scored master, replaces richest-row-wins):
+  + has email (attributes.email or an email alias)   strongest identity signal
+  + full name (>=2 name tokens)                       a real rendering, not a stub
+  + corroboration (provenance event count)            grounded in more events
+  + fact count                                        the node that holds the picture
+  - bare single-token name                            penalize stub
+  - ID-like (digit ratio > 0.5)                       penalize 7B numeric-id junk
+
+OUTPUT: per cluster — master, losers, why, and the repoint impact (facts +
+relationships that would move onto the master). No DB writes in dry-run
+(the session is forced read-only). --apply would execute via the reviewed
+fusion_drive merge executor + entity_merges audit (NOT enabled here).
+
+Usage:
+    python fusion_defrag.py --arena 'pentatonic-team%' --surname vickers
+    python fusion_defrag.py --arena 'pentatonic-team%' --surname vickers --json out.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+import uuid
+from collections import defaultdict
+
+
+def _connect(dsn: str):
+    import psycopg
+    import psycopg.rows
+    return psycopg.connect(dsn, row_factory=psycopg.rows.dict_row)
+
+
+_TOKEN_RE = re.compile(r"[^a-z0-9]+")
+_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
+
+
+def name_tokens(name: str) -> list[str]:
+    return [t for t in _TOKEN_RE.split((name or "").lower()) if t]
+
+
+def digit_ratio(name: str) -> float:
+    s = re.sub(r"\s+", "", name or "")
+    return sum(c.isdigit() for c in s) / len(s) if s else 0.0
+
+
+# Honorifics/titles are NOT first names — strip them so "Herr Johann Boedecker"
+# matches "Johann Boedecker", and a title-only "Herr Boedecker" reduces to a bare
+# surname (held for review, not merged on the title).
+_TITLES = {"herr", "frau", "fr", "dr", "prof", "mr", "mrs", "ms", "miss",
+           "sir", "dame", "mx", "mme", "mlle", "hr"}
+
+
+def first_name_tokens(name: str, surname: str) -> list[str]:
+    """Name tokens minus the surname (first occurrence) and minus honorifics."""
+    toks = name_tokens(name)
+    out, dropped = [], False
+    for t in toks:
+        if not dropped and t == surname:
+            dropped = True
+            continue
+        if t in _TITLES:
+            continue
+        out.append(t)
+    return out
+
+
+def first_names_compatible(a: list[str], b: list[str]) -> bool:
+    """Compatible iff the leading given-name tokens don't CONFLICT. Equal /
+    initial-of / prefix-of are compatible; two distinct full names are not.
+    Empty (bare surname) is handled separately by the caller — NOT here."""
+    if not a or not b:
+        return False  # bare names never auto-bridge via this predicate
+    x, y = a[0], b[0]
+    if x == y:
+        return True
+    # initial ↔ full (w / william)
+    if (len(x) == 1 and y.startswith(x)) or (len(y) == 1 and x.startswith(y)):
+        return True
+    # nickname/prefix (will / william) — require >=3 chars to avoid junk
+    if len(x) >= 3 and y.startswith(x):
+        return True
+    if len(y) >= 3 and x.startswith(y):
+        return True
+    return False
+
+
+class UnionFind:
+    def __init__(self, ids):
+        self.p = {i: i for i in ids}
+
+    def find(self, i):
+        while self.p[i] != i:
+            self.p[i] = self.p[self.p[i]]
+            i = self.p[i]
+        return i
+
+    def union(self, a, b):
+        ra, rb = self.find(a), self.find(b)
+        if ra != rb:
+            self.p[ra] = rb
+
+
+def has_email(ent: dict) -> bool:
+    attrs = ent.get("attributes") or {}
+    if isinstance(attrs, dict) and attrs.get("email"):
+        return True
+    return any(isinstance(a, str) and _EMAIL_RE.match(a) for a in (ent.get("aliases") or []))
+
+
+def master_score(ent: dict, surname: str) -> float:
+    fn = first_name_tokens(ent["canonical_name"], surname)
+    score = 0.0
+    if has_email(ent):
+        score += 3.0
+    if len(name_tokens(ent["canonical_name"])) >= 2:
+        score += 2.0
+    score += min(len(ent.get("provenance_event_ids") or []), 20) * 0.2
+    score += min(ent.get("fact_n", 0), 40) * 0.05
+    if not fn:  # bare surname
+        score -= 3.0
+    if digit_ratio(ent["canonical_name"]) > 0.5:
+        score -= 5.0
+    return score
+
+
+def _union(*lists):
+    seen = {}
+    for lst in lists:
+        for x in lst or []:
+            seen.setdefault(x, None)
+    return list(seen.keys())
+
+
+def apply_cluster(cur, conn, arena: str, master: dict, losers: list[dict]) -> dict:
+    """Fold `losers` into `master` within ONE transaction. Faithful inline of the
+    reviewed fusion_drive _execute_entity_plan + build_entity_merge_plan: repoint
+    facts (subject/object) and relationships (endpoints, summing weight on the
+    post-repoint (from,to,type) collision), accrete aliases+provenance onto the
+    master, write one entity_merges audit row per loser (rollback_payload = full
+    loser row), then delete the losers. Re-validates losers still exist first."""
+    loser_ids = [l["id"] for l in losers]
+    # Load edges/facts touching the losers AND THE MASTER. Including the master is
+    # load-bearing for relationships: a loser edge repointed onto the master can
+    # collide with an edge the master ALREADY has (or another loser's) on the
+    # UNIQUE(arena,from,to,type) key — if we don't see the master's existing edges
+    # in collision detection, the repoint UPDATE hits a duplicate-key violation
+    # (caught the hard way on the first Vickers apply; the txn rolled back clean).
+    # Facts have no such unique key, so master facts are loaded but never repointed
+    # (repoint decisions key on the loser set only).
+    targets = loser_ids + [master["id"]]
+    cur.execute(
+        "SELECT id, subject_entity_id, object_entity_id FROM facts "
+        "WHERE arena = %s AND (subject_entity_id = ANY(%s) OR object_entity_id = ANY(%s))",
+        (arena, targets, targets),
+    )
+    facts = cur.fetchall()
+    cur.execute(
+        "SELECT id, from_entity_id, to_entity_id, relationship_type, weight, "
+        "provenance_event_ids FROM relationships "
+        "WHERE arena = %s AND (from_entity_id = ANY(%s) OR to_entity_id = ANY(%s))",
+        (arena, targets, targets),
+    )
+    rels = cur.fetchall()
+
+    lset = set(loser_ids)
+    aliases = _union(master.get("aliases") or [],
+                     [l["canonical_name"] for l in losers],
+                     *[l.get("aliases") or [] for l in losers])
+    aliases = [a for a in aliases if a != master["canonical_name"]]
+    provenance = _union(master.get("provenance_event_ids") or [],
+                        *[l.get("provenance_event_ids") or [] for l in losers])
+    fact_subj = [f["id"] for f in facts if f["subject_entity_id"] in lset]
+    fact_obj = [f["id"] for f in facts if f["object_entity_id"] in lset]
+
+    def rk(r):
+        frm = master["id"] if r["from_entity_id"] in lset else r["from_entity_id"]
+        to = master["id"] if r["to_entity_id"] in lset else r["to_entity_id"]
+        return (frm, to, r["relationship_type"])
+    by_key, rel_repoints, rel_collisions = {}, [], []
+    for r in rels:
+        touches = r["from_entity_id"] in lset or r["to_entity_id"] in lset
+        key = rk(r)
+        if key in by_key:
+            keep = by_key[key]
+            rel_collisions.append({
+                "keep": keep["id"], "drop": r["id"],
+                "summed_weight": round((keep.get("weight") or 1.0) + (r.get("weight") or 1.0), 4),
+                "provenance": _union(keep.get("provenance_event_ids") or [],
+                                     r.get("provenance_event_ids") or []),
+            })
+        else:
+            by_key[key] = r
+            if touches:
+                rel_repoints.append(r["id"])
+
+    with conn.transaction():
+        live = set()
+        cur.execute("SELECT id FROM entities WHERE id = ANY(%s)", (loser_ids,))
+        live = {r["id"] for r in cur.fetchall()}
+        if live != lset:
+            return {"applied": False, "reason": "stale: some losers already gone"}
+        cur.execute("UPDATE entities SET aliases=%s, provenance_event_ids=%s, last_seen=NOW() "
+                    "WHERE id=%s", (aliases, provenance, master["id"]))
+        for fid in fact_subj:
+            cur.execute("UPDATE facts SET subject_entity_id=%s WHERE id=%s", (master["id"], fid))
+        for fid in fact_obj:
+            cur.execute("UPDATE facts SET object_entity_id=%s WHERE id=%s", (master["id"], fid))
+        # DELETE colliding edges BEFORE repointing — else repointing a "keep" edge
+        # onto the master collides with the not-yet-deleted "drop" on the UNIQUE
+        # (arena,from,to,type) key. Carry each drop's weight+provenance onto its keep.
+        for col in rel_collisions:
+            cur.execute("UPDATE relationships SET weight=%s, provenance_event_ids=%s WHERE id=%s",
+                        (col["summed_weight"], col["provenance"], col["keep"]))
+            cur.execute("DELETE FROM relationships WHERE id=%s", (col["drop"],))
+        for rid in rel_repoints:
+            cur.execute(
+                "UPDATE relationships SET "
+                "from_entity_id = CASE WHEN from_entity_id = ANY(%s) THEN %s ELSE from_entity_id END, "
+                "to_entity_id   = CASE WHEN to_entity_id   = ANY(%s) THEN %s ELSE to_entity_id END "
+                "WHERE id=%s",
+                (loser_ids, master["id"], loser_ids, master["id"], rid))
+        for l in losers:
+            cur.execute(
+                "INSERT INTO entity_merges (id, arena, canonical_id, deprecated_id, "
+                "deprecated_canonical_name, deprecated_aliases, merge_signal, "
+                "facts_repointed, relationships_repointed, merged_by, rollback_payload) "
+                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb)",
+                ("em_" + uuid.uuid4().hex[:20], arena, master["id"], l["id"],
+                 l["canonical_name"], l.get("aliases") or [], "heuristic",
+                 len(fact_subj) + len(fact_obj), len(rel_repoints), "fusion-defrag",
+                 json.dumps(l, default=str)))
+        cur.execute("DELETE FROM entities WHERE id = ANY(%s)", (loser_ids,))
+    return {"applied": True, "facts_repointed": len(fact_subj) + len(fact_obj),
+            "rels_repointed": len(rel_repoints), "rel_collisions": len(rel_collisions),
+            "tombstoned": len(loser_ids)}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--arena", required=True, help="arena LIKE filter (REQUIRED)")
+    ap.add_argument("--surname", required=True, help="surname token to scope (e.g. vickers)")
+    ap.add_argument("--pg-dsn", default="", help="Postgres DSN (or PG_DSN env)")
+    ap.add_argument("--json", help="write the proposal as JSON")
+    ap.add_argument("--apply", action="store_true",
+                    help="EXECUTE the merges (default: dry-run). Requires --i-have-a-snapshot.")
+    ap.add_argument("--i-have-a-snapshot", action="store_true",
+                    help="operator asserts a DB/row snapshot exists for rollback (required with --apply)")
+    args = ap.parse_args()
+    if args.apply and not args.i_have_a_snapshot:
+        print("REFUSED: --apply requires --i-have-a-snapshot (take a snapshot first; "
+              "merges repoint facts/rels + tombstone nodes — entity_merges holds rollback "
+              "payloads but a row/DB snapshot is the real safety net).", file=sys.stderr)
+        return 2
+
+    import os
+    dsn = args.pg_dsn or os.environ.get("PG_DSN", "")
+    if not dsn:
+        print("FATAL: no --pg-dsn / PG_DSN", file=sys.stderr)
+        return 2
+    surname = args.surname.lower()
+
+    with _connect(dsn) as conn:
+        with conn.cursor() as cur:
+            if not args.apply:
+                cur.execute("SET default_transaction_read_only = on")  # dry-run safety
+            cur.execute("SET max_parallel_workers_per_gather = 0")
+            # Person fragments whose CANONICAL NAME carries the surname as a real
+            # token. Critically NOT alias-scoped: an @vickers-oil.com email domain
+            # in aliases would otherwise drag in unrelated employees (Paul Vann,
+            # Matt Tooze) who merely WORK at a Vickers company — the over-merge the
+            # first dry-run caught. Email-named stubs (canonical_name has '@') are
+            # also excluded: tokenizing an email is not a safe surname signal (they
+            # converge via the alias-resolution path instead).
+            cur.execute(
+                """
+                SELECT e.id, e.arena, e.canonical_name, e.aliases, e.provenance_event_ids,
+                       e.attributes, e.last_seen,
+                       (SELECT count(*) FROM facts f
+                          WHERE f.provenance_event_ids && e.provenance_event_ids
+                            AND (f.subject_entity_id = e.id OR f.object_entity_id = e.id)) AS fact_n
+                  FROM entities e
+                 WHERE e.arena LIKE %s AND e.entity_type = 'person'
+                   AND position('@' in e.canonical_name) = 0
+                   AND lower(e.canonical_name) ~ %s
+                """,
+                (args.arena, rf"(^|[^a-z]){surname}([^a-z]|$)"),
+            )
+            ents = cur.fetchall()
+            # Confirm the surname is a real NAME token (regex is a coarse guard).
+            ents = [e for e in ents if surname in name_tokens(e["canonical_name"])]
+            print(f"[defrag] arena={args.arena} surname={surname!r}: "
+                  f"{len(ents)} person fragments")
+            if not ents:
+                return 0
+
+            by_id = {e["id"]: e for e in ents}
+            non_bare = [e for e in ents if first_name_tokens(e["canonical_name"], surname)]
+            bare = [e for e in ents if not first_name_tokens(e["canonical_name"], surname)]
+
+            # Union-find over compatible non-bare names.
+            uf = UnionFind([e["id"] for e in non_bare])
+            for i in range(len(non_bare)):
+                for j in range(i + 1, len(non_bare)):
+                    a, b = non_bare[i], non_bare[j]
+                    # Never union across exact arenas — entity id = hash(arena|
+                    # type|name), so cross-arena same-name nodes are genuinely
+                    # different scoped entities; merging them would be wrong.
+                    if a["arena"] != b["arena"]:
+                        continue
+                    if first_names_compatible(
+                        first_name_tokens(a["canonical_name"], surname),
+                        first_name_tokens(b["canonical_name"], surname),
+                    ):
+                        uf.union(a["id"], b["id"])
+            clusters: dict[str, list[dict]] = defaultdict(list)
+            for e in non_bare:
+                clusters[uf.find(e["id"])].append(e)
+
+            # Bare surnames: fold in ONLY if exactly one non-bare cluster exists.
+            bare_note = ""
+            if bare:
+                if len(clusters) == 1:
+                    only = next(iter(clusters))
+                    cl_arena = clusters[only][0]["arena"]
+                    same = [b for b in bare if b["arena"] == cl_arena]
+                    clusters[only].extend(same)
+                    bare_note = f"{len(same)} bare-'{surname}' node(s) folded into the single cluster"
+                else:
+                    bare_note = (f"{len(bare)} bare-'{surname}' node(s) LEFT FOR REVIEW "
+                                 f"({len(clusters)} distinct name-clusters — ambiguous which person)")
+
+            proposals = []
+            for cid, members in clusters.items():
+                if len(members) < 2:
+                    continue
+                master = max(members, key=lambda e: (master_score(e, surname),
+                                                     len(e.get("provenance_event_ids") or []),
+                                                     len(e["canonical_name"])))
+                losers = [e for e in members if e["id"] != master["id"]]
+                loser_ids = [l["id"] for l in losers]
+                # Repoint impact (read-only counts).
+                cur.execute(
+                    "SELECT count(*) AS n FROM facts WHERE arena LIKE %s AND "
+                    "(subject_entity_id = ANY(%s) OR object_entity_id = ANY(%s))",
+                    (args.arena, loser_ids, loser_ids),
+                )
+                facts_repointed = cur.fetchone()["n"]
+                cur.execute(
+                    "SELECT count(*) AS n FROM relationships WHERE arena LIKE %s AND "
+                    "(from_entity_id = ANY(%s) OR to_entity_id = ANY(%s))",
+                    (args.arena, loser_ids, loser_ids),
+                )
+                rels_repointed = cur.fetchone()["n"]
+                proposals.append({
+                    "arena": master["arena"],
+                    "master_row": master,   # full row for apply
+                    "loser_rows": losers,
+                    "master": {"id": master["id"], "name": master["canonical_name"],
+                               "facts": master.get("fact_n", 0),
+                               "prov": len(master.get("provenance_event_ids") or []),
+                               "email": has_email(master),
+                               "score": round(master_score(master, surname), 2)},
+                    "losers": [{"id": l["id"], "name": l["canonical_name"],
+                                "facts": l.get("fact_n", 0),
+                                "prov": len(l.get("provenance_event_ids") or []),
+                                "email": has_email(l)} for l in losers],
+                    "facts_repointed": facts_repointed,
+                    "rels_repointed": rels_repointed,
+                })
+
+            if args.apply and proposals:
+                conn.rollback()  # end the read-only probe txn cleanly before writes
+                print(f"\n[defrag] APPLYING {len(proposals)} cluster(s) — arena-scoped, transactional…")
+                for p in proposals:
+                    p["apply_result"] = apply_cluster(cur, conn, p["arena"],
+                                                       p["master_row"], p["loser_rows"])
+                    print(f"  master={p['master']['name']!r} ({p['arena']}): {p['apply_result']}")
+
+    # ---- report ----
+    mode = "APPLIED" if args.apply else "PROPOSED (dry-run, no writes)"
+    print(f"\n=== {mode} MERGES — surname '{surname}' ===")
+    if bare_note:
+        print(f"  note: {bare_note}")
+    if not proposals:
+        print("  (no multi-node clusters — nothing to merge)")
+    tot_dep = tot_f = tot_r = 0
+    for i, p in enumerate(proposals, 1):
+        m = p["master"]
+        print(f"\n[{i}] MASTER ← {m['name']!r}  ({m['id'][:10]}…) "
+              f"score={m['score']} facts={m['facts']} prov={m['prov']} email={m['email']}")
+        for l in p["losers"]:
+            print(f"      merge: {l['name']!r}  ({l['id'][:10]}…) "
+                  f"facts={l['facts']} prov={l['prov']} email={l['email']}")
+        print(f"      → would repoint {p['facts_repointed']} facts, "
+              f"{p['rels_repointed']} relationships onto the master; "
+              f"{len(p['losers'])} node(s) tombstoned")
+        tot_dep += len(p["losers"]); tot_f += p["facts_repointed"]; tot_r += p["rels_repointed"]
+    tail = ("APPLIED — rollback via entity_merges (merged_by='fusion-defrag') + snapshot."
+            if args.apply else "DRY-RUN — nothing written.")
+    print(f"\nTOTAL: {len(proposals)} cluster(s), {tot_dep} nodes tombstoned, "
+          f"{tot_f} facts + {tot_r} relationships repointed. {tail}")
+    if args.json:
+        with open(args.json, "w") as f:
+            json.dump({"surname": surname, "arena": args.arena, "applied": args.apply,
+                       "bare_note": bare_note,
+                       "proposals": [{k: v for k, v in p.items()
+                                      if k not in ("master_row", "loser_rows")}
+                                     for p in proposals]}, f, indent=2)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())