@pentatonic-ai/ai-agent-sdk 0.10.1 → 0.10.2

package/packages/memory-engine-v2/org-model/migrations/002_entity_merges_audit.sql

@@ -0,0 +1,53 @@
+-- pentatonic-memory-engine v2: entity reconciliation audit table.
+--
+-- Backs the entity-reconciliation RFC's backfill (step 3) and the
+-- online merge path. Every time a duplicate entity row is merged
+-- into a canonical row, one record is written here recording:
+--
+--   - which canonical row absorbed the merge
+--   - which row was deprecated (deleted from `entities`)
+--   - what signal triggered the merge (co_occurrence | alias_overlap |
+--     heuristic | online_resolver)
+--   - how many facts / relationships were repointed
+--   - a `rollback_payload` JSONB snapshot of the deprecated row's
+--     state, sufficient to recreate it if the merge proves wrong
+--
+-- The losing row in `entities` IS deleted on merge (otherwise the
+-- alias-GIN + canonical-name lookups still find both, defeating the
+-- whole point). This table is the receipt + rollback substrate.
+--
+-- See RFC: packages/memory-engine-v2/RFC-entity-reconciliation.md §3
+
+CREATE TABLE IF NOT EXISTS entity_merges (
+  id                          TEXT PRIMARY KEY,
+  arena                       TEXT NOT NULL,
+  canonical_id                TEXT NOT NULL REFERENCES entities(id) ON DELETE CASCADE,
+  deprecated_id               TEXT NOT NULL,                  -- no FK; row is deleted
+  deprecated_canonical_name   TEXT NOT NULL,                  -- preserve for forensics
+  deprecated_aliases          TEXT[] NOT NULL DEFAULT '{}',   -- preserve for forensics
+  merge_signal                TEXT NOT NULL CHECK (
+    merge_signal IN ('co_occurrence', 'alias_overlap', 'heuristic', 'online_resolver')
+  ),
+  facts_repointed             INTEGER NOT NULL DEFAULT 0,
+  relationships_repointed     INTEGER NOT NULL DEFAULT 0,
+  merged_at                   TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  merged_by                   TEXT NOT NULL,                  -- 'backfill-YYYY-MM'|'online-resolver'
+
+  -- Rollback snapshot: enough state to recreate the deprecated row
+  -- if the merge is judged wrong. Includes everything that was
+  -- specific to the deprecated row (provenance_event_ids etc).
+  rollback_payload            JSONB NOT NULL DEFAULT '{}'::jsonb
+);
+
+-- Look up "what merged into this entity" (canonical-side query).
+CREATE INDEX IF NOT EXISTS idx_entity_merges_canonical
+  ON entity_merges(canonical_id);
+
+-- Look up "was this id ever a separate entity that got merged" so
+-- callers holding a stale id can resolve it forward.
+CREATE INDEX IF NOT EXISTS idx_entity_merges_deprecated
+  ON entity_merges(deprecated_id);
+
+-- Per-arena audit listing (e.g. dry-run reports).
+CREATE INDEX IF NOT EXISTS idx_entity_merges_arena_merged_at
+  ON entity_merges(arena, merged_at DESC);

package/packages/memory-engine-v2/org-model/migrations/003_distillation_traces.sql

@@ -0,0 +1,60 @@
+-- pentatonic-memory-engine v2: distillation trace audit table.
+--
+-- Append-only log of raw LLM teacher I/O per event distillation. We
+-- already keep the *parsed* output as facts/entities/relationships,
+-- but the org-model's normal-form storage is post-merge and
+-- post-filter — perfect for retrieval, wrong shape for training a
+-- student model on the teacher's distribution.
+--
+-- This table is the teacher-distribution record:
+--
+--   user_prompt   = the per-event block we fed into the LLM (i.e.
+--                   build_event_block(i, ev) — what the model SAW)
+--   raw_response  = the per-event slice of the model's pipe-delimited
+--                   output (i.e. everything between `=== event K ===`
+--                   and the next header — what the model PRODUCED)
+--
+-- Together they form a (input, output) pair suitable for fine-tuning
+-- a seq2seq student (BART/FLAN-T5) to mimic the teacher's extraction
+-- behaviour. `system_prompt_hash` lets us segment training data by
+-- teacher version: when the BATCH_SYSTEM_PROMPT changes, don't train
+-- a student on outputs produced under the old prompt.
+--
+-- Forget semantics: ON DELETE CASCADE from events. A FORGET_MEMORY
+-- that deletes the source event also removes its trace — training
+-- data inherits the same right-to-erasure contract as the rest of
+-- the org-model.
+
+CREATE TABLE IF NOT EXISTS distillation_traces (
+  id                  BIGSERIAL PRIMARY KEY,
+  event_id            TEXT NOT NULL REFERENCES events(id) ON DELETE CASCADE,
+
+  -- Teacher I/O for THIS event. Both are bounded by MAX_CONTENT_CHARS
+  -- + LLM_MAX_TOKENS_PER_EVENT at write time, so storage is
+  -- predictable (~2-3KB/row).
+  user_prompt         TEXT NOT NULL,
+  raw_response        TEXT NOT NULL,
+
+  -- Teacher identity. Lets us filter when the prompt or model changes
+  -- (don't train students on outputs from a retired prompt). Hash is
+  -- truncated sha256(BATCH_SYSTEM_PROMPT)[:16] — long enough to be a
+  -- collision-free identifier, short enough to index cheaply.
+  llm_model           TEXT NOT NULL,
+  system_prompt_hash  TEXT NOT NULL,
+
+  -- LLM call latency (chunk-level — one LLM call distills N events
+  -- in one request). Useful for setting student-latency targets.
+  llm_chunk_ms        REAL,
+
+  created_at          TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_distillation_traces_event_id
+  ON distillation_traces(event_id);
+
+CREATE INDEX IF NOT EXISTS idx_distillation_traces_created_at
+  ON distillation_traces(created_at DESC);
+
+-- Segment by teacher version when exporting training data.
+CREATE INDEX IF NOT EXISTS idx_distillation_traces_prompt_hash
+  ON distillation_traces(system_prompt_hash);

package/packages/memory-engine-v2/scripts/backfill_entity_reconciliation.py

@@ -0,0 +1,581 @@
+#!/usr/bin/env python3
+"""Backfill: merge fragmented person entities (and optionally other
+types) into a single canonical row per person.
+
+Implements RFC §3 (backfill merge). Dry-run by default; --apply
+required to actually write. One arena at a time. Audit records every
+merge to the `entity_merges` table with enough state in
+`rollback_payload` to recreate the deprecated row if a merge proves
+wrong.
+
+Grouping signal priority (per RFC):
+
+  1. CO-OCCURRENCE  — events whose `attributes` carry both a name and
+     an email for the same person (gmail from_name+from_email,
+     calendar attendee displayName+email, slack profile rows). This is
+     ground-truth pairing.
+
+  2. ALIAS_OVERLAP  — entity rows where A.canonical_name appears in
+     B.aliases (or vice versa). Catches post-fix merges still missing
+     from the legacy backfill.
+
+  3. HEURISTIC       — (OFF by default; --heuristic-merge to enable)
+     local-part vs name-tokens overlap for cases where no event ever
+     paired the surface forms. Risky; only with explicit operator
+     flag + manual audit of the dry-run.
+
+Usage:
+
+    python3 backfill_entity_reconciliation.py \\
+        --arena <arena-id> \\
+        --pg-dsn postgresql://... \\
+        [--apply]                  # write; default is dry-run
+        [--entity-type person]     # which type to reconcile; default person
+        [--heuristic-merge]        # enable signal 3 (off by default)
+        [--out /tmp/merges.jsonl]  # where to write the merge report
+
+Exit codes:
+  0  success (dry-run report written, or --apply completed)
+  1  partial failure (some merges failed; report shows which)
+  2  bad arguments
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import sys
+import unicodedata
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+
+import psycopg
+import psycopg.rows
+
+
+# ----------------------------------------------------------------------
+# Constants & helpers
+# ----------------------------------------------------------------------
+
+# Person-shaped attribute keys to look up paired (name, email) on
+# events. Producer-agnostic: any source that follows the conventional
+# `<role>_email` / `<role>_name` shape contributes co-occurrence pairs.
+PERSON_ROLE_PAIRS = [
+    ("from_email", "from_name"),
+    ("to_email", "to_name"),
+    ("cc_email", "cc_name"),
+    ("reply_to_email", "reply_to_name"),
+    ("sender_email", "sender_name"),
+    ("organizer_email", "organizer_name"),
+    ("organizer_email", "organizer_display_name"),
+]
+EMAIL_RE = re.compile(r"\b([a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,})\b")
+HEURISTIC_LOCAL_PART_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9._\-]*)@")
+
+
+def _normalize_surface(s: str) -> str:
+    """MIRROR of extractor-{sync,async}'s _normalize_surface."""
+    return re.sub(r"\s+", " ", unicodedata.normalize("NFKC", s)).strip().lower()
+
+
+def _looks_like_email(s: str) -> bool:
+    return isinstance(s, str) and "@" in s and " " not in s and "." in s.split("@", 1)[-1]
+
+
+def _local_part(email: str) -> str:
+    m = HEURISTIC_LOCAL_PART_RE.match(email)
+    return m.group(1) if m else ""
+
+
+def _name_tokens(name: str) -> set[str]:
+    """Split a display name into lowercased tokens for heuristic match."""
+    return {t for t in re.split(r"[\s\.\-_]+", name.lower()) if t}
+
+
+def _local_part_tokens(local: str) -> set[str]:
+    return {t for t in re.split(r"[\.\-_]+", local.lower()) if t}
+
+
+# ----------------------------------------------------------------------
+# Data classes
+# ----------------------------------------------------------------------
+
+@dataclass
+class Entity:
+    id: str
+    canonical_name: str
+    aliases: list[str]
+    provenance_event_ids: list[str]
+    fact_count: int = 0       # filled later
+    rel_count: int = 0        # filled later
+    # All surface forms (normalized) for fast set-overlap matching.
+    norm_forms: set[str] = field(default_factory=set)
+
+
+@dataclass
+class MergeProposal:
+    canonical: Entity
+    deprecated: list[Entity]
+    signal: str   # 'co_occurrence' | 'alias_overlap' | 'heuristic'
+
+
+# ----------------------------------------------------------------------
+# Load + signal extraction
+# ----------------------------------------------------------------------
+
+def load_entities(conn: psycopg.Connection, arena: str, entity_type: str) -> list[Entity]:
+    """Pull all entities of the given type for this arena, with
+    fact_count + rel_count for canonical-selection (richest wins)."""
+    out: list[Entity] = []
+    with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:
+        cur.execute(
+            """
+            WITH ent AS (
+              SELECT id, canonical_name, aliases, provenance_event_ids
+              FROM entities
+              WHERE arena = %s AND entity_type = %s
+            ),
+            f AS (
+              SELECT subject_entity_id AS eid, COUNT(*) AS n FROM facts
+              WHERE arena = %s GROUP BY 1
+              UNION ALL
+              SELECT object_entity_id AS eid, COUNT(*) AS n FROM facts
+              WHERE arena = %s AND object_entity_id IS NOT NULL GROUP BY 1
+            ),
+            r AS (
+              SELECT from_entity_id AS eid, COUNT(*) AS n FROM relationships
+              WHERE arena = %s GROUP BY 1
+              UNION ALL
+              SELECT to_entity_id AS eid, COUNT(*) AS n FROM relationships
+              WHERE arena = %s GROUP BY 1
+            )
+            SELECT
+              ent.id, ent.canonical_name, ent.aliases, ent.provenance_event_ids,
+              COALESCE((SELECT SUM(n) FROM f WHERE eid = ent.id), 0) AS fact_count,
+              COALESCE((SELECT SUM(n) FROM r WHERE eid = ent.id), 0) AS rel_count
+            FROM ent
+            """,
+            (arena, entity_type, arena, arena, arena, arena),
+        )
+        for r in cur.fetchall():
+            forms = {_normalize_surface(r["canonical_name"])}
+            for a in r["aliases"] or []:
+                forms.add(_normalize_surface(a))
+            out.append(Entity(
+                id=r["id"],
+                canonical_name=r["canonical_name"],
+                aliases=list(r["aliases"] or []),
+                provenance_event_ids=list(r["provenance_event_ids"] or []),
+                fact_count=int(r["fact_count"]),
+                rel_count=int(r["rel_count"]),
+                norm_forms=forms,
+            ))
+    return out
+
+
+def collect_cooccurrence_pairs(
+    conn: psycopg.Connection, arena: str
+) -> set[tuple[str, str]]:
+    """Scan events.attributes for paired (name, email) where both
+    appear for the same person in the same event. Returns
+    set of (normalized_name, normalized_email) pairs."""
+    pairs: set[tuple[str, str]] = set()
+    with conn.cursor(row_factory=psycopg.rows.dict_row) as cur:
+        cur.execute(
+            "SELECT attributes FROM events WHERE arena = %s",
+            (arena,),
+        )
+        for row in cur:
+            attrs = row["attributes"] or {}
+            if not isinstance(attrs, dict):
+                continue
+            for email_key, name_key in PERSON_ROLE_PAIRS:
+                email = attrs.get(email_key)
+                name = attrs.get(name_key)
+                if isinstance(email, str) and isinstance(name, str) \
+                        and _looks_like_email(email) and name.strip():
+                    pairs.add((_normalize_surface(name), _normalize_surface(email)))
+            # Structured attendee objects (calendar producers).
+            attendees = attrs.get("attendees") or attrs.get("attendee_objects")
+            if isinstance(attendees, list):
+                for a in attendees:
+                    if isinstance(a, dict):
+                        email = a.get("email")
+                        name = a.get("displayName") or a.get("name")
+                        if isinstance(email, str) and isinstance(name, str) \
+                                and _looks_like_email(email) and name.strip():
+                            pairs.add((_normalize_surface(name),
+                                       _normalize_surface(email)))
+    return pairs
+
+
+# ----------------------------------------------------------------------
+# Merge proposal building
+# ----------------------------------------------------------------------
+
+def build_proposals(
+    entities: list[Entity],
+    cooccurrence_pairs: set[tuple[str, str]],
+    use_heuristic: bool,
+) -> list[MergeProposal]:
+    """Union-find over entities grouped by overlap of any of:
+      1. co-occurrence pairs (highest priority)
+      2. alias overlap (cheap, ground-truth via existing aliases)
+      3. heuristic local-part vs name-tokens (optional)
+
+    Returns one MergeProposal per group with >= 2 entities; the
+    `signal` is set to the *highest-priority* signal that contributed.
+    """
+    # parent: entity-id → root-id (union-find)
+    parent: dict[str, str] = {e.id: e.id for e in entities}
+    # Track which signal connected each pair; resolved per-group at the end.
+    edge_signal: dict[frozenset, str] = {}
+
+    def find(x: str) -> str:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(a: str, b: str, signal: str) -> None:
+        ra, rb = find(a), find(b)
+        if ra == rb:
+            return
+        parent[ra] = rb
+        edge_signal[frozenset({ra, rb})] = signal
+
+    # ---- Signal 1: co-occurrence pairs ---------------------------------
+    # Build a (normalized_form → entity_id) lookup, then for each
+    # (name, email) pair, if both forms map to entities, union them.
+    form_to_entity: dict[str, str] = {}
+    for e in entities:
+        for f in e.norm_forms:
+            # First-write wins on collision; co-occurrence-driven
+            # merges happen below anyway.
+            form_to_entity.setdefault(f, e.id)
+    for n_name, n_email in cooccurrence_pairs:
+        eid_name = form_to_entity.get(n_name)
+        eid_email = form_to_entity.get(n_email)
+        if eid_name and eid_email and eid_name != eid_email:
+            union(eid_name, eid_email, "co_occurrence")
+
+    # ---- Signal 2: alias overlap ---------------------------------------
+    # Two entities that share any normalized form (one's canonical
+    # appears in the other's aliases, etc.) should already be one.
+    # Group by each form and union all entities sharing it.
+    form_to_entities: dict[str, set[str]] = defaultdict(set)
+    for e in entities:
+        for f in e.norm_forms:
+            form_to_entities[f].add(e.id)
+    for ents in form_to_entities.values():
+        if len(ents) <= 1:
+            continue
+        ents_list = sorted(ents)
+        for other in ents_list[1:]:
+            union(ents_list[0], other, "alias_overlap")
+
+    # ---- Signal 3 (optional): heuristic local-part vs name tokens ------
+    if use_heuristic:
+        # For every email-only canonical that hasn't been unioned via
+        # 1 or 2, try matching its local-part tokens against
+        # name-canonicals' tokens. Last-resort; can produce false
+        # positives (e.g. 'sam' matches 'Sam Patel' AND 'Sam Jones').
+        # Operator MUST eyeball the dry-run report.
+        email_entities: list[Entity] = [
+            e for e in entities
+            if _looks_like_email(e.canonical_name)
+        ]
+        name_entities_by_token: dict[str, list[Entity]] = defaultdict(list)
+        for e in entities:
+            if not _looks_like_email(e.canonical_name):
+                for t in _name_tokens(e.canonical_name):
+                    name_entities_by_token[t].append(e)
+        for ee in email_entities:
+            local = _local_part(ee.canonical_name)
+            tokens = _local_part_tokens(local)
+            candidates: dict[str, int] = defaultdict(int)
+            for t in tokens:
+                for ne in name_entities_by_token.get(t, []):
+                    if find(ee.id) == find(ne.id):
+                        continue
+                    candidates[ne.id] += 1
+            # Require >= 2 token-overlap to consider, OR a single
+            # token that's both >= 3 chars and appears in only one
+            # candidate (unambiguous nickname-style).
+            best: tuple[int, str] | None = None
+            for cand_id, hits in candidates.items():
+                if hits >= 2 or (hits == 1 and len(tokens) == 1
+                                  and len(next(iter(tokens))) >= 3
+                                  and len(candidates) == 1):
+                    if best is None or hits > best[0]:
+                        best = (hits, cand_id)
+            if best is not None:
+                union(ee.id, best[1], "heuristic")
+
+    # ---- Materialise groups → proposals --------------------------------
+    groups: dict[str, list[Entity]] = defaultdict(list)
+    for e in entities:
+        groups[find(e.id)].append(e)
+
+    proposals: list[MergeProposal] = []
+    for group in groups.values():
+        if len(group) < 2:
+            continue
+        # Canonical = richest (most facts, then most rels, then most
+        # provenance events, then lex-smallest id for determinism).
+        group_sorted = sorted(
+            group,
+            key=lambda e: (-e.fact_count, -e.rel_count,
+                           -len(e.provenance_event_ids), e.id),
+        )
+        canonical = group_sorted[0]
+        deprecated = group_sorted[1:]
+
+        # Choose strongest signal that connected this group.
+        ids = {e.id for e in group}
+        signals_in_group = {
+            sig for edge, sig in edge_signal.items()
+            if edge & ids
+        }
+        signal_priority = ("co_occurrence", "alias_overlap", "heuristic")
+        chosen = next((s for s in signal_priority if s in signals_in_group),
+                      "alias_overlap")
+        proposals.append(MergeProposal(canonical=canonical,
+                                       deprecated=deprecated,
+                                       signal=chosen))
+    return proposals
+
+
+# ----------------------------------------------------------------------
+# Apply (--apply)
+# ----------------------------------------------------------------------
+
+def apply_proposals(
+    conn: psycopg.Connection,
+    arena: str,
+    proposals: list[MergeProposal],
+    merged_by: str,
+) -> tuple[int, int, list[str]]:
+    """Apply merges in one transaction per proposal (so a failure
+    doesn't roll back successful merges in the same batch).
+
+    Returns (succeeded_count, failed_count, errors)."""
+    succeeded = 0
+    failed = 0
+    errors: list[str] = []
+    for p in proposals:
+        try:
+            with conn.transaction():
+                with conn.cursor() as cur:
+                    # Lock the canonical row + every deprecated row to
+                    # avoid concurrent online-resolver writes during
+                    # the merge.
+                    ids = [p.canonical.id, *(d.id for d in p.deprecated)]
+                    cur.execute(
+                        "SELECT id FROM entities WHERE id = ANY(%s) FOR UPDATE",
+                        (ids,),
+                    )
+                    for dep in p.deprecated:
+                        # Repoint facts.
+                        cur.execute(
+                            """
+                            UPDATE facts SET subject_entity_id = %s
+                            WHERE arena = %s AND subject_entity_id = %s
+                            """,
+                            (p.canonical.id, arena, dep.id),
+                        )
+                        facts_repointed = cur.rowcount
+                        cur.execute(
+                            """
+                            UPDATE facts SET object_entity_id = %s
+                            WHERE arena = %s AND object_entity_id = %s
+                            """,
+                            (p.canonical.id, arena, dep.id),
+                        )
+                        facts_repointed += cur.rowcount
+                        # Repoint relationships.
+                        cur.execute(
+                            """
+                            UPDATE relationships SET from_entity_id = %s
+                            WHERE arena = %s AND from_entity_id = %s
+                            """,
+                            (p.canonical.id, arena, dep.id),
+                        )
+                        rels_repointed = cur.rowcount
+                        cur.execute(
+                            """
+                            UPDATE relationships SET to_entity_id = %s
+                            WHERE arena = %s AND to_entity_id = %s
+                            """,
+                            (p.canonical.id, arena, dep.id),
+                        )
+                        rels_repointed += cur.rowcount
+
+                        # Merge aliases + provenance into canonical.
+                        cur.execute(
+                            """
+                            UPDATE entities SET
+                              aliases = ARRAY(SELECT DISTINCT UNNEST(
+                                aliases || %s::text[] || ARRAY[%s]
+                              )),
+                              provenance_event_ids = ARRAY(SELECT DISTINCT UNNEST(
+                                provenance_event_ids || %s::text[]
+                              )),
+                              last_seen = NOW()
+                            WHERE id = %s
+                            """,
+                            (dep.aliases, dep.canonical_name,
+                             dep.provenance_event_ids, p.canonical.id),
+                        )
+
+                        # Audit + rollback payload.
+                        rollback_payload = {
+                            "id": dep.id,
+                            "canonical_name": dep.canonical_name,
+                            "aliases": dep.aliases,
+                            "provenance_event_ids": dep.provenance_event_ids,
+                        }
+                        merge_id = "m_" + hashlib.sha256(
+                            f"{arena}|{dep.id}|{p.canonical.id}".encode()
+                        ).hexdigest()[:24]
+                        cur.execute(
+                            """
+                            INSERT INTO entity_merges (
+                              id, arena, canonical_id, deprecated_id,
+                              deprecated_canonical_name, deprecated_aliases,
+                              merge_signal, facts_repointed,
+                              relationships_repointed, merged_by, rollback_payload
+                            ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb)
+                            ON CONFLICT (id) DO NOTHING
+                            """,
+                            (
+                                merge_id, arena, p.canonical.id, dep.id,
+                                dep.canonical_name, dep.aliases,
+                                p.signal, facts_repointed, rels_repointed,
+                                merged_by, json.dumps(rollback_payload),
+                            ),
+                        )
+
+                        # Delete the deprecated row.
+                        cur.execute(
+                            "DELETE FROM entities WHERE id = %s",
+                            (dep.id,),
+                        )
+            succeeded += 1
+        except Exception as e:
+            failed += 1
+            errors.append(f"{p.canonical.id} <- {[d.id for d in p.deprecated]}: {e}")
+    return succeeded, failed, errors
+
+
+# ----------------------------------------------------------------------
+# Report writer
+# ----------------------------------------------------------------------
+
+def write_report(proposals: list[MergeProposal], path: str) -> None:
+    """JSONL with one record per proposal. Operator inspects this
+    before --apply."""
+    with open(path, "w") as f:
+        for p in proposals:
+            f.write(json.dumps({
+                "canonical": {
+                    "id": p.canonical.id,
+                    "canonical_name": p.canonical.canonical_name,
+                    "fact_count": p.canonical.fact_count,
+                    "rel_count": p.canonical.rel_count,
+                },
+                "deprecated": [
+                    {
+                        "id": d.id,
+                        "canonical_name": d.canonical_name,
+                        "aliases": d.aliases,
+                        "fact_count": d.fact_count,
+                        "rel_count": d.rel_count,
+                    } for d in p.deprecated
+                ],
+                "signal": p.signal,
+            }) + "\n")
+
+
+# ----------------------------------------------------------------------
+# CLI
+# ----------------------------------------------------------------------
+
+def parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    p.add_argument("--arena", required=True,
+                   help="arena id to reconcile (one at a time)")
+    p.add_argument("--pg-dsn", default=os.environ.get("PG_DSN"),
+                   help="postgres DSN; defaults to $PG_DSN")
+    p.add_argument("--entity-type", default="person",
+                   help="entity type to reconcile (default: person)")
+    p.add_argument("--apply", action="store_true",
+                   help="actually run the merges; default is dry-run")
+    p.add_argument("--heuristic-merge", action="store_true",
+                   help="enable signal 3 (off by default; risky)")
+    p.add_argument("--out", default=None,
+                   help="merge-report JSONL path (default: stdout-only "
+                        "summary, no jsonl)")
+    p.add_argument("--merged-by", default=None,
+                   help="audit tag (default: backfill-YYYY-MM)")
+    return p.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    if not args.pg_dsn:
+        print("error: --pg-dsn (or $PG_DSN) required", file=sys.stderr)
+        return 2
+
+    merged_by = args.merged_by or f"backfill-{datetime.now(timezone.utc):%Y-%m}"
+
+    with psycopg.connect(args.pg_dsn, autocommit=False) as conn:
+        print(f"[backfill] arena={args.arena} type={args.entity_type} "
+              f"apply={args.apply} heuristic={args.heuristic_merge}")
+        entities = load_entities(conn, args.arena, args.entity_type)
+        print(f"[backfill] loaded {len(entities)} {args.entity_type} entities")
+
+        cooc = collect_cooccurrence_pairs(conn, args.arena) \
+            if args.entity_type == "person" else set()
+        print(f"[backfill] collected {len(cooc)} co-occurrence pairs from events")
+
+        proposals = build_proposals(entities, cooc, args.heuristic_merge)
+        print(f"[backfill] built {len(proposals)} merge proposals "
+              f"({sum(len(p.deprecated) for p in proposals)} rows would deprecate)")
+
+        # Summarise by signal.
+        by_signal: dict[str, int] = defaultdict(int)
+        for p in proposals:
+            by_signal[p.signal] += 1
+        for sig, n in sorted(by_signal.items()):
+            print(f"  - {sig}: {n} groups")
+
+        if args.out:
+            write_report(proposals, args.out)
+            print(f"[backfill] wrote merge report → {args.out}")
+
+        if not args.apply:
+            print("[backfill] dry-run only; pass --apply to execute")
+            return 0
+
+        succeeded, failed, errors = apply_proposals(
+            conn, args.arena, proposals, merged_by
+        )
+        conn.commit()
+        print(f"[backfill] applied: {succeeded} succeeded, {failed} failed")
+        for err in errors[:20]:
+            print(f"  ERR: {err}")
+        if len(errors) > 20:
+            print(f"  ... and {len(errors) - 20} more (see --out for full report)")
+        return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())