@pentatonic-ai/ai-agent-sdk 0.10.7 → 0.10.8

package/packages/memory-engine-v2/fusion_drive/test_salience.py

@@ -0,0 +1,93 @@
+"""Unit tests for Fusion Drive salience scoring + decay (pure, no DB)."""
+
+from __future__ import annotations
+
+import salience as S
+
+
+class TestBornSalience:
+    def test_base_for_single_uncorroborated_clean_node(self):
+        assert S.born_salience(n_sources=1) == S.BASE_SALIENCE
+
+    def test_corroboration_raises_but_caps(self):
+        assert S.born_salience(n_sources=2) > S.BASE_SALIENCE
+        capped = S.born_salience(n_sources=100)
+        assert capped == round(S.BASE_SALIENCE + S.CORROB_CAP, 4)
+
+    def test_junk_is_born_below_decay_sweep_threshold(self):
+        # Any single hard-junk flag must sink it under the 0.3 sweep line
+        # so decay targets pollution without needing a fusion match.
+        for flag in ("noise_name", "numeric_id_person", "hallucinated_email"):
+            assert S.born_salience(n_sources=1, quality_flags=[flag]) < 0.3, flag
+
+    def test_combined_flags_drive_to_floor(self):
+        s = S.born_salience(
+            n_sources=1, quality_flags=["numeric_id_person", "hallucinated_email", "ungrounded"]
+        )
+        assert s == S.SALIENCE_FLOOR
+
+    def test_corroboration_cannot_rescue_hard_junk(self):
+        # A heavily-corroborated numeric-ID person is still junk-leaning.
+        s = S.born_salience(n_sources=5, quality_flags=["numeric_id_person"])
+        assert s < S.BASE_SALIENCE
+
+    def test_never_exceeds_ceiling_or_below_floor(self):
+        assert S.born_salience(n_sources=1000) <= S.SALIENCE_CEIL
+        assert S.born_salience(n_sources=1, quality_flags=["noise_name"] * 10) == S.SALIENCE_FLOOR
+
+
+class TestHalfLife:
+    def test_durable_categories_barely_decay(self):
+        assert S.half_life_days("fact", "decision") >= 3650
+        assert S.half_life_days("fact", "commitment") >= 3650
+
+    def test_ephemeral_categories_fade_fast(self):
+        assert S.half_life_days("fact", "mention") <= 30
+        assert S.half_life_days("fact", "observation") <= 30
+
+    def test_unknown_category_uses_default(self):
+        assert S.half_life_days("fact", "made_up") == S.FACT_HALF_LIFE_DEFAULT
+
+    def test_entity_and_relationship_kinds(self):
+        assert S.half_life_days("entity") == S.ENTITY_HALF_LIFE_DAYS
+        assert S.half_life_days("relationship") == S.RELATIONSHIP_HALF_LIFE_DAYS
+
+
+class TestDecay:
+    def test_one_half_life_halves_salience(self):
+        assert S.decayed_salience(0.8, age_days=30, hl_days=30) == round(0.4, 4)
+
+    def test_two_half_lives_quarters(self):
+        assert S.decayed_salience(0.8, age_days=60, hl_days=30) == round(0.2, 4)
+
+    def test_no_age_no_decay(self):
+        assert S.decayed_salience(0.6, age_days=0, hl_days=30) == 0.6
+
+    def test_durable_fact_after_a_year_barely_moves(self):
+        hl = S.half_life_days("fact", "decision")
+        assert S.decayed_salience(0.7, age_days=365, hl_days=hl) > 0.65
+
+    def test_mention_after_three_months_is_near_floor(self):
+        hl = S.half_life_days("fact", "mention")  # 30d
+        assert S.decayed_salience(0.5, age_days=90, hl_days=hl) < 0.1
+
+
+class TestEvictable:
+    def test_low_salience_old_unreferenced_is_evictable(self):
+        assert S.is_evictable(current_salience=0.02, age_days=60, referenced_by_live_node=False)
+
+    def test_referenced_node_survives(self):
+        assert not S.is_evictable(current_salience=0.0, age_days=999, referenced_by_live_node=True)
+
+    def test_restricted_never_auto_evicted(self):
+        assert not S.is_evictable(
+            current_salience=0.0, age_days=999, referenced_by_live_node=False,
+            disclosure_class="restricted",
+        )
+
+    def test_recent_low_salience_not_yet_evictable(self):
+        # Below threshold but too young — grace window protects it.
+        assert not S.is_evictable(current_salience=0.01, age_days=5, referenced_by_live_node=False)
+
+    def test_healthy_salience_never_evictable(self):
+        assert not S.is_evictable(current_salience=0.5, age_days=9999, referenced_by_live_node=False)

package/packages/memory-engine-v2/org-model/migrations/006_fusion_drive.sql

@@ -0,0 +1,80 @@
+-- 006: Fusion Drive foundations — salience + audit/ledger tables.
+--
+-- See RFC-fusion-drive.md. This migration is Phase 1 (scoring only, NO
+-- eviction): it adds the columns the decay pass scores against and the
+-- audit/ledger tables fusion + decay write to. Nothing in this migration
+-- deletes or evicts; the decay pass ships dry-run first.
+--
+-- KEY DESIGN CALL: salience is SEPARATE from confidence. `confidence`
+-- means "how corroborated/true is this fact" and must only ever move up
+-- with corroboration — decaying it would lie about truth. `salience` is
+-- "retention priority" and is what decays with time + rises with access.
+-- Eviction (a later phase) keys on salience, never on confidence.
+--
+-- All idempotent (IF NOT EXISTS) — safe to re-run.
+
+-- Retention scoring columns. Default 0.5 (neutral); the decay pass and
+-- the extractor's born-salience seeding set real values. last_accessed
+-- is bumped by /search when a node is returned (keeps used memories
+-- alive); NULL means never retrieved.
+ALTER TABLE entities       ADD COLUMN IF NOT EXISTS salience REAL NOT NULL DEFAULT 0.5;
+ALTER TABLE entities       ADD COLUMN IF NOT EXISTS last_accessed TIMESTAMPTZ;
+ALTER TABLE facts          ADD COLUMN IF NOT EXISTS salience REAL NOT NULL DEFAULT 0.5;
+ALTER TABLE facts          ADD COLUMN IF NOT EXISTS last_accessed TIMESTAMPTZ;
+ALTER TABLE relationships  ADD COLUMN IF NOT EXISTS salience REAL NOT NULL DEFAULT 0.5;
+ALTER TABLE relationships  ADD COLUMN IF NOT EXISTS last_accessed TIMESTAMPTZ;
+
+-- Partial indexes for the decay sweep: it scans low-salience nodes only.
+CREATE INDEX IF NOT EXISTS idx_entities_salience      ON entities(arena, salience)      WHERE salience < 0.3;
+CREATE INDEX IF NOT EXISTS idx_facts_salience         ON facts(arena, salience)         WHERE salience < 0.3;
+CREATE INDEX IF NOT EXISTS idx_relationships_salience ON relationships(arena, salience) WHERE salience < 0.3;
+
+-- fact_merges — mirror of entity_merges (002) for fact fusion. Same
+-- receipt + rollback substrate: the losing fact row is deleted on merge,
+-- this records what absorbed it and enough to recreate it.
+CREATE TABLE IF NOT EXISTS fact_merges (
+  id                      TEXT PRIMARY KEY,
+  arena                   TEXT NOT NULL,
+  canonical_id            TEXT NOT NULL REFERENCES facts(id) ON DELETE CASCADE,
+  deprecated_id           TEXT NOT NULL,                 -- no FK; row is deleted
+  deprecated_statement    TEXT NOT NULL,                 -- preserve for forensics
+  merge_signal            TEXT NOT NULL CHECK (
+    merge_signal IN ('exact_triple', 'statement_embedding', 'llm_adjudication')
+  ),
+  provenance_unioned      INTEGER NOT NULL DEFAULT 0,
+  rollback_payload        JSONB NOT NULL,
+  merged_at               TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+CREATE INDEX IF NOT EXISTS idx_fact_merges_canonical ON fact_merges(canonical_id);
+
+-- Run ledgers — observability for the Fusion Drive passes. Each pass
+-- (fusion or decay) writes one row: what it scanned, what it changed,
+-- and whether it was a dry-run.
+CREATE TABLE IF NOT EXISTS fusion_drive_runs (
+  id              TEXT PRIMARY KEY,
+  arena           TEXT NOT NULL,
+  pass_kind       TEXT NOT NULL CHECK (pass_kind IN ('fusion', 'decay')),
+  mode            TEXT NOT NULL CHECK (mode IN ('dry_run', 'apply')),
+  scanned         INTEGER NOT NULL DEFAULT 0,
+  changed         INTEGER NOT NULL DEFAULT 0,   -- merged (fusion) / evicted (decay)
+  detail          JSONB NOT NULL DEFAULT '{}'::jsonb,
+  started_at      TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  finished_at     TIMESTAMPTZ
+);
+CREATE INDEX IF NOT EXISTS idx_fusion_drive_runs_arena ON fusion_drive_runs(arena, started_at DESC);
+
+-- node_evictions — rollback receipts for decay eviction. The decay pass
+-- only deletes nodes its salience math already classified evictable
+-- (low salience, aged past the grace floor, unreferenced, non-restricted);
+-- this records the full deleted row so an eviction can be undone. Mirrors
+-- the merge-audit pattern (002 / fact_merges).
+CREATE TABLE IF NOT EXISTS node_evictions (
+  id               TEXT PRIMARY KEY,
+  arena            TEXT NOT NULL,
+  node_kind        TEXT NOT NULL CHECK (node_kind IN ('entity', 'fact', 'relationship')),
+  node_id          TEXT NOT NULL,
+  salience_at_evict REAL,
+  rollback_payload JSONB NOT NULL,
+  evicted_at       TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+CREATE INDEX IF NOT EXISTS idx_node_evictions_arena ON node_evictions(arena, evicted_at DESC);

package/packages/memory-engine-v2/scripts/fusion_drive_born_salience_backfill.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Fusion Drive — born-salience backfill for EXISTING rows.
+
+Migration 006 defaults every pre-existing entity/fact to salience 0.5,
+including accumulated 7B-era junk (e.g. the 87k pip-agents events). At 0.5 a
+junk entity would take ~3+ years to decay under the entity half-life — so the
+"junk self-evicts" cure is inert for existing data until their salience is
+re-seeded from the same quality flags the worker now stamps at insert.
+
+This pass recomputes born-salience for existing entities + facts in an arena
+and writes it back, so the decay pass can act on historical pollution. It only
+LOWERS salience where the quality flags fire (never raises a node above what
+it already has — re-corroboration, not this pass, raises salience). Reads the
+same digit-ratio / subject-undeclared signals the worker uses; deeper signals
+(ungrounded vs source content) are a follow-up.
+
+Arena-scoped (required), dry-run default, --apply to write.
+
+Usage:
+  fusion_drive_born_salience_backfill.py --arena 'X'          # report
+  fusion_drive_born_salience_backfill.py --arena 'X' --apply  # write salience
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "fusion_drive"))
+import salience as S  # noqa: E402
+
+try:
+    import psycopg
+    from psycopg.rows import dict_row
+except ModuleNotFoundError:
+    print("psycopg required", file=sys.stderr)
+    raise
+
+
+def _digit_ratio(s: str) -> float:
+    stripped = "".join((s or "").split())
+    return sum(c.isdigit() for c in stripped) / len(stripped) if stripped else 0.0
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--arena", required=True)
+    ap.add_argument("--pg-dsn", default=os.environ.get("PG_DSN"))
+    ap.add_argument("--apply", action="store_true")
+    args = ap.parse_args()
+    if not args.pg_dsn:
+        print("PG_DSN required", file=sys.stderr)
+        return 2
+
+    ent_lowered = fact_lowered = ent_scanned = fact_scanned = 0
+    with psycopg.connect(args.pg_dsn, row_factory=dict_row) as conn:
+        with conn.cursor() as cur:
+            # entities: numeric-ID-as-person born low
+            cur.execute(
+                "SELECT id, entity_type, canonical_name, salience, provenance_event_ids "
+                "FROM entities WHERE arena = %s", (args.arena,))
+            for e in cur.fetchall():
+                ent_scanned += 1
+                flags = []
+                if e["entity_type"] == "person" and _digit_ratio(e["canonical_name"]) > 0.5:
+                    flags.append("numeric_id_person")
+                if not flags:
+                    continue
+                new_sal = S.born_salience(n_sources=len(e["provenance_event_ids"] or []) or 1,
+                                          quality_flags=flags)
+                if new_sal < e["salience"]:
+                    ent_lowered += 1
+                    if args.apply:
+                        with conn.cursor() as w:
+                            w.execute("UPDATE entities SET salience = %s WHERE id = %s",
+                                      (new_sal, e["id"]))
+
+            # facts: subject-undeclared / low-signal born low
+            cur.execute(
+                "SELECT id, subject_entity_id, statement, salience, provenance_event_ids "
+                "FROM facts WHERE arena = %s", (args.arena,))
+            for f in cur.fetchall():
+                fact_scanned += 1
+                flags = []
+                if f["subject_entity_id"] is None:
+                    flags.append("subject_undeclared")
+                if len((f["statement"] or "")) < 60:
+                    flags.append("low_signal")
+                if not flags:
+                    continue
+                new_sal = S.born_salience(n_sources=len(f["provenance_event_ids"] or []) or 1,
+                                          quality_flags=flags)
+                if new_sal < f["salience"]:
+                    fact_lowered += 1
+                    if args.apply:
+                        with conn.cursor() as w:
+                            w.execute("UPDATE facts SET salience = %s WHERE id = %s",
+                                      (new_sal, f["id"]))
+        if args.apply:
+            conn.commit()
+
+    mode = "APPLY" if args.apply else "DRY-RUN"
+    print(f"[fusion-drive:born-salience-backfill] {mode} arena={args.arena}")
+    print(f"  entities: scanned={ent_scanned} lowered={ent_lowered}")
+    print(f"  facts:    scanned={fact_scanned} lowered={fact_lowered}")
+    if not args.apply:
+        print("  (dry-run — run --apply to write. Lowers junk salience only; never raises.)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/packages/memory-engine-v2/scripts/fusion_drive_decay.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Fusion Drive — decay pass (scoring + eviction).
+
+Recomputes time-decayed salience for an arena's facts/entities/relationships,
+reports eviction candidates, and — only with --evict — deletes them, writing a
+node_evictions rollback receipt per deletion. Three escalating modes:
+
+  (default)  DRY-RUN  — report candidates, change nothing.
+  --apply             — persist recomputed salience back to rows (no deletion).
+  --evict             — additionally DELETE evictable nodes (implies --apply),
+                        each with a full rollback_payload in node_evictions.
+
+Pollution cure (recap): junk born at low salience (extractor quality flags)
+falls below the eviction threshold purely from age, so it self-evicts without
+needing a fusion match. Eviction safety: arena-scoped, restricted disclosure
+never evicted, entities referenced by a surviving fact never evicted, every
+deletion reversible from node_evictions, one transaction.
+
+Usage:
+  fusion_drive_decay.py --arena 'X'           # dry-run report
+  fusion_drive_decay.py --arena 'X' --apply   # persist salience, no deletion
+  fusion_drive_decay.py --arena 'X' --evict   # delete evictable (reversible)
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import uuid
+from datetime import datetime, timezone
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "fusion_drive"))
+import json  # noqa: E402
+import salience as S  # noqa: E402
+
+try:
+    import psycopg
+except ModuleNotFoundError:
+    print("psycopg required", file=sys.stderr)
+    raise
+
+
+def _age_days(ref: datetime | None, now: datetime) -> float:
+    if ref is None:
+        return 0.0
+    if ref.tzinfo is None:
+        ref = ref.replace(tzinfo=timezone.utc)
+    return max(0.0, (now - ref).total_seconds() / 86400.0)
+
+
+def _scan(cur, arena: str, now: datetime) -> tuple[dict, list[dict]]:
+    """Return (report, evictable). evictable carries only {node_kind, id,
+    salience} — the FULL row for the rollback receipt is re-selected at
+    delete time in _evict (so the receipt is genuinely complete and
+    reversible, not a partial stub). Reads only."""
+    report: dict = {}
+    evictable: list[dict] = []
+
+    # facts: decay clock = most recent of (last_accessed, asserted_at)
+    cur.execute(
+        """SELECT id, category, salience, asserted_at, last_accessed, disclosure_class
+           FROM facts WHERE arena = %s""",
+        (arena,),
+    )
+    rows = cur.fetchall()
+    fcand = 0
+    for fid, category, sal, asserted, accessed, disc in rows:
+        clock = max([t for t in (accessed, asserted) if t is not None], default=None)
+        age = _age_days(clock, now)
+        cur_sal = S.decayed_salience(sal, age, S.half_life_days("fact", category))
+        if S.is_evictable(current_salience=cur_sal, age_days=age,
+                          referenced_by_live_node=False, disclosure_class=disc or "private"):
+            fcand += 1
+            evictable.append({"node_kind": "fact", "id": fid, "salience": cur_sal})
+    report["facts"] = {"scanned": len(rows), "evict_candidates": fcand}
+
+    # entities: an entity referenced by ANY surviving fact OR relationship is
+    # NOT evictable. The relationship check is essential — relationships FK
+    # entities with ON DELETE CASCADE, so evicting a rel-endpoint entity would
+    # silently cascade-delete the relationship with no rollback receipt.
+    cur.execute(
+        """SELECT e.id, e.entity_type, e.salience, e.last_seen, e.last_accessed, e.disclosure_class,
+                  (EXISTS (SELECT 1 FROM facts f WHERE f.arena = e.arena
+                           AND (f.subject_entity_id = e.id OR f.object_entity_id = e.id))
+                   OR EXISTS (SELECT 1 FROM relationships r WHERE r.arena = e.arena
+                              AND (r.from_entity_id = e.id OR r.to_entity_id = e.id))) AS referenced
+           FROM entities e WHERE e.arena = %s""",
+        (arena,),
+    )
+    rows = cur.fetchall()
+    ecand = 0
+    for eid, etype, sal, last_seen, accessed, disc, referenced in rows:
+        clock = max([t for t in (accessed, last_seen) if t is not None], default=None)
+        age = _age_days(clock, now)
+        cur_sal = S.decayed_salience(sal, age, S.half_life_days("entity"))
+        if S.is_evictable(current_salience=cur_sal, age_days=age,
+                          referenced_by_live_node=bool(referenced), disclosure_class=disc or "private"):
+            ecand += 1
+            evictable.append({"node_kind": "entity", "id": eid, "salience": cur_sal})
+    report["entities"] = {"scanned": len(rows), "evict_candidates": ecand}
+
+    # NOTE: relationship DECAY/eviction is intentionally NOT done here yet
+    # (the migration adds salience to relationships, but seeding + a clock
+    # policy for edges is a follow-up). Relationships only leave via the
+    # entity-merge collision path or cascade — and the guard above prevents
+    # cascade from silently dropping a live edge.
+    return report, evictable
+
+
+def _evict(cur, evictable: list[dict], now: datetime) -> int:
+    """Delete evictable nodes, one COMPLETE node_evictions receipt each. The
+    full row is re-selected as JSON immediately before deletion so the
+    rollback_payload can actually recreate the row (the headline reversibility
+    guarantee). Facts before entities. Runs in the caller's transaction."""
+    table = {"fact": "facts", "entity": "entities", "relationship": "relationships"}
+    evicted = 0
+    for kind in ("fact", "entity", "relationship"):
+        tbl = table[kind]
+        for node in [n for n in evictable if n["node_kind"] == kind]:
+            cur.execute(f"SELECT to_jsonb(t) FROM {tbl} t WHERE id = %s", (node["id"],))
+            row = cur.fetchone()
+            if not row:
+                continue  # already gone (e.g. fact whose entity cascade-nulled it elsewhere)
+            full_row = row[0]
+            cur.execute(
+                """INSERT INTO node_evictions (id, arena, node_kind, node_id, salience_at_evict, rollback_payload)
+                   VALUES (%s, %s, %s, %s, %s, %s::jsonb)""",
+                ("nev_" + uuid.uuid4().hex[:20], full_row.get("arena"), kind, node["id"],
+                 node["salience"], json.dumps(full_row, default=str)),
+            )
+            cur.execute(f"DELETE FROM {tbl} WHERE id = %s", (node["id"],))
+            evicted += 1
+    return evicted
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--arena", required=True, help="arena to scan (required — never global)")
+    ap.add_argument("--pg-dsn", default=os.environ.get("PG_DSN"))
+    ap.add_argument("--apply", action="store_true",
+                    help="persist recomputed salience back to rows (no deletion)")
+    ap.add_argument("--evict", action="store_true",
+                    help="DELETE evictable nodes (reversible via node_evictions); implies --apply")
+    args = ap.parse_args()
+    if not args.pg_dsn:
+        print("PG_DSN required (env or --pg-dsn)", file=sys.stderr)
+        return 2
+
+    now = datetime.now(timezone.utc)
+    evicted = 0
+    with psycopg.connect(args.pg_dsn) as conn:   # NOT autocommit — eviction is one txn
+        with conn.cursor() as cur:
+            report, evictable = _scan(cur, args.arena, now)
+            if args.evict:
+                evicted = _evict(cur, evictable, now)
+            run_id = "fdr_" + uuid.uuid4().hex[:20]
+            scanned = sum(r["scanned"] for r in report.values())
+            mode = "apply" if (args.evict or args.apply) else "dry_run"
+            cur.execute(
+                """INSERT INTO fusion_drive_runs (id, arena, pass_kind, mode, scanned, changed, detail, finished_at)
+                   VALUES (%s, %s, 'decay', %s, %s, %s, %s::jsonb, NOW())""",
+                (run_id, args.arena, mode, scanned, evicted, json.dumps(report)),
+            )
+        conn.commit()
+
+    label = ("EVICT (deleted, reversible via node_evictions)" if args.evict
+             else "APPLY (salience only)" if args.apply else "DRY-RUN")
+    print(f"[fusion-drive:decay] {label} arena={args.arena}")
+    for kind, r in report.items():
+        print(f"  {kind}: scanned={r['scanned']} evict_candidates={r['evict_candidates']}")
+    if args.evict:
+        print(f"  EVICTED {evicted} node(s) — rollback receipts in node_evictions")
+    else:
+        print(f"  {sum(r['evict_candidates'] for r in report.values())} would evict (run --evict to delete)")
+    print(f"  ledger: {run_id}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())