@pentatonic-ai/ai-agent-sdk 0.10.8 → 0.10.9

package/packages/memory-engine-v2/extractor-sync/server.py

@@ -27,10 +27,14 @@ import os
 import re
 import time
 from contextlib import asynccontextmanager
+from datetime import datetime  # noqa: F401  (used in type hints)
 from typing import Any
 
 # Canonical entity-ID scheme — byte-identical copy in extractor-async (entity_id.py).
+from confidence import born_salience
 from entity_id import entity_id, normalize_surface_form  # noqa: F401
+# Source-time parsing — byte-identical copy in extractor-async (source_time.py).
+from source_time import event_source_time
 
 import psycopg
 import psycopg.rows
@@ -394,17 +398,27 @@ RULES = {
 
 async def _upsert_event(cur: psycopg.AsyncCursor, req: ExtractRequest,
                         event_id: str, content_hash: str) -> None:
-    """ON CONFLICT DO NOTHING — re-emitting the same event is a no-op."""
+    """ON CONFLICT DO NOTHING — re-emitting the same event is a no-op.
+
+    `emitted_at` is the SOURCE time of the content (when the
+    email/meeting/message actually happened), parsed from
+    `attributes.timestamp`; `received_at` keeps its NOW() default and
+    means ingest time — exactly the split the schema comment at
+    001_init.sql:112 promises. When the source time is absent or
+    unparseable we fall back to NOW() via COALESCE (never NULL a
+    NOT NULL column)."""
+    emitted_at = event_source_time({"attributes": req.attributes})
     await cur.execute(
         """
         INSERT INTO events (
           id, arena, client_id, user_id, event_type, source_kind,
           source_id, content, content_hash, participant_set,
-          participant_kind, disclosure_class, attributes
+          participant_kind, disclosure_class, attributes, emitted_at
         ) VALUES (
           %s, %s, %s, %s, %s, %s::source_kind,
           %s, %s, %s, %s,
-          %s::participant_kind, %s::disclosure_class, %s::jsonb
+          %s::participant_kind, %s::disclosure_class, %s::jsonb,
+          COALESCE(%s, NOW())
         )
         ON CONFLICT (id) DO NOTHING
         """,
@@ -416,13 +430,26 @@ async def _upsert_event(cur: psycopg.AsyncCursor, req: ExtractRequest,
             req.attributes.get("participant_kind", "unknown"),
             req.attributes.get("disclosure_class", "private"),
             psycopg.types.json.Json(req.attributes),
+            emitted_at,
         ),
     )
 
 
-async def _upsert_entities(cur: psycopg.AsyncCursor, entities: list[dict]) -> None:
+async def _upsert_entities(
+    cur: psycopg.AsyncCursor,
+    entities: list[dict],
+    event_time: "datetime | None",
+) -> None:
     """Alias-aware idempotent entity upsert.
 
+    `event_time` is the SOURCE time of the originating event (parsed from
+    `attributes.timestamp`); it stamps `first_seen`/`last_seen` so the
+    graph tracks when the evidence actually happened, not when we
+    ingested it. `None` (no/garbage source time) falls back to NOW() via
+    COALESCE. On re-corroboration we widen the window with
+    LEAST(first_seen, ...) / GREATEST(last_seen, ...): "most recent
+    evidence" = newest SOURCE time, not newest ingest.
+
     For each entity, before inserting, look for an existing row in the
     same (arena, entity_type) whose canonical_name OR aliases overlap
     any of the incoming surface forms. If found, merge aliases +
@@ -488,23 +515,40 @@ async def _upsert_entities(cur: psycopg.AsyncCursor, entities: list[dict]) -> No
                 UPDATE entities SET
                   aliases = ARRAY(SELECT DISTINCT UNNEST(aliases || %s::text[])),
                   provenance_event_ids = ARRAY(SELECT DISTINCT UNNEST(provenance_event_ids || %s::text[])),
-                  last_seen = NOW()
+                  -- Widen the seen-window with this event's SOURCE time,
+                  -- not NOW(): newest evidence = newest source time.
+                  last_seen = GREATEST(last_seen, COALESCE(%s, NOW())),
+                  first_seen = LEAST(first_seen, COALESCE(%s, NOW()))
                 WHERE id = %s
                 """,
-                (e["aliases"], e["provenance_event_ids"], existing_id),
+                (e["aliases"], e["provenance_event_ids"],
+                 event_time, event_time, existing_id),
             )
         else:
             # 3b. No match — insert new. ON CONFLICT (id) is a belt-
             # and-braces fallback for the rare case where two writers
             # collide on the same id under different surface forms;
             # the advisory lock above is the primary defence.
+            # Fusion Drive born-salience via the SHARED born_salience (no
+            # inline constants — they'd drift from the async path; #96 review
+            # §4). Sync entities are deterministic (names from structured
+            # email/calendar fields) so they're high-quality; the one junk
+            # class sync can still emit is a numeric-ID-as-person, flagged so
+            # it's born low and decay can evict it. The async distiller owns
+            # the full quality-flag set.
+            _digits = sum(c.isdigit() for c in e["canonical_name"] if not c.isspace())
+            _nonspace = sum(1 for c in e["canonical_name"] if not c.isspace()) or 1
+            _flags = ["numeric_id_person"] if (e["entity_type"] == "person" and _digits / _nonspace > 0.5) else []
+            _sal = born_salience(1, _flags)
             await cur.execute(
                 """
                 INSERT INTO entities (
                   id, arena, entity_type, canonical_name, aliases,
-                  provenance_event_ids, participant_set, disclosure_class
+                  provenance_event_ids, participant_set, disclosure_class,
+                  first_seen, last_seen, salience
                 ) VALUES (
-                  %s, %s, %s, %s, %s, %s, %s, %s::disclosure_class
+                  %s, %s, %s, %s, %s, %s, %s, %s::disclosure_class,
+                  COALESCE(%s, NOW()), COALESCE(%s, NOW()), %s
                 )
                 ON CONFLICT (id) DO UPDATE SET
                   aliases = (
@@ -513,11 +557,14 @@ async def _upsert_entities(cur: psycopg.AsyncCursor, entities: list[dict]) -> No
                   provenance_event_ids = (
                     SELECT ARRAY(SELECT DISTINCT UNNEST(entities.provenance_event_ids || EXCLUDED.provenance_event_ids))
                   ),
-                  last_seen = NOW()
+                  salience = GREATEST(entities.salience, EXCLUDED.salience),
+                  last_seen = GREATEST(entities.last_seen, EXCLUDED.last_seen),
+                  first_seen = LEAST(entities.first_seen, EXCLUDED.first_seen)
                 """,
                 (e["id"], e["arena"], e["entity_type"], e["canonical_name"],
                  e["aliases"], e["provenance_event_ids"],
-                 e["participant_set"], e["disclosure_class"]),
+                 e["participant_set"], e["disclosure_class"],
+                 event_time, event_time, _sal),
             )
 
 
@@ -584,7 +631,10 @@ async def extract(req: ExtractRequest):
     async with _pool.connection() as conn:
         async with conn.cursor() as cur:
             await _upsert_event(cur, req, event_id, content_hash)
-            await _upsert_entities(cur, entities)
+            # Source time of THIS event — stamps the graph rows so
+            # first/last_seen track content time, not ingest time.
+            event_time = event_source_time({"attributes": req.attributes})
+            await _upsert_entities(cur, entities, event_time)
             # Facts + relationships are deliberately left to the async
             # distillation worker — the deterministic path can't
             # reliably extract decisions/commitments without LLM context.

package/packages/memory-engine-v2/extractor-sync/source_time.py

@@ -0,0 +1,63 @@
+"""source_time — robust ISO-8601 source-time parsing for graph stamping.
+
+The memory graph must stamp `events.emitted_at` and the graph rows'
+`first_seen` / `last_seen` / `asserted_at` from the SOURCE time of the
+content (when the email/meeting/message actually happened), NOT the
+ingest wall-clock (`NOW()`). The source time is carried on the event as
+`attributes.timestamp` (ISO-8601). This helper promotes it.
+
+Mirrors `compat/server.py:_parse_ts` (handles the bare `Z` suffix that
+`datetime.fromisoformat` only learned in 3.11) but returns a tz-aware
+`datetime` rather than a unix float, because the destination columns are
+`TIMESTAMPTZ` and we want psycopg to bind a datetime, not an epoch.
+
+CONTRACT (load-bearing): callers MUST fall back to the existing default
+(received / NOW) when the source time is absent or unparseable. This
+helper NEVER raises and returns `None` on anything it can't parse — the
+caller is responsible for the `or NOW()` fallback so we never NULL a
+NOT NULL column or crash the ingest/distill path.
+
+NOTE: keep this byte-identical with the copy in extractor-sync/. Same
+convention as entity_id.py — two services, one parsing rule.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+
+def parse_source_time(value: Any) -> datetime | None:
+    """Best-effort ISO-8601 -> tz-aware datetime. Returns None on
+    anything we can't parse (caller falls back to NOW()).
+
+    Accepts both the bare `Z` suffix and explicit offsets. A parsed
+    value with no offset is assumed UTC (the producers emit UTC ISO
+    strings; a naive datetime would break TIMESTAMPTZ comparisons)."""
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        # `fromisoformat` handles `+00:00` but not the bare `Z` suffix
+        # until Python 3.11; normalise to be safe across runtime
+        # versions on the engine box.
+        dt = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except Exception:
+        return None
+    if dt.tzinfo is None:
+        # Producer emitted a naive ISO string; treat as UTC rather than
+        # letting psycopg interpret it in the server's local zone.
+        dt = dt.replace(tzinfo=timezone.utc)
+    return dt
+
+
+def event_source_time(event: dict[str, Any]) -> datetime | None:
+    """Pull the source time off an event dict's attributes.
+
+    Precedence: `attributes.timestamp` (the source/content time) wins
+    over `attributes.emitted_at` (a producer-supplied emit-now, which is
+    closer to ingest time). Returns None if neither parses — caller
+    falls back to NOW()."""
+    attrs = event.get("attributes") or {}
+    return parse_source_time(attrs.get("timestamp")) or parse_source_time(
+        attrs.get("emitted_at")
+    )

package/packages/memory-engine-v2/extractor-sync/test_confidence_parity.py

@@ -0,0 +1,18 @@
+"""extractor-sync/confidence.py must stay byte-identical to extractor-async's
+copy — both carry born_salience, whose scale must match the Fusion Drive decay
+side. Same drift guard as test_entity_id_parity.py across the build contexts."""
+
+from __future__ import annotations
+
+import os
+
+
+def test_sync_confidence_is_byte_identical_to_async():
+    here = os.path.dirname(__file__)
+    sync = os.path.join(here, "confidence.py")
+    async_ = os.path.join(here, "..", "extractor-async", "confidence.py")
+    with open(sync, "rb") as f:
+        a = f.read()
+    with open(async_, "rb") as f:
+        b = f.read()
+    assert a == b, "extractor-sync/confidence.py drifted from extractor-async/confidence.py"

package/packages/memory-engine-v2/extractor-sync/test_paired_extraction.py

@@ -273,7 +273,7 @@ def test_pool_keeps_default_tuple_row_factory() -> None:
 def test_upsert_entities_merge_branch_with_tuple_rows() -> None:
     """Entity already exists → UPDATE branch runs, id taken from row[0]."""
     cur = _FakeCursor(existing_id="e_existing")
-    asyncio.run(sync_server._upsert_entities(cur, [_entity_stub()]))
+    asyncio.run(sync_server._upsert_entities(cur, [_entity_stub()], None))
     updates = [(s, p) for s, p in cur.executed if s.startswith("UPDATE entities")]
     assert len(updates) == 1
     _, params = updates[0]
@@ -283,7 +283,7 @@ def test_upsert_entities_merge_branch_with_tuple_rows() -> None:
 
 def test_upsert_entities_insert_branch_when_no_match() -> None:
     cur = _FakeCursor(existing_id=None)
-    asyncio.run(sync_server._upsert_entities(cur, [_entity_stub()]))
+    asyncio.run(sync_server._upsert_entities(cur, [_entity_stub()], None))
     inserts = [s for s, _ in cur.executed if s.startswith("INSERT INTO entities")]
     assert len(inserts) == 1
     assert not any(s.startswith("UPDATE entities") for s, _ in cur.executed)

package/packages/memory-engine-v2/fusion_drive/adjudicate.py

@@ -0,0 +1,85 @@
+"""Fusion Drive — LLM adjudication via the self-hosted distiller (no egress).
+
+When fusion's deterministic tiers (exact-name, cross-run-shared-provenance,
+exact-triple facts) leave AMBIGUOUS candidates — two entities in the
+0.75–0.92 embedding band, or two facts that look like the same assertion in
+different words — we ask the **in-VPC distiller** (Qwen3.6-27B-FP8, the same
+LLM that extracted this content) to adjudicate. Using the distiller instead
+of a hosted API means the memory content NEVER leaves the VPC: no third-party
+egress, no disclosure_class sign-off, no per-token cost.
+
+This module is pure: the HTTP call is injected (`post_fn`), so verdict parsing
+and prompt construction are unit-tested without a GPU. The caller supplies a
+`post_fn(messages) -> str` that hits the distiller's OpenAI /v1/chat/completions
+(temperature 0, chat_template_kwargs enable_thinking=false — same shape the
+worker uses). If post_fn raises / returns None, adjudication is treated as
+UNSURE (never auto-merge on an LLM failure) — graceful degradation when no
+distiller box is up.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+
+# Conservative default: anything that isn't a clear "yes" does NOT merge.
+ENTITY_PROMPT = (
+    "You decide whether two extracted entities refer to the SAME real-world "
+    "thing. Be conservative: only say yes if you are confident they are the "
+    "same. Two different people who merely share a first name are NOT the same.\n\n"
+    "Entity A: type={a_type} name={a_name} aliases={a_aliases}\n"
+    "Context A (facts mentioning A): {a_ctx}\n\n"
+    "Entity B: type={b_type} name={b_name} aliases={b_aliases}\n"
+    "Context B (facts mentioning B): {b_ctx}\n\n"
+    'Reply with ONLY a JSON object: {{"same": true|false, "reason": "<short>"}}'
+)
+
+FACT_PROMPT = (
+    "You decide whether two statements assert the SAME fact (same subject, "
+    "same claim), even if worded differently. Be conservative.\n\n"
+    "Statement A: {a}\n"
+    "Statement B: {b}\n\n"
+    'Reply with ONLY a JSON object: {{"same": true|false, "reason": "<short>"}}'
+)
+
+
+def _parse_verdict(raw: str | None) -> dict:
+    """Parse the model's JSON verdict. Anything unparseable / non-affirmative
+    → {'same': False} (fail closed — never merge on doubt)."""
+    if not raw:
+        return {"same": False, "reason": "no response (unsure)"}
+    m = re.search(r"\{.*\}", raw, re.DOTALL)
+    if not m:
+        return {"same": False, "reason": "unparseable verdict"}
+    try:
+        v = json.loads(m.group(0))
+    except ValueError:
+        return {"same": False, "reason": "invalid json verdict"}
+    return {"same": bool(v.get("same") is True), "reason": str(v.get("reason", ""))[:200]}
+
+
+def adjudicate_entities(a: dict, b: dict, post_fn) -> dict:
+    """a/b: {entity_type, canonical_name, aliases, context (list[str] of facts)}.
+    Returns {'same': bool, 'reason': str}. Fail-closed on any error."""
+    msg = ENTITY_PROMPT.format(
+        a_type=a.get("entity_type"), a_name=a.get("canonical_name"),
+        a_aliases=", ".join(a.get("aliases") or []) or "(none)",
+        a_ctx=" | ".join((a.get("context") or [])[:5]) or "(none)",
+        b_type=b.get("entity_type"), b_name=b.get("canonical_name"),
+        b_aliases=", ".join(b.get("aliases") or []) or "(none)",
+        b_ctx=" | ".join((b.get("context") or [])[:5]) or "(none)",
+    )
+    try:
+        raw = post_fn([{"role": "user", "content": msg}])
+    except Exception:
+        return {"same": False, "reason": "adjudicator unreachable (unsure)"}
+    return _parse_verdict(raw)
+
+
+def adjudicate_facts(stmt_a: str, stmt_b: str, post_fn) -> dict:
+    msg = FACT_PROMPT.format(a=stmt_a, b=stmt_b)
+    try:
+        raw = post_fn([{"role": "user", "content": msg}])
+    except Exception:
+        return {"same": False, "reason": "adjudicator unreachable (unsure)"}
+    return _parse_verdict(raw)

package/packages/memory-engine-v2/fusion_drive/test_adjudicate.py

@@ -0,0 +1,65 @@
+"""Unit tests for distiller-based adjudication (pure; post_fn injected)."""
+
+from __future__ import annotations
+
+from adjudicate import adjudicate_entities, adjudicate_facts, _parse_verdict
+
+
+class TestParseVerdict:
+    def test_clean_yes(self):
+        assert _parse_verdict('{"same": true, "reason": "same person"}')["same"] is True
+
+    def test_clean_no(self):
+        assert _parse_verdict('{"same": false, "reason": "different"}')["same"] is False
+
+    def test_prose_wrapped_json(self):
+        assert _parse_verdict('Sure: {"same": true, "reason": "x"} done')["same"] is True
+
+    def test_none_is_unsure_false(self):
+        assert _parse_verdict(None)["same"] is False
+
+    def test_unparseable_is_false(self):
+        assert _parse_verdict("yeah probably the same tbh")["same"] is False
+
+    def test_invalid_json_is_false(self):
+        assert _parse_verdict('{"same": tru')["same"] is False
+
+    def test_missing_same_key_is_false(self):
+        assert _parse_verdict('{"reason": "hmm"}')["same"] is False
+
+
+class TestAdjudicateEntities:
+    def test_yes_path(self):
+        post = lambda msgs: '{"same": true, "reason": "same"}'
+        v = adjudicate_entities({"canonical_name": "Phil"}, {"canonical_name": "Philip"}, post)
+        assert v["same"] is True
+
+    def test_post_fn_raises_is_failclosed(self):
+        def boom(msgs):
+            raise RuntimeError("no distiller up")
+        v = adjudicate_entities({"canonical_name": "A"}, {"canonical_name": "B"}, boom)
+        assert v["same"] is False and "unsure" in v["reason"]
+
+    def test_prompt_includes_both_entities(self):
+        captured = {}
+        def post(msgs):
+            captured["content"] = msgs[0]["content"]
+            return '{"same": false}'
+        adjudicate_entities(
+            {"entity_type": "person", "canonical_name": "Katie Cooper", "aliases": ["KC"],
+             "context": ["Katie organised Ramen Day"]},
+            {"entity_type": "person", "canonical_name": "1716801984", "context": []},
+            post,
+        )
+        assert "Katie Cooper" in captured["content"] and "1716801984" in captured["content"]
+
+
+class TestAdjudicateFacts:
+    def test_same_assertion(self):
+        post = lambda msgs: '{"same": true, "reason": "same claim"}'
+        assert adjudicate_facts("joined Acme", "works at Acme", post)["same"] is True
+
+    def test_unreachable_is_failclosed(self):
+        def boom(msgs):
+            raise ConnectionError()
+        assert adjudicate_facts("a", "b", boom)["same"] is False

package/packages/memory-engine-v2/scripts/fusion_drive_decay.py

@@ -100,11 +100,30 @@ def _scan(cur, arena: str, now: datetime) -> tuple[dict, list[dict]]:
             evictable.append({"node_kind": "entity", "id": eid, "salience": cur_sal})
     report["entities"] = {"scanned": len(rows), "evict_candidates": ecand}
 
-    # NOTE: relationship DECAY/eviction is intentionally NOT done here yet
-    # (the migration adds salience to relationships, but seeding + a clock
-    # policy for edges is a follow-up). Relationships only leave via the
-    # entity-merge collision path or cascade — and the guard above prevents
-    # cascade from silently dropping a live edge.
+    # relationships: an edge between two surviving entities is "referenced"
+    # (it IS the evidence they interacted) and is kept regardless of salience.
+    # Only a dangling/decayed edge — low salience, stale, and missing at least
+    # one endpoint — is evictable. (rels FK entities ON DELETE CASCADE, so a
+    # live-endpoint edge is never orphaned by the entity pass either.)
+    cur.execute(
+        """SELECT r.id, r.relationship_type, r.salience, r.last_seen, r.last_accessed, r.disclosure_class,
+                  (EXISTS (SELECT 1 FROM entities e WHERE e.id = r.from_entity_id)
+                   AND EXISTS (SELECT 1 FROM entities e WHERE e.id = r.to_entity_id)) AS both_live
+           FROM relationships r WHERE r.arena = %s""",
+        (arena,),
+    )
+    rows = cur.fetchall()
+    rcand = 0
+    for rid, rtype, sal, last_seen, accessed, disc, both_live in rows:
+        clock = max([t for t in (accessed, last_seen) if t is not None], default=None)
+        age = _age_days(clock, now)
+        cur_sal = S.decayed_salience(sal, age, S.half_life_days("relationship"))
+        if S.is_evictable(current_salience=cur_sal, age_days=age,
+                          referenced_by_live_node=bool(both_live), disclosure_class=disc or "private"):
+            rcand += 1
+            evictable.append({"node_kind": "relationship", "id": rid, "salience": cur_sal})
+    report["relationships"] = {"scanned": len(rows), "evict_candidates": rcand}
+
     return report, evictable