@pentatonic-ai/ai-agent-sdk 0.10.30 → 0.10.31

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.30";
+var VERSION = "0.10.31";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.30";
+var VERSION = "0.10.31";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.30",
+  "version": "0.10.31",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -153,6 +153,42 @@ MAX_INFLIGHT_BATCHES = int(os.environ.get("MAX_INFLIGHT_BATCHES", "4"))
 # continuous stream of calls instead of per-batch bursts. None until amain runs.
 _STUDENT_SEM: "asyncio.Semaphore | None" = None
 _TEACHER_SEM: "asyncio.Semaphore | None" = None
+
+# Cap on provenance_event_ids array length per node (entity/fact/relationship).
+# UNBOUNDED accretion was the drain-throughput bottleneck: hot nodes re-mentioned
+# thousands of times grew arrays to 50k+ event IDs (~2MB TOASTed), so every
+# subsequent upsert read+rewrote the whole array off disk (16s DataFileRead
+# measured; entities TOAST hit 233GB, relationships 66GB — 2026-07-08). Keep only
+# the most-recent N distinct IDs — provenance drives corroboration counts +
+# citations, and the avg node has ~6 (only a handful of super-nodes exceed this),
+# so 200 preserves all real provenance while bounding the pathological tail.
+# Capping the STORED array is self-healing: the first capped write shrinks a
+# bloated node to N, then its upserts stay cheap.
+PROVENANCE_MAX_EVENT_IDS = int(os.environ.get("PROVENANCE_MAX_EVENT_IDS", "200"))
+
+
+def _cap_prov(existing: str, incoming: str) -> str:
+    """SQL expression: merge the `existing` and `incoming` text[] provenance
+    arrays, dedupe, and keep the PROVENANCE_MAX_EVENT_IDS most-recent IDs (by
+    position — appended IDs are newest), re-ordered oldest→newest to preserve the
+    append convention readers see. Drop-in replacement for the old unbounded
+    `ARRAY(SELECT DISTINCT UNNEST(existing || incoming))`. `incoming` may carry a
+    single %s placeholder; it stays in the same param position."""
+    return (
+        "(SELECT COALESCE(array_agg(ev ORDER BY ord), ARRAY[]::text[]) "
+        "FROM (SELECT ev, max(ord) AS ord "
+        f"FROM unnest(({existing}) || ({incoming})) WITH ORDINALITY AS _pu(ev, ord) "
+        f"GROUP BY ev ORDER BY max(ord) DESC LIMIT {PROVENANCE_MAX_EVENT_IDS}) _pk)"
+    )
+
+
+# Precomputed capped-provenance SQL fragments for each upsert site (built once).
+_PROV_ENT_UPDATE = _cap_prov("provenance_event_ids", "%s::text[]")
+_PROV_ENT_CONFLICT = _cap_prov("entities.provenance_event_ids", "EXCLUDED.provenance_event_ids")
+_PROV_FACT = _cap_prov("facts.provenance_event_ids", "EXCLUDED.provenance_event_ids")
+_PROV_REL = _cap_prov("relationships.provenance_event_ids", "EXCLUDED.provenance_event_ids")
+
+
 # Fraction of student-passing events ALSO sent to the teacher — NOT a quality
 # lever, this is the monitoring + active-learning sample (drift detection and
 # fresh teacher-gold on live traffic). Default conservative-ish 5%.
@@ -1910,7 +1946,7 @@ def upsert_entities(
                     """
                     UPDATE entities SET
                       aliases = ARRAY(SELECT DISTINCT UNNEST(aliases || %s::text[])),
-                      provenance_event_ids = ARRAY(SELECT DISTINCT UNNEST(provenance_event_ids || %s::text[])),
+                      provenance_event_ids = """ + _PROV_ENT_UPDATE + """,
                       -- Merge the org-domain hard key (no-op for the `{}` default
                       -- on non-orgs / unmatched orgs; never clobbers an existing key).
                       attributes = entities.attributes || %s::jsonb,
@@ -1986,9 +2022,7 @@ def upsert_entities(
                       aliases = (
                         SELECT ARRAY(SELECT DISTINCT UNNEST(entities.aliases || EXCLUDED.aliases))
                       ),
-                      provenance_event_ids = (
-                        SELECT ARRAY(SELECT DISTINCT UNNEST(entities.provenance_event_ids || EXCLUDED.provenance_event_ids))
-                      ),
+                      provenance_event_ids = """ + _PROV_ENT_CONFLICT + """,
                       -- merge the org-domain hard key (no-op for `{}`)
                       attributes = entities.attributes || EXCLUDED.attributes,
                       -- re-corroboration can only RAISE salience, never lower it
@@ -2107,11 +2141,7 @@ def upsert_facts(
                   COALESCE(%s, NOW()), %s, %s
                 )
                 ON CONFLICT (id) DO UPDATE SET
-                  provenance_event_ids = (
-                    SELECT ARRAY(SELECT DISTINCT UNNEST(
-                      facts.provenance_event_ids || EXCLUDED.provenance_event_ids
-                    ))
-                  ),
+                  provenance_event_ids = """ + _PROV_FACT + """,
                   salience = GREATEST(facts.salience, EXCLUDED.salience),
                   -- Confidence bumps with each additional independent
                   -- source. The cardinality of the merged provenance
@@ -2222,9 +2252,7 @@ def upsert_relationships(
                 -- distillation + the SEE-490 backfill on merged-entity edges.
                 ON CONFLICT (arena, from_entity_id, to_entity_id, relationship_type) DO UPDATE SET
                   weight = relationships.weight + EXCLUDED.weight,
-                  provenance_event_ids = (
-                    SELECT ARRAY(SELECT DISTINCT UNNEST(relationships.provenance_event_ids || EXCLUDED.provenance_event_ids))
-                  ),
+                  provenance_event_ids = """ + _PROV_REL + """,
                   attributes = relationships.attributes || EXCLUDED.attributes,
                   last_seen = GREATEST(relationships.last_seen, EXCLUDED.last_seen),
                   first_seen = LEAST(relationships.first_seen, EXCLUDED.first_seen)

package/packages/memory-engine-v2/extractor-sync/server.py

@@ -49,6 +49,30 @@ log = logging.getLogger("extractor-sync")
 
 PG_DSN = os.environ.get("PG_DSN", "postgresql://pme:local-dev-pw@org-model:5432/org_model")
 
+# Cap on provenance_event_ids array length. Unbounded accretion bloated the
+# entities table to 249GB (233GB TOAST) — hot nodes grew 50k+-element arrays so
+# every upsert read+rewrote ~2MB off disk (16s DataFileRead, 2026-07-08). Keep
+# only the most-recent N distinct IDs. Byte-identical intent to extractor-async's
+# _cap_prov (kept as a local copy per the sibling-service duplication convention,
+# like confidence.py / source_time.py).
+PROVENANCE_MAX_EVENT_IDS = int(os.environ.get("PROVENANCE_MAX_EVENT_IDS", "200"))
+
+
+def _cap_prov(existing: str, incoming: str) -> str:
+    """Merge `existing`||`incoming` text[] provenance, dedupe, keep the
+    PROVENANCE_MAX_EVENT_IDS most-recent IDs (append order = newest last).
+    Drop-in for the old unbounded `ARRAY(SELECT DISTINCT UNNEST(...))`."""
+    return (
+        "(SELECT COALESCE(array_agg(ev ORDER BY ord), ARRAY[]::text[]) "
+        "FROM (SELECT ev, max(ord) AS ord "
+        f"FROM unnest(({existing}) || ({incoming})) WITH ORDINALITY AS _pu(ev, ord) "
+        f"GROUP BY ev ORDER BY max(ord) DESC LIMIT {PROVENANCE_MAX_EVENT_IDS}) _pk)"
+    )
+
+
+_PROV_ENT_UPDATE = _cap_prov("provenance_event_ids", "%s::text[]")
+_PROV_ENT_CONFLICT = _cap_prov("entities.provenance_event_ids", "EXCLUDED.provenance_event_ids")
+
 # Connection pool. Sized for high-concurrency bursts: with TES at
 # shardCount=8 and BATCH_SIZE=50 in the DO drain, compat's parallel
 # asyncio.gather in /store-batch can fan ~400 simultaneous _extract
@@ -534,7 +558,7 @@ async def _upsert_entities(
                 """
                 UPDATE entities SET
                   aliases = ARRAY(SELECT DISTINCT UNNEST(aliases || %s::text[])),
-                  provenance_event_ids = ARRAY(SELECT DISTINCT UNNEST(provenance_event_ids || %s::text[])),
+                  provenance_event_ids = """ + _PROV_ENT_UPDATE + """,
                   -- Widen the seen-window with this event's SOURCE time,
                   -- not NOW(): newest evidence = newest source time.
                   last_seen = GREATEST(last_seen, COALESCE(%s, NOW())),
@@ -574,9 +598,7 @@ async def _upsert_entities(
                   aliases = (
                     SELECT ARRAY(SELECT DISTINCT UNNEST(entities.aliases || EXCLUDED.aliases))
                   ),
-                  provenance_event_ids = (
-                    SELECT ARRAY(SELECT DISTINCT UNNEST(entities.provenance_event_ids || EXCLUDED.provenance_event_ids))
-                  ),
+                  provenance_event_ids = """ + _PROV_ENT_CONFLICT + """,
                   -- merge the email/domain hard key (no-op for the `{}` default
                   -- on entity types that don't stamp it); never clobbers.
                   attributes = entities.attributes || EXCLUDED.attributes,

package/packages/memory-engine-v2/fusion_drive/merge.py

@@ -17,6 +17,13 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from typing import Any
 
+# Defensive bound on merged provenance arrays. A merge of many fragmented
+# entities could otherwise union thousands of provenance IDs into the master,
+# re-bloating the entities/relationships TOAST that the extractor cap fixes
+# (see extractor-async/worker.py PROVENANCE_MAX_EVENT_IDS). Kept a literal here
+# to preserve this module's no-env/no-clock purity; must match the extractor.
+PROVENANCE_MAX_EVENT_IDS = 200
+
 
 # ── entity fusion plan ───────────────────────────────────────────────
 @dataclass
@@ -75,7 +82,7 @@ def build_entity_merge_plan(
     provenance = _union(
         master.get("provenance_event_ids", []),
         *[l.get("provenance_event_ids", []) for l in losers],
-    )
+    )[:PROVENANCE_MAX_EVENT_IDS]
 
     plan = EntityMergePlan(
         arena=arena,
@@ -112,7 +119,7 @@ def build_entity_merge_plan(
                 "drop": r["id"],
                 "summed_weight": round(keep.get("weight", 1.0) + r.get("weight", 1.0), 4),
                 "provenance": _union(keep.get("provenance_event_ids", []),
-                                     r.get("provenance_event_ids", [])),
+                                     r.get("provenance_event_ids", []))[:PROVENANCE_MAX_EVENT_IDS],
             })
         else:
             by_key[key] = r
@@ -147,7 +154,7 @@ def build_fact_merge_plan(*, arena: str, dup_facts: list[dict]) -> dict | None:
     )
     master, losers = ranked[0], ranked[1:]
     provenance = _union(master.get("provenance_event_ids", []),
-                        *[l.get("provenance_event_ids", []) for l in losers])
+                        *[l.get("provenance_event_ids", []) for l in losers])[:PROVENANCE_MAX_EVENT_IDS]
     return {
         "arena": arena,
         "master_id": master["id"],

package/packages/memory-engine-v2/scripts/cap_provenance.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""cap_provenance.py — one-time truncation of over-long provenance_event_ids
+arrays on the graph PG (org_model).
+
+Unbounded provenance accretion bloated `entities` to 249GB (233GB TOAST) and
+`relationships` to 72GB (66GB TOAST): a handful of hot nodes grew arrays to
+50k+ event IDs (~2MB TOASTed), so every upsert read+rewrote the whole array off
+disk — a 16s DataFileRead per update, which was THE distillation drain-throughput
+bottleneck (2026-07-08). The extractor cap (worker.py / server.py
+PROVENANCE_MAX_EVENT_IDS) stops NEW growth; this script shrinks the arrays that
+are ALREADY over the cap so their next upsert is cheap immediately, rather than
+waiting for each hot node to be re-mentioned.
+
+It keeps the most-recent CAP distinct IDs (same recency-preserving expression as
+the extractor write path). Runs in bounded batches with a commit per batch so it
+never holds a long lock against live distillation, and is fully idempotent
+(re-running skips already-capped rows via the cardinality guard).
+
+DRY-RUN by default (reports how many rows would be capped, writes nothing).
+Pass --apply to write.
+
+  # in the extractor-async container (has psycopg + PG_DSN):
+  python3 cap_provenance.py --table entities --apply
+  python3 cap_provenance.py --table relationships --apply
+
+Disk reclaim: truncation turns the old 2MB array versions into dead TOAST
+tuples. autovacuum frees them for REUSE (so the table stops growing) but does
+NOT shrink the file. To return the ~299GB to disk, run VACUUM FULL (locks) or
+pg_repack (online) in a maintenance window — non-urgent; performance recovers
+as soon as the live arrays are small.
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+
+import psycopg
+
+# Same recency-preserving cap as the extractor write path (worker.py._cap_prov):
+# dedupe existing, keep the CAP most-recent by position, re-order oldest→newest.
+_CAP_SQL = (
+    "(SELECT COALESCE(array_agg(ev ORDER BY ord), ARRAY[]::text[]) "
+    "FROM (SELECT ev, max(ord) AS ord "
+    "FROM unnest(provenance_event_ids) WITH ORDINALITY AS _pu(ev, ord) "
+    "GROUP BY ev ORDER BY max(ord) DESC LIMIT %(cap)s) _pk)"
+)
+
+ALLOWED_TABLES = {"entities", "facts", "relationships"}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--table", required=True, choices=sorted(ALLOWED_TABLES))
+    ap.add_argument("--cap", type=int, default=int(os.environ.get("PROVENANCE_MAX_EVENT_IDS", "200")))
+    ap.add_argument("--batch", type=int, default=200, help="ids updated per commit")
+    ap.add_argument("--apply", action="store_true", help="write (default: dry-run)")
+    args = ap.parse_args()
+
+    # choices= already guards this; belt-and-braces since the table name is
+    # interpolated into DDL-ish SQL (can't be a bound parameter).
+    if args.table not in ALLOWED_TABLES:
+        print(f"refusing unknown table {args.table!r}", file=sys.stderr)
+        return 2
+    tbl = args.table
+
+    dsn = os.environ["PG_DSN"]
+    with psycopg.connect(dsn, autocommit=True) as conn:
+        over = conn.execute(
+            f"SELECT count(*) FROM {tbl} WHERE cardinality(provenance_event_ids) > %s",
+            (args.cap,),
+        ).fetchone()[0]
+        mx = conn.execute(
+            f"SELECT COALESCE(max(cardinality(provenance_event_ids)), 0) FROM {tbl}"
+        ).fetchone()[0]
+        print(f"{tbl}: {over} row(s) over cap={args.cap} (current max cardinality={mx})")
+        if not args.apply:
+            print("dry-run — pass --apply to truncate")
+            return 0
+        if over == 0:
+            print("nothing to do")
+            return 0
+
+        ids = [
+            r[0]
+            for r in conn.execute(
+                f"SELECT id FROM {tbl} WHERE cardinality(provenance_event_ids) > %s",
+                (args.cap,),
+            ).fetchall()
+        ]
+        total = len(ids)
+        done = 0
+        t0 = time.monotonic()
+        for i in range(0, total, args.batch):
+            chunk = ids[i : i + args.batch]
+            # Re-check cardinality in the WHERE so a concurrent write that already
+            # shrank a row is skipped (idempotent, no wasted rewrite).
+            conn.execute(
+                f"UPDATE {tbl} SET provenance_event_ids = {_CAP_SQL} "
+                f"WHERE id = ANY(%(ids)s) AND cardinality(provenance_event_ids) > %(cap)s",
+                {"ids": chunk, "cap": args.cap},
+            )
+            done += len(chunk)
+            print(f"  capped {done}/{total} ({time.monotonic()-t0:.0f}s)", flush=True)
+
+        mx2 = conn.execute(
+            f"SELECT COALESCE(max(cardinality(provenance_event_ids)), 0) FROM {tbl}"
+        ).fetchone()[0]
+        print(f"done: {total} row(s) capped; max cardinality now {mx2} (<= {args.cap})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())