@pentatonic-ai/ai-agent-sdk 0.10.22 → 0.10.24

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.22";
+var VERSION = "0.10.24";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.22";
+var VERSION = "0.10.24";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.22",
+  "version": "0.10.24",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/Dockerfile

@@ -15,5 +15,10 @@ COPY worker.py .
 # container crash-loop on import at startup (observed 2026-06-08 deploy).
 # The test_*.py files are intentionally excluded; pytest only, not runtime.
 COPY noise_filter.py confidence.py entity_id.py sensitive_filter.py extraction_schema.py source_time.py ./
+# SEE-490 one-off backfill — a standalone script (not a worker.py import), but it
+# does `from worker import …` so it must sit beside worker.py in /app. #141 added
+# the script but not this COPY, so it never reached the image and the runbook's
+# `/app/backfill_structured_edges.py` apply step had nothing to run.
+COPY backfill_structured_edges.py ./
 
 CMD ["python", "worker.py"]

package/packages/memory-engine-v2/extractor-async/backfill_structured_edges.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""Backfill deterministic structured edges (SEE-490) over the EXISTING corpus.
+
+The structured-edge builders (`structured_graph_elements` in worker.py) run inside
+the distill flow, so every NEW event gets its deterministic edges — but the
+HISTORICAL graph does not. This one-off script closes that gap WITHOUT a re-distill.
+
+Why a dedicated script and not a re-distill:
+  - The structured edges are pure Python (no model call), but they're wired INSIDE
+    `_apply_extraction`, so a literal re-distill would re-run the teacher LLM over
+    ~700k events just to add edges that needed no model — slow and expensive.
+  - This script runs ONLY `structured_graph_elements` per event and upserts the
+    result via the SAME `upsert_entities` / `upsert_relationships` the distiller
+    uses. No LLM, fast, cheap.
+
+Why it is SAFE (and needs NO delete-first, unlike the hard-key re-distill runbook):
+  - PURELY ADDITIVE. It changes NO existing node id — it upserts deterministic
+    edges and their hard-keyed endpoint nodes (person→email, doc→drive-file-id,
+    channel→channel-id, meeting→meeting-id) idempotently on `(arena, from, to,
+    type)`. Re-running is a no-op-shaped upsert; it cannot double the graph.
+  - Rollback is a single scoped DELETE (the edges carry a provenance stamp):
+        DELETE FROM relationships
+        WHERE arena LIKE '<scope>' AND attributes->>'derivation' = 'structured';
+    (The few email/file-id-keyed endpoint nodes it may mint are benign additions
+    that converge with the LLM/sync passes; they need no cleanup.)
+
+Multi-tenant blast radius: scoped by `--arena` (default `pentatonic-team%`). It
+NEVER touches `pip-agents` (legacy) or test tenants unless you widen the scope.
+
+Usage (on the engine box, AFTER the SDK carrying SEE-490 is deployed + the
+extractor-async container rebuilt — verify `structured_graph_elements` is present
+in the installed package first):
+
+    python backfill_structured_edges.py --dry-run            # report only, no writes
+    python backfill_structured_edges.py                      # apply (pentatonic-team)
+    python backfill_structured_edges.py --sources gmail,drive  # one/few sources
+    python backfill_structured_edges.py --arena 'pentatonic-team%' --commit-every 500
+
+Reads `PG_DSN` from the environment (same as the worker).
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+import time
+from typing import Any
+
+import psycopg
+
+# Reuse the distiller's own functions so behaviour is byte-identical to the live
+# path — the builders, the entity resolution/keying, and the edge upsert.
+from worker import (  # type: ignore
+    structured_graph_elements,
+    upsert_entities,
+    upsert_relationships,
+    event_source_time,
+)
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
+)
+log = logging.getLogger("backfill-structured-edges")
+
+# Producer labels that actually yield structured edges today — used only to narrow
+# the scan (and to honour --sources). Keep in sync with worker._STRUCTURED_EDGE_BUILDERS.
+_EDGE_SOURCES = {
+    "gmail": "pip-gmail-ingest",
+    "drive": "pip-drive-ingest",
+    "slack": "pip-slack-ingest",
+    "granola": "pip-granola-ingest",
+    "calendar": "pip-calendar-ingest",
+}
+
+
+def _parse_args(argv: list[str]) -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "--arena", default="pentatonic-team%",
+        help="arena LIKE filter (default 'pentatonic-team%%'). Blast-radius guard.",
+    )
+    p.add_argument(
+        "--sources", default="",
+        help="comma list of "
+        + "/".join(_EDGE_SOURCES)
+        + " to limit the scan (default: all edge-producing sources).",
+    )
+    p.add_argument(
+        "--dry-run", action="store_true",
+        help="compute + report edges that WOULD be created; write nothing.",
+    )
+    p.add_argument("--limit", type=int, default=0, help="cap events scanned (0 = all).")
+    p.add_argument(
+        "--commit-every", type=int, default=500,
+        help="commit the write txn every N edge-producing events (default 500).",
+    )
+    return p.parse_args(argv)
+
+
+def _source_filter(sources_arg: str) -> list[str]:
+    if not sources_arg.strip():
+        return list(_EDGE_SOURCES.values())
+    out: list[str] = []
+    for s in sources_arg.split(","):
+        s = s.strip().lower()
+        if s not in _EDGE_SOURCES:
+            raise SystemExit(f"unknown source {s!r}; valid: {', '.join(_EDGE_SOURCES)}")
+        out.append(_EDGE_SOURCES[s])
+    return out
+
+
+def main(argv: list[str]) -> int:
+    args = _parse_args(argv)
+    dsn = os.environ.get("PG_DSN")
+    if not dsn:
+        raise SystemExit("PG_DSN not set")
+    sources = _source_filter(args.sources)
+
+    log.info(
+        "backfill start: arena LIKE %r sources=%s dry_run=%s limit=%s",
+        args.arena, sources, args.dry_run, args.limit or "all",
+    )
+
+    # Separate connections: a server-side (named) cursor STREAMS the read so we
+    # never load the whole events table into memory; a second connection does the
+    # batched writes. The read connection MUST NOT be autocommit — psycopg3's
+    # named server-side cursor issues DECLARE CURSOR, which requires a
+    # transaction block ("DECLARE CURSOR can only be used in transaction
+    # blocks"). It's read-only (SELECT), never committed, just closed at the end.
+    read_conn = psycopg.connect(dsn)
+    write_conn = psycopg.connect(dsn)
+
+    scanned = 0
+    with_edges = 0
+    edges_total = 0
+    ents_total = 0
+    t0 = time.perf_counter()
+    pending_in_txn = 0
+
+    try:
+        with read_conn.cursor(name="backfill_events") as cur:
+            cur.itersize = 1000
+            sql = """
+                SELECT id, arena, attributes, participant_set, disclosure_class,
+                       emitted_at
+                FROM events
+                WHERE arena LIKE %s
+                  AND attributes->>'source' = ANY(%s)
+                ORDER BY received_at
+            """
+            params: tuple[Any, ...] = (args.arena, sources)
+            if args.limit:
+                sql += " LIMIT %s"
+                params = (*params, args.limit)
+            cur.execute(sql, params)
+
+            for event_id, arena, attributes, participant_set, disclosure, emitted_at in cur:
+                scanned += 1
+                attrs = attributes or {}
+                det_ents, det_rels = structured_graph_elements(arena, attrs)
+                if not det_rels:
+                    continue
+                with_edges += 1
+                edges_total += len(det_rels)
+                ents_total += len(det_ents)
+
+                if args.dry_run:
+                    if with_edges <= 5:
+                        log.info(
+                            "would add %d edges from %s (%s): %s",
+                            len(det_rels), event_id, attrs.get("source"),
+                            sorted({r["type"] for r in det_rels}),
+                        )
+                else:
+                    pset = participant_set or [arena]
+                    disc = disclosure or "private"
+                    event_time = event_source_time({"attributes": attrs}) or emitted_at
+                    name_to_id = upsert_entities(
+                        write_conn, arena, event_id, pset, disc, det_ents,
+                        event_time, attrs,
+                    )
+                    upsert_relationships(
+                        write_conn, arena, event_id, pset, disc, det_rels,
+                        name_to_id, event_time,
+                    )
+                    pending_in_txn += 1
+                    if pending_in_txn >= args.commit_every:
+                        write_conn.commit()
+                        pending_in_txn = 0
+
+                if with_edges % 5000 == 0 and with_edges:
+                    rate = scanned / max(time.perf_counter() - t0, 1e-6)
+                    log.info(
+                        "progress: scanned=%d with_edges=%d edges=%d (%.0f ev/s)",
+                        scanned, with_edges, edges_total, rate,
+                    )
+
+        if not args.dry_run:
+            write_conn.commit()
+    except Exception:
+        if not args.dry_run:
+            write_conn.rollback()
+        log.exception("backfill failed; write txn rolled back")
+        return 1
+    finally:
+        read_conn.close()
+        write_conn.close()
+
+    dur = time.perf_counter() - t0
+    log.info(
+        "backfill %s: scanned=%d events, edge-producing=%d, edges=%d, endpoint-nodes=%d, %.1fs",
+        "DRY-RUN (no writes)" if args.dry_run else "DONE",
+        scanned, with_edges, edges_total, ents_total, dur,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/packages/memory-engine-v2/extractor-async/test_backfill_structured_edges.py

@@ -0,0 +1,57 @@
+"""Pure-logic tests for the structured-edge backfill (SEE-490). The DB path
+(stream read → upsert) is exercised by the live run; here we pin the CLI scope
+parsing + the source-label mapping (the blast-radius / --sources guards)."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+_THIS = Path(__file__).resolve().parent
+_SPEC = importlib.util.spec_from_file_location(
+    "backfill_structured_edges", _THIS / "backfill_structured_edges.py"
+)
+assert _SPEC and _SPEC.loader
+bf = importlib.util.module_from_spec(_SPEC)
+try:
+    _SPEC.loader.exec_module(bf)
+except ImportError as e:
+    pytest.skip(f"extractor-async deps unavailable: {e}", allow_module_level=True)
+
+
+def test_source_filter_default_is_all_edge_sources():
+    got = set(bf._source_filter(""))
+    assert got == set(bf._EDGE_SOURCES.values())
+    # all five live edge-producing producers
+    assert "pip-gmail-ingest" in got and "pip-calendar-ingest" in got
+
+
+def test_source_filter_named_subset():
+    assert bf._source_filter("gmail,drive") == ["pip-gmail-ingest", "pip-drive-ingest"]
+    assert bf._source_filter(" Slack , Granola ") == [
+        "pip-slack-ingest", "pip-granola-ingest",
+    ]
+
+
+def test_source_filter_rejects_unknown():
+    with pytest.raises(SystemExit):
+        bf._source_filter("teams")  # not an edge source (no envelope)
+
+
+def test_default_scope_is_pentatonic_team():
+    # The blast-radius guard: default arena filter must be tenant-scoped, never
+    # the whole multi-tenant table.
+    args = bf._parse_args([])
+    assert args.arena == "pentatonic-team%"
+    assert args.dry_run is False
+    assert args.commit_every == 500
+
+
+def test_dry_run_and_sources_flags():
+    args = bf._parse_args(["--dry-run", "--sources", "gmail", "--limit", "10"])
+    assert args.dry_run is True
+    assert args.limit == 10
+    assert bf._source_filter(args.sources) == ["pip-gmail-ingest"]