@pentatonic-ai/ai-agent-sdk 0.10.23 → 0.10.25

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.23";
+var VERSION = "0.10.25";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.23";
+var VERSION = "0.10.25";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.23",
+  "version": "0.10.25",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/Dockerfile

@@ -15,5 +15,10 @@ COPY worker.py .
 # container crash-loop on import at startup (observed 2026-06-08 deploy).
 # The test_*.py files are intentionally excluded; pytest only, not runtime.
 COPY noise_filter.py confidence.py entity_id.py sensitive_filter.py extraction_schema.py source_time.py ./
+# SEE-490 one-off backfill — a standalone script (not a worker.py import), but it
+# does `from worker import …` so it must sit beside worker.py in /app. #141 added
+# the script but not this COPY, so it never reached the image and the runbook's
+# `/app/backfill_structured_edges.py` apply step had nothing to run.
+COPY backfill_structured_edges.py ./
 
 CMD ["python", "worker.py"]

package/packages/memory-engine-v2/extractor-async/backfill_structured_edges.py

@@ -37,6 +37,17 @@ in the installed package first):
     python backfill_structured_edges.py --arena 'pentatonic-team%' --commit-every 500
 
 Reads `PG_DSN` from the environment (same as the worker).
+
+Concurrent-writer contention (SEE-490 prod incident):
+  When run against PROD this backfill shares the `relationships`/`entities` rows
+  with the LIVE distiller worker, so the two writers can deadlock on the same
+  rows. A single unhandled `DeadlockDetected` used to kill the whole ~17h run
+  (the prod run died at ~78%). The write path is now wrapped in a bounded
+  retry-with-backoff loop and — crucially — the pending batch is BUFFERED
+  (computed, not yet written) so that on a deadlock we can `rollback()` and
+  re-apply the *entire* buffered batch, without re-reading from the streaming
+  cursor. Everything is idempotent (`ON CONFLICT (id) DO UPDATE`), so already
+  committed work is durable and a fresh resume simply re-does the tail.
 """
 
 from __future__ import annotations
@@ -44,12 +55,33 @@ from __future__ import annotations
 import argparse
 import logging
 import os
+import random
 import sys
 import time
 from typing import Any
 
 import psycopg
 
+# Transient, contention-driven failures that a retry can clear. A backfill that
+# runs alongside the live distiller WILL hit these — Postgres picks one of two
+# deadlocking transactions as the victim and aborts it; a serialization failure
+# is the MVCC-level equivalent; OperationalError is the broad psycopg umbrella
+# (dropped/again-busy connection) we also want to survive rather than abort a
+# 17h run over. All are safe to retry here because every write is an idempotent
+# ON CONFLICT (id) DO UPDATE upsert.
+_RETRYABLE_ERRORS: tuple[type[BaseException], ...] = (
+    psycopg.errors.DeadlockDetected,
+    psycopg.errors.SerializationFailure,
+    psycopg.errors.OperationalError,
+)
+
+# Bounded exponential backoff with jitter. Real wall-clock sleep is fine in this
+# operational one-off; the *decision* helpers below are pure so they stay unit
+# testable without a live DB.
+_MAX_WRITE_RETRIES = 5
+_BACKOFF_BASE_S = 0.5
+_BACKOFF_CAP_S = 30.0
+
 # Reuse the distiller's own functions so behaviour is byte-identical to the live
 # path — the builders, the entity resolution/keying, and the edge upsert.
 from worker import (  # type: ignore
@@ -111,6 +143,116 @@ def _source_filter(sources_arg: str) -> list[str]:
     return out
 
 
+def _is_retryable(exc: BaseException) -> bool:
+    """True iff `exc` is a transient contention failure worth retrying.
+
+    Pure predicate (no I/O) so the retry decision is unit-testable without a DB.
+    Retryable types are the psycopg contention errors captured in
+    `_RETRYABLE_ERRORS`; anything else (a real data/logic error) must propagate.
+    """
+    return isinstance(exc, _RETRYABLE_ERRORS)
+
+
+def _should_retry(exc: BaseException, attempt: int, max_retries: int = _MAX_WRITE_RETRIES) -> bool:
+    """Decide retry-vs-raise for `exc` on `attempt` (1-based) of a batch write.
+
+    Retry only when the error is contention-driven AND we have attempts left.
+    `attempt` counts the try that just failed, so attempt N == max_retries is the
+    last one we allow before giving up (re-raise). Pure — no sleeping, no I/O.
+    """
+    return _is_retryable(exc) and attempt < max_retries
+
+
+def _backoff_seconds(
+    attempt: int,
+    base: float = _BACKOFF_BASE_S,
+    cap: float = _BACKOFF_CAP_S,
+    jitter: float | None = None,
+) -> float:
+    """Exponential backoff (base * 2**(attempt-1)), capped, plus small jitter.
+
+    `attempt` is 1-based. Jitter defaults to a random fraction of the step so
+    contending writers de-sync; pass an explicit `jitter` in [0,1) for
+    deterministic tests. Pure aside from the optional RNG draw.
+    """
+    step = min(base * (2 ** max(attempt - 1, 0)), cap)
+    if jitter is None:
+        jitter = random.random()
+    return step + jitter * base
+
+
+def _apply_event_edges(
+    write_conn: "psycopg.Connection[Any]",
+    item: dict[str, Any],
+) -> None:
+    """Upsert one buffered edge-producing event's entities + relationships.
+
+    Uses the distiller's own upsert helpers so behaviour matches the live path.
+    Called inside the retry loop, so it must be safe to re-run after a rollback —
+    it is, because both upserts are idempotent ON CONFLICT (id) DO UPDATE.
+    """
+    name_to_id = upsert_entities(
+        write_conn,
+        item["arena"],
+        item["event_id"],
+        item["pset"],
+        item["disc"],
+        item["det_ents"],
+        item["event_time"],
+        item["attrs"],
+    )
+    upsert_relationships(
+        write_conn,
+        item["arena"],
+        item["event_id"],
+        item["pset"],
+        item["disc"],
+        item["det_rels"],
+        name_to_id,
+        item["event_time"],
+    )
+
+
+def _commit_batch_with_retry(
+    write_conn: "psycopg.Connection[Any]",
+    batch: list[dict[str, Any]],
+) -> None:
+    """Apply the whole buffered `batch` and commit, retrying on contention.
+
+    The batch is re-applied FROM ITS BUFFER on every attempt — on a deadlock we
+    `rollback()` (discarding the partially-applied txn) and replay the entire
+    buffered batch, so the streaming READ cursor never has to rewind. Bounded by
+    `_MAX_WRITE_RETRIES`; a non-retryable error, or exhausted retries, re-raises.
+    """
+    if not batch:
+        return
+    lo = batch[0]["event_id"]
+    hi = batch[-1]["event_id"]
+    attempt = 0
+    while True:
+        attempt += 1
+        try:
+            for item in batch:
+                _apply_event_edges(write_conn, item)
+            write_conn.commit()
+            return
+        except Exception as exc:  # noqa: BLE001 — classified by _should_retry
+            try:
+                write_conn.rollback()
+            except Exception:  # rollback best-effort; don't mask the original
+                log.warning("rollback after write error also failed", exc_info=True)
+            if not _should_retry(exc, attempt):
+                raise
+            delay = _backoff_seconds(attempt)
+            log.warning(
+                "write contention (%s) on batch events %s..%s (%d edge-events); "
+                "retry attempt %d/%d after %.2fs backoff",
+                type(exc).__name__, lo, hi, len(batch),
+                attempt, _MAX_WRITE_RETRIES, delay,
+            )
+            time.sleep(delay)
+
+
 def main(argv: list[str]) -> int:
     args = _parse_args(argv)
     dsn = os.environ.get("PG_DSN")
@@ -125,8 +267,11 @@ def main(argv: list[str]) -> int:
 
     # Separate connections: a server-side (named) cursor STREAMS the read so we
     # never load the whole events table into memory; a second connection does the
-    # batched writes. Read connection is autocommit (no long-lived read txn).
-    read_conn = psycopg.connect(dsn, autocommit=True)
+    # batched writes. The read connection MUST NOT be autocommit — psycopg3's
+    # named server-side cursor issues DECLARE CURSOR, which requires a
+    # transaction block ("DECLARE CURSOR can only be used in transaction
+    # blocks"). It's read-only (SELECT), never committed, just closed at the end.
+    read_conn = psycopg.connect(dsn)
     write_conn = psycopg.connect(dsn)
 
     scanned = 0
@@ -134,7 +279,12 @@ def main(argv: list[str]) -> int:
     edges_total = 0
     ents_total = 0
     t0 = time.perf_counter()
-    pending_in_txn = 0
+    # Buffer of edge-producing events computed since the last successful commit.
+    # We hold the computed upsert args (NOT yet written) so that on a deadlock we
+    # can rollback the write txn and replay the whole batch without rewinding the
+    # streaming read cursor. Flushed (applied + committed, with retry) every
+    # --commit-every edge-producing events.
+    pending_batch: list[dict[str, Any]] = []
 
     try:
         with read_conn.cursor(name="backfill_events") as cur:
@@ -174,18 +324,22 @@ def main(argv: list[str]) -> int:
                     pset = participant_set or [arena]
                     disc = disclosure or "private"
                     event_time = event_source_time({"attributes": attrs}) or emitted_at
-                    name_to_id = upsert_entities(
-                        write_conn, arena, event_id, pset, disc, det_ents,
-                        event_time, attrs,
-                    )
-                    upsert_relationships(
-                        write_conn, arena, event_id, pset, disc, det_rels,
-                        name_to_id, event_time,
-                    )
-                    pending_in_txn += 1
-                    if pending_in_txn >= args.commit_every:
-                        write_conn.commit()
-                        pending_in_txn = 0
+                    # Buffer the computed upsert args; do NOT touch the write conn
+                    # yet. The whole buffer is applied + committed together (with
+                    # deadlock retry) at flush time.
+                    pending_batch.append({
+                        "arena": arena,
+                        "event_id": event_id,
+                        "pset": pset,
+                        "disc": disc,
+                        "det_ents": det_ents,
+                        "det_rels": det_rels,
+                        "event_time": event_time,
+                        "attrs": attrs,
+                    })
+                    if len(pending_batch) >= args.commit_every:
+                        _commit_batch_with_retry(write_conn, pending_batch)
+                        pending_batch = []
 
                 if with_edges % 5000 == 0 and with_edges:
                     rate = scanned / max(time.perf_counter() - t0, 1e-6)
@@ -195,11 +349,20 @@ def main(argv: list[str]) -> int:
                     )
 
         if not args.dry_run:
-            write_conn.commit()
+            # Flush the final partial batch (same retry-on-contention path).
+            _commit_batch_with_retry(write_conn, pending_batch)
+            pending_batch = []
     except Exception:
         if not args.dry_run:
-            write_conn.rollback()
-        log.exception("backfill failed; write txn rolled back")
+            try:
+                write_conn.rollback()
+            except Exception:
+                log.warning("final rollback also failed", exc_info=True)
+        log.exception(
+            "backfill failed; write txn rolled back. Already-committed batches are "
+            "durable/idempotent — re-run to resume (ON CONFLICT makes it a no-op "
+            "over prior work)."
+        )
         return 1
     finally:
         read_conn.close()

package/packages/memory-engine-v2/extractor-async/test_backfill_structured_edges.py

@@ -55,3 +55,44 @@ def test_dry_run_and_sources_flags():
     assert args.dry_run is True
     assert args.limit == 10
     assert bf._source_filter(args.sources) == ["pip-gmail-ingest"]
+
+
+# --- retry / backoff decision (SEE-490 prod deadlock survival) ----------------
+# Pure-logic guards for the contention-retry path. The live DB replay is
+# exercised by the run itself; here we pin the retry-vs-raise decision and the
+# backoff schedule so a regression in either is caught without a live DB.
+
+def test_deadlock_is_retryable_but_bounded():
+    # A deadlock (the exact prod failure) is retried while attempts remain and
+    # re-raised once they're exhausted.
+    exc = bf.psycopg.errors.DeadlockDetected("deadlock detected")
+    assert bf._is_retryable(exc) is True
+    assert bf._should_retry(exc, attempt=1, max_retries=5) is True
+    assert bf._should_retry(exc, attempt=4, max_retries=5) is True
+    # attempt == max_retries is the last allowed try; no further retry.
+    assert bf._should_retry(exc, attempt=5, max_retries=5) is False
+
+
+def test_serialization_and_operational_are_retryable():
+    assert bf._is_retryable(bf.psycopg.errors.SerializationFailure("s")) is True
+    assert bf._is_retryable(bf.psycopg.errors.OperationalError("o")) is True
+
+
+def test_non_contention_error_never_retries():
+    # A logic/data error (not contention) must propagate immediately, never retry.
+    exc = ValueError("bad data")
+    assert bf._is_retryable(exc) is False
+    assert bf._should_retry(exc, attempt=1, max_retries=5) is False
+
+
+def test_backoff_is_exponential_capped_and_deterministic_with_fixed_jitter():
+    # With jitter pinned to 0 the schedule is pure exponential: base * 2**(n-1).
+    base, cap = 0.5, 30.0
+    assert bf._backoff_seconds(1, base=base, cap=cap, jitter=0.0) == 0.5
+    assert bf._backoff_seconds(2, base=base, cap=cap, jitter=0.0) == 1.0
+    assert bf._backoff_seconds(3, base=base, cap=cap, jitter=0.0) == 2.0
+    # Later attempts saturate at the cap (+ the deterministic jitter term).
+    assert bf._backoff_seconds(20, base=base, cap=cap, jitter=0.0) == cap
+    # Jitter adds at most `base`, keeping the value within [step, step+base).
+    jittered = bf._backoff_seconds(1, base=base, cap=cap, jitter=0.5)
+    assert 0.5 <= jittered < 0.5 + base