@pentatonic-ai/ai-agent-sdk 0.10.37 → 0.10.38

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.37";
+var VERSION = "0.10.38";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.37";
+var VERSION = "0.10.38";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.37",
+  "version": "0.10.38",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -2646,7 +2646,7 @@ def claim_next_batch(conn: psycopg.Connection) -> list[dict[str, Any]]:
               ) AND attempts < %s
               FOR UPDATE SKIP LOCKED
             )
-            RETURNING id, event_id, attempts
+            RETURNING id, event_id, attempts, transient_attempts
             """,
             (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, MAX_ATTEMPTS,
              BATCH_SIZE, BATCH_SIZE, MAX_ATTEMPTS),
@@ -2713,6 +2713,68 @@ def _attempts_exhausted(attempts: int) -> bool:
     return attempts + 1 >= MAX_ATTEMPTS
 
 
+# ── transient vs genuine failures ─────────────────────────────────────
+# Transient = infra hiccups that resolve on their own: the teacher timing out
+# under load (ReadTimeout) or the teacher-lb returning 502/503/504 while the
+# fleet is cold-starting from zero. These must NOT burn the genuine `attempts`
+# budget (3) — 3 quick ReadTimeouts used to march a row to `failed`, and failed
+# rows are never requeued, so the extraction was silently lost. Instead they get
+# their own larger budget (MAX_TRANSIENT_ATTEMPTS) on the `transient_attempts`
+# counter (migration 013), leaving `attempts` untouched so the row stays
+# claimable (`attempts < MAX_ATTEMPTS` in claim_next_batch). ReadTimeouts are
+# self-spaced by LLM_TIMEOUT_SEC and the fair-share round-robin spreads re-claims
+# across the backlog, so this does not hot-loop; MAX_TRANSIENT_ATTEMPTS bounds a
+# genuine poison chunk (one that times out even on a healthy multi-box fleet).
+MAX_TRANSIENT_ATTEMPTS = int(os.environ.get("MAX_TRANSIENT_ATTEMPTS", "12"))
+_TRANSIENT_MARKERS = (
+    "ReadTimeout",
+    "TimeoutException",
+    "ConnectTimeout",
+    "ConnectError",
+    "PoolTimeout",
+    "502",
+    "503",
+    "504",
+    "Bad Gateway",
+    "Gateway Time-out",
+    "Service Unavailable",
+    "Temporarily Unavailable",
+)
+
+
+def _is_transient(err: str) -> bool:
+    """True if `err` looks like a recoverable infra hiccup (teacher timeout /
+    LB gateway error) rather than a genuine extraction/DB failure."""
+    return any(m in err for m in _TRANSIENT_MARKERS)
+
+
+def requeue_transient(
+    conn: psycopg.Connection, queue_id: int, transient_attempts: int, error: str
+) -> None:
+    """Recoverable INFRA failure. Release the claim back to pending WITHOUT
+    touching `attempts` (preserves the genuine-failure budget), counting instead
+    against `transient_attempts`. Once that budget is exhausted the row IS marked
+    failed — a chunk that times out this many times is genuinely unprocessable,
+    not a passing hiccup."""
+    if transient_attempts + 1 >= MAX_TRANSIENT_ATTEMPTS:
+        mark_failed(conn, queue_id, f"transient-exhausted: {error}")
+        return
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            UPDATE distillation_queue SET
+              status = 'pending',
+              claimed_by = NULL,
+              claimed_at = NULL,
+              claim_expires_at = NULL,
+              transient_attempts = transient_attempts + 1,
+              last_error = %s
+            WHERE id = %s
+            """,
+            (error[:1024], queue_id),
+        )
+
+
 # --------------------------------------------------------------------
 # Main loop
 # --------------------------------------------------------------------
@@ -2924,7 +2986,11 @@ async def _run_teacher(
                 log.warning(
                     f"extraction failed queue_id={queue_id} attempts={attempts}: {err}"
                 )
-                if _attempts_exhausted(attempts):
+                if _is_transient(err):
+                    requeue_transient(
+                        conn, queue_id, item.get("transient_attempts", 0), err
+                    )
+                elif _attempts_exhausted(attempts):
                     mark_failed(conn, queue_id, err)
                 else:
                     release_claim(conn, queue_id, err)
@@ -3022,8 +3088,13 @@ def _apply_extraction(
             f"db upsert failed queue_id={queue_id} attempts={attempts}: {err}"
         )
         # The transaction rolled back; the claim bookkeeping runs on the
-        # now-autocommit connection so it persists independently.
-        if _attempts_exhausted(attempts):
+        # now-autocommit connection so it persists independently. A transient DB
+        # blip (connection/pool timeout) requeues without burning the genuine
+        # budget; real errors (UniqueViolation, schema mismatch) don't match
+        # _is_transient and take the normal attempts path.
+        if _is_transient(err):
+            requeue_transient(conn, queue_id, item.get("transient_attempts", 0), err)
+        elif _attempts_exhausted(attempts):
             mark_failed(conn, queue_id, err)
         else:
             release_claim(conn, queue_id, err)

package/packages/memory-engine-v2/org-model/migrations/013_distillation_transient_attempts.sql

@@ -0,0 +1,27 @@
+-- 013 — separate the TRANSIENT retry budget from the genuine-failure budget.
+--
+-- The distiller shares one `attempts` counter for every failure, capped at
+-- MAX_ATTEMPTS (3) and enforced by the claim query's `attempts < MAX_ATTEMPTS`
+-- filter. That conflates two very different failures:
+--   * genuine  — parse error, NULL-subject fact, schema mismatch: real, should
+--                give up after a few tries.
+--   * transient — teacher ReadTimeout under load, teacher-lb 502/503/504 during
+--                a scale-from-zero window: infra hiccups that resolve on their
+--                own once the fleet catches up / a box finishes booting.
+--
+-- Because they shared a counter, a transient hiccup burned the genuine budget:
+-- 3 quick ReadTimeouts → `failed`, and since failed rows are never requeued the
+-- extraction was silently LOST (measured 154 ReadTimeout losses in a single day
+-- during a bulk-import whale, 2026-07-14).
+--
+-- This adds a dedicated `transient_attempts` counter. worker.py routes transient
+-- errors here (incrementing THIS, leaving `attempts` untouched so the row stays
+-- claimable — `attempts` remains < MAX_ATTEMPTS) up to MAX_TRANSIENT_ATTEMPTS,
+-- after which the row is marked failed like any genuinely stuck work. Genuine
+-- errors keep using `attempts` exactly as before.
+--
+-- Metadata-only on PostgreSQL (constant DEFAULT, no table rewrite) — safe to run
+-- online against the ~250k-row prod table. Idempotent.
+
+ALTER TABLE distillation_queue
+  ADD COLUMN IF NOT EXISTS transient_attempts integer NOT NULL DEFAULT 0;