@pentatonic-ai/ai-agent-sdk 0.10.36 → 0.10.38

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.36";
+var VERSION = "0.10.38";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.36";
+var VERSION = "0.10.38";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.36",
+  "version": "0.10.38",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/compat/server.py

@@ -37,6 +37,7 @@ import uuid
 from contextlib import asynccontextmanager
 from datetime import datetime
 from typing import Any
+from urllib.parse import urlsplit
 
 import httpx
 import numpy as np
@@ -59,6 +60,14 @@ NV_EMBED_PROVIDER = os.environ.get("NV_EMBED_PROVIDER", "openai")  # 'openai' |
 EMBED_DIM = int(os.environ.get("EMBED_DIM", "4096"))
 # Window (minutes) over which /health/deep computes the distillation drain rate.
 HEALTH_DRAIN_WINDOW_MIN = int(os.environ.get("HEALTH_DRAIN_WINDOW_MIN", "5"))
+# GPU/compute-tier endpoints probed by the /health/deep stack-online gauge. The
+# teacher is probed THROUGH its load-balancer (a stable service name that is up
+# even when the fleet is scaled to zero) — see health_deep for why the probe
+# treats a no-upstream 502 as offline. STUDENT_ENDPOINT is the vLLM student box;
+# when unset (not wired into compat's env) the student tier is simply omitted
+# from the gauge rather than counted as down.
+STUDENT_ENDPOINT = os.environ.get("STUDENT_ENDPOINT", "")
+TEACHER_HEALTH_URL = os.environ.get("TEACHER_HEALTH_URL", "http://teacher-lb:8500/v1/models")
 
 COLLECTION_NAME = "evidence"
 
@@ -651,10 +660,36 @@ async def health():
     return {"status": "healthy", "service": "pme2-compat", "version": "0.1.0"}
 
 
+def _models_url(endpoint: str) -> str:
+    """Rewrite any endpoint URL to its OpenAI-style /v1/models path — a cheap
+    GET the vLLM student/teacher boxes and the embed gateways all answer. Keeps
+    only scheme+host+port so it works whether the source URL was /v1/embeddings,
+    /v1/chat/completions, etc."""
+    p = urlsplit(endpoint)
+    return f"{p.scheme}://{p.netloc}/v1/models"
+
+
+async def _tier_online(url: str, *, healthy_below: int = 600) -> bool:
+    """Liveness probe for one compute tier. Online iff the endpoint answers with
+    an HTTP status < healthy_below within the timeout. Connection refused /
+    timeout / DNS failure → offline.
+
+    healthy_below defaults to 600 (any HTTP answer = the box is powered on and
+    serving). Pass 500 for a tier fronted by an always-up load balancer (the
+    teacher LB): with the fleet scaled to zero the LB itself still answers, but
+    with a 502 (no healthy upstream) — which we must read as OFFLINE, not "the
+    LB replied, so it's up"."""
+    try:
+        r = await _http.get(url, timeout=3.0)
+        return r.status_code < healthy_below
+    except Exception:
+        return False
+
+
 @app.get("/health/deep")
 async def health_deep():
-    """Round-trips all three stores + the embed gateway. Slow; do not
-    use as a docker healthcheck."""
+    """Round-trips all three stores + the embed gateway, then computes a
+    stack-online gauge. Slow; do not use as a docker healthcheck."""
     result = {"compat": "ok", "stores": {}}
 
     # org-model
@@ -710,6 +745,44 @@ async def health_deep():
     except Exception as e:
         result["stores"]["embed_gateway"] = {"status": "error", "error": str(e)}
 
+    # Stack-online gauge: what fraction of the compute stack is powered on RIGHT
+    # NOW. Deliberately counts every tier — including the scale-to-zero teacher
+    # fleet — so a parked teacher legitimately lowers the %. This is a "what's
+    # running" dial, not a pure health signal: the distillation drain/backlog
+    # block above says whether a parked teacher is actually a problem. Tiers the
+    # store round-trips already assessed are reused (no re-probe); the GPU tiers
+    # are probed concurrently for liveness.
+    components: dict[str, bool] = {
+        "compat": True,  # we are the one answering
+        "org_model": result["stores"].get("org_model", {}).get("status") == "ok",
+        "vector_index": result["stores"].get("vector_index", {}).get("status") == "ok",
+        "embed_interactive": result["stores"].get("embed_gateway", {}).get("status") == "ok",
+    }
+    # GPU/compute tiers not covered by the store probes. (name, url, healthy_below)
+    gpu_probes = [("teacher", TEACHER_HEALTH_URL, 500)]
+    # Bulk embedder only counts as a distinct tier when it's a separate box.
+    if NV_EMBED_URL_BULK and NV_EMBED_URL_BULK != NV_EMBED_URL:
+        gpu_probes.append(("embed_bulk", _models_url(NV_EMBED_URL_BULK), 600))
+    if STUDENT_ENDPOINT:
+        gpu_probes.append(("student", _models_url(STUDENT_ENDPOINT), 600))
+    try:
+        probe_results = await asyncio.gather(
+            *(_tier_online(url, healthy_below=hb) for _, url, hb in gpu_probes)
+        )
+        for (name, _, _), up in zip(gpu_probes, probe_results):
+            components[name] = up
+    except Exception as e:  # pragma: no cover — probing must never 500 the endpoint
+        result["stack_probe_error"] = str(e)
+
+    online = sum(1 for up in components.values() if up)
+    total = len(components)
+    result["stack"] = {
+        "online_pct": round(100 * online / total, 1) if total else 0.0,
+        "online": online,
+        "total": total,
+        "components": {k: ("up" if up else "down") for k, up in components.items()},
+    }
+
     return result
 
 

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -2646,7 +2646,7 @@ def claim_next_batch(conn: psycopg.Connection) -> list[dict[str, Any]]:
               ) AND attempts < %s
               FOR UPDATE SKIP LOCKED
             )
-            RETURNING id, event_id, attempts
+            RETURNING id, event_id, attempts, transient_attempts
             """,
             (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, MAX_ATTEMPTS,
              BATCH_SIZE, BATCH_SIZE, MAX_ATTEMPTS),
@@ -2713,6 +2713,68 @@ def _attempts_exhausted(attempts: int) -> bool:
     return attempts + 1 >= MAX_ATTEMPTS
 
 
+# ── transient vs genuine failures ─────────────────────────────────────
+# Transient = infra hiccups that resolve on their own: the teacher timing out
+# under load (ReadTimeout) or the teacher-lb returning 502/503/504 while the
+# fleet is cold-starting from zero. These must NOT burn the genuine `attempts`
+# budget (3) — 3 quick ReadTimeouts used to march a row to `failed`, and failed
+# rows are never requeued, so the extraction was silently lost. Instead they get
+# their own larger budget (MAX_TRANSIENT_ATTEMPTS) on the `transient_attempts`
+# counter (migration 013), leaving `attempts` untouched so the row stays
+# claimable (`attempts < MAX_ATTEMPTS` in claim_next_batch). ReadTimeouts are
+# self-spaced by LLM_TIMEOUT_SEC and the fair-share round-robin spreads re-claims
+# across the backlog, so this does not hot-loop; MAX_TRANSIENT_ATTEMPTS bounds a
+# genuine poison chunk (one that times out even on a healthy multi-box fleet).
+MAX_TRANSIENT_ATTEMPTS = int(os.environ.get("MAX_TRANSIENT_ATTEMPTS", "12"))
+_TRANSIENT_MARKERS = (
+    "ReadTimeout",
+    "TimeoutException",
+    "ConnectTimeout",
+    "ConnectError",
+    "PoolTimeout",
+    "502",
+    "503",
+    "504",
+    "Bad Gateway",
+    "Gateway Time-out",
+    "Service Unavailable",
+    "Temporarily Unavailable",
+)
+
+
+def _is_transient(err: str) -> bool:
+    """True if `err` looks like a recoverable infra hiccup (teacher timeout /
+    LB gateway error) rather than a genuine extraction/DB failure."""
+    return any(m in err for m in _TRANSIENT_MARKERS)
+
+
+def requeue_transient(
+    conn: psycopg.Connection, queue_id: int, transient_attempts: int, error: str
+) -> None:
+    """Recoverable INFRA failure. Release the claim back to pending WITHOUT
+    touching `attempts` (preserves the genuine-failure budget), counting instead
+    against `transient_attempts`. Once that budget is exhausted the row IS marked
+    failed — a chunk that times out this many times is genuinely unprocessable,
+    not a passing hiccup."""
+    if transient_attempts + 1 >= MAX_TRANSIENT_ATTEMPTS:
+        mark_failed(conn, queue_id, f"transient-exhausted: {error}")
+        return
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            UPDATE distillation_queue SET
+              status = 'pending',
+              claimed_by = NULL,
+              claimed_at = NULL,
+              claim_expires_at = NULL,
+              transient_attempts = transient_attempts + 1,
+              last_error = %s
+            WHERE id = %s
+            """,
+            (error[:1024], queue_id),
+        )
+
+
 # --------------------------------------------------------------------
 # Main loop
 # --------------------------------------------------------------------
@@ -2924,7 +2986,11 @@ async def _run_teacher(
                 log.warning(
                     f"extraction failed queue_id={queue_id} attempts={attempts}: {err}"
                 )
-                if _attempts_exhausted(attempts):
+                if _is_transient(err):
+                    requeue_transient(
+                        conn, queue_id, item.get("transient_attempts", 0), err
+                    )
+                elif _attempts_exhausted(attempts):
                     mark_failed(conn, queue_id, err)
                 else:
                     release_claim(conn, queue_id, err)
@@ -3022,8 +3088,13 @@ def _apply_extraction(
             f"db upsert failed queue_id={queue_id} attempts={attempts}: {err}"
         )
         # The transaction rolled back; the claim bookkeeping runs on the
-        # now-autocommit connection so it persists independently.
-        if _attempts_exhausted(attempts):
+        # now-autocommit connection so it persists independently. A transient DB
+        # blip (connection/pool timeout) requeues without burning the genuine
+        # budget; real errors (UniqueViolation, schema mismatch) don't match
+        # _is_transient and take the normal attempts path.
+        if _is_transient(err):
+            requeue_transient(conn, queue_id, item.get("transient_attempts", 0), err)
+        elif _attempts_exhausted(attempts):
             mark_failed(conn, queue_id, err)
         else:
             release_claim(conn, queue_id, err)

package/packages/memory-engine-v2/org-model/migrations/013_distillation_transient_attempts.sql

@@ -0,0 +1,27 @@
+-- 013 — separate the TRANSIENT retry budget from the genuine-failure budget.
+--
+-- The distiller shares one `attempts` counter for every failure, capped at
+-- MAX_ATTEMPTS (3) and enforced by the claim query's `attempts < MAX_ATTEMPTS`
+-- filter. That conflates two very different failures:
+--   * genuine  — parse error, NULL-subject fact, schema mismatch: real, should
+--                give up after a few tries.
+--   * transient — teacher ReadTimeout under load, teacher-lb 502/503/504 during
+--                a scale-from-zero window: infra hiccups that resolve on their
+--                own once the fleet catches up / a box finishes booting.
+--
+-- Because they shared a counter, a transient hiccup burned the genuine budget:
+-- 3 quick ReadTimeouts → `failed`, and since failed rows are never requeued the
+-- extraction was silently LOST (measured 154 ReadTimeout losses in a single day
+-- during a bulk-import whale, 2026-07-14).
+--
+-- This adds a dedicated `transient_attempts` counter. worker.py routes transient
+-- errors here (incrementing THIS, leaving `attempts` untouched so the row stays
+-- claimable — `attempts` remains < MAX_ATTEMPTS) up to MAX_TRANSIENT_ATTEMPTS,
+-- after which the row is marked failed like any genuinely stuck work. Genuine
+-- errors keep using `attempts` exactly as before.
+--
+-- Metadata-only on PostgreSQL (constant DEFAULT, no table rewrite) — safe to run
+-- online against the ~250k-row prod table. Idempotent.
+
+ALTER TABLE distillation_queue
+  ADD COLUMN IF NOT EXISTS transient_attempts integer NOT NULL DEFAULT 0;