@pentatonic-ai/ai-agent-sdk 0.10.29 → 0.10.30

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.29";
+var VERSION = "0.10.30";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.29";
+var VERSION = "0.10.30";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.29",
+  "version": "0.10.30",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -131,6 +131,28 @@ STUDENT_MAX_TOKENS = int(os.environ.get("STUDENT_MAX_TOKENS", "768"))
 # the box's KV-cache headroom. Do NOT raise CONCURRENT_LLM_CALLS to fix student
 # throughput — that also raises teacher 27B concurrency (OOM risk).
 STUDENT_CONCURRENT_CALLS = int(os.environ.get("STUDENT_CONCURRENT_CALLS", "48"))
+# How many claimed batches the worker processes CONCURRENTLY. The prior loop
+# claimed a batch, awaited it end-to-end (student gather → straggler tail →
+# teacher tail), and only then claimed the next — a hard barrier. Wall-clock
+# profiling of the 25k-backlog incident (2026-07-08) showed the student L40S
+# burst to Running:50 for ~10s then sit at Running:1 for ~3min draining a
+# straggler tail while the (LB'd) teacher fleet idled — the GPUs were starved by
+# the loop, not by capacity. Overlapping N batches, all sharing ONE student pool
+# and ONE teacher pool (the shared semaphores below), keeps those pools
+# continuously full: batch B's fresh student calls take the slots batch A's
+# stragglers free. Total LLM concurrency is still bounded by the shared sems
+# (STUDENT_CONCURRENT_CALLS / CONCURRENT_LLM_CALLS) REGARDLESS of this value —
+# this only controls how many independent claim→apply pipelines feed them (each
+# on its own PG connection; applies within a batch stay sequential, so one conn
+# per batch is never touched concurrently). Raise to feed the pools harder; the
+# sems, not this, cap GPU load.
+MAX_INFLIGHT_BATCHES = int(os.environ.get("MAX_INFLIGHT_BATCHES", "4"))
+# Process-global shared concurrency pools, created once inside amain's event
+# loop (NOT per-batch — per-batch pools are what caused the burst-then-idle).
+# All in-flight batches acquire from these, so the student/teacher GPUs see one
+# continuous stream of calls instead of per-batch bursts. None until amain runs.
+_STUDENT_SEM: "asyncio.Semaphore | None" = None
+_TEACHER_SEM: "asyncio.Semaphore | None" = None
 # Fraction of student-passing events ALSO sent to the teacher — NOT a quality
 # lever, this is the monitoring + active-learning sample (drift detection and
 # fresh teacher-gold on live traffic). Default conservative-ish 5%.
@@ -2662,7 +2684,11 @@ async def _process_cascade(
     # STUDENT_CONCURRENT_CALLS, not CONCURRENT_LLM_CALLS: the student is a cheap
     # single-event 4B call that batches far higher than the teacher's 27B
     # batch-calls (see the config note). Reusing the teacher knob starved it.
-    sem = asyncio.Semaphore(STUDENT_CONCURRENT_CALLS)
+    # SHARED across all in-flight batches (see MAX_INFLIGHT_BATCHES) so the
+    # student pool stays continuously full — a per-batch semaphore burst-then-
+    # idled the GPU. amain initialises it before any batch runs.
+    sem = _STUDENT_SEM
+    assert sem is not None, "amain must initialise _STUDENT_SEM before processing"
 
     async def _student(item):
         async with sem:
@@ -2719,8 +2745,12 @@ async def _run_teacher(
         chunks.append((chunk_items, chunk_events))
 
     # Bound parallelism with a semaphore — gather only waits for slots,
-    # not for everything to be ready at once.
-    sem = asyncio.Semaphore(CONCURRENT_LLM_CALLS)
+    # not for everything to be ready at once. SHARED across all in-flight
+    # batches so the teacher fleet (behind the LB) sees a continuous stream and
+    # its total concurrency stays capped at CONCURRENT_LLM_CALLS regardless of
+    # how many batches are escalating at once. amain initialises it.
+    sem = _TEACHER_SEM
+    assert sem is not None, "amain must initialise _TEACHER_SEM before processing"
 
     async def run_one(chunk_items, chunk_events):
         async with sem:
@@ -2917,20 +2947,65 @@ async def amain():
     if stub_mode:
         log.warning("LLM_ENDPOINT not set — running in stub mode (no extraction).")
 
+    # Shared concurrency pools — created HERE, inside the running event loop, so
+    # ONE student pool and ONE teacher pool are shared by every in-flight batch
+    # (see MAX_INFLIGHT_BATCHES). Per-batch pools burst the GPU to Running:~50
+    # then idled it at Running:1 through each batch's straggler/teacher tail; one
+    # shared pool stays continuously full across overlapping batches.
+    global _STUDENT_SEM, _TEACHER_SEM
+    _STUDENT_SEM = asyncio.Semaphore(STUDENT_CONCURRENT_CALLS)
+    _TEACHER_SEM = asyncio.Semaphore(CONCURRENT_LLM_CALLS)
+    log.info(
+        f"pipelined loop: max_inflight_batches={MAX_INFLIGHT_BATCHES}, "
+        f"student_pool={STUDENT_CONCURRENT_CALLS}, teacher_pool={CONCURRENT_LLM_CALLS}"
+    )
+
     # Single async client across the loop. The httpx default limits
-    # (max_connections=100, max_keepalive=20) easily cover BATCH_SIZE
+    # (max_connections=100, max_keepalive=20) easily cover the shared-pool
     # concurrent inflight LLM calls.
     async with httpx.AsyncClient(timeout=LLM_TIMEOUT_SEC) as http:
+
+        async def _run_one_batch() -> int:
+            """Claim + fully process one batch on a DEDICATED connection.
+            Returns the number of items claimed (0 = queue empty). Applies within
+            a batch happen sequentially (after each gather), so a batch's
+            connection is never touched concurrently — one conn per batch is
+            safe. The `with` closes the conn on both success and exception; a
+            crash leaves that batch's items claimed until CLAIM_TTL_SEC expires
+            and they are reclaimed (unchanged from the prior loop)."""
+            with psycopg.connect(PG_DSN, autocommit=True) as conn:
+                items = claim_next_batch(conn)
+                if not items:
+                    return 0
+                log.info(f"claimed {len(items)} item(s)")
+                await process_batch(http, conn, items, stub_mode)
+                return len(items)
+
+        # Keep up to MAX_INFLIGHT_BATCHES independent claim→apply pipelines
+        # running at once. Their student/teacher calls all draw from the shared
+        # pools above, so while one batch drains its slow tail the others keep
+        # the GPUs fed. Total GPU concurrency is capped by the sems, NOT by this
+        # count.
+        inflight: set[asyncio.Task] = set()
         while True:
             try:
-                with psycopg.connect(PG_DSN, autocommit=True) as conn:
-                    items = claim_next_batch(conn)
-                    if not items:
-                        await asyncio.sleep(POLL_INTERVAL_SEC)
-                        continue
-
-                    log.info(f"claimed {len(items)} item(s)")
-                    await process_batch(http, conn, items, stub_mode)
+                while len(inflight) < MAX_INFLIGHT_BATCHES:
+                    inflight.add(asyncio.create_task(_run_one_batch()))
+                done, inflight = await asyncio.wait(
+                    inflight, return_when=asyncio.FIRST_COMPLETED
+                )
+                claimed_any = False
+                for t in done:
+                    try:
+                        if t.result() > 0:
+                            claimed_any = True
+                    except Exception as exc:
+                        log.error(f"batch task error: {exc}")
+                # All finished batches were empty claims ⇒ queue drained; back
+                # off before re-claiming so idle workers don't hot-spin. Under a
+                # real backlog every claim returns items and this never sleeps.
+                if not claimed_any:
+                    await asyncio.sleep(POLL_INTERVAL_SEC)
             except Exception as exc:
                 log.error(f"worker loop error: {exc}")
                 await asyncio.sleep(POLL_INTERVAL_SEC * 2)