@pentatonic-ai/ai-agent-sdk 0.10.28 → 0.10.30

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.28";
+var VERSION = "0.10.30";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.28";
+var VERSION = "0.10.30";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.28",
+  "version": "0.10.30",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -131,6 +131,28 @@ STUDENT_MAX_TOKENS = int(os.environ.get("STUDENT_MAX_TOKENS", "768"))
 # the box's KV-cache headroom. Do NOT raise CONCURRENT_LLM_CALLS to fix student
 # throughput — that also raises teacher 27B concurrency (OOM risk).
 STUDENT_CONCURRENT_CALLS = int(os.environ.get("STUDENT_CONCURRENT_CALLS", "48"))
+# How many claimed batches the worker processes CONCURRENTLY. The prior loop
+# claimed a batch, awaited it end-to-end (student gather → straggler tail →
+# teacher tail), and only then claimed the next — a hard barrier. Wall-clock
+# profiling of the 25k-backlog incident (2026-07-08) showed the student L40S
+# burst to Running:50 for ~10s then sit at Running:1 for ~3min draining a
+# straggler tail while the (LB'd) teacher fleet idled — the GPUs were starved by
+# the loop, not by capacity. Overlapping N batches, all sharing ONE student pool
+# and ONE teacher pool (the shared semaphores below), keeps those pools
+# continuously full: batch B's fresh student calls take the slots batch A's
+# stragglers free. Total LLM concurrency is still bounded by the shared sems
+# (STUDENT_CONCURRENT_CALLS / CONCURRENT_LLM_CALLS) REGARDLESS of this value —
+# this only controls how many independent claim→apply pipelines feed them (each
+# on its own PG connection; applies within a batch stay sequential, so one conn
+# per batch is never touched concurrently). Raise to feed the pools harder; the
+# sems, not this, cap GPU load.
+MAX_INFLIGHT_BATCHES = int(os.environ.get("MAX_INFLIGHT_BATCHES", "4"))
+# Process-global shared concurrency pools, created once inside amain's event
+# loop (NOT per-batch — per-batch pools are what caused the burst-then-idle).
+# All in-flight batches acquire from these, so the student/teacher GPUs see one
+# continuous stream of calls instead of per-batch bursts. None until amain runs.
+_STUDENT_SEM: "asyncio.Semaphore | None" = None
+_TEACHER_SEM: "asyncio.Semaphore | None" = None
 # Fraction of student-passing events ALSO sent to the teacher — NOT a quality
 # lever, this is the monitoring + active-learning sample (drift detection and
 # fresh teacher-gold on live traffic). Default conservative-ish 5%.
@@ -2662,7 +2684,11 @@ async def _process_cascade(
     # STUDENT_CONCURRENT_CALLS, not CONCURRENT_LLM_CALLS: the student is a cheap
     # single-event 4B call that batches far higher than the teacher's 27B
     # batch-calls (see the config note). Reusing the teacher knob starved it.
-    sem = asyncio.Semaphore(STUDENT_CONCURRENT_CALLS)
+    # SHARED across all in-flight batches (see MAX_INFLIGHT_BATCHES) so the
+    # student pool stays continuously full — a per-batch semaphore burst-then-
+    # idled the GPU. amain initialises it before any batch runs.
+    sem = _STUDENT_SEM
+    assert sem is not None, "amain must initialise _STUDENT_SEM before processing"
 
     async def _student(item):
         async with sem:
@@ -2719,8 +2745,12 @@ async def _run_teacher(
         chunks.append((chunk_items, chunk_events))
 
     # Bound parallelism with a semaphore — gather only waits for slots,
-    # not for everything to be ready at once.
-    sem = asyncio.Semaphore(CONCURRENT_LLM_CALLS)
+    # not for everything to be ready at once. SHARED across all in-flight
+    # batches so the teacher fleet (behind the LB) sees a continuous stream and
+    # its total concurrency stays capped at CONCURRENT_LLM_CALLS regardless of
+    # how many batches are escalating at once. amain initialises it.
+    sem = _TEACHER_SEM
+    assert sem is not None, "amain must initialise _TEACHER_SEM before processing"
 
     async def run_one(chunk_items, chunk_events):
         async with sem:
@@ -2805,70 +2835,91 @@ def _apply_extraction(
     det_ents, det_rels = structured_graph_elements(arena, event.get("attributes"))
 
     try:
-        name_to_id = upsert_entities(
-            conn, arena, event_id, participant_set, disclosure, ents + det_ents,
-            event_time, event.get("attributes"),
-        )
-        n_facts = upsert_facts(
-            conn, arena, event_id, participant_set, disclosure, facts, name_to_id,
-            event_time, due_at, src,
-        )
-        n_rels = upsert_relationships(
-            conn, arena, event_id, participant_set, disclosure, rels + det_rels, name_to_id,
-            event_time,
-        )
-        mark_done(conn, queue_id)
-        log.info(
-            f"completed queue_id={queue_id} event_id={event_id} producer={producer} "
-            f"entities={len(name_to_id)} facts={n_facts} relationships={n_rels}"
-            f" det_edges={len(det_rels)}"
-            + (f" llm_ms={llm_ms:.0f}" if not stub_mode and llm_ms else "")
-        )
-        # Trace logging — best-effort. ONLY the teacher produces training gold;
-        # logging student output would train the student on itself.
-        if DISTILL_TRACE_ENABLED and not stub_mode and producer == "teacher":
-            try:
-                _insert_trace(
-                    conn,
-                    event_id=event_id,
-                    user_prompt=build_event_block(local_idx, event),
-                    raw_response=result.get("raw_slice", ""),
-                    llm_chunk_ms=llm_ms,
-                )
-            except Exception as trace_exc:
-                log.warning(
-                    f"trace insert failed queue_id={queue_id} "
-                    f"event_id={event_id}: {trace_exc}"
-                )
-        # Cascade ledger — best-effort, records which producer wrote this
-        # event's rows (only when the cascade is on).
-        if CASCADE_ENABLED:
-            try:
-                _record_distillation(
-                    conn,
-                    event_id=event_id,
-                    producer=producer,
-                    llm_model=LLM_MODEL if producer == "teacher" else STUDENT_MODEL,
-                    escalated=escalated,
-                    escalate_reason=escalate_reason,
-                )
-            except Exception as ledger_exc:
-                log.warning(
-                    f"ledger insert failed queue_id={queue_id} "
-                    f"event_id={event_id}: {ledger_exc}"
-                )
-        return True
+        # ONE transaction per event. The connection is autocommit (amain), so
+        # without this each statement fsync-commits on its own — ~20 commits per
+        # event (entity resolution SELECTs/locks/upserts + facts + rels +
+        # mark_done), which is the round-trip bottleneck under a backlog: the
+        # graph PG sits idle between the worker's serial commits (profiled
+        # 2026-07-08 — 39 conns idle, 1 active). Wrapping commits once per event.
+        #
+        # NB: this also makes upsert_entities' `pg_advisory_xact_lock` actually
+        # span the event's resolution. Under autocommit each lock released after
+        # its own single-statement txn — effectively a NO-OP — so the
+        # per-surface-form anti-fragmentation lock never protected the
+        # SELECT-overlap→INSERT window. Now it holds for the whole event: correct
+        # (the lock finally does its job), but it serialises concurrent workers
+        # that touch the same surface form. Load-test before trusting on prod.
+        with conn.transaction():
+            name_to_id = upsert_entities(
+                conn, arena, event_id, participant_set, disclosure, ents + det_ents,
+                event_time, event.get("attributes"),
+            )
+            n_facts = upsert_facts(
+                conn, arena, event_id, participant_set, disclosure, facts, name_to_id,
+                event_time, due_at, src,
+            )
+            n_rels = upsert_relationships(
+                conn, arena, event_id, participant_set, disclosure, rels + det_rels, name_to_id,
+                event_time,
+            )
+            mark_done(conn, queue_id)
     except Exception as exc:
         err = f"{type(exc).__name__}: {exc}"
         log.warning(
             f"db upsert failed queue_id={queue_id} attempts={attempts}: {err}"
         )
+        # The transaction rolled back; the claim bookkeeping runs on the
+        # now-autocommit connection so it persists independently.
         if _attempts_exhausted(attempts):
             mark_failed(conn, queue_id, err)
         else:
             release_claim(conn, queue_id, err)
         return False
 
+    log.info(
+        f"completed queue_id={queue_id} event_id={event_id} producer={producer} "
+        f"entities={len(name_to_id)} facts={n_facts} relationships={n_rels}"
+        f" det_edges={len(det_rels)}"
+        + (f" llm_ms={llm_ms:.0f}" if not stub_mode and llm_ms else "")
+    )
+    # Best-effort side-writes, AFTER the graph commit and OUTSIDE the transaction:
+    # a failure here must not roll back the committed graph rows, and an error
+    # inside the txn would poison it (a swallowed error leaves the txn aborted).
+    # Trace: ONLY the teacher produces training gold; logging student output would
+    # train the student on itself.
+    if DISTILL_TRACE_ENABLED and not stub_mode and producer == "teacher":
+        try:
+            _insert_trace(
+                conn,
+                event_id=event_id,
+                user_prompt=build_event_block(local_idx, event),
+                raw_response=result.get("raw_slice", ""),
+                llm_chunk_ms=llm_ms,
+            )
+        except Exception as trace_exc:
+            log.warning(
+                f"trace insert failed queue_id={queue_id} "
+                f"event_id={event_id}: {trace_exc}"
+            )
+    # Cascade ledger — best-effort, records which producer wrote this
+    # event's rows (only when the cascade is on).
+    if CASCADE_ENABLED:
+        try:
+            _record_distillation(
+                conn,
+                event_id=event_id,
+                producer=producer,
+                llm_model=LLM_MODEL if producer == "teacher" else STUDENT_MODEL,
+                escalated=escalated,
+                escalate_reason=escalate_reason,
+            )
+        except Exception as ledger_exc:
+            log.warning(
+                f"ledger insert failed queue_id={queue_id} "
+                f"event_id={event_id}: {ledger_exc}"
+            )
+    return True
+
 
 async def amain():
     log.info(
@@ -2896,20 +2947,65 @@ async def amain():
     if stub_mode:
         log.warning("LLM_ENDPOINT not set — running in stub mode (no extraction).")
 
+    # Shared concurrency pools — created HERE, inside the running event loop, so
+    # ONE student pool and ONE teacher pool are shared by every in-flight batch
+    # (see MAX_INFLIGHT_BATCHES). Per-batch pools burst the GPU to Running:~50
+    # then idled it at Running:1 through each batch's straggler/teacher tail; one
+    # shared pool stays continuously full across overlapping batches.
+    global _STUDENT_SEM, _TEACHER_SEM
+    _STUDENT_SEM = asyncio.Semaphore(STUDENT_CONCURRENT_CALLS)
+    _TEACHER_SEM = asyncio.Semaphore(CONCURRENT_LLM_CALLS)
+    log.info(
+        f"pipelined loop: max_inflight_batches={MAX_INFLIGHT_BATCHES}, "
+        f"student_pool={STUDENT_CONCURRENT_CALLS}, teacher_pool={CONCURRENT_LLM_CALLS}"
+    )
+
     # Single async client across the loop. The httpx default limits
-    # (max_connections=100, max_keepalive=20) easily cover BATCH_SIZE
+    # (max_connections=100, max_keepalive=20) easily cover the shared-pool
     # concurrent inflight LLM calls.
     async with httpx.AsyncClient(timeout=LLM_TIMEOUT_SEC) as http:
+
+        async def _run_one_batch() -> int:
+            """Claim + fully process one batch on a DEDICATED connection.
+            Returns the number of items claimed (0 = queue empty). Applies within
+            a batch happen sequentially (after each gather), so a batch's
+            connection is never touched concurrently — one conn per batch is
+            safe. The `with` closes the conn on both success and exception; a
+            crash leaves that batch's items claimed until CLAIM_TTL_SEC expires
+            and they are reclaimed (unchanged from the prior loop)."""
+            with psycopg.connect(PG_DSN, autocommit=True) as conn:
+                items = claim_next_batch(conn)
+                if not items:
+                    return 0
+                log.info(f"claimed {len(items)} item(s)")
+                await process_batch(http, conn, items, stub_mode)
+                return len(items)
+
+        # Keep up to MAX_INFLIGHT_BATCHES independent claim→apply pipelines
+        # running at once. Their student/teacher calls all draw from the shared
+        # pools above, so while one batch drains its slow tail the others keep
+        # the GPUs fed. Total GPU concurrency is capped by the sems, NOT by this
+        # count.
+        inflight: set[asyncio.Task] = set()
         while True:
             try:
-                with psycopg.connect(PG_DSN, autocommit=True) as conn:
-                    items = claim_next_batch(conn)
-                    if not items:
-                        await asyncio.sleep(POLL_INTERVAL_SEC)
-                        continue
-
-                    log.info(f"claimed {len(items)} item(s)")
-                    await process_batch(http, conn, items, stub_mode)
+                while len(inflight) < MAX_INFLIGHT_BATCHES:
+                    inflight.add(asyncio.create_task(_run_one_batch()))
+                done, inflight = await asyncio.wait(
+                    inflight, return_when=asyncio.FIRST_COMPLETED
+                )
+                claimed_any = False
+                for t in done:
+                    try:
+                        if t.result() > 0:
+                            claimed_any = True
+                    except Exception as exc:
+                        log.error(f"batch task error: {exc}")
+                # All finished batches were empty claims ⇒ queue drained; back
+                # off before re-claiming so idle workers don't hot-spin. Under a
+                # real backlog every claim returns items and this never sleeps.
+                if not claimed_any:
+                    await asyncio.sleep(POLL_INTERVAL_SEC)
             except Exception as exc:
                 log.error(f"worker loop error: {exc}")
                 await asyncio.sleep(POLL_INTERVAL_SEC * 2)