@pentatonic-ai/ai-agent-sdk 0.10.33 → 0.10.35

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.33";
+var VERSION = "0.10.35";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.33";
+var VERSION = "0.10.35";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.33",
+  "version": "0.10.35",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -507,6 +507,18 @@ ACTIVE_SYSTEM_PROMPT = (
 # guided-JSON traces never mix in a training export).
 SYSTEM_PROMPT_HASH = hashlib.sha256(ACTIVE_SYSTEM_PROMPT.encode()).hexdigest()[:16]
 
+# Sentinel system_prompt_hash for STUDENT traces. The student runs with NO
+# system prompt, so it has no natural hash — and crucially, every trace CONSUMER
+# keys off the teacher's hash: build_retrain_corpus.py filters
+# `system_prompt_hash = <clean teacher hash>` and redistill.py treats a trace at
+# that hash as "already re-distilled clean". Tagging student traces with a
+# distinct sentinel (never a 16-hex teacher hash) keeps them out of BOTH — so
+# student output can never be mistaken for training gold or a clean re-distill.
+# `llm_model` (= STUDENT_MODEL) also distinguishes them; the sentinel is the
+# belt to that suspenders. These rows exist purely as the student side of the
+# student-vs-teacher agreement corpus.
+STUDENT_TRACE_PROMPT_HASH = f"student:{STUDENT_MODEL}"
+
 
 # --------------------------------------------------------------------
 # Content cleaner — strip HTML/CSS so email + doc styling never reaches
@@ -2296,6 +2308,8 @@ def _insert_trace(
     user_prompt: str,
     raw_response: str,
     llm_chunk_ms: float | None,
+    llm_model: str = LLM_MODEL,
+    system_prompt_hash: str = SYSTEM_PROMPT_HASH,
 ) -> None:
     """Append a (user_prompt, raw_response) pair to distillation_traces.
 
@@ -2303,7 +2317,12 @@ def _insert_trace(
     caller wraps this in a try/except and never lets a trace-insert
     failure poison the upsert path. Skip rows with empty raw_response
     (no signal to train on); the model occasionally emits a header
-    with no body."""
+    with no body.
+
+    Defaults tag the row as the teacher's training gold. The student path
+    passes llm_model=STUDENT_MODEL + system_prompt_hash=STUDENT_TRACE_PROMPT_HASH
+    so its rows form the agreement corpus WITHOUT being pulled into the retrain
+    corpus / redistill (both key off the teacher hash)."""
     if not raw_response.strip():
         return
     with conn.cursor() as cur:
@@ -2316,7 +2335,7 @@ def _insert_trace(
             """,
             (
                 event_id, user_prompt, raw_response,
-                LLM_MODEL, SYSTEM_PROMPT_HASH, llm_chunk_ms,
+                llm_model, system_prompt_hash, llm_chunk_ms,
             ),
         )
 
@@ -2551,19 +2570,65 @@ def claim_next_batch(conn: psycopg.Connection) -> list[dict[str, Any]]:
               -- increment lived here and ~3 deploys could push a row to
               -- attempts=MAX, making it forever-ineligible for reclaim AND never
               -- marked failed → orphaned in `claimed`. See gotcha #11.)
+            -- FAIR-SHARE CLAIM (migration 012). Was `ORDER BY id` — global
+            -- FIFO, so one tenant's 20k-event backfill head-of-line-blocked
+            -- every other tenant. Now each client is a logical queue ordered
+            -- by (priority, emit_ts) — priority is client-set per event
+            -- (envelope attribute high/medium/low → 10/100/200, enriched onto
+            -- the row by trigger) — and the batch interleaves clients
+            -- round-robin: rank rn=1 of every client, then rn=2, ... so every
+            -- active client gets a proportional slice of each claim.
+            --
+            -- Shape: per-client candidates come from a bounded LATERAL scan on
+            -- idx_distillation_fair_claim (client, priority, emit_ts) — the
+            -- window function never sees more than clients x BATCH_SIZE rows.
+            -- FOR UPDATE cannot sit in the same query level as a window
+            -- function, so the ranked ids are re-locked in an inner
+            -- re-check (status/lease guards repeated) — a concurrently-taken
+            -- row is simply skipped, same semantics as before (a claim may
+            -- return fewer than BATCH_SIZE under contention).
             WHERE id IN (
               SELECT id FROM distillation_queue
-              WHERE (
+              WHERE id = ANY (ARRAY(
+                SELECT cand.id FROM (
+                  SELECT x.id,
+                         ROW_NUMBER() OVER (
+                           PARTITION BY c.client
+                           ORDER BY x.priority, x.emit_ts, x.id
+                         ) AS rn,
+                         x.priority, x.emit_ts
+                  FROM (
+                    SELECT DISTINCT client FROM distillation_queue
+                    WHERE (
+                      status = 'pending'
+                      OR (status = 'claimed' AND claim_expires_at < NOW())
+                    ) AND attempts < %s
+                  ) c
+                  CROSS JOIN LATERAL (
+                    SELECT id, COALESCE(priority, 100) AS priority, emit_ts
+                    FROM distillation_queue
+                    WHERE client IS NOT DISTINCT FROM c.client
+                      AND (
+                        status = 'pending'
+                        OR (status = 'claimed' AND claim_expires_at < NOW())
+                      ) AND attempts < %s
+                    ORDER BY COALESCE(priority, 100), emit_ts, id
+                    LIMIT %s
+                  ) x
+                ) cand
+                ORDER BY cand.rn, cand.priority, cand.emit_ts, cand.id
+                LIMIT %s
+              ))
+              AND (
                 status = 'pending'
                 OR (status = 'claimed' AND claim_expires_at < NOW())
               ) AND attempts < %s
-              ORDER BY id
               FOR UPDATE SKIP LOCKED
-              LIMIT %s
             )
             RETURNING id, event_id, attempts
             """,
-            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, BATCH_SIZE),
+            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, MAX_ATTEMPTS,
+             BATCH_SIZE, BATCH_SIZE, MAX_ATTEMPTS),
         )
         return cur.fetchall()
 
@@ -2748,6 +2813,30 @@ async def _process_cascade(
         if escalate:
             escalate_items.append(item)
             reason_by_qid[item["id"]] = reason or "escalated"
+            # Capture the student's (discarded) output on escalated events —
+            # the ONLY place the teacher ALSO runs, so the ONLY place a
+            # student-vs-teacher diff is possible. This is the agreement corpus
+            # that makes student quality measurable (esp. the random_sample
+            # subset = an unbiased paired sample of student-passing traffic, and
+            # the high_value_class lane the student now carries). Tagged student
+            # (model + sentinel hash) so no consumer mistakes it for teacher
+            # gold. sresult is None only on student_parse_fail (nothing to
+            # trace). Best-effort — never blocks escalation.
+            if DISTILL_TRACE_ENABLED and event is not None and sresult is not None:
+                try:
+                    _insert_trace(
+                        conn,
+                        event_id=item["event_id"],  # same key the teacher trace uses → rows pair
+                        user_prompt=build_event_block(0, event),
+                        raw_response=sresult.get("raw_slice", ""),
+                        llm_chunk_ms=None,
+                        llm_model=STUDENT_MODEL,
+                        system_prompt_hash=STUDENT_TRACE_PROMPT_HASH,
+                    )
+                except Exception as trace_exc:
+                    log.warning(
+                        f"student trace insert failed event_id={event.get('id')}: {trace_exc}"
+                    )
         else:
             _apply_extraction(
                 conn, item=item, event=event, result=sresult, llm_ms=0.0,

package/packages/memory-engine-v2/org-model/migrations/012_queue_priority_fair_share.sql

@@ -0,0 +1,88 @@
+-- 012_queue_priority_fair_share.sql — client-set priority + per-client fair
+-- usage for the distillation queue.
+--
+-- The claim was global FIFO (ORDER BY id): one tenant's 20k-event Drive dump
+-- head-of-line-blocked every other tenant — at 30-40-user onboarding scale the
+-- last user's memory would go live only after everyone ahead of them drained.
+--
+-- Model (decided 2026-07-09): every client effectively gets their OWN queue,
+-- drained round-robin across clients; within a client, items order by
+-- (priority, emit timestamp). Clients set priority per event at emit via the
+-- envelope attribute `priority: "high" | "medium" | "low"` (default medium).
+-- Implemented logically in the one queue table (PARTITION BY client in the
+-- claim) rather than physical per-client tables — identical semantics, none of
+-- the per-tenant DDL/monitoring sprawl; PG LIST-partitioning by client remains
+-- a drop-in physical upgrade under the same claim SQL if ever needed.
+--
+-- Lower priority value = drained first: high=10, medium=100, low=200 (gaps
+-- leave room for future classes without re-mapping).
+--
+-- The queue row is enriched by a BEFORE INSERT trigger (client = arena prefix,
+-- emit_ts = events.emitted_at, priority = mapped envelope attribute unless the
+-- writer set one explicitly). A trigger — not writer-side code — so EVERY
+-- writer (extractor-sync enqueue, the redistill gardener, backfill scripts)
+-- gets consistent enrichment with no emitter/TES changes.
+--
+-- On an already-initialised prod DB (migrations only auto-run on a fresh
+-- volume): apply the ALTERs + trigger + backfill via psql, and build the index
+-- with CREATE INDEX CONCURRENTLY.
+
+ALTER TABLE distillation_queue
+  ADD COLUMN IF NOT EXISTS client   TEXT,
+  ADD COLUMN IF NOT EXISTS priority SMALLINT,
+  ADD COLUMN IF NOT EXISTS emit_ts  TIMESTAMPTZ;
+
+CREATE OR REPLACE FUNCTION distillation_queue_enrich() RETURNS trigger AS $$
+DECLARE
+  ev RECORD;
+BEGIN
+  IF NEW.client IS NULL OR NEW.emit_ts IS NULL OR NEW.priority IS NULL THEN
+    SELECT split_part(e.arena, ':', 1)              AS client,
+           e.emitted_at                             AS emitted_at,
+           lower(e.attributes->>'priority')         AS prio
+      INTO ev
+      FROM events e WHERE e.id = NEW.event_id;
+    IF FOUND THEN
+      NEW.client  := COALESCE(NEW.client, ev.client);
+      NEW.emit_ts := COALESCE(NEW.emit_ts, ev.emitted_at);
+      -- An explicit writer-set priority wins (e.g. the redistill gardener
+      -- self-demoting to low); otherwise the client's envelope attribute;
+      -- otherwise medium.
+      NEW.priority := COALESCE(
+        NEW.priority,
+        CASE ev.prio WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END
+      );
+    ELSE
+      -- Orphan enqueue (no event row): defaults keep the row claimable.
+      NEW.client   := COALESCE(NEW.client, 'unknown');
+      NEW.emit_ts  := COALESCE(NEW.emit_ts, NOW());
+      NEW.priority := COALESCE(NEW.priority, 100);
+    END IF;
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_distillation_queue_enrich ON distillation_queue;
+CREATE TRIGGER trg_distillation_queue_enrich
+  BEFORE INSERT ON distillation_queue
+  FOR EACH ROW EXECUTE FUNCTION distillation_queue_enrich();
+
+-- Backfill live rows (pending/claimed). done/failed rows are history — the
+-- daily prune retires them and the claim never reads them.
+UPDATE distillation_queue dq
+   SET client   = COALESCE(dq.client,  split_part(e.arena, ':', 1)),
+       emit_ts  = COALESCE(dq.emit_ts, e.emitted_at),
+       priority = COALESCE(dq.priority,
+                    CASE lower(e.attributes->>'priority')
+                      WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END)
+  FROM events e
+ WHERE dq.event_id = e.id
+   AND dq.status IN ('pending', 'claimed')
+   AND (dq.client IS NULL OR dq.emit_ts IS NULL OR dq.priority IS NULL);
+
+-- Serves both halves of the fair-share claim: the DISTINCT-client scan and
+-- each client's (priority, emit_ts)-ordered candidate scan.
+CREATE INDEX IF NOT EXISTS idx_distillation_fair_claim
+  ON distillation_queue (client, priority, emit_ts)
+  WHERE status = 'pending';

package/packages/memory-engine-v2/scripts/redistill.py

@@ -212,8 +212,12 @@ def main() -> int:
                     if args.supersede_facts:
                         deleted += dump_and_delete_solo_facts(cur, eid, rb)
                     cur.execute(
-                        "INSERT INTO distillation_queue (event_id, status, attempts) "
-                        "VALUES (%s, 'pending', 0)", (eid,))
+                        # priority=200 (low): re-distills are maintenance — they
+                        # must never compete with live client traffic. Explicit
+                        # value wins over the enrichment trigger's mapping
+                        # (migration 012).
+                        "INSERT INTO distillation_queue (event_id, status, attempts, priority) "
+                        "VALUES (%s, 'pending', 0, 200)", (eid,))
                     enq += 1
                     led.write(json.dumps({
                         "event_id": eid, "arena": args.arena,