@pentatonic-ai/ai-agent-sdk 0.10.32 → 0.10.34

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.32";
+var VERSION = "0.10.34";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.32";
+var VERSION = "0.10.34";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.32",
+  "version": "0.10.34",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -193,13 +193,24 @@ _PROV_REL = _cap_prov("relationships.provenance_event_ids", "EXCLUDED.provenance
 # lever, this is the monitoring + active-learning sample (drift detection and
 # fresh teacher-gold on live traffic). Default conservative-ish 5%.
 STUDENT_SAMPLE_RATE = float(os.environ.get("STUDENT_SAMPLE_RATE", "0.05"))
-# Fact categories that always go to the teacher regardless of the student's
-# output (high-value, cheap to over-escalate). Comma-separated, lowercased.
+# Fact categories that go to the teacher regardless of the student's output
+# (high-value, cheap to over-escalate). Comma-separated, lowercased.
 HIGH_VALUE_CATEGORIES = {
     c.strip().lower()
     for c in os.environ.get("HIGH_VALUE_CATEGORIES", "decision,commitment").split(",")
     if c.strip()
 }
+# Fraction of high-value-class events that ESCALATE to the teacher. 1.0 (default)
+# = the original always-teacher gate. The gate was set at cascade launch when the
+# student was unproven; since then the clean student has zero quality-fail
+# escalations in prod and shadow-eval parity with the teacher — while the HV gate
+# routed ~57% of a decision/commitment-heavy backlog to the teacher (90% of all
+# escalations, 2026-07-09), pinning drain to teacher capacity with the student
+# idle. Lowering this (e.g. 0.25) makes HV student-primary with an elevated
+# teacher sample as the safety net/monitoring corpus on exactly the high-value
+# class. Instant rollback: set back to 1.0. Escalated-anyway HV events keep the
+# 'high_value_class' ledger reason, so the mix stays queryable.
+HIGH_VALUE_SAMPLE_RATE = float(os.environ.get("HIGH_VALUE_SAMPLE_RATE", "1.0"))
 
 
 # KV-text output format constants. We dropped JSON output (and the
@@ -1078,10 +1089,14 @@ def escalation_decision(
     for em in {e.lower() for e in _EMAIL_RE.findall(blob)}:
         if em not in known:
             return True, "grounding_violation"
-    # 2. High-value fact class — always teacher.
+    # 2. High-value fact class — teacher at HIGH_VALUE_SAMPLE_RATE (1.0 = the
+    # original always-teacher gate; lower = student-primary with an elevated
+    # teacher sample on exactly this class — see the config note).
     for f in student_result.get("facts", []):
         if (f.get("category") or "").lower() in HIGH_VALUE_CATEGORIES:
-            return True, "high_value_class"
+            if HIGH_VALUE_SAMPLE_RATE >= 1.0 or random.random() < HIGH_VALUE_SAMPLE_RATE:
+                return True, "high_value_class"
+            break  # HV but sampled-out → student writes; still eligible for gate 3
     # 3. Random monitoring/active-learning sample (not a quality lever).
     if STUDENT_SAMPLE_RATE > 0 and random.random() < STUDENT_SAMPLE_RATE:
         return True, "random_sample"
@@ -2536,19 +2551,65 @@ def claim_next_batch(conn: psycopg.Connection) -> list[dict[str, Any]]:
               -- increment lived here and ~3 deploys could push a row to
               -- attempts=MAX, making it forever-ineligible for reclaim AND never
               -- marked failed → orphaned in `claimed`. See gotcha #11.)
+            -- FAIR-SHARE CLAIM (migration 012). Was `ORDER BY id` — global
+            -- FIFO, so one tenant's 20k-event backfill head-of-line-blocked
+            -- every other tenant. Now each client is a logical queue ordered
+            -- by (priority, emit_ts) — priority is client-set per event
+            -- (envelope attribute high/medium/low → 10/100/200, enriched onto
+            -- the row by trigger) — and the batch interleaves clients
+            -- round-robin: rank rn=1 of every client, then rn=2, ... so every
+            -- active client gets a proportional slice of each claim.
+            --
+            -- Shape: per-client candidates come from a bounded LATERAL scan on
+            -- idx_distillation_fair_claim (client, priority, emit_ts) — the
+            -- window function never sees more than clients x BATCH_SIZE rows.
+            -- FOR UPDATE cannot sit in the same query level as a window
+            -- function, so the ranked ids are re-locked in an inner
+            -- re-check (status/lease guards repeated) — a concurrently-taken
+            -- row is simply skipped, same semantics as before (a claim may
+            -- return fewer than BATCH_SIZE under contention).
             WHERE id IN (
               SELECT id FROM distillation_queue
-              WHERE (
+              WHERE id = ANY (ARRAY(
+                SELECT cand.id FROM (
+                  SELECT x.id,
+                         ROW_NUMBER() OVER (
+                           PARTITION BY c.client
+                           ORDER BY x.priority, x.emit_ts, x.id
+                         ) AS rn,
+                         x.priority, x.emit_ts
+                  FROM (
+                    SELECT DISTINCT client FROM distillation_queue
+                    WHERE (
+                      status = 'pending'
+                      OR (status = 'claimed' AND claim_expires_at < NOW())
+                    ) AND attempts < %s
+                  ) c
+                  CROSS JOIN LATERAL (
+                    SELECT id, COALESCE(priority, 100) AS priority, emit_ts
+                    FROM distillation_queue
+                    WHERE client IS NOT DISTINCT FROM c.client
+                      AND (
+                        status = 'pending'
+                        OR (status = 'claimed' AND claim_expires_at < NOW())
+                      ) AND attempts < %s
+                    ORDER BY COALESCE(priority, 100), emit_ts, id
+                    LIMIT %s
+                  ) x
+                ) cand
+                ORDER BY cand.rn, cand.priority, cand.emit_ts, cand.id
+                LIMIT %s
+              ))
+              AND (
                 status = 'pending'
                 OR (status = 'claimed' AND claim_expires_at < NOW())
               ) AND attempts < %s
-              ORDER BY id
               FOR UPDATE SKIP LOCKED
-              LIMIT %s
             )
             RETURNING id, event_id, attempts
             """,
-            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, BATCH_SIZE),
+            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, MAX_ATTEMPTS,
+             BATCH_SIZE, BATCH_SIZE, MAX_ATTEMPTS),
         )
         return cur.fetchall()
 
@@ -2964,7 +3025,8 @@ async def amain():
             f"cascade ENABLED — student-primary "
             f"(model={STUDENT_MODEL}, endpoint={STUDENT_ENDPOINT or '(unset!)'}, "
             f"sample_rate={STUDENT_SAMPLE_RATE}, "
-            f"high_value={sorted(HIGH_VALUE_CATEGORIES)})"
+            f"high_value={sorted(HIGH_VALUE_CATEGORIES)}, "
+            f"hv_sample_rate={HIGH_VALUE_SAMPLE_RATE})"
         )
         if not STUDENT_ENDPOINT:
             log.warning(

package/packages/memory-engine-v2/org-model/migrations/012_queue_priority_fair_share.sql

@@ -0,0 +1,88 @@
+-- 012_queue_priority_fair_share.sql — client-set priority + per-client fair
+-- usage for the distillation queue.
+--
+-- The claim was global FIFO (ORDER BY id): one tenant's 20k-event Drive dump
+-- head-of-line-blocked every other tenant — at 30-40-user onboarding scale the
+-- last user's memory would go live only after everyone ahead of them drained.
+--
+-- Model (decided 2026-07-09): every client effectively gets their OWN queue,
+-- drained round-robin across clients; within a client, items order by
+-- (priority, emit timestamp). Clients set priority per event at emit via the
+-- envelope attribute `priority: "high" | "medium" | "low"` (default medium).
+-- Implemented logically in the one queue table (PARTITION BY client in the
+-- claim) rather than physical per-client tables — identical semantics, none of
+-- the per-tenant DDL/monitoring sprawl; PG LIST-partitioning by client remains
+-- a drop-in physical upgrade under the same claim SQL if ever needed.
+--
+-- Lower priority value = drained first: high=10, medium=100, low=200 (gaps
+-- leave room for future classes without re-mapping).
+--
+-- The queue row is enriched by a BEFORE INSERT trigger (client = arena prefix,
+-- emit_ts = events.emitted_at, priority = mapped envelope attribute unless the
+-- writer set one explicitly). A trigger — not writer-side code — so EVERY
+-- writer (extractor-sync enqueue, the redistill gardener, backfill scripts)
+-- gets consistent enrichment with no emitter/TES changes.
+--
+-- On an already-initialised prod DB (migrations only auto-run on a fresh
+-- volume): apply the ALTERs + trigger + backfill via psql, and build the index
+-- with CREATE INDEX CONCURRENTLY.
+
+ALTER TABLE distillation_queue
+  ADD COLUMN IF NOT EXISTS client   TEXT,
+  ADD COLUMN IF NOT EXISTS priority SMALLINT,
+  ADD COLUMN IF NOT EXISTS emit_ts  TIMESTAMPTZ;
+
+CREATE OR REPLACE FUNCTION distillation_queue_enrich() RETURNS trigger AS $$
+DECLARE
+  ev RECORD;
+BEGIN
+  IF NEW.client IS NULL OR NEW.emit_ts IS NULL OR NEW.priority IS NULL THEN
+    SELECT split_part(e.arena, ':', 1)              AS client,
+           e.emitted_at                             AS emitted_at,
+           lower(e.attributes->>'priority')         AS prio
+      INTO ev
+      FROM events e WHERE e.id = NEW.event_id;
+    IF FOUND THEN
+      NEW.client  := COALESCE(NEW.client, ev.client);
+      NEW.emit_ts := COALESCE(NEW.emit_ts, ev.emitted_at);
+      -- An explicit writer-set priority wins (e.g. the redistill gardener
+      -- self-demoting to low); otherwise the client's envelope attribute;
+      -- otherwise medium.
+      NEW.priority := COALESCE(
+        NEW.priority,
+        CASE ev.prio WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END
+      );
+    ELSE
+      -- Orphan enqueue (no event row): defaults keep the row claimable.
+      NEW.client   := COALESCE(NEW.client, 'unknown');
+      NEW.emit_ts  := COALESCE(NEW.emit_ts, NOW());
+      NEW.priority := COALESCE(NEW.priority, 100);
+    END IF;
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_distillation_queue_enrich ON distillation_queue;
+CREATE TRIGGER trg_distillation_queue_enrich
+  BEFORE INSERT ON distillation_queue
+  FOR EACH ROW EXECUTE FUNCTION distillation_queue_enrich();
+
+-- Backfill live rows (pending/claimed). done/failed rows are history — the
+-- daily prune retires them and the claim never reads them.
+UPDATE distillation_queue dq
+   SET client   = COALESCE(dq.client,  split_part(e.arena, ':', 1)),
+       emit_ts  = COALESCE(dq.emit_ts, e.emitted_at),
+       priority = COALESCE(dq.priority,
+                    CASE lower(e.attributes->>'priority')
+                      WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END)
+  FROM events e
+ WHERE dq.event_id = e.id
+   AND dq.status IN ('pending', 'claimed')
+   AND (dq.client IS NULL OR dq.emit_ts IS NULL OR dq.priority IS NULL);
+
+-- Serves both halves of the fair-share claim: the DISTINCT-client scan and
+-- each client's (priority, emit_ts)-ordered candidate scan.
+CREATE INDEX IF NOT EXISTS idx_distillation_fair_claim
+  ON distillation_queue (client, priority, emit_ts)
+  WHERE status = 'pending';

package/packages/memory-engine-v2/scripts/redistill.py

@@ -212,8 +212,12 @@ def main() -> int:
                     if args.supersede_facts:
                         deleted += dump_and_delete_solo_facts(cur, eid, rb)
                     cur.execute(
-                        "INSERT INTO distillation_queue (event_id, status, attempts) "
-                        "VALUES (%s, 'pending', 0)", (eid,))
+                        # priority=200 (low): re-distills are maintenance — they
+                        # must never compete with live client traffic. Explicit
+                        # value wins over the enrichment trigger's mapping
+                        # (migration 012).
+                        "INSERT INTO distillation_queue (event_id, status, attempts, priority) "
+                        "VALUES (%s, 'pending', 0, 200)", (eid,))
                     enq += 1
                     led.write(json.dumps({
                         "event_id": eid, "arena": args.arena,