@pentatonic-ai/ai-agent-sdk 0.10.33 → 0.10.34

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.33";
+var VERSION = "0.10.34";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.33";
+var VERSION = "0.10.34";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.33",
+  "version": "0.10.34",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -2551,19 +2551,65 @@ def claim_next_batch(conn: psycopg.Connection) -> list[dict[str, Any]]:
               -- increment lived here and ~3 deploys could push a row to
               -- attempts=MAX, making it forever-ineligible for reclaim AND never
               -- marked failed → orphaned in `claimed`. See gotcha #11.)
+            -- FAIR-SHARE CLAIM (migration 012). Was `ORDER BY id` — global
+            -- FIFO, so one tenant's 20k-event backfill head-of-line-blocked
+            -- every other tenant. Now each client is a logical queue ordered
+            -- by (priority, emit_ts) — priority is client-set per event
+            -- (envelope attribute high/medium/low → 10/100/200, enriched onto
+            -- the row by trigger) — and the batch interleaves clients
+            -- round-robin: rank rn=1 of every client, then rn=2, ... so every
+            -- active client gets a proportional slice of each claim.
+            --
+            -- Shape: per-client candidates come from a bounded LATERAL scan on
+            -- idx_distillation_fair_claim (client, priority, emit_ts) — the
+            -- window function never sees more than clients x BATCH_SIZE rows.
+            -- FOR UPDATE cannot sit in the same query level as a window
+            -- function, so the ranked ids are re-locked in an inner
+            -- re-check (status/lease guards repeated) — a concurrently-taken
+            -- row is simply skipped, same semantics as before (a claim may
+            -- return fewer than BATCH_SIZE under contention).
             WHERE id IN (
               SELECT id FROM distillation_queue
-              WHERE (
+              WHERE id = ANY (ARRAY(
+                SELECT cand.id FROM (
+                  SELECT x.id,
+                         ROW_NUMBER() OVER (
+                           PARTITION BY c.client
+                           ORDER BY x.priority, x.emit_ts, x.id
+                         ) AS rn,
+                         x.priority, x.emit_ts
+                  FROM (
+                    SELECT DISTINCT client FROM distillation_queue
+                    WHERE (
+                      status = 'pending'
+                      OR (status = 'claimed' AND claim_expires_at < NOW())
+                    ) AND attempts < %s
+                  ) c
+                  CROSS JOIN LATERAL (
+                    SELECT id, COALESCE(priority, 100) AS priority, emit_ts
+                    FROM distillation_queue
+                    WHERE client IS NOT DISTINCT FROM c.client
+                      AND (
+                        status = 'pending'
+                        OR (status = 'claimed' AND claim_expires_at < NOW())
+                      ) AND attempts < %s
+                    ORDER BY COALESCE(priority, 100), emit_ts, id
+                    LIMIT %s
+                  ) x
+                ) cand
+                ORDER BY cand.rn, cand.priority, cand.emit_ts, cand.id
+                LIMIT %s
+              ))
+              AND (
                 status = 'pending'
                 OR (status = 'claimed' AND claim_expires_at < NOW())
               ) AND attempts < %s
-              ORDER BY id
               FOR UPDATE SKIP LOCKED
-              LIMIT %s
             )
             RETURNING id, event_id, attempts
             """,
-            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, BATCH_SIZE),
+            (WORKER_ID, CLAIM_TTL_SEC, MAX_ATTEMPTS, MAX_ATTEMPTS,
+             BATCH_SIZE, BATCH_SIZE, MAX_ATTEMPTS),
         )
         return cur.fetchall()
 

package/packages/memory-engine-v2/org-model/migrations/012_queue_priority_fair_share.sql

@@ -0,0 +1,88 @@
+-- 012_queue_priority_fair_share.sql — client-set priority + per-client fair
+-- usage for the distillation queue.
+--
+-- The claim was global FIFO (ORDER BY id): one tenant's 20k-event Drive dump
+-- head-of-line-blocked every other tenant — at 30-40-user onboarding scale the
+-- last user's memory would go live only after everyone ahead of them drained.
+--
+-- Model (decided 2026-07-09): every client effectively gets their OWN queue,
+-- drained round-robin across clients; within a client, items order by
+-- (priority, emit timestamp). Clients set priority per event at emit via the
+-- envelope attribute `priority: "high" | "medium" | "low"` (default medium).
+-- Implemented logically in the one queue table (PARTITION BY client in the
+-- claim) rather than physical per-client tables — identical semantics, none of
+-- the per-tenant DDL/monitoring sprawl; PG LIST-partitioning by client remains
+-- a drop-in physical upgrade under the same claim SQL if ever needed.
+--
+-- Lower priority value = drained first: high=10, medium=100, low=200 (gaps
+-- leave room for future classes without re-mapping).
+--
+-- The queue row is enriched by a BEFORE INSERT trigger (client = arena prefix,
+-- emit_ts = events.emitted_at, priority = mapped envelope attribute unless the
+-- writer set one explicitly). A trigger — not writer-side code — so EVERY
+-- writer (extractor-sync enqueue, the redistill gardener, backfill scripts)
+-- gets consistent enrichment with no emitter/TES changes.
+--
+-- On an already-initialised prod DB (migrations only auto-run on a fresh
+-- volume): apply the ALTERs + trigger + backfill via psql, and build the index
+-- with CREATE INDEX CONCURRENTLY.
+
+ALTER TABLE distillation_queue
+  ADD COLUMN IF NOT EXISTS client   TEXT,
+  ADD COLUMN IF NOT EXISTS priority SMALLINT,
+  ADD COLUMN IF NOT EXISTS emit_ts  TIMESTAMPTZ;
+
+CREATE OR REPLACE FUNCTION distillation_queue_enrich() RETURNS trigger AS $$
+DECLARE
+  ev RECORD;
+BEGIN
+  IF NEW.client IS NULL OR NEW.emit_ts IS NULL OR NEW.priority IS NULL THEN
+    SELECT split_part(e.arena, ':', 1)              AS client,
+           e.emitted_at                             AS emitted_at,
+           lower(e.attributes->>'priority')         AS prio
+      INTO ev
+      FROM events e WHERE e.id = NEW.event_id;
+    IF FOUND THEN
+      NEW.client  := COALESCE(NEW.client, ev.client);
+      NEW.emit_ts := COALESCE(NEW.emit_ts, ev.emitted_at);
+      -- An explicit writer-set priority wins (e.g. the redistill gardener
+      -- self-demoting to low); otherwise the client's envelope attribute;
+      -- otherwise medium.
+      NEW.priority := COALESCE(
+        NEW.priority,
+        CASE ev.prio WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END
+      );
+    ELSE
+      -- Orphan enqueue (no event row): defaults keep the row claimable.
+      NEW.client   := COALESCE(NEW.client, 'unknown');
+      NEW.emit_ts  := COALESCE(NEW.emit_ts, NOW());
+      NEW.priority := COALESCE(NEW.priority, 100);
+    END IF;
+  END IF;
+  RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+DROP TRIGGER IF EXISTS trg_distillation_queue_enrich ON distillation_queue;
+CREATE TRIGGER trg_distillation_queue_enrich
+  BEFORE INSERT ON distillation_queue
+  FOR EACH ROW EXECUTE FUNCTION distillation_queue_enrich();
+
+-- Backfill live rows (pending/claimed). done/failed rows are history — the
+-- daily prune retires them and the claim never reads them.
+UPDATE distillation_queue dq
+   SET client   = COALESCE(dq.client,  split_part(e.arena, ':', 1)),
+       emit_ts  = COALESCE(dq.emit_ts, e.emitted_at),
+       priority = COALESCE(dq.priority,
+                    CASE lower(e.attributes->>'priority')
+                      WHEN 'high' THEN 10 WHEN 'medium' THEN 100 WHEN 'low' THEN 200 ELSE 100 END)
+  FROM events e
+ WHERE dq.event_id = e.id
+   AND dq.status IN ('pending', 'claimed')
+   AND (dq.client IS NULL OR dq.emit_ts IS NULL OR dq.priority IS NULL);
+
+-- Serves both halves of the fair-share claim: the DISTINCT-client scan and
+-- each client's (priority, emit_ts)-ordered candidate scan.
+CREATE INDEX IF NOT EXISTS idx_distillation_fair_claim
+  ON distillation_queue (client, priority, emit_ts)
+  WHERE status = 'pending';

package/packages/memory-engine-v2/scripts/redistill.py

@@ -212,8 +212,12 @@ def main() -> int:
                     if args.supersede_facts:
                         deleted += dump_and_delete_solo_facts(cur, eid, rb)
                     cur.execute(
-                        "INSERT INTO distillation_queue (event_id, status, attempts) "
-                        "VALUES (%s, 'pending', 0)", (eid,))
+                        # priority=200 (low): re-distills are maintenance — they
+                        # must never compete with live client traffic. Explicit
+                        # value wins over the enrichment trigger's mapping
+                        # (migration 012).
+                        "INSERT INTO distillation_queue (event_id, status, attempts, priority) "
+                        "VALUES (%s, 'pending', 0, 200)", (eid,))
                     enq += 1
                     led.write(json.dumps({
                         "event_id": eid, "arena": args.arena,