@pentatonic-ai/ai-agent-sdk 0.10.0 → 0.10.2

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.0";
+var VERSION = "0.10.2";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.10.0";
+var VERSION = "0.10.2";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory-engine-v2/compat/server.py

@@ -355,16 +355,48 @@ def _content_hash(arena: str, content: str) -> str:
 # gateway healthy and queues the rest in compat instead of pushing
 # the failure back through the DO retry loop (which causes DLQ on
 # repeated 502s — observed 2026-05-17). Pair with retry below.
-_EMBED_SEMAPHORE = asyncio.Semaphore(4)
+#
+# ── Embed lane separation ────────────────────────────────────────────
+# Live /search query embeds and bulk ingest /store(-batch) content embeds
+# share one embedder. A bulk re-embed/ingest job can then saturate the
+# embedder (GPU + the shared semaphore) and starve interactive chat —
+# observed 2026-06-06: a corpus re-embed pinned the embedder, every chat
+# query-embed timed out, semantic search returned 0. Two backward-
+# compatible levers fix it:
+#   1. NV_EMBED_URL_BULK — optional SEPARATE embedder for the bulk lane.
+#      Defaults to NV_EMBED_URL, so behaviour is unchanged until a second
+#      embedder is provisioned and this is set.
+#   2. Per-lane semaphores — the interactive lane gets its own reserved
+#      in-flight slots, so a saturated bulk lane cannot consume the slots
+#      live chat needs, EVEN when both lanes share one embedder.
+NV_EMBED_URL_BULK = os.environ.get("NV_EMBED_URL_BULK", NV_EMBED_URL)
+_EMBED_SEMAPHORE = asyncio.Semaphore(
+    int(os.environ.get("NV_EMBED_BULK_CONCURRENCY", "4"))
+)
+_EMBED_SEMAPHORE_INTERACTIVE = asyncio.Semaphore(
+    int(os.environ.get("NV_EMBED_INTERACTIVE_CONCURRENCY", "4"))
+)
 _EMBED_RETRY_STATUSES = {502, 503, 504, 429}
 _EMBED_MAX_ATTEMPTS = 5
 
 
-async def _embed_batch(texts: list[str]) -> list[list[float]]:
+async def _embed_batch(
+    texts: list[str], lane: str = "bulk"
+) -> list[list[float]]:
     """Call the external embed gateway. Both 'openai' and
-    'pentatonic-gateway' provider shapes supported."""
+    'pentatonic-gateway' provider shapes supported.
+
+    `lane` selects the embed lane (see NV_EMBED_URL_BULK above):
+      - 'interactive' — live /search query embeds. Uses NV_EMBED_URL and
+        a dedicated semaphore so chat is never starved by bulk ingest.
+      - 'bulk' (default) — ingest /store(-batch) content embeds. Uses
+        NV_EMBED_URL_BULK (defaults to NV_EMBED_URL).
+    """
     if not texts:
         return []
+    interactive = lane == "interactive"
+    url = NV_EMBED_URL if interactive else NV_EMBED_URL_BULK
+    sem = _EMBED_SEMAPHORE_INTERACTIVE if interactive else _EMBED_SEMAPHORE
     headers = {"Content-Type": "application/json"}
     if NV_EMBED_API_KEY:
         if NV_EMBED_PROVIDER == "pentatonic-gateway":
@@ -374,7 +406,7 @@ async def _embed_batch(texts: list[str]) -> list[list[float]]:
 
     body = {"input": texts, "model": "nv-embed-v2"}
 
-    async with _EMBED_SEMAPHORE:
+    async with sem:
         # Retry transient gateway failures (502/503/504/429) with
         # exponential backoff before bubbling up to the caller. Without
         # this a single GPU hiccup propagates a 500 to the TES DO,
@@ -382,7 +414,7 @@ async def _embed_batch(texts: list[str]) -> list[list[float]]:
         last_exc: Exception | None = None
         for attempt in range(_EMBED_MAX_ATTEMPTS):
             try:
-                r = await _http.post(NV_EMBED_URL, json=body, headers=headers)
+                r = await _http.post(url, json=body, headers=headers)
                 if r.status_code in _EMBED_RETRY_STATUSES:
                     last_exc = httpx.HTTPStatusError(
                         f"embed gateway {r.status_code}", request=r.request, response=r,
@@ -813,7 +845,7 @@ async def search(req: SearchRequest):
         # rejects to force callers to be explicit.
         raise HTTPException(400, "arena or arenas required")
 
-    qvec = (await _embed_batch([req.query]))[0]
+    qvec = (await _embed_batch([req.query], lane="interactive"))[0]
     # Compose Qdrant Filter: arena scope is always required, plus any
     # caller-supplied metadata_filter keys ANDed in. Mirrors how
     # /forget's `metadata_contains` already builds containment filters

package/packages/memory-engine-v2/extractor-async/Dockerfile

@@ -7,8 +7,10 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 COPY worker.py .
 # Pure helper modules — sibling imports inside worker.py
-# (noise_filter, confidence). The test_*.py files are intentionally
-# excluded; they're for local pytest, not container runtime.
-COPY noise_filter.py confidence.py ./
+# (noise_filter, confidence, entity_id). entity_id.py is byte-identical to
+# extractor-sync's copy (per-service build contexts prevent a shared module;
+# tests/test_entity_id_parity.py guards drift). The test_*.py files are
+# intentionally excluded; they're for local pytest, not container runtime.
+COPY noise_filter.py confidence.py entity_id.py ./
 
 CMD ["python", "worker.py"]

package/packages/memory-engine-v2/extractor-async/entity_id.py

@@ -0,0 +1,57 @@
+"""Canonical entity-ID scheme — SHARED, byte-identical across extractor-sync and
+extractor-async.
+
+The two extractors run as separate Docker services with PER-SERVICE build contexts
+(docker-compose `context: ./extractor-sync` / `./extractor-async`), so a single
+importable module can't be COPY'd into both. This file is therefore DUPLICATED in
+each service dir, and tests/test_entity_id_parity.py fails if the copies ever drift.
+
+Why this exists: both passes must key an entity (person / org / …) by the SAME id so
+the same entity converges across the deterministic (sync) and LLM (async) passes.
+Before this, the two services keyed entities DIFFERENTLY — sync as
+`e_` + sha256("{arena}|{type}|{name.lower().strip()}")[:24]; async as
+sha256("\\x1f".join(parts))[:32] (no lowercasing, no prefix) — so even identical
+names produced different ids and never merged. We unify on the sync scheme: sync's
+existing rows are unaffected, and the async pass converges onto them.
+
+Step 1 of RFC-entity-reconciliation.md (the foundation for alias-aware resolution).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+import unicodedata
+
+_WHITESPACE = re.compile(r"\s+")
+
+
+def normalize_surface_form(value: str) -> str:
+    """Normalize a surface form (person name, email, org name, …) for identity
+    keying. Steps, in order:
+
+      1. None → "" (defensive; some producers can hand a missing field as None).
+      2. Unicode NFKC (compatibility decomposition + canonical composition) —
+         collapses width / ligature / decomposed-accent variants. Without this
+         "Café" (precomposed U+00E9) and "Cafe\\u0301" (decomposed e+combining
+         acute) — which render identically — would key as different entities.
+         Same for fullwidth Latin ("ＣＡＲＬＹ" ↔ "CARLY") and ligatures
+         ("fi" U+FB01 ↔ "fi"). Real-world relevant for vCard imports, Mac
+         pasteboard, IME inputs, internationalised name sources.
+      3. Trim outer whitespace, then collapse internal `\\s+` to a single space —
+         "Carly  Snider" (slack-autocomplete double-space) ↔ "Carly Snider".
+      4. Lowercase. Email casing is case-insensitive per spec; person-name
+         casing varies by producer (gmail header casing vs slack profile).
+    """
+    s = unicodedata.normalize("NFKC", value or "")
+    return _WHITESPACE.sub(" ", s.strip()).lower()
+
+
+def entity_id(arena: str, entity_type: str, canonical_name: str) -> str:
+    """Deterministic entity id. The same (arena, entity_type, normalized
+    canonical_name) yields the same id across BOTH extractor passes, so re-extraction
+    and cross-pass extraction converge. Format is preserved from extractor-sync
+    (`e_` + 24 hex of sha256) so its existing rows are unaffected.
+    """
+    key = f"{arena}|{entity_type}|{normalize_surface_form(canonical_name)}"
+    return "e_" + hashlib.sha256(key.encode()).hexdigest()[:24]

package/packages/memory-engine-v2/extractor-async/sensitive_filter.py

@@ -0,0 +1,51 @@
+"""sensitive_filter — content-guardrail skip for the distillation worker.
+
+Seesa's ingest classifier tags content that is directed interpersonal
+sentiment / characterization about a named colleague (gossip) with
+`sensitivity_class == 'interpersonal'` and a `sensitive_about` subject
+set (see Seesa docs/permissions/content-guardrail-spec.md). Such content
+must NEVER be distilled into the entity graph: the SUBJECT of gossip has
+no standing in an entity graph, and turning "X thinks Y is checked out"
+into a fact ABOUT Y is the worst pollution — and a real HR / legal
+hazard. The extractor filters these events at claim time (a SQL pre-pass)
+AND per-event via the pure predicate here (defense in depth).
+
+commercial_secret is deliberately NOT filtered: it stays in the
+producer's own per-user arena; its cross-user spread is governed upstream
+(L0 / Layer-P), not by the per-arena graph.
+
+Pure module — no I/O, no deps. Importable from worker.py without pulling
+in psycopg / httpx, and importable from tests without fixtures (mirrors
+noise_filter.py).
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+# Default ON; tunable off for back-compat. Inert until Seesa stamps the
+# tag upstream (the interpersonal detector is opt-in), but safe to ship on
+# so it takes effect the moment tags appear.
+SKIP_SENSITIVE_CONTENT: bool = os.environ.get(
+    "DISTILL_SKIP_SENSITIVE_CONTENT", "true"
+).strip().lower() not in ("false", "0", "no", "off")
+
+
+def is_sensitive_event(event: dict[str, Any]) -> bool:
+    """True iff the event was tagged interpersonal gossip about a colleague
+    (`sensitivity_class == 'interpersonal'`) or carries a non-empty
+    `sensitive_about` subject set. Tolerant of missing / odd attribute
+    shapes — defaults to False (fail-open: the SQL pre-filter is the
+    primary line; a malformed attribute bag never crashes the worker)."""
+    if not isinstance(event, dict):
+        return False
+    attrs = event.get("attributes") or {}
+    if not isinstance(attrs, dict):
+        return False
+    if attrs.get("sensitivity_class") == "interpersonal":
+        return True
+    sa = attrs.get("sensitive_about")
+    return isinstance(sa, list) and any(
+        isinstance(s, str) and s.strip() for s in sa
+    )

package/packages/memory-engine-v2/extractor-async/test_async_ent_parser.py

@@ -0,0 +1,258 @@
+"""Unit tests for entity reconciliation logic in extractor-async.
+
+Covers the parts of RFC §1 (normalization), §2 (alias-aware upsert
+helper construction), and §4 (LLM prompt 4-field ENT parsing) that
+don't require a real Postgres connection. The integration scenarios
+(advisory-lock concurrency, end-to-end alias resolution) need a DB
+and live in `packages/memory-engine-v2/tests/`.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+_THIS = Path(__file__).resolve().parent
+_SPEC = importlib.util.spec_from_file_location("extractor_async_worker",
+                                                _THIS / "worker.py")
+assert _SPEC and _SPEC.loader
+worker = importlib.util.module_from_spec(_SPEC)
+try:
+    _SPEC.loader.exec_module(worker)
+except ImportError as e:
+    pytest.skip(f"extractor-async deps unavailable: {e}", allow_module_level=True)
+
+
+
+def test_content_id_does_not_normalize() -> None:
+    """`_content_id` is used for fact / relationship ids — those are
+    content-hashed and intentionally case-sensitive. Entity ids go
+    through `entity_id()` (entity_id.py) instead, which DOES normalize.
+    Locks in that separation: changing `_content_id` to normalize
+    would silently break fact/rel content addressing."""
+    a = worker._content_id("foo", "bar")
+    b = worker._content_id("FOO", "bar")
+    assert a != b
+
+
+# ----------------------------------------------------------------------
+# Prompt parser — ENT|type|name|email? (RFC §Async extractor)
+# ----------------------------------------------------------------------
+
+def test_ent_parser_three_fields_unchanged() -> None:
+    """Back-compat: 3-field ENT lines (no email) parse as before."""
+    out = worker._parse_kv_records(
+        "=== event 0 ===\nENT|person|Alex Wong\n", expected_n=1
+    )
+    assert out[0]["entities"] == [{"type": "person", "name": "Alex Wong"}]
+
+
+def test_ent_parser_four_fields_promotes_email_to_aliases() -> None:
+    """New: 4-field ENT|person|<name>|<email> → email goes into aliases."""
+    out = worker._parse_kv_records(
+        "=== event 0 ===\nENT|person|Carly Snider|carly@example.com\n",
+        expected_n=1,
+    )
+    assert out[0]["entities"] == [{
+        "type": "person",
+        "name": "Carly Snider",
+        "aliases": ["carly@example.com"],
+    }]
+
+
+def test_ent_parser_four_fields_non_email_dropped() -> None:
+    """Junk 4th field (not an email) is silently dropped — better to
+    strip noise than poison the aliases list with non-email tokens."""
+    out = worker._parse_kv_records(
+        "=== event 0 ===\nENT|person|Sam Patel|something random\n",
+        expected_n=1,
+    )
+    assert out[0]["entities"] == [{"type": "person", "name": "Sam Patel"}]
+
+
+def test_ent_parser_mixed_three_and_four_field() -> None:
+    """A batch can mix 3-field and 4-field ENT lines (model behaviour
+    will vary by what's visible per event)."""
+    out = worker._parse_kv_records(
+        (
+            "=== event 0 ===\n"
+            "ENT|person|Alex Wong\n"
+            "ENT|person|Carly Snider|carly@example.com\n"
+            "ENT|org|Acme Corp\n"
+        ),
+        expected_n=1,
+    )
+    ents = out[0]["entities"]
+    assert {"type": "person", "name": "Alex Wong"} in ents
+    assert {"type": "person", "name": "Carly Snider",
+            "aliases": ["carly@example.com"]} in ents
+    assert {"type": "org", "name": "Acme Corp"} in ents
+
+
+def test_ent_parser_email_with_dot_and_subdomain() -> None:
+    """Realistic email forms (subdomains, plus-tags, dots) get
+    accepted into aliases."""
+    out = worker._parse_kv_records(
+        (
+            "=== event 0 ===\n"
+            "ENT|person|Dot Person|first.last@sub.example.com\n"
+            "ENT|person|Plus Tag|user+tag@example.co.uk\n"
+        ),
+        expected_n=1,
+    )
+    aliases = [e.get("aliases") for e in out[0]["entities"]]
+    assert ["first.last@sub.example.com"] in aliases
+    assert ["user+tag@example.co.uk"] in aliases
+
+
+# ----------------------------------------------------------------------
+# Raw-slice splitter — for distillation trace logging (migration 003).
+# ----------------------------------------------------------------------
+
+def test_split_event_blocks_basic() -> None:
+    """Two events, each with content; both slices come back verbatim
+    (sans trailing whitespace)."""
+    text = (
+        "=== event 0 ===\n"
+        "ENT|person|Alex\n"
+        "FCT|mention|Alex|works at|Acme|Alex works at Acme\n"
+        "=== event 1 ===\n"
+        "ENT|org|Acme\n"
+    )
+    slices = worker._split_event_blocks(text, expected_n=2)
+    assert slices[0] == (
+        "ENT|person|Alex\nFCT|mention|Alex|works at|Acme|Alex works at Acme"
+    )
+    assert slices[1] == "ENT|org|Acme"
+
+
+def test_split_event_blocks_pads_missing() -> None:
+    """If the model skips an event (header absent) we still return
+    expected_n entries — missing ones come back as empty strings.
+    Parity with `_parse_kv_records` so trace logging and parsing line
+    up at the same indices."""
+    text = "=== event 0 ===\nENT|person|Alex\n"  # no event 1, 2
+    slices = worker._split_event_blocks(text, expected_n=3)
+    assert slices[0] == "ENT|person|Alex"
+    assert slices[1] == ""
+    assert slices[2] == ""
+
+
+def test_split_event_blocks_ignores_preamble() -> None:
+    """Lines before the first event header (model preamble like
+    'Sure, here are the extractions:') don't poison any slice."""
+    text = (
+        "Here are the extractions:\n\n"
+        "=== event 0 ===\n"
+        "ENT|person|Alex\n"
+    )
+    slices = worker._split_event_blocks(text, expected_n=1)
+    assert slices[0] == "ENT|person|Alex"
+
+
+def test_split_event_blocks_out_of_range_header_dropped() -> None:
+    """If the model emits `=== event 7 ===` when N=2, the rogue header
+    + its body get dropped without crashing or corrupting other slices."""
+    text = (
+        "=== event 0 ===\n"
+        "ENT|person|Alex\n"
+        "=== event 7 ===\n"
+        "ENT|person|Mystery\n"
+        "=== event 1 ===\n"
+        "ENT|org|Acme\n"
+    )
+    slices = worker._split_event_blocks(text, expected_n=2)
+    assert slices[0] == "ENT|person|Alex"
+    assert slices[1] == "ENT|org|Acme"
+    assert all("Mystery" not in s for s in slices)
+
+
+def test_split_and_parse_align_at_same_indices() -> None:
+    """Contract: for the same text + expected_n, the i-th slice
+    corresponds to the i-th parsed-records dict. The worker relies on
+    this to attach raw_slice to the right record."""
+    text = (
+        "=== event 0 ===\n"
+        "ENT|person|Alex\n"
+        "=== event 1 ===\n"
+        "ENT|org|Acme\n"
+    )
+    parsed = worker._parse_kv_records(text, expected_n=2)
+    slices = worker._split_event_blocks(text, expected_n=2)
+    assert len(parsed) == len(slices) == 2
+    # event 0 — Alex is in both the parsed entities AND the slice
+    assert any(e["name"] == "Alex" for e in parsed[0]["entities"])
+    assert "Alex" in slices[0]
+    assert "Acme" in slices[1]
+
+
+# ----------------------------------------------------------------------
+# build_event_block format contract — frozen reference for out-of-tree
+# tooling (the Pentatonic-internal training-data export, future student
+# inference, etc.) to match against. The script that generates training
+# data lives OUTSIDE the SDK so it can't import this directly; this
+# test instead asserts a fixed (event_dict, rendered) pair as the
+# canonical contract. If you change build_event_block, also update
+# anything external that reproduces the format.
+# ----------------------------------------------------------------------
+
+def test_build_event_block_format_contract_full() -> None:
+    """All optional attributes (emitted_at, author) present."""
+    event = {
+        "source_kind": "chat",
+        "content": "hello world",
+        "attributes": {
+            "emitted_at": "2026-06-01T12:00:00Z",
+            "author": "phil@example.com",
+        },
+    }
+    assert worker.build_event_block(0, event) == (
+        "[event 0]\n"
+        "source_kind: chat\n"
+        "when: 2026-06-01T12:00:00Z\n"
+        "author: phil@example.com\n"
+        "---\n"
+        "hello world"
+    )
+
+
+def test_build_event_block_format_contract_minimal() -> None:
+    """Only source_kind + content. Missing optional attrs drop their
+    header lines entirely — not blank lines, not '-' placeholders."""
+    event = {
+        "source_kind": "doc",
+        "content": "abc",
+        "attributes": {},
+    }
+    assert worker.build_event_block(0, event) == (
+        "[event 0]\n"
+        "source_kind: doc\n"
+        "---\n"
+        "abc"
+    )
+
+
+def test_build_event_block_format_contract_idx_passthrough() -> None:
+    """The idx argument names the event in the header — must round-trip
+    so that batched extractions can reattach to the right event."""
+    block = worker.build_event_block(7, {
+        "source_kind": "note", "content": "x", "attributes": {},
+    })
+    assert block.startswith("[event 7]\n")
+
+
+def test_build_event_block_format_contract_truncation() -> None:
+    """Content longer than MAX_CONTENT_CHARS gets truncated. External
+    reconstructors must mirror this — otherwise the student sees a
+    different input distribution at training vs inference time."""
+    long_content = "x" * (worker.MAX_CONTENT_CHARS + 500)
+    block = worker.build_event_block(0, {
+        "source_kind": "doc",
+        "content": long_content,
+        "attributes": {},
+    })
+    content_part = block.split("---\n", 1)[1]
+    assert len(content_part) == worker.MAX_CONTENT_CHARS

package/packages/memory-engine-v2/extractor-async/test_sensitive_filter.py

@@ -0,0 +1,61 @@
+"""Tests for sensitive_filter — content-guardrail distillation skip.
+
+The rule: never distil interpersonal gossip about a colleague into the
+entity graph. Mirrors Seesa docs/permissions/content-guardrail-spec.md.
+
+Run: pytest packages/memory-engine-v2/extractor-async/test_sensitive_filter.py
+"""
+
+from __future__ import annotations
+
+from sensitive_filter import is_sensitive_event
+
+
+class TestSensitiveEventsAreFiltered:
+    def test_interpersonal_class(self):
+        ev = {"attributes": {"sensitivity_class": "interpersonal"}}
+        assert is_sensitive_event(ev)
+
+    def test_non_empty_sensitive_about(self):
+        ev = {"attributes": {"sensitive_about": ["usr_sarah"]}}
+        assert is_sensitive_event(ev)
+
+    def test_both(self):
+        ev = {"attributes": {"sensitivity_class": "interpersonal", "sensitive_about": ["usr_x"]}}
+        assert is_sensitive_event(ev)
+
+
+class TestNonSensitiveEventsPass:
+    def test_clean_event(self):
+        ev = {"attributes": {"source": "pip-granola-ingest", "content": "Q3 roadmap"}}
+        assert not is_sensitive_event(ev)
+
+    def test_commercial_secret_is_NOT_filtered(self):
+        # commercial_secret stays in the producer's own arena — not graph-filtered.
+        ev = {"attributes": {"sensitivity_class": "commercial_secret"}}
+        assert not is_sensitive_event(ev)
+
+    def test_empty_sensitive_about(self):
+        ev = {"attributes": {"sensitive_about": []}}
+        assert not is_sensitive_event(ev)
+
+    def test_blank_only_subjects(self):
+        ev = {"attributes": {"sensitive_about": ["", "  "]}}
+        assert not is_sensitive_event(ev)
+
+
+class TestMalformedShapesDefaultFalse:
+    def test_no_attributes(self):
+        assert not is_sensitive_event({})
+
+    def test_attributes_none(self):
+        assert not is_sensitive_event({"attributes": None})
+
+    def test_attributes_not_dict(self):
+        assert not is_sensitive_event({"attributes": "oops"})
+
+    def test_event_not_dict(self):
+        assert not is_sensitive_event("nope")  # type: ignore[arg-type]
+
+    def test_sensitive_about_not_list(self):
+        assert not is_sensitive_event({"attributes": {"sensitive_about": "usr_x"}})