@pentatonic-ai/ai-agent-sdk 0.10.5 → 0.10.6

package/packages/memory-engine-v2/eval/retrieval_golden.seed.json

@@ -0,0 +1,69 @@
+{
+  "_instructions": [
+    "Retrieval-eval golden seed for hybrid BM25+RRF retrieval (BET 3).",
+    "Each question maps to the event_ids that a correct /search MUST surface.",
+    "HOW TO FILL IN: replace each 'EVENT_ID_PLACEHOLDER_*' with a real",
+    "`events.id` from the engine's Postgres (the same id /search returns as",
+    "`results[].id`). Find them by running the question through /search on",
+    "the live engine and/or `SELECT id, content FROM events WHERE arena = $1",
+    "AND content ILIKE '%<distinctive phrase>%'`, then manually verifying the",
+    "event actually answers the question. relevance: 2 = directly answers,",
+    "1 = useful supporting context. Leave a question's `relevant` empty to",
+    "exclude it from metrics (recall_at_k.py skips unfilled questions).",
+    "Add questions freely — lexical-heavy ones (exact names, codes, file",
+    "names, invoice numbers) are the cases hybrid BM25 is meant to win;",
+    "keep paraphrase-style ones too so dense regressions are caught.",
+    "No live calls happen from this file — it is data for eval/recall_at_k.py."
+  ],
+  "version": 1,
+  "default_arena": "REPLACE_WITH_ARENA (e.g. pentatonic-team:usr_xxx)",
+  "questions": [
+    {
+      "id": "q-exact-name-1",
+      "class": "lexical",
+      "note": "Exact proper-noun lookup — BM25 should dominate.",
+      "query": "REPLACE: a question naming a specific person/company verbatim",
+      "relevant": [
+        {"event_id": "EVENT_ID_PLACEHOLDER_1A", "relevance": 2},
+        {"event_id": "EVENT_ID_PLACEHOLDER_1B", "relevance": 1}
+      ]
+    },
+    {
+      "id": "q-exact-code-1",
+      "class": "lexical",
+      "note": "Identifier/code/file-name lookup (e.g. invoice no, PR #, doc title) — the classic dense-retrieval miss.",
+      "query": "REPLACE: a question containing an exact identifier",
+      "relevant": [
+        {"event_id": "EVENT_ID_PLACEHOLDER_2A", "relevance": 2}
+      ]
+    },
+    {
+      "id": "q-paraphrase-1",
+      "class": "semantic",
+      "note": "Paraphrase with zero keyword overlap — dense should carry; guards against hybrid regressing semantic recall.",
+      "query": "REPLACE: a question that paraphrases the source content",
+      "relevant": [
+        {"event_id": "EVENT_ID_PLACEHOLDER_3A", "relevance": 2}
+      ]
+    },
+    {
+      "id": "q-temporal-1",
+      "class": "temporal",
+      "note": "'last meeting'-class query — checks the temporal re-rank still works on RRF-fused candidates.",
+      "query": "REPLACE: when did we last meet <person>?",
+      "relevant": [
+        {"event_id": "EVENT_ID_PLACEHOLDER_4A", "relevance": 2}
+      ]
+    },
+    {
+      "id": "q-mixed-1",
+      "class": "mixed",
+      "note": "Named entity + semantic intent — the case RRF fusion is built for.",
+      "query": "REPLACE: e.g. summary of the <Company> contract discussion",
+      "relevant": [
+        {"event_id": "EVENT_ID_PLACEHOLDER_5A", "relevance": 2},
+        {"event_id": "EVENT_ID_PLACEHOLDER_5B", "relevance": 1}
+      ]
+    }
+  ]
+}

package/packages/memory-engine-v2/extractor-async/Dockerfile

@@ -14,6 +14,6 @@ COPY worker.py .
 # add a new sibling module, add it here too — missing COPY makes the
 # container crash-loop on import at startup (observed 2026-06-08 deploy).
 # The test_*.py files are intentionally excluded; pytest only, not runtime.
-COPY noise_filter.py confidence.py entity_id.py sensitive_filter.py ./
+COPY noise_filter.py confidence.py entity_id.py sensitive_filter.py extraction_schema.py ./
 
 CMD ["python", "worker.py"]

package/packages/memory-engine-v2/extractor-async/extraction_schema.py

@@ -0,0 +1,246 @@
+"""extraction_schema — JSON Schema for guided-JSON distiller output.
+
+Used by worker.py when DISTILL_OUTPUT_MODE=guided_json: the schema is
+handed to vLLM's structured-output engine (xgrammar/outlines), which
+masks logits during decoding so the model is INCAPABLE of emitting
+schema-invalid bytes. This is what makes JSON output safe to revisit
+after the 2026-05-18 removal of guided_json (see worker.py ~87-92):
+
+  - The old failure ("one bad char in a 13k-char JSON blob nukes the
+    whole 15-event chunk") is answered twice over: (a) guided decoding
+    means the server, not the model's goodwill, guarantees well-formed
+    output; (b) the schema is an ARRAY OF PER-EVENT OBJECTS, so one
+    event's content can never corrupt another event's parse — the only
+    residual failure mode is max_tokens truncation, and the parser
+    salvages every complete event object before the cut.
+  - The old "server half-ignored structured output" applied to the
+    Qwen3-VL-30B gateway deployment; the self-hosted Qwen2.5-7B vLLM
+    box enforces it.
+
+This module is the single source of truth for the allowed entity
+types and fact categories (worker.py imports them from here — leaf
+module, so no circular import). The fact object carries EXACTLY the
+same 5 semantic fields the KV parser yields (category, subject,
+predicate, object, statement) so the upsert path is untouched.
+
+Pure module — no I/O, stdlib only. Importable from worker.py, tests,
+and scripts/bakeoff_guided_vs_kv.py without psycopg/httpx.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+# Allowed-value enums. Moved here from worker.py (which now imports
+# them) so the schema pins to the SAME constants the KV prompt and
+# downstream normalisation use — change them in one place only.
+ALLOWED_ENT_TYPES = {
+    "person", "org", "product", "place", "project",
+    "concept", "topic", "date", "other",
+}
+ALLOWED_FCT_CATEGORIES = {
+    "decision", "commitment", "state", "mention",
+    "observation", "preference",
+}
+
+# Hard caps per event — mirror BATCH_SYSTEM_PROMPT's "HARD CAPS per
+# event: 8 ENT, 6 FCT, 6 REL" so guided decoding enforces what the KV
+# prompt could only request.
+MAX_ENTITIES_PER_EVENT = 8
+MAX_FACTS_PER_EVENT = 6
+MAX_RELATIONSHIPS_PER_EVENT = 6
+MAX_STATEMENT_CHARS = 140
+
+# sorted() so the schema (and anything hashed from it) is byte-stable
+# across processes — set iteration order is hash-randomised.
+_ENT_TYPE_ENUM = sorted(ALLOWED_ENT_TYPES)
+_FCT_CATEGORY_ENUM = sorted(ALLOWED_FCT_CATEGORIES)
+
+EXTRACTION_SCHEMA: dict[str, Any] = {
+    "type": "object",
+    "properties": {
+        "events": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    # Zero-indexed position of the event in the input
+                    # batch — mirrors the `[event K]` header so parsed
+                    # objects reattach to the right queue item even if
+                    # the model reorders or a truncation drops the tail.
+                    "index": {"type": "integer", "minimum": 0},
+                    "entities": {
+                        "type": "array",
+                        "maxItems": MAX_ENTITIES_PER_EVENT,
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string", "minLength": 1},
+                                "type": {"type": "string", "enum": _ENT_TYPE_ENUM},
+                                # Optional, person-only (prompt rule);
+                                # promoted into aliases by the parser
+                                # exactly like the KV 4th field.
+                                "email": {"type": "string"},
+                            },
+                            "required": ["name", "type"],
+                            "additionalProperties": False,
+                        },
+                    },
+                    "facts": {
+                        "type": "array",
+                        "maxItems": MAX_FACTS_PER_EVENT,
+                        "items": {
+                            "type": "object",
+                            # EXACTLY the 5 semantic fields the KV
+                            # parser yields (FCT line = literal `FCT`
+                            # + these 5) — upserts stay untouched.
+                            "properties": {
+                                "category": {"type": "string", "enum": _FCT_CATEGORY_ENUM},
+                                "subject": {"type": "string", "minLength": 1},
+                                "predicate": {"type": "string", "minLength": 1},
+                                # null when absent — the KV format's `-`.
+                                "object": {"type": ["string", "null"]},
+                                "statement": {
+                                    "type": "string",
+                                    "minLength": 1,
+                                    "maxLength": MAX_STATEMENT_CHARS,
+                                },
+                            },
+                            "required": [
+                                "category", "subject", "predicate",
+                                "object", "statement",
+                            ],
+                            "additionalProperties": False,
+                        },
+                    },
+                    "relationships": {
+                        "type": "array",
+                        "maxItems": MAX_RELATIONSHIPS_PER_EVENT,
+                        "items": {
+                            "type": "object",
+                            # Mirror the KV REL fields: REL|from|to|rel_type.
+                            "properties": {
+                                "from": {"type": "string", "minLength": 1},
+                                "to": {"type": "string", "minLength": 1},
+                                "type": {"type": "string", "minLength": 1},
+                            },
+                            "required": ["from", "to", "type"],
+                            "additionalProperties": False,
+                        },
+                    },
+                },
+                "required": ["index", "entities", "facts", "relationships"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    "required": ["events"],
+    "additionalProperties": False,
+}
+
+
+def extraction_schema_json() -> str:
+    """Stable serialisation of EXTRACTION_SCHEMA (sorted keys) for
+    request bodies and fingerprinting."""
+    return json.dumps(EXTRACTION_SCHEMA, sort_keys=True, separators=(",", ":"))
+
+
+# ----------------------------------------------------------------------
+# Hand-rolled payload validation.
+#
+# The runtime guarantee comes from vLLM's logit masking, NOT from this
+# function — it exists so tests and the bake-off script can check
+# payloads against the same constraints without adding a `jsonschema`
+# dependency (not currently in requirements.txt). It covers exactly the
+# constraints EXTRACTION_SCHEMA expresses; if you extend the schema,
+# extend this too.
+# ----------------------------------------------------------------------
+
+
+def validate_payload(payload: Any) -> list[str]:
+    """Return a list of human-readable violations ([] == valid)."""
+    errors: list[str] = []
+    if not isinstance(payload, dict):
+        return ["payload: not an object"]
+    if set(payload.keys()) - {"events"}:
+        errors.append("payload: unexpected top-level keys")
+    events = payload.get("events")
+    if not isinstance(events, list):
+        return errors + ["events: missing or not an array"]
+    for i, ev in enumerate(events):
+        if not isinstance(ev, dict):
+            errors.append(f"events[{i}]: not an object")
+            continue
+        if set(ev.keys()) - {"index", "entities", "facts", "relationships"}:
+            errors.append(f"events[{i}]: unexpected keys")
+        idx = ev.get("index")
+        if not isinstance(idx, int) or isinstance(idx, bool) or idx < 0:
+            errors.append(f"events[{i}].index: not a non-negative integer")
+        ents = ev.get("entities")
+        if not isinstance(ents, list):
+            errors.append(f"events[{i}].entities: not an array")
+            ents = []
+        if len(ents) > MAX_ENTITIES_PER_EVENT:
+            errors.append(f"events[{i}].entities: exceeds {MAX_ENTITIES_PER_EVENT}")
+        for j, e in enumerate(ents):
+            loc = f"events[{i}].entities[{j}]"
+            if not isinstance(e, dict):
+                errors.append(f"{loc}: not an object")
+                continue
+            if set(e.keys()) - {"name", "type", "email"}:
+                errors.append(f"{loc}: unexpected keys")
+            if not (isinstance(e.get("name"), str) and e.get("name")):
+                errors.append(f"{loc}.name: missing/empty")
+            if e.get("type") not in ALLOWED_ENT_TYPES:
+                errors.append(f"{loc}.type: not in ALLOWED_ENT_TYPES")
+            if "email" in e and not isinstance(e["email"], str):
+                errors.append(f"{loc}.email: not a string")
+        facts = ev.get("facts")
+        if not isinstance(facts, list):
+            errors.append(f"events[{i}].facts: not an array")
+            facts = []
+        if len(facts) > MAX_FACTS_PER_EVENT:
+            errors.append(f"events[{i}].facts: exceeds {MAX_FACTS_PER_EVENT}")
+        for j, f in enumerate(facts):
+            loc = f"events[{i}].facts[{j}]"
+            if not isinstance(f, dict):
+                errors.append(f"{loc}: not an object")
+                continue
+            required = {"category", "subject", "predicate", "object", "statement"}
+            if set(f.keys()) != required:
+                errors.append(f"{loc}: keys != {{category,subject,predicate,object,statement}}")
+                continue
+            if f["category"] not in ALLOWED_FCT_CATEGORIES:
+                errors.append(f"{loc}.category: not in ALLOWED_FCT_CATEGORIES")
+            if not (isinstance(f["subject"], str) and f["subject"]):
+                errors.append(f"{loc}.subject: missing/empty")
+            if not (isinstance(f["predicate"], str) and f["predicate"]):
+                errors.append(f"{loc}.predicate: missing/empty")
+            if f["object"] is not None and not isinstance(f["object"], str):
+                errors.append(f"{loc}.object: not string-or-null")
+            stmt = f["statement"]
+            if not (isinstance(stmt, str) and stmt):
+                errors.append(f"{loc}.statement: missing/empty")
+            elif len(stmt) > MAX_STATEMENT_CHARS:
+                errors.append(f"{loc}.statement: exceeds {MAX_STATEMENT_CHARS} chars")
+        rels = ev.get("relationships")
+        if not isinstance(rels, list):
+            errors.append(f"events[{i}].relationships: not an array")
+            rels = []
+        if len(rels) > MAX_RELATIONSHIPS_PER_EVENT:
+            errors.append(
+                f"events[{i}].relationships: exceeds {MAX_RELATIONSHIPS_PER_EVENT}"
+            )
+        for j, r in enumerate(rels):
+            loc = f"events[{i}].relationships[{j}]"
+            if not isinstance(r, dict):
+                errors.append(f"{loc}: not an object")
+                continue
+            if set(r.keys()) != {"from", "to", "type"}:
+                errors.append(f"{loc}: keys != {{from,to,type}}")
+                continue
+            for k in ("from", "to", "type"):
+                if not (isinstance(r[k], str) and r[k]):
+                    errors.append(f"{loc}.{k}: missing/empty")
+    return errors