@pentatonic-ai/ai-agent-sdk 0.9.5 → 0.10.0

package/packages/memory-engine/scripts/wipe-legacy-l3-entities.py

@@ -1,128 +0,0 @@
-#!/usr/bin/env python3
-"""Wipe pre-arena-scoping :Entity nodes from the L3 Neo4j graph.
-
-Run this AFTER the engine has been deployed with the arena-scoped
-writer paths, not before. Sequence:
-
-    1. Deploy l2-hybridrag-proxy with arena-scoped MERGE patterns.
-    2. Verify new ingest is creating arena-tagged entities (run
-       --dry-run on this script first; it reports legacy vs new).
-    3. Run this script with --confirm to wipe legacy entities.
-    4. Future ingest re-extracts entities from existing :Chunk nodes
-       on-demand (search-side touches them; new stores recreate them
-       from scratch under the right arena).
-
-Why wipe vs. backfill: pre-fix entities collapsed cross-tenant by name,
-so their MENTIONS edges connect to chunks across multiple arenas.
-Splitting them by mentions is doable but error-prone (edge cases for
-many-arena entities, orphans, no-mention entities). The Hebbian weights
-on those edges were also corrupted by cross-tenant traffic, so they
-weren't worth saving. Chunks are preserved either way — re-extraction
-is cheap.
-
-Usage:
-    # report counts only
-    python wipe-legacy-l3-entities.py --neo4j-uri bolt://localhost:7687
-
-    # actually wipe
-    python wipe-legacy-l3-entities.py --neo4j-uri bolt://localhost:7687 --confirm
-
-The script is idempotent — running it twice on a clean graph deletes
-zero rows.
-"""
-from __future__ import annotations
-
-import argparse
-import os
-import sys
-
-try:
-    from neo4j import GraphDatabase
-except ImportError:
-    print("ERROR: neo4j driver not installed. `pip install neo4j` first.", file=sys.stderr)
-    sys.exit(1)
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
-    parser.add_argument(
-        "--neo4j-uri",
-        default=os.environ.get("NEO4J_URI", "bolt://localhost:7687"),
-        help="Neo4j bolt URI (default: NEO4J_URI env or bolt://localhost:7687)",
-    )
-    parser.add_argument(
-        "--neo4j-user",
-        default=os.environ.get("NEO4J_USER", "neo4j"),
-    )
-    parser.add_argument(
-        "--neo4j-password",
-        default=os.environ.get("NEO4J_PASSWORD"),
-    )
-    parser.add_argument(
-        "--confirm",
-        action="store_true",
-        help="Actually delete. Without this flag, runs in dry-run mode "
-        "(reports counts only).",
-    )
-    args = parser.parse_args()
-
-    if not args.neo4j_password:
-        print("ERROR: --neo4j-password or NEO4J_PASSWORD env required", file=sys.stderr)
-        return 2
-
-    driver = GraphDatabase.driver(
-        args.neo4j_uri, auth=(args.neo4j_user, args.neo4j_password),
-    )
-    try:
-        with driver.session() as session:
-            # Count legacy vs arena-scoped entities so the operator can
-            # eyeball whether the new writer path has actually started
-            # producing arena-scoped rows before deleting anything.
-            legacy = session.run(
-                "MATCH (e:Entity) WHERE e.arena IS NULL RETURN count(e) AS n"
-            ).single()["n"]
-            scoped = session.run(
-                "MATCH (e:Entity) WHERE e.arena IS NOT NULL RETURN count(e) AS n"
-            ).single()["n"]
-            print(f"L3 Entity nodes: {legacy} legacy (no arena) / {scoped} arena-scoped")
-
-            # Same breakdown for chunks — they should already all be
-            # arena-tagged but worth verifying before/after.
-            chunk_legacy = session.run(
-                "MATCH (c:Chunk) WHERE c.arena IS NULL RETURN count(c) AS n"
-            ).single()["n"]
-            chunk_scoped = session.run(
-                "MATCH (c:Chunk) WHERE c.arena IS NOT NULL RETURN count(c) AS n"
-            ).single()["n"]
-            print(f"L3 Chunk nodes:  {chunk_legacy} legacy (no arena) / {chunk_scoped} arena-scoped")
-
-            if not args.confirm:
-                print("\nDry run — pass --confirm to actually delete legacy entities.")
-                return 0
-
-            if legacy == 0:
-                print("\nNothing to do — all entities already arena-scoped.")
-                return 0
-
-            print(f"\nWiping {legacy} legacy entities…")
-            result = session.run(
-                "MATCH (e:Entity) WHERE e.arena IS NULL DETACH DELETE e RETURN count(e) AS n"
-            ).single()
-            deleted = result["n"]
-            print(f"Deleted {deleted} legacy entities.")
-
-            # Drop the old single-key entity index now that the data
-            # is gone — safe to do because the new writer path uses
-            # the compound (arena, name) index.
-            try:
-                session.run("DROP INDEX entity_name IF EXISTS")
-                print("Dropped legacy `entity_name` index.")
-            except Exception as e:  # noqa: BLE001
-                print(f"Note: could not drop entity_name index ({e}); ok if absent.")
-        return 0
-    finally:
-        driver.close()
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/packages/memory-engine/tests/e2e_arena.sh

@@ -1,259 +0,0 @@
-#!/usr/bin/env bash
-# e2e_arena.sh — multi-tenant store/retrieve smoke test against a live
-# memory-engine stack. Exercises /store, arena-scoped /search, and
-# /forget end-to-end across L0/L4/L5/L6 + the compat shim.
-#
-# Run after `docker compose -f docker-compose.yml -f docker-compose.test.yml \
-#   up -d --wait l3 l4 l5 l6 l2 compat embed-stub`.
-set -eu
-
-BASE="${BASE:-http://localhost:8099}"
-WAIT_HEALTH_SECS="${WAIT_HEALTH_SECS:-180}"
-PASS=0
-FAIL=0
-
-ok()   { echo "  ✅ $1"; PASS=$((PASS+1)); }
-fail() { echo "  ❌ $1"; FAIL=$((FAIL+1)); }
-
-# ---------------------------------------------------------------------------
-# Wait for the compat shim to come up. Its /health aggregates layer
-# health; we accept "ok" or "degraded" (l3 cosmetic 404 is known and
-# doesn't block functional paths).
-# ---------------------------------------------------------------------------
-
-echo "=== waiting for $BASE/health (up to ${WAIT_HEALTH_SECS}s) ==="
-deadline=$(( $(date +%s) + WAIT_HEALTH_SECS ))
-while :; do
-  if H=$(curl -sf --max-time 5 "$BASE/health"); then
-    s=$(echo "$H" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("status",""))')
-    if [ "$s" = "ok" ] || [ "$s" = "degraded" ]; then
-      echo "  health: $s"
-      break
-    fi
-  fi
-  if [ "$(date +%s)" -ge "$deadline" ]; then
-    echo "  ❌ engine never became healthy"
-    exit 1
-  fi
-  sleep 3
-done
-
-# ---------------------------------------------------------------------------
-# /store — two arenas, two distinct documents per arena.
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== /store ==="
-post() {
-  curl -sf -X POST "$BASE/store" \
-    -H "Content-Type: application/json" \
-    -d "$1"
-}
-
-R1=$(post '{"content":"Alpha team owns project Atlas","metadata":{"arena":"e2e-tenant-a","kind":"note","probe":"e2e-arena"}}')
-R2=$(post '{"content":"Alpha team owns project Borealis","metadata":{"arena":"e2e-tenant-b","kind":"note","probe":"e2e-arena"}}')
-R3=$(post '{"content":"Bravo team owns project Cobalt","metadata":{"arena":"e2e-tenant-a","kind":"note","probe":"e2e-arena"}}')
-R4=$(post '{"content":"Bravo team owns project Diamond","metadata":{"arena":"e2e-tenant-b","kind":"note","probe":"e2e-arena"}}')
-
-[ -n "$R1" ] && [ -n "$R2" ] && [ -n "$R3" ] && [ -n "$R4" ] \
-  && ok "stored 4 docs across 2 arenas" \
-  || fail "store"
-
-# Indexing is async on some layers — give the stack a brief settle.
-sleep 4
-
-# ---------------------------------------------------------------------------
-# /search — arena scoping. tenant-a should never see Borealis/Diamond,
-# tenant-b should never see Atlas/Cobalt.
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== /search arena=e2e-tenant-a ==="
-SA=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"team project","limit":20,"arena":"e2e-tenant-a"}')
-echo "  hits: $(echo "$SA" | python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("results",[])))')"
-
-leak_a=$(echo "$SA" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-bad=[r for r in data if "Borealis" in r.get("content","") or "Diamond" in r.get("content","")]
-print(len(bad))')
-[ "$leak_a" = "0" ] && ok "tenant-a: no Borealis/Diamond leakage" \
-  || fail "tenant-a leaked $leak_a tenant-b docs"
-
-found_atlas=$(echo "$SA" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Atlas" in r.get("content","") for r in data) else "no")')
-[ "$found_atlas" = "yes" ] && ok "tenant-a: Atlas recovered" \
-  || fail "tenant-a missing Atlas"
-
-echo ""
-echo "=== /search arena=e2e-tenant-b ==="
-SB=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"team project","limit":20,"arena":"e2e-tenant-b"}')
-echo "  hits: $(echo "$SB" | python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("results",[])))')"
-
-leak_b=$(echo "$SB" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-bad=[r for r in data if "Atlas" in r.get("content","") or "Cobalt" in r.get("content","")]
-print(len(bad))')
-[ "$leak_b" = "0" ] && ok "tenant-b: no Atlas/Cobalt leakage" \
-  || fail "tenant-b leaked $leak_b tenant-a docs"
-
-found_borealis=$(echo "$SB" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Borealis" in r.get("content","") for r in data) else "no")')
-[ "$found_borealis" = "yes" ] && ok "tenant-b: Borealis recovered" \
-  || fail "tenant-b missing Borealis"
-
-# ---------------------------------------------------------------------------
-# /search with metadata_filter — arena+probe combo should still scope.
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== /search metadata_filter probe=e2e-arena ==="
-SF=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"team","limit":20,"arena":"e2e-tenant-a","metadata_filter":{"probe":"e2e-arena"}}')
-all_match=$(echo "$SF" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-ok=all(r.get("metadata",{}).get("probe")=="e2e-arena" and r.get("metadata",{}).get("arena")=="e2e-tenant-a" for r in data)
-print("yes" if ok and data else "no")')
-[ "$all_match" = "yes" ] && ok "metadata_filter scopes to probe + arena" \
-  || fail "metadata_filter let other rows through"
-
-# ---------------------------------------------------------------------------
-# User-scope vs tenant-wide arenas — proves the multi-arena search model.
-#
-#   tenant-wide row    arena=acme               (visible to every user in acme)
-#   user-A's row       arena=acme:user-a        (only user-A retrieves it)
-#   user-B's row       arena=acme:user-b        (only user-B retrieves it)
-#
-# A user-scoped search sends arenas=[acme, acme:userX] so the user sees
-# tenant-wide AND own user-scope, but never another user's user-scope.
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== user-scope vs tenant-wide ==="
-post '{"content":"acme tenant-wide rules of engagement","metadata":{"arena":"acme","probe":"e2e-arena"}}' >/dev/null
-post '{"content":"alice private note about Project Mercury","metadata":{"arena":"acme:alice","probe":"e2e-arena"}}' >/dev/null
-post '{"content":"bob private note about Project Saturn","metadata":{"arena":"acme:bob","probe":"e2e-arena"}}' >/dev/null
-sleep 3
-
-# Search as alice: arenas=[acme, acme:alice] — should see tenant-wide + own
-SAlice=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"Project rules note","limit":20,"arenas":["acme","acme:alice"]}')
-
-alice_sees_tenant=$(echo "$SAlice" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("tenant-wide" in r.get("content","") for r in data) else "no")')
-alice_sees_own=$(echo "$SAlice" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Mercury" in r.get("content","") for r in data) else "no")')
-alice_leak_bob=$(echo "$SAlice" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print(sum(1 for r in data if "Saturn" in r.get("content","")))')
-
-[ "$alice_sees_tenant" = "yes" ] && ok "alice: tenant-wide visible" \
-  || fail "alice: missing tenant-wide row"
-[ "$alice_sees_own" = "yes" ] && ok "alice: own user-scope visible" \
-  || fail "alice: missing own user-scope row"
-[ "$alice_leak_bob" = "0" ] && ok "alice: no leakage of bob's user-scope" \
-  || fail "alice leaked $alice_leak_bob bob rows (cross-user!)"
-
-# Search as bob: arenas=[acme, acme:bob] — should see tenant-wide + own
-SBob=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"Project rules note","limit":20,"arenas":["acme","acme:bob"]}')
-
-bob_sees_own=$(echo "$SBob" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Saturn" in r.get("content","") for r in data) else "no")')
-bob_leak_alice=$(echo "$SBob" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print(sum(1 for r in data if "Mercury" in r.get("content","")))')
-
-[ "$bob_sees_own" = "yes" ] && ok "bob: own user-scope visible" \
-  || fail "bob: missing own user-scope row"
-[ "$bob_leak_alice" = "0" ] && ok "bob: no leakage of alice's user-scope" \
-  || fail "bob leaked $bob_leak_alice alice rows (cross-user!)"
-
-# ---------------------------------------------------------------------------
-# Same content across two arenas — proves the arena-aware id derivation.
-# Pre-v0.7.8, identical content collapsed to one row in L4/L5/L6 because
-# the id was sha1(content); the second tenant's write overwrote the first.
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== same-content-across-arenas ==="
-post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-x","probe":"e2e-arena"}}' >/dev/null
-post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-y","probe":"e2e-arena"}}' >/dev/null
-sleep 3
-
-SX=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-x"}')
-SY=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-y"}')
-
-x_has_phrase=$(echo "$SX" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
-y_has_phrase=$(echo "$SY" | python3 -c '
-import json,sys
-data=json.load(sys.stdin).get("results",[])
-print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
-
-[ "$x_has_phrase" = "yes" ] && ok "tenant-x: shared phrase preserved" \
-  || fail "tenant-x lost the shared phrase (id collision?)"
-[ "$y_has_phrase" = "yes" ] && ok "tenant-y: shared phrase preserved" \
-  || fail "tenant-y lost the shared phrase (id collision?)"
-
-# ---------------------------------------------------------------------------
-# /forget — tenant-scoped delete by arena.
-#
-# Pre-arena, /forget would unconditionally trigger a global wipe of
-# L0+L4+L3 inside the shim, regardless of what filters were passed. The
-# smoke test happily asserted "deleted >= 1" because the shim returned
-# the count of every row globally, even rows the caller didn't ask to
-# remove. That was the right thing for a bench reset, the wrong thing
-# for any real tenant operation.
-#
-# Now /forget honours the arena in the metadata filter:
-#   - {metadata_contains: {arena: <tenant>}}     → tenant-scoped wipe
-#   - {metadata_contains: {<other>: <val>}}      → L6 targeted only
-#   - {} (empty body)                            → bench reset (gated
-#                                                  by GLOBAL_WIPE in the
-#                                                  internal endpoint)
-#
-# We assert here that wiping arena=e2e-tenant-a actually removes ≥1 row,
-# AND that arena=e2e-tenant-b's rows survived — proves real tenant
-# isolation rather than just "shim returned non-zero".
-# ---------------------------------------------------------------------------
-
-echo ""
-echo "=== /forget arena=e2e-tenant-a ==="
-F=$(curl -sf -X POST "$BASE/forget" -H "Content-Type: application/json" \
-  -d '{"metadata_contains":{"arena":"e2e-tenant-a"}}')
-deleted=$(echo "$F" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("deleted",0))')
-echo "  deleted: $deleted"
-[ "$deleted" -ge "1" ] && ok "/forget removed tenant-a rows" || fail "/forget tenant-a wipe"
-
-# tenant-b should still have its rows.
-B=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
-  -d '{"arena":"e2e-tenant-b","query":"Borealis","limit":4}')
-b_hits=$(echo "$B" | python3 -c 'import json,sys; print(len(json.load(sys.stdin).get("results",[])))')
-[ "$b_hits" -ge "1" ] && ok "/forget left tenant-b intact" || fail "/forget leaked into tenant-b"
-
-echo ""
-echo "=== Result ==="
-echo "  PASS: $PASS"
-echo "  FAIL: $FAIL"
-exit $FAIL

package/packages/memory-engine/tests/embed_stub/Dockerfile

@@ -1,13 +0,0 @@
-FROM python:3.12-slim
-
-WORKDIR /app
-
-RUN pip install --no-cache-dir fastapi "uvicorn[standard]" pydantic
-
-COPY server.py /app/server.py
-
-ENV EMBED_DIM=4096
-
-EXPOSE 8041
-
-CMD ["python", "-m", "uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8041"]

package/packages/memory-engine/tests/embed_stub/server.py

@@ -1,80 +0,0 @@
-"""Deterministic embedding stub for hermetic CI runs.
-
-Returns a fixed-dim vector per input string, derived from a hash so the
-same text always maps to the same vector. Cosine similarity between two
-embeddings equals 1.0 only for identical input strings, and decreases
-roughly with edit distance — enough to exercise the engine's vector
-search paths in CI without an actual embedding model.
-
-Speaks both shapes the engine uses:
-  POST /v1/embeddings  { input, model }   -> { data:[{embedding:[...] }] }
-  POST /v1/embed       { input, model }   -> { embeddings:[[...]] }
-
-Run:
-  EMBED_DIM=4096 uvicorn server:app --host 0.0.0.0 --port 8041
-"""
-from __future__ import annotations
-
-import hashlib
-import math
-import os
-from typing import Any
-
-from fastapi import FastAPI
-from pydantic import BaseModel
-
-EMBED_DIM = int(os.environ.get("EMBED_DIM", "4096"))
-
-app = FastAPI(title="embed-stub")
-
-
-class EmbedRequest(BaseModel):
-    input: Any
-    model: str | None = None
-
-
-def _vector_for(text: str) -> list[float]:
-    """Deterministic vector: hash the text, expand to EMBED_DIM, L2-normalise."""
-    text = text or ""
-    seed = hashlib.sha256(text.encode("utf-8")).digest()
-    raw: list[int] = []
-    counter = 0
-    while len(raw) < EMBED_DIM:
-        chunk = hashlib.sha256(seed + counter.to_bytes(4, "big")).digest()
-        raw.extend(chunk)
-        counter += 1
-    floats = [(b - 127.5) / 127.5 for b in raw[:EMBED_DIM]]
-    norm = math.sqrt(sum(x * x for x in floats)) or 1.0
-    return [x / norm for x in floats]
-
-
-def _normalise_inputs(inp: Any) -> list[str]:
-    if isinstance(inp, str):
-        return [inp]
-    if isinstance(inp, list):
-        return [str(x) for x in inp]
-    return [str(inp)]
-
-
-@app.get("/health")
-def health() -> dict:
-    return {"status": "ok", "dim": EMBED_DIM}
-
-
-@app.post("/v1/embeddings")
-def openai_embeddings(req: EmbedRequest) -> dict:
-    texts = _normalise_inputs(req.input)
-    return {
-        "object": "list",
-        "data": [
-            {"object": "embedding", "index": i, "embedding": _vector_for(t)}
-            for i, t in enumerate(texts)
-        ],
-        "model": req.model or "embed-stub",
-    }
-
-
-@app.post("/v1/embed")
-def lambda_gateway_embed(req: EmbedRequest) -> dict:
-    texts = _normalise_inputs(req.input)
-    return {"embeddings": [_vector_for(t) for t in texts]}