@pentatonic-ai/ai-agent-sdk 0.9.6 → 0.10.0

package/packages/memory-engine/docker-compose.test.yml

@@ -1,59 +0,0 @@
-# docker-compose.test.yml — overlay for hermetic CI runs.
-#
-# Replaces the nv-embed GPU service with a deterministic embedding
-# stub that mimics both the OpenAI /v1/embeddings shape and the
-# lambda-gateway /v1/embed shape. Lets CI exercise every layer's
-# vector path without an actual model.
-#
-# Usage:
-#   docker compose -f docker-compose.yml -f docker-compose.test.yml \
-#     up -d --wait l3 l4 l5 l6 l2 compat embed-stub
-#
-# The base nv-embed service is intentionally NOT started in CI
-# (requires a GPU). l4/l5/l6 are pointed at embed-stub via env.
-
-services:
-  embed-stub:
-    build:
-      context: ./tests/embed_stub
-      dockerfile: Dockerfile
-    container_name: pme-embed-stub
-    networks:
-      - engine-net
-    environment:
-      EMBED_DIM: "4096"
-    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8041/health',timeout=3)"]
-      interval: 5s
-      timeout: 3s
-      retries: 20
-      start_period: 5s
-
-  # Pin the embedding dim explicitly across layers, independent of any
-  # developer-local .env (which may set EMBED_DIM=768 for Ollama-based
-  # local dev). The stub returns 4096; layers must agree.
-  l5:
-    environment:
-      L5_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
-      L5_EMBED_API_KEY: ""
-      L5_EMBED_DIM: "4096"
-
-  l6:
-    environment:
-      L6_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
-      L6_EMBED_API_KEY: ""
-      L6_EMBED_DIM: "4096"
-
-  l2:
-    environment:
-      PME_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
-
-  compat:
-    environment:
-      NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
-    depends_on:
-      embed-stub:
-        condition: service_healthy
-      l2: { condition: service_started }
-      l5: { condition: service_started }
-      l6: { condition: service_started }

package/packages/memory-engine/docker-compose.yml

@@ -1,255 +0,0 @@
-name: pentatonic-memory-engine
-
-# pentatonic-memory-engine v0.1
-#
-# Drop-in HTTP-compatible replacement for `pentatonic-memory` v0.5,
-# backed by the 7-layer hybrid retrieval engine.
-#
-# Bring up:        docker compose up -d
-# API surface:     http://localhost:8099  (matches pentatonic-memory v0.5)
-# Health:          curl http://localhost:8099/health
-#
-# Layers:
-#   l0 (in-process)  — BM25 over SQLite FTS5 (inside L2 + L6)
-#   l1 (in-process)  — Always-loaded core files (inside L2)
-#   l2  :8031        — HybridRAG proxy (RRF fusion)
-#   l3  :7474        — Knowledge Graph (Neo4j Community)
-#   l4  :8042        — sqlite-vec vector index
-#   l5  :8034        — Milvus comms layer
-#   l6  :8037        — Document store + cross-encoder reranker
-#   compat :8099     — pentatonic-memory v0.5 API translation shim
-#
-#   nv-embed :8041   — NV-Embed-v2 4096d embedding service
-#
-# The nv-embed service requires a GPU. If running without GPU,
-# override NV_EMBED_URL to point at an external embedding endpoint:
-#   - OpenAI's /v1/embeddings (text-embedding-3-large)
-#   - the Pentatonic AI Gateway
-#   - any OpenAI-compat embedding endpoint
-
-x-engine-base: &engine-base
-  restart: unless-stopped
-  networks:
-    - engine-net
-
-services:
-  # --------------------------------------------------------------------
-  # NV-Embed-v2 — 4096-dim embedding service (GPU required)
-  # --------------------------------------------------------------------
-  nv-embed:
-    <<: *engine-base
-    build:
-      context: ./engine/services/nv-embed
-      dockerfile: Dockerfile
-    container_name: pme-nv-embed
-    ports: ["127.0.0.1:${PME_NV_EMBED_PORT:-8041}:8041"]
-    volumes:
-      - pme-nv-embed-cache:/cache
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
-    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8041/health',timeout=3)"]
-      interval: 15s
-      timeout: 10s
-      retries: 20
-      start_period: 120s
-
-  # --------------------------------------------------------------------
-  # L3 — Knowledge Graph (Neo4j Community)
-  # --------------------------------------------------------------------
-  l3:
-    <<: *engine-base
-    image: neo4j:5-community
-    container_name: pme-l3
-    ports:
-      - "127.0.0.1:${PME_L3_HTTP_PORT:-7475}:7474"   # HTTP browser
-      - "127.0.0.1:${PME_L3_BOLT_PORT:-7688}:7687"   # bolt protocol
-    environment:
-      NEO4J_AUTH: ${NEO4J_AUTH:-neo4j/local-dev-pw}
-      NEO4J_PLUGINS: '["apoc"]'
-      # Heap defaults were 512m hardcoded — fine for an empty dev
-      # graph, catastrophic at production scale. A 2026-05-14 prod
-      # incident on a ~10M-relationship KG saw L3 sit at >600% CPU
-      # locked in parallel GC, blocking the L2 write fan-out and
-      # triggering cascading 5xx through L6 and the embed gateway.
-      # The graph fit in RAM fine; the JVM just had nowhere to put
-      # short-lived allocations.
-      #
-      # Defaults now sized for a small-but-realistic local graph
-      # (~1M relationships): 1g heap + 256m initial + 512m pagecache.
-      # Production deployments override via PME_L3_HEAP_MAX etc.
-      # (the AWS overlay sets 4g/1g/1g — see thing-event-system
-      # modules/pentatonic-memory/deploy/docker-compose.aws.yml).
-      NEO4J_dbms_memory_heap_max__size: ${PME_L3_HEAP_MAX:-1g}
-      NEO4J_dbms_memory_heap_initial__size: ${PME_L3_HEAP_INITIAL:-256m}
-      NEO4J_dbms_memory_pagecache_size: ${PME_L3_PAGECACHE:-512m}
-    volumes:
-      - pme-l3-data:/data
-    healthcheck:
-      test: ["CMD-SHELL", "wget -qO- http://localhost:7474 >/dev/null 2>&1 || exit 1"]
-      interval: 10s
-      timeout: 5s
-      retries: 30
-      start_period: 30s
-
-  # --------------------------------------------------------------------
-  # L5 — Qdrant comms layer
-  # --------------------------------------------------------------------
-  l5:
-    <<: *engine-base
-    build:
-      context: ./engine/services
-      dockerfile: l5/Dockerfile
-    container_name: pme-l5
-    # Default 18034 to avoid port collisions on 8034.
-    # Override via PME_L5_PORT for bench setups that intentionally replace it.
-    ports: ["127.0.0.1:${PME_L5_PORT:-18034}:8034"]
-    environment:
-      L5_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
-      L5_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
-      L5_EMBED_API_KEY: ${EMBED_API_KEY:-}
-      L5_EMBED_PROVIDER: ${EMBED_PROVIDER:-openai}
-      L5_EMBED_AUTODETECT: ${EMBED_AUTODETECT:-true}
-      L5_EMBED_DIM: ${EMBED_DIM:-4096}
-      L5_OLLAMA_DIM: ${OLLAMA_DIM:-768}
-      L5_OLLAMA_EMBED_URL: ${L5_OLLAMA_EMBED_URL:-http://host.docker.internal:11434/api/embed}
-      L5_OLLAMA_EMBED_MODEL: ${L5_OLLAMA_EMBED_MODEL:-nomic-embed-text}
-      L5_DB_PATH: /data/comms.db
-      PME_WORKSPACE: /data/workspace
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - pme-l5-data:/data
-
-  # --------------------------------------------------------------------
-  # L6 — Document store + cross-encoder reranker
-  # --------------------------------------------------------------------
-  l6:
-    <<: *engine-base
-    build:
-      context: ./engine/services
-      dockerfile: l6/Dockerfile
-    container_name: pme-l6
-    # Default 18037 to avoid colliding with Spark Core L6 doc-store on 8037.
-    # Override via PME_L6_PORT for bench setups that intentionally replace it.
-    ports: ["127.0.0.1:${PME_L6_PORT:-18037}:8037"]
-    environment:
-      L6_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
-      L6_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
-      L6_EMBED_API_KEY: ${EMBED_API_KEY:-}
-      L6_EMBED_PROVIDER: ${EMBED_PROVIDER:-openai}
-      L6_EMBED_AUTODETECT: ${EMBED_AUTODETECT:-true}
-      L6_EMBED_DIM: ${EMBED_DIM:-4096}
-      L6_DATA_DIR: /data
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - pme-l6-data:/data
-
-  # --------------------------------------------------------------------
-  # L2 — HybridRAG orchestrator (RRF fusion across all layers)
-  # --------------------------------------------------------------------
-  l2:
-    <<: *engine-base
-    build:
-      context: ./engine/services
-      dockerfile: l2/Dockerfile
-    container_name: pme-l2
-    ports: ["127.0.0.1:${PME_L2_PORT:-8131}:8031"]
-    environment:
-      PME_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
-      PME_EMBED_API_KEY: ${EMBED_API_KEY:-}
-      PME_EMBED_PROVIDER: ${EMBED_PROVIDER:-openai}
-      PME_EMBED_AUTODETECT: ${EMBED_AUTODETECT:-true}
-      PME_NV_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
-      PME_NEO4J_URI: bolt://l3:7687
-      PME_NEO4J_PASSWORD: ${NEO4J_PASSWORD:-local-dev-pw}
-      NEO4J_PASSWORD: ${NEO4J_PASSWORD:-local-dev-pw}
-      PME_L5_URL: http://l5:8034
-      PME_L6_URL: http://l6:8037
-      # HyDE off (deliberate — hurts precision on retrieval-graded benches)
-      PME_HYDE_ENABLED: "false"
-      # Cross-encoder reranker on (boosts top-K precision)
-      PME_RERANK_ENABLED: "true"
-      PME_WORKSPACE: /data/workspace
-      PME_QMD_DB: /data/qmd.sqlite
-      PME_MEMORY_DB: /data/workspace.db
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - pme-l2-data:/data
-    depends_on:
-      l3: { condition: service_healthy }
-      l5: { condition: service_started }
-      l6: { condition: service_started }
-
-  # --------------------------------------------------------------------
-  # Compat shim — exposes pentatonic-memory v0.5 API on :8099
-  #
-  # Now routes /search through the L2 7-layer proxy by default. L6-only
-  # bypass (BYPASS_L2_PROXY=1) is still available for isolated testing.
-  # --------------------------------------------------------------------
-  compat:
-    <<: *engine-base
-    build:
-      # Build context is the memory-engine root so the Dockerfile can
-      # COPY both compat/server.py and engine/services/_shared (shared
-      # EmbedClient for /store-batch dedup).
-      context: .
-      dockerfile: compat/Dockerfile
-    container_name: pme-compat
-    ports:
-      - "127.0.0.1:${PME_PORT:-8099}:8099"
-    environment:
-      L0_URL: http://l2:8031
-      L2_PROXY_URL: http://l2:8031
-      L3_KG_URL: http://l3:7474
-      L5_MILVUS_URL: http://l5:8034
-      L6_DOC_URL: http://l6:8037
-      NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
-      # PME_ prefix vars feed the shim's EmbedClient for shared-embed
-      # mode on /store-batch (one embed call across all 3 indexers vs
-      # 3 redundant calls). Match the L2 config block so both clients
-      # hit the same gateway with the same model. Set
-      # PME_SHARE_EMBEDDINGS=false to revert to per-layer embedding.
-      PME_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
-      PME_EMBED_API_KEY: ${EMBED_API_KEY:-}
-      PME_EMBED_PROVIDER: ${EMBED_PROVIDER:-openai}
-      PME_EMBED_AUTODETECT: ${EMBED_AUTODETECT:-true}
-      PME_NV_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
-      PME_SHARE_EMBEDDINGS: ${PME_SHARE_EMBEDDINGS:-true}
-      BYPASS_L2_PROXY: ${BYPASS_L2_PROXY:-0}
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    depends_on:
-      l2: { condition: service_started }
-      l5: { condition: service_started }
-      l6: { condition: service_started }
-    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8099/health',timeout=3)"]
-      interval: 10s
-      timeout: 5s
-      retries: 30
-      # 180s gives L2 enough time to finish Neo4j schema + index creation
-      # on a cold start before compat's healthcheck starts counting failures.
-      # Observed concretely on the v0.9.4 deploy (2026-05-14): L2 took
-      # ~90s to warm up; with start_period: 60s, compat went unhealthy
-      # mid-startup, cloudflared's `depends_on: condition: service_healthy`
-      # failed, and `docker compose up` errored out before wait_for_health
-      # could observe the eventual recovery.
-      start_period: 180s
-
-networks:
-  engine-net:
-
-volumes:
-  pme-nv-embed-cache:
-  pme-l2-data:
-  pme-l3-data:
-  pme-l5-data:
-  pme-l6-data:

package/packages/memory-engine/engine/README.md

@@ -1,52 +0,0 @@
-# engine/
-
-Bundled engine layers for the Pentatonic Memory Engine.
-
-| File | Layer | LOC | Purpose |
-|---|---|---|---|
-| `l2-hybridrag-proxy.py` | L2 | ~1.5k | RRF fusion across all layers, exposed on `:8031` |
-| `l5-comms-layer.py` | L5 | ~0.7k | Milvus comms layer for chat/email/contact/memory collections, exposed on `:8034` |
-| `l6-document-store.py` | L6 | ~1.5k | Document store + cross-encoder reranker, exposed on `:8037` |
-| `services/nv-embed/server.py` | — | ~150 | NV-Embed-v2 4096-dim embedding service, exposed on `:8041` |
-
-## pme_memory SDK
-
-The `pme_memory/` package at the repo root is an installable Python SDK for the L5 communications layer. It provides:
-
-- **store.py** — Milvus client and collection management (chats, emails, contacts, memory)
-- **search.py** — Semantic search across collections
-- **embed.py** — Dual-stack embedding (NV-Embed-v2 primary, Ollama fallback)
-- **indexer.py** — Data ingestion pipeline (JSONL chats, email archives, contacts, memory files)
-- **scoring.py** — Pressure scoring for need signals (recency, novelty, centrality, priority)
-- **synthesis.py** — Deterministic multi-parent artifact merge
-- **artifacts.py** — Append-only artifact DAG store (JSONL)
-- **hygiene.py** — DAG maintenance (dedup, conflict detection, orphan pruning)
-- **health.py** — L5 health check
-- **needs.py** — Need signal indexing
-- **provenance.py** — Lineage visualization
-
-Install: `pip install -e ".[full]"` — CLI: `pme-memory health|stats|index|search|serve`
-
-## KG Extraction Scripts
-
-The `scripts/` directory contains Knowledge Graph population tools:
-
-- **kg-extractor.py** — spaCy + regex entity/relationship extraction from memory files → Neo4j
-- **kg-preflexor-v2.py** — 2-pass concurrent LLM-based extraction via Ollama (14 structured entity types + native graph discovery)
-
-## Where L0, L3 and the embedding service live
-
-- **L0 BM25** — provided by SQLite FTS5; the L2 proxy queries it directly via `sqlite3`. No separate service binary.
-- **L3 Knowledge Graph** — provided by Neo4j Community (free, OSS) running in a sibling container. The proxy queries it via the bolt protocol on `:7687`.
-- **NV-Embed-v2 embedding service** — see `services/nv-embed/` for the Docker context. Exposes the OpenAI-compatible `/v1/embeddings` endpoint on `:8041`.
-
-## Dependencies
-
-Each service has its own `requirements.txt` in `services/<layer>/`. Common heavy deps:
-
-- `pymilvus>=2.6.12` (L5)
-- `sentence-transformers` (L6 reranker, NV-Embed)
-- `httpx`, `fastapi`, `uvicorn` (all)
-- `spacy` (L6 entity extraction)
-
-NV-Embed needs Torch + the model weights (auto-downloaded on first run from Hugging Face).