@pentatonic-ai/ai-agent-sdk 0.6.0 → 0.7.1

package/packages/memory-engine/docker-compose.yml

@@ -0,0 +1,243 @@
+name: pentatonic-memory-engine
+
+# pentatonic-memory-engine v0.1
+#
+# Drop-in HTTP-compatible replacement for `pentatonic-memory` v0.5,
+# backed by the 7-layer hybrid retrieval engine.
+#
+# Bring up:        docker compose up -d
+# API surface:     http://localhost:8099  (matches pentatonic-memory v0.5)
+# Health:          curl http://localhost:8099/health
+#
+# Layers:
+#   l0 (in-process)  — BM25 over SQLite FTS5 (inside L2 + L6)
+#   l1 (in-process)  — Always-loaded core files (inside L2)
+#   l2  :8031        — HybridRAG proxy (RRF fusion)
+#   l3  :7474        — Knowledge Graph (Neo4j Community)
+#   l4  :8042        — sqlite-vec vector index
+#   l5  :8034        — Milvus comms layer
+#   l6  :8037        — Document store + cross-encoder reranker
+#   compat :8099     — pentatonic-memory v0.5 API translation shim
+#
+#   nv-embed :8041   — NV-Embed-v2 4096d embedding service
+#
+# The nv-embed service requires a GPU. If running without GPU,
+# override NV_EMBED_URL to point at an external embedding endpoint:
+#   - OpenAI's /v1/embeddings (text-embedding-3-large)
+#   - the Pentatonic AI Gateway
+#   - any OpenAI-compat embedding endpoint
+
+x-engine-base: &engine-base
+  restart: unless-stopped
+  networks:
+    - engine-net
+
+services:
+  # --------------------------------------------------------------------
+  # NV-Embed-v2 — 4096-dim embedding service (GPU required)
+  # --------------------------------------------------------------------
+  nv-embed:
+    <<: *engine-base
+    build:
+      context: ./engine/services/nv-embed
+      dockerfile: Dockerfile
+    container_name: pme-nv-embed
+    ports: ["127.0.0.1:${PME_NV_EMBED_PORT:-8041}:8041"]
+    volumes:
+      - pme-nv-embed-cache:/cache
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8041/health',timeout=3)"]
+      interval: 15s
+      timeout: 10s
+      retries: 20
+      start_period: 120s
+
+  # --------------------------------------------------------------------
+  # L3 — Knowledge Graph (Neo4j Community)
+  # --------------------------------------------------------------------
+  l3:
+    <<: *engine-base
+    image: neo4j:5-community
+    container_name: pme-l3
+    ports:
+      - "127.0.0.1:${PME_L3_HTTP_PORT:-7475}:7474"   # HTTP browser
+      - "127.0.0.1:${PME_L3_BOLT_PORT:-7688}:7687"   # bolt protocol
+    environment:
+      NEO4J_AUTH: ${NEO4J_AUTH:-neo4j/local-dev-pw}
+      NEO4J_PLUGINS: '["apoc"]'
+      NEO4J_dbms_memory_heap_max__size: 512m
+    volumes:
+      - pme-l3-data:/data
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:7474 >/dev/null 2>&1 || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+      start_period: 30s
+
+  # --------------------------------------------------------------------
+  # L4 — sqlite-vec sidecar
+  # --------------------------------------------------------------------
+  l4:
+    <<: *engine-base
+    build:
+      context: ./engine/services/l4
+      dockerfile: Dockerfile
+    container_name: pme-l4
+    # Default 18042 to avoid port collisions on 8042.
+    # Override via PME_L4_PORT for bench setups that intentionally replace it.
+    ports: ["127.0.0.1:${PME_L4_PORT:-18042}:8042"]
+    environment:
+      L4_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
+      L4_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
+      L4_EMBED_API_KEY: ${EMBED_API_KEY:-}
+      L4_EMBED_DIM: ${EMBED_DIM:-4096}
+      L4_DB_PATH: /data/vec.db
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - pme-l4-data:/data
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request,sys; urllib.request.urlopen('http://localhost:8042/health',timeout=3)"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  # --------------------------------------------------------------------
+  # L5 — Qdrant comms layer
+  # --------------------------------------------------------------------
+  l5:
+    <<: *engine-base
+    build:
+      context: ./engine/services/l5
+      dockerfile: Dockerfile
+    container_name: pme-l5
+    # Default 18034 to avoid port collisions on 8034.
+    # Override via PME_L5_PORT for bench setups that intentionally replace it.
+    ports: ["127.0.0.1:${PME_L5_PORT:-18034}:8034"]
+    environment:
+      L5_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
+      L5_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
+      L5_EMBED_API_KEY: ${EMBED_API_KEY:-}
+      L5_EMBED_DIM: ${EMBED_DIM:-4096}
+      L5_OLLAMA_DIM: ${OLLAMA_DIM:-768}
+      L5_OLLAMA_EMBED_URL: ${L5_OLLAMA_EMBED_URL:-http://host.docker.internal:11434/api/embed}
+      L5_OLLAMA_EMBED_MODEL: ${L5_OLLAMA_EMBED_MODEL:-nomic-embed-text}
+      L5_DB_PATH: /data/comms.db
+      PME_WORKSPACE: /data/workspace
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - pme-l5-data:/data
+
+  # --------------------------------------------------------------------
+  # L6 — Document store + cross-encoder reranker
+  # --------------------------------------------------------------------
+  l6:
+    <<: *engine-base
+    build:
+      context: ./engine/services/l6
+      dockerfile: Dockerfile
+    container_name: pme-l6
+    # Default 18037 to avoid colliding with Spark Core L6 doc-store on 8037.
+    # Override via PME_L6_PORT for bench setups that intentionally replace it.
+    ports: ["127.0.0.1:${PME_L6_PORT:-18037}:8037"]
+    environment:
+      L6_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
+      L6_EMBED_MODEL: ${EMBED_MODEL_NAME:-nv-embed-v2}
+      L6_EMBED_API_KEY: ${EMBED_API_KEY:-}
+      L6_EMBED_DIM: ${EMBED_DIM:-4096}
+      L6_DATA_DIR: /data
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - pme-l6-data:/data
+
+  # --------------------------------------------------------------------
+  # L2 — HybridRAG orchestrator (RRF fusion across all layers)
+  # --------------------------------------------------------------------
+  l2:
+    <<: *engine-base
+    build:
+      context: ./engine/services/l2
+      dockerfile: Dockerfile
+    container_name: pme-l2
+    ports: ["127.0.0.1:${PME_L2_PORT:-8131}:8031"]
+    environment:
+      PME_NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
+      PME_NEO4J_URI: bolt://l3:7687
+      PME_NEO4J_PASSWORD: ${NEO4J_PASSWORD:-local-dev-pw}
+      NEO4J_PASSWORD: ${NEO4J_PASSWORD:-local-dev-pw}
+      PME_L5_URL: http://l5:8034
+      PME_L6_URL: http://l6:8037
+      # HyDE off (deliberate — hurts precision on retrieval-graded benches)
+      PME_HYDE_ENABLED: "false"
+      # Cross-encoder reranker on (boosts top-K precision)
+      PME_RERANK_ENABLED: "true"
+      PME_WORKSPACE: /data/workspace
+      PME_QMD_DB: /data/qmd.sqlite
+      PME_MEMORY_DB: /data/workspace.db
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - pme-l2-data:/data
+    depends_on:
+      l3: { condition: service_healthy }
+      l5: { condition: service_started }
+      l6: { condition: service_started }
+
+  # --------------------------------------------------------------------
+  # Compat shim — exposes pentatonic-memory v0.5 API on :8099
+  #
+  # Now routes /search through the L2 7-layer proxy by default. L6-only
+  # bypass (BYPASS_L2_PROXY=1) is still available for isolated testing.
+  # --------------------------------------------------------------------
+  compat:
+    <<: *engine-base
+    build:
+      context: ./compat
+      dockerfile: Dockerfile
+    container_name: pme-compat
+    ports:
+      - "127.0.0.1:${PME_PORT:-8099}:8099"
+    environment:
+      L0_URL: http://l2:8031
+      L2_PROXY_URL: http://l2:8031
+      L3_KG_URL: http://l3:7474
+      L4_VEC_URL: http://l4:8042
+      L5_MILVUS_URL: http://l5:8034
+      L6_DOC_URL: http://l6:8037
+      NV_EMBED_URL: ${NV_EMBED_URL:-http://host.docker.internal:8041/v1/embeddings}
+      CLIENT_ID: ${CLIENT_ID:-default}
+      BYPASS_L2_PROXY: ${BYPASS_L2_PROXY:-0}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    depends_on:
+      l2: { condition: service_started }
+      l4: { condition: service_started }
+      l5: { condition: service_started }
+      l6: { condition: service_started }
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8099/health',timeout=3)"]
+      interval: 10s
+      timeout: 5s
+      retries: 30
+      start_period: 60s
+
+networks:
+  engine-net:
+
+volumes:
+  pme-nv-embed-cache:
+  pme-l2-data:
+  pme-l3-data:
+  pme-l4-data:
+  pme-l5-data:
+  pme-l6-data:

package/packages/memory-engine/engine/README.md

@@ -0,0 +1,52 @@
+# engine/
+
+Bundled engine layers for the Pentatonic Memory Engine.
+
+| File | Layer | LOC | Purpose |
+|---|---|---|---|
+| `l2-hybridrag-proxy.py` | L2 | ~1.5k | RRF fusion across all layers, exposed on `:8031` |
+| `l5-comms-layer.py` | L5 | ~0.7k | Milvus comms layer for chat/email/contact/memory collections, exposed on `:8034` |
+| `l6-document-store.py` | L6 | ~1.5k | Document store + cross-encoder reranker, exposed on `:8037` |
+| `services/nv-embed/server.py` | — | ~150 | NV-Embed-v2 4096-dim embedding service, exposed on `:8041` |
+
+## pme_memory SDK
+
+The `pme_memory/` package at the repo root is an installable Python SDK for the L5 communications layer. It provides:
+
+- **store.py** — Milvus client and collection management (chats, emails, contacts, memory)
+- **search.py** — Semantic search across collections
+- **embed.py** — Dual-stack embedding (NV-Embed-v2 primary, Ollama fallback)
+- **indexer.py** — Data ingestion pipeline (JSONL chats, email archives, contacts, memory files)
+- **scoring.py** — Pressure scoring for need signals (recency, novelty, centrality, priority)
+- **synthesis.py** — Deterministic multi-parent artifact merge
+- **artifacts.py** — Append-only artifact DAG store (JSONL)
+- **hygiene.py** — DAG maintenance (dedup, conflict detection, orphan pruning)
+- **health.py** — L5 health check
+- **needs.py** — Need signal indexing
+- **provenance.py** — Lineage visualization
+
+Install: `pip install -e ".[full]"` — CLI: `pme-memory health|stats|index|search|serve`
+
+## KG Extraction Scripts
+
+The `scripts/` directory contains Knowledge Graph population tools:
+
+- **kg-extractor.py** — spaCy + regex entity/relationship extraction from memory files → Neo4j
+- **kg-preflexor-v2.py** — 2-pass concurrent LLM-based extraction via Ollama (14 structured entity types + native graph discovery)
+
+## Where L0, L3 and the embedding service live
+
+- **L0 BM25** — provided by SQLite FTS5; the L2 proxy queries it directly via `sqlite3`. No separate service binary.
+- **L3 Knowledge Graph** — provided by Neo4j Community (free, OSS) running in a sibling container. The proxy queries it via the bolt protocol on `:7687`.
+- **NV-Embed-v2 embedding service** — see `services/nv-embed/` for the Docker context. Exposes the OpenAI-compatible `/v1/embeddings` endpoint on `:8041`.
+
+## Dependencies
+
+Each service has its own `requirements.txt` in `services/<layer>/`. Common heavy deps:
+
+- `pymilvus>=2.6.12` (L5)
+- `sentence-transformers` (L6 reranker, NV-Embed)
+- `httpx`, `fastapi`, `uvicorn` (all)
+- `spacy` (L6 entity extraction)
+
+NV-Embed needs Torch + the model weights (auto-downloaded on first run from Hugging Face).