@pentatonic-ai/ai-agent-sdk 0.9.6 → 0.10.1

package/README.md

@@ -114,7 +114,7 @@ sudo systemctl restart ollama
 
 ```bash
 git clone https://github.com/Pentatonic-Ltd/ai-agent-sdk.git
-cd ai-agent-sdk/packages/memory-engine
+cd ai-agent-sdk/packages/memory-engine-v2
 
 # Default .env points at Ollama on the host. Edit if your Ollama is
 # elsewhere or you want to use a higher-quality model (e.g. mxbai-embed-large
@@ -238,7 +238,7 @@ await adapter.init();
 await adapter.ingestChunk('User prefers dark mode', { kind: 'note' });
 ```
 
-For raw `/search` and `/store`, just `fetch()` against `${engineUrl}/search` etc. The wire format is documented in `packages/memory-engine/MIGRATION.md`.
+For raw `/search` and `/store`, just `fetch()` against `${engineUrl}/search` etc. The wire format is documented in `packages/memory-engine-v2/MIGRATION.md`.
 
 ---
 
@@ -504,7 +504,7 @@ const { content, model, usage, toolCalls } = normalizeResponse(openaiResponse);
 
 Thin HTTP client for the memory engine. `config = { engineUrl, arena, apiKey? }`. Returns `{ ingestChunk(content, metadata), deleteByCorpusFile(repoAbs, relPath), init() }`. See [Use as a library](#use-as-a-library).
 
-For raw `/store` / `/search` calls, just `fetch()` against `${engineUrl}` directly — the wire format is documented in `packages/memory-engine/MIGRATION.md`.
+For raw `/store` / `/search` calls, just `fetch()` against `${engineUrl}` directly — the wire format is documented in `packages/memory-engine-v2/MIGRATION.md`.
 
 ---
 

package/bin/cli.js

@@ -78,7 +78,7 @@ function ask(question) {
 // in favour of:
 //   - `tes config local`  → writes the plugin config + prints engine
 //                           bring-up instructions
-//   - `cd packages/memory-engine && docker compose up -d` → runs the
+//   - `cd packages/memory-engine-v2 && docker compose up -d` → runs the
 //                           actual engine
 // `ask` is kept for any future interactive prompts.
 

package/bin/commands/config.js

@@ -131,7 +131,7 @@ async function runConfigLocal(opts) {
   log("");
   log("  2. Bring up the engine docker stack:");
   log("");
-  log("       cd packages/memory-engine");
+  log("       cd packages/memory-engine-v2");
   log("       cp .env.example .env             # if no .env yet");
   log("       # edit .env if you want a different embedding model/dim");
   log("       docker compose up -d --scale nv-embed=0");

package/dist/index.cjs

@@ -878,7 +878,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.9.6";
+var VERSION = "0.10.1";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/dist/index.js

@@ -847,7 +847,7 @@ function fireAndForgetEmit(clientConfig, sessionOpts, messages, result, model) {
 }
 
 // src/telemetry.js
-var VERSION = "0.9.6";
+var VERSION = "0.10.1";
 var TELEMETRY_URL = "https://sdk-telemetry.philip-134.workers.dev";
 function machineId() {
   const raw = typeof process !== "undefined" ? `${process.env?.USER || process.env?.USERNAME || "u"}:${process.platform || "x"}:${process.arch || "x"}` : "browser";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.9.6",
+  "version": "0.10.1",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -27,7 +27,7 @@
     "src",
     "bin",
     "packages/memory",
-    "packages/memory-engine",
+    "packages/memory-engine-v2",
     "packages/doctor",
     "build.js",
     "README.md",

package/packages/doctor/src/checks/local-memory.js

@@ -2,7 +2,7 @@
  * Local memory engine checks.
  *
  * Targets the engine stack started by:
- *   cd packages/memory-engine && docker compose up -d
+ *   cd packages/memory-engine-v2 && docker compose up -d
  *
  * The engine exposes a compat HTTP shim on port 8099 (or whatever
  * memory_url is set to in the user's plugin config). All checks are
@@ -202,7 +202,7 @@ function checkEmbeddingPath() {
         }
         return {
           ok: false,
-          msg: `nv_embed=${nv} — L4/L5/L6 indexing will fail. Check NV_EMBED_URL in packages/memory-engine/.env`,
+          msg: `nv_embed=${nv} — L4/L5/L6 indexing will fail. Check NV_EMBED_URL in packages/memory-engine-v2/.env`,
           detail: { nv_embed: nv },
         };
       } catch (err) {

package/packages/memory/README.md

@@ -4,11 +4,11 @@
 >
 > This package is the **legacy** single-process MCP server backed by
 > PostgreSQL + pgvector + Ollama. It's superseded by the **7-layer
-> memory engine** at [`packages/memory-engine/`](../memory-engine/),
+> memory engine** at [`packages/memory-engine-v2/`](../memory-engine/),
 > which is what the top-level SDK README walks users into and what
 > TES production runs.
 >
-> | | This package (legacy) | `packages/memory-engine/` (current) |
+> | | This package (legacy) | `packages/memory-engine-v2/` (current) |
 > |---|---|---|
 > | Wire | MCP over stdio | HTTP (`/store`, `/search`, ...) |
 > | Storage | One Postgres table, one embedding | 7 layers fused via RRF |

package/packages/memory/openclaw-plugin/README.md

@@ -7,7 +7,7 @@ Persistent, searchable memory for OpenClaw. Local (Docker + Ollama) or hosted (P
 > The `database_url` / `embedding_url` / `llm_url` config fields shown
 > in this README configure the **legacy** `packages/memory/` Postgres+
 > Ollama+pgvector MCP server, which is being retired in favour of the
-> 7-layer memory engine at `packages/memory-engine/`. Both backends
+> 7-layer memory engine at `packages/memory-engine-v2/`. Both backends
 > still work; the legacy one will be removed in v1.0.
 >
 > **For new installs, prefer:**
@@ -23,7 +23,7 @@ Persistent, searchable memory for OpenClaw. Local (Docker + Ollama) or hosted (P
 > ```
 >
 > …with the engine brought up via `docker compose up -d` from
-> `packages/memory-engine/`. See the
+> `packages/memory-engine-v2/`. See the
 > [top-level SDK README](../../../README.md#local-self-hosted) for the
 > walkthrough.
 

package/packages/memory/openclaw-plugin/openclaw.plugin.json

@@ -17,7 +17,7 @@
       "memory_url": {
         "type": "string",
         "default": "http://localhost:8099",
-        "description": "Memory engine HTTP URL (local mode). Default 8099 = packages/memory-engine compat shim. (3333 was the legacy single-process MCP server port — deprecated.)"
+        "description": "Memory engine HTTP URL (local mode). Default 8099 = packages/memory-engine-v2 compat shim. (3333 was the legacy single-process MCP server port — deprecated.)"
       },
       "database_url": {
         "type": "string",

package/packages/memory/src/server.js

@@ -42,14 +42,14 @@ process.on("unhandledRejection", (err) => {
 
 // Deprecation notice — see packages/memory/README.md for context.
 // This MCP server (Postgres+pgvector+Ollama, single-process) is being
-// retired in favour of the 7-layer engine at packages/memory-engine/.
+// retired in favour of the 7-layer engine at packages/memory-engine-v2/.
 // Targeted for removal in v1.0; in the meantime everything keeps
 // working. Print once on startup so operators see the signal in logs
 // without flooding the conversation surface.
 if (process.env.PENTATONIC_DEPRECATION_QUIET !== "1") {
   process.stderr.write(
     "[memory-server] DEPRECATED: this server (Postgres+pgvector+Ollama MCP) " +
-      "is superseded by the 7-layer memory engine at packages/memory-engine/. " +
+      "is superseded by the 7-layer memory engine at packages/memory-engine-v2/. " +
       "Existing deployments keep working; removal targeted for v1.0. " +
       "See README → Memory → Local for the migration path. " +
       "Suppress this warning with PENTATONIC_DEPRECATION_QUIET=1.\n"

package/packages/memory-engine-v2/.env.example

@@ -0,0 +1,30 @@
+# pentatonic-memory-engine v2 environment.
+# Copy to .env on the engine box; never commit a populated copy.
+
+# --- Embed gateway (same as v1) -----------------------------------
+NV_EMBED_URL=https://lambda-gateway.pentatonic.com/v1/embed
+PENTATONIC_AI_GATEWAY_KEY=<rotate-on-deploy>
+NV_EMBED_PROVIDER=pentatonic-gateway
+
+# --- Postgres (org-model) -----------------------------------------
+PME_V2_PG_USER=pme
+PME_V2_PG_PASSWORD=<random-strong-password>
+PME_V2_PG_DB=org_model
+PME_V2_PG_DSN=postgresql://pme:<password>@org-model:5432/org_model
+
+# --- Ports (host-side) --------------------------------------------
+# v1 runs on 8099; if you want v2 alongside, give it a different port
+# until the cutover. Compose maps these to internal 8099 / 5432 / 6333.
+PME_V2_COMPAT_PORT=8199                 # v2 alongside v1
+PME_V2_ORG_MODEL_PORT=15433
+PME_V2_QDRANT_HTTP_PORT=16335
+PME_V2_QDRANT_GRPC_PORT=16336
+
+# --- Embedding dimension ------------------------------------------
+PME_V2_EMBED_DIM=4096
+
+# --- Cloudflared (optional, only set if exposing v2 publicly) -----
+# TUNNEL_TOKEN_V2=<cut-a-new-tunnel-or-reuse-v1>
+
+# --- LLM endpoint (extractor-async; stub mode if empty) -----------
+# PME_V2_LLM_ENDPOINT=https://lambda-gateway.pentatonic.com/v1/chat/completions

package/packages/memory-engine-v2/README.md

@@ -0,0 +1,125 @@
+# pentatonic-memory-engine v2
+
+Keystone-first rebuild. Three stores, not seven. Wire-compatible with
+v1's compat shim so TES can flip via a single env var.
+
+## Status
+
+- Infrastructure scaffolded (this directory)
+- Not yet released to npm; not yet CI'd
+- Currently deployed to AWS engine box at `/opt/engine-v2` via rsync
+- Real LLM extraction (extractor-async) stubbed pending keystone spec
+  (#285 extraction-objectives.md)
+
+## Layout
+
+```
+packages/memory-engine-v2/
+  docker-compose.yml            # 5 services: org-model, vector-index,
+                                # extractor-sync, extractor-async, compat
+  docker-compose.aws.yml        # AWS overlay (bind-mount volumes, prod env)
+  compat/                       # FastAPI v1-wire-compatible shim
+    Dockerfile + server.py + requirements.txt
+  extractor-sync/               # Deterministic per-source extraction
+    Dockerfile + server.py + requirements.txt
+  extractor-async/              # LLM distillation worker (stub for now)
+    Dockerfile + worker.py + requirements.txt
+  org-model/migrations/         # Postgres schema
+    001_init.sql                # Initial schema with provenance cols
+```
+
+## What's in here vs v1
+
+| v1 has | v2 has | Notes |
+|---|---|---|
+| L0 (sqlite FTS5) inside L2 | not yet — falls back to vector-only on /search | Keyword fallback comes once the typed router lands |
+| L2 HybridRAG proxy | compat shim does the routing | Single shim, no separate proxy service |
+| L3 Neo4j knowledge graph | recursive-CTE on org-model (pending spike #278) | If quality acceptable, Neo4j retires entirely |
+| L4 sqlite-vec sidecar | gone | Already removed in SDK PR #51 |
+| L4 QMD (vectors as JSON text in SQLite) | gone | Replaced by Qdrant + content-hash IDs |
+| L5 Milvus chats | folded into Qdrant via source_kind=chat | One vector store, payload-filtered |
+| L6 Milvus doc-store | folded into Qdrant via source_kind=doc | Same |
+| Compat shim | compat (same wire contract) | Same external surface, totally new internals |
+| nv-embed service | external (NV_EMBED_URL points at GH200) | Same as v1's AWS deploy |
+| HyDE per query | gone | Confidence-gated query rewrite if needed later |
+
+## Deploy (AWS engine box, via rsync — no CI)
+
+```bash
+# From SDK repo on your laptop:
+rsync -av --delete \
+  packages/memory-engine-v2/ \
+  ubuntu@<engine-host>:/opt/engine-v2/
+
+# On the box:
+ssh ubuntu@<engine-host>
+cd /opt/engine-v2
+# CRITICAL: production deploys MUST include `-f docker-compose.aws.yml`.
+# The base `docker-compose.yml` uses named docker volumes (anonymous,
+# project-scoped). The AWS overlay re-points those to bind mounts under
+# `/var/lib/pme-v2/<service>` — that's where the actual production data
+# lives. Omitting the overlay creates fresh empty volumes and effectively
+# "hides" the real data until the overlay is re-applied. The data is not
+# lost; the container just mounts the wrong path.
+sudo docker compose --env-file .env \
+  -f docker-compose.yml -f docker-compose.aws.yml \
+  up -d --build
+
+# Verify:
+curl http://localhost:8099/health        # cheap liveness
+curl http://localhost:8099/health/deep   # round-trips all stores
+
+# Single-service redeploy (e.g. just rebuild extractor-async after a
+# code change). --no-deps avoids touching other healthy services.
+# Same overlay rule applies here.
+sudo docker compose --env-file .env \
+  -f docker-compose.yml -f docker-compose.aws.yml \
+  up -d --no-deps --build extractor-async
+```
+
+Required env vars (set in `.env` next to the compose file):
+
+```bash
+NV_EMBED_URL=https://lambda-gateway.pentatonic.com/v1/embed
+PENTATONIC_AI_GATEWAY_KEY=<the key>
+NV_EMBED_PROVIDER=pentatonic-gateway
+PME_V2_PG_PASSWORD=<random>
+PME_V2_PG_DSN=postgresql://pme:<password>@org-model:5432/org_model
+
+# Optional:
+PME_V2_LLM_ENDPOINT=https://lambda-gateway.pentatonic.com/v1/chat/completions
+PME_V2_COMPAT_PORT=8099  # match v1 so the env-var flip is one line
+```
+
+## Cutover from v1 → v2
+
+```bash
+# On TES wrangler secrets:
+wrangler pages secret put MEMORY_ENGINE_URL --project-name <project>
+# Enter: https://memory-engine-v2.thingeventsystem.ai (or whichever
+# cloudflared tunnel hostname routes to v2's compat:8099)
+```
+
+Then v1's containers can be stopped:
+
+```bash
+cd /opt/engine
+sudo docker compose stop l2 l3 l5 l6 compat
+```
+
+Roll back by reverting the wrangler secret.
+
+## Open work (not in this scaffold)
+
+1. **Keystone (PR #285)** — extraction-objectives.md. Per-source typed
+   schemas, κ / false-omission bars. Drives the real extractor-async
+   prompts. Until this lands, extractor-async is a stub that drains
+   the queue at no-op speed.
+2. **Typed router** in /search — intent classify → org-model and/or
+   vector-index → fuse. Currently /search is vector-only.
+3. **BM25 fallback** — sqlite FTS5 service or inline in compat.
+4. **Graph spike (#278)** — recursive-CTE personFacets/peopleList over
+   org-model. Outcome decides whether Neo4j retires.
+5. **Memory-proxy Worker** (separate from v2) — removes the public
+   tunnel surface. Design doc at
+   `thing-event-system/modules/pentatonic-memory/proxy/DESIGN.md`.

package/packages/memory-engine-v2/compat/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY server.py .
+
+EXPOSE 8099
+CMD ["uvicorn", "server:app", "--host", "0.0.0.0", "--port", "8099", "--workers", "4"]

package/packages/memory-engine-v2/compat/requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.115.0
+uvicorn[standard]==0.32.0
+psycopg[binary,pool]==3.2.3
+httpx==0.27.2
+qdrant-client==1.12.1
+pydantic==2.9.2