@pentatonic-ai/ai-agent-sdk 0.7.5 → 0.7.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.7.5",
+  "version": "0.7.7",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -15,7 +15,9 @@
     "./memory/openclaw": "./packages/memory/src/openclaw/index.js",
     "./doctor": "./packages/doctor/src/index.js",
     "./memory/hosted": "./packages/memory/src/hosted.js",
-    "./memory/corpus": "./packages/memory/src/corpus/index.js"
+    "./memory/corpus": "./packages/memory/src/corpus/index.js",
+    "./memory/engine": "./packages/memory/src/engine.js",
+    "./memory/engine-layers": "./packages/memory/src/engine-layers.js"
   },
   "bin": {
     "ai-agent-sdk": "./bin/cli.js"

package/packages/memory/src/__tests__/engine-layers.test.js

@@ -0,0 +1,62 @@
+import { describe, it, expect } from "@jest/globals";
+import {
+  STATIC_LAYER_TYPES,
+  buildStaticLayers,
+  layerIdToType,
+} from "../engine-layers.js";
+
+describe("engine-layers", () => {
+  it("STATIC_LAYER_TYPES is the documented 4-layer surface", () => {
+    expect(STATIC_LAYER_TYPES).toEqual([
+      "episodic",
+      "semantic",
+      "procedural",
+      "working",
+    ]);
+  });
+
+  describe("buildStaticLayers", () => {
+    it("returns one row per layer type with stable id shape", () => {
+      const rows = buildStaticLayers("acme");
+      expect(rows.map((r) => r.id)).toEqual([
+        "ml_acme_episodic",
+        "ml_acme_semantic",
+        "ml_acme_procedural",
+        "ml_acme_working",
+      ]);
+      for (const r of rows) {
+        expect(r.client_id).toBe("acme");
+        expect(r.layer_type).toBe(r.name);
+        expect(r.is_active).toBe(true);
+        expect(r.memory_count).toBe(null);
+        expect(r.capacity).toBe(null);
+        expect(r.decay_policy).toBe(null);
+      }
+    });
+
+    it("created_at is install-time stable (epoch) so dashboards don't churn", () => {
+      const a = buildStaticLayers("acme");
+      const b = buildStaticLayers("acme");
+      expect(a[0].created_at).toBe(b[0].created_at);
+      expect(a[0].created_at).toBe(new Date(0).toISOString());
+    });
+  });
+
+  describe("layerIdToType", () => {
+    it("extracts the type suffix", () => {
+      expect(layerIdToType("ml_acme_episodic")).toBe("episodic");
+      expect(layerIdToType("ml_acme_semantic")).toBe("semantic");
+      expect(layerIdToType("ml_acme_procedural")).toBe("procedural");
+      expect(layerIdToType("ml_acme_working")).toBe("working");
+    });
+
+    it("falls back to episodic for unknown / malformed ids", () => {
+      expect(layerIdToType("ml_acme_other")).toBe("episodic");
+      expect(layerIdToType("garbage")).toBe("episodic");
+      expect(layerIdToType("")).toBe("episodic");
+      expect(layerIdToType(null)).toBe("episodic");
+      expect(layerIdToType(undefined)).toBe("episodic");
+      expect(layerIdToType(42)).toBe("episodic");
+    });
+  });
+});

package/packages/memory/src/__tests__/engine.test.js

@@ -0,0 +1,214 @@
+import { describe, it, expect, beforeEach, jest } from "@jest/globals";
+import {
+  fetchEngine,
+  engineStore,
+  engineSearch,
+  engineForget,
+  DEFAULT_ENGINE_URL,
+} from "../engine.js";
+
+describe("engine HTTP client", () => {
+  let originalFetch;
+  let calls;
+
+  beforeEach(() => {
+    originalFetch = global.fetch;
+    calls = [];
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  function mockOk(responseBody) {
+    global.fetch = jest.fn(async (url, init) => {
+      calls.push({ url, init });
+      return {
+        ok: true,
+        status: 200,
+        json: async () => responseBody,
+      };
+    });
+  }
+
+  function mockHttp(status, text) {
+    global.fetch = jest.fn(async (url, init) => {
+      calls.push({ url, init });
+      return {
+        ok: false,
+        status,
+        text: async () => text,
+        json: async () => ({}),
+      };
+    });
+  }
+
+  function mockNetworkErr(message) {
+    global.fetch = jest.fn(async () => {
+      throw new Error(message);
+    });
+  }
+
+  describe("fetchEngine", () => {
+    it("uses provided engineUrl + path verbatim", async () => {
+      mockOk({ ok: true });
+      await fetchEngine("https://x.example", "/store", { a: 1 });
+      expect(calls[0].url).toBe("https://x.example/store");
+      expect(calls[0].init.method).toBe("POST");
+      expect(calls[0].init.headers["content-type"]).toBe("application/json");
+      expect(JSON.parse(calls[0].init.body)).toEqual({ a: 1 });
+    });
+
+    it("falls back to DEFAULT_ENGINE_URL when engineUrl is empty", async () => {
+      mockOk({ ok: true });
+      await fetchEngine("", "/health", {});
+      expect(calls[0].url).toBe(`${DEFAULT_ENGINE_URL}/health`);
+    });
+
+    it("throws engine_<status> with detail on HTTP non-2xx", async () => {
+      mockHttp(503, "engine down");
+      await expect(fetchEngine("https://x", "/store", {})).rejects.toMatchObject({
+        message: "engine_503",
+        detail: "engine down",
+      });
+    });
+
+    it("throws engine_network on transport failure", async () => {
+      mockNetworkErr("ECONNREFUSED");
+      await expect(fetchEngine("https://x", "/store", {})).rejects.toThrow(
+        /engine_network: ECONNREFUSED/
+      );
+    });
+  });
+
+  describe("engineStore", () => {
+    it("builds canonical /store body with arena=clientId", async () => {
+      mockOk({ id: "abc", content: "hello", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", {
+        clientId: "acme",
+        content: "hello",
+        layerType: "episodic",
+        actorUserId: "u-1",
+        metadata: { kind: "note" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/store");
+      expect(body).toEqual({
+        content: "hello",
+        metadata: {
+          kind: "note",
+          arena: "acme",
+          layer_type: "episodic",
+          actor_user_id: "u-1",
+        },
+      });
+    });
+
+    it("omits layer_type and actor_user_id when not provided", async () => {
+      mockOk({ id: "x", content: "x", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", { clientId: "acme", content: "x" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata).toEqual({ arena: "acme" });
+    });
+
+    it("does NOT let caller override arena via metadata", async () => {
+      mockOk({ id: "x", content: "x", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", {
+        clientId: "acme",
+        content: "x",
+        // attempted hostile arena spoof:
+        metadata: { arena: "tenant-b" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata.arena).toBe("acme");
+    });
+
+    it("rejects missing clientId or content", async () => {
+      mockOk({});
+      await expect(engineStore("https://e", { content: "x" })).rejects.toThrow(
+        /clientId/
+      );
+      await expect(engineStore("https://e", { clientId: "a" })).rejects.toThrow(
+        /content/
+      );
+    });
+  });
+
+  describe("engineSearch", () => {
+    it("builds canonical /search body and forwards arena/limit/min_score", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hello",
+        limit: 5,
+        minScore: 0.5,
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/search");
+      expect(body).toEqual({
+        arena: "acme",
+        query: "hello",
+        limit: 5,
+        min_score: 0.5,
+      });
+    });
+
+    it("includes metadata_filter only when non-empty", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hi",
+        metadataFilter: { kind: "note" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata_filter).toEqual({ kind: "note" });
+
+      calls.length = 0;
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hi",
+        metadataFilter: {},
+      });
+      const body2 = JSON.parse(calls[0].init.body);
+      expect(body2.metadata_filter).toBeUndefined();
+    });
+
+    it("uses defaults for limit + minScore", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", { clientId: "acme", query: "hi" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.limit).toBe(10);
+      expect(body.min_score).toBe(0.3);
+    });
+  });
+
+  describe("engineForget", () => {
+    it("forwards id when provided", async () => {
+      mockOk({ deleted: 1 });
+      await engineForget("https://e", { clientId: "acme", id: "abc" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/forget");
+      expect(body).toEqual({ arena: "acme", id: "abc" });
+    });
+
+    it("forwards metadata_contains when provided", async () => {
+      mockOk({ deleted: 5 });
+      await engineForget("https://e", {
+        clientId: "acme",
+        metadataContains: { source_repo: "monorepo" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body).toEqual({
+        arena: "acme",
+        metadata_contains: { source_repo: "monorepo" },
+      });
+    });
+
+    it("requires id or metadataContains", async () => {
+      await expect(
+        engineForget("https://e", { clientId: "acme" })
+      ).rejects.toThrow(/id or metadataContains/);
+    });
+  });
+});

package/packages/memory/src/engine-layers.js

@@ -0,0 +1,72 @@
+/**
+ * 4-layer projection over the memory engine.
+ *
+ * The engine's internal architecture has 7 storage layers (L0 BM25,
+ * L1 system files, L2 HybridRAG, L3 Neo4j KG, L4 sqlite-vec, L5 Milvus
+ * comms, L6 doc-store). The user-facing surface preferred by the SDK
+ * and the dashboard is a simpler 4-layer view: episodic / semantic /
+ * procedural / working.
+ *
+ * The engine itself doesn't model these — it stores everything as
+ * chunks tagged with `arena` + arbitrary `metadata`. The 4-layer view
+ * is synthesised here at the host boundary so callers see a familiar
+ * shape regardless of the storage architecture below.
+ *
+ * @example
+ *   import {
+ *     buildStaticLayers,
+ *     layerIdToType,
+ *     STATIC_LAYER_TYPES,
+ *   } from "@pentatonic-ai/ai-agent-sdk/memory/engine-layers";
+ *
+ *   // Resolver for a `memoryLayers(clientId)` field:
+ *   const layers = buildStaticLayers("acme");
+ *   // → 4 entries with stable ids ml_acme_episodic, ml_acme_semantic, ...
+ */
+
+export const STATIC_LAYER_TYPES = [
+  "episodic",
+  "semantic",
+  "procedural",
+  "working",
+];
+
+/**
+ * Synthesise the 4-layer surface for a tenant.
+ *
+ * Returned shape matches the legacy `memory_layers` row schema so
+ * existing GraphQL types and dashboard code work unchanged. Per-layer
+ * memory counts are null because the engine doesn't track them.
+ *
+ * @param {string} clientId
+ * @returns {Array<{id: string, client_id: string, name: string, layer_type: string, capacity: null, decay_policy: null, is_active: true, created_at: string, memory_count: null}>}
+ */
+export function buildStaticLayers(clientId) {
+  const nowIso = new Date(0).toISOString(); // stable, install-time
+  return STATIC_LAYER_TYPES.map((type) => ({
+    id: `ml_${clientId}_${type}`,
+    client_id: clientId,
+    name: type,
+    layer_type: type,
+    capacity: null,
+    decay_policy: null,
+    is_active: true,
+    created_at: nowIso,
+    memory_count: null,
+  }));
+}
+
+/**
+ * Extract the layer type from a layer id of the form
+ * `ml_<clientId>_<type>`. Falls back to "episodic" for inputs that
+ * don't match — keeps writes alive when the SDK is ahead of the host
+ * on layer naming.
+ *
+ * @param {string} layerId
+ * @returns {"episodic" | "semantic" | "procedural" | "working"}
+ */
+export function layerIdToType(layerId) {
+  if (typeof layerId !== "string") return "episodic";
+  const last = layerId.split("_").pop();
+  return STATIC_LAYER_TYPES.includes(last) ? last : "episodic";
+}

package/packages/memory/src/engine.js

@@ -0,0 +1,198 @@
+/**
+ * HTTP client for the Pentatonic memory engine (compat shim API).
+ *
+ * The engine speaks four endpoints:
+ *
+ *   POST /store     { content, metadata }              → { id, content, layerId, engine }
+ *   POST /store-batch { records[], arena? }            → { ids, inserted, engine }
+ *   POST /search    { query, limit?, arena?, ... }     → { results }
+ *   POST /forget    { id? } | { metadata_contains? }   → { deleted }
+ *
+ * This module is the canonical wire-shape contract: any host (TES,
+ * Workers, Lambda) imports the same builders so the engine never sees
+ * inconsistent payloads. The host owns auth, audit, multi-tenant
+ * routing, and the surface format it exposes to its callers.
+ *
+ * No Node-only APIs — Workers / browser-fetch friendly.
+ *
+ * @example
+ *   import { engineStore, engineSearch, engineForget }
+ *     from "@pentatonic-ai/ai-agent-sdk/memory/engine";
+ *
+ *   const ENGINE = process.env.MEMORY_ENGINE_URL;
+ *
+ *   const stored = await engineStore(ENGINE, {
+ *     clientId: "acme",
+ *     content: "Alice owns project Atlas",
+ *     metadata: { kind: "note" },
+ *     layerType: "episodic",
+ *   });
+ *
+ *   const { results } = await engineSearch(ENGINE, {
+ *     clientId: "acme",
+ *     query: "Alice",
+ *     limit: 6,
+ *   });
+ */
+
+export const DEFAULT_ENGINE_URL = "https://memory-engine.thingeventsystem.ai";
+export const DEFAULT_LIMIT = 10;
+export const DEFAULT_MIN_SCORE = 0.3;
+
+/**
+ * @typedef {object} EngineSearchHit
+ * @property {string} id
+ * @property {string} content
+ * @property {object} [metadata]
+ * @property {number} [similarity]
+ * @property {number} [score]
+ * @property {string} [layer_id]
+ * @property {string} [engine_layer]
+ * @property {string} [source]
+ */
+
+/**
+ * @typedef {object} EngineStoreResult
+ * @property {string} id
+ * @property {string} content
+ * @property {string} layerId
+ * @property {object} [engine]   - per-layer index counts
+ */
+
+/**
+ * Low-level POST to the engine HTTP API. Mostly an internal helper —
+ * prefer the higher-level builders (`engineStore`, `engineSearch`,
+ * `engineForget`) which encode the canonical body shape.
+ *
+ * @param {string} engineUrl - engine base URL (no trailing slash)
+ * @param {string} path      - "/store" | "/search" | "/forget" | "/health" | "/store-batch"
+ * @param {object} body      - JSON body, serialised verbatim
+ * @returns {Promise<object>} parsed JSON response
+ * @throws {Error} - "engine_<status>" with `.detail` set to response text
+ *                   on HTTP non-2xx, or "engine_network: <msg>" on
+ *                   transport failure.
+ */
+export async function fetchEngine(engineUrl, path, body) {
+  const base = engineUrl || DEFAULT_ENGINE_URL;
+  const url = `${base}${path}`;
+  let res;
+  try {
+    res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    throw new Error(`engine_network: ${err.message}`);
+  }
+  if (!res.ok) {
+    let detail = "";
+    try {
+      detail = await res.text();
+    } catch {
+      // ignore body-read failure; status alone is enough to fail
+    }
+    const e = new Error(`engine_${res.status}`);
+    e.detail = detail;
+    throw e;
+  }
+  return res.json();
+}
+
+/**
+ * Store a single memory in the engine.
+ *
+ * Builds the canonical /store body: `arena = clientId` is set on
+ * metadata so the engine's multi-tenant scoping works. Caller-supplied
+ * metadata fields take precedence on conflict.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId      tenant id (becomes engine arena)
+ * @param {string} opts.content
+ * @param {object} [opts.metadata]    extra metadata; merged into engine body
+ * @param {string} [opts.layerType]   "episodic" | "semantic" | "procedural" | "working"
+ * @param {string} [opts.actorUserId] passes through as metadata.actor_user_id
+ * @returns {Promise<EngineStoreResult>}
+ */
+export async function engineStore(engineUrl, opts) {
+  const {
+    clientId,
+    content,
+    metadata = {},
+    layerType,
+    actorUserId,
+  } = opts || {};
+  if (!clientId) throw new Error("engineStore: clientId required");
+  if (typeof content !== "string") throw new Error("engineStore: content required");
+  const body = {
+    content,
+    metadata: {
+      ...metadata,
+      arena: clientId,
+      ...(layerType ? { layer_type: layerType } : {}),
+      ...(actorUserId !== undefined ? { actor_user_id: actorUserId } : {}),
+    },
+  };
+  return fetchEngine(engineUrl, "/store", body);
+}
+
+/**
+ * Search the engine, scoped to a tenant.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId
+ * @param {string} opts.query
+ * @param {number} [opts.limit=10]
+ * @param {number} [opts.minScore=0.3]
+ * @param {object} [opts.metadataFilter]  arbitrary equality filter on result metadata
+ * @returns {Promise<{results: EngineSearchHit[]}>}
+ */
+export async function engineSearch(engineUrl, opts) {
+  const {
+    clientId,
+    query,
+    limit = DEFAULT_LIMIT,
+    minScore = DEFAULT_MIN_SCORE,
+    metadataFilter,
+  } = opts || {};
+  if (!clientId) throw new Error("engineSearch: clientId required");
+  if (typeof query !== "string") throw new Error("engineSearch: query required");
+  const body = {
+    arena: clientId,
+    query,
+    limit,
+    min_score: minScore,
+    ...(metadataFilter && Object.keys(metadataFilter).length
+      ? { metadata_filter: metadataFilter }
+      : {}),
+  };
+  return fetchEngine(engineUrl, "/search", body);
+}
+
+/**
+ * Forget memories by id or by metadata equality.
+ *
+ * Caller must supply exactly one of `id` or `metadataContains`.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId
+ * @param {string} [opts.id]                forget a single record by engine id
+ * @param {object} [opts.metadataContains]  forget all records matching every key=value pair
+ * @returns {Promise<{deleted: number}>}
+ */
+export async function engineForget(engineUrl, opts) {
+  const { clientId, id, metadataContains } = opts || {};
+  if (!clientId) throw new Error("engineForget: clientId required");
+  if (!id && !metadataContains) {
+    throw new Error("engineForget: provide id or metadataContains");
+  }
+  const body = {
+    arena: clientId,
+    ...(id ? { id } : {}),
+    ...(metadataContains ? { metadata_contains: metadataContains } : {}),
+  };
+  return fetchEngine(engineUrl, "/forget", body);
+}

package/packages/memory-engine/compat/server.py

@@ -420,12 +420,19 @@ async def health():
     if failures:
         out["status"] = "degraded" if failures < 3 else "down"
 
-    # Memory count: query L6 doc-store as authoritative.
+    # Memory count: query L6 doc-store as authoritative. L6 /stats
+    # returns vector_chunks (Milvus) and fts_chunks (sqlite content
+    # table). Under healthy operation they're equal — take the max so
+    # the count is honest if one side is mid-rebuild.
     try:
         r = await _client().get(f"{L6_DOC_URL}/stats", timeout=3.0)
         if r.status_code == 200:
             stats = r.json()
-            out["memories"] = stats.get("total_chunks") or stats.get("fts_chunks") or 0
+            out["memories"] = max(
+                int(stats.get("vector_chunks") or 0),
+                int(stats.get("fts_chunks") or 0),
+                int(stats.get("total_chunks") or 0),
+            )
     except Exception:
         out["memories"] = None
     return out

package/packages/memory-engine/docker-compose.test.yml

@@ -29,20 +29,26 @@ services:
       retries: 20
       start_period: 5s
 
+  # Pin the embedding dim explicitly across layers, independent of any
+  # developer-local .env (which may set EMBED_DIM=768 for Ollama-based
+  # local dev). The stub returns 4096; layers must agree.
   l4:
     environment:
       L4_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
       L4_EMBED_API_KEY: ""
+      L4_EMBED_DIM: "4096"
 
   l5:
     environment:
       L5_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
       L5_EMBED_API_KEY: ""
+      L5_EMBED_DIM: "4096"
 
   l6:
     environment:
       L6_NV_EMBED_URL: http://embed-stub:8041/v1/embeddings
       L6_EMBED_API_KEY: ""
+      L6_EMBED_DIM: "4096"
 
   l2:
     environment:

package/packages/memory-engine/engine/services/l6/l6-document-store.py

@@ -911,16 +911,34 @@ def serve(port: int = DEFAULT_PORT):
             milvus.insert(collection_name=COLLECTION_NAME, data=rows)
         insert_ms = (_time.time() - t1) * 1000.0
 
-        # Single FTS write (best-effort — search still works without it).
+        # Single FTS write — into the `chunks` content table; the
+        # AFTER INSERT trigger replicates rows into the chunks_fts
+        # virtual table so BM25 search (which JOINs chunks ON rowid)
+        # actually finds them. Earlier versions wrote directly to
+        # chunks_fts, leaving `chunks` empty — which made BM25 return
+        # zero hits AND broke the /stats fts_chunks counter.
         try:
             fts_conn = get_fts_db()
             for r, txt in zip(records, texts):
                 rid = r.get("id") or _hashlib.sha1(txt.encode("utf-8")).hexdigest()[:32]
+                chunk_id = f"l6:{rid}:0"[:63]
                 fts_conn.execute(
-                    "INSERT INTO chunks_fts(text, source_file, arena, heading, entities_json) "
-                    "VALUES (?, ?, ?, ?, ?)",
-                    (txt, (r.get("source_file") or f"{rid}.md"), arena,
-                     (r.get("heading") or ""), "[]"),
+                    "INSERT OR REPLACE INTO chunks "
+                    "(id, text, source_file, arena, doc_type, heading, "
+                    " chunk_index, content_hash, entities_json, indexed_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        chunk_id,
+                        txt,
+                        (r.get("source_file") or f"{rid}.md"),
+                        arena,
+                        (r.get("doc_type") or "general"),
+                        (r.get("heading") or ""),
+                        0,
+                        _hashlib.sha1(txt.encode("utf-8")).hexdigest()[:20],
+                        "[]",
+                        now,
+                    ),
                 )
             fts_conn.commit()
             fts_conn.close()