@pentatonic-ai/ai-agent-sdk 0.7.6 → 0.7.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.7.6",
+  "version": "0.7.8",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -15,7 +15,9 @@
     "./memory/openclaw": "./packages/memory/src/openclaw/index.js",
     "./doctor": "./packages/doctor/src/index.js",
     "./memory/hosted": "./packages/memory/src/hosted.js",
-    "./memory/corpus": "./packages/memory/src/corpus/index.js"
+    "./memory/corpus": "./packages/memory/src/corpus/index.js",
+    "./memory/engine": "./packages/memory/src/engine.js",
+    "./memory/engine-layers": "./packages/memory/src/engine-layers.js"
   },
   "bin": {
     "ai-agent-sdk": "./bin/cli.js"

package/packages/memory/src/__tests__/engine-layers.test.js

@@ -0,0 +1,62 @@
+import { describe, it, expect } from "@jest/globals";
+import {
+  STATIC_LAYER_TYPES,
+  buildStaticLayers,
+  layerIdToType,
+} from "../engine-layers.js";
+
+describe("engine-layers", () => {
+  it("STATIC_LAYER_TYPES is the documented 4-layer surface", () => {
+    expect(STATIC_LAYER_TYPES).toEqual([
+      "episodic",
+      "semantic",
+      "procedural",
+      "working",
+    ]);
+  });
+
+  describe("buildStaticLayers", () => {
+    it("returns one row per layer type with stable id shape", () => {
+      const rows = buildStaticLayers("acme");
+      expect(rows.map((r) => r.id)).toEqual([
+        "ml_acme_episodic",
+        "ml_acme_semantic",
+        "ml_acme_procedural",
+        "ml_acme_working",
+      ]);
+      for (const r of rows) {
+        expect(r.client_id).toBe("acme");
+        expect(r.layer_type).toBe(r.name);
+        expect(r.is_active).toBe(true);
+        expect(r.memory_count).toBe(null);
+        expect(r.capacity).toBe(null);
+        expect(r.decay_policy).toBe(null);
+      }
+    });
+
+    it("created_at is install-time stable (epoch) so dashboards don't churn", () => {
+      const a = buildStaticLayers("acme");
+      const b = buildStaticLayers("acme");
+      expect(a[0].created_at).toBe(b[0].created_at);
+      expect(a[0].created_at).toBe(new Date(0).toISOString());
+    });
+  });
+
+  describe("layerIdToType", () => {
+    it("extracts the type suffix", () => {
+      expect(layerIdToType("ml_acme_episodic")).toBe("episodic");
+      expect(layerIdToType("ml_acme_semantic")).toBe("semantic");
+      expect(layerIdToType("ml_acme_procedural")).toBe("procedural");
+      expect(layerIdToType("ml_acme_working")).toBe("working");
+    });
+
+    it("falls back to episodic for unknown / malformed ids", () => {
+      expect(layerIdToType("ml_acme_other")).toBe("episodic");
+      expect(layerIdToType("garbage")).toBe("episodic");
+      expect(layerIdToType("")).toBe("episodic");
+      expect(layerIdToType(null)).toBe("episodic");
+      expect(layerIdToType(undefined)).toBe("episodic");
+      expect(layerIdToType(42)).toBe("episodic");
+    });
+  });
+});

package/packages/memory/src/__tests__/engine.test.js

@@ -0,0 +1,214 @@
+import { describe, it, expect, beforeEach, jest } from "@jest/globals";
+import {
+  fetchEngine,
+  engineStore,
+  engineSearch,
+  engineForget,
+  DEFAULT_ENGINE_URL,
+} from "../engine.js";
+
+describe("engine HTTP client", () => {
+  let originalFetch;
+  let calls;
+
+  beforeEach(() => {
+    originalFetch = global.fetch;
+    calls = [];
+  });
+
+  afterEach(() => {
+    global.fetch = originalFetch;
+  });
+
+  function mockOk(responseBody) {
+    global.fetch = jest.fn(async (url, init) => {
+      calls.push({ url, init });
+      return {
+        ok: true,
+        status: 200,
+        json: async () => responseBody,
+      };
+    });
+  }
+
+  function mockHttp(status, text) {
+    global.fetch = jest.fn(async (url, init) => {
+      calls.push({ url, init });
+      return {
+        ok: false,
+        status,
+        text: async () => text,
+        json: async () => ({}),
+      };
+    });
+  }
+
+  function mockNetworkErr(message) {
+    global.fetch = jest.fn(async () => {
+      throw new Error(message);
+    });
+  }
+
+  describe("fetchEngine", () => {
+    it("uses provided engineUrl + path verbatim", async () => {
+      mockOk({ ok: true });
+      await fetchEngine("https://x.example", "/store", { a: 1 });
+      expect(calls[0].url).toBe("https://x.example/store");
+      expect(calls[0].init.method).toBe("POST");
+      expect(calls[0].init.headers["content-type"]).toBe("application/json");
+      expect(JSON.parse(calls[0].init.body)).toEqual({ a: 1 });
+    });
+
+    it("falls back to DEFAULT_ENGINE_URL when engineUrl is empty", async () => {
+      mockOk({ ok: true });
+      await fetchEngine("", "/health", {});
+      expect(calls[0].url).toBe(`${DEFAULT_ENGINE_URL}/health`);
+    });
+
+    it("throws engine_<status> with detail on HTTP non-2xx", async () => {
+      mockHttp(503, "engine down");
+      await expect(fetchEngine("https://x", "/store", {})).rejects.toMatchObject({
+        message: "engine_503",
+        detail: "engine down",
+      });
+    });
+
+    it("throws engine_network on transport failure", async () => {
+      mockNetworkErr("ECONNREFUSED");
+      await expect(fetchEngine("https://x", "/store", {})).rejects.toThrow(
+        /engine_network: ECONNREFUSED/
+      );
+    });
+  });
+
+  describe("engineStore", () => {
+    it("builds canonical /store body with arena=clientId", async () => {
+      mockOk({ id: "abc", content: "hello", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", {
+        clientId: "acme",
+        content: "hello",
+        layerType: "episodic",
+        actorUserId: "u-1",
+        metadata: { kind: "note" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/store");
+      expect(body).toEqual({
+        content: "hello",
+        metadata: {
+          kind: "note",
+          arena: "acme",
+          layer_type: "episodic",
+          actor_user_id: "u-1",
+        },
+      });
+    });
+
+    it("omits layer_type and actor_user_id when not provided", async () => {
+      mockOk({ id: "x", content: "x", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", { clientId: "acme", content: "x" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata).toEqual({ arena: "acme" });
+    });
+
+    it("does NOT let caller override arena via metadata", async () => {
+      mockOk({ id: "x", content: "x", layerId: "ml_acme_episodic" });
+      await engineStore("https://e", {
+        clientId: "acme",
+        content: "x",
+        // attempted hostile arena spoof:
+        metadata: { arena: "tenant-b" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata.arena).toBe("acme");
+    });
+
+    it("rejects missing clientId or content", async () => {
+      mockOk({});
+      await expect(engineStore("https://e", { content: "x" })).rejects.toThrow(
+        /clientId/
+      );
+      await expect(engineStore("https://e", { clientId: "a" })).rejects.toThrow(
+        /content/
+      );
+    });
+  });
+
+  describe("engineSearch", () => {
+    it("builds canonical /search body and forwards arena/limit/min_score", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hello",
+        limit: 5,
+        minScore: 0.5,
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/search");
+      expect(body).toEqual({
+        arena: "acme",
+        query: "hello",
+        limit: 5,
+        min_score: 0.5,
+      });
+    });
+
+    it("includes metadata_filter only when non-empty", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hi",
+        metadataFilter: { kind: "note" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata_filter).toEqual({ kind: "note" });
+
+      calls.length = 0;
+      mockOk({ results: [] });
+      await engineSearch("https://e", {
+        clientId: "acme",
+        query: "hi",
+        metadataFilter: {},
+      });
+      const body2 = JSON.parse(calls[0].init.body);
+      expect(body2.metadata_filter).toBeUndefined();
+    });
+
+    it("uses defaults for limit + minScore", async () => {
+      mockOk({ results: [] });
+      await engineSearch("https://e", { clientId: "acme", query: "hi" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.limit).toBe(10);
+      expect(body.min_score).toBe(0.3);
+    });
+  });
+
+  describe("engineForget", () => {
+    it("forwards id when provided", async () => {
+      mockOk({ deleted: 1 });
+      await engineForget("https://e", { clientId: "acme", id: "abc" });
+      const body = JSON.parse(calls[0].init.body);
+      expect(calls[0].url).toBe("https://e/forget");
+      expect(body).toEqual({ arena: "acme", id: "abc" });
+    });
+
+    it("forwards metadata_contains when provided", async () => {
+      mockOk({ deleted: 5 });
+      await engineForget("https://e", {
+        clientId: "acme",
+        metadataContains: { source_repo: "monorepo" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body).toEqual({
+        arena: "acme",
+        metadata_contains: { source_repo: "monorepo" },
+      });
+    });
+
+    it("requires id or metadataContains", async () => {
+      await expect(
+        engineForget("https://e", { clientId: "acme" })
+      ).rejects.toThrow(/id or metadataContains/);
+    });
+  });
+});

package/packages/memory/src/engine-layers.js

@@ -0,0 +1,72 @@
+/**
+ * 4-layer projection over the memory engine.
+ *
+ * The engine's internal architecture has 7 storage layers (L0 BM25,
+ * L1 system files, L2 HybridRAG, L3 Neo4j KG, L4 sqlite-vec, L5 Milvus
+ * comms, L6 doc-store). The user-facing surface preferred by the SDK
+ * and the dashboard is a simpler 4-layer view: episodic / semantic /
+ * procedural / working.
+ *
+ * The engine itself doesn't model these — it stores everything as
+ * chunks tagged with `arena` + arbitrary `metadata`. The 4-layer view
+ * is synthesised here at the host boundary so callers see a familiar
+ * shape regardless of the storage architecture below.
+ *
+ * @example
+ *   import {
+ *     buildStaticLayers,
+ *     layerIdToType,
+ *     STATIC_LAYER_TYPES,
+ *   } from "@pentatonic-ai/ai-agent-sdk/memory/engine-layers";
+ *
+ *   // Resolver for a `memoryLayers(clientId)` field:
+ *   const layers = buildStaticLayers("acme");
+ *   // → 4 entries with stable ids ml_acme_episodic, ml_acme_semantic, ...
+ */
+
+export const STATIC_LAYER_TYPES = [
+  "episodic",
+  "semantic",
+  "procedural",
+  "working",
+];
+
+/**
+ * Synthesise the 4-layer surface for a tenant.
+ *
+ * Returned shape matches the legacy `memory_layers` row schema so
+ * existing GraphQL types and dashboard code work unchanged. Per-layer
+ * memory counts are null because the engine doesn't track them.
+ *
+ * @param {string} clientId
+ * @returns {Array<{id: string, client_id: string, name: string, layer_type: string, capacity: null, decay_policy: null, is_active: true, created_at: string, memory_count: null}>}
+ */
+export function buildStaticLayers(clientId) {
+  const nowIso = new Date(0).toISOString(); // stable, install-time
+  return STATIC_LAYER_TYPES.map((type) => ({
+    id: `ml_${clientId}_${type}`,
+    client_id: clientId,
+    name: type,
+    layer_type: type,
+    capacity: null,
+    decay_policy: null,
+    is_active: true,
+    created_at: nowIso,
+    memory_count: null,
+  }));
+}
+
+/**
+ * Extract the layer type from a layer id of the form
+ * `ml_<clientId>_<type>`. Falls back to "episodic" for inputs that
+ * don't match — keeps writes alive when the SDK is ahead of the host
+ * on layer naming.
+ *
+ * @param {string} layerId
+ * @returns {"episodic" | "semantic" | "procedural" | "working"}
+ */
+export function layerIdToType(layerId) {
+  if (typeof layerId !== "string") return "episodic";
+  const last = layerId.split("_").pop();
+  return STATIC_LAYER_TYPES.includes(last) ? last : "episodic";
+}

package/packages/memory/src/engine.js

@@ -0,0 +1,198 @@
+/**
+ * HTTP client for the Pentatonic memory engine (compat shim API).
+ *
+ * The engine speaks four endpoints:
+ *
+ *   POST /store     { content, metadata }              → { id, content, layerId, engine }
+ *   POST /store-batch { records[], arena? }            → { ids, inserted, engine }
+ *   POST /search    { query, limit?, arena?, ... }     → { results }
+ *   POST /forget    { id? } | { metadata_contains? }   → { deleted }
+ *
+ * This module is the canonical wire-shape contract: any host (TES,
+ * Workers, Lambda) imports the same builders so the engine never sees
+ * inconsistent payloads. The host owns auth, audit, multi-tenant
+ * routing, and the surface format it exposes to its callers.
+ *
+ * No Node-only APIs — Workers / browser-fetch friendly.
+ *
+ * @example
+ *   import { engineStore, engineSearch, engineForget }
+ *     from "@pentatonic-ai/ai-agent-sdk/memory/engine";
+ *
+ *   const ENGINE = process.env.MEMORY_ENGINE_URL;
+ *
+ *   const stored = await engineStore(ENGINE, {
+ *     clientId: "acme",
+ *     content: "Alice owns project Atlas",
+ *     metadata: { kind: "note" },
+ *     layerType: "episodic",
+ *   });
+ *
+ *   const { results } = await engineSearch(ENGINE, {
+ *     clientId: "acme",
+ *     query: "Alice",
+ *     limit: 6,
+ *   });
+ */
+
+export const DEFAULT_ENGINE_URL = "https://memory-engine.thingeventsystem.ai";
+export const DEFAULT_LIMIT = 10;
+export const DEFAULT_MIN_SCORE = 0.3;
+
+/**
+ * @typedef {object} EngineSearchHit
+ * @property {string} id
+ * @property {string} content
+ * @property {object} [metadata]
+ * @property {number} [similarity]
+ * @property {number} [score]
+ * @property {string} [layer_id]
+ * @property {string} [engine_layer]
+ * @property {string} [source]
+ */
+
+/**
+ * @typedef {object} EngineStoreResult
+ * @property {string} id
+ * @property {string} content
+ * @property {string} layerId
+ * @property {object} [engine]   - per-layer index counts
+ */
+
+/**
+ * Low-level POST to the engine HTTP API. Mostly an internal helper —
+ * prefer the higher-level builders (`engineStore`, `engineSearch`,
+ * `engineForget`) which encode the canonical body shape.
+ *
+ * @param {string} engineUrl - engine base URL (no trailing slash)
+ * @param {string} path      - "/store" | "/search" | "/forget" | "/health" | "/store-batch"
+ * @param {object} body      - JSON body, serialised verbatim
+ * @returns {Promise<object>} parsed JSON response
+ * @throws {Error} - "engine_<status>" with `.detail` set to response text
+ *                   on HTTP non-2xx, or "engine_network: <msg>" on
+ *                   transport failure.
+ */
+export async function fetchEngine(engineUrl, path, body) {
+  const base = engineUrl || DEFAULT_ENGINE_URL;
+  const url = `${base}${path}`;
+  let res;
+  try {
+    res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    throw new Error(`engine_network: ${err.message}`);
+  }
+  if (!res.ok) {
+    let detail = "";
+    try {
+      detail = await res.text();
+    } catch {
+      // ignore body-read failure; status alone is enough to fail
+    }
+    const e = new Error(`engine_${res.status}`);
+    e.detail = detail;
+    throw e;
+  }
+  return res.json();
+}
+
+/**
+ * Store a single memory in the engine.
+ *
+ * Builds the canonical /store body: `arena = clientId` is set on
+ * metadata so the engine's multi-tenant scoping works. Caller-supplied
+ * metadata fields take precedence on conflict.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId      tenant id (becomes engine arena)
+ * @param {string} opts.content
+ * @param {object} [opts.metadata]    extra metadata; merged into engine body
+ * @param {string} [opts.layerType]   "episodic" | "semantic" | "procedural" | "working"
+ * @param {string} [opts.actorUserId] passes through as metadata.actor_user_id
+ * @returns {Promise<EngineStoreResult>}
+ */
+export async function engineStore(engineUrl, opts) {
+  const {
+    clientId,
+    content,
+    metadata = {},
+    layerType,
+    actorUserId,
+  } = opts || {};
+  if (!clientId) throw new Error("engineStore: clientId required");
+  if (typeof content !== "string") throw new Error("engineStore: content required");
+  const body = {
+    content,
+    metadata: {
+      ...metadata,
+      arena: clientId,
+      ...(layerType ? { layer_type: layerType } : {}),
+      ...(actorUserId !== undefined ? { actor_user_id: actorUserId } : {}),
+    },
+  };
+  return fetchEngine(engineUrl, "/store", body);
+}
+
+/**
+ * Search the engine, scoped to a tenant.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId
+ * @param {string} opts.query
+ * @param {number} [opts.limit=10]
+ * @param {number} [opts.minScore=0.3]
+ * @param {object} [opts.metadataFilter]  arbitrary equality filter on result metadata
+ * @returns {Promise<{results: EngineSearchHit[]}>}
+ */
+export async function engineSearch(engineUrl, opts) {
+  const {
+    clientId,
+    query,
+    limit = DEFAULT_LIMIT,
+    minScore = DEFAULT_MIN_SCORE,
+    metadataFilter,
+  } = opts || {};
+  if (!clientId) throw new Error("engineSearch: clientId required");
+  if (typeof query !== "string") throw new Error("engineSearch: query required");
+  const body = {
+    arena: clientId,
+    query,
+    limit,
+    min_score: minScore,
+    ...(metadataFilter && Object.keys(metadataFilter).length
+      ? { metadata_filter: metadataFilter }
+      : {}),
+  };
+  return fetchEngine(engineUrl, "/search", body);
+}
+
+/**
+ * Forget memories by id or by metadata equality.
+ *
+ * Caller must supply exactly one of `id` or `metadataContains`.
+ *
+ * @param {string} engineUrl
+ * @param {object} opts
+ * @param {string} opts.clientId
+ * @param {string} [opts.id]                forget a single record by engine id
+ * @param {object} [opts.metadataContains]  forget all records matching every key=value pair
+ * @returns {Promise<{deleted: number}>}
+ */
+export async function engineForget(engineUrl, opts) {
+  const { clientId, id, metadataContains } = opts || {};
+  if (!clientId) throw new Error("engineForget: clientId required");
+  if (!id && !metadataContains) {
+    throw new Error("engineForget: provide id or metadataContains");
+  }
+  const body = {
+    arena: clientId,
+    ...(id ? { id } : {}),
+    ...(metadataContains ? { metadata_contains: metadataContains } : {}),
+  };
+  return fetchEngine(engineUrl, "/forget", body);
+}

package/packages/memory-engine/compat/server.py

@@ -438,12 +438,25 @@ async def health():
     return out
 
 
+def _arena_scoped_rid(arena: str, content: str) -> str:
+    """Stable, arena-scoped record id.
+
+    Same content in different arenas hashes to *different* ids — so two
+    tenants storing the byte-identical chunk get distinct primary keys
+    in the underlying stores (L4 sqlite-vec, L5 Milvus, L6 Milvus).
+    Without this, arena=tenant-b's write would overwrite arena=tenant-a's
+    row of the same content, silently breaking multi-tenant isolation
+    for any duplicated phrase.
+    """
+    return hashlib.sha1(f"{arena}:{content}".encode()).hexdigest()[:32]
+
+
 @app.post("/store")
 async def store(req: StoreRequest):
     """Single-record ingest. Same wire format as pentatonic-memory v0.5."""
-    rid = (req.metadata or {}).get("id") or hashlib.sha1(req.content.encode()).hexdigest()[:32]
-    record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
     arena = (req.metadata or {}).get("arena", "general")
+    rid = (req.metadata or {}).get("id") or _arena_scoped_rid(arena, req.content)
+    record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
 
     # Stash the full metadata under every key shape any layer could echo back.
     # L5/L6 use derivatives of rid; L2-internal returns paths shaped like
@@ -483,16 +496,16 @@ async def store_batch(req: StoreBatchRequest):
         return {"inserted": 0, "ids": []}
 
     # Normalise each record to {id, content, metadata}.
+    arena = req.arena or "general"
     normalised = []
     for r in req.records:
         content = r.get("content") or r.get("text") or ""
         if not content:
             continue
-        rid = r.get("id") or hashlib.sha1(content.encode()).hexdigest()[:32]
+        rid = r.get("id") or _arena_scoped_rid(arena, content)
         normalised.append({"id": rid, "content": content, "metadata": r.get("metadata") or {}})
 
     # Stash metadata for every record so /search can re-attach it.
-    arena = req.arena or "general"
     for r in normalised:
         _stash_all_keys(r["id"], r.get("metadata") or {}, arena)
 

package/packages/memory-engine/engine/services/l6/l6-document-store.py

@@ -970,6 +970,77 @@ def serve(port: int = DEFAULT_PORT):
         milvus.load_collection(COLLECTION_NAME)
         return {"status": "rebuilt"}
 
+    @api.post("/repair-fts")
+    def api_repair_fts():
+        """Backfill the SQLite `chunks` content table from Milvus.
+
+        Pre-v0.7.6 the /index-batch path wrote straight to the FTS5
+        virtual table and never populated `chunks`. The result: BM25
+        search (which JOINs chunks ON rowid) returned zero hits even
+        though Milvus had the data, and /stats fts_chunks reported 0.
+
+        v0.7.6 fixed new writes; this endpoint cleans up old rows by
+        walking Milvus and INSERT-OR-REPLACE'ing every row into chunks.
+        The trigger on chunks then mirrors them into chunks_fts. Idempotent.
+        Runs in-process so the L6 service's already-open Milvus handle
+        is reused — no file-lock conflict (Milvus Lite locks the .db).
+        """
+        milvus = get_milvus()
+        # Milvus Lite caps query() at 16384 per call; page through.
+        page_size = 16384
+        offset = 0
+        rows: list = []
+        while True:
+            page = milvus.query(
+                COLLECTION_NAME,
+                filter="id != ''",
+                output_fields=[
+                    "id", "text", "source_file", "arena", "doc_type",
+                    "heading", "chunk_index", "content_hash",
+                    "entities_json", "indexed_at",
+                ],
+                limit=page_size,
+                offset=offset,
+            )
+            if not page:
+                break
+            rows.extend(page)
+            if len(page) < page_size:
+                break
+            offset += page_size
+        fts_conn = get_fts_db()
+        repaired = 0
+        for r in rows:
+            try:
+                fts_conn.execute(
+                    "INSERT OR REPLACE INTO chunks "
+                    "(id, text, source_file, arena, doc_type, heading, "
+                    " chunk_index, content_hash, entities_json, indexed_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        r.get("id"),
+                        r.get("text", "") or "",
+                        r.get("source_file", "") or "",
+                        r.get("arena", "") or "general",
+                        r.get("doc_type", "") or "general",
+                        r.get("heading", "") or "",
+                        int(r.get("chunk_index", 0) or 0),
+                        r.get("content_hash", "") or "",
+                        r.get("entities_json", "") or "[]",
+                        r.get("indexed_at", "") or "",
+                    ),
+                )
+                repaired += 1
+            except Exception as exc:
+                log.warning("repair-fts: skipping row %s: %s", r.get("id"), exc)
+        fts_conn.commit()
+        fts_conn.close()
+        return {
+            "status": "ok",
+            "milvus_rows": len(rows),
+            "repaired": repaired,
+        }
+
     log.info(f"L6 Document Store — http://127.0.0.1:{port}")
     uvicorn.run(api, host=os.environ.get("HOST","127.0.0.1"), port=port, log_level="info")
 

package/packages/memory-engine/tests/e2e_arena.sh

@@ -125,6 +125,37 @@ print("yes" if ok and data else "no")')
 [ "$all_match" = "yes" ] && ok "metadata_filter scopes to probe + arena" \
   || fail "metadata_filter let other rows through"
 
+# ---------------------------------------------------------------------------
+# Same content across two arenas — proves the arena-aware id derivation.
+# Pre-v0.7.8, identical content collapsed to one row in L4/L5/L6 because
+# the id was sha1(content); the second tenant's write overwrote the first.
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "=== same-content-across-arenas ==="
+post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-x","probe":"e2e-arena"}}' >/dev/null
+post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-y","probe":"e2e-arena"}}' >/dev/null
+sleep 3
+
+SX=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
+  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-x"}')
+SY=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
+  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-y"}')
+
+x_has_phrase=$(echo "$SX" | python3 -c '
+import json,sys
+data=json.load(sys.stdin).get("results",[])
+print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
+y_has_phrase=$(echo "$SY" | python3 -c '
+import json,sys
+data=json.load(sys.stdin).get("results",[])
+print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
+
+[ "$x_has_phrase" = "yes" ] && ok "tenant-x: shared phrase preserved" \
+  || fail "tenant-x lost the shared phrase (id collision?)"
+[ "$y_has_phrase" = "yes" ] && ok "tenant-y: shared phrase preserved" \
+  || fail "tenant-y lost the shared phrase (id collision?)"
+
 # ---------------------------------------------------------------------------
 # /forget — by metadata_contains. Cleans up so reruns are idempotent.
 # ---------------------------------------------------------------------------