@pentatonic-ai/ai-agent-sdk 0.7.7 → 0.7.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.7.7",
+  "version": "0.7.9",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory/src/ingest.js

@@ -41,6 +41,18 @@ import { distill } from "./distill.js";
  *   ids) — pass the raw form here so retries of the same logical event
  *   match across runs whose prefixes differ by a few ms. Defaults to
  *   `content`.
+ * @param {"client" | "user"} [opts.dedupScope="client"] - Scope of the
+ *   dedup match. Default `"client"`: byte-equal content for the tenant
+ *   collapses to one row regardless of who emitted it (today's behaviour,
+ *   appropriate when each row is single-owner). Set `"user"` to also
+ *   require `user_id` equality, which lets multiple users legitimately
+ *   own their own copy of the same shared content (private chat-channel
+ *   members, group meeting attendees) — emit one ingest per member with
+ *   the same content but different `userId`, all with
+ *   `dedupScope: "user"`, and each user gets their own row with native
+ *   per-user access counting / recency / decay. Requires `userId` to be
+ *   set; when `userId` is null the option degrades to `"client"` scope
+ *   (a global shared row still collapses cross-emit).
  * @param {number} [opts.dedupLegacyWindowDays=7] - How far back the
  *   `[<iso>] <content>` legacy-form `LIKE` match scans. Default 7 days.
  *   The leading-wildcard `LIKE` can't use a btree index, so without a
@@ -94,6 +106,11 @@ export async function ingest(db, ai, llm, content, opts = {}) {
       opts.dedupLegacyWindowDays === undefined
         ? 7
         : Number(opts.dedupLegacyWindowDays);
+    // Per-user dedup requires `userId`; degrade to client-scope otherwise so
+    // a misuse can't accidentally relax the dedup boundary (we'd rather over-
+    // collapse than fragment the corpus on a missing userId).
+    const userScopedDedup = opts.dedupScope === "user" && !!opts.userId;
+    const userClause = userScopedDedup ? ` AND user_id = $4` : "";
     try {
       const sql =
         legacyWindowDays > 0
@@ -107,17 +124,18 @@ export async function ingest(db, ai, llm, content, opts = {}) {
                     content LIKE '%] ' || $2
                     AND created_at > NOW() - ($3 || ' days')::interval
                   )
-                )
+                )${userScopedDedup ? "\n                AND user_id = $4" : ""}
               LIMIT 1`
           : `SELECT id, 'exact' AS match_kind
                FROM memory_nodes
               WHERE client_id = $1
-                AND content = $2
+                AND content = $2${userClause}
               LIMIT 1`;
-      const params =
+      const baseParams =
         legacyWindowDays > 0
           ? [clientId, dedupKey, String(legacyWindowDays)]
           : [clientId, dedupKey];
+      const params = userScopedDedup ? [...baseParams, opts.userId] : baseParams;
       const dupCheck = await db(sql, params);
       if (dupCheck.rows?.length) {
         const matchKind = dupCheck.rows[0].match_kind || "exact";

package/packages/memory-engine/compat/server.py

@@ -438,12 +438,25 @@ async def health():
     return out
 
 
+def _arena_scoped_rid(arena: str, content: str) -> str:
+    """Stable, arena-scoped record id.
+
+    Same content in different arenas hashes to *different* ids — so two
+    tenants storing the byte-identical chunk get distinct primary keys
+    in the underlying stores (L4 sqlite-vec, L5 Milvus, L6 Milvus).
+    Without this, arena=tenant-b's write would overwrite arena=tenant-a's
+    row of the same content, silently breaking multi-tenant isolation
+    for any duplicated phrase.
+    """
+    return hashlib.sha1(f"{arena}:{content}".encode()).hexdigest()[:32]
+
+
 @app.post("/store")
 async def store(req: StoreRequest):
     """Single-record ingest. Same wire format as pentatonic-memory v0.5."""
-    rid = (req.metadata or {}).get("id") or hashlib.sha1(req.content.encode()).hexdigest()[:32]
-    record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
     arena = (req.metadata or {}).get("arena", "general")
+    rid = (req.metadata or {}).get("id") or _arena_scoped_rid(arena, req.content)
+    record = {"id": rid, "content": req.content, "metadata": req.metadata or {}}
 
     # Stash the full metadata under every key shape any layer could echo back.
     # L5/L6 use derivatives of rid; L2-internal returns paths shaped like
@@ -483,16 +496,16 @@ async def store_batch(req: StoreBatchRequest):
         return {"inserted": 0, "ids": []}
 
     # Normalise each record to {id, content, metadata}.
+    arena = req.arena or "general"
     normalised = []
     for r in req.records:
         content = r.get("content") or r.get("text") or ""
         if not content:
             continue
-        rid = r.get("id") or hashlib.sha1(content.encode()).hexdigest()[:32]
+        rid = r.get("id") or _arena_scoped_rid(arena, content)
         normalised.append({"id": rid, "content": content, "metadata": r.get("metadata") or {}})
 
     # Stash metadata for every record so /search can re-attach it.
-    arena = req.arena or "general"
     for r in normalised:
         _stash_all_keys(r["id"], r.get("metadata") or {}, arena)
 

package/packages/memory-engine/engine/services/l6/l6-document-store.py

@@ -970,6 +970,77 @@ def serve(port: int = DEFAULT_PORT):
         milvus.load_collection(COLLECTION_NAME)
         return {"status": "rebuilt"}
 
+    @api.post("/repair-fts")
+    def api_repair_fts():
+        """Backfill the SQLite `chunks` content table from Milvus.
+
+        Pre-v0.7.6 the /index-batch path wrote straight to the FTS5
+        virtual table and never populated `chunks`. The result: BM25
+        search (which JOINs chunks ON rowid) returned zero hits even
+        though Milvus had the data, and /stats fts_chunks reported 0.
+
+        v0.7.6 fixed new writes; this endpoint cleans up old rows by
+        walking Milvus and INSERT-OR-REPLACE'ing every row into chunks.
+        The trigger on chunks then mirrors them into chunks_fts. Idempotent.
+        Runs in-process so the L6 service's already-open Milvus handle
+        is reused — no file-lock conflict (Milvus Lite locks the .db).
+        """
+        milvus = get_milvus()
+        # Milvus Lite caps query() at 16384 per call; page through.
+        page_size = 16384
+        offset = 0
+        rows: list = []
+        while True:
+            page = milvus.query(
+                COLLECTION_NAME,
+                filter="id != ''",
+                output_fields=[
+                    "id", "text", "source_file", "arena", "doc_type",
+                    "heading", "chunk_index", "content_hash",
+                    "entities_json", "indexed_at",
+                ],
+                limit=page_size,
+                offset=offset,
+            )
+            if not page:
+                break
+            rows.extend(page)
+            if len(page) < page_size:
+                break
+            offset += page_size
+        fts_conn = get_fts_db()
+        repaired = 0
+        for r in rows:
+            try:
+                fts_conn.execute(
+                    "INSERT OR REPLACE INTO chunks "
+                    "(id, text, source_file, arena, doc_type, heading, "
+                    " chunk_index, content_hash, entities_json, indexed_at) "
+                    "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                    (
+                        r.get("id"),
+                        r.get("text", "") or "",
+                        r.get("source_file", "") or "",
+                        r.get("arena", "") or "general",
+                        r.get("doc_type", "") or "general",
+                        r.get("heading", "") or "",
+                        int(r.get("chunk_index", 0) or 0),
+                        r.get("content_hash", "") or "",
+                        r.get("entities_json", "") or "[]",
+                        r.get("indexed_at", "") or "",
+                    ),
+                )
+                repaired += 1
+            except Exception as exc:
+                log.warning("repair-fts: skipping row %s: %s", r.get("id"), exc)
+        fts_conn.commit()
+        fts_conn.close()
+        return {
+            "status": "ok",
+            "milvus_rows": len(rows),
+            "repaired": repaired,
+        }
+
     log.info(f"L6 Document Store — http://127.0.0.1:{port}")
     uvicorn.run(api, host=os.environ.get("HOST","127.0.0.1"), port=port, log_level="info")
 

package/packages/memory-engine/tests/e2e_arena.sh

@@ -125,6 +125,37 @@ print("yes" if ok and data else "no")')
 [ "$all_match" = "yes" ] && ok "metadata_filter scopes to probe + arena" \
   || fail "metadata_filter let other rows through"
 
+# ---------------------------------------------------------------------------
+# Same content across two arenas — proves the arena-aware id derivation.
+# Pre-v0.7.8, identical content collapsed to one row in L4/L5/L6 because
+# the id was sha1(content); the second tenant's write overwrote the first.
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "=== same-content-across-arenas ==="
+post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-x","probe":"e2e-arena"}}' >/dev/null
+post '{"content":"shared phrase about Eclipse","metadata":{"arena":"e2e-tenant-y","probe":"e2e-arena"}}' >/dev/null
+sleep 3
+
+SX=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
+  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-x"}')
+SY=$(curl -sf -X POST "$BASE/search" -H "Content-Type: application/json" \
+  -d '{"query":"Eclipse","limit":10,"arena":"e2e-tenant-y"}')
+
+x_has_phrase=$(echo "$SX" | python3 -c '
+import json,sys
+data=json.load(sys.stdin).get("results",[])
+print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
+y_has_phrase=$(echo "$SY" | python3 -c '
+import json,sys
+data=json.load(sys.stdin).get("results",[])
+print("yes" if any("Eclipse" in r.get("content","") for r in data) else "no")')
+
+[ "$x_has_phrase" = "yes" ] && ok "tenant-x: shared phrase preserved" \
+  || fail "tenant-x lost the shared phrase (id collision?)"
+[ "$y_has_phrase" = "yes" ] && ok "tenant-y: shared phrase preserved" \
+  || fail "tenant-y lost the shared phrase (id collision?)"
+
 # ---------------------------------------------------------------------------
 # /forget — by metadata_contains. Cleans up so reruns are idempotent.
 # ---------------------------------------------------------------------------