ltcai 11.2.0 → 11.4.0

package/latticeai/services/memory_service/maintenance.py

@@ -0,0 +1,138 @@
+"""The mutating half: prune, compact, rebuild, clear.
+
+Every deletion path goes through :meth:`prune`, and :meth:`prune` intersects
+its targets with the caller's *own* memories first. That ownership guard is the
+point: a forged id belonging to another user is refused and reported as
+``skipped``, never silently deleted. :meth:`clear` is a thin scope router over
+the same guard, and refuses ``graph`` outright because the graph is not
+workspace-scoped.
+
+Partial failure is reported, not swallowed: ``status`` becomes ``partial`` when
+some deletions landed and ``error`` when none did.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from ._contract import MemoryCore as _Core
+from .constants import LOGGER, WORKSPACE_KINDS
+
+
+class MemoryMaintenanceMixin(_Core):
+    """Prune / compact / rebuild / clear. Mixed into ``MemoryService``."""
+
+    # ── mutating operations ───────────────────────────────────────────────
+    def prune(
+        self,
+        *,
+        ids: Optional[List[str]] = None,
+        kind: Optional[str] = None,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        # Ownership guard: a caller may only prune memories they own. Both the
+        # explicit-id and kind paths are intersected with the caller's own
+        # memories, so a forged id for another user's memory is refused, not
+        # silently deleted.
+        owned_ids = {
+            m["id"]
+            for m in self._workspace_memories(user_email=user_email, workspace_id=workspace_id)
+            if m.get("id")
+        }
+        removed: List[str] = []
+        failed: List[Dict[str, str]] = []
+        skipped: List[str] = []
+        target_ids: List[str] = []
+        seen: set = set()
+        for mid in (ids or []):
+            if mid in seen:
+                continue
+            seen.add(mid)
+            if mid in owned_ids:
+                target_ids.append(mid)
+            else:
+                skipped.append(mid)
+        if kind:
+            for m in self._workspace_memories(user_email=user_email, workspace_id=workspace_id):
+                if m.get("kind") == kind and m.get("id") and m["id"] not in seen:
+                    seen.add(m["id"])
+                    target_ids.append(m["id"])
+        for mid in target_ids:
+            try:
+                self._store.delete_memory(mid)
+                removed.append(mid)
+            except Exception as exc:
+                LOGGER.exception("memory deletion failed for %s", mid)
+                failed.append({"id": mid, "detail": str(exc)})
+        result: Dict[str, Any] = {"removed": removed, "count": len(removed)}
+        if skipped:
+            result["skipped"] = skipped
+        if failed:
+            result["failed"] = failed
+            result["status"] = "partial" if removed else "error"
+        return result
+
+    def compact(
+        self,
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Dedupe workspace memories with identical (kind, content)."""
+        seen: set = set()
+        removed: List[str] = []
+        failed: List[Dict[str, str]] = []
+        # Oldest first so the first occurrence (oldest) is kept.
+        memories = list(reversed(self._workspace_memories(user_email=user_email, workspace_id=workspace_id)))
+        for m in memories:
+            key = (m.get("kind"), str(m.get("content") or "").strip())
+            if key in seen:
+                if m.get("id"):
+                    try:
+                        self._store.delete_memory(m["id"])
+                        removed.append(m["id"])
+                    except Exception as exc:
+                        LOGGER.exception("memory compaction deletion failed for %s", m["id"])
+                        failed.append({"id": m["id"], "detail": str(exc)})
+            else:
+                seen.add(key)
+        return {
+            "compacted": len(removed),
+            "removed": removed,
+            "remaining": len(seen),
+            "failed": failed,
+            "status": "partial" if failed and removed else "error" if failed else "ok",
+        }
+
+    def rebuild(self, target: str = "vector") -> Dict[str, Any]:
+        if target in {"vector", "index", "vector_index"}:
+            if not self._enable_graph:
+                return {"status": "unavailable", "detail": "Knowledge graph / vector index disabled."}
+            try:
+                result = self._kg.rebuild_vector_index()
+                return {"status": "ok", "target": "vector_index", "result": result}
+            except Exception as exc:
+                return {"status": "error", "detail": str(exc)}
+        return {"status": "error", "detail": f"Unknown rebuild target: {target}"}
+
+    def clear(
+        self,
+        *,
+        scope: str,
+        confirm: bool = False,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        if not confirm:
+            raise ValueError("clear requires confirm=true")
+        if scope in WORKSPACE_KINDS:
+            result = self.prune(kind=scope, user_email=user_email, workspace_id=workspace_id)
+            return {"cleared": scope, **result}
+        if scope == "workspace":  # pragma: no cover — shadowed: "workspace" is itself a WORKSPACE_KINDS member, so the by-kind branch above always wins; reordering would widen what clear() deletes, which is an owner decision
+            ids = [m["id"] for m in self._workspace_memories(user_email=user_email, workspace_id=workspace_id) if m.get("id")]
+            result = self.prune(ids=ids, user_email=user_email, workspace_id=workspace_id)
+            return {"cleared": "workspace", **result}
+        if scope == "graph":
+            raise ValueError("graph clear is disabled from Memory Manager because it is not workspace-scoped")
+        raise ValueError(f"unsupported clear scope: {scope}")

package/latticeai/services/memory_service/manager.py

@@ -0,0 +1,186 @@
+"""The Memory Manager: one cross-tier report over every backing store.
+
+:meth:`manager` is the service's centre of gravity — sources, counts, sizes,
+health and the Brain readiness signal in a single read that the brief, the
+proof and the API all build on. Nothing here is estimated: a tier with no
+backing reports ``unavailable`` and contributes ``None``, never a zero dressed
+up as a measurement.
+
+``_brain_readiness`` lives here rather than in the frontend so the growth score
+is auditable and the Memory Manager stays the single owner of cross-tier
+signals.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+from latticeai.core.timeutil import now_iso as _now
+from latticeai.core.workspace_os_utils import _file_size
+
+from ._contract import MemoryCore as _Core
+from .constants import TIERS
+
+
+class MemoryManagerMixin(_Core):
+    """Sources / usage / health. Mixed into ``MemoryService``."""
+
+    # ── Memory Manager: sources / usage / health ──────────────────────────
+    def _brain_readiness(
+        self,
+        *,
+        memory_count: int,
+        concept_count: Optional[int],
+        relationship_count: Optional[int],
+        healthy_sources: int,
+    ) -> Dict[str, Any]:
+        """Summarize how ready the user's Brain is for the product UI.
+
+        The frontend should render this signal instead of re-deriving it from
+        UI fragments. Keeping it here makes the score auditable and keeps the
+        Memory Manager as the single owner of cross-tier Brain growth signals.
+        """
+        concepts = max(0, int(concept_count or 0))
+        relationships = max(0, int(relationship_count or 0))
+        memories = max(0, int(memory_count or 0))
+        healthy = max(0, int(healthy_sources or 0))
+        score = min(100, round(memories * 12 + concepts * 8 + relationships * 4 + healthy * 3))
+
+        if memories < 1 and concepts < 1:
+            state = "quiet"
+            depth = 2
+            title_key = "brain.readiness.quiet"
+            action_key = "brain.readiness.start"
+            score = max(12, score)
+        elif concepts < 3 or relationships < 2:
+            state = "forming"
+            depth = 3 if concepts < 3 else 4
+            title_key = "brain.readiness.forming"
+            action_key = "brain.readiness.grow"
+            score = max(38, score)
+        else:
+            state = "alive"
+            depth = 5
+            title_key = "brain.readiness.alive"
+            action_key = "brain.readiness.map"
+            score = max(72, score)
+
+        return {
+            "score": score,
+            "state": state,
+            "depth": depth,
+            "title_key": title_key,
+            "action_key": action_key,
+            "signals": {
+                "memory_count": memories,
+                "concept_count": concepts,
+                "relationship_count": relationships,
+                "healthy_sources": healthy,
+            },
+            "source": "memory_service",
+        }
+
+    def manager(self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None) -> Dict[str, Any]:
+        ws_mem = self._workspace_memories(user_email=user_email, workspace_id=workspace_id or "personal")
+        if workspace_id is None:
+            project_mem = [m for m in self._all_memories() if (m.get("workspace_id") or "personal") != "personal"]
+        else:
+            project_mem = self._workspace_memories(user_email=user_email, workspace_id=workspace_id)
+        snaps = self._snapshots(workspace_id=workspace_id)
+        convs = self._scoped_conversations(user_email=user_email, workspace_id=workspace_id)
+        kg_stats = self._kg_stats()
+        kg_index = self._kg_index()
+
+        ws_bytes = _file_size(self._data_dir / "workspace_os.json")
+        kg_bytes = _file_size(self._data_dir / "knowledge_graph.sqlite")
+        if self._conversation_store is not None:
+            conv_bytes = int(getattr(self._conversation_store, "size_bytes", lambda: 0)())
+        else:
+            conv_bytes = _file_size(self._history_file)
+
+        node_total = sum((kg_stats or {}).get("nodes", {}).values()) if kg_stats else None
+        edge_total = sum((kg_stats or {}).get("edges", {}).values()) if kg_stats else None
+        vector_total = None
+        if kg_index and isinstance(kg_index.get("vector_counts"), dict):
+            vector_total = sum(kg_index["vector_counts"].values())
+        elif kg_index:
+            vector_total = kg_index.get("indexed") or kg_index.get("ready")
+
+        sources: List[Dict[str, Any]] = [
+            {
+                "id": "workspace", "type": "workspace", "label": "Workspace Memory",
+                "count": len(ws_mem), "size_bytes": ws_bytes if ws_mem else 0,
+                "health": "ok", "detail": "Personal workspace knowledge, by kind.",
+            },
+            {
+                "id": "project", "type": "project", "label": "Project Memory",
+                "count": len(project_mem), "size_bytes": 0,
+                "health": "ok", "detail": "Memory scoped to organization workspaces.",
+            },
+            {
+                "id": "agent", "type": "agent", "label": "Agent Memory",
+                "count": len(snaps), "size_bytes": 0,
+                "health": "ok", "detail": "Per-run agent memory snapshots.",
+            },
+            {
+                "id": "conversation", "type": "conversation", "label": "Conversation Memory",
+                "count": len(convs), "size_bytes": conv_bytes,
+                "health": "ok" if (self._conversation_store is not None or self._history_file.exists()) else "empty",
+                "detail": "Historical interaction memory from chat.",
+            },
+            {
+                "id": "graph", "type": "graph", "label": "Graph Memory",
+                "count": node_total, "size_bytes": kg_bytes,
+                "health": "ok" if kg_stats else "unavailable",
+                "detail": "Knowledge Graph entities and relations." if kg_stats else "Knowledge graph disabled or unavailable.",
+                "edges": edge_total,
+            },
+            {
+                "id": "vector", "type": "vector", "label": "Vector Memory",
+                "count": vector_total, "size_bytes": 0,
+                "health": "ok" if kg_index else "unavailable",
+                "detail": "Local embedding vector index." if kg_index else "Vector index unavailable.",
+            },
+        ]
+        total_items = sum(int(s["count"] or 0) for s in sources)
+        total_bytes = ws_bytes + kg_bytes + conv_bytes
+        healthy = sum(1 for s in sources if s["health"] == "ok")
+        overall = "ok" if healthy >= 4 else "degraded" if healthy >= 1 else "unavailable"
+        memory_ids = {m.get("id") for m in [*ws_mem, *project_mem] if m.get("id")}
+        memory_count = len(memory_ids) + len(snaps) + len(convs)
+        return {
+            "sources": sources,
+            "recent_memories": self._manager_recent_memories([*ws_mem, *project_mem], limit=8),
+            "tiers": list(TIERS),
+            "usage": {"total_items": total_items, "total_bytes": total_bytes, "sources": len(sources)},
+            "brain_readiness": self._brain_readiness(
+                memory_count=memory_count,
+                concept_count=node_total,
+                relationship_count=edge_total,
+                healthy_sources=healthy,
+            ),
+            "health": overall,
+            "graph_enabled": self._enable_graph,
+            "generated_at": _now(),
+        }
+
+    @staticmethod
+    def _manager_recent_memories(memories: List[Dict[str, Any]], *, limit: int = 8) -> List[Dict[str, Any]]:
+        rows: List[Dict[str, Any]] = []
+        for item in memories[: max(1, limit)]:
+            metadata = item.get("metadata") if isinstance(item.get("metadata"), dict) else {}
+            rows.append({
+                "id": item.get("id") or "",
+                "kind": item.get("kind") or "memory",
+                "content": str(item.get("content") or "")[:320],
+                "tags": item.get("tags") if isinstance(item.get("tags"), list) else [],
+                "metadata": metadata,
+                "workspace_id": item.get("workspace_id") or "personal",
+                "created_at": item.get("created_at"),
+                "updated_at": item.get("updated_at"),
+            })
+        return rows
+
+    def brain_quality_summary(self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None) -> Dict[str, Any]:
+        """Return the backend-owned Brain readiness signal for API consumers."""
+        return self.manager(user_email=user_email, workspace_id=workspace_id)["brain_readiness"]

package/latticeai/services/memory_service/proof.py

@@ -0,0 +1,136 @@
+"""Brain Proof: the durable evidence behind "your Brain outlives the model".
+
+Backend-owned on purpose. The first-run Brain screen must show the stores it
+can actually recall from, and the model-continuity claim has to be independent
+of whichever LLM happens to be loaded — a UI-derived version of this would be
+an assertion, not a proof.
+
+The two are deliberately separated: ``capability`` is always true (the design
+is model-independent), while ``proven`` stays false until there is durable
+evidence on disk to recall from.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from latticeai.core.timeutil import now_iso as _now
+
+from ._contract import MemoryCore as _Core
+
+
+class MemoryProofMixin(_Core):
+    """The proof points and the query they are demonstrated with."""
+
+    def brain_proof(
+        self,
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        active_model: Optional[str] = None,
+        recall_query: str = "",
+        limit: int = 3,
+    ) -> Dict[str, Any]:
+        """Return the proof points that make Brain feel model-independent.
+
+        This is intentionally backend-owned instead of UI-derived: the first-run
+        Brain screen needs to show the durable stores it can actually recall
+        from, and the model continuity claim must be independent from whichever
+        LLM is currently loaded.
+        """
+        manager = self.manager(user_email=user_email, workspace_id=workspace_id)
+        sources = {str(source.get("id")): source for source in manager.get("sources", []) if isinstance(source, dict)}
+        readiness = manager.get("brain_readiness") or {}
+        conversation_count = int(sources.get("conversation", {}).get("count") or 0)
+        workspace_count = int(sources.get("workspace", {}).get("count") or 0)
+        graph_count = int(sources.get("graph", {}).get("count") or 0)
+        vector_count = int(sources.get("vector", {}).get("count") or 0)
+        query = recall_query.strip() or self._latest_recall_query(user_email=user_email, workspace_id=workspace_id)
+        recall: Dict[str, Any] = (
+            self.recall(query, user_email=user_email, workspace_id=workspace_id, limit=limit)
+            if query
+            else {"query": "", "results": [], "count": 0, "source": "live"}
+        )
+        recall_items = [
+            {
+                "id": item.get("id"),
+                "source": item.get("source"),
+                "title": item.get("title"),
+                "snippet": item.get("snippet"),
+                "score": item.get("score", 0),
+                # Evidence explainability: why this item was recalled, so the
+                # citation UI can show the matched terms instead of a bare score.
+                "matched_terms": item.get("matched_terms") or [],
+                "confidence": item.get("confidence") or "low",
+                # v11.1.0: what kind of memory this is, and — for a picture —
+                # the caption and inline thumbnail the Evidence panel renders.
+                "kind": item.get("kind") or "",
+                **({"caption": item["caption"]} if item.get("caption") else {}),
+                **({"thumbnail": item["thumbnail"]} if item.get("thumbnail") else {}),
+            }
+            for item in list(recall.get("results", []))[: max(1, min(limit, 8))]
+        ]
+        durable_items = workspace_count + conversation_count + graph_count
+        # Capability and proof are deliberately separated. The brain is
+        # architecturally model-independent, so the capability is always true.
+        # The proof stays false until there is durable evidence on disk.
+        has_durable_evidence = durable_items > 0
+        proven_continuity = has_durable_evidence
+        return {
+            "status": readiness.get("state") or "quiet",
+            "readiness": readiness,
+            "model_continuity": {
+                "active_model": active_model or "",
+                "brain_owner": "lattice_brain",
+                # Design capability: the brain is built to outlive any model.
+                "capability": True,
+                # Proof: only true when durable evidence exists to recall.
+                "survives_model_switch": proven_continuity,
+                "proven": proven_continuity,
+                "context_store": "workspace + conversation + graph + vector",
+            },
+            "proofs": {
+                "durable_items": durable_items,
+                "has_durable_evidence": has_durable_evidence,
+                "workspace_memories": workspace_count,
+                "conversations": conversation_count,
+                "graph_concepts": graph_count,
+                "vector_items": vector_count,
+                "healthy_sources": readiness.get("signals", {}).get("healthy_sources", 0),
+            },
+            "recall": {
+                "query": recall.get("query") or query,
+                "items": recall_items,
+                "count": recall.get("count", 0),
+            },
+            "claims": {
+                "can_recall_user_context": bool(recall_items or durable_items > 0),
+                # Proven, not asserted: no durable evidence means no continuity claim.
+                "keeps_context_across_models": proven_continuity,
+                "is_knowledge_store": bool(graph_count or vector_count or workspace_count or conversation_count),
+            },
+            "generated_at": _now(),
+        }
+
+    def _latest_recall_query(self, *, user_email: Optional[str], workspace_id: Optional[str]) -> str:
+        for memory in self._workspace_memories(user_email=user_email, workspace_id=workspace_id or "personal"):
+            content = str(memory.get("content") or "").strip()
+            if content:
+                return content[:96]
+        for conversation in self._conversations():
+            messages = conversation.get("messages") or []
+            if not isinstance(messages, list):
+                continue
+            for message in reversed(messages[-8:]):
+                if not isinstance(message, dict):
+                    continue
+                if user_email and message.get("user_email") != user_email:
+                    continue
+                message_workspace = message.get("workspace_id") or "personal"
+                target_workspace = workspace_id or "personal"
+                if message_workspace != target_workspace:
+                    continue
+                content = str(message.get("content") or "").strip()
+                if content:
+                    return content[:96]
+        return ""