ltcai 11.5.2 → 11.6.0

package/latticeai/services/local_knowledge.py

@@ -1,465 +0,0 @@
-"""Local folder knowledge-source API and optional filesystem watcher."""
-
-import logging
-import threading
-from pathlib import Path
-from typing import Any, Callable, Dict, Optional
-
-from fastapi import APIRouter, HTTPException, Request
-from pydantic import BaseModel, Field
-
-
-class LocalTreeRequest(BaseModel):
-    path: str
-    max_items: int = 200
-    approved: bool = False
-    approval_token: Optional[str] = None
-
-
-class LocalKnowledgeAuditRequest(BaseModel):
-    path: str
-    include_ocr: bool = False
-    max_files: int = 50_000
-    approved: bool = False
-    approval_token: Optional[str] = None
-
-
-class LocalKnowledgeIndexRequest(BaseModel):
-    path: str
-    include_ocr: bool = False
-    watch_enabled: bool = False
-    max_files: int = 5_000
-    consent: Dict[str, Any] = Field(default_factory=dict)
-    approved: bool = False
-    approval_token: Optional[str] = None
-
-
-class LocalKnowledgeWatchRequest(BaseModel):
-    source_id: str
-
-
-class _LocalWatchHandler:
-    def __init__(self, schedule_change: Callable[[], None]):
-        self._schedule_change = schedule_change
-
-    def on_any_event(self, event):  # pragma: no cover - exercised by OS watcher
-        if getattr(event, "is_directory", False):
-            return
-        self._schedule_change()
-
-
-class LocalKnowledgeWatcher:
-    """Debounced watchdog wrapper for approved local knowledge sources."""
-
-    def __init__(self, get_graph: Callable[[], Any], *, debounce_seconds: float = 5.0, hooks: Any = None):
-        self._get_graph = get_graph
-        self._debounce_seconds = debounce_seconds
-        self._hooks = hooks
-        self._lock = threading.Lock()
-        self._watched: Dict[str, Dict[str, Any]] = {}
-        # watchdog is optional; these stay None when it is not installed.
-        self._observer_cls: Optional[Any] = None
-        self._event_handler_base: Optional[Any] = None
-        self._import_error = ""
-        try:
-            from watchdog.events import FileSystemEventHandler
-            from watchdog.observers import Observer
-
-            self._observer_cls = Observer
-            self._event_handler_base = FileSystemEventHandler
-        except Exception as exc:  # pragma: no cover - depends on optional dependency
-            self._import_error = str(exc)
-
-    @property
-    def available(self) -> bool:
-        return self._observer_cls is not None and self._event_handler_base is not None
-
-    def status(self) -> Dict[str, Any]:
-        with self._lock:
-            active = {
-                source_id: {
-                    "root_path": item["source"].get("root_path"),
-                    "last_event_at": item.get("last_event_at"),
-                    "last_indexed_at": item.get("last_indexed_at"),
-                    "last_error": item.get("last_error"),
-                }
-                for source_id, item in self._watched.items()
-            }
-        return {
-            "available": self.available,
-            "error": "" if self.available else self._import_error or "watchdog is not installed",
-            "debounce_seconds": self._debounce_seconds,
-            "active": active,
-        }
-
-    def restore_enabled_sources(self) -> Dict[str, Any]:
-        graph = self._get_graph()
-        if graph is None:
-            return {"restored": 0, "available": self.available}
-        restored = 0
-        try:
-            for source in graph.local_sources().get("sources", []):
-                if source.get("watch_enabled"):
-                    consent = source.get("consent") or {}
-                    restored_source = {
-                        **source,
-                        # Older sources had no explicit scope. Their local
-                        # knowledge belongs to the personal Brain, never every
-                        # workspace.
-                        "workspace_id": source.get("workspace_id") or consent.get("workspace_id") or "personal",
-                    }
-                    result = self.start_source(restored_source)
-                    if result.get("watching"):
-                        restored += 1
-        except Exception as exc:
-            logging.warning("local knowledge watcher restore failed: %s", exc)
-        return {"restored": restored, "available": self.available}
-
-    def start_source(self, source: Dict[str, Any]) -> Dict[str, Any]:
-        source_id = str(source.get("id") or "")
-        root_path = str(source.get("root_path") or "")
-        if not source_id or not root_path:
-            return {"watching": False, "error": "source_id and root_path are required"}
-        if not self.available:
-            return {"watching": False, "source_id": source_id, "error": self._import_error or "watchdog is not installed"}
-        root = Path(root_path).expanduser().resolve()
-        if not root.exists() or not root.is_dir():
-            return {"watching": False, "source_id": source_id, "error": "source folder is not available"}
-
-        self.stop_source(source_id)
-
-        handler_base = self._event_handler_base
-        observer_cls = self._observer_cls
-        if handler_base is None or observer_cls is None:
-            return {
-                "watching": False,
-                "source_id": source_id,
-                "error": self._import_error or "watchdog is not installed",
-            }
-
-        class Handler(_LocalWatchHandler, handler_base):  # type: ignore[misc, valid-type]
-            def __init__(handler_self):
-                handler_base.__init__(handler_self)  # type: ignore[misc]
-                _LocalWatchHandler.__init__(handler_self, lambda: self._schedule(source_id))
-
-        observer = observer_cls()
-        try:
-            observer.schedule(Handler(), str(root), recursive=True)
-            observer.start()
-        except Exception as exc:
-            logging.warning("local knowledge watcher start failed for %s: %s", root, exc)
-            return {"watching": False, "source_id": source_id, "error": str(exc)}
-
-        with self._lock:
-            self._watched[source_id] = {
-                "observer": observer,
-                "timer": None,
-                "source": dict(source),
-                "last_event_at": None,
-                "last_indexed_at": None,
-                "last_error": None,
-            }
-        return {"watching": True, "source_id": source_id, "root_path": str(root)}
-
-    def stop_source(self, source_id: str) -> Dict[str, Any]:
-        with self._lock:
-            item = self._watched.pop(source_id, None)
-        if not item:
-            return {"stopped": False, "source_id": source_id}
-        timer = item.get("timer")
-        if timer:
-            timer.cancel()
-        observer = item.get("observer")
-        try:
-            if observer is None:
-                raise RuntimeError("watch entry has no observer")
-            observer.stop()
-            observer.join(timeout=3)
-        except Exception as exc:
-            logging.warning("local knowledge watcher stop failed for %s: %s", source_id, exc)
-        return {"stopped": True, "source_id": source_id}
-
-    def stop_all(self) -> None:
-        for source_id in list(self.status().get("active", {}).keys()):
-            self.stop_source(source_id)
-
-    def _schedule(self, source_id: str) -> None:
-        with self._lock:
-            item = self._watched.get(source_id)
-            if not item:
-                return
-            timer = item.get("timer")
-            if timer:
-                timer.cancel()
-            item["last_event_at"] = _now_seconds()
-            timer = threading.Timer(self._debounce_seconds, self._run_index, args=(source_id,))
-            timer.daemon = True
-            item["timer"] = timer
-            timer.start()
-
-    def _run_index(self, source_id: str) -> None:
-        with self._lock:
-            item = self._watched.get(source_id)
-            if not item:
-                return
-            source = dict(item["source"])
-            item["timer"] = None
-        graph = self._get_graph()
-        if graph is None:
-            return
-        consent = source.get("consent") or {}
-        root = source.get("root_path")
-        if self._hooks is not None:
-            self._hooks.fire_hook("pre_index", "folder.reindex",
-                                  payload={"source_id": source_id, "root_path": root, "trigger": "watch"})
-        try:
-            result = graph.index_local_folder(
-                Path(source["root_path"]),
-                include_ocr=bool(source.get("include_ocr")),
-                watch_enabled=True,
-                user_email=consent.get("approved_by"),
-                workspace_id=source.get("workspace_id") or consent.get("workspace_id") or "personal",
-                consent=consent,
-                source_id_override=source_id,
-            )
-            with self._lock:
-                if source_id in self._watched:
-                    self._watched[source_id]["last_indexed_at"] = _now_seconds()
-                    self._watched[source_id]["last_error"] = None
-            if self._hooks is not None:
-                self._hooks.fire_hook("post_index", "folder.reindex",
-                                      payload={"source_id": source_id, "root_path": root, "trigger": "watch", "status": "ok"})
-                counts = (result.get("counts") or {}) if isinstance(result, dict) else {}
-                if int(counts.get("indexed") or 0) + int(counts.get("deleted") or 0) > 0:
-                    self._hooks.fire_hook(
-                        "post_tool",
-                        "tool.kg_ingest.local_folder",
-                        payload={
-                            "tool": "kg_ingest.local_folder",
-                            "status": "ok",
-                            "source": "ingestion",
-                            "source_type": "local_folder",
-                            "source_id": source_id,
-                        },
-                        user_email=consent.get("approved_by"),
-                        workspace_id=source.get("workspace_id"),
-                    )
-        except Exception as exc:
-            logging.warning("local knowledge watcher reindex failed for %s: %s", source_id, exc)
-            with self._lock:
-                if source_id in self._watched:
-                    self._watched[source_id]["last_error"] = str(exc)
-            if self._hooks is not None:
-                self._hooks.fire_hook("post_index", "folder.reindex",
-                                      payload={"source_id": source_id, "root_path": root, "trigger": "watch", "status": "error", "error": str(exc)})
-
-
-def _now_seconds() -> float:
-    import time
-
-    return time.time()
-
-
-def create_local_knowledge_router(
-    *,
-    get_graph: Callable[[], Any],
-    require_graph: Callable[[], None],
-    require_user: Callable[[Request], str],
-    require_local_user: Callable[[Request], str],
-    local_permission_response: Callable[..., dict],
-    require_local_approval: Callable[..., None],
-    watcher: Optional[LocalKnowledgeWatcher] = None,
-    hooks: Any = None,
-    workspace_service: Any = None,
-) -> APIRouter:
-    router = APIRouter()
-
-    def graph():
-        require_graph()
-        return get_graph()
-
-    def write_workspace(request: Request, user: str) -> Optional[str]:
-        requested = request.headers.get("X-Workspace-Id")
-        requested = requested.strip() if requested and requested.strip() else None
-        if workspace_service is None:
-            return requested
-        try:
-            return workspace_service.resolve_write_scope(requested, user or None)
-        except PermissionError as exc:
-            raise HTTPException(status_code=403, detail=str(exc)) from exc
-
-    @router.get("/knowledge-graph/local/roots")
-    async def knowledge_graph_local_roots(request: Request):
-        require_user(request)
-        return graph().discover_local_roots()
-
-    @router.get("/knowledge-graph/local/sources")
-    async def knowledge_graph_local_sources(request: Request):
-        require_user(request)
-        payload = graph().local_sources()
-        watch_status: Dict[str, Any] = (
-            watcher.status() if watcher else {"available": False, "active": {}}
-        )
-        active: Dict[str, Any] = dict(watch_status.get("active") or {})
-        for source in payload.get("sources", []):
-            source["watch_active"] = source.get("id") in active
-            source["watch_status"] = active.get(source.get("id"))
-        payload["watch"] = watch_status
-        return payload
-
-    @router.get("/knowledge-graph/local/health")
-    async def knowledge_graph_local_health(request: Request, error_samples: int = 3):
-        """Per-folder memory state (v9.9.7): indexing coverage, failures with
-        their stored reasons, and watch state.
-
-        Vector freshness rides along as an explicitly **global** figure — the
-        vector index is not per-folder, and claiming otherwise would invent a
-        number.
-        """
-        require_user(request)
-        kg = graph()
-        payload = kg.local_source_health(error_samples=error_samples)
-        watch_status: Dict[str, Any] = (
-            watcher.status() if watcher else {"available": False, "active": {}}
-        )
-        active = dict(watch_status.get("active") or {})
-        for folder in payload.get("folders", []):
-            folder["watch_active"] = folder.get("id") in active
-        payload["watch"] = watch_status
-        try:
-            payload["vector_freshness_global"] = kg.vector_freshness()
-        except Exception as exc:  # noqa: BLE001 — one missing signal, not a 500
-            payload["vector_freshness_global"] = {
-                "status": "unavailable", "detail": str(exc),
-                "pending_items": 0, "total_items": 0,
-            }
-        return payload
-
-    @router.get("/knowledge-graph/local/watch/status")
-    async def knowledge_graph_local_watch_status(request: Request):
-        require_user(request)
-        graph()
-        return watcher.status() if watcher else {"available": False, "active": {}, "error": "watcher unavailable"}
-
-    @router.post("/knowledge-graph/local/watch/stop")
-    async def knowledge_graph_local_watch_stop(req: LocalKnowledgeWatchRequest, request: Request):
-        require_user(request)
-        kg = graph()
-        try:
-            kg.set_local_source_watch(req.source_id, False)
-        except ValueError as exc:
-            raise HTTPException(status_code=404, detail=str(exc)) from exc
-        result = watcher.stop_source(req.source_id) if watcher else {"stopped": False, "source_id": req.source_id}
-        return {"status": "ok", "watch": result}
-
-    @router.post("/knowledge-graph/local/tree")
-    async def knowledge_graph_local_tree(req: LocalTreeRequest, request: Request):
-        current_user = require_local_user(request)
-        kg = graph()
-        if not req.approved:
-            return local_permission_response(req.path, "list", current_user)
-        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
-        try:
-            return kg.preview_local_tree(Path(req.path), max_items=req.max_items)
-        except ValueError as exc:
-            raise HTTPException(status_code=400, detail=str(exc)) from exc
-
-    @router.post("/knowledge-graph/local/audit")
-    async def knowledge_graph_local_audit(req: LocalKnowledgeAuditRequest, request: Request):
-        current_user = require_local_user(request)
-        kg = graph()
-        if not req.approved:
-            return local_permission_response(req.path, "list", current_user)
-        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
-        try:
-            return kg.audit_local_folder(Path(req.path), include_ocr=req.include_ocr, max_files=req.max_files)
-        except ValueError as exc:
-            raise HTTPException(status_code=400, detail=str(exc)) from exc
-
-    @router.post("/knowledge-graph/local/index")
-    async def knowledge_graph_local_index(req: LocalKnowledgeIndexRequest, request: Request):
-        current_user = require_local_user(request)
-        workspace_id = write_workspace(request, current_user)
-        kg = graph()
-        if not req.approved:
-            return local_permission_response(req.path, "read", current_user)
-        require_local_approval(token=req.approval_token, path=req.path, action="read", user_email=current_user)
-        consent = {
-            **(req.consent or {}),
-            "approved_by": current_user,
-            "workspace_id": workspace_id or "personal",
-        }
-        source_id_override = None
-        try:
-            target_root = Path(req.path).expanduser().resolve()
-            target_scope = workspace_id or "personal"
-            for source in kg.local_sources().get("sources", []):
-                source_consent = source.get("consent") or {}
-                source_scope = source_consent.get("workspace_id") or "personal"
-                source_root_value = str(source.get("root_path") or "").strip()
-                if not source_root_value:
-                    continue
-                source_root = Path(source_root_value).expanduser().resolve()
-                if source_scope == target_scope and source_root == target_root:
-                    source_id_override = source.get("id")
-                    break
-        except Exception:
-            # Source reuse is a compatibility optimization; indexing still has
-            # a deterministic workspace-scoped ID when discovery is unavailable.
-            source_id_override = None
-        if hooks is not None:
-            hooks.fire_hook("pre_index", "folder.index",
-                            payload={"root_path": req.path, "trigger": "connect", "watch": req.watch_enabled},
-                            user_email=current_user, workspace_id=workspace_id)
-        try:
-            result = kg.index_local_folder(
-                Path(req.path),
-                include_ocr=req.include_ocr,
-                watch_enabled=req.watch_enabled,
-                user_email=current_user,
-                workspace_id=workspace_id or "personal",
-                consent=consent,
-                max_files=req.max_files,
-                source_id_override=source_id_override,
-            )
-        except ValueError as exc:
-            if hooks is not None:
-                hooks.fire_hook("post_index", "folder.index",
-                                payload={"root_path": req.path, "trigger": "connect", "status": "error", "error": str(exc)},
-                                user_email=current_user, workspace_id=workspace_id)
-            raise HTTPException(status_code=400, detail=str(exc)) from exc
-        if hooks is not None:
-            _idx = (result.get("index") or {}) if isinstance(result, dict) else {}
-            hooks.fire_hook("post_index", "folder.index",
-                            payload={"root_path": req.path, "trigger": "connect", "status": "ok",
-                                     "indexed": _idx.get("indexed") or (result or {}).get("indexed")},
-                            user_email=current_user, workspace_id=workspace_id)
-            counts = (result.get("counts") or {}) if isinstance(result, dict) else {}
-            if int(counts.get("indexed") or 0) + int(counts.get("deleted") or 0) > 0:
-                hooks.fire_hook(
-                    "post_tool",
-                    "tool.kg_ingest.local_folder",
-                    payload={
-                        "tool": "kg_ingest.local_folder",
-                        "status": "ok",
-                        "source": "ingestion",
-                        "source_type": "local_folder",
-                        "source_id": (result.get("source") or {}).get("id") if isinstance(result, dict) else None,
-                    },
-                    user_email=current_user,
-                    workspace_id=workspace_id,
-                )
-
-        if watcher:
-            if req.watch_enabled:
-                source_payload = {
-                    **result.get("source", {}),
-                    "workspace_id": workspace_id or "personal",
-                    "consent": consent,
-                }
-                result["watch"] = watcher.start_source(source_payload)
-            else:
-                result["watch"] = watcher.stop_source(result.get("source", {}).get("id", ""))
-        return result
-
-    return router

package/latticeai/services/memory_service/__init__.py

@@ -1,52 +0,0 @@
-"""Long-Term Memory platform + Memory Manager (v3.2.0).
-
-Parts 7, 8 and 13. Lattice AI already persists memory in several real stores;
-before this service they were unrelated. ``MemoryService`` unifies them behind
-one façade and adds a Memory Manager that reports usage / sources / health /
-size / type and supports recall / inspect / prune / compact / rebuild / clear.
-
-Memory tiers and their real backing store (nothing is fabricated — a tier with
-no backing reports ``unavailable``):
-
-* **workspace**     — personal workspace memories (``WorkspaceOS`` memories)
-* **project**       — memories scoped to a non-personal (organization) workspace
-* **agent**         — agent memory snapshots captured during runs
-* **conversation**  — chat history conversations
-* **graph**         — Knowledge Graph nodes (entities + relations)
-* **vector**        — local embedding vector index
-
-The service never invents counts or health: every number is read from the
-underlying store, and missing stores surface as ``unavailable``.
-
-Split into cohesive submodules in v11.3.0 (no behaviour change): ``constants``
-(tier vocabulary + the visual passthrough), ``stores`` (reads over the real
-backends), ``manager`` (the cross-tier report), ``brief`` and ``proof`` (the
-home screen's briefing and its evidence), ``recall`` (unified retrieval and
-per-tier inspection), ``maintenance`` (the mutating half), and ``service`` (the
-composed class). This module re-exports every name the single file exposed, so
-``latticeai.services.memory_service.X`` keeps working.
-"""
-
-from __future__ import annotations
-
-# The single file had no ``__all__``, so its public surface was "every module
-# global" — including the names it imported for its own use. The redundant-alias
-# form reproduces exactly that surface and marks each name as a deliberate
-# re-export rather than a leftover import.
-#
-# Stubbing note: rebinding one of these *here* changes only this module's name.
-# The submodule that calls it holds its own reference, so a test standing in for
-# ``LOGGER`` or ``_now`` patches the submodule that uses it.
-# noqa on the next line, not a redundant alias: the single file exposed this
-# name *renamed* (``memory_service._now``), and reproducing that surface exactly
-# is the point — importing it as ``now_iso`` would add a name the module never had.
-from latticeai.core.timeutil import now_iso as _now  # noqa: F401
-from latticeai.core.workspace_os_utils import _file_size as _file_size
-
-from .constants import LOGGER as LOGGER
-from .constants import MAX_RECALL_THUMBNAIL_CHARS as MAX_RECALL_THUMBNAIL_CHARS
-from .constants import TIERS as TIERS
-from .constants import WORKSPACE_KINDS as WORKSPACE_KINDS
-from .constants import MemoryServiceError as MemoryServiceError
-from .constants import _visual_fields as _visual_fields
-from .service import MemoryService as MemoryService

package/latticeai/services/memory_service/_contract.py

@@ -1,100 +0,0 @@
-"""The seam the memory mixins share.
-
-``MemoryService`` is assembled from the store-read, manager, brief, proof,
-recall and maintenance mixins. Every one of them reads constructor state it
-does not own, and most of them call methods another mixin implements. That
-contract existed as an unwritten convention inside one class; this module
-writes it down.
-
-Typing-only, exactly like :mod:`lattice_brain.portability._contract`: the
-declarations below are never executed as behaviour — each body is a bare
-``raise NotImplementedError`` that the composed class always overrides — so the
-MRO and every method resolution stay what the single-file class had. Adding a
-cross-mixin call without declaring it here is a type error.
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-
-class MemoryCore:
-    """What any memory mixin may assume about ``self``.
-
-    Members are declared, not implemented: the implementation lives in
-    ``service.py`` (constructor state) or in whichever mixin owns the method.
-    """
-
-    # ── State owned by MemoryService.__init__ ────────────────────────────────
-    _store: Any
-    _kg: Any
-    _enable_graph: bool
-    _data_dir: Path
-    _history_file: Path
-    _conversation_store: Any
-
-    # ── stores.py: the reads every other surface is built on ─────────────────
-    def _workspace_memories(
-        self, *, user_email: Optional[str], workspace_id: Optional[str]
-    ) -> List[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _all_memories(self) -> List[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _snapshots(self, *, workspace_id: Optional[str]) -> List[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _conversations(self) -> List[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _scoped_conversations(
-        self, *, user_email: Optional[str], workspace_id: Optional[str]
-    ) -> List[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _kg_stats(self) -> Optional[Dict[str, Any]]:
-        raise NotImplementedError
-
-    def _kg_index(self) -> Optional[Dict[str, Any]]:
-        raise NotImplementedError
-
-    # ── manager.py: the cross-tier report the brief and the proof rest on ────
-    def manager(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        raise NotImplementedError
-
-    # ── proof.py / recall.py: what the brief composes ────────────────────────
-    def brain_proof(
-        self,
-        *,
-        user_email: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        active_model: Optional[str] = None,
-        recall_query: str = "",
-        limit: int = 3,
-    ) -> Dict[str, Any]:
-        raise NotImplementedError
-
-    def recall(
-        self,
-        query: str,
-        *,
-        user_email: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        limit: int = 20,
-    ) -> Dict[str, Any]:
-        raise NotImplementedError
-
-    # ── maintenance.py: clear() delegates every scope to prune() ─────────────
-    def prune(
-        self,
-        *,
-        ids: Optional[List[str]] = None,
-        kind: Optional[str] = None,
-        user_email: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        raise NotImplementedError