ltcai 9.0.0 → 9.2.0

package/latticeai/services/local_knowledge.py

@@ -0,0 +1,423 @@
+"""Local folder knowledge-source API and optional filesystem watcher."""
+
+import logging
+import threading
+from pathlib import Path
+from typing import Any, Callable, Dict, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+
+class LocalTreeRequest(BaseModel):
+    path: str
+    max_items: int = 200
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeAuditRequest(BaseModel):
+    path: str
+    include_ocr: bool = False
+    max_files: int = 50_000
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeIndexRequest(BaseModel):
+    path: str
+    include_ocr: bool = False
+    watch_enabled: bool = False
+    max_files: int = 5_000
+    consent: Dict[str, Any] = Field(default_factory=dict)
+    approved: bool = False
+    approval_token: Optional[str] = None
+
+
+class LocalKnowledgeWatchRequest(BaseModel):
+    source_id: str
+
+
+class _LocalWatchHandler:
+    def __init__(self, schedule_change: Callable[[], None]):
+        self._schedule_change = schedule_change
+
+    def on_any_event(self, event):  # pragma: no cover - exercised by OS watcher
+        if getattr(event, "is_directory", False):
+            return
+        self._schedule_change()
+
+
+class LocalKnowledgeWatcher:
+    """Debounced watchdog wrapper for approved local knowledge sources."""
+
+    def __init__(self, get_graph: Callable[[], Any], *, debounce_seconds: float = 5.0, hooks: Any = None):
+        self._get_graph = get_graph
+        self._debounce_seconds = debounce_seconds
+        self._hooks = hooks
+        self._lock = threading.Lock()
+        self._watched: Dict[str, Dict[str, Any]] = {}
+        self._observer_cls = None
+        self._event_handler_base = None
+        self._import_error = ""
+        try:
+            from watchdog.events import FileSystemEventHandler
+            from watchdog.observers import Observer
+
+            self._observer_cls = Observer
+            self._event_handler_base = FileSystemEventHandler
+        except Exception as exc:  # pragma: no cover - depends on optional dependency
+            self._import_error = str(exc)
+
+    @property
+    def available(self) -> bool:
+        return self._observer_cls is not None and self._event_handler_base is not None
+
+    def status(self) -> Dict[str, Any]:
+        with self._lock:
+            active = {
+                source_id: {
+                    "root_path": item["source"].get("root_path"),
+                    "last_event_at": item.get("last_event_at"),
+                    "last_indexed_at": item.get("last_indexed_at"),
+                    "last_error": item.get("last_error"),
+                }
+                for source_id, item in self._watched.items()
+            }
+        return {
+            "available": self.available,
+            "error": "" if self.available else self._import_error or "watchdog is not installed",
+            "debounce_seconds": self._debounce_seconds,
+            "active": active,
+        }
+
+    def restore_enabled_sources(self) -> Dict[str, Any]:
+        graph = self._get_graph()
+        if graph is None:
+            return {"restored": 0, "available": self.available}
+        restored = 0
+        try:
+            for source in graph.local_sources().get("sources", []):
+                if source.get("watch_enabled"):
+                    consent = source.get("consent") or {}
+                    restored_source = {
+                        **source,
+                        # Older sources had no explicit scope. Their local
+                        # knowledge belongs to the personal Brain, never every
+                        # workspace.
+                        "workspace_id": source.get("workspace_id") or consent.get("workspace_id") or "personal",
+                    }
+                    result = self.start_source(restored_source)
+                    if result.get("watching"):
+                        restored += 1
+        except Exception as exc:
+            logging.warning("local knowledge watcher restore failed: %s", exc)
+        return {"restored": restored, "available": self.available}
+
+    def start_source(self, source: Dict[str, Any]) -> Dict[str, Any]:
+        source_id = str(source.get("id") or "")
+        root_path = str(source.get("root_path") or "")
+        if not source_id or not root_path:
+            return {"watching": False, "error": "source_id and root_path are required"}
+        if not self.available:
+            return {"watching": False, "source_id": source_id, "error": self._import_error or "watchdog is not installed"}
+        root = Path(root_path).expanduser().resolve()
+        if not root.exists() or not root.is_dir():
+            return {"watching": False, "source_id": source_id, "error": "source folder is not available"}
+
+        self.stop_source(source_id)
+
+        class Handler(_LocalWatchHandler, self._event_handler_base):  # type: ignore[misc, valid-type]
+            def __init__(handler_self):
+                self._event_handler_base.__init__(handler_self)
+                _LocalWatchHandler.__init__(handler_self, lambda: self._schedule(source_id))
+
+        observer = self._observer_cls()
+        try:
+            observer.schedule(Handler(), str(root), recursive=True)
+            observer.start()
+        except Exception as exc:
+            logging.warning("local knowledge watcher start failed for %s: %s", root, exc)
+            return {"watching": False, "source_id": source_id, "error": str(exc)}
+
+        with self._lock:
+            self._watched[source_id] = {
+                "observer": observer,
+                "timer": None,
+                "source": dict(source),
+                "last_event_at": None,
+                "last_indexed_at": None,
+                "last_error": None,
+            }
+        return {"watching": True, "source_id": source_id, "root_path": str(root)}
+
+    def stop_source(self, source_id: str) -> Dict[str, Any]:
+        with self._lock:
+            item = self._watched.pop(source_id, None)
+        if not item:
+            return {"stopped": False, "source_id": source_id}
+        timer = item.get("timer")
+        if timer:
+            timer.cancel()
+        observer = item.get("observer")
+        try:
+            observer.stop()
+            observer.join(timeout=3)
+        except Exception as exc:
+            logging.warning("local knowledge watcher stop failed for %s: %s", source_id, exc)
+        return {"stopped": True, "source_id": source_id}
+
+    def stop_all(self) -> None:
+        for source_id in list(self.status().get("active", {}).keys()):
+            self.stop_source(source_id)
+
+    def _schedule(self, source_id: str) -> None:
+        with self._lock:
+            item = self._watched.get(source_id)
+            if not item:
+                return
+            timer = item.get("timer")
+            if timer:
+                timer.cancel()
+            item["last_event_at"] = _now_seconds()
+            timer = threading.Timer(self._debounce_seconds, self._run_index, args=(source_id,))
+            timer.daemon = True
+            item["timer"] = timer
+            timer.start()
+
+    def _run_index(self, source_id: str) -> None:
+        with self._lock:
+            item = self._watched.get(source_id)
+            if not item:
+                return
+            source = dict(item["source"])
+            item["timer"] = None
+        graph = self._get_graph()
+        if graph is None:
+            return
+        consent = source.get("consent") or {}
+        root = source.get("root_path")
+        if self._hooks is not None:
+            self._hooks.fire_hook("pre_index", "folder.reindex",
+                                  payload={"source_id": source_id, "root_path": root, "trigger": "watch"})
+        try:
+            result = graph.index_local_folder(
+                Path(source["root_path"]),
+                include_ocr=bool(source.get("include_ocr")),
+                watch_enabled=True,
+                user_email=consent.get("approved_by"),
+                workspace_id=source.get("workspace_id") or consent.get("workspace_id") or "personal",
+                consent=consent,
+                source_id_override=source_id,
+            )
+            with self._lock:
+                if source_id in self._watched:
+                    self._watched[source_id]["last_indexed_at"] = _now_seconds()
+                    self._watched[source_id]["last_error"] = None
+            if self._hooks is not None:
+                self._hooks.fire_hook("post_index", "folder.reindex",
+                                      payload={"source_id": source_id, "root_path": root, "trigger": "watch", "status": "ok"})
+                counts = (result.get("counts") or {}) if isinstance(result, dict) else {}
+                if int(counts.get("indexed") or 0) + int(counts.get("deleted") or 0) > 0:
+                    self._hooks.fire_hook(
+                        "post_tool",
+                        "tool.kg_ingest.local_folder",
+                        payload={
+                            "tool": "kg_ingest.local_folder",
+                            "status": "ok",
+                            "source": "ingestion",
+                            "source_type": "local_folder",
+                            "source_id": source_id,
+                        },
+                        user_email=consent.get("approved_by"),
+                        workspace_id=source.get("workspace_id"),
+                    )
+        except Exception as exc:
+            logging.warning("local knowledge watcher reindex failed for %s: %s", source_id, exc)
+            with self._lock:
+                if source_id in self._watched:
+                    self._watched[source_id]["last_error"] = str(exc)
+            if self._hooks is not None:
+                self._hooks.fire_hook("post_index", "folder.reindex",
+                                      payload={"source_id": source_id, "root_path": root, "trigger": "watch", "status": "error", "error": str(exc)})
+
+
+def _now_seconds() -> float:
+    import time
+
+    return time.time()
+
+
+def create_local_knowledge_router(
+    *,
+    get_graph: Callable[[], Any],
+    require_graph: Callable[[], None],
+    require_user: Callable[[Request], str],
+    require_local_user: Callable[[Request], str],
+    local_permission_response: Callable[..., dict],
+    require_local_approval: Callable[..., None],
+    watcher: Optional[LocalKnowledgeWatcher] = None,
+    hooks: Any = None,
+    workspace_service: Any = None,
+) -> APIRouter:
+    router = APIRouter()
+
+    def graph():
+        require_graph()
+        return get_graph()
+
+    def write_workspace(request: Request, user: str) -> Optional[str]:
+        requested = request.headers.get("X-Workspace-Id")
+        requested = requested.strip() if requested and requested.strip() else None
+        if workspace_service is None:
+            return requested
+        try:
+            return workspace_service.resolve_write_scope(requested, user or None)
+        except PermissionError as exc:
+            raise HTTPException(status_code=403, detail=str(exc)) from exc
+
+    @router.get("/knowledge-graph/local/roots")
+    async def knowledge_graph_local_roots(request: Request):
+        require_user(request)
+        return graph().discover_local_roots()
+
+    @router.get("/knowledge-graph/local/sources")
+    async def knowledge_graph_local_sources(request: Request):
+        require_user(request)
+        payload = graph().local_sources()
+        watch_status = watcher.status() if watcher else {"available": False, "active": {}}
+        active = watch_status.get("active", {})
+        for source in payload.get("sources", []):
+            source["watch_active"] = source.get("id") in active
+            source["watch_status"] = active.get(source.get("id"))
+        payload["watch"] = watch_status
+        return payload
+
+    @router.get("/knowledge-graph/local/watch/status")
+    async def knowledge_graph_local_watch_status(request: Request):
+        require_user(request)
+        graph()
+        return watcher.status() if watcher else {"available": False, "active": {}, "error": "watcher unavailable"}
+
+    @router.post("/knowledge-graph/local/watch/stop")
+    async def knowledge_graph_local_watch_stop(req: LocalKnowledgeWatchRequest, request: Request):
+        require_user(request)
+        kg = graph()
+        try:
+            kg.set_local_source_watch(req.source_id, False)
+        except ValueError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        result = watcher.stop_source(req.source_id) if watcher else {"stopped": False, "source_id": req.source_id}
+        return {"status": "ok", "watch": result}
+
+    @router.post("/knowledge-graph/local/tree")
+    async def knowledge_graph_local_tree(req: LocalTreeRequest, request: Request):
+        current_user = require_local_user(request)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "list", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
+        try:
+            return kg.preview_local_tree(Path(req.path), max_items=req.max_items)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/knowledge-graph/local/audit")
+    async def knowledge_graph_local_audit(req: LocalKnowledgeAuditRequest, request: Request):
+        current_user = require_local_user(request)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "list", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="list", user_email=current_user)
+        try:
+            return kg.audit_local_folder(Path(req.path), include_ocr=req.include_ocr, max_files=req.max_files)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+    @router.post("/knowledge-graph/local/index")
+    async def knowledge_graph_local_index(req: LocalKnowledgeIndexRequest, request: Request):
+        current_user = require_local_user(request)
+        workspace_id = write_workspace(request, current_user)
+        kg = graph()
+        if not req.approved:
+            return local_permission_response(req.path, "read", current_user)
+        require_local_approval(token=req.approval_token, path=req.path, action="read", user_email=current_user)
+        consent = {
+            **(req.consent or {}),
+            "approved_by": current_user,
+            "workspace_id": workspace_id or "personal",
+        }
+        source_id_override = None
+        try:
+            target_root = Path(req.path).expanduser().resolve()
+            target_scope = workspace_id or "personal"
+            for source in kg.local_sources().get("sources", []):
+                source_consent = source.get("consent") or {}
+                source_scope = source_consent.get("workspace_id") or "personal"
+                source_root_value = str(source.get("root_path") or "").strip()
+                if not source_root_value:
+                    continue
+                source_root = Path(source_root_value).expanduser().resolve()
+                if source_scope == target_scope and source_root == target_root:
+                    source_id_override = source.get("id")
+                    break
+        except Exception:
+            # Source reuse is a compatibility optimization; indexing still has
+            # a deterministic workspace-scoped ID when discovery is unavailable.
+            source_id_override = None
+        if hooks is not None:
+            hooks.fire_hook("pre_index", "folder.index",
+                            payload={"root_path": req.path, "trigger": "connect", "watch": req.watch_enabled},
+                            user_email=current_user, workspace_id=workspace_id)
+        try:
+            result = kg.index_local_folder(
+                Path(req.path),
+                include_ocr=req.include_ocr,
+                watch_enabled=req.watch_enabled,
+                user_email=current_user,
+                workspace_id=workspace_id or "personal",
+                consent=consent,
+                max_files=req.max_files,
+                source_id_override=source_id_override,
+            )
+        except ValueError as exc:
+            if hooks is not None:
+                hooks.fire_hook("post_index", "folder.index",
+                                payload={"root_path": req.path, "trigger": "connect", "status": "error", "error": str(exc)},
+                                user_email=current_user, workspace_id=workspace_id)
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        if hooks is not None:
+            _idx = (result.get("index") or {}) if isinstance(result, dict) else {}
+            hooks.fire_hook("post_index", "folder.index",
+                            payload={"root_path": req.path, "trigger": "connect", "status": "ok",
+                                     "indexed": _idx.get("indexed") or (result or {}).get("indexed")},
+                            user_email=current_user, workspace_id=workspace_id)
+            counts = (result.get("counts") or {}) if isinstance(result, dict) else {}
+            if int(counts.get("indexed") or 0) + int(counts.get("deleted") or 0) > 0:
+                hooks.fire_hook(
+                    "post_tool",
+                    "tool.kg_ingest.local_folder",
+                    payload={
+                        "tool": "kg_ingest.local_folder",
+                        "status": "ok",
+                        "source": "ingestion",
+                        "source_type": "local_folder",
+                        "source_id": (result.get("source") or {}).get("id") if isinstance(result, dict) else None,
+                    },
+                    user_email=current_user,
+                    workspace_id=workspace_id,
+                )
+
+        if watcher:
+            if req.watch_enabled:
+                source_payload = {
+                    **result.get("source", {}),
+                    "workspace_id": workspace_id or "personal",
+                    "consent": consent,
+                }
+                result["watch"] = watcher.start_source(source_payload)
+            else:
+                result["watch"] = watcher.stop_source(result.get("source", {}).get("id", ""))
+        return result
+
+    return router

package/latticeai/services/memory_service.py

@@ -22,11 +22,12 @@ underlying store, and missing stores surface as ``unavailable``.
 from __future__ import annotations
 
 import json
-from datetime import datetime
+import logging
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from latticeai.core.workspace_os_utils import _file_size
+from latticeai.core.timeutil import now_iso as _now
 
 # Personal workspace memory kinds (from WorkspaceOS.MEMORY_KINDS).
 WORKSPACE_KINDS = (
@@ -40,10 +41,11 @@ WORKSPACE_KINDS = (
 )
 
 TIERS = ("workspace", "project", "agent", "conversation", "graph", "vector")
+LOGGER = logging.getLogger(__name__)
 
 
-def _now() -> str:
-    return datetime.now().isoformat(timespec="seconds")
+class MemoryServiceError(RuntimeError):
+    """Raised when a configured memory backend cannot be read reliably."""
 
 
 class MemoryService:
@@ -70,20 +72,23 @@ class MemoryService:
     def _workspace_memories(self, *, user_email: Optional[str], workspace_id: Optional[str]) -> List[Dict[str, Any]]:
         try:
             return list(self._store.list_memories(user_email=user_email, workspace_id=workspace_id).get("memories", []))
-        except Exception:
-            return []
+        except Exception as exc:
+            LOGGER.exception("workspace memory read failed")
+            raise MemoryServiceError("workspace memory backend unavailable") from exc
 
     def _all_memories(self) -> List[Dict[str, Any]]:
         try:
             return list(self._store.list_memories().get("memories", []))
-        except Exception:
-            return []
+        except Exception as exc:
+            LOGGER.exception("global memory read failed")
+            raise MemoryServiceError("memory backend unavailable") from exc
 
     def _snapshots(self, *, workspace_id: Optional[str]) -> List[Dict[str, Any]]:
         try:
             return list(self._store.list_memory_snapshots(workspace_id=workspace_id, limit=200).get("snapshots", []))
-        except Exception:
-            return []
+        except Exception as exc:
+            LOGGER.exception("memory snapshot read failed")
+            raise MemoryServiceError("memory snapshot backend unavailable") from exc
 
     def _conversations(self) -> List[Dict[str, Any]]:
         if self._conversation_store is not None:
@@ -92,15 +97,17 @@ class MemoryService:
                 for item in self._conversation_store.history():
                     grouped.setdefault(item.get("conversation_id") or "legacy-previous-history", []).append(item)
                 return [{"id": conv_id, "messages": msgs} for conv_id, msgs in grouped.items()]
-            except Exception:
-                return []
+            except Exception as exc:
+                LOGGER.exception("conversation store read failed")
+                raise MemoryServiceError("conversation backend unavailable") from exc
         if not self._history_file.exists():
             return []
         try:
             with open(self._history_file, "r", encoding="utf-8") as fh:
                 data = json.load(fh)
-        except Exception:
-            return []
+        except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc:
+            LOGGER.exception("legacy conversation history read failed")
+            raise MemoryServiceError("conversation history is unreadable") from exc
         if isinstance(data, dict):
             convs = data.get("conversations")
             if isinstance(convs, list):
@@ -136,6 +143,7 @@ class MemoryService:
         try:
             return self._kg.stats()
         except Exception:
+            LOGGER.exception("knowledge graph stats read failed")
             return None
 
     def _kg_index(self) -> Optional[Dict[str, Any]]:
@@ -144,6 +152,7 @@ class MemoryService:
         try:
             return self._kg.index_status()
         except Exception:
+            LOGGER.exception("knowledge graph index status read failed")
             return None
 
     # ── Memory Manager: sources / usage / health ──────────────────────────
@@ -847,9 +856,12 @@ class MemoryService:
 
         results: List[Dict[str, Any]] = []
 
+        errors: List[Dict[str, str]] = []
         try:
             mem = self._store.search_memories(q, user_email=user_email, limit=limit, workspace_id=workspace_id).get("memories", [])
-        except Exception:
+        except Exception as exc:
+            LOGGER.exception("workspace memory search failed")
+            errors.append({"source": "workspace", "detail": str(exc)})
             mem = []
         for m in mem:
             matched = _matched_terms(m.get("content"), " ".join(m.get("tags") or []), m.get("kind"))
@@ -867,8 +879,15 @@ class MemoryService:
         if self._enable_graph and q:
             try:
                 # KnowledgeGraph.search returns {"query": ..., "matches": [...]}.
-                hits = self._kg.search(q, limit).get("matches", [])
-            except Exception:
+                search_kwargs = (
+                    {"allowed_workspaces": {workspace_id}}
+                    if workspace_id is not None
+                    else {}
+                )
+                hits = self._kg.search(q, limit, **search_kwargs).get("matches", [])
+            except Exception as exc:
+                LOGGER.exception("knowledge graph memory search failed")
+                errors.append({"source": "graph", "detail": str(exc)})
                 hits = []
             for hit in hits[:limit]:
                 matched = _matched_terms(hit.get("title"), hit.get("name"), hit.get("summary"), hit.get("content"))
@@ -898,6 +917,8 @@ class MemoryService:
             "results": results[: max(1, min(limit, 100))],
             "count": len(results),
             "source": "live",
+            "status": "degraded" if errors else "ok",
+            "errors": errors,
             "quality_gate": {
                 "candidates": candidates,
                 "passed": len(results),
@@ -949,6 +970,7 @@ class MemoryService:
             if m.get("id")
         }
         removed: List[str] = []
+        failed: List[Dict[str, str]] = []
         skipped: List[str] = []
         target_ids: List[str] = []
         seen: set = set()
@@ -969,11 +991,15 @@ class MemoryService:
             try:
                 self._store.delete_memory(mid)
                 removed.append(mid)
-            except Exception:
-                continue
+            except Exception as exc:
+                LOGGER.exception("memory deletion failed for %s", mid)
+                failed.append({"id": mid, "detail": str(exc)})
         result: Dict[str, Any] = {"removed": removed, "count": len(removed)}
         if skipped:
             result["skipped"] = skipped
+        if failed:
+            result["failed"] = failed
+            result["status"] = "partial" if removed else "error"
         return result
 
     def compact(
@@ -985,6 +1011,7 @@ class MemoryService:
         """Dedupe workspace memories with identical (kind, content)."""
         seen: set = set()
         removed: List[str] = []
+        failed: List[Dict[str, str]] = []
         # Oldest first so the first occurrence (oldest) is kept.
         memories = list(reversed(self._workspace_memories(user_email=user_email, workspace_id=workspace_id)))
         for m in memories:
@@ -994,11 +1021,18 @@ class MemoryService:
                     try:
                         self._store.delete_memory(m["id"])
                         removed.append(m["id"])
-                    except Exception:
-                        continue
+                    except Exception as exc:
+                        LOGGER.exception("memory compaction deletion failed for %s", m["id"])
+                        failed.append({"id": m["id"], "detail": str(exc)})
             else:
                 seen.add(key)
-        return {"compacted": len(removed), "removed": removed, "remaining": len(seen)}
+        return {
+            "compacted": len(removed),
+            "removed": removed,
+            "remaining": len(seen),
+            "failed": failed,
+            "status": "partial" if failed and removed else "error" if failed else "ok",
+        }
 
     def rebuild(self, target: str = "vector") -> Dict[str, Any]:
         if target in {"vector", "index", "vector_index"}: