ltcai 11.1.0 → 11.2.0

package/latticeai/services/folder_watch.py

@@ -29,6 +29,7 @@ import uuid
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
+from lattice_brain.gates import FeatureGate
 from lattice_brain.ingestion import (
     DEFAULT_FOLDER_EXTENSIONS,
     DEFAULT_MAX_FILE_BYTES,
@@ -46,6 +47,33 @@ WATCH_INTERVAL_ENV = "LATTICEAI_FOLDER_WATCH_INTERVAL"
 DEFAULT_WATCH_INTERVAL_SECONDS = 30.0
 MAX_FILES_PER_SCAN = 200  # per-watch per-scan ingest cap (thrash guard)
 
+# ── vault watch (v11.2.0) ────────────────────────────────────────────────────
+#: A watched *vault* is not a watched folder. 11.1.0 refused to schedule vault
+#: syncs for a real reason — link edges need the node ids that only a completed
+#: inline ingest has, so a per-file incremental pass would report edges it never
+#: wrote. The resolution is not to make the bridge incremental: it is to notice
+#: that the vault changed and then run the **whole** bridge sync, which does
+#: produce those edges and is idempotent by content hash. What the snapshot buys
+#: is skipping the sync entirely when nothing moved.
+WATCH_KIND_FOLDER = "folder"
+WATCH_KIND_VAULT = "vault"
+WATCH_KINDS = (WATCH_KIND_FOLDER, WATCH_KIND_VAULT)
+VAULT_WATCH_ENV = "LATTICEAI_VAULT_WATCH"
+#: Opt-in, default off, resolved when asked so a settings toggle can move it.
+VAULT_WATCH_GATE = FeatureGate(
+    VAULT_WATCH_ENV,
+    default=False,
+    name="vault_watch",
+    detail=(
+        "Watching an external vault re-syncs it in the background; it is off "
+        "until you turn it on."
+    ),
+)
+VAULT_WATCH_DISABLED_DETAIL = (
+    "vault watch is off by default; turn it on to let an external vault "
+    f"re-sync in the background ({VAULT_WATCH_ENV})"
+)
+
 
 def _default_interval() -> float:
     raw = os.getenv(WATCH_INTERVAL_ENV, "").strip()
@@ -65,8 +93,13 @@ class FolderWatchService:
         pipeline: Any,
         config_path: Path,
         interval_seconds: Optional[float] = None,
+        vault_bridge: Optional[Any] = None,
     ) -> None:
         self._pipeline = pipeline
+        # Injected rather than imported at call time so a vault watch can be
+        # driven by a test double, and so this module keeps knowing nothing
+        # about how a vault is parsed.
+        self._vault_bridge = vault_bridge
         self._config_path = Path(config_path)
         self._interval = float(interval_seconds) if interval_seconds else _default_interval()
         self._lock = threading.Lock()
@@ -100,13 +133,23 @@ class FolderWatchService:
         owner: Optional[str] = None,
         workspace_id: Optional[str] = None,
         recursive: bool = True,
+        kind: str = WATCH_KIND_FOLDER,
     ) -> Dict[str, Any]:
         """Explicit opt-in: snapshot the folder now, ingest only future changes.
 
         The folder is assumed to have been ingested already (the UI offers
         watch after a folder ingest); the enable-time snapshot is the baseline,
         so enabling never re-ingests the whole folder.
+
+        ``kind="vault"`` watches an external Obsidian vault instead, re-running
+        the whole bridge sync when the snapshot moves. That mode carries its own
+        opt-in gate on top of this one and refuses while it is off.
         """
+        kind = str(kind or WATCH_KIND_FOLDER).strip().lower()
+        if kind not in WATCH_KINDS:
+            return {"status": "failed", "detail": f"unknown watch kind: {kind}"}
+        if kind == WATCH_KIND_VAULT and not VAULT_WATCH_GATE.enabled():
+            return {"status": "disabled", "detail": VAULT_WATCH_DISABLED_DETAIL}
         root = Path(path).expanduser()
         if not root.is_dir():
             return {"status": "failed", "detail": f"not a directory: {root}"}
@@ -116,6 +159,7 @@ class FolderWatchService:
                 same_scope = (watch.get("workspace_id") or None) == (workspace_id or None)
                 if watch.get("path") == str(root) and same_scope:
                     watch["enabled"] = True
+                    watch["kind"] = kind
                     watch["snapshot"] = self._snapshot(root, recursive=bool(watch.get("recursive", True)))
                     self._save_config()
                     self._ensure_thread()
@@ -127,6 +171,7 @@ class FolderWatchService:
                 "owner": owner,
                 "workspace_id": workspace_id,
                 "recursive": bool(recursive),
+                "kind": kind,
                 "enabled": True,
                 "created_at": _now_iso(),
                 "last_scan_at": None,
@@ -172,6 +217,11 @@ class FolderWatchService:
             "polling": polling,
             "interval_seconds": self._interval,
             "watches": watches,
+            "kinds": list(WATCH_KINDS),
+            "vault_watch": {
+                **VAULT_WATCH_GATE.describe(),
+                "bridge_wired": self._vault_bridge is not None,
+            },
             "note": (
                 "Watch mode is opt-in and off by default. Deleted files are "
                 "counted but never auto-removed from the Brain."
@@ -208,10 +258,12 @@ class FolderWatchService:
             recursive = bool(watch.get("recursive", True))
             owner = watch.get("owner")
             workspace_id = watch.get("workspace_id")
+            kind = str(watch.get("kind") or WATCH_KIND_FOLDER)
 
         result: Dict[str, Any] = {
             "status": "ok",
             "watch_id": watch_id,
+            "kind": kind,
             "new": 0,
             "changed": 0,
             "removed": 0,
@@ -237,6 +289,17 @@ class FolderWatchService:
                 changed_rel.append(rel)
         result["removed"] = sum(1 for rel in snapshot if rel not in current)
 
+        if kind == WATCH_KIND_VAULT:
+            self._vault_scan(
+                result,
+                root=root,
+                changed=bool(changed_rel) or bool(result["removed"]),
+                owner=owner,
+                workspace_id=workspace_id,
+            )
+            self._record_scan(watch_id, result, snapshot=current)
+            return result
+
         for rel in changed_rel[:MAX_FILES_PER_SCAN]:
             path = root / rel
             ext = path.suffix.lower()
@@ -287,6 +350,53 @@ class FolderWatchService:
         return result
 
     # ── internals ────────────────────────────────────────────────────────────
+    def _vault_scan(
+        self,
+        result: Dict[str, Any],
+        *,
+        root: Path,
+        changed: bool,
+        owner: Optional[str],
+        workspace_id: Optional[str],
+    ) -> None:
+        """Re-run the whole vault sync — but only when something actually moved.
+
+        A full sync is what makes the link edges real; the snapshot is only
+        there to avoid paying for one when the vault is untouched. The gate is
+        re-checked here, not just at enable time, so turning vault watch off
+        stops the background work immediately instead of at the next restart.
+        """
+        if not VAULT_WATCH_GATE.enabled():
+            result.update(status="disabled", detail=VAULT_WATCH_DISABLED_DETAIL)
+            return
+        if not changed:
+            result["synced"] = False
+            result["detail"] = "vault unchanged since the last scan"
+            return
+        if self._vault_bridge is None:
+            result.update(
+                status="failed",
+                synced=False,
+                detail="no vault bridge is wired; the vault was not re-synced",
+            )
+            return
+        try:
+            summary = self._vault_bridge.sync(
+                str(root), owner=owner, workspace_id=workspace_id, user_email=owner,
+            )
+        except Exception as exc:  # noqa: BLE001 — one bad sync must not kill the poller
+            result.update(status="failed", synced=False, detail=f"vault sync failed: {exc}")
+            return
+        result["synced"] = True
+        result["ingested"] = int(summary.get("ingested") or 0)
+        result["duplicate"] = int(summary.get("duplicate") or 0)
+        result["failed"] = int(summary.get("failed") or 0)
+        result["errors"] = list(summary.get("errors") or [])[:20]
+        result["edges"] = summary.get("edges")
+        result["status"] = "ok" if summary.get("status") in ("ok", "dry_run") else str(
+            summary.get("status") or "ok"
+        )
+
     def _record_scan(
         self,
         watch_id: str,
@@ -355,6 +465,7 @@ class FolderWatchService:
             "owner": watch.get("owner"),
             "workspace_id": watch.get("workspace_id"),
             "recursive": bool(watch.get("recursive", True)),
+            "kind": str(watch.get("kind") or WATCH_KIND_FOLDER),
             "enabled": bool(watch.get("enabled")),
             "created_at": watch.get("created_at"),
             "last_scan_at": watch.get("last_scan_at"),
@@ -396,4 +507,14 @@ class FolderWatchService:
                     continue
 
 
-__all__ = ["FolderWatchService", "DEFAULT_WATCH_INTERVAL_SECONDS", "WATCH_INTERVAL_ENV"]
+__all__ = [
+    "DEFAULT_WATCH_INTERVAL_SECONDS",
+    "VAULT_WATCH_DISABLED_DETAIL",
+    "VAULT_WATCH_ENV",
+    "VAULT_WATCH_GATE",
+    "WATCH_INTERVAL_ENV",
+    "WATCH_KINDS",
+    "WATCH_KIND_FOLDER",
+    "WATCH_KIND_VAULT",
+    "FolderWatchService",
+]

package/latticeai/services/hybrid_chat.py

@@ -2,6 +2,13 @@
 
 Phase 1–2 entry point used by the chat path when NetworkBoundaryMode is
 CLOUD_ALLOWED. Local-only sessions never enter this module's cloud path.
+
+Both turn functions take the Review Center sink and the hybrid policy's
+``auto_commit`` decision as arguments rather than reaching for them: the
+caller that knows *whose* turn this is is the only one that can resolve a
+per-user policy, and a service that reads a process singleton cannot be
+tested against both branches. Defaults are the safe ones — no sink, no
+auto-commit — so a headless caller stages nothing rather than writing.
 """
 
 from __future__ import annotations
@@ -36,6 +43,35 @@ def _scope_key(user_email: Optional[str], workspace_id: Optional[str]) -> str:
     return f"{user_email or 'anon'}|{workspace_id or 'global'}"
 
 
+def _ingest_cloud_expansion(
+    result: CloudTurnResult,
+    *,
+    knowledge_graph: Any,
+    review_queue: Any,
+    auto_commit: bool,
+    user_email: Optional[str],
+    workspace_id: Optional[str],
+) -> Dict[str, Any]:
+    """Stage what the cloud answer taught the Brain (v11.2.0 wiring).
+
+    ``plan_kg_expansion_rich`` builds every plan with ``auto_commit=False``
+    because extraction has no idea what the user consented to; the policy dial
+    does, and this is where the two meet. With the sink bound the plan lands in
+    the Review Center as a ``change_proposal``, which is what makes
+    cloud-derived memory growth a thing the user approves rather than a thing
+    that happens to them.
+    """
+    plan = plan_kg_expansion_rich(result)
+    plan.auto_commit = bool(auto_commit)
+    ingestor = CloudResponseIngestor(
+        store=knowledge_graph,
+        review_queue=review_queue,
+        user_email=user_email,
+        workspace_id=workspace_id,
+    )
+    return ingestor.ingest(plan)
+
+
 async def run_hybrid_cloud_turn(
     *,
     user_message: str,
@@ -46,6 +82,8 @@ async def run_hybrid_cloud_turn(
     model: Optional[str] = None,
     top_k: int = 6,
     adapter: Optional[Any] = None,
+    review_queue: Any = None,
+    auto_commit: bool = False,
 ) -> CloudTurnResult:
     """Non-streaming helper: full answer + expansion plan."""
     mode = normalize_network_mode(mode)
@@ -85,9 +123,14 @@ async def run_hybrid_cloud_turn(
         mode=mode,
         model=model,
     )
-    plan = plan_kg_expansion_rich(result)
-    ingestor = CloudResponseIngestor(store=knowledge_graph)
-    ingest_status = ingestor.ingest(plan)
+    ingest_status = _ingest_cloud_expansion(
+        result,
+        knowledge_graph=knowledge_graph,
+        review_queue=review_queue,
+        auto_commit=auto_commit,
+        user_email=user_email,
+        workspace_id=workspace_id,
+    )
     used = minimal.token_estimate + max(1, len(result.answer_text) // 4)
     budget.record(used)
     result.usage = {
@@ -116,6 +159,8 @@ async def stream_hybrid_cloud_turn(
     history_user: Optional[Dict[str, Any]] = None,
     notify: Any = None,
     source: Optional[str] = None,
+    review_queue: Any = None,
+    auto_commit: bool = False,
 ) -> AsyncIterator[str]:
     """SSE generator for a hybrid cloud turn (Phase 2 chat path).
 
@@ -220,8 +265,14 @@ async def stream_hybrid_cloud_turn(
                 }
             )
 
-        plan = plan_kg_expansion_rich(result)
-        ingest_status = CloudResponseIngestor(store=knowledge_graph).ingest(plan)
+        ingest_status = _ingest_cloud_expansion(
+            result,
+            knowledge_graph=knowledge_graph,
+            review_queue=review_queue,
+            auto_commit=auto_commit,
+            user_email=user_email,
+            workspace_id=workspace_id,
+        )
         used = minimal.token_estimate + max(1, len(result.answer_text) // 4)
         budget.record(used)