ltcai 9.9.2 → 9.9.4

package/latticeai/services/automation_intelligence.py

@@ -560,6 +560,8 @@ class AutomationIntelligenceService:
             except Exception:
                 LOGGER.exception("automation intelligence workflow read failed")
                 workflows = []
+            from latticeai.services.automation_execution import last_execution_view
+
             for workflow in workflows:
                 metadata = (workflow or {}).get("metadata") or {}
                 if metadata.get("created_from") not in {"automation_suggestion", "brain_automation_recipe"}:
@@ -573,6 +575,11 @@ class AutomationIntelligenceService:
                     "enabled": metadata.get("automation_state") == "enabled",
                     "requires_user_enable": bool(metadata.get("requires_user_enable", True)),
                     "creates": metadata.get("creates") or [],
+                    # Execution log surfacing (backlog #6): the newest of the
+                    # run-now stamp vs. the latest persisted workflow run.
+                    "last_execution": last_execution_view(
+                        workflow, store=self._store, workspace_id=workspace_id
+                    ),
                 })
         return {
             "suggestions": suggestion_report["suggestions"],

package/latticeai/services/command_center.py

@@ -23,9 +23,10 @@ calls a model.
 from __future__ import annotations
 
 import logging
+from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional
 
-from latticeai.core.timeutil import now_iso as _now
+from latticeai.core.timeutil import local_now, now_iso as _now, parse_iso
 
 LOGGER = logging.getLogger(__name__)
 
@@ -33,6 +34,13 @@ _RECENT_NODE_LIMIT = 6
 _SEARCH_HISTORY_LIMIT = 2000
 _BRIEFING_HISTORY_LIMIT = 2000
 
+# Graph-hygiene advisory (review 2026-07-25 Wave 2.5): once the graph passes
+# this many nodes with no noise-curate pass in the stale window, the briefing
+# suggests a *dry-run* /knowledge-graph/curate/noise. Advisory only — nothing
+# is deleted until the user explicitly applies the job.
+_HYGIENE_MIN_NODES = 200
+_HYGIENE_STALE_DAYS = 7
+
 
 def _clip(text: Any, limit: int = 160) -> str:
     value = str(text or "").strip()
@@ -147,18 +155,37 @@ class CommandCenterService:
         workflows = self._workflows(workspace_id=workspace_id)
         enabled = 0
         drafts = 0
+        last_execution: Optional[Dict[str, Any]] = None
         for workflow in workflows:
             metadata = (workflow or {}).get("metadata") or {}
             if metadata.get("automation_state") == "enabled":
                 enabled += 1
             elif metadata.get("automation_state") == "draft_disabled":
                 drafts += 1
-        return {
+            # Execution log surfacing (backlog #6): the briefing carries the
+            # most recent stamped automation execution so "did my automation
+            # run, and how did it go?" is answered on the home surface.
+            stamped = metadata.get("last_execution")
+            if isinstance(stamped, dict) and str(stamped.get("finished_at") or "") > str(
+                (last_execution or {}).get("finished_at") or ""
+            ):
+                last_execution = {
+                    "workflow_id": workflow.get("id"),
+                    "name": _clip(workflow.get("name"), 120),
+                    "mode": stamped.get("mode"),
+                    "status": stamped.get("status"),
+                    "summary": _clip(stamped.get("summary"), 200),
+                    "finished_at": stamped.get("finished_at"),
+                }
+        section = {
             "available": True,
             "total": len(workflows),
             "enabled": enabled,
             "drafts": drafts,
         }
+        if last_execution is not None:
+            section["last_execution"] = last_execution
+        return section
 
     def _review_section(
         self, *, user_email: Optional[str], workspace_id: Optional[str]
@@ -221,6 +248,73 @@ class CommandCenterService:
         ]
         return {"available": True, "count": len(suggestions), "top": top}
 
+    def _hygiene_section(self) -> Dict[str, Any]:
+        """Advisory graph-hygiene signal (review 2026-07-25 Wave 2.5).
+
+        Suggests a dry-run noise-curate pass when the graph has grown past
+        ``_HYGIENE_MIN_NODES`` nodes and the last applied pass is missing or
+        older than ``_HYGIENE_STALE_DAYS`` days. Fail-open: any backend error
+        degrades to "no suggestion" — the briefing itself never breaks.
+        """
+        section: Dict[str, Any] = {
+            "available": False,
+            "suggest_noise_curate": False,
+            "reason": "",
+            "last_noise_curate_at": None,
+            "node_count": 0,
+        }
+        if not self._enable_graph or not hasattr(self._kg, "stats"):
+            return section
+        try:
+            stats = self._kg.stats() or {}
+            node_count = sum(
+                int(count or 0) for count in (stats.get("nodes") or {}).values()
+            )
+            last = (
+                self._kg.last_noise_curate_at()
+                if hasattr(self._kg, "last_noise_curate_at")
+                else None
+            )
+            section.update(
+                {
+                    "available": True,
+                    "node_count": node_count,
+                    "last_noise_curate_at": last,
+                }
+            )
+            if node_count < _HYGIENE_MIN_NODES:
+                return section
+            if last is not None and not self._older_than_days(
+                last, _HYGIENE_STALE_DAYS
+            ):
+                return section
+            section["suggest_noise_curate"] = True
+            section["reason"] = (
+                f"{node_count} nodes and no noise curation in the last "
+                f"{_HYGIENE_STALE_DAYS} days"
+                if last
+                else f"{node_count} nodes and no noise curation recorded"
+            )
+        except Exception:
+            LOGGER.exception("command center hygiene read failed")
+            return {
+                "available": False,
+                "suggest_noise_curate": False,
+                "reason": "",
+                "last_noise_curate_at": None,
+                "node_count": 0,
+            }
+        return section
+
+    @staticmethod
+    def _older_than_days(stamp: str, days: int) -> bool:
+        parsed = parse_iso(stamp)
+        if parsed is None:
+            # Unreadable stamp → treat as stale; suggesting a dry-run is safe.
+            return True
+        now = datetime.now(parsed.tzinfo) if parsed.tzinfo else local_now()
+        return (now - parsed) > timedelta(days=days)
+
     # ── quick actions ────────────────────────────────────────────────────
 
     def _quick_actions(self, sections: Dict[str, Dict[str, Any]]) -> List[Dict[str, Any]]:
@@ -276,6 +370,19 @@ class CommandCenterService:
                     "target": "/brain/graph",
                 }
             )
+        hygiene = sections.get("hygiene") or {}
+        if hygiene.get("suggest_noise_curate"):
+            actions.append(
+                {
+                    "id": "curate-noise",
+                    "kind": "hygiene",
+                    "count": int(hygiene.get("node_count") or 0),
+                    "target": "/brain/graph",
+                    # Dry-run by default on the API side: this endpoint only
+                    # *reports* removals until dry_run=false is sent explicitly.
+                    "endpoint": "/knowledge-graph/curate/noise",
+                }
+            )
         if not actions:
             actions.append(
                 {"id": "ask-brain", "kind": "chat", "count": 0, "target": "/brain"}
@@ -300,6 +407,7 @@ class CommandCenterService:
             "suggestions": self._suggestion_section(
                 user_email=user_email, workspace_id=workspace_id
             ),
+            "hygiene": self._hygiene_section(),
         }
         return {
             "generated_at": _now(),

package/latticeai/services/folder_watch.py

@@ -0,0 +1,396 @@
+"""Opt-in folder watch mode for the unified ingestion pipeline (backlog #8).
+
+Watches previously-ingested folders and incrementally re-ingests new/changed
+files through the normal :class:`IngestionPipeline` door (same filters, same
+quality pipeline, same workspace scoping, same provenance).
+
+Design decisions (review §7.2 B + risk section):
+
+* **Default OFF, explicit opt-in.** A watch exists only after an explicit API
+  call. The stored config is the durable record of that consent; ``restore()``
+  resumes only entries persisted with ``enabled: true`` and never invents one.
+* **Polling with mtime snapshots**, not an OS watcher. ``watchdog`` is already
+  a dependency, but polling is deterministic, portable, and testable
+  (``scan_once`` runs synchronously); the interval is configurable via
+  ``LATTICEAI_FOLDER_WATCH_INTERVAL`` (seconds, default 30).
+* **Reuses the folder-ingest filters** (skip dirs, ``.latticeignore``,
+  extension allow-list, size cap) so watch mode can never ingest something a
+  manual folder ingest would have refused.
+* Deleted files are *counted* but their graph nodes are not removed — node
+  deletion is a destructive operation that stays behind explicit user flows.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from lattice_brain.ingestion import (
+    DEFAULT_FOLDER_EXTENSIONS,
+    DEFAULT_MAX_FILE_BYTES,
+    FOLDER_DEFAULT_SKIP_DIRS,
+    FOLDER_DOCUMENT_EXTENSIONS,
+    LATTICEIGNORE_FILENAME,
+    IngestionItem,
+    _load_latticeignore,
+    _matches_ignore,
+)
+from latticeai.core.timeutil import now_iso as _now_iso
+
+WATCH_INTERVAL_ENV = "LATTICEAI_FOLDER_WATCH_INTERVAL"
+DEFAULT_WATCH_INTERVAL_SECONDS = 30.0
+MAX_FILES_PER_SCAN = 200  # per-watch per-scan ingest cap (thrash guard)
+
+
+def _default_interval() -> float:
+    raw = os.getenv(WATCH_INTERVAL_ENV, "").strip()
+    try:
+        value = float(raw) if raw else DEFAULT_WATCH_INTERVAL_SECONDS
+    except ValueError:
+        value = DEFAULT_WATCH_INTERVAL_SECONDS
+    return max(1.0, value)
+
+
+class FolderWatchService:
+    """Poll-based incremental re-ingestion of explicitly opted-in folders."""
+
+    def __init__(
+        self,
+        *,
+        pipeline: Any,
+        config_path: Path,
+        interval_seconds: Optional[float] = None,
+    ) -> None:
+        self._pipeline = pipeline
+        self._config_path = Path(config_path)
+        self._interval = float(interval_seconds) if interval_seconds else _default_interval()
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+        self._watches: Dict[str, Dict[str, Any]] = self._load_config()
+
+    # ── persistence ──────────────────────────────────────────────────────────
+    def _load_config(self) -> Dict[str, Dict[str, Any]]:
+        try:
+            payload = json.loads(self._config_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return {}
+        watches = payload.get("watches")
+        return dict(watches) if isinstance(watches, dict) else {}
+
+    def _save_config(self) -> None:
+        self._config_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self._config_path.with_suffix(".tmp")
+        tmp.write_text(
+            json.dumps({"watches": self._watches}, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        tmp.replace(self._config_path)
+
+    # ── public API ───────────────────────────────────────────────────────────
+    def enable(
+        self,
+        path: Any,
+        *,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        recursive: bool = True,
+    ) -> Dict[str, Any]:
+        """Explicit opt-in: snapshot the folder now, ingest only future changes.
+
+        The folder is assumed to have been ingested already (the UI offers
+        watch after a folder ingest); the enable-time snapshot is the baseline,
+        so enabling never re-ingests the whole folder.
+        """
+        root = Path(path).expanduser()
+        if not root.is_dir():
+            return {"status": "failed", "detail": f"not a directory: {root}"}
+        root = root.resolve()
+        with self._lock:
+            for watch in self._watches.values():
+                same_scope = (watch.get("workspace_id") or None) == (workspace_id or None)
+                if watch.get("path") == str(root) and same_scope:
+                    watch["enabled"] = True
+                    watch["snapshot"] = self._snapshot(root, recursive=bool(watch.get("recursive", True)))
+                    self._save_config()
+                    self._ensure_thread()
+                    return {"status": "ok", "watch": self._public_watch(watch), "already_watching": True}
+            watch_id = f"watch_{uuid.uuid4().hex[:12]}"
+            watch = {
+                "id": watch_id,
+                "path": str(root),
+                "owner": owner,
+                "workspace_id": workspace_id,
+                "recursive": bool(recursive),
+                "enabled": True,
+                "created_at": _now_iso(),
+                "last_scan_at": None,
+                "last_result": None,
+                "snapshot": self._snapshot(root, recursive=bool(recursive)),
+            }
+            self._watches[watch_id] = watch
+            self._save_config()
+            self._ensure_thread()
+            return {"status": "ok", "watch": self._public_watch(watch), "already_watching": False}
+
+    def disable(
+        self,
+        *,
+        watch_id: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Opt back out: remove the stored consent record entirely."""
+        with self._lock:
+            target_id = None
+            if watch_id and watch_id in self._watches:
+                target_id = watch_id
+            elif path:
+                wanted = str(Path(path).expanduser().resolve())
+                for wid, watch in self._watches.items():
+                    if watch.get("path") == wanted:
+                        target_id = wid
+                        break
+            if target_id is None:
+                return {"status": "not_found"}
+            removed = self._watches.pop(target_id)
+            self._save_config()
+            if not any(w.get("enabled") for w in self._watches.values()):
+                self._stop_thread_locked()
+        return {"status": "ok", "watch": self._public_watch(removed)}
+
+    def status(self) -> Dict[str, Any]:
+        with self._lock:
+            watches = [self._public_watch(w) for w in self._watches.values()]
+            polling = self._thread is not None and self._thread.is_alive()
+        return {
+            "enabled_count": sum(1 for w in watches if w.get("enabled")),
+            "polling": polling,
+            "interval_seconds": self._interval,
+            "watches": watches,
+            "note": (
+                "Watch mode is opt-in and off by default. Deleted files are "
+                "counted but never auto-removed from the Brain."
+            ),
+        }
+
+    def restore(self) -> Dict[str, Any]:
+        """Resume polling for watches persisted with the explicit opt-in.
+
+        Never starts anything when the stored config has no enabled entries —
+        an empty/missing config means the user never opted in.
+        """
+        with self._lock:
+            enabled = [w for w in self._watches.values() if w.get("enabled")]
+            if enabled:
+                self._ensure_thread()
+        return {"restored": len(enabled), "polling": bool(enabled)}
+
+    def stop_all(self) -> None:
+        """Stop the polling thread (keeps the stored opt-in for next start)."""
+        with self._lock:
+            self._stop_thread_locked()
+
+    def scan_once(self, watch_id: str) -> Dict[str, Any]:
+        """Synchronously scan one watch and ingest new/changed files."""
+        with self._lock:
+            watch = self._watches.get(watch_id)
+            if watch is None:
+                return {"status": "not_found", "watch_id": watch_id}
+            if not watch.get("enabled"):
+                return {"status": "disabled", "watch_id": watch_id}
+            snapshot = dict(watch.get("snapshot") or {})
+            root = Path(watch["path"])
+            recursive = bool(watch.get("recursive", True))
+            owner = watch.get("owner")
+            workspace_id = watch.get("workspace_id")
+
+        result: Dict[str, Any] = {
+            "status": "ok",
+            "watch_id": watch_id,
+            "new": 0,
+            "changed": 0,
+            "removed": 0,
+            "ingested": 0,
+            "duplicate": 0,
+            "failed": 0,
+            "errors": [],
+        }
+        if not root.is_dir():
+            result.update(status="failed", detail=f"folder unavailable: {root}")
+            self._record_scan(watch_id, result, snapshot=None)
+            return result
+
+        current = self._snapshot(root, recursive=recursive)
+        changed_rel: List[str] = []
+        for rel, stamp in current.items():
+            previous = snapshot.get(rel)
+            if previous is None:
+                result["new"] += 1
+                changed_rel.append(rel)
+            elif previous != stamp:
+                result["changed"] += 1
+                changed_rel.append(rel)
+        result["removed"] = sum(1 for rel in snapshot if rel not in current)
+
+        for rel in changed_rel[:MAX_FILES_PER_SCAN]:
+            path = root / rel
+            ext = path.suffix.lower()
+            metadata: Dict[str, Any] = {
+                "relative_path": rel,
+                "folder_watch": True,
+                "watch_id": watch_id,
+            }
+            if ext in FOLDER_DOCUMENT_EXTENSIONS:
+                source_type = "pdf"
+            else:
+                source_type = "file"
+                try:
+                    content = path.read_text(encoding="utf-8", errors="ignore")
+                except OSError as exc:
+                    result["failed"] += 1
+                    if len(result["errors"]) < 20:
+                        result["errors"].append({"path": rel, "detail": f"read failed: {exc}"})
+                    continue
+                metadata["extracted"] = {"content": content, "chars": len(content)}
+            outcome = self._pipeline.ingest(
+                IngestionItem(
+                    source_type=source_type,
+                    title=path.name,
+                    path=str(path),
+                    source_uri=str(path),
+                    owner=owner,
+                    workspace_id=workspace_id,
+                    metadata=metadata,
+                ),
+                user_email=owner,
+            )
+            if outcome.status == "ok":
+                if outcome.duplicate:
+                    result["duplicate"] += 1
+                else:
+                    result["ingested"] += 1
+            else:
+                result["failed"] += 1
+                if len(result["errors"]) < 20:
+                    result["errors"].append({
+                        "path": rel,
+                        "detail": outcome.detail or outcome.status,
+                    })
+        if len(changed_rel) > MAX_FILES_PER_SCAN:
+            result["truncated"] = True
+        self._record_scan(watch_id, result, snapshot=current)
+        return result
+
+    # ── internals ────────────────────────────────────────────────────────────
+    def _record_scan(
+        self,
+        watch_id: str,
+        result: Dict[str, Any],
+        *,
+        snapshot: Optional[Dict[str, Any]],
+    ) -> None:
+        with self._lock:
+            watch = self._watches.get(watch_id)
+            if watch is None:
+                return
+            watch["last_scan_at"] = _now_iso()
+            watch["last_result"] = {
+                key: result[key]
+                for key in ("status", "new", "changed", "removed", "ingested", "duplicate", "failed")
+                if key in result
+            }
+            # Watch trust UI (review Wave 1.3): keep a small sample of the
+            # latest scan's failures so the home health card can show *why*
+            # a watch is unhealthy, not just a count.
+            watch["last_errors"] = list(result.get("errors") or [])[:3]
+            if snapshot is not None:
+                watch["snapshot"] = snapshot
+            self._save_config()
+
+    def _snapshot(self, root: Path, *, recursive: bool) -> Dict[str, List[float]]:
+        """{relative_posix_path: [mtime, size]} using the folder-ingest filters."""
+        patterns = _load_latticeignore(root)
+        snapshot: Dict[str, List[float]] = {}
+        for dirpath, dirnames, filenames in os.walk(root):
+            current = Path(dirpath)
+            rel_dir = current.relative_to(root)
+            kept: List[str] = []
+            for name in sorted(dirnames):
+                if name in FOLDER_DEFAULT_SKIP_DIRS or name.startswith("."):
+                    continue
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if _matches_ignore(rel, name, is_dir=True, patterns=patterns):
+                    continue
+                kept.append(name)
+            dirnames[:] = kept if recursive else []
+            for name in sorted(filenames):
+                if name == LATTICEIGNORE_FILENAME or name.startswith("."):
+                    continue
+                path = current / name
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if _matches_ignore(rel, name, is_dir=False, patterns=patterns):
+                    continue
+                if path.suffix.lower() not in DEFAULT_FOLDER_EXTENSIONS:
+                    continue
+                try:
+                    stat = path.stat()
+                except OSError:
+                    continue
+                if stat.st_size > DEFAULT_MAX_FILE_BYTES:
+                    continue
+                snapshot[rel] = [round(stat.st_mtime, 3), stat.st_size]
+        return snapshot
+
+    @staticmethod
+    def _public_watch(watch: Dict[str, Any]) -> Dict[str, Any]:
+        return {
+            "id": watch.get("id"),
+            "path": watch.get("path"),
+            "owner": watch.get("owner"),
+            "workspace_id": watch.get("workspace_id"),
+            "recursive": bool(watch.get("recursive", True)),
+            "enabled": bool(watch.get("enabled")),
+            "created_at": watch.get("created_at"),
+            "last_scan_at": watch.get("last_scan_at"),
+            "last_result": watch.get("last_result"),
+            "last_errors": list(watch.get("last_errors") or []),
+            "tracked_files": len(watch.get("snapshot") or {}),
+        }
+
+    def _ensure_thread(self) -> None:
+        """Start the poller (caller holds the lock)."""
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop = threading.Event()
+        self._thread = threading.Thread(
+            target=self._poll_loop, name="folder-watch-poller", daemon=True,
+        )
+        self._thread.start()
+
+    def _stop_thread_locked(self) -> None:
+        self._stop.set()
+        self._thread = None
+
+    def _poll_loop(self) -> None:
+        stop = self._stop
+        while not stop.wait(self._interval):
+            with self._lock:
+                enabled_ids = [
+                    wid for wid, watch in self._watches.items() if watch.get("enabled")
+                ]
+            if not enabled_ids:
+                return
+            for watch_id in enabled_ids:
+                if stop.is_set():
+                    return
+                try:
+                    self.scan_once(watch_id)
+                except Exception:  # noqa: BLE001 — one bad watch must not kill the poller
+                    continue
+
+
+__all__ = ["FolderWatchService", "DEFAULT_WATCH_INTERVAL_SECONDS", "WATCH_INTERVAL_ENV"]