ltcai 9.9.2 → 9.9.3

package/latticeai/services/folder_watch.py

@@ -0,0 +1,391 @@
+"""Opt-in folder watch mode for the unified ingestion pipeline (backlog #8).
+
+Watches previously-ingested folders and incrementally re-ingests new/changed
+files through the normal :class:`IngestionPipeline` door (same filters, same
+quality pipeline, same workspace scoping, same provenance).
+
+Design decisions (review §7.2 B + risk section):
+
+* **Default OFF, explicit opt-in.** A watch exists only after an explicit API
+  call. The stored config is the durable record of that consent; ``restore()``
+  resumes only entries persisted with ``enabled: true`` and never invents one.
+* **Polling with mtime snapshots**, not an OS watcher. ``watchdog`` is already
+  a dependency, but polling is deterministic, portable, and testable
+  (``scan_once`` runs synchronously); the interval is configurable via
+  ``LATTICEAI_FOLDER_WATCH_INTERVAL`` (seconds, default 30).
+* **Reuses the folder-ingest filters** (skip dirs, ``.latticeignore``,
+  extension allow-list, size cap) so watch mode can never ingest something a
+  manual folder ingest would have refused.
+* Deleted files are *counted* but their graph nodes are not removed — node
+  deletion is a destructive operation that stays behind explicit user flows.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import uuid
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from lattice_brain.ingestion import (
+    DEFAULT_FOLDER_EXTENSIONS,
+    DEFAULT_MAX_FILE_BYTES,
+    FOLDER_DEFAULT_SKIP_DIRS,
+    FOLDER_DOCUMENT_EXTENSIONS,
+    LATTICEIGNORE_FILENAME,
+    IngestionItem,
+    _load_latticeignore,
+    _matches_ignore,
+)
+from latticeai.core.timeutil import now_iso as _now_iso
+
+WATCH_INTERVAL_ENV = "LATTICEAI_FOLDER_WATCH_INTERVAL"
+DEFAULT_WATCH_INTERVAL_SECONDS = 30.0
+MAX_FILES_PER_SCAN = 200  # per-watch per-scan ingest cap (thrash guard)
+
+
+def _default_interval() -> float:
+    raw = os.getenv(WATCH_INTERVAL_ENV, "").strip()
+    try:
+        value = float(raw) if raw else DEFAULT_WATCH_INTERVAL_SECONDS
+    except ValueError:
+        value = DEFAULT_WATCH_INTERVAL_SECONDS
+    return max(1.0, value)
+
+
+class FolderWatchService:
+    """Poll-based incremental re-ingestion of explicitly opted-in folders."""
+
+    def __init__(
+        self,
+        *,
+        pipeline: Any,
+        config_path: Path,
+        interval_seconds: Optional[float] = None,
+    ) -> None:
+        self._pipeline = pipeline
+        self._config_path = Path(config_path)
+        self._interval = float(interval_seconds) if interval_seconds else _default_interval()
+        self._lock = threading.Lock()
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+        self._watches: Dict[str, Dict[str, Any]] = self._load_config()
+
+    # ── persistence ──────────────────────────────────────────────────────────
+    def _load_config(self) -> Dict[str, Dict[str, Any]]:
+        try:
+            payload = json.loads(self._config_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return {}
+        watches = payload.get("watches")
+        return dict(watches) if isinstance(watches, dict) else {}
+
+    def _save_config(self) -> None:
+        self._config_path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = self._config_path.with_suffix(".tmp")
+        tmp.write_text(
+            json.dumps({"watches": self._watches}, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        tmp.replace(self._config_path)
+
+    # ── public API ───────────────────────────────────────────────────────────
+    def enable(
+        self,
+        path: Any,
+        *,
+        owner: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        recursive: bool = True,
+    ) -> Dict[str, Any]:
+        """Explicit opt-in: snapshot the folder now, ingest only future changes.
+
+        The folder is assumed to have been ingested already (the UI offers
+        watch after a folder ingest); the enable-time snapshot is the baseline,
+        so enabling never re-ingests the whole folder.
+        """
+        root = Path(path).expanduser()
+        if not root.is_dir():
+            return {"status": "failed", "detail": f"not a directory: {root}"}
+        root = root.resolve()
+        with self._lock:
+            for watch in self._watches.values():
+                same_scope = (watch.get("workspace_id") or None) == (workspace_id or None)
+                if watch.get("path") == str(root) and same_scope:
+                    watch["enabled"] = True
+                    watch["snapshot"] = self._snapshot(root, recursive=bool(watch.get("recursive", True)))
+                    self._save_config()
+                    self._ensure_thread()
+                    return {"status": "ok", "watch": self._public_watch(watch), "already_watching": True}
+            watch_id = f"watch_{uuid.uuid4().hex[:12]}"
+            watch = {
+                "id": watch_id,
+                "path": str(root),
+                "owner": owner,
+                "workspace_id": workspace_id,
+                "recursive": bool(recursive),
+                "enabled": True,
+                "created_at": _now_iso(),
+                "last_scan_at": None,
+                "last_result": None,
+                "snapshot": self._snapshot(root, recursive=bool(recursive)),
+            }
+            self._watches[watch_id] = watch
+            self._save_config()
+            self._ensure_thread()
+            return {"status": "ok", "watch": self._public_watch(watch), "already_watching": False}
+
+    def disable(
+        self,
+        *,
+        watch_id: Optional[str] = None,
+        path: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Opt back out: remove the stored consent record entirely."""
+        with self._lock:
+            target_id = None
+            if watch_id and watch_id in self._watches:
+                target_id = watch_id
+            elif path:
+                wanted = str(Path(path).expanduser().resolve())
+                for wid, watch in self._watches.items():
+                    if watch.get("path") == wanted:
+                        target_id = wid
+                        break
+            if target_id is None:
+                return {"status": "not_found"}
+            removed = self._watches.pop(target_id)
+            self._save_config()
+            if not any(w.get("enabled") for w in self._watches.values()):
+                self._stop_thread_locked()
+        return {"status": "ok", "watch": self._public_watch(removed)}
+
+    def status(self) -> Dict[str, Any]:
+        with self._lock:
+            watches = [self._public_watch(w) for w in self._watches.values()]
+            polling = self._thread is not None and self._thread.is_alive()
+        return {
+            "enabled_count": sum(1 for w in watches if w.get("enabled")),
+            "polling": polling,
+            "interval_seconds": self._interval,
+            "watches": watches,
+            "note": (
+                "Watch mode is opt-in and off by default. Deleted files are "
+                "counted but never auto-removed from the Brain."
+            ),
+        }
+
+    def restore(self) -> Dict[str, Any]:
+        """Resume polling for watches persisted with the explicit opt-in.
+
+        Never starts anything when the stored config has no enabled entries —
+        an empty/missing config means the user never opted in.
+        """
+        with self._lock:
+            enabled = [w for w in self._watches.values() if w.get("enabled")]
+            if enabled:
+                self._ensure_thread()
+        return {"restored": len(enabled), "polling": bool(enabled)}
+
+    def stop_all(self) -> None:
+        """Stop the polling thread (keeps the stored opt-in for next start)."""
+        with self._lock:
+            self._stop_thread_locked()
+
+    def scan_once(self, watch_id: str) -> Dict[str, Any]:
+        """Synchronously scan one watch and ingest new/changed files."""
+        with self._lock:
+            watch = self._watches.get(watch_id)
+            if watch is None:
+                return {"status": "not_found", "watch_id": watch_id}
+            if not watch.get("enabled"):
+                return {"status": "disabled", "watch_id": watch_id}
+            snapshot = dict(watch.get("snapshot") or {})
+            root = Path(watch["path"])
+            recursive = bool(watch.get("recursive", True))
+            owner = watch.get("owner")
+            workspace_id = watch.get("workspace_id")
+
+        result: Dict[str, Any] = {
+            "status": "ok",
+            "watch_id": watch_id,
+            "new": 0,
+            "changed": 0,
+            "removed": 0,
+            "ingested": 0,
+            "duplicate": 0,
+            "failed": 0,
+            "errors": [],
+        }
+        if not root.is_dir():
+            result.update(status="failed", detail=f"folder unavailable: {root}")
+            self._record_scan(watch_id, result, snapshot=None)
+            return result
+
+        current = self._snapshot(root, recursive=recursive)
+        changed_rel: List[str] = []
+        for rel, stamp in current.items():
+            previous = snapshot.get(rel)
+            if previous is None:
+                result["new"] += 1
+                changed_rel.append(rel)
+            elif previous != stamp:
+                result["changed"] += 1
+                changed_rel.append(rel)
+        result["removed"] = sum(1 for rel in snapshot if rel not in current)
+
+        for rel in changed_rel[:MAX_FILES_PER_SCAN]:
+            path = root / rel
+            ext = path.suffix.lower()
+            metadata: Dict[str, Any] = {
+                "relative_path": rel,
+                "folder_watch": True,
+                "watch_id": watch_id,
+            }
+            if ext in FOLDER_DOCUMENT_EXTENSIONS:
+                source_type = "pdf"
+            else:
+                source_type = "file"
+                try:
+                    content = path.read_text(encoding="utf-8", errors="ignore")
+                except OSError as exc:
+                    result["failed"] += 1
+                    if len(result["errors"]) < 20:
+                        result["errors"].append({"path": rel, "detail": f"read failed: {exc}"})
+                    continue
+                metadata["extracted"] = {"content": content, "chars": len(content)}
+            outcome = self._pipeline.ingest(
+                IngestionItem(
+                    source_type=source_type,
+                    title=path.name,
+                    path=str(path),
+                    source_uri=str(path),
+                    owner=owner,
+                    workspace_id=workspace_id,
+                    metadata=metadata,
+                ),
+                user_email=owner,
+            )
+            if outcome.status == "ok":
+                if outcome.duplicate:
+                    result["duplicate"] += 1
+                else:
+                    result["ingested"] += 1
+            else:
+                result["failed"] += 1
+                if len(result["errors"]) < 20:
+                    result["errors"].append({
+                        "path": rel,
+                        "detail": outcome.detail or outcome.status,
+                    })
+        if len(changed_rel) > MAX_FILES_PER_SCAN:
+            result["truncated"] = True
+        self._record_scan(watch_id, result, snapshot=current)
+        return result
+
+    # ── internals ────────────────────────────────────────────────────────────
+    def _record_scan(
+        self,
+        watch_id: str,
+        result: Dict[str, Any],
+        *,
+        snapshot: Optional[Dict[str, Any]],
+    ) -> None:
+        with self._lock:
+            watch = self._watches.get(watch_id)
+            if watch is None:
+                return
+            watch["last_scan_at"] = _now_iso()
+            watch["last_result"] = {
+                key: result[key]
+                for key in ("status", "new", "changed", "removed", "ingested", "duplicate", "failed")
+                if key in result
+            }
+            if snapshot is not None:
+                watch["snapshot"] = snapshot
+            self._save_config()
+
+    def _snapshot(self, root: Path, *, recursive: bool) -> Dict[str, List[float]]:
+        """{relative_posix_path: [mtime, size]} using the folder-ingest filters."""
+        patterns = _load_latticeignore(root)
+        snapshot: Dict[str, List[float]] = {}
+        for dirpath, dirnames, filenames in os.walk(root):
+            current = Path(dirpath)
+            rel_dir = current.relative_to(root)
+            kept: List[str] = []
+            for name in sorted(dirnames):
+                if name in FOLDER_DEFAULT_SKIP_DIRS or name.startswith("."):
+                    continue
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if _matches_ignore(rel, name, is_dir=True, patterns=patterns):
+                    continue
+                kept.append(name)
+            dirnames[:] = kept if recursive else []
+            for name in sorted(filenames):
+                if name == LATTICEIGNORE_FILENAME or name.startswith("."):
+                    continue
+                path = current / name
+                rel = name if str(rel_dir) == "." else (rel_dir / name).as_posix()
+                if _matches_ignore(rel, name, is_dir=False, patterns=patterns):
+                    continue
+                if path.suffix.lower() not in DEFAULT_FOLDER_EXTENSIONS:
+                    continue
+                try:
+                    stat = path.stat()
+                except OSError:
+                    continue
+                if stat.st_size > DEFAULT_MAX_FILE_BYTES:
+                    continue
+                snapshot[rel] = [round(stat.st_mtime, 3), stat.st_size]
+        return snapshot
+
+    @staticmethod
+    def _public_watch(watch: Dict[str, Any]) -> Dict[str, Any]:
+        return {
+            "id": watch.get("id"),
+            "path": watch.get("path"),
+            "owner": watch.get("owner"),
+            "workspace_id": watch.get("workspace_id"),
+            "recursive": bool(watch.get("recursive", True)),
+            "enabled": bool(watch.get("enabled")),
+            "created_at": watch.get("created_at"),
+            "last_scan_at": watch.get("last_scan_at"),
+            "last_result": watch.get("last_result"),
+            "tracked_files": len(watch.get("snapshot") or {}),
+        }
+
+    def _ensure_thread(self) -> None:
+        """Start the poller (caller holds the lock)."""
+        if self._thread is not None and self._thread.is_alive():
+            return
+        self._stop = threading.Event()
+        self._thread = threading.Thread(
+            target=self._poll_loop, name="folder-watch-poller", daemon=True,
+        )
+        self._thread.start()
+
+    def _stop_thread_locked(self) -> None:
+        self._stop.set()
+        self._thread = None
+
+    def _poll_loop(self) -> None:
+        stop = self._stop
+        while not stop.wait(self._interval):
+            with self._lock:
+                enabled_ids = [
+                    wid for wid, watch in self._watches.items() if watch.get("enabled")
+                ]
+            if not enabled_ids:
+                return
+            for watch_id in enabled_ids:
+                if stop.is_set():
+                    return
+                try:
+                    self.scan_once(watch_id)
+                except Exception:  # noqa: BLE001 — one bad watch must not kill the poller
+                    continue
+
+
+__all__ = ["FolderWatchService", "DEFAULT_WATCH_INTERVAL_SECONDS", "WATCH_INTERVAL_ENV"]

package/latticeai/services/funnel_metrics.py

@@ -0,0 +1,202 @@
+"""UX funnel metrics — lightweight runtime counters (backlog #16, review §4.3).
+
+Tracks the product's core value funnel with honest, local-only counters:
+
+* ``file_requests``          — chat requests recognized as file intents;
+* ``real_file_delivered``    — file intents that produced actual artifacts;
+* ``code_only_responses``    — file intents that finished with only a
+  code/prose answer (the failure mode the >95% real-file goal watches);
+* ``agent_runs``             — completed chat agent runs (rate denominator);
+* ``needs_review_runs``      — agent runs that ended in ``NEEDS_REVIEW``;
+* ``ingest_completions``     — successful ingestion pipeline completions;
+* ``recall_successes``       — chat turns whose context was grounded in at
+  least one Brain node.
+
+TTFV ("time to first value") derives from two first-occurrence timestamps:
+the first successful ingest and the first grounded recall/answer.
+
+Design constraints (deliberate):
+
+* **Cheap.** One JSON file under the data dir, atomic replace on write, a
+  single ``threading.Lock`` — no background threads, no new dependencies.
+* **Never breaks the product.** Every public method swallows I/O errors;
+  metrics are advisory observability, not a gate.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import tempfile
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+LOGGER = logging.getLogger(__name__)
+
+COUNTER_NAMES = (
+    "file_requests",
+    "real_file_delivered",
+    "code_only_responses",
+    "agent_runs",
+    "needs_review_runs",
+    "ingest_completions",
+    "recall_successes",
+)
+
+_FIRST_NAMES = ("first_ingest_at", "first_value_at")
+
+
+def _utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _parse_iso(value: Any) -> Optional[datetime]:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed
+
+
+def _rate(numerator: int, denominator: int) -> Optional[float]:
+    """Honest rate: ``None`` (not 0.0) when there is no denominator yet."""
+    if denominator <= 0:
+        return None
+    return round(numerator / denominator, 4)
+
+
+class FunnelMetricsService:
+    """Thread-safe, JSON-persisted funnel counters."""
+
+    def __init__(self, path: Any) -> None:
+        self._path = Path(path)
+        self._lock = threading.Lock()
+        self._state: Dict[str, Any] = self._load()
+
+    # ── persistence ──────────────────────────────────────────────────────
+
+    def _load(self) -> Dict[str, Any]:
+        state: Dict[str, Any] = {name: 0 for name in COUNTER_NAMES}
+        for name in _FIRST_NAMES:
+            state[name] = None
+        try:
+            raw = json.loads(self._path.read_text(encoding="utf-8"))
+        except FileNotFoundError:
+            return state
+        except Exception as exc:  # noqa: BLE001 — corrupt metrics never break startup
+            LOGGER.warning("funnel metrics load failed (%s); starting fresh", exc)
+            return state
+        if isinstance(raw, dict):
+            for name in COUNTER_NAMES:
+                try:
+                    state[name] = max(0, int(raw.get(name) or 0))
+                except (TypeError, ValueError):
+                    state[name] = 0
+            for name in _FIRST_NAMES:
+                value = raw.get(name)
+                state[name] = str(value) if value else None
+        return state
+
+    def _save_locked(self) -> None:
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            fd, tmp_name = tempfile.mkstemp(
+                prefix=self._path.name, dir=str(self._path.parent)
+            )
+            try:
+                with os.fdopen(fd, "w", encoding="utf-8") as handle:
+                    json.dump(self._state, handle, ensure_ascii=False, indent=2)
+                os.replace(tmp_name, self._path)
+            finally:
+                if os.path.exists(tmp_name):
+                    try:
+                        os.unlink(tmp_name)
+                    except OSError:
+                        pass
+        except Exception as exc:  # noqa: BLE001 — metrics persistence is best-effort
+            LOGGER.warning("funnel metrics save failed: %s", exc)
+
+    # ── mutation ─────────────────────────────────────────────────────────
+
+    def increment(self, name: str, by: int = 1) -> None:
+        """Increment a known counter; unknown names are ignored (logged)."""
+        if name not in COUNTER_NAMES:
+            LOGGER.warning("funnel metrics: unknown counter %r ignored", name)
+            return
+        try:
+            step = int(by)
+        except (TypeError, ValueError):
+            step = 1
+        if step <= 0:
+            return
+        with self._lock:
+            self._state[name] = int(self._state.get(name) or 0) + step
+            self._save_locked()
+
+    def _mark_first_locked(self, name: str) -> None:
+        if not self._state.get(name):
+            self._state[name] = _utc_now_iso()
+
+    def record_ingest(self, *, duplicate: bool = False) -> None:
+        """A successful ingestion completed; the first one starts the TTFV clock."""
+        with self._lock:
+            self._state["ingest_completions"] = (
+                int(self._state.get("ingest_completions") or 0) + 1
+            )
+            self._mark_first_locked("first_ingest_at")
+            self._save_locked()
+
+    def record_recall_success(self) -> None:
+        """A chat answer was grounded in the Brain; the first one ends TTFV."""
+        with self._lock:
+            self._state["recall_successes"] = (
+                int(self._state.get("recall_successes") or 0) + 1
+            )
+            # First value only counts after knowledge actually entered the Brain.
+            if self._state.get("first_ingest_at"):
+                self._mark_first_locked("first_value_at")
+            self._save_locked()
+
+    # ── reads ────────────────────────────────────────────────────────────
+
+    def ttfv_seconds(self) -> Optional[float]:
+        with self._lock:
+            first_ingest = _parse_iso(self._state.get("first_ingest_at"))
+            first_value = _parse_iso(self._state.get("first_value_at"))
+        if first_ingest is None or first_value is None:
+            return None
+        delta = (first_value - first_ingest).total_seconds()
+        return round(delta, 1) if delta >= 0 else None
+
+    def snapshot(self) -> Dict[str, Any]:
+        """Counters + derived rates for the admin surface."""
+        with self._lock:
+            counters = {name: int(self._state.get(name) or 0) for name in COUNTER_NAMES}
+            firsts = {name: self._state.get(name) for name in _FIRST_NAMES}
+        return {
+            "counters": counters,
+            "firsts": firsts,
+            "rates": {
+                "real_file_rate": _rate(
+                    counters["real_file_delivered"], counters["file_requests"]
+                ),
+                "code_only_rate": _rate(
+                    counters["code_only_responses"], counters["file_requests"]
+                ),
+                "needs_review_rate": _rate(
+                    counters["needs_review_runs"], counters["agent_runs"]
+                ),
+            },
+            "ttfv_seconds": self.ttfv_seconds(),
+            "generated_at": _utc_now_iso(),
+        }
+
+
+__all__ = ["FunnelMetricsService", "COUNTER_NAMES"]

package/latticeai/services/product_readiness.py

@@ -18,7 +18,7 @@ from typing import Any, Dict, List
 
 from latticeai.services.architecture_readiness import architecture_readiness
 
-PRODUCT_VERSION_TARGET = "9.9.2"
+PRODUCT_VERSION_TARGET = "9.9.3"
 
 
 @dataclass(frozen=True)

package/latticeai/services/search_service.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 from dataclasses import dataclass
 from typing import Any, Dict, Mapping, Optional
 
+from lattice_brain.graph.fusion import fusion_profile
+
 
 DEFAULT_HYBRID_WEIGHTS = {
     "keyword": 0.35,
@@ -402,7 +404,18 @@ class SearchService:
         allowed_workspaces=None,
         include_legacy_global: bool = False,
     ) -> Dict[str, Any]:
-        weights = {**DEFAULT_HYBRID_WEIGHTS, **dict(weights or {})}
+        # Query-class fusion (backlog #5): when the caller does not pin
+        # explicit weights, detect the query class (fact/code/person/recency)
+        # and use its per-class channel weights. Explicit weights still win,
+        # and the "fact" class equals DEFAULT_HYBRID_WEIGHTS, so pinned-weight
+        # callers and fact-class queries behave exactly as before.
+        query_class: Optional[str] = None
+        if weights is None:
+            profile = fusion_profile(query)
+            query_class = profile["query_class"]
+            weights = dict(profile["weights"])
+        else:
+            weights = {**DEFAULT_HYBRID_WEIGHTS, **dict(weights)}
         # Scope each channel at the source so out-of-scope rows never enter the
         # fusion set (defense-in-depth — the fused result is re-scoped below too).
         channels = {
@@ -464,9 +477,12 @@ class SearchService:
                 "weights": weights,
                 "sources": match.get("sources", []),
             }
+            if query_class is not None:
+                match["fusion"]["query_class"] = query_class
         return {
             "query": query,
             "mode": "hybrid",
+            "query_class": query_class,
             "weights": weights,
             "channels": {
                 name: {