ltcai 9.9.9 → 10.2.0

package/latticeai/services/hybrid_chat.py

@@ -0,0 +1,265 @@
+"""Hybrid chat turn: minimal KG context → cloud stream → local KG expansion.
+
+Phase 1–2 entry point used by the chat path when NetworkBoundaryMode is
+CLOUD_ALLOWED. Local-only sessions never enter this module's cloud path.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, AsyncIterator, Dict, Optional
+
+from latticeai.core.network_boundary import (
+    NetworkBoundaryMode,
+    normalize_network_mode,
+)
+from latticeai.services.cloud_egress_audit import record_cloud_egress
+from latticeai.services.cloud_extraction import plan_kg_expansion_rich
+from latticeai.services.cloud_streaming import (
+    CloudResponseIngestor,
+    CloudStreamingBridge,
+    CloudTurnResult,
+)
+from latticeai.services.cloud_token_guard import budget_for
+from latticeai.services.hybrid_context import MinimalContext, build_minimal_context
+from latticeai.services.openai_compatible_adapter import OpenAICompatibleAdapter
+
+logger = logging.getLogger(__name__)
+
+
+def _sse(data: Dict[str, Any]) -> str:
+    return f"data: {json.dumps(data, ensure_ascii=False)}\n\n"
+
+
+def _scope_key(user_email: Optional[str], workspace_id: Optional[str]) -> str:
+    return f"{user_email or 'anon'}|{workspace_id or 'global'}"
+
+
+async def run_hybrid_cloud_turn(
+    *,
+    user_message: str,
+    knowledge_graph: Any,
+    mode: NetworkBoundaryMode | str,
+    workspace_id: Optional[str] = None,
+    user_email: Optional[str] = None,
+    model: Optional[str] = None,
+    top_k: int = 6,
+    adapter: Optional[Any] = None,
+) -> CloudTurnResult:
+    """Non-streaming helper: full answer + expansion plan."""
+    mode = normalize_network_mode(mode)
+    if mode != NetworkBoundaryMode.CLOUD_ALLOWED:
+        raise PermissionError(
+            f"hybrid cloud turn refused under NetworkBoundaryMode={mode.value!r}"
+        )
+
+    minimal = build_minimal_context(
+        user_message,
+        store=knowledge_graph,
+        mode=mode,
+        top_k=top_k,
+        allowed_workspaces={workspace_id} if workspace_id else None,
+    )
+    budget = budget_for(_scope_key(user_email, workspace_id))
+    refusal = budget.check_turn(minimal.token_estimate)
+    if refusal:
+        record_cloud_egress(
+            node_ids=minimal.node_ids, token_estimate=minimal.token_estimate,
+            mode=mode.value, provider="(refused)", model=model,
+            user_email=user_email, workspace_id=workspace_id,
+            outcome="refused_token_guard", detail=refusal,
+        )
+        raise PermissionError(f"cloud token guard: {refusal}")
+
+    chosen_adapter = adapter or OpenAICompatibleAdapter()
+    record_cloud_egress(
+        node_ids=minimal.node_ids, token_estimate=minimal.token_estimate,
+        mode=mode.value, provider=getattr(chosen_adapter, "name", type(chosen_adapter).__name__),
+        model=model, user_email=user_email, workspace_id=workspace_id,
+    )
+    bridge = CloudStreamingBridge(adapter=chosen_adapter)
+    result = await bridge.run_turn(
+        user_message=user_message,
+        minimal=minimal,
+        mode=mode,
+        model=model,
+    )
+    plan = plan_kg_expansion_rich(result)
+    ingestor = CloudResponseIngestor(store=knowledge_graph)
+    ingest_status = ingestor.ingest(plan)
+    used = minimal.token_estimate + max(1, len(result.answer_text) // 4)
+    budget.record(used)
+    result.usage = {
+        **(result.usage or {}),
+        "minimal_nodes": len(minimal.node_ids),
+        "token_estimate": minimal.token_estimate,
+        "context_quality": minimal.quality,
+        "kg_expansion": ingest_status,
+        "token_budget": budget.snapshot(),
+    }
+    return result
+
+
+async def stream_hybrid_cloud_turn(
+    *,
+    user_message: str,
+    knowledge_graph: Any,
+    mode: NetworkBoundaryMode | str,
+    workspace_id: Optional[str] = None,
+    user_email: Optional[str] = None,
+    model: Optional[str] = None,
+    top_k: int = 6,
+    adapter: Optional[Any] = None,
+    chat_service: Any = None,
+    history_meta: Optional[Dict[str, Any]] = None,
+    history_user: Optional[Dict[str, Any]] = None,
+    notify: Any = None,
+    source: Optional[str] = None,
+) -> AsyncIterator[str]:
+    """SSE generator for a hybrid cloud turn (Phase 2 chat path).
+
+    Events:
+    * ``hybrid_context`` — which local nodes were selected
+    * ``token`` / classic ``chunk`` — streamed text
+    * ``hybrid_done`` — final answer + KG expansion status
+    * ``error`` — honest failure
+    """
+    mode = normalize_network_mode(mode)
+    if mode != NetworkBoundaryMode.CLOUD_ALLOWED:
+        yield _sse(
+            {
+                "type": "error",
+                "detail": f"NetworkBoundaryMode is {mode.value!r}; cloud path disabled",
+            }
+        )
+        yield "data: [DONE]\n\n"
+        return
+
+    minimal = build_minimal_context(
+        user_message,
+        store=knowledge_graph,
+        mode=mode,
+        top_k=top_k,
+        allowed_workspaces={workspace_id} if workspace_id else None,
+    )
+    budget = budget_for(_scope_key(user_email, workspace_id))
+    refusal = budget.check_turn(minimal.token_estimate)
+    if refusal:
+        # A refusal is auditable too: "nothing left the machine, and here is why".
+        record_cloud_egress(
+            node_ids=minimal.node_ids, token_estimate=minimal.token_estimate,
+            mode=mode.value, provider="(refused)", model=model,
+            user_email=user_email, workspace_id=workspace_id,
+            outcome="refused_token_guard", detail=refusal,
+        )
+        yield _sse({"type": "error", "detail": f"cloud token guard: {refusal}"})
+        yield "data: [DONE]\n\n"
+        return
+
+    yield _sse(
+        {
+            "type": "hybrid_context",
+            "node_ids": minimal.node_ids,
+            "keywords": minimal.keywords,
+            "token_estimate": minimal.token_estimate,
+            "quality": minimal.quality,
+            "titles": [str(n.get("title") or n.get("id") or "") for n in minimal.nodes],
+            "token_budget": budget.snapshot(),
+        }
+    )
+
+    chosen_adapter = adapter or OpenAICompatibleAdapter()
+    bridge = CloudStreamingBridge(adapter=chosen_adapter)
+
+    # Recorded before the call, not after: if the provider hangs or the process
+    # dies mid-stream, the record of what was about to leave must already exist.
+    record_cloud_egress(
+        node_ids=minimal.node_ids, token_estimate=minimal.token_estimate,
+        mode=mode.value, provider=getattr(chosen_adapter, "name", type(chosen_adapter).__name__),
+        model=model, user_email=user_email, workspace_id=workspace_id,
+    )
+
+    try:
+        chunks: list[str] = []
+        if chosen_adapter is not None and hasattr(chosen_adapter, "stream"):
+            system = (
+                "You are assisting a user whose private Knowledge Graph lives on their machine. "
+                "Use only the provided context. If the context is insufficient, say so honestly."
+            )
+            async for piece in chosen_adapter.stream(
+                system=system,
+                user=user_message,
+                context=minimal.compact_text,
+                model=model,
+            ):
+                chunks.append(piece)
+                # dual shape: hybrid token + classic chunk for existing clients
+                yield _sse({"type": "token", "text": piece, "chunk": piece, "model": model})
+            answer = "".join(chunks)
+            result = CloudTurnResult(
+                user_message=user_message,
+                answer_text=answer,
+                sent_node_ids=list(minimal.node_ids),
+                provider=getattr(chosen_adapter, "provider_name", "cloud"),
+                model=model or getattr(chosen_adapter, "default_model", ""),
+            )
+        else:
+            result = await bridge.run_turn(
+                user_message=user_message,
+                minimal=minimal,
+                mode=mode,
+                model=model,
+            )
+            yield _sse(
+                {
+                    "type": "token",
+                    "text": result.answer_text,
+                    "chunk": result.answer_text,
+                    "model": model,
+                }
+            )
+
+        plan = plan_kg_expansion_rich(result)
+        ingest_status = CloudResponseIngestor(store=knowledge_graph).ingest(plan)
+        used = minimal.token_estimate + max(1, len(result.answer_text) // 4)
+        budget.record(used)
+
+        if chat_service is not None:
+            try:
+                await chat_service.persist_entry(
+                    "assistant",
+                    result.answer_text,
+                    history_meta=history_meta or {},
+                    history_user=history_user or {},
+                )
+                if notify is not None:
+                    notify("assistant", result.answer_text, source)
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("hybrid chat persistence failed: %s", exc)
+
+        yield _sse(
+            {
+                "type": "hybrid_done",
+                "chunk": "",
+                "answer": result.answer_text,
+                "sent_node_ids": result.sent_node_ids,
+                "provider": result.provider,
+                "model": result.model,
+                "kg_expansion": ingest_status,
+                "token_estimate": minimal.token_estimate,
+                "token_budget": budget.snapshot(),
+            }
+        )
+        yield "data: [DONE]\n\n"
+    except Exception as exc:  # noqa: BLE001 — surface honest error to client
+        logger.warning("hybrid cloud turn failed: %s", exc)
+        yield _sse({"type": "error", "detail": str(exc), "error": str(exc)})
+        yield "data: [DONE]\n\n"
+
+
+__all__ = [
+    "run_hybrid_cloud_turn",
+    "stream_hybrid_cloud_turn",
+    "MinimalContext",
+]

package/latticeai/services/hybrid_context.py

@@ -0,0 +1,228 @@
+"""Minimal context extraction for hybrid cloud LLM turns.
+
+Given a user message and a local Knowledge Graph store, select only the
+smallest useful set of related nodes, assemble a compact text payload, and
+estimate token cost. This is the only knowledge that is ever allowed to leave
+the machine when NetworkBoundaryMode is CLOUD_ALLOWED.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Protocol, Sequence
+
+from latticeai.core.network_boundary import (
+    NetworkBoundaryMode,
+    is_node_blocked_for_cloud,
+    normalize_network_mode,
+)
+from latticeai.core.security import redact_secret_text
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MinimalContext:
+    """What may be sent to a cloud LLM."""
+
+    query: str
+    keywords: List[str] = field(default_factory=list)
+    node_ids: List[str] = field(default_factory=list)
+    compact_text: str = ""
+    nodes: List[Dict[str, Any]] = field(default_factory=list)
+    token_estimate: int = 0
+    quality: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "query": self.query,
+            "keywords": list(self.keywords),
+            "node_ids": list(self.node_ids),
+            "compact_text": self.compact_text,
+            "nodes": list(self.nodes),
+            "token_estimate": self.token_estimate,
+            "quality": dict(self.quality),
+        }
+
+
+class SupportsHybridSearch(Protocol):
+    """Minimal surface required from a KnowledgeGraph store."""
+
+    def hybrid_search(
+        self,
+        query: str,
+        *,
+        top_k: int = 20,
+        allowed_workspaces: Any = None,
+        include_legacy_global: bool = False,
+    ) -> Dict[str, Any]:
+        ...
+
+    def context_for_query_with_meta(
+        self,
+        query: str,
+        limit: int = 6,
+        *,
+        allowed_workspaces: Any = None,
+        include_legacy_global: bool = False,
+        use_hybrid: bool = True,
+    ) -> Dict[str, Any]:
+        ...
+
+
+def _rough_token_estimate(text: str) -> int:
+    """Cheap heuristic: ~4 characters per token for mixed EN/KO."""
+    if not text:
+        return 0
+    return max(1, (len(text) + 3) // 4)
+
+
+def _extract_keywords(message: str, limit: int = 12) -> List[str]:
+    """Very lightweight keyword / entity candidates.
+
+    Real extraction can later be replaced by a small local model or the
+    existing topic candidate helpers inside lattice_brain. For scaffolding we
+    keep this deterministic and dependency-free.
+    """
+    import re
+
+    text = str(message or "").strip()
+    if not text:
+        return []
+    # Split on whitespace and common punctuation; keep tokens of length >= 2.
+    tokens = re.findall(r"[\w가-힣]{2,}", text, flags=re.UNICODE)
+    seen: set[str] = set()
+    out: List[str] = []
+    for tok in tokens:
+        key = tok.lower()
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(tok)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def build_minimal_context(
+    message: str,
+    *,
+    store: Optional[SupportsHybridSearch] = None,
+    mode: NetworkBoundaryMode | str = NetworkBoundaryMode.LOCAL_ONLY,
+    top_k: int = 6,
+    allowed_workspaces: Any = None,
+    include_legacy_global: bool = False,
+    preferred_types: Optional[Sequence[str]] = None,
+) -> MinimalContext:
+    """Select the smallest useful set of local nodes for a cloud turn.
+
+    When mode is LOCAL_ONLY this still returns a MinimalContext (for local
+    RAG) but callers must not transmit it.
+    """
+    mode = normalize_network_mode(mode)
+    query = str(message or "").strip()
+    keywords = _extract_keywords(query)
+
+    empty = MinimalContext(
+        query=query,
+        keywords=keywords,
+        quality={"mode": "none", "nodes": 0, "limited": True, "reason": "no store or empty query"},
+    )
+    if not query or store is None:
+        return empty
+
+    preferred = set(
+        preferred_types
+        or (
+            "Decision",
+            "Concept",
+            "Task",
+            "Document",
+            "File",
+            "CodeFile",
+            "Person",
+            "Feature",
+        )
+    )
+
+    # One retrieval, not two. This used to call
+    # ``context_for_query_with_meta`` first — a full retrieval — and throw away
+    # everything but its ``quality`` dict, then run ``hybrid_search`` for the
+    # matches it actually needed. Every cloud turn paid for retrieval twice.
+    # ``context_quality_signal`` derives the same signal from the match list we
+    # already have.
+    matches: List[Dict[str, Any]] = []
+    quality: Dict[str, Any] = {}
+
+    try:
+        hybrid = store.hybrid_search(
+            query,
+            top_k=max(top_k * 2, top_k),
+            allowed_workspaces=allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+        )
+        matches = list(hybrid.get("matches") or [])
+        from lattice_brain.graph.retrieval import context_quality_signal
+
+        quality = context_quality_signal(
+            str(hybrid.get("mode") or "hybrid"),
+            len(matches),
+        )
+    except Exception:  # noqa: BLE001
+        # Fail closed: an unusable retrieval sends nothing rather than
+        # falling back to a broader, unfiltered context.
+        logger.warning("hybrid context retrieval failed; sending no context", exc_info=True)
+        matches = []
+
+    # Filter blocked + prefer useful types, then cut to top_k.
+    selected: List[Dict[str, Any]] = []
+    for match in matches:
+        if is_node_blocked_for_cloud(match):
+            continue
+        # Soft preference: keep preferred types first by stable partition later.
+        selected.append(match)
+
+    selected.sort(
+        key=lambda m: (
+            0 if str(m.get("type") or "") in preferred else 1,
+            -float(m.get("score") or 0.0),
+            str(m.get("node_id") or m.get("id") or ""),
+        )
+    )
+    selected = selected[: max(1, min(int(top_k), 12))]
+
+    lines: List[str] = []
+    node_ids: List[str] = []
+    for match in selected:
+        nid = str(match.get("node_id") or match.get("id") or "")
+        if not nid:
+            continue
+        node_ids.append(nid)
+        title = str(match.get("title") or nid)
+        summary = str(match.get("summary") or "")[:400]
+        ntype = str(match.get("type") or "Node")
+        lines.append(f"- [{ntype}] {title}: {summary}".rstrip(": "))
+
+    # Last gate before the text can leave the machine. `redact_secret_text` was
+    # already applied to logs, audit records, and API previews — everywhere
+    # except the one path where bytes actually go to a third party. A token
+    # pasted into a note is not marked sensitive by anyone, so pattern
+    # redaction is the only thing standing between it and the provider.
+    compact = redact_secret_text("\n".join(lines))
+    return MinimalContext(
+        query=query,
+        keywords=keywords,
+        node_ids=node_ids,
+        compact_text=compact,
+        nodes=selected,
+        token_estimate=_rough_token_estimate(compact),
+        quality=quality or {"mode": "none", "nodes": len(selected), "limited": len(selected) <= 1},
+    )
+
+
+__all__ = [
+    "MinimalContext",
+    "SupportsHybridSearch",
+    "build_minimal_context",
+]

package/latticeai/services/hybrid_policy.py

@@ -0,0 +1,178 @@
+"""User-configurable hybrid policy (Phase 3).
+
+Controls:
+
+* which node types / metadata flags are blocked from leaving the machine
+* whether cloud-derived KG expansion auto-commits or stages for review
+* whether multimodal (video) cloud calls are allowed when cloud mode is on
+
+Persisted under the data dir, scoped like NetworkBoundaryService
+(workspace → user → default).
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Set
+
+from latticeai.core.io_utils import atomic_write_json
+from latticeai.core.network_boundary import (
+    HARD_BLOCK_METADATA_FLAGS,
+    HARD_BLOCK_NODE_TYPES,
+)
+
+DEFAULT_BLOCKED_TYPES: List[str] = []
+DEFAULT_BLOCKED_FLAGS: List[str] = sorted(HARD_BLOCK_METADATA_FLAGS)
+DEFAULT_AUTO_COMMIT = False
+DEFAULT_ALLOW_MULTIMODAL = False
+
+
+def _default_policy() -> Dict[str, Any]:
+    return {
+        "blocked_node_types": list(DEFAULT_BLOCKED_TYPES),
+        "blocked_metadata_flags": list(DEFAULT_BLOCKED_FLAGS),
+        "auto_commit": DEFAULT_AUTO_COMMIT,
+        "allow_multimodal": DEFAULT_ALLOW_MULTIMODAL,
+        "min_extraction_confidence": 0.55,
+    }
+
+
+class HybridPolicyService:
+    """Load / save hybrid cloud policy."""
+
+    def __init__(
+        self,
+        *,
+        data_dir: Path,
+        audit: Optional[Callable[..., None]] = None,
+    ) -> None:
+        self._path = Path(data_dir) / "hybrid_policy.json"
+        self._audit = audit or (lambda *a, **kw: None)
+        self._lock = threading.Lock()
+
+    def rebind_data_dir(self, data_dir: Path) -> None:
+        with self._lock:
+            self._path = Path(data_dir) / "hybrid_policy.json"
+
+    def rebind_audit(self, audit: Callable[..., None]) -> None:
+        with self._lock:
+            self._audit = audit
+
+    def _read(self) -> Dict[str, Any]:
+        base = {
+            "default": _default_policy(),
+            "users": {},
+            "workspaces": {},
+        }
+        if not self._path.exists():
+            return base
+        try:
+            data = json.loads(self._path.read_text(encoding="utf-8"))
+        except Exception:
+            return base
+        if not isinstance(data, dict):
+            return base
+        data.setdefault("default", _default_policy())
+        data.setdefault("users", {})
+        data.setdefault("workspaces", {})
+        return data
+
+    def _write(self, data: Dict[str, Any]) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        atomic_write_json(self._path, data)
+
+    def _resolve_raw(
+        self,
+        data: Dict[str, Any],
+        *,
+        user_email: Optional[str],
+        workspace_id: Optional[str],
+    ) -> Dict[str, Any]:
+        policy = dict(data.get("default") or _default_policy())
+        if user_email:
+            user = (data.get("users") or {}).get(str(user_email).lower())
+            if isinstance(user, dict):
+                policy.update(user)
+        if workspace_id:
+            ws = (data.get("workspaces") or {}).get(str(workspace_id))
+            if isinstance(ws, dict):
+                policy.update(ws)
+        # Always union hard-coded circuit breakers.
+        blocked_types: Set[str] = set(policy.get("blocked_node_types") or [])
+        blocked_types |= set(HARD_BLOCK_NODE_TYPES)
+        blocked_flags: Set[str] = set(policy.get("blocked_metadata_flags") or [])
+        blocked_flags |= set(HARD_BLOCK_METADATA_FLAGS)
+        policy["blocked_node_types"] = sorted(blocked_types)
+        policy["blocked_metadata_flags"] = sorted(blocked_flags)
+        policy["auto_commit"] = bool(policy.get("auto_commit", False))
+        policy["allow_multimodal"] = bool(policy.get("allow_multimodal", False))
+        try:
+            policy["min_extraction_confidence"] = float(
+                policy.get("min_extraction_confidence", 0.55)
+            )
+        except (TypeError, ValueError):
+            policy["min_extraction_confidence"] = 0.55
+        return policy
+
+    def resolve(
+        self,
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        with self._lock:
+            data = self._read()
+        return self._resolve_raw(data, user_email=user_email, workspace_id=workspace_id)
+
+    def set_policy(
+        self,
+        patch: Dict[str, Any],
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        source: str = "api",
+    ) -> Dict[str, Any]:
+        allowed = {
+            "blocked_node_types",
+            "blocked_metadata_flags",
+            "auto_commit",
+            "allow_multimodal",
+            "min_extraction_confidence",
+        }
+        clean = {k: v for k, v in (patch or {}).items() if k in allowed}
+        with self._lock:
+            data = self._read()
+            previous = self._resolve_raw(
+                data, user_email=user_email, workspace_id=workspace_id
+            )
+            if workspace_id:
+                bucket = data.setdefault("workspaces", {})
+                current = dict(bucket.get(str(workspace_id)) or {})
+                current.update(clean)
+                bucket[str(workspace_id)] = current
+            elif user_email:
+                bucket = data.setdefault("users", {})
+                key = str(user_email).lower()
+                current = dict(bucket.get(key) or {})
+                current.update(clean)
+                bucket[key] = current
+            else:
+                current = dict(data.get("default") or _default_policy())
+                current.update(clean)
+                data["default"] = current
+            self._write(data)
+        resolved = self.resolve(user_email=user_email, workspace_id=workspace_id)
+        self._audit(
+            "hybrid_policy_changed",
+            user_email=user_email,
+            workspace_id=workspace_id,
+            previous=previous,
+            policy=resolved,
+            source=source,
+        )
+        return resolved
+
+
+__all__ = ["HybridPolicyService", "DEFAULT_AUTO_COMMIT", "DEFAULT_ALLOW_MULTIMODAL"]

package/latticeai/services/memory_service.py

@@ -26,8 +26,8 @@ import logging
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from latticeai.core.workspace_os_utils import _file_size
 from latticeai.core.timeutil import now_iso as _now
+from latticeai.core.workspace_os_utils import _file_size
 
 # Personal workspace memory kinds (from WorkspaceOS.MEMORY_KINDS).
 WORKSPACE_KINDS = (