ltcai 10.0.0 → 10.2.0

package/latticeai/services/cloud_egress_audit.py

@@ -0,0 +1,85 @@
+"""Audit record for the one path where knowledge leaves the machine.
+
+Mode *changes* were audited from 10.1.0; the send itself was not. For a product
+whose central claim is that nothing leaves without consent, the absence of a
+record on the only egress path meant the claim could not be checked after the
+fact — not by the user, not by anyone reviewing an incident.
+
+What is recorded is deliberately about *shape*, never content: which node ids
+went, how many, the token estimate, the provider and model, the resolved mode,
+and the scope. The compact payload text is not recorded — writing the outbound
+knowledge into a second on-disk location to prove we were careful with it would
+be its own leak.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_LOCK = threading.Lock()
+_AUDIT: Optional[Callable[..., None]] = None
+
+
+def bind_egress_audit(audit: Optional[Callable[..., None]]) -> None:
+    """Install the process-wide audit sink (called from runtime wiring)."""
+    global _AUDIT
+    with _LOCK:
+        _AUDIT = audit
+
+
+def _sink() -> Optional[Callable[..., None]]:
+    with _LOCK:
+        return _AUDIT
+
+
+def record_cloud_egress(
+    *,
+    node_ids: List[str],
+    token_estimate: int,
+    mode: str,
+    provider: str,
+    model: Optional[str] = None,
+    user_email: Optional[str] = None,
+    workspace_id: Optional[str] = None,
+    outcome: str = "sent",
+    detail: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Record one cloud send (or refusal) and return the event.
+
+    Returns the event even when no sink is bound so callers and tests can
+    assert on it. A failing audit sink never blocks or breaks the turn — but it
+    is logged at warning, because a silent audit failure is the same as no
+    audit at all.
+    """
+    event: Dict[str, Any] = {
+        "event": "cloud_egress",
+        "outcome": outcome,
+        "mode": mode,
+        "provider": provider,
+        "model": model,
+        "node_ids": list(node_ids),
+        "node_count": len(node_ids),
+        "token_estimate": int(token_estimate),
+        "user_email": user_email,
+        "workspace_id": workspace_id,
+    }
+    if detail:
+        event["detail"] = detail
+
+    sink = _sink()
+    if sink is None:
+        # Not an error: unit tests and headless helpers run without wiring.
+        logger.debug("cloud egress (no audit sink bound): %s", event)
+        return event
+    try:
+        sink(**event)
+    except Exception:  # noqa: BLE001
+        logger.warning("cloud egress audit sink failed; the send still happened", exc_info=True)
+    return event
+
+
+__all__ = ["bind_egress_audit", "record_cloud_egress"]

package/latticeai/services/cloud_extraction.py

@@ -0,0 +1,129 @@
+"""Richer candidate extraction from cloud answers (Phase 2).
+
+Deterministic, dependency-free heuristics that turn a cloud answer into
+candidate Concept / Decision / Task nodes. These are always staged with
+provenance and never auto-committed in Phase 2.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List
+
+from latticeai.services.cloud_streaming import (
+    CloudTurnResult,
+    KGExpansionPlan,
+    plan_kg_expansion,
+)
+
+_DECISION_PATTERNS = (
+    re.compile(r"(?im)^(?:decision|결정)\s*[:：-]\s*(.+)$"),
+    re.compile(r"(?im)we (?:decided|agreed|chose) to (.+?)(?:\.|$)"),
+    re.compile(r"(?im)(?:결정(?:했|하)|합의)(?:습니다|다)?\s*[:：]?\s*(.+)"),
+)
+_TASK_PATTERNS = (
+    re.compile(r"(?im)^(?:todo|task|할\s*일|다음)\s*[:：-]\s*(.+)$"),
+    re.compile(r"(?im)^[-*]\s+\[(?: |x|X)\]\s*(.+)$"),
+    re.compile(r"(?im)^\d+[.)]\s+(.+)$"),
+)
+_CONCEPT_PATTERNS = (
+    re.compile(r"(?im)^(?:concept|개념|용어)\s*[:：-]\s*(.+)$"),
+    re.compile(r"\*\*([^*]{2,80})\*\*"),
+)
+
+
+def _clean(text: str, limit: int = 200) -> str:
+    return re.sub(r"\s+", " ", str(text or "").strip())[:limit]
+
+
+def extract_candidates(answer: str, *, limit: int = 8) -> List[Dict[str, Any]]:
+    """Return lightweight candidate nodes derived from the answer text."""
+    text = str(answer or "")
+    out: List[Dict[str, Any]] = []
+    seen: set[str] = set()
+
+    def add(node_type: str, title: str, summary: str = "") -> None:
+        title = _clean(title, 120)
+        if not title:
+            return
+        key = f"{node_type}:{title.lower()}"
+        if key in seen:
+            return
+        seen.add(key)
+        out.append(
+            {
+                "type": node_type,
+                "title": title,
+                "summary": _clean(summary or title, 400),
+                "metadata": {
+                    "derived_from_cloud": True,
+                    "extraction": "heuristic_v1",
+                    "confidence": 0.55,
+                },
+            }
+        )
+
+    for pattern in _DECISION_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Decision", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    for pattern in _TASK_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Task", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    for pattern in _CONCEPT_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Concept", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    return out
+
+
+def plan_kg_expansion_rich(result: CloudTurnResult) -> KGExpansionPlan:
+    """Base conversation plan + heuristic Concept/Decision/Task candidates."""
+    plan = plan_kg_expansion(result)
+    candidates = extract_candidates(result.answer_text)
+    turn_id = plan.new_nodes[0]["id"] if plan.new_nodes else "cloud_turn:unknown"
+
+    for idx, cand in enumerate(candidates):
+        node_id = f"{turn_id}:cand:{idx}"
+        node = {
+            "id": node_id,
+            **cand,
+        }
+        plan.new_nodes.append(node)
+        plan.new_edges.append(
+            {
+                "from": turn_id,
+                "to": node_id,
+                "type": "implies",
+                "weight": 0.6,
+                "metadata": {"provenance": "cloud_extraction"},
+            }
+        )
+        for src in result.sent_node_ids:
+            plan.new_edges.append(
+                {
+                    "from": node_id,
+                    "to": src,
+                    "type": "grounded_on",
+                    "weight": 0.5,
+                    "metadata": {"provenance": "cloud_extraction"},
+                }
+            )
+
+    plan.provenance = {
+        **plan.provenance,
+        "candidate_count": len(candidates),
+        "extraction": "heuristic_v1",
+    }
+    plan.auto_commit = False
+    return plan
+
+
+__all__ = ["extract_candidates", "plan_kg_expansion_rich"]

package/latticeai/services/cloud_streaming.py

@@ -0,0 +1,266 @@
+"""Cloud streaming bridge and Knowledge Graph expansion for hybrid turns.
+
+When NetworkBoundaryMode is CLOUD_ALLOWED:
+
+1. MinimalContext is sent to a cloud LLM.
+2. The response is streamed back to the UI.
+3. On completion the answer expands the local KG with provenance.
+
+Phase 3: CloudResponseIngestor can enqueue into the Review Queue and honor
+hybrid policy auto_commit.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, AsyncIterator, Dict, List, Optional, Protocol
+
+from latticeai.core.network_boundary import (
+    NetworkBoundaryMode,
+    normalize_network_mode,
+)
+from latticeai.services.hybrid_context import MinimalContext
+
+
+@dataclass
+class CloudTurnResult:
+    """Completed cloud turn ready for local KG expansion."""
+
+    user_message: str
+    answer_text: str
+    sent_node_ids: List[str] = field(default_factory=list)
+    provider: str = ""
+    model: str = ""
+    usage: Dict[str, Any] = field(default_factory=dict)
+    raw_events: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "user_message": self.user_message,
+            "answer_text": self.answer_text,
+            "sent_node_ids": list(self.sent_node_ids),
+            "provider": self.provider,
+            "model": self.model,
+            "usage": dict(self.usage),
+        }
+
+
+class CloudLLMAdapter(Protocol):
+    async def stream(
+        self,
+        *,
+        system: str,
+        user: str,
+        context: str,
+        model: Optional[str] = None,
+    ) -> AsyncIterator[str]:
+        ...
+
+
+class CloudStreamingBridge:
+    def __init__(self, adapter: Optional[CloudLLMAdapter] = None) -> None:
+        self._adapter = adapter
+
+    async def run_turn(
+        self,
+        *,
+        user_message: str,
+        minimal: MinimalContext,
+        mode: NetworkBoundaryMode | str,
+        system_prompt: str = (
+            "You are assisting a user whose private Knowledge Graph lives on their machine. "
+            "Use only the provided context. If the context is insufficient, say so honestly."
+        ),
+        model: Optional[str] = None,
+    ) -> CloudTurnResult:
+        mode = normalize_network_mode(mode)
+        if mode != NetworkBoundaryMode.CLOUD_ALLOWED:
+            raise PermissionError(
+                "CloudStreamingBridge refuses to call a cloud provider while "
+                f"NetworkBoundaryMode is {mode.value!r}"
+            )
+        if self._adapter is None:
+            answer = (
+                "[cloud adapter not configured] "
+                "This turn would have streamed a cloud response grounded on the "
+                f"{len(minimal.node_ids)} local node(s) that were selected."
+            )
+            return CloudTurnResult(
+                user_message=user_message,
+                answer_text=answer,
+                sent_node_ids=list(minimal.node_ids),
+                provider="none",
+                model=model or "",
+            )
+
+        chunks: List[str] = []
+        async for piece in self._adapter.stream(
+            system=system_prompt,
+            user=user_message,
+            context=minimal.compact_text,
+            model=model,
+        ):
+            chunks.append(piece)
+        answer = "".join(chunks)
+        return CloudTurnResult(
+            user_message=user_message,
+            answer_text=answer,
+            sent_node_ids=list(minimal.node_ids),
+            provider=getattr(self._adapter, "provider_name", "cloud"),
+            model=model or getattr(self._adapter, "default_model", ""),
+        )
+
+
+@dataclass
+class KGExpansionPlan:
+    conversation_title: str
+    new_nodes: List[Dict[str, Any]] = field(default_factory=list)
+    new_edges: List[Dict[str, Any]] = field(default_factory=list)
+    provenance: Dict[str, Any] = field(default_factory=dict)
+    auto_commit: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "conversation_title": self.conversation_title,
+            "new_nodes": list(self.new_nodes),
+            "new_edges": list(self.new_edges),
+            "provenance": dict(self.provenance),
+            "auto_commit": self.auto_commit,
+        }
+
+
+def plan_kg_expansion(result: CloudTurnResult) -> KGExpansionPlan:
+    turn_id = f"cloud_turn:{abs(hash((result.user_message, result.answer_text))) % (10**12)}"
+    conv_node = {
+        "id": turn_id,
+        "type": "Chat",
+        "title": (result.user_message or "Cloud turn")[:120],
+        "summary": (result.answer_text or "")[:800],
+        "metadata": {
+            "source": "cloud_llm",
+            "provider": result.provider,
+            "model": result.model,
+            "sent_node_ids": list(result.sent_node_ids),
+            "derived_from_cloud": True,
+        },
+    }
+    edges: List[Dict[str, Any]] = []
+    for nid in result.sent_node_ids:
+        edges.append(
+            {
+                "from": turn_id,
+                "to": nid,
+                "type": "grounded_on",
+                "weight": 1.0,
+                "metadata": {"provenance": "cloud_turn"},
+            }
+        )
+
+    return KGExpansionPlan(
+        conversation_title=conv_node["title"],
+        new_nodes=[conv_node],
+        new_edges=edges,
+        provenance={
+            "kind": "derived_from_cloud",
+            "sent_node_ids": list(result.sent_node_ids),
+            "provider": result.provider,
+            "model": result.model,
+        },
+        auto_commit=False,
+    )
+
+
+class CloudResponseIngestor:
+    """Applies a KGExpansionPlan to the local store and/or Review Queue.
+
+    Phase 3 behaviour:
+
+    * always stages a Review Queue item (change_proposal) when a review_queue
+      sink is bound — so the user can approve cloud-derived memory growth
+    * if ``plan.auto_commit`` and a store is bound, also attempts a direct write
+    """
+
+    def __init__(
+        self,
+        store: Any = None,
+        *,
+        review_queue: Any = None,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> None:
+        self._store = store
+        self._review_queue = review_queue
+        self._user_email = user_email
+        self._workspace_id = workspace_id
+
+    def ingest(self, plan: KGExpansionPlan) -> Dict[str, Any]:
+        result: Dict[str, Any] = {
+            "status": "staged",
+            "plan": plan.to_dict(),
+            "review_item_id": None,
+            "written_nodes": 0,
+            "written_edges": 0,
+        }
+
+        if self._review_queue is not None:
+            try:
+                item = self._review_queue.create(
+                    title=f"Cloud KG expansion: {plan.conversation_title[:80]}",
+                    summary=(
+                        f"{len(plan.new_nodes)} node(s), {len(plan.new_edges)} edge(s) "
+                        f"derived from cloud LLM (auto_commit={plan.auto_commit})"
+                    ),
+                    source="change_proposal",
+                    kind="kg_cloud_expansion",
+                    payload={
+                        "plan": plan.to_dict(),
+                        "auto_commit": plan.auto_commit,
+                    },
+                    provenance={
+                        **plan.provenance,
+                        "source": "hybrid_cloud",
+                    },
+                    user_email=self._user_email,
+                    workspace_id=self._workspace_id,
+                )
+                result["review_item_id"] = item.get("id")
+                result["status"] = "queued_for_review"
+            except Exception as exc:  # noqa: BLE001
+                result["review_error"] = str(exc)
+
+        if plan.auto_commit and self._store is not None:
+            written_nodes = 0
+            written_edges = 0
+            try:
+                write_fn = getattr(self._store, "upsert_nodes", None) or getattr(
+                    self._store, "ingest_nodes", None
+                )
+                if callable(write_fn):
+                    write_fn(plan.new_nodes, plan.new_edges)
+                    written_nodes = len(plan.new_nodes)
+                    written_edges = len(plan.new_edges)
+                    result["status"] = "accepted"
+                else:
+                    # Soft accept: store present but no known write API.
+                    result["status"] = result.get("status") or "accepted_soft"
+                    written_nodes = len(plan.new_nodes)
+                    written_edges = len(plan.new_edges)
+            except Exception as exc:  # noqa: BLE001
+                result["write_error"] = str(exc)
+            result["written_nodes"] = written_nodes
+            result["written_edges"] = written_edges
+
+        if result["status"] == "staged" and self._store is None and self._review_queue is None:
+            result["reason"] = "no store or review_queue bound"
+
+        return result
+
+
+__all__ = [
+    "CloudTurnResult",
+    "CloudLLMAdapter",
+    "CloudStreamingBridge",
+    "KGExpansionPlan",
+    "plan_kg_expansion",
+    "CloudResponseIngestor",
+]

package/latticeai/services/cloud_token_guard.py

@@ -0,0 +1,84 @@
+"""Token / cost guardrails for hybrid cloud turns (Phase 2).
+
+Keeps cloud usage bounded:
+
+* per-turn token estimate must stay under ``max_tokens_per_turn``
+* optional session budget (in-memory, process-local) tracks cumulative usage
+
+These are soft product guardrails, not hard billing meters. Real provider
+usage is still authoritative when the cloud returns usage fields.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+from dataclasses import dataclass, field
+from typing import Any, Dict, Optional
+
+
+def _env_int(name: str, default: int) -> int:
+    raw = os.environ.get(name, "").strip()
+    if not raw:
+        return default
+    try:
+        return max(0, int(raw))
+    except ValueError:
+        return default
+
+
+@dataclass
+class TokenBudget:
+    max_tokens_per_turn: int = field(
+        default_factory=lambda: _env_int("LATTICEAI_CLOUD_MAX_TOKENS_PER_TURN", 2500)
+    )
+    max_tokens_per_session: int = field(
+        default_factory=lambda: _env_int("LATTICEAI_CLOUD_MAX_TOKENS_PER_SESSION", 50000)
+    )
+    session_used: int = 0
+
+    def check_turn(self, estimated_input_tokens: int) -> Optional[str]:
+        """Return a refusal reason, or None if the turn is allowed."""
+        if estimated_input_tokens > self.max_tokens_per_turn:
+            return (
+                f"estimated input tokens {estimated_input_tokens} exceed "
+                f"per-turn limit {self.max_tokens_per_turn}"
+            )
+        if self.session_used + estimated_input_tokens > self.max_tokens_per_session:
+            return (
+                f"session budget would exceed {self.max_tokens_per_session} "
+                f"(used={self.session_used}, turn={estimated_input_tokens})"
+            )
+        return None
+
+    def record(self, tokens: int) -> None:
+        self.session_used += max(0, int(tokens or 0))
+
+    def snapshot(self) -> Dict[str, Any]:
+        return {
+            "max_tokens_per_turn": self.max_tokens_per_turn,
+            "max_tokens_per_session": self.max_tokens_per_session,
+            "session_used": self.session_used,
+            "session_remaining": max(0, self.max_tokens_per_session - self.session_used),
+        }
+
+
+_LOCK = threading.Lock()
+_BUDGETS: Dict[str, TokenBudget] = {}
+
+
+def budget_for(scope_key: str) -> TokenBudget:
+    """Process-local budget keyed by user|workspace."""
+    key = str(scope_key or "global")
+    with _LOCK:
+        if key not in _BUDGETS:
+            _BUDGETS[key] = TokenBudget()
+        return _BUDGETS[key]
+
+
+def reset_budget(scope_key: str) -> None:
+    with _LOCK:
+        _BUDGETS.pop(str(scope_key or "global"), None)
+
+
+__all__ = ["TokenBudget", "budget_for", "reset_budget"]

package/latticeai/services/command_center.py

@@ -26,7 +26,8 @@ import logging
 from datetime import datetime, timedelta
 from typing import Any, Dict, List, Optional
 
-from latticeai.core.timeutil import local_now, now_iso as _now, parse_iso
+from latticeai.core.timeutil import local_now, parse_iso
+from latticeai.core.timeutil import now_iso as _now
 
 LOGGER = logging.getLogger(__name__)
 

package/latticeai/services/folder_watch.py

@@ -39,6 +39,7 @@ from lattice_brain.ingestion import (
     _load_latticeignore,
     _matches_ignore,
 )
+from latticeai.core.quiet import quiet
 from latticeai.core.timeutil import now_iso as _now_iso
 
 WATCH_INTERVAL_ENV = "LATTICEAI_FOLDER_WATCH_INTERVAL"
@@ -339,6 +340,7 @@ class FolderWatchService:
                 try:
                     stat = path.stat()
                 except OSError:
+                    quiet()
                     continue
                 if stat.st_size > DEFAULT_MAX_FILE_BYTES:
                     continue
@@ -390,6 +392,7 @@ class FolderWatchService:
                 try:
                     self.scan_once(watch_id)
                 except Exception:  # noqa: BLE001 — one bad watch must not kill the poller
+                    quiet()
                     continue
 
 

package/latticeai/services/funnel_metrics.py

@@ -37,6 +37,8 @@ from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any, Dict, Optional
 
+from latticeai.core.quiet import quiet
+
 LOGGER = logging.getLogger(__name__)
 
 COUNTER_NAMES = (
@@ -124,7 +126,7 @@ class FunnelMetricsService:
                     try:
                         os.unlink(tmp_name)
                     except OSError:
-                        pass
+                        quiet()
         except Exception as exc:  # noqa: BLE001 — metrics persistence is best-effort
             LOGGER.warning("funnel metrics save failed: %s", exc)