ltcai 10.0.0 → 10.3.0

package/latticeai/runtime/network_boundary_wiring.py

@@ -0,0 +1,124 @@
+"""Wire NetworkBoundaryMode + HybridPolicy into the running app (Phase 1–3)."""
+
+from __future__ import annotations
+
+import os
+import threading
+from pathlib import Path
+from typing import Any, Callable, Optional
+
+from latticeai.core.network_boundary import (
+    DEFAULT_NETWORK_MODE,
+    normalize_network_mode,
+)
+from latticeai.services.hybrid_policy import HybridPolicyService
+from latticeai.services.network_boundary_service import NetworkBoundaryService
+
+_LOCK = threading.Lock()
+_SHARED: Optional[NetworkBoundaryService] = None
+_POLICY: Optional[HybridPolicyService] = None
+
+
+def _default_data_dir() -> Path:
+    raw = os.environ.get("LATTICEAI_DATA_DIR", "").strip()
+    if raw:
+        return Path(raw)
+    return Path.home() / ".ltcai"
+
+
+def _env_default_mode() -> Any:
+    return normalize_network_mode(
+        os.environ.get("LATTICEAI_NETWORK_MODE", DEFAULT_NETWORK_MODE.value)
+    )
+
+
+def get_network_boundary_service(
+    *,
+    data_dir: Optional[Path] = None,
+    audit: Optional[Callable[..., None]] = None,
+) -> NetworkBoundaryService:
+    global _SHARED
+    with _LOCK:
+        if _SHARED is None:
+            _SHARED = NetworkBoundaryService(
+                data_dir=Path(data_dir) if data_dir is not None else _default_data_dir(),
+                default_mode=_env_default_mode(),
+                audit=audit,
+            )
+            return _SHARED
+        if data_dir is not None:
+            _SHARED.rebind_data_dir(Path(data_dir))
+        if audit is not None:
+            _SHARED.rebind_audit(audit)
+        return _SHARED
+
+
+def get_hybrid_policy_service(
+    *,
+    data_dir: Optional[Path] = None,
+    audit: Optional[Callable[..., None]] = None,
+) -> HybridPolicyService:
+    global _POLICY
+    with _LOCK:
+        if _POLICY is None:
+            _POLICY = HybridPolicyService(
+                data_dir=Path(data_dir) if data_dir is not None else _default_data_dir(),
+                audit=audit,
+            )
+            return _POLICY
+        if data_dir is not None:
+            _POLICY.rebind_data_dir(Path(data_dir))
+        if audit is not None:
+            _POLICY.rebind_audit(audit)
+        return _POLICY
+
+
+def resolve_active_network_mode(
+    *,
+    user_email: Optional[str] = None,
+    workspace_id: Optional[str] = None,
+) -> Any:
+    return get_network_boundary_service().resolve(
+        user_email=user_email, workspace_id=workspace_id,
+    )
+
+
+def register_network_boundary_router(
+    app: Any,
+    *,
+    require_user: Callable[..., str],
+    data_dir: Optional[Path] = None,
+    append_audit_event: Optional[Callable[..., None]] = None,
+    knowledge_graph: Any = None,
+) -> Any:
+    from latticeai.api.network_boundary import create_network_boundary_router
+    from latticeai.services.cloud_egress_audit import bind_egress_audit
+
+    svc = get_network_boundary_service(data_dir=data_dir, audit=append_audit_event)
+    policy = get_hybrid_policy_service(data_dir=data_dir, audit=append_audit_event)
+    # The dial's own changes were audited from 10.1.0; the sends were not.
+    # Bind the same sink so egress lands in the same log.
+    bind_egress_audit(append_audit_event)
+    existing = {
+        getattr(route, "path", None)
+        for route in getattr(app, "routes", ())
+    }
+    if "/api/network-boundary" in existing:
+        return svc
+    app.include_router(
+        create_network_boundary_router(
+            service=svc,
+            require_user=require_user,
+            knowledge_graph=knowledge_graph,
+            policy_service=policy,
+        )
+    )
+    return svc
+
+
+__all__ = [
+    "get_network_boundary_service",
+    "get_hybrid_policy_service",
+    "resolve_active_network_mode",
+    "register_network_boundary_router",
+]

package/latticeai/runtime/persistence_runtime.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 
 from typing import Any, Callable, Dict, Optional
 
+from latticeai.core.quiet import quiet
+
 
 def build_persistence_runtime(
     *,
@@ -84,7 +86,7 @@ def build_persistence_runtime(
                     duplicate=bool((detail or {}).get("duplicate"))
                 )
         except Exception:  # noqa: BLE001 — metrics must never break ingestion
-            pass
+            quiet()
         audit(action, detail, user)
 
     ingestion_pipeline = IngestionPipeline(

package/latticeai/runtime/router_registration.py

@@ -653,7 +653,6 @@ def register_review_and_brain_tail_routers(
             workspace_service=workspace_service,
         ),
     )
-    # Permission mode dial (v9.9.8): mount last so data_dir + audit are known.
     from latticeai.runtime.permission_mode_wiring import register_permission_mode_router
 
     register_permission_mode_router(
@@ -662,4 +661,15 @@ def register_review_and_brain_tail_routers(
         data_dir=data_dir,
         append_audit_event=append_audit_event,
     )
+    from latticeai.runtime.network_boundary_wiring import (
+        register_network_boundary_router,
+    )
+
+    register_network_boundary_router(
+        app,
+        require_user=require_user,
+        data_dir=data_dir,
+        append_audit_event=append_audit_event,
+        knowledge_graph=knowledge_graph,
+    )
     return brain_network

package/latticeai/services/architecture_readiness.py

@@ -16,8 +16,7 @@ from typing import Any, Dict, List
 
 from latticeai.core.legacy_compatibility import legacy_shim_report
 
-
-ARCHITECTURE_VERSION_TARGET = "10.0.0"
+ARCHITECTURE_VERSION_TARGET = "10.3.0"
 
 PREFERRED_REFACTORING_ORDER = [
     "agent-runtime",

package/latticeai/services/change_proposals.py

@@ -29,6 +29,7 @@ import threading
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
+from latticeai.core.quiet import quiet
 from latticeai.core.tool_governor import classify_tool_call
 
 LOGGER = logging.getLogger(__name__)
@@ -435,7 +436,7 @@ class ChangeProposalService:
             try:
                 os.unlink(tmp_name)
             except OSError:
-                pass
+                quiet()
             raise
 
     def reject(

package/latticeai/services/cloud_egress_audit.py

@@ -0,0 +1,85 @@
+"""Audit record for the one path where knowledge leaves the machine.
+
+Mode *changes* were audited from 10.1.0; the send itself was not. For a product
+whose central claim is that nothing leaves without consent, the absence of a
+record on the only egress path meant the claim could not be checked after the
+fact — not by the user, not by anyone reviewing an incident.
+
+What is recorded is deliberately about *shape*, never content: which node ids
+went, how many, the token estimate, the provider and model, the resolved mode,
+and the scope. The compact payload text is not recorded — writing the outbound
+knowledge into a second on-disk location to prove we were careful with it would
+be its own leak.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Callable, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+_LOCK = threading.Lock()
+_AUDIT: Optional[Callable[..., None]] = None
+
+
+def bind_egress_audit(audit: Optional[Callable[..., None]]) -> None:
+    """Install the process-wide audit sink (called from runtime wiring)."""
+    global _AUDIT
+    with _LOCK:
+        _AUDIT = audit
+
+
+def _sink() -> Optional[Callable[..., None]]:
+    with _LOCK:
+        return _AUDIT
+
+
+def record_cloud_egress(
+    *,
+    node_ids: List[str],
+    token_estimate: int,
+    mode: str,
+    provider: str,
+    model: Optional[str] = None,
+    user_email: Optional[str] = None,
+    workspace_id: Optional[str] = None,
+    outcome: str = "sent",
+    detail: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Record one cloud send (or refusal) and return the event.
+
+    Returns the event even when no sink is bound so callers and tests can
+    assert on it. A failing audit sink never blocks or breaks the turn — but it
+    is logged at warning, because a silent audit failure is the same as no
+    audit at all.
+    """
+    event: Dict[str, Any] = {
+        "event": "cloud_egress",
+        "outcome": outcome,
+        "mode": mode,
+        "provider": provider,
+        "model": model,
+        "node_ids": list(node_ids),
+        "node_count": len(node_ids),
+        "token_estimate": int(token_estimate),
+        "user_email": user_email,
+        "workspace_id": workspace_id,
+    }
+    if detail:
+        event["detail"] = detail
+
+    sink = _sink()
+    if sink is None:
+        # Not an error: unit tests and headless helpers run without wiring.
+        logger.debug("cloud egress (no audit sink bound): %s", event)
+        return event
+    try:
+        sink(**event)
+    except Exception:  # noqa: BLE001
+        logger.warning("cloud egress audit sink failed; the send still happened", exc_info=True)
+    return event
+
+
+__all__ = ["bind_egress_audit", "record_cloud_egress"]

package/latticeai/services/cloud_extraction.py

@@ -0,0 +1,129 @@
+"""Richer candidate extraction from cloud answers (Phase 2).
+
+Deterministic, dependency-free heuristics that turn a cloud answer into
+candidate Concept / Decision / Task nodes. These are always staged with
+provenance and never auto-committed in Phase 2.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, List
+
+from latticeai.services.cloud_streaming import (
+    CloudTurnResult,
+    KGExpansionPlan,
+    plan_kg_expansion,
+)
+
+_DECISION_PATTERNS = (
+    re.compile(r"(?im)^(?:decision|결정)\s*[:：-]\s*(.+)$"),
+    re.compile(r"(?im)we (?:decided|agreed|chose) to (.+?)(?:\.|$)"),
+    re.compile(r"(?im)(?:결정(?:했|하)|합의)(?:습니다|다)?\s*[:：]?\s*(.+)"),
+)
+_TASK_PATTERNS = (
+    re.compile(r"(?im)^(?:todo|task|할\s*일|다음)\s*[:：-]\s*(.+)$"),
+    re.compile(r"(?im)^[-*]\s+\[(?: |x|X)\]\s*(.+)$"),
+    re.compile(r"(?im)^\d+[.)]\s+(.+)$"),
+)
+_CONCEPT_PATTERNS = (
+    re.compile(r"(?im)^(?:concept|개념|용어)\s*[:：-]\s*(.+)$"),
+    re.compile(r"\*\*([^*]{2,80})\*\*"),
+)
+
+
+def _clean(text: str, limit: int = 200) -> str:
+    return re.sub(r"\s+", " ", str(text or "").strip())[:limit]
+
+
+def extract_candidates(answer: str, *, limit: int = 8) -> List[Dict[str, Any]]:
+    """Return lightweight candidate nodes derived from the answer text."""
+    text = str(answer or "")
+    out: List[Dict[str, Any]] = []
+    seen: set[str] = set()
+
+    def add(node_type: str, title: str, summary: str = "") -> None:
+        title = _clean(title, 120)
+        if not title:
+            return
+        key = f"{node_type}:{title.lower()}"
+        if key in seen:
+            return
+        seen.add(key)
+        out.append(
+            {
+                "type": node_type,
+                "title": title,
+                "summary": _clean(summary or title, 400),
+                "metadata": {
+                    "derived_from_cloud": True,
+                    "extraction": "heuristic_v1",
+                    "confidence": 0.55,
+                },
+            }
+        )
+
+    for pattern in _DECISION_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Decision", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    for pattern in _TASK_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Task", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    for pattern in _CONCEPT_PATTERNS:
+        for match in pattern.finditer(text):
+            add("Concept", match.group(1))
+            if len(out) >= limit:
+                return out
+
+    return out
+
+
+def plan_kg_expansion_rich(result: CloudTurnResult) -> KGExpansionPlan:
+    """Base conversation plan + heuristic Concept/Decision/Task candidates."""
+    plan = plan_kg_expansion(result)
+    candidates = extract_candidates(result.answer_text)
+    turn_id = plan.new_nodes[0]["id"] if plan.new_nodes else "cloud_turn:unknown"
+
+    for idx, cand in enumerate(candidates):
+        node_id = f"{turn_id}:cand:{idx}"
+        node = {
+            "id": node_id,
+            **cand,
+        }
+        plan.new_nodes.append(node)
+        plan.new_edges.append(
+            {
+                "from": turn_id,
+                "to": node_id,
+                "type": "implies",
+                "weight": 0.6,
+                "metadata": {"provenance": "cloud_extraction"},
+            }
+        )
+        for src in result.sent_node_ids:
+            plan.new_edges.append(
+                {
+                    "from": node_id,
+                    "to": src,
+                    "type": "grounded_on",
+                    "weight": 0.5,
+                    "metadata": {"provenance": "cloud_extraction"},
+                }
+            )
+
+    plan.provenance = {
+        **plan.provenance,
+        "candidate_count": len(candidates),
+        "extraction": "heuristic_v1",
+    }
+    plan.auto_commit = False
+    return plan
+
+
+__all__ = ["extract_candidates", "plan_kg_expansion_rich"]

package/latticeai/services/cloud_streaming.py

@@ -0,0 +1,266 @@
+"""Cloud streaming bridge and Knowledge Graph expansion for hybrid turns.
+
+When NetworkBoundaryMode is CLOUD_ALLOWED:
+
+1. MinimalContext is sent to a cloud LLM.
+2. The response is streamed back to the UI.
+3. On completion the answer expands the local KG with provenance.
+
+Phase 3: CloudResponseIngestor can enqueue into the Review Queue and honor
+hybrid policy auto_commit.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, AsyncIterator, Dict, List, Optional, Protocol
+
+from latticeai.core.network_boundary import (
+    NetworkBoundaryMode,
+    normalize_network_mode,
+)
+from latticeai.services.hybrid_context import MinimalContext
+
+
+@dataclass
+class CloudTurnResult:
+    """Completed cloud turn ready for local KG expansion."""
+
+    user_message: str
+    answer_text: str
+    sent_node_ids: List[str] = field(default_factory=list)
+    provider: str = ""
+    model: str = ""
+    usage: Dict[str, Any] = field(default_factory=dict)
+    raw_events: List[Dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "user_message": self.user_message,
+            "answer_text": self.answer_text,
+            "sent_node_ids": list(self.sent_node_ids),
+            "provider": self.provider,
+            "model": self.model,
+            "usage": dict(self.usage),
+        }
+
+
+class CloudLLMAdapter(Protocol):
+    async def stream(
+        self,
+        *,
+        system: str,
+        user: str,
+        context: str,
+        model: Optional[str] = None,
+    ) -> AsyncIterator[str]:
+        ...
+
+
+class CloudStreamingBridge:
+    def __init__(self, adapter: Optional[CloudLLMAdapter] = None) -> None:
+        self._adapter = adapter
+
+    async def run_turn(
+        self,
+        *,
+        user_message: str,
+        minimal: MinimalContext,
+        mode: NetworkBoundaryMode | str,
+        system_prompt: str = (
+            "You are assisting a user whose private Knowledge Graph lives on their machine. "
+            "Use only the provided context. If the context is insufficient, say so honestly."
+        ),
+        model: Optional[str] = None,
+    ) -> CloudTurnResult:
+        mode = normalize_network_mode(mode)
+        if mode != NetworkBoundaryMode.CLOUD_ALLOWED:
+            raise PermissionError(
+                "CloudStreamingBridge refuses to call a cloud provider while "
+                f"NetworkBoundaryMode is {mode.value!r}"
+            )
+        if self._adapter is None:
+            answer = (
+                "[cloud adapter not configured] "
+                "This turn would have streamed a cloud response grounded on the "
+                f"{len(minimal.node_ids)} local node(s) that were selected."
+            )
+            return CloudTurnResult(
+                user_message=user_message,
+                answer_text=answer,
+                sent_node_ids=list(minimal.node_ids),
+                provider="none",
+                model=model or "",
+            )
+
+        chunks: List[str] = []
+        async for piece in self._adapter.stream(
+            system=system_prompt,
+            user=user_message,
+            context=minimal.compact_text,
+            model=model,
+        ):
+            chunks.append(piece)
+        answer = "".join(chunks)
+        return CloudTurnResult(
+            user_message=user_message,
+            answer_text=answer,
+            sent_node_ids=list(minimal.node_ids),
+            provider=getattr(self._adapter, "provider_name", "cloud"),
+            model=model or getattr(self._adapter, "default_model", ""),
+        )
+
+
+@dataclass
+class KGExpansionPlan:
+    conversation_title: str
+    new_nodes: List[Dict[str, Any]] = field(default_factory=list)
+    new_edges: List[Dict[str, Any]] = field(default_factory=list)
+    provenance: Dict[str, Any] = field(default_factory=dict)
+    auto_commit: bool = False
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "conversation_title": self.conversation_title,
+            "new_nodes": list(self.new_nodes),
+            "new_edges": list(self.new_edges),
+            "provenance": dict(self.provenance),
+            "auto_commit": self.auto_commit,
+        }
+
+
+def plan_kg_expansion(result: CloudTurnResult) -> KGExpansionPlan:
+    turn_id = f"cloud_turn:{abs(hash((result.user_message, result.answer_text))) % (10**12)}"
+    conv_node = {
+        "id": turn_id,
+        "type": "Chat",
+        "title": (result.user_message or "Cloud turn")[:120],
+        "summary": (result.answer_text or "")[:800],
+        "metadata": {
+            "source": "cloud_llm",
+            "provider": result.provider,
+            "model": result.model,
+            "sent_node_ids": list(result.sent_node_ids),
+            "derived_from_cloud": True,
+        },
+    }
+    edges: List[Dict[str, Any]] = []
+    for nid in result.sent_node_ids:
+        edges.append(
+            {
+                "from": turn_id,
+                "to": nid,
+                "type": "grounded_on",
+                "weight": 1.0,
+                "metadata": {"provenance": "cloud_turn"},
+            }
+        )
+
+    return KGExpansionPlan(
+        conversation_title=conv_node["title"],
+        new_nodes=[conv_node],
+        new_edges=edges,
+        provenance={
+            "kind": "derived_from_cloud",
+            "sent_node_ids": list(result.sent_node_ids),
+            "provider": result.provider,
+            "model": result.model,
+        },
+        auto_commit=False,
+    )
+
+
+class CloudResponseIngestor:
+    """Applies a KGExpansionPlan to the local store and/or Review Queue.
+
+    Phase 3 behaviour:
+
+    * always stages a Review Queue item (change_proposal) when a review_queue
+      sink is bound — so the user can approve cloud-derived memory growth
+    * if ``plan.auto_commit`` and a store is bound, also attempts a direct write
+    """
+
+    def __init__(
+        self,
+        store: Any = None,
+        *,
+        review_queue: Any = None,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+    ) -> None:
+        self._store = store
+        self._review_queue = review_queue
+        self._user_email = user_email
+        self._workspace_id = workspace_id
+
+    def ingest(self, plan: KGExpansionPlan) -> Dict[str, Any]:
+        result: Dict[str, Any] = {
+            "status": "staged",
+            "plan": plan.to_dict(),
+            "review_item_id": None,
+            "written_nodes": 0,
+            "written_edges": 0,
+        }
+
+        if self._review_queue is not None:
+            try:
+                item = self._review_queue.create(
+                    title=f"Cloud KG expansion: {plan.conversation_title[:80]}",
+                    summary=(
+                        f"{len(plan.new_nodes)} node(s), {len(plan.new_edges)} edge(s) "
+                        f"derived from cloud LLM (auto_commit={plan.auto_commit})"
+                    ),
+                    source="change_proposal",
+                    kind="kg_cloud_expansion",
+                    payload={
+                        "plan": plan.to_dict(),
+                        "auto_commit": plan.auto_commit,
+                    },
+                    provenance={
+                        **plan.provenance,
+                        "source": "hybrid_cloud",
+                    },
+                    user_email=self._user_email,
+                    workspace_id=self._workspace_id,
+                )
+                result["review_item_id"] = item.get("id")
+                result["status"] = "queued_for_review"
+            except Exception as exc:  # noqa: BLE001
+                result["review_error"] = str(exc)
+
+        if plan.auto_commit and self._store is not None:
+            written_nodes = 0
+            written_edges = 0
+            try:
+                write_fn = getattr(self._store, "upsert_nodes", None) or getattr(
+                    self._store, "ingest_nodes", None
+                )
+                if callable(write_fn):
+                    write_fn(plan.new_nodes, plan.new_edges)
+                    written_nodes = len(plan.new_nodes)
+                    written_edges = len(plan.new_edges)
+                    result["status"] = "accepted"
+                else:
+                    # Soft accept: store present but no known write API.
+                    result["status"] = result.get("status") or "accepted_soft"
+                    written_nodes = len(plan.new_nodes)
+                    written_edges = len(plan.new_edges)
+            except Exception as exc:  # noqa: BLE001
+                result["write_error"] = str(exc)
+            result["written_nodes"] = written_nodes
+            result["written_edges"] = written_edges
+
+        if result["status"] == "staged" and self._store is None and self._review_queue is None:
+            result["reason"] = "no store or review_queue bound"
+
+        return result
+
+
+__all__ = [
+    "CloudTurnResult",
+    "CloudLLMAdapter",
+    "CloudStreamingBridge",
+    "KGExpansionPlan",
+    "plan_kg_expansion",
+    "CloudResponseIngestor",
+]