ltcai 3.5.0 → 4.0.0

package/latticeai/services/memory_service.py

@@ -24,7 +24,7 @@ from __future__ import annotations
 import json
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 # Personal workspace memory kinds (from WorkspaceOS.MEMORY_KINDS).
 WORKSPACE_KINDS = (
@@ -60,12 +60,16 @@ class MemoryService:
         knowledge_graph: Any = None,
         enable_graph: bool = True,
         history_file: Optional[Path] = None,
+        conversation_store: Any = None,
     ):
         self._store = store
         self._kg = knowledge_graph
         self._enable_graph = bool(enable_graph and knowledge_graph is not None)
         self._data_dir = Path(data_dir)
         self._history_file = Path(history_file) if history_file else (self._data_dir / "chat_history.json")
+        # v4: the durable SQLite conversation store supersedes the JSON file
+        # as the conversation tier's backing store when provided.
+        self._conversation_store = conversation_store
 
     # ── helpers over the underlying stores ────────────────────────────────
     def _workspace_memories(self, *, user_email: Optional[str], workspace_id: Optional[str]) -> List[Dict[str, Any]]:
@@ -87,6 +91,14 @@ class MemoryService:
             return []
 
     def _conversations(self) -> List[Dict[str, Any]]:
+        if self._conversation_store is not None:
+            try:
+                grouped: Dict[str, List[Dict[str, Any]]] = {}
+                for item in self._conversation_store.history():
+                    grouped.setdefault(item.get("conversation_id") or "legacy-previous-history", []).append(item)
+                return [{"id": conv_id, "messages": msgs} for conv_id, msgs in grouped.items()]
+            except Exception:
+                return []
         if not self._history_file.exists():
             return []
         try:
@@ -130,7 +142,10 @@ class MemoryService:
 
         ws_bytes = _file_size(self._data_dir / "workspace_os.json")
         kg_bytes = _file_size(self._data_dir / "knowledge_graph.sqlite")
-        conv_bytes = _file_size(self._history_file)
+        if self._conversation_store is not None:
+            conv_bytes = int(getattr(self._conversation_store, "size_bytes", lambda: 0)())
+        else:
+            conv_bytes = _file_size(self._history_file)
 
         node_total = sum((kg_stats or {}).get("nodes", {}).values()) if kg_stats else None
         edge_total = sum((kg_stats or {}).get("edges", {}).values()) if kg_stats else None
@@ -159,7 +174,7 @@ class MemoryService:
             {
                 "id": "conversation", "type": "conversation", "label": "Conversation Memory",
                 "count": len(convs), "size_bytes": conv_bytes,
-                "health": "ok" if self._history_file.exists() else "empty",
+                "health": "ok" if (self._conversation_store is not None or self._history_file.exists()) else "empty",
                 "detail": "Historical interaction memory from chat.",
             },
             {
@@ -202,6 +217,18 @@ class MemoryService:
         limit: int = 20,
     ) -> Dict[str, Any]:
         q = str(query or "").strip()
+        query_tokens = [tok for tok in q.lower().split() if tok]
+
+        def _lexical_score(*texts: Any) -> float:
+            # Honest, comparable relevance: fraction of query tokens present.
+            # Both tiers share this scorer so the cross-tier ranking is real,
+            # not an artifact of per-tier constants.
+            if not query_tokens:
+                return 0.0
+            haystack = " ".join(str(t or "") for t in texts).lower()
+            hits = sum(1 for tok in query_tokens if tok in haystack)
+            return round(hits / len(query_tokens), 4)
+
         results: List[Dict[str, Any]] = []
 
         try:
@@ -215,23 +242,24 @@ class MemoryService:
                 "title": (m.get("kind") or "memory"),
                 "snippet": str(m.get("content") or "")[:240],
                 "kind": m.get("kind"),
-                "score": 0.6,
+                "score": _lexical_score(m.get("content"), " ".join(m.get("tags") or []), m.get("kind")),
                 "tags": m.get("tags") or [],
             })
 
         if self._enable_graph and q:
             try:
-                hits = self._kg.search(q, limit).get("results", [])
+                # KnowledgeGraph.search returns {"query": ..., "matches": [...]}.
+                hits = self._kg.search(q, limit).get("matches", [])
             except Exception:
                 hits = []
-            for hsit in hits[:limit]:
+            for hit in hits[:limit]:
                 results.append({
                     "source": "graph",
-                    "id": hsit.get("id") or hsit.get("node_id"),
-                    "title": hsit.get("title") or hsit.get("name") or "node",
-                    "snippet": str(hsit.get("summary") or hsit.get("content") or "")[:240],
-                    "kind": hsit.get("type") or "node",
-                    "score": float(hsit.get("score") or 0.5),
+                    "id": hit.get("id") or hit.get("node_id"),
+                    "title": hit.get("title") or hit.get("name") or "node",
+                    "snippet": str(hit.get("summary") or hit.get("content") or "")[:240],
+                    "kind": hit.get("type") or "node",
+                    "score": _lexical_score(hit.get("title"), hit.get("name"), hit.get("summary"), hit.get("content")),
                 })
 
         results.sort(key=lambda r: r.get("score", 0), reverse=True)

package/latticeai/services/model_runtime.py

@@ -18,7 +18,6 @@ import re
 import shutil
 import subprocess
 import sys
-import tempfile
 import threading
 import time
 import urllib.error
@@ -26,16 +25,14 @@ import urllib.request
 from pathlib import Path
 from typing import AsyncIterator, Dict, List, Optional
 
-import httpx
 from fastapi import HTTPException, Request
 
-from llm_router import (
+from latticeai.models.router import (
     AsyncOpenAI,
     HF_MODELS_ROOT,
     OPENAI_COMPATIBLE_PROVIDERS,
     ensure_mlx_runtime,
     hf_model_dir,
-    normalize_branding,
     parse_model_ref,
 )
 from latticeai.core.model_compat import (
@@ -89,7 +86,7 @@ def configure_model_runtime(**deps) -> None:
 # Catalog data + version-dedup helpers live in ``model_catalog``; re-exported
 # here so existing ``from ...model_runtime import ENGINE_MODEL_CATALOG`` imports
 # keep working.
-from latticeai.services.model_catalog import (  # noqa: F401  (re-export)
+from latticeai.services.model_catalog import (  # noqa: E402, F401 (re-export after the module globals it documents)
     ENGINE_INSTALLERS,
     ENGINE_MODEL_CATALOG,
     MODEL_ENGINE_ALIASES,

package/latticeai/services/platform_runtime.py

@@ -18,8 +18,9 @@ from typing import Any, Callable, Dict, Optional, Set
 from fastapi import HTTPException, Request
 
 from latticeai.core.hooks import dispatch_tool
-from latticeai.core.multi_agent import MultiAgentOrchestrator, default_role_runner
-from latticeai.core.workflow_engine import WorkflowEngine
+from latticeai.core.multi_agent import MultiAgentOrchestrator, default_role_runner, llm_role_runner
+from latticeai.core.workflow_engine import ApprovalRequired, WorkflowEngine
+from tools import execute_tool
 
 
 class PlatformRuntime:
@@ -34,6 +35,9 @@ class PlatformRuntime:
         workspace_scope_from_request: Callable[[Request], Optional[str]],
         get_tool_permission: Callable[..., Dict[str, Any]],
         hooks: Any = None,
+        llm_generate: Optional[Callable[..., str]] = None,
+        llm_available: Optional[Callable[[], bool]] = None,
+        agent_registry: Any = None,
     ):
         self.store = store
         self.svc = workspace_service
@@ -45,6 +49,12 @@ class PlatformRuntime:
         # Lifecycle hooks registry — wires the workflow runtime + workflow tool
         # nodes into the same pre_*/post_* lifecycle as the HTTP + agent paths.
         self.hooks = hooks
+        # v4 (T7b): a synchronous model bridge. When a model is loaded,
+        # build_orchestrator returns the REAL (mode='llm') runner; otherwise
+        # the deterministic runner, honestly labeled mode='simulation'.
+        self.llm_generate = llm_generate
+        self.llm_available = llm_available or (lambda: False)
+        self.agent_registry = agent_registry
 
     # ── request gating ────────────────────────────────────────────────────
 
@@ -77,31 +87,57 @@ class PlatformRuntime:
     # ── shared node runners ───────────────────────────────────────────────
 
     def _tool_node_runner(self):
-        """Workflow tool node: records the invocation + governance decision but
-        never silently executes exec/destructive tools (those need approval)."""
+        """Workflow tool node: EXECUTES the tool under governance (v4).
+
+        Auto-approve tools run immediately through the shared dispatch_tool
+        lifecycle. Tools whose policy requires approval raise
+        :class:`ApprovalRequired` so the engine pauses the run into
+        ``awaiting_approval`` — never a silent ``{recorded: true}`` success,
+        never an unapproved execution. A resumed run carries the approved
+        node id in ``context['__approved_nodes__']``.
+        """
         def runner(*, node, context):
             cfg = node.get("config") or {}
             name = cfg.get("tool") or ""
             args = cfg.get("args") or {}
-
-            def _record():
-                try:
-                    permission = dict(self.get_tool_permission(name))
-                except Exception:
-                    permission = {"tool": name, "risk": "unknown"}
-                return {"tool": name, "args": args, "recorded": True, "permission": permission}
+            if not name:
+                raise ValueError("tool node has no tool configured")
+            try:
+                permission = dict(self.get_tool_permission(name, args))
+            except TypeError:
+                permission = dict(self.get_tool_permission(name))
+            approved_nodes = set(context.get("__approved_nodes__") or [])
+            if permission.get("requires_approval") and node.get("id") not in approved_nodes:
+                raise ApprovalRequired(
+                    f"tool '{name}' requires explicit approval before a workflow may run it",
+                    tool=name, args=args, permission=permission,
+                )
+
+            def _execute():
+                return execute_tool(name, args)
 
             # Same tool lifecycle as the HTTP + agent paths (a pre_tool block
             # raises PermissionError, surfaced as the node error by the engine).
-            return dispatch_tool(self.hooks, name or "tool", args, _record, source="workflow")
+            result = dispatch_tool(self.hooks, name, args, _execute, source="workflow")
+            return {"tool": name, "args": args, "executed": True,
+                    "permission": permission, "result": result}
         return runner
 
     def _skill_node_runner(self):
+        """Skill nodes refuse honestly: a skill is an instruction package for
+        an LLM; without a model-driven executor there is nothing to run, and
+        pretending otherwise (the pre-v4 existence check that reported 'ok')
+        is exactly the fake functionality v4 bans."""
         def runner(*, node, context):
             cfg = node.get("config") or {}
             name = cfg.get("skill") or ""
             entry = self.store.load_state().get("skill_registry", {}).get(name) or {}
-            return {"skill": name, "found": bool(entry), "enabled": bool(entry.get("enabled"))}
+            if not entry:
+                raise ValueError(f"skill '{name}' is not installed")
+            raise RuntimeError(
+                f"skill '{name}' requires LLM-driven execution, which workflow "
+                "skill nodes do not provide in this build — refusing to fake a result"
+            )
         return runner
 
     def _context_provider(self, user, scope):
@@ -116,15 +152,25 @@ class PlatformRuntime:
     def plugin_capability_runners(self, user, scope) -> Dict[str, Callable[..., Any]]:
         """Runners the Plugin SDK boundary dispatches to (one per capability)."""
         def run_skill(*, plugin_id, action, args, manifest):
-            return {"plugin": plugin_id, "ran_skills": manifest.provides.get("skills", [])}
+            raise RuntimeError(
+                f"plugin '{plugin_id}' skill execution requires an LLM-driven "
+                "runner, which this build does not provide — refusing to fake a result"
+            )
 
         def run_tool(*, plugin_id, action, args, manifest):
             tool = args.get("tool") or (manifest.provides.get("tools") or [None])[0]
-            try:
-                permission = dict(self.get_tool_permission(tool)) if tool else {}
-            except Exception:
-                permission = {}
-            return {"plugin": plugin_id, "tool": tool, "permission": permission, "recorded": True}
+            if not tool:
+                raise ValueError(f"plugin '{plugin_id}' run_tool needs a tool name")
+            permission = dict(self.get_tool_permission(tool))
+            if permission.get("requires_approval"):
+                raise ApprovalRequired(
+                    f"plugin tool '{tool}' requires explicit approval",
+                    tool=tool, args=args, permission=permission,
+                )
+            result = dispatch_tool(self.hooks, tool, args, lambda: execute_tool(tool, args),
+                                   source=f"plugin:{plugin_id}")
+            return {"plugin": plugin_id, "tool": tool, "permission": permission,
+                    "executed": True, "result": result}
 
         def run_workflow(*, plugin_id, action, args, manifest):
             wf_id = args.get("workflow_id")
@@ -175,6 +221,9 @@ class PlatformRuntime:
             workflow_id=workflow_id, name=workflow.get("name") or "workflow",
             status=result.status, timeline=result.timeline, outputs=result.outputs,
             user_email=user, graph=self.workspace_graph(), workspace_id=scope,
+            mode="live",
+            pause={"node": result.paused_node, "pending": result.pending_approval,
+                   "context": result.paused_context} if result.status == "awaiting_approval" else None,
         )
         return {"workflow_run_id": run["id"], "status": result.status}
 
@@ -209,8 +258,36 @@ class PlatformRuntime:
         }
 
     def build_orchestrator(self, user, scope) -> MultiAgentOrchestrator:
+        workflow_runner = lambda wf_ref, ctx: self.run_workflow_by_id(wf_ref, user, scope, with_agent=False, inputs=ctx.inputs)  # noqa: E731
+        plugin_runner = lambda pid, ctx: self.registry.execute_action(pid, "run_skill", {}, runners=self.plugin_capability_runners(user, scope), workspace_id=scope).as_dict()  # noqa: E731
+        context_provider = self._context_provider(user, scope)
+        custom_agents = {}
+        if self.agent_registry is not None:
+            try:
+                custom_agents = {
+                    a["id"]: a for a in self.agent_registry.all()
+                    if str(a.get("id", "")).startswith("agent:custom:") and a.get("enabled", True)
+                }
+            except Exception:
+                custom_agents = {}
+        if self.llm_generate is not None and self.llm_available():
+            from latticeai.core.agent_prompts import CRITIC_PROMPT, PLANNER_PROMPT
+
+            return MultiAgentOrchestrator(
+                role_runner=llm_role_runner(
+                    generate=self.llm_generate,
+                    planner_prompt=PLANNER_PROMPT,
+                    critic_prompt=CRITIC_PROMPT,
+                    context_provider=context_provider,
+                    workflow_runner=workflow_runner,
+                    plugin_runner=plugin_runner,
+                    custom_agents=custom_agents,
+                ),
+                mode="llm",
+                custom_agents=custom_agents,
+            )
         return MultiAgentOrchestrator(role_runner=default_role_runner(
-            workflow_runner=lambda wf_ref, ctx: self.run_workflow_by_id(wf_ref, user, scope, with_agent=False, inputs=ctx.inputs),
-            plugin_runner=lambda pid, ctx: self.registry.execute_action(pid, "run_skill", {}, runners=self.plugin_capability_runners(user, scope), workspace_id=scope).as_dict(),
-            context_provider=self._context_provider(user, scope),
-        ))
+            workflow_runner=workflow_runner,
+            plugin_runner=plugin_runner,
+            context_provider=context_provider,
+        ), mode="simulation", custom_agents=custom_agents)

package/latticeai/services/search_service.py

@@ -7,7 +7,7 @@ keyword search into UI-ready contracts without tying routers to store internals.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, Dict, List, Mapping, Optional
+from typing import Any, Dict, Mapping, Optional
 
 
 DEFAULT_HYBRID_WEIGHTS = {
@@ -34,7 +34,15 @@ class SearchService:
             raise ValueError("knowledge graph is disabled")
         return self.graph_store
 
-    def keyword_search(self, query: str, *, limit: int = 30) -> Dict[str, Any]:
+    def _scope(self, matches, allowed_workspaces):
+        """Drop matches scoped to workspaces the caller is not a member of
+        (None = no scoping; legacy-global rows stay visible — documented)."""
+        if allowed_workspaces is None:
+            return matches
+        graph = self._require_graph()
+        return graph.filter_scoped_nodes(matches, allowed_workspaces)
+
+    def keyword_search(self, query: str, *, limit: int = 30, allowed_workspaces=None) -> Dict[str, Any]:
         graph = self._require_graph()
         payload = graph.search(query, limit)
         matches = []
@@ -53,9 +61,9 @@ class SearchService:
                 "metadata": match.get("metadata") or {},
                 "updated_at": match.get("updated_at"),
             })
-        return {"query": query, "mode": "keyword", "matches": matches}
+        return {"query": query, "mode": "keyword", "matches": self._scope(matches, allowed_workspaces)}
 
-    def vector_search(self, query: str, *, limit: int = 30, min_score: float = 0.0) -> Dict[str, Any]:
+    def vector_search(self, query: str, *, limit: int = 30, min_score: float = 0.0, allowed_workspaces=None) -> Dict[str, Any]:
         graph = self._require_graph()
         payload = graph.vector_search(query, limit=limit, min_score=min_score)
         matches = []
@@ -80,10 +88,10 @@ class SearchService:
             "mode": "vector",
             "embedding_model": payload.get("embedding_model"),
             "embedding_dim": payload.get("embedding_dim"),
-            "matches": matches,
+            "matches": self._scope(matches, allowed_workspaces),
         }
 
-    def graph_search(self, query: str, *, limit: int = 30, expand_depth: int = 1) -> Dict[str, Any]:
+    def graph_search(self, query: str, *, limit: int = 30, expand_depth: int = 1, allowed_workspaces=None) -> Dict[str, Any]:
         graph = self._require_graph()
         limit = max(1, min(int(limit or 30), 100))
         expand_depth = max(0, min(int(expand_depth or 1), 3))
@@ -157,7 +165,7 @@ class SearchService:
             match["rank"] = rank
             match["score"] = round(float(match["score"]), 6)
             match["source_scores"]["graph"] = round(float(match["source_scores"]["graph"]), 6)
-        return {"query": query, "mode": "graph", "expand_depth": expand_depth, "matches": matches}
+        return {"query": query, "mode": "graph", "expand_depth": expand_depth, "matches": self._scope(matches, allowed_workspaces)}
 
     def hybrid_search(
         self,
@@ -168,6 +176,7 @@ class SearchService:
         vector_limit: int = 30,
         graph_limit: int = 30,
         weights: Optional[Mapping[str, float]] = None,
+        allowed_workspaces=None,
     ) -> Dict[str, Any]:
         weights = {**DEFAULT_HYBRID_WEIGHTS, **dict(weights or {})}
         channels = {
@@ -202,7 +211,7 @@ class SearchService:
                     current.setdefault("graph_context", [])
                     current["graph_context"].extend(result.get("graph_context") or [])
 
-        matches = sorted(fused.values(), key=lambda item: item["score"], reverse=True)[: max(1, min(limit, 100))]
+        matches = self._scope(sorted(fused.values(), key=lambda item: item["score"], reverse=True), allowed_workspaces)[: max(1, min(limit, 100))]
         for rank, match in enumerate(matches, start=1):
             match["rank"] = rank
             match["score"] = round(float(match["score"]), 6)

package/latticeai/services/tool_dispatch.py

@@ -22,8 +22,16 @@ from latticeai.core.tool_registry import ToolPermission, ToolPolicy
 from tools import AGENT_ROOT, DEFAULT_TOOL_REGISTRY, ToolError, ensure_agent_root
 
 
-_load_users: Callable[[], Dict[str, Any]] = lambda: {}
-_get_user_role: Callable[..., str] = lambda _email, _users=None: "user"
+def _default_load_users() -> Dict[str, Any]:
+    return {}
+
+
+def _default_get_user_role(_email, _users=None) -> str:
+    return "user"
+
+
+_load_users: Callable[[], Dict[str, Any]] = _default_load_users
+_get_user_role: Callable[..., str] = _default_get_user_role
 
 FILE_CREATE_ACTIONS = set(DEFAULT_TOOL_REGISTRY.file_create_actions)
 TOOL_GOVERNANCE: Dict[str, ToolPolicy] = dict(DEFAULT_TOOL_REGISTRY.governance)
@@ -104,6 +112,7 @@ def build_agent_runtime(
     knowledge_save: Callable[..., Dict[str, Any]],
     audit: Callable[..., None],
     hooks: Any = None,
+    brain_memory: Any = None,
 ) -> AgentRuntime:
     ensure_agent_root()
     deps = AgentDeps(
@@ -125,6 +134,7 @@ def build_agent_runtime(
         memory_updater_prompt=MEMORY_UPDATER_PROMPT,
         agent_root=AGENT_ROOT,
         hooks=hooks,
+        brain_memory=brain_memory,
     )
     return AgentRuntime(deps)