ltcai 2.2.7 → 3.1.0

package/latticeai/core/local_embeddings.py

@@ -0,0 +1,86 @@
+"""Local deterministic embeddings for Lattice AI search.
+
+The v3 backend needs a local-first vector signal without introducing a cloud
+runtime requirement. This module provides a small feature-hashing embedder that
+is deterministic, cheap to run, and good enough for indexing/search tests. A
+future runtime can swap the implementation behind the same interface when a
+local model server is available.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import re
+import struct
+from dataclasses import dataclass
+from typing import Iterable, List
+
+
+DEFAULT_EMBEDDING_DIM = int(os.getenv("LATTICEAI_VECTOR_DIM", "384"))
+EMBEDDING_MODEL_ID = f"lattice-local-hash-v1:{DEFAULT_EMBEDDING_DIM}"
+
+
+def _tokenize(text: str) -> List[str]:
+    raw = str(text or "").lower()
+    tokens = re.findall(r"[a-z0-9][a-z0-9_.:/+-]{1,}|[가-힣]{2,}", raw)
+    features: List[str] = []
+    for token in tokens:
+        features.append(f"tok:{token}")
+        if len(token) >= 5 and re.search(r"[a-z]", token):
+            for i in range(0, len(token) - 2):
+                features.append(f"tri:{token[i:i+3]}")
+        if re.search(r"[가-힣]", token) and len(token) >= 3:
+            for i in range(0, len(token) - 1):
+                features.append(f"ko:{token[i:i+2]}")
+    return features
+
+
+def _hash_to_index(feature: str, dim: int) -> tuple[int, float]:
+    digest = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest()
+    value = int.from_bytes(digest, "big", signed=False)
+    sign = 1.0 if (value & 1) == 0 else -1.0
+    return value % dim, sign
+
+
+@dataclass(frozen=True)
+class LocalEmbeddingModel:
+    """Deterministic feature-hashing embedder.
+
+    The output vectors are L2-normalized, so cosine similarity is just a dot
+    product. No network access, model download, or global mutable state is
+    required.
+    """
+
+    dim: int = DEFAULT_EMBEDDING_DIM
+    model_id: str = EMBEDDING_MODEL_ID
+
+    def embed(self, text: str) -> List[float]:
+        vector = [0.0] * self.dim
+        features = _tokenize(text)
+        if not features:
+            return vector
+        for feature in features:
+            index, sign = _hash_to_index(feature, self.dim)
+            vector[index] += sign
+        norm = math.sqrt(sum(value * value for value in vector))
+        if norm <= 0:
+            return vector
+        return [value / norm for value in vector]
+
+    def similarity(self, left: Iterable[float], right: Iterable[float]) -> float:
+        return float(sum(a * b for a, b in zip(left, right)))
+
+    def encode(self, vector: Iterable[float]) -> bytes:
+        values = list(vector)
+        return struct.pack(f"<{len(values)}f", *values)
+
+    def decode(self, payload: bytes, dim: int | None = None) -> List[float]:
+        if not payload:
+            return []
+        count = int(dim or self.dim)
+        expected = count * 4
+        if len(payload) != expected:
+            count = len(payload) // 4
+        return list(struct.unpack(f"<{count}f", payload[: count * 4]))

package/latticeai/core/workspace_os.py

@@ -18,7 +18,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
 
-WORKSPACE_OS_VERSION = "2.2.7"
+WORKSPACE_OS_VERSION = "3.1.0"
 
 # Workspace types separate single-user Personal workspaces from shared
 # Organization workspaces. Both keep the same local-first JSON store; the type

package/latticeai/server_app.py

@@ -72,6 +72,9 @@ from latticeai.core.enterprise import (
 from latticeai.services.workspace_service import WorkspaceService
 from latticeai.services.model_service import ModelService
 from latticeai.services.chat_service import ChatService
+from latticeai.services.search_service import SearchService
+from latticeai.core.embedding_providers import resolve_embedder, resolve_embedding_profile
+from latticeai.services.agent_runtime import AgentRuntime
 from latticeai.services.model_runtime import (
     CLOUD_VERIFY_TTL_SECONDS,
     ENGINE_MODEL_CATALOG,
@@ -105,6 +108,7 @@ from latticeai.api.realtime import create_realtime_router
 from latticeai.api.marketplace import create_marketplace_router
 from latticeai.api.models import create_models_router
 from latticeai.api.chat import create_chat_router
+from latticeai.api.search import create_search_router
 from latticeai.api.tools import create_tools_router
 from latticeai.api.static_routes import create_static_routes_router
 from latticeai.api.garden import create_garden_router
@@ -244,7 +248,37 @@ VPC_FILE = DATA_DIR / "vpc_config.json"
 MCP_FILE = DATA_DIR / "mcp_installs.json"
 AUDIT_FILE = DATA_DIR / "audit_log.json"
 SSO_FILE = DATA_DIR / "sso_config.json"
-KNOWLEDGE_GRAPH = KnowledgeGraphStore(DATA_DIR / "knowledge_graph.sqlite", DATA_DIR / "knowledge_graph_blobs") if ENABLE_GRAPH else None
+# Resolve the configured embedding provider once at startup. Degrades to the
+# offline hash fallback when the requested provider is unavailable, while
+# recording the requested-vs-active provider for the Embeddings status surface.
+try:
+    EMBEDDING_PROFILE = resolve_embedding_profile(CONFIG.embedding_profile)
+except ValueError as exc:
+    logging.warning("Embedding profile ignored: %s", exc)
+    EMBEDDING_PROFILE = {}
+_embedding_provider = CONFIG.embedding_provider
+_embedding_model = CONFIG.embedding_model or str(EMBEDDING_PROFILE.get("model") or "")
+_embedding_dim = CONFIG.embedding_dim or int(EMBEDDING_PROFILE.get("dimensions") or 0)
+if CONFIG.embedding_profile and CONFIG.embedding_provider in {"", "hash", "local", "fallback"}:
+    _embedding_provider = str(EMBEDDING_PROFILE.get("provider") or CONFIG.embedding_provider)
+
+EMBEDDER = resolve_embedder(
+    _embedding_provider,
+    model=_embedding_model,
+    base_url=CONFIG.embedding_base_url,
+    api_key=CONFIG.embedding_api_key,
+    dim=_embedding_dim,
+    timeout=CONFIG.embedding_timeout,
+    extra={"target": CONFIG.embedding_custom_target},
+    probe=_embedding_provider not in {"", "hash", "local", "fallback"},
+)
+if EMBEDDER.fell_back:
+    logging.warning("Embedding provider %s unavailable: %s", EMBEDDER.requested, EMBEDDER.detail)
+KNOWLEDGE_GRAPH = KnowledgeGraphStore(
+    DATA_DIR / "knowledge_graph.sqlite",
+    DATA_DIR / "knowledge_graph_blobs",
+    embedder=EMBEDDER.provider,
+) if ENABLE_GRAPH else None
 LOCAL_KG_WATCHER = LocalKnowledgeWatcher(lambda: KNOWLEDGE_GRAPH) if ENABLE_GRAPH else None
 # ── v2 Realtime bus: constructed first so the store can fan every timeline
 # event into the realtime feed via a single additive sink (no per-call wiring).
@@ -782,6 +816,8 @@ def _require_local_approval(
 
 def require_admin(request: Request) -> tuple[str, Dict]:
     users = load_users()
+    if not REQUIRE_AUTH:
+        return "", users
     token = _extract_bearer_token(request)
     if token:
         email = get_session_email(token)
@@ -1077,6 +1113,7 @@ app.include_router(create_auth_router(
     get_sso_settings=get_sso_settings, get_sso_discovery=_get_sso_discovery,
     public_sso_config=public_sso_config,
     open_registration=OPEN_REGISTRATION, session_ttl=_SESSION_TTL,
+    require_auth=REQUIRE_AUTH,
 ))
 
 def _graph_stats_safe():
@@ -1171,6 +1208,9 @@ def _workspace_graph():
     return KNOWLEDGE_GRAPH if (ENABLE_GRAPH and KNOWLEDGE_GRAPH) else None
 
 
+SEARCH_SERVICE = SearchService(graph_store=_workspace_graph())
+
+
 # ── Workspace OS + Organization router (latticeai.api.workspace, v1.2.0) ──────
 app.include_router(create_workspace_router(
     service=WORKSPACE_SERVICE,
@@ -1217,6 +1257,14 @@ PLATFORM = PlatformRuntime(
     get_tool_permission=get_tool_permission,
 )
 
+# Single AgentRuntime boundary over the orchestrator + run store.
+AGENT_RUNTIME = AgentRuntime(
+    store=WORKSPACE_OS,
+    orchestrator_factory=PLATFORM.build_orchestrator,
+    workspace_graph=_workspace_graph,
+    append_audit_event=append_audit_event,
+)
+
 app.include_router(create_plugins_router(
     registry=PLUGIN_REGISTRY,
     require_user=require_user,
@@ -1252,6 +1300,7 @@ app.include_router(create_agents_router(
     append_audit_event=append_audit_event,
     ui_file_response=ui_file_response,
     static_dir=STATIC_DIR,
+    agent_runtime=AGENT_RUNTIME,
 ))
 
 app.include_router(create_marketplace_router(
@@ -1351,6 +1400,21 @@ app.include_router(create_chat_router(
     base_dir=BASE_DIR,
 ))
 
+def _embedding_info() -> dict:
+    from latticeai.core.embedding_providers import PROVIDER_TYPES, embedding_provider_profiles
+    info = EMBEDDER.as_dict()
+    info["available_providers"] = list(PROVIDER_TYPES)
+    info["profile"] = CONFIG.embedding_profile or ""
+    info["profiles"] = embedding_provider_profiles()
+    return info
+
+
+app.include_router(create_search_router(
+    service=SEARCH_SERVICE,
+    require_user=require_user,
+    embedding_info=_embedding_info,
+))
+
 app.include_router(create_tools_router(
     config=CONFIG,
     data_dir=DATA_DIR,

package/latticeai/services/agent_runtime.py

@@ -0,0 +1,245 @@
+"""AgentRuntime — the single boundary for agent execution and observability.
+
+Before this module the agent concern was spread across three places: the
+:class:`~latticeai.core.multi_agent.MultiAgentOrchestrator` (role pipeline),
+the :class:`~latticeai.services.platform_runtime.PlatformRuntime` (cross-system
+wiring + an ad-hoc ``run_agent``), and ``api/agents.py`` (HTTP transport that
+also owned orchestration + persistence + audit). The frontend reached past all
+of them into the workspace store via ``/workspace/agents``.
+
+``AgentRuntime`` collapses that into one façade with a small, stable surface:
+
+* **configuration** — :meth:`config`, :meth:`roles`
+* **status / health** — :meth:`status`, :meth:`health`
+* **execution**       — :meth:`start`, :meth:`stop`
+* **events / state**  — :meth:`list_runs`, :meth:`get_run`, :meth:`events`, :meth:`replay`
+
+It *wraps* the existing orchestrator and run store rather than reimplementing
+them — execution semantics are unchanged, but every caller (HTTP router and, via
+it, the frontend) now depends on this boundary instead of internal paths.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional
+
+from latticeai.core.multi_agent import (
+    AGENT_ROLES,
+    CORE_PIPELINE,
+    MULTI_AGENT_VERSION,
+    ROLE_AGENT_IDS,
+)
+
+ROLE_DESCRIPTIONS = {
+    "researcher": "Gathers workspace context and memory for the goal.",
+    "planner": "Decomposes the goal into an ordered, bounded plan.",
+    "executor": "Executes each planned step, invoking tools and workflows.",
+    "reviewer": "Reviews the executed work and approves, rejects, or retries.",
+    "release": "Finalizes and summarizes the approved outcome.",
+}
+
+# Run statuses the orchestrator can emit that mean "still working". The default
+# orchestrator runs synchronously, so persisted runs are always terminal; this
+# set lets the runtime report live work if a future async runner lands.
+_ACTIVE_STATUSES = {"running", "in_progress", "queued", "retrying"}
+_TERMINAL_STATUSES = {"ok", "retried_ok", "failed", "rejected", "cancelled"}
+
+
+class AgentRuntime:
+    def __init__(
+        self,
+        *,
+        store: Any,
+        orchestrator_factory: Callable[[Optional[str], Optional[str]], Any],
+        workspace_graph: Callable[[], Any],
+        append_audit_event: Callable[..., None],
+        max_retries_cap: int = 5,
+    ):
+        self._store = store
+        self._orchestrator_factory = orchestrator_factory
+        self._workspace_graph = workspace_graph
+        self._append_audit_event = append_audit_event
+        self._max_retries_cap = int(max_retries_cap)
+
+    # ── configuration ─────────────────────────────────────────────────────
+    def config(self) -> Dict[str, Any]:
+        return {
+            "version": MULTI_AGENT_VERSION,
+            "roles": list(AGENT_ROLES),
+            "default_pipeline": list(CORE_PIPELINE),
+            "max_retries_cap": self._max_retries_cap,
+            "execution_mode": "synchronous",
+        }
+
+    def roles(self) -> List[Dict[str, Any]]:
+        return [
+            {
+                "role": role,
+                "agent_id": ROLE_AGENT_IDS.get(role, f"agent:{role}"),
+                "description": ROLE_DESCRIPTIONS.get(role, ""),
+                "terminal": role not in {"researcher", "planner", "executor", "reviewer"},
+            }
+            for role in AGENT_ROLES
+        ]
+
+    # ── health ────────────────────────────────────────────────────────────
+    def health(self) -> Dict[str, Any]:
+        checks: Dict[str, Any] = {}
+        ok = True
+        try:
+            self._store.list_agents(workspace_id=None)
+            checks["run_store"] = {"status": "ok"}
+        except Exception as exc:  # pragma: no cover - defensive
+            ok = False
+            checks["run_store"] = {"status": "error", "detail": str(exc)}
+        try:
+            self._orchestrator_factory(None, None)
+            checks["orchestrator"] = {"status": "ok"}
+        except Exception as exc:  # pragma: no cover - defensive
+            ok = False
+            checks["orchestrator"] = {"status": "error", "detail": str(exc)}
+        return {"status": "ok" if ok else "degraded", "checks": checks}
+
+    # ── roster + status ───────────────────────────────────────────────────
+    def _roster(self, runs: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Canonical role roster enriched with real run statistics."""
+        by_agent: Dict[str, Dict[str, Any]] = {}
+        for run in runs:
+            aid = str(run.get("agent_id") or "")
+            entry = by_agent.setdefault(aid, {"runs": 0, "last_status": None, "last_at": None})
+            entry["runs"] += 1
+            if entry["last_at"] is None:  # runs are newest-first
+                entry["last_status"] = run.get("status")
+                entry["last_at"] = run.get("created_at") or run.get("completed_at")
+
+        roster: List[Dict[str, Any]] = []
+        order = list(CORE_PIPELINE)  # planner, executor, reviewer first
+        ordered_roles = order + [r for r in AGENT_ROLES if r not in order]
+        for role in ordered_roles:
+            agent_id = ROLE_AGENT_IDS.get(role, f"agent:{role}")
+            stats = by_agent.get(agent_id, {"runs": 0, "last_status": None, "last_at": None})
+            handoffs = []
+            if role == "planner":
+                handoffs = [ROLE_AGENT_IDS["executor"]]
+            elif role == "executor":
+                handoffs = [ROLE_AGENT_IDS["reviewer"]]
+            roster.append({
+                "id": agent_id,
+                "name": role.capitalize(),
+                "role": ROLE_DESCRIPTIONS.get(role, ""),
+                "state": "available" if role != "release" else "idle",
+                "runs": stats["runs"],
+                "last_status": stats["last_status"],
+                "last_at": stats["last_at"],
+                "handoffs": handoffs,
+            })
+        return roster
+
+    def status(self, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        try:
+            listing = self._store.list_agents(workspace_id=scope)
+        except Exception as exc:  # pragma: no cover - defensive
+            listing = {"agents": [], "runs": [], "error": str(exc)}
+        runs = list(listing.get("runs") or [])
+        active = sum(1 for r in runs if str(r.get("status")) in _ACTIVE_STATUSES)
+        return {
+            "runtime": {
+                "ready": True,
+                "version": MULTI_AGENT_VERSION,
+                "execution_mode": "synchronous",
+                "default_pipeline": list(CORE_PIPELINE),
+                "total_runs": len(runs),
+                "active_runs": active,
+            },
+            "health": self.health(),
+            "roles": self.roles(),
+            "agents": self._roster(runs),
+            "runs": runs[:25],
+        }
+
+    # ── events / state ────────────────────────────────────────────────────
+    def list_runs(self, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        return self._store.list_agents(workspace_id=scope)
+
+    def get_run(self, run_id: str, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        return {"run": self._store.get_agent_run(run_id, workspace_id=scope)}
+
+    def replay(self, run_id: str, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        return {"replay": self._store.replay_agent_run(run_id, workspace_id=scope)}
+
+    def events(self, run_id: str, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        run = self._store.get_agent_run(run_id, workspace_id=scope)
+        status = str(run.get("status") or "")
+        return {
+            "run_id": run_id,
+            "status": status,
+            "is_final": status in _TERMINAL_STATUSES or status not in _ACTIVE_STATUSES,
+            "current_role": run.get("current_role"),
+            "timeline": run.get("timeline") or [],
+            "handoffs": run.get("handoffs") or [],
+        }
+
+    # ── execution ─────────────────────────────────────────────────────────
+    def start(
+        self,
+        goal: str,
+        *,
+        user_email: Optional[str],
+        scope: Optional[str],
+        roles: Optional[List[str]] = None,
+        inputs: Optional[Dict[str, Any]] = None,
+        max_retries: int = 2,
+    ) -> Dict[str, Any]:
+        if not str(goal or "").strip():
+            raise ValueError("goal is required")
+        orchestrator = self._orchestrator_factory(user_email or None, scope)
+        result = orchestrator.run(
+            goal,
+            user_email=user_email or None,
+            workspace_id=scope,
+            inputs=inputs or {},
+            roles=roles or None,
+            max_retries=max(0, min(int(max_retries or 0), self._max_retries_cap)),
+        )
+        run = self._store.record_agent_run(
+            agent_id=result.agent_id,
+            status=result.status,
+            input_text=goal,
+            output_text=result.output,
+            timeline=result.timeline,
+            relationships=[ROLE_AGENT_IDS.get(r, f"agent:{r}") for r in result.roles_run],
+            handoffs=result.handoffs,
+            context_packets=result.context_packets,
+            plan=result.plan,
+            plan_review=result.plan_review,
+            review_history=result.review_history,
+            retry_history=result.retry_history,
+            memory_snapshots=result.memory_snapshots,
+            user_email=user_email or None,
+            graph=self._workspace_graph(),
+            workspace_id=scope,
+        )
+        self._append_audit_event(
+            "multi_agent_run",
+            user_email=user_email,
+            agent_id=result.agent_id,
+            status=result.status,
+            retries=result.retries,
+        )
+        return {"run": run, "result": result.as_dict()}
+
+    def stop(self, run_id: str, *, scope: Optional[str] = None) -> Dict[str, Any]:
+        """Best-effort stop.
+
+        The default runtime executes synchronously, so by the time a run id
+        exists the run has already completed. Report that honestly rather than
+        pretending a cancellation occurred.
+        """
+        try:
+            run = self._store.get_agent_run(run_id, workspace_id=scope)
+        except FileNotFoundError:
+            return {"stopped": False, "reason": "run not found", "run_id": run_id}
+        status = str(run.get("status") or "")
+        if status in _ACTIVE_STATUSES:
+            return {"stopped": False, "reason": "asynchronous cancellation is not supported by the synchronous runtime", "run_id": run_id, "status": status}
+        return {"stopped": False, "reason": "run already finished", "run_id": run_id, "status": status}