ltcai 3.0.1 → 3.2.0

package/latticeai/api/marketplace.py

@@ -16,6 +16,10 @@ class TemplateInstallRequest(BaseModel):
     data: Dict[str, Any] = {}
 
 
+class TemplateCloneRequest(BaseModel):
+    name: Optional[str] = None
+
+
 def create_marketplace_router(
     *,
     store,
@@ -72,6 +76,15 @@ def create_marketplace_router(
             raise HTTPException(status_code=400, detail=str(exc)) from exc
         return {"installed": installed}
 
+    @router.post("/marketplace/templates/{kind}/{template_id}/clone")
+    async def clone_template(kind: str, template_id: str, req: TemplateCloneRequest, request: Request):
+        require_user(request)
+        gate_read(request)
+        try:
+            return {"template": catalog.clone_template(kind, template_id, req.name)}
+        except MarketplaceError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+
     @router.get("/marketplace/templates/registry")
     async def template_registry(request: Request):
         require_user(request)

package/latticeai/api/memory.py

@@ -0,0 +1,109 @@
+"""Memory platform + Memory Manager API router (v3.2.0).
+
+Exposes :class:`~latticeai.services.memory_service.MemoryService` so the /app
+Memory view can inspect every memory tier, recall across them, and run manager
+operations (prune / compact / rebuild / clear). Full paths in decorators.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, List, Optional
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel
+
+from latticeai.services.memory_service import MemoryService
+
+
+class RecallRequest(BaseModel):
+    query: str = ""
+    limit: int = 20
+
+
+class PruneRequest(BaseModel):
+    ids: List[str] = []
+    kind: Optional[str] = None
+
+
+class RebuildRequest(BaseModel):
+    target: str = "vector"
+
+
+class ClearRequest(BaseModel):
+    scope: str
+    confirm: bool = False
+
+
+def create_memory_router(
+    *,
+    service: MemoryService,
+    require_user: Callable[[Request], str],
+    get_current_user: Callable[[Request], Optional[str]],
+    gate_read: Callable[[Request], Optional[str]],
+    gate_write: Callable[[Request], Optional[str]],
+    append_audit_event: Callable[..., None],
+) -> APIRouter:
+    router = APIRouter()
+
+    @router.get("/api/memory/manager")
+    async def memory_manager(request: Request):
+        user = require_user(request)
+        scope = gate_read(request)
+        return service.manager(user_email=user, workspace_id=scope)
+
+    @router.get("/api/memory/tiers")
+    async def memory_tiers(request: Request):
+        require_user(request)
+        return service.tiers()
+
+    @router.get("/api/memory/inspect")
+    async def memory_inspect(request: Request, source: str, limit: int = 50):
+        user = require_user(request)
+        scope = gate_read(request)
+        try:
+            return service.inspect(source, user_email=user, workspace_id=scope, limit=limit)
+        except KeyError as exc:
+            raise HTTPException(status_code=404, detail=f"Unknown memory source: {source}") from exc
+
+    @router.post("/api/memory/recall")
+    async def memory_recall(req: RecallRequest, request: Request):
+        user = require_user(request)
+        scope = gate_read(request)
+        return service.recall(req.query, user_email=user, workspace_id=scope, limit=req.limit)
+
+    @router.post("/api/memory/prune")
+    async def memory_prune(req: PruneRequest, request: Request):
+        user = require_user(request)
+        gate_write(request)
+        result = service.prune(ids=req.ids, kind=req.kind, user_email=user)
+        append_audit_event("memory_prune", user_email=user, count=result.get("count", 0))
+        return result
+
+    @router.post("/api/memory/compact")
+    async def memory_compact(request: Request):
+        user = require_user(request)
+        gate_write(request)
+        result = service.compact(user_email=user)
+        append_audit_event("memory_compact", user_email=user, compacted=result.get("compacted", 0))
+        return result
+
+    @router.post("/api/memory/rebuild")
+    async def memory_rebuild(req: RebuildRequest, request: Request):
+        user = require_user(request)
+        gate_write(request)
+        result = service.rebuild(req.target)
+        append_audit_event("memory_rebuild", user_email=user, target=req.target, status=result.get("status"))
+        return result
+
+    @router.post("/api/memory/clear")
+    async def memory_clear(req: ClearRequest, request: Request):
+        user = require_user(request)
+        gate_write(request)
+        try:
+            result = service.clear(scope=req.scope, confirm=req.confirm, user_email=user)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        append_audit_event("memory_clear", user_email=user, scope=req.scope)
+        return result
+
+    return router

package/latticeai/api/search.py

@@ -7,6 +7,7 @@ from typing import Any, Callable, Dict, Optional
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, Field
 
+from latticeai.core.embedding_providers import embedding_provider_profiles
 from latticeai.services.search_service import DEFAULT_HYBRID_WEIGHTS, SearchService
 
 
@@ -213,9 +214,12 @@ def create_search_router(
     async def embeddings_providers(request: Request) -> Dict[str, Any]:
         require_user(request)
         resolved = embedding_info() if embedding_info else {}
+        profiles = resolved.get("profiles") or embedding_provider_profiles()
         return {
             "active": resolved.get("active_provider"),
             "requested": resolved.get("requested_provider"),
+            "profile": resolved.get("profile") or "",
+            "profiles": profiles,
             "providers": [
                 {"id": "hash", "label": "Local hash (fallback)", "grade": "fallback",
                  "requires": [], "detail": "Deterministic offline vectors — always available."},

package/latticeai/core/agent_registry.py

@@ -0,0 +1,234 @@
+"""Agent Registry — registration, discovery, metadata, versioning, capabilities.
+
+Part 2 of the v3.2.0 platform. Before this module the agent roster was derived
+ad-hoc from the hardcoded ``AGENT_ROLES`` tuple wherever it was needed. The
+registry makes every agent — built-in role and user-registered custom agent — a
+first-class entry with stable metadata: ``id``, ``type``, ``version``,
+``capabilities``, ``description`` and a mutable ``config``. The HTTP surface and
+the /app views read this registry instead of any hardcoded list.
+
+Built-in role agents are projected from the single source of truth in
+:mod:`latticeai.core.multi_agent` (so adding a role there flows through here).
+Custom agents and config overrides are persisted to
+``data_dir/agent_registry.json``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from latticeai.core.multi_agent import (
+    AGENT_ROLES,
+    CORE_PIPELINE,
+    MULTI_AGENT_VERSION,
+    ROLE_AGENT_IDS,
+)
+
+AGENT_TYPES = ("planner", "researcher", "executor", "reviewer", "release", "custom")
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+# Capabilities + descriptions for the built-in role agents. Kept here as the
+# registry's metadata projection of the roles defined in multi_agent.py.
+ROLE_META: Dict[str, Dict[str, Any]] = {
+    "researcher": {
+        "description": "Gathers workspace context, memory, and graph signal for the goal.",
+        "capabilities": ["context-retrieval", "memory-recall", "graph-read", "hybrid-search"],
+    },
+    "planner": {
+        "description": "Decomposes the goal into an ordered, bounded, reviewable plan.",
+        "capabilities": ["task-decomposition", "plan-review", "delegation"],
+    },
+    "executor": {
+        "description": "Executes each planned step, invoking tools, workflows, and plugins.",
+        "capabilities": ["tool-use", "workflow-run", "plugin-run", "file-write"],
+    },
+    "reviewer": {
+        "description": "Reviews executed work and approves, rejects, or requests a retry.",
+        "capabilities": ["verification", "retry-control", "approval"],
+    },
+    "release": {
+        "description": "Finalizes and summarizes the approved outcome.",
+        "capabilities": ["summarize", "finalize"],
+    },
+}
+
+
+class AgentRegistry:
+    """Persisted registry of built-in role agents + user-registered agents."""
+
+    def __init__(self, path: Path):
+        self.path = Path(path)
+        self._state: Dict[str, Any] = self._load()
+
+    # ── persistence ───────────────────────────────────────────────────────
+    def _load(self) -> Dict[str, Any]:
+        if self.path.exists():
+            try:
+                with open(self.path, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+                if isinstance(data, dict):
+                    data.setdefault("custom", [])
+                    data.setdefault("config_overrides", {})
+                    return data
+            except Exception:
+                pass
+        return {"custom": [], "config_overrides": {}}
+
+    def _save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=str(self.path.parent), suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump(self._state, fh, ensure_ascii=False, indent=2)
+            os.replace(tmp, self.path)
+        finally:
+            if os.path.exists(tmp):
+                os.unlink(tmp)
+
+    # ── projection ────────────────────────────────────────────────────────
+    def _builtin_agents(self) -> List[Dict[str, Any]]:
+        overrides = self._state.get("config_overrides", {})
+        agents: List[Dict[str, Any]] = []
+        for role in AGENT_ROLES:
+            meta = ROLE_META.get(role, {"description": "", "capabilities": []})
+            agent_id = ROLE_AGENT_IDS.get(role, f"agent:{role}")
+            handoffs: List[str] = []
+            if role == "planner":
+                handoffs = [ROLE_AGENT_IDS["executor"]]
+            elif role == "executor":
+                handoffs = [ROLE_AGENT_IDS["reviewer"]]
+            ov = overrides.get(agent_id, {})
+            agents.append({
+                "id": agent_id,
+                "name": role.capitalize(),
+                "type": role,
+                "version": MULTI_AGENT_VERSION,
+                "description": meta["description"],
+                "capabilities": list(meta["capabilities"]),
+                "handoffs": handoffs,
+                "in_default_pipeline": role in CORE_PIPELINE,
+                "source": "builtin",
+                "removable": False,
+                "enabled": bool(ov.get("enabled", True)),
+                "config": ov.get("config", {}),
+            })
+        return agents
+
+    def _custom_agents(self) -> List[Dict[str, Any]]:
+        out: List[Dict[str, Any]] = []
+        for entry in self._state.get("custom", []):
+            agent = dict(entry)
+            agent["source"] = "user"
+            agent["removable"] = True
+            agent.setdefault("enabled", True)
+            agent.setdefault("handoffs", [])
+            out.append(agent)
+        return out
+
+    def all(self) -> List[Dict[str, Any]]:
+        return self._builtin_agents() + self._custom_agents()
+
+    # ── reads ─────────────────────────────────────────────────────────────
+    def list(self, agent_type: Optional[str] = None) -> Dict[str, Any]:
+        agents = self.all()
+        if agent_type:
+            agents = [a for a in agents if a["type"] == agent_type]
+        counts: Dict[str, int] = {}
+        for a in self.all():
+            counts[a["type"]] = counts.get(a["type"], 0) + 1
+        return {
+            "agents": agents,
+            "types": list(AGENT_TYPES),
+            "counts": counts,
+            "total": len(agents),
+            "version": MULTI_AGENT_VERSION,
+            "default_pipeline": list(CORE_PIPELINE),
+            "generated_at": _now(),
+        }
+
+    def get(self, agent_id: str) -> Optional[Dict[str, Any]]:
+        return next((a for a in self.all() if a["id"] == agent_id), None)
+
+    def capabilities(self) -> Dict[str, List[str]]:
+        """Inverted index: capability -> [agent_id, …]."""
+        index: Dict[str, List[str]] = {}
+        for a in self.all():
+            for cap in a.get("capabilities", []):
+                index.setdefault(cap, []).append(a["id"])
+        return index
+
+    def discover(self, capability: str) -> List[Dict[str, Any]]:
+        cap = str(capability or "").lower().strip()
+        return [a for a in self.all() if any(cap == c.lower() for c in a.get("capabilities", []))]
+
+    # ── mutations ─────────────────────────────────────────────────────────
+    def register(
+        self,
+        *,
+        name: str,
+        agent_type: str = "custom",
+        description: str = "",
+        capabilities: Optional[List[str]] = None,
+        config: Optional[Dict[str, Any]] = None,
+        version: str = "1.0.0",
+    ) -> Dict[str, Any]:
+        if not str(name).strip():
+            raise ValueError("name is required")
+        if agent_type not in AGENT_TYPES:
+            raise ValueError(f"type must be one of {', '.join(AGENT_TYPES)}")
+        slug = str(name).strip().lower().replace(" ", "-")
+        agent_id = f"agent:custom:{slug}"
+        existing = {c["id"] for c in self._state.get("custom", [])}
+        if agent_id in existing:
+            agent_id = f"agent:custom:{slug}-{len(existing) + 1}"
+        entry = {
+            "id": agent_id,
+            "name": str(name).strip(),
+            "type": agent_type,
+            "version": str(version or "1.0.0"),
+            "description": str(description or "").strip(),
+            "capabilities": list(capabilities or []),
+            "config": dict(config or {}),
+            "enabled": True,
+            "created_at": _now(),
+        }
+        self._state.setdefault("custom", []).append(entry)
+        self._save()
+        return entry
+
+    def update_config(self, agent_id: str, config: Dict[str, Any], *, enabled: Optional[bool] = None) -> Dict[str, Any]:
+        if self.get(agent_id) is None:
+            raise KeyError(agent_id)
+        if agent_id.startswith("agent:custom:"):
+            for entry in self._state.get("custom", []):
+                if entry["id"] == agent_id:
+                    entry["config"] = dict(config or {})
+                    if enabled is not None:
+                        entry["enabled"] = bool(enabled)
+                    entry["updated_at"] = _now()
+        else:
+            ov = self._state.setdefault("config_overrides", {}).setdefault(agent_id, {})
+            ov["config"] = dict(config or {})
+            if enabled is not None:
+                ov["enabled"] = bool(enabled)
+        self._save()
+        return self.get(agent_id)  # type: ignore[return-value]
+
+    def remove(self, agent_id: str) -> Dict[str, Any]:
+        if not agent_id.startswith("agent:custom:"):
+            raise ValueError("Built-in role agents cannot be removed; disable them via config instead.")
+        before = len(self._state.get("custom", []))
+        self._state["custom"] = [c for c in self._state.get("custom", []) if c["id"] != agent_id]
+        if len(self._state["custom"]) == before:
+            raise KeyError(agent_id)
+        self._save()
+        return {"removed": agent_id}

package/latticeai/core/config.py

@@ -95,6 +95,7 @@ class Config:
 
     # ── embeddings (retrieval vector signal) ────────────────────────
     embedding_provider: str
+    embedding_profile: str
     embedding_model: str
     embedding_base_url: str
     embedding_api_key: str
@@ -174,6 +175,7 @@ class Config:
             local_draft_model=_value(env, "LATTICEAI_LOCAL_DRAFT_MODEL", ""),
             auto_read_chat_paths=_bool(env, "LATTICEAI_AUTO_READ_CHAT_PATHS", default=False),
             embedding_provider=_value(env, "LATTICEAI_EMBEDDING_PROVIDER", "hash").strip().lower(),
+            embedding_profile=_value(env, "LATTICEAI_EMBEDDING_PROFILE", "").strip().lower(),
             embedding_model=_value(env, "LATTICEAI_EMBEDDING_MODEL", ""),
             embedding_base_url=_value(env, "LATTICEAI_EMBEDDING_BASE_URL", ""),
             embedding_api_key=_value(env, "LATTICEAI_EMBEDDING_API_KEY", ""),

package/latticeai/core/embedding_providers.py

@@ -51,6 +51,7 @@ class EmbeddingUnavailable(RuntimeError):
 # identity is stable before the first (possibly remote) call. A configured
 # ``dim`` always wins; an unknown model falls back to a one-time live probe.
 _KNOWN_DIMS = {
+    "bge-m3": 1024,
     "nomic-embed-text": 768,
     "mxbai-embed-large": 1024,
     "all-minilm": 384,
@@ -61,12 +62,131 @@ _KNOWN_DIMS = {
     "gte-small": 384,
     "gte-base": 768,
     "gte-large": 1024,
+    "e5-large": 1024,
+    "multilingual-e5-large": 1024,
     "text-embedding-3-small": 1536,
     "text-embedding-3-large": 3072,
     "text-embedding-ada-002": 1536,
 }
 
 
+PRODUCTION_PROVIDER_PROFILES: Dict[str, Dict[str, Any]] = {
+    "local:bge-m3": {
+        "id": "local:bge-m3",
+        "provider": "mlx",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "BGE-M3 local",
+        "detail": "Multilingual semantic embeddings for local retrieval.",
+    },
+    "local:nomic-embed-text": {
+        "id": "local:nomic-embed-text",
+        "provider": "ollama",
+        "model": "nomic-embed-text",
+        "dimensions": 768,
+        "grade": "production",
+        "family": "local",
+        "label": "Nomic Embed Text local",
+        "detail": "General-purpose local semantic embeddings.",
+    },
+    "local:e5-large": {
+        "id": "local:e5-large",
+        "provider": "mlx",
+        "model": "e5-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "E5 Large local",
+        "detail": "High-recall local retrieval profile.",
+    },
+    "local:gte-large": {
+        "id": "local:gte-large",
+        "provider": "mlx",
+        "model": "gte-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "GTE Large local",
+        "detail": "Large local semantic embedding profile.",
+    },
+    "ollama:nomic-embed-text": {
+        "id": "ollama:nomic-embed-text",
+        "provider": "ollama",
+        "model": "nomic-embed-text",
+        "dimensions": 768,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama Nomic Embed Text",
+        "detail": "Production semantic embeddings through Ollama.",
+    },
+    "ollama:mxbai-embed-large": {
+        "id": "ollama:mxbai-embed-large",
+        "provider": "ollama",
+        "model": "mxbai-embed-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama MXBAI Embed Large",
+        "detail": "High-quality local semantic embeddings through Ollama.",
+    },
+    "ollama:bge-m3": {
+        "id": "ollama:bge-m3",
+        "provider": "ollama",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama BGE-M3-compatible",
+        "detail": "BGE-M3-compatible providers exposed through Ollama.",
+    },
+    "mlx:bge-m3": {
+        "id": "mlx:bge-m3",
+        "provider": "mlx",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "mlx",
+        "label": "MLX BGE-M3",
+        "detail": "Apple Silicon optimized local embeddings.",
+    },
+    "openai:text-embedding-3-small": {
+        "id": "openai:text-embedding-3-small",
+        "provider": "openai",
+        "model": "text-embedding-3-small",
+        "dimensions": 1536,
+        "grade": "production",
+        "family": "openai-compatible",
+        "label": "OpenAI-compatible small",
+        "detail": "OpenAI-compatible /v1/embeddings endpoint.",
+    },
+    "openai:text-embedding-3-large": {
+        "id": "openai:text-embedding-3-large",
+        "provider": "openai",
+        "model": "text-embedding-3-large",
+        "dimensions": 3072,
+        "grade": "production",
+        "family": "openai-compatible",
+        "label": "OpenAI-compatible large",
+        "detail": "Highest-dimensional OpenAI-compatible embedding profile.",
+    },
+}
+
+
+def embedding_provider_profiles() -> List[Dict[str, Any]]:
+    return [dict(PRODUCTION_PROVIDER_PROFILES[key]) for key in sorted(PRODUCTION_PROVIDER_PROFILES)]
+
+
+def resolve_embedding_profile(profile: str) -> Dict[str, Any]:
+    if not profile:
+        return {}
+    key = str(profile).strip().lower()
+    if key in PRODUCTION_PROVIDER_PROFILES:
+        return dict(PRODUCTION_PROVIDER_PROFILES[key])
+    raise ValueError(f"unknown embedding profile: {profile!r}")
+
+
 def _guess_dim(model: str, default: int) -> int:
     key = str(model or "").split("/")[-1].strip().lower()
     key = key.split(":")[0]
@@ -498,5 +618,8 @@ __all__ = [
     "ResolvedEmbedder",
     "build_embedding_provider",
     "resolve_embedder",
+    "resolve_embedding_profile",
+    "embedding_provider_profiles",
+    "PRODUCTION_PROVIDER_PROFILES",
     "PROVIDER_TYPES",
 ]