ltcai 3.0.1 → 3.2.0

package/latticeai/core/hooks.py

@@ -0,0 +1,284 @@
+"""Hooks platform — a persisted registry of lifecycle extension points.
+
+Lattice AI runs several behaviours at well-defined points in the agent / tool /
+workflow lifecycle (audit logging, secret redaction, sensitive-data
+classification, tool-permission gating, memory snapshots, workflow replay
+logging). v3.2.0 makes those points *first-class and inspectable*: every hook is
+listed, ordered, and individually enable/disable-able, and users can register
+their own custom hooks alongside the built-ins.
+
+The registry owns metadata, ordering and the enabled flag (persisted to
+``data_dir/hooks.json``). Built-in hooks carry ``source="builtin"`` and map onto
+behaviour the platform already performs; ``managed`` records whether the
+behaviour is enforced by the platform (``platform``) or is an advisory hook the
+user has registered (``user``). The registry never silently drops a hook: the
+full set is always returned so the UI can show exactly what runs and in which
+order.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+HOOK_KINDS = (
+    "pre_run",
+    "post_run",
+    "pre_tool",
+    "post_tool",
+    "agent",
+    "pipeline",
+    "workflow",
+)
+
+
+def _now() -> str:
+    return datetime.now().isoformat(timespec="seconds")
+
+
+# Built-in hooks describe lifecycle points the platform already exercises. They
+# are honest reflections of existing behaviour (see the `binding` field), made
+# visible and orderable here. Disabling a `managed="platform"` hook is recorded
+# and surfaced, but core safety behaviours remain enforced by their owning
+# subsystem — the UI states this explicitly so nothing is misrepresented.
+BUILTIN_HOOKS: List[Dict[str, Any]] = [
+    {
+        "id": "builtin:redact-secrets",
+        "name": "Redact secrets",
+        "kind": "pre_run",
+        "order": 10,
+        "description": "Strip secret-like fields (token, password, api_key…) from agent context packets before a run.",
+        "binding": "latticeai.core.multi_agent._redact",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:research-memory-snapshot",
+        "name": "Research memory snapshot",
+        "kind": "agent",
+        "order": 20,
+        "description": "Capture a short-term memory snapshot after the researcher stage gathers context.",
+        "binding": "latticeai.core.multi_agent.default_role_runner",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:tool-permission-gate",
+        "name": "Tool permission gate",
+        "kind": "pre_tool",
+        "order": 10,
+        "description": "Require explicit approval for tools whose governance policy is not auto-approve.",
+        "binding": "latticeai.core.tool_registry.ToolRegistry.permission",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:sensitive-data-guard",
+        "name": "Sensitive-data guard",
+        "kind": "pre_tool",
+        "order": 20,
+        "description": "Classify outgoing content for sensitive data before tool execution.",
+        "binding": "server_app.classify_sensitive_message",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:audit-agent-run",
+        "name": "Audit agent run",
+        "kind": "post_run",
+        "order": 10,
+        "description": "Append every completed agent run to the workspace audit log.",
+        "binding": "latticeai.services.agent_runtime.AgentRuntime.start",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:workflow-replay-log",
+        "name": "Workflow replay log",
+        "kind": "workflow",
+        "order": 10,
+        "description": "Record each workflow run's timeline so it can be replayed step by step.",
+        "binding": "latticeai.api.workflow_designer",
+        "managed": "platform",
+    },
+    {
+        "id": "builtin:pipeline-index-status",
+        "name": "Pipeline index status",
+        "kind": "pipeline",
+        "order": 10,
+        "description": "Publish ingest / embed / graph-build pipeline state to the retrieval index status.",
+        "binding": "latticeai.api.search",
+        "managed": "platform",
+    },
+]
+
+
+class HooksRegistry:
+    """Persisted registry of lifecycle hooks (built-in + user-registered)."""
+
+    def __init__(self, path: Path):
+        self.path = Path(path)
+        self._state: Dict[str, Any] = self._load()
+
+    # ── persistence ───────────────────────────────────────────────────────
+    def _load(self) -> Dict[str, Any]:
+        if self.path.exists():
+            try:
+                with open(self.path, "r", encoding="utf-8") as fh:
+                    data = json.load(fh)
+                if isinstance(data, dict):
+                    data.setdefault("custom", [])
+                    data.setdefault("overrides", {})
+                    return data
+            except Exception:
+                pass
+        return {"custom": [], "overrides": {}}
+
+    def _save(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(dir=str(self.path.parent), suffix=".tmp")
+        try:
+            with os.fdopen(fd, "w", encoding="utf-8") as fh:
+                json.dump(self._state, fh, ensure_ascii=False, indent=2)
+            os.replace(tmp, self.path)
+        finally:
+            if os.path.exists(tmp):
+                os.unlink(tmp)
+
+    # ── views ─────────────────────────────────────────────────────────────
+    def _materialize(self) -> List[Dict[str, Any]]:
+        overrides: Dict[str, Any] = self._state.get("overrides", {})
+        hooks: List[Dict[str, Any]] = []
+        for base in BUILTIN_HOOKS:
+            ov = overrides.get(base["id"], {})
+            hook = dict(base)
+            hook["source"] = "builtin"
+            hook["enabled"] = bool(ov.get("enabled", True))
+            if "order" in ov:
+                hook["order"] = ov["order"]
+            hook["removable"] = False
+            hooks.append(hook)
+        for custom in self._state.get("custom", []):
+            hook = dict(custom)
+            hook["source"] = "user"
+            hook.setdefault("managed", "user")
+            hook.setdefault("binding", "advisory")
+            hook["enabled"] = bool(custom.get("enabled", True))
+            hook["removable"] = True
+            hooks.append(hook)
+        hooks.sort(key=lambda h: (HOOK_KINDS.index(h["kind"]) if h["kind"] in HOOK_KINDS else 99, h.get("order", 100), h["id"]))
+        return hooks
+
+    def list(self, kind: Optional[str] = None) -> Dict[str, Any]:
+        hooks = self._materialize()
+        if kind:
+            hooks = [h for h in hooks if h["kind"] == kind]
+        counts: Dict[str, Dict[str, int]] = {}
+        for h in self._materialize():
+            bucket = counts.setdefault(h["kind"], {"total": 0, "enabled": 0})
+            bucket["total"] += 1
+            if h["enabled"]:
+                bucket["enabled"] += 1
+        return {
+            "hooks": hooks,
+            "kinds": list(HOOK_KINDS),
+            "counts": counts,
+            "total": len(hooks),
+            "enabled": sum(1 for h in hooks if h["enabled"]),
+            "generated_at": _now(),
+        }
+
+    def get(self, hook_id: str) -> Optional[Dict[str, Any]]:
+        return next((h for h in self._materialize() if h["id"] == hook_id), None)
+
+    def inspect(self, hook_id: str) -> Dict[str, Any]:
+        hook = self.get(hook_id)
+        if hook is None:
+            raise KeyError(hook_id)
+        detail = dict(hook)
+        detail["advisory"] = hook.get("managed") != "platform"
+        detail["note"] = (
+            "Enforced by its owning subsystem; the registry controls visibility and ordering."
+            if hook.get("managed") == "platform"
+            else "User-registered hook: listed, ordered and inspectable; runs advisory in this build."
+        )
+        return detail
+
+    # ── mutations ─────────────────────────────────────────────────────────
+    def set_enabled(self, hook_id: str, enabled: bool) -> Dict[str, Any]:
+        if self.get(hook_id) is None:
+            raise KeyError(hook_id)
+        if hook_id.startswith("builtin:"):
+            self._state.setdefault("overrides", {}).setdefault(hook_id, {})["enabled"] = bool(enabled)
+        else:
+            for custom in self._state.get("custom", []):
+                if custom["id"] == hook_id:
+                    custom["enabled"] = bool(enabled)
+        self._save()
+        return self.get(hook_id)  # type: ignore[return-value]
+
+    def set_order(self, hook_id: str, order: int) -> Dict[str, Any]:
+        if self.get(hook_id) is None:
+            raise KeyError(hook_id)
+        order = int(order)
+        if hook_id.startswith("builtin:"):
+            self._state.setdefault("overrides", {}).setdefault(hook_id, {})["order"] = order
+        else:
+            for custom in self._state.get("custom", []):
+                if custom["id"] == hook_id:
+                    custom["order"] = order
+        self._save()
+        return self.get(hook_id)  # type: ignore[return-value]
+
+    def reorder(self, kind: str, ordered_ids: List[str]) -> Dict[str, Any]:
+        for idx, hook_id in enumerate(ordered_ids):
+            try:
+                self.set_order(hook_id, (idx + 1) * 10)
+            except KeyError:
+                continue
+        return self.list(kind=kind)
+
+    def register(
+        self,
+        *,
+        name: str,
+        kind: str,
+        description: str = "",
+        command: str = "",
+        order: Optional[int] = None,
+        enabled: bool = True,
+    ) -> Dict[str, Any]:
+        if not str(name).strip():
+            raise ValueError("name is required")
+        if kind not in HOOK_KINDS:
+            raise ValueError(f"kind must be one of {', '.join(HOOK_KINDS)}")
+        slug = str(name).strip().lower().replace(" ", "-")
+        hook_id = f"user:{slug}"
+        existing = {c["id"] for c in self._state.get("custom", [])}
+        if hook_id in existing:
+            hook_id = f"user:{slug}-{len(existing) + 1}"
+        entry = {
+            "id": hook_id,
+            "name": str(name).strip(),
+            "kind": kind,
+            "description": str(description or "").strip(),
+            "command": str(command or "").strip(),
+            "order": int(order) if order is not None else 100,
+            "enabled": bool(enabled),
+            "managed": "user",
+            "binding": "advisory",
+            "created_at": _now(),
+        }
+        self._state.setdefault("custom", []).append(entry)
+        self._save()
+        return entry
+
+    def remove(self, hook_id: str) -> Dict[str, Any]:
+        if hook_id.startswith("builtin:"):
+            raise ValueError("Built-in hooks cannot be removed; disable them instead.")
+        before = len(self._state.get("custom", []))
+        self._state["custom"] = [c for c in self._state.get("custom", []) if c["id"] != hook_id]
+        if len(self._state["custom"]) == before:
+            raise KeyError(hook_id)
+        self._save()
+        return {"removed": hook_id}

package/latticeai/core/marketplace.py

@@ -11,10 +11,39 @@ from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
 
-MARKETPLACE_VERSION = "2.2.0"
+MARKETPLACE_VERSION = "3.2.0"
 TEMPLATE_KINDS = ("plugin", "workflow", "agent")
 
 
+def _agent_template(
+    template_id: str,
+    name: str,
+    description: str,
+    *,
+    roles: List[str],
+    capabilities: List[str],
+    suggested_tools: List[str],
+    category: str,
+    max_retries: int = 2,
+) -> Dict[str, Any]:
+    """Build a portable agent template entry (Part 4 reusable templates)."""
+    return {
+        "id": template_id,
+        "kind": "agent",
+        "name": name,
+        "version": "1.0.0",
+        "description": description,
+        "metadata": {"category": category, "installable": True, "agent_template": True},
+        "definition": {
+            "roles": roles,
+            "max_retries": max_retries,
+            "capabilities": capabilities,
+            "suggested_tools": suggested_tools,
+            "constraints": ["workspace scoped", "no secret leakage", "replayable timeline"],
+        },
+    }
+
+
 class MarketplaceError(Exception):
     """Raised for invalid template operations."""
 
@@ -73,7 +102,47 @@ BUILTIN_TEMPLATES: Dict[str, List[Dict[str, Any]]] = {
                 "max_retries": 2,
                 "constraints": ["workspace scoped", "no secret leakage", "replayable timeline"],
             },
-        }
+        },
+        _agent_template(
+            "agent-research-assistant", "Research Assistant",
+            "Retrieves workspace context, plans an inquiry, and synthesizes a reviewed answer.",
+            roles=["researcher", "planner", "reviewer"],
+            capabilities=["context-retrieval", "hybrid-search", "memory-recall", "synthesis"],
+            suggested_tools=["knowledge_search", "knowledge_graph_search", "read_file"],
+            category="research",
+        ),
+        _agent_template(
+            "agent-coding-assistant", "Coding Assistant",
+            "Plans a change, edits files, runs the build, and reviews the result before finishing.",
+            roles=["planner", "executor", "reviewer"],
+            capabilities=["task-decomposition", "tool-use", "file-write", "verification"],
+            suggested_tools=["edit_file", "write_file", "run_command", "build_project", "git_diff"],
+            category="coding",
+        ),
+        _agent_template(
+            "agent-knowledge-curator", "Knowledge Curator",
+            "Captures, structures, and saves knowledge into the graph and memory.",
+            roles=["researcher", "executor"],
+            capabilities=["context-retrieval", "graph-read", "knowledge-save"],
+            suggested_tools=["knowledge_save", "knowledge_graph_ingest", "knowledge_tree"],
+            category="knowledge",
+        ),
+        _agent_template(
+            "agent-documentation-writer", "Documentation Writer",
+            "Plans a document, drafts and writes it, and reviews for completeness.",
+            roles=["planner", "executor", "reviewer"],
+            capabilities=["task-decomposition", "file-write", "summarize", "verification"],
+            suggested_tools=["create_docx", "create_pdf", "write_file", "read_document"],
+            category="documentation",
+        ),
+        _agent_template(
+            "agent-workflow-builder", "Workflow Builder",
+            "Plans and assembles a multi-step workflow definition for repeatable automation.",
+            roles=["planner", "executor"],
+            capabilities=["task-decomposition", "workflow-run", "delegation"],
+            suggested_tools=["todo_write", "workspace_tree"],
+            category="automation",
+        ),
     ],
 }
 
@@ -110,6 +179,22 @@ class TemplateCatalog:
                 return deepcopy(template)
         raise MarketplaceError(f"template not found: {item_kind}/{template_id}")
 
+    def clone_template(self, kind: str, template_id: str, new_name: Optional[str] = None) -> Dict[str, Any]:
+        """Return an editable copy of a template with a fresh id (Part 4 clone)."""
+        template = self.get_template(kind, template_id)
+        name = (new_name or f"{template['name']} (Copy)").strip()
+        slug = name.lower().replace(" ", "-").replace("(", "").replace(")", "")
+        clone = deepcopy(template)
+        clone["id"] = f"{template['id']}-copy-{slug}"[:80]
+        clone["name"] = name
+        clone["version"] = "1.0.0"
+        clone["metadata"] = {
+            **(template.get("metadata") or {}),
+            "cloned_from": template_id,
+            "editable": True,
+        }
+        return clone
+
     def export_template(self, kind: str, template_id: str) -> Dict[str, Any]:
         template = self.get_template(kind, template_id)
         return {

package/latticeai/core/multi_agent.py

@@ -14,7 +14,7 @@ from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
 
-MULTI_AGENT_VERSION = "2.2.0"
+MULTI_AGENT_VERSION = "3.2.0"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/latticeai/core/workspace_os.py

@@ -18,7 +18,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
 
-WORKSPACE_OS_VERSION = "3.0.1"
+WORKSPACE_OS_VERSION = "3.2.0"
 
 # Workspace types separate single-user Personal workspaces from shared
 # Organization workspaces. Both keep the same local-first JSON store; the type

package/latticeai/server_app.py

@@ -73,7 +73,7 @@ from latticeai.services.workspace_service import WorkspaceService
 from latticeai.services.model_service import ModelService
 from latticeai.services.chat_service import ChatService
 from latticeai.services.search_service import SearchService
-from latticeai.core.embedding_providers import resolve_embedder
+from latticeai.core.embedding_providers import resolve_embedder, resolve_embedding_profile
 from latticeai.services.agent_runtime import AgentRuntime
 from latticeai.services.model_runtime import (
     CLOUD_VERIFY_TTL_SECONDS,
@@ -113,6 +113,12 @@ from latticeai.api.tools import create_tools_router
 from latticeai.api.static_routes import create_static_routes_router
 from latticeai.api.garden import create_garden_router
 from latticeai.api.setup import create_setup_router
+from latticeai.api.hooks import create_hooks_router
+from latticeai.core.hooks import HooksRegistry
+from latticeai.api.agent_registry import create_agent_registry_router
+from latticeai.core.agent_registry import AgentRegistry
+from latticeai.api.memory import create_memory_router
+from latticeai.services.memory_service import MemoryService
 from latticeai.services.tool_dispatch import (
     LOCAL_WRITE_BLOCKED_PREFIXES as _LOCAL_WRITE_BLOCKED_PREFIXES,
     TOOL_GOVERNANCE,
@@ -122,6 +128,7 @@ from latticeai.services.tool_dispatch import (
     configure_tool_dispatch,
     get_tool_permission,
     list_tool_permissions,
+    tool_response as _tool_response,
 )
 from latticeai.core.tool_registry import TOOL_CATALOG_BRIEF as _TOOL_CATALOG_BRIEF
 from mcp_registry import (
@@ -251,15 +258,26 @@ SSO_FILE = DATA_DIR / "sso_config.json"
 # Resolve the configured embedding provider once at startup. Degrades to the
 # offline hash fallback when the requested provider is unavailable, while
 # recording the requested-vs-active provider for the Embeddings status surface.
+try:
+    EMBEDDING_PROFILE = resolve_embedding_profile(CONFIG.embedding_profile)
+except ValueError as exc:
+    logging.warning("Embedding profile ignored: %s", exc)
+    EMBEDDING_PROFILE = {}
+_embedding_provider = CONFIG.embedding_provider
+_embedding_model = CONFIG.embedding_model or str(EMBEDDING_PROFILE.get("model") or "")
+_embedding_dim = CONFIG.embedding_dim or int(EMBEDDING_PROFILE.get("dimensions") or 0)
+if CONFIG.embedding_profile and CONFIG.embedding_provider in {"", "hash", "local", "fallback"}:
+    _embedding_provider = str(EMBEDDING_PROFILE.get("provider") or CONFIG.embedding_provider)
+
 EMBEDDER = resolve_embedder(
-    CONFIG.embedding_provider,
-    model=CONFIG.embedding_model,
+    _embedding_provider,
+    model=_embedding_model,
     base_url=CONFIG.embedding_base_url,
     api_key=CONFIG.embedding_api_key,
-    dim=CONFIG.embedding_dim,
+    dim=_embedding_dim,
     timeout=CONFIG.embedding_timeout,
     extra={"target": CONFIG.embedding_custom_target},
-    probe=CONFIG.embedding_provider not in {"", "hash", "local", "fallback"},
+    probe=_embedding_provider not in {"", "hash", "local", "fallback"},
 )
 if EMBEDDER.fell_back:
     logging.warning("Embedding provider %s unavailable: %s", EMBEDDER.requested, EMBEDDER.detail)
@@ -280,6 +298,19 @@ WORKSPACE_SERVICE = WorkspaceService(WORKSPACE_OS)
 PLUGINS_DIR = Path(os.getenv("LATTICEAI_PLUGINS_DIR") or (BASE_DIR / "plugins"))
 PLUGIN_REGISTRY = PluginRegistry(PLUGINS_DIR, store=WORKSPACE_OS)
 TEMPLATE_CATALOG = TemplateCatalog()
+# ── v3.2 platform registries: lifecycle hooks + agent registry, persisted under
+# DATA_DIR so the /app Hooks and Agent Registry views read/write real state.
+HOOKS_REGISTRY = HooksRegistry(DATA_DIR / "hooks.json")
+AGENT_REGISTRY = AgentRegistry(DATA_DIR / "agent_registry.json")
+# Unified long-term memory platform fronting workspace memories, agent
+# snapshots, conversation history, and the KG graph/vector index.
+MEMORY_SERVICE = MemoryService(
+    store=WORKSPACE_OS,
+    data_dir=DATA_DIR,
+    knowledge_graph=KNOWLEDGE_GRAPH,
+    enable_graph=ENABLE_GRAPH,
+    history_file=HISTORY_FILE,
+)
 
 def _require_graph():
     if not ENABLE_GRAPH or KNOWLEDGE_GRAPH is None:
@@ -805,6 +836,8 @@ def _require_local_approval(
 
 def require_admin(request: Request) -> tuple[str, Dict]:
     users = load_users()
+    if not REQUIRE_AUTH:
+        return "", users
     token = _extract_bearer_token(request)
     if token:
         email = get_session_email(token)
@@ -1100,6 +1133,7 @@ app.include_router(create_auth_router(
     get_sso_settings=get_sso_settings, get_sso_discovery=_get_sso_discovery,
     public_sso_config=public_sso_config,
     open_registration=OPEN_REGISTRATION, session_ttl=_SESSION_TTL,
+    require_auth=REQUIRE_AUTH,
 ))
 
 def _graph_stats_safe():
@@ -1387,9 +1421,11 @@ app.include_router(create_chat_router(
 ))
 
 def _embedding_info() -> dict:
-    from latticeai.core.embedding_providers import PROVIDER_TYPES
+    from latticeai.core.embedding_providers import PROVIDER_TYPES, embedding_provider_profiles
     info = EMBEDDER.as_dict()
     info["available_providers"] = list(PROVIDER_TYPES)
+    info["profile"] = CONFIG.embedding_profile or ""
+    info["profiles"] = embedding_provider_profiles()
     return info
 
 
@@ -1423,6 +1459,27 @@ app.include_router(create_tools_router(
     mcp_public_item=mcp_public_item,
 ))
 
+app.include_router(create_hooks_router(
+    registry=HOOKS_REGISTRY,
+    require_user=require_user,
+    append_audit_event=append_audit_event,
+))
+
+app.include_router(create_agent_registry_router(
+    registry=AGENT_REGISTRY,
+    require_user=require_user,
+    append_audit_event=append_audit_event,
+))
+
+app.include_router(create_memory_router(
+    service=MEMORY_SERVICE,
+    require_user=require_user,
+    get_current_user=get_current_user,
+    gate_read=PLATFORM.gate_read,
+    gate_write=PLATFORM.gate_write,
+    append_audit_event=append_audit_event,
+))
+
 app.include_router(create_garden_router(gardener=gardener, require_user=require_user))
 app.include_router(create_setup_router(model_router=router, require_user=require_user))