ltcai 5.6.0 → 6.0.0

package/latticeai/core/marketplace.py

@@ -11,7 +11,7 @@ from copy import deepcopy
 from typing import Any, Dict, List, Optional
 
 
-MARKETPLACE_VERSION = "5.6.0"
+MARKETPLACE_VERSION = "6.0.0"
 TEMPLATE_KINDS = ("plugin", "workflow", "agent")
 
 

package/latticeai/core/workspace_os.py

@@ -19,7 +19,7 @@ from pathlib import Path
 from typing import Any, Callable, Dict, Iterable, List, Optional
 
 
-WORKSPACE_OS_VERSION = "5.6.0"
+WORKSPACE_OS_VERSION = "6.0.0"
 
 # Workspace types separate single-user Personal workspaces from shared
 # Organization workspaces. Both keep the same local-first JSON store; the type

package/latticeai/runtime/app_context_runtime.py

@@ -0,0 +1,13 @@
+"""Application dependency context assembly for app startup."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def build_app_context(**deps: Any) -> Any:
+    """Construct the typed dependency context consumed by API routers."""
+
+    from latticeai.services.app_context import AppContext
+
+    return AppContext(**deps)

package/latticeai/runtime/automation_runtime.py

@@ -0,0 +1,64 @@
+"""Automation runtime assembly for review, trigger, agent, and run execution.
+
+This module keeps the automation object graph behind one construction seam.
+Router registration remains in ``app_factory`` so route order stays unchanged.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict
+
+
+def build_automation_runtime(
+    *,
+    store: Any,
+    platform: Any,
+    data_dir: Any,
+    workspace_graph: Callable[..., Any],
+    append_audit_event: Callable[..., Any],
+    hooks: Any,
+) -> Dict[str, Any]:
+    """Construct automation services and attach the run executor boundary."""
+
+    from lattice_brain.runtime.agent_runtime import AgentRuntime
+    from latticeai.services.review_queue import ReviewQueueService
+    from latticeai.services.run_executor import RunExecutor
+    from latticeai.services.triggers import TriggerService
+
+    review_queue = ReviewQueueService(store=store)
+    trigger_service = TriggerService(
+        store=store,
+        run_workflow=lambda wf_id, inputs: platform.run_workflow_by_id(
+            wf_id,
+            None,
+            None,
+            with_agent=False,
+            inputs=inputs,
+        ),
+        data_dir=data_dir,
+        review_sink=review_queue,
+    )
+    agent_runtime = AgentRuntime(
+        store=store,
+        orchestrator_factory=platform.build_orchestrator,
+        workspace_graph=workspace_graph,
+        append_audit_event=append_audit_event,
+        hooks=hooks,
+    )
+    run_executor = RunExecutor(
+        store=store,
+        agent_runtime=agent_runtime,
+        build_workflow_runners=platform.build_workflow_runners,
+        workspace_graph=workspace_graph,
+        append_audit_event=append_audit_event,
+        hooks=hooks,
+        review_sink=review_queue,
+    )
+    agent_runtime.attach_executor(run_executor)
+
+    return {
+        "REVIEW_QUEUE": review_queue,
+        "TRIGGER_SERVICE": trigger_service,
+        "AGENT_RUNTIME": agent_runtime,
+        "RUN_EXECUTOR": run_executor,
+    }

package/latticeai/runtime/bootstrap.py

@@ -0,0 +1,48 @@
+"""Session bootstrap runtime: session store and token helpers.
+
+Extracted from ``app_factory._build`` as a composition seam. The session
+token helpers stay closures over a single ``SessionStore`` so the factory
+keeps one source of truth for token lifecycle. ``user_id_resolver`` is the
+factory's ``user_id_for_email`` — injected so the store stays decoupled from
+user persistence. Heavy imports stay inside the function so importing the
+module has no side effects.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional
+
+_DEFAULT_SESSION_TTL = 60 * 60 * 24
+
+
+def build_session_runtime(
+    *,
+    user_id_resolver: Callable[[Optional[str]], Optional[str]],
+    ttl_seconds: int = _DEFAULT_SESSION_TTL,
+) -> Dict[str, Any]:
+    """Construct the session store and its token helper closures."""
+
+    from latticeai.core.sessions import SessionStore
+
+    session_store = SessionStore()
+
+    def create_session(email: str) -> str:
+        return session_store.create(user_id_resolver(email) or email, email=email)
+
+    def get_session_email(token: str) -> Optional[str]:
+        return session_store.get_email(token)
+
+    def get_session_user_id(token: str) -> Optional[str]:
+        return session_store.get_subject(token)
+
+    def invalidate_session(token: str) -> None:
+        session_store.invalidate(token)
+
+    return {
+        "_SESSION_TTL": ttl_seconds,
+        "_session_store": session_store,
+        "create_session": create_session,
+        "get_session_email": get_session_email,
+        "get_session_user_id": get_session_user_id,
+        "invalidate_session": invalidate_session,
+    }

package/latticeai/runtime/context_runtime.py

@@ -0,0 +1,43 @@
+"""Retrieval/context runtime assembly for app startup."""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict
+
+
+def build_context_runtime(
+    *,
+    graph_store: Any,
+    ingestion_pipeline: Any,
+    memory_service: Any,
+    gardener: Any,
+    require_auth: bool,
+    allowed_scopes_for_user: Callable[[Any], Any],
+) -> Dict[str, Any]:
+    """Construct search, brain memory, and context assembly services."""
+
+    from lattice_brain.context import ContextAssembler
+    from lattice_brain.memory import BrainMemory
+    from latticeai.services.search_service import SearchService
+
+    search_service = SearchService(graph_store=graph_store)
+    brain_memory = BrainMemory(ingestion_pipeline)
+
+    def scoped_hybrid_search(q, user_email=None, **kw):
+        allowed = None
+        if require_auth and user_email:
+            allowed = allowed_scopes_for_user(user_email)
+        return search_service.hybrid_search(q, allowed_workspaces=allowed, **kw)
+
+    context_assembler = ContextAssembler(
+        memory_recall=memory_service.recall,
+        hybrid_search=scoped_hybrid_search,
+        notes_context=gardener.get_relevant_context,
+    )
+
+    return {
+        "SEARCH_SERVICE": search_service,
+        "BRAIN_MEMORY": brain_memory,
+        "CONTEXT_ASSEMBLER": context_assembler,
+        "_scoped_hybrid_search": scoped_hybrid_search,
+    }

package/latticeai/runtime/hooks_runtime.py

@@ -0,0 +1,77 @@
+"""Hooks + local-knowledge watcher runtime assembly for app startup.
+
+Extracted from ``app_factory._build`` as a composition seam: the hooks
+registry must be constructed *ahead* of the local-knowledge watcher so
+folder-watch reindexes can fire the ``pre_index``/``post_index`` lifecycle
+hooks. Heavy imports stay inside the function so importing the module has no
+side effects.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict
+
+
+def build_hooks_runtime(
+    *,
+    data_dir: Any,
+    enable_graph: bool,
+    knowledge_graph_getter: Callable[[], Any],
+) -> Dict[str, Any]:
+    """Construct the hooks registry and local-knowledge watcher behind one seam."""
+
+    from lattice_brain.runtime.hooks import HooksRegistry
+    from local_knowledge_api import LocalKnowledgeWatcher
+
+    hooks_registry = HooksRegistry(data_dir / "hooks.json")
+    local_kg_watcher = (
+        LocalKnowledgeWatcher(knowledge_graph_getter, hooks=hooks_registry)
+        if enable_graph
+        else None
+    )
+    return {
+        "HOOKS_REGISTRY": hooks_registry,
+        "LOCAL_KG_WATCHER": local_kg_watcher,
+    }
+
+
+def bind_trigger_hook_runner(*, registry: Any, trigger_service: Any) -> str:
+    """Ensure the brain-event trigger hook exists and bind its runtime runner."""
+
+    from latticeai.services.triggers import TRIGGER_HOOK_NAME
+
+    trigger_hook_id = next(
+        (
+            h.get("id")
+            for h in registry._state.get("custom", [])
+            if h.get("name") == TRIGGER_HOOK_NAME
+        ),
+        None,
+    )
+    if trigger_hook_id is None:
+        trigger_hook_id = registry.register(
+            name=TRIGGER_HOOK_NAME,
+            kind="post_tool",
+            description="Fires brain_event workflow triggers when knowledge enters the brain.",
+        )["id"]
+    registry.register_hook(trigger_hook_id, trigger_service.hook_runner())
+    return trigger_hook_id
+
+
+def bind_builtin_hook_runners(
+    *,
+    registry: Any,
+    append_audit_event: Callable[..., Any],
+    get_tool_permission: Callable[..., Any],
+    classify_sensitive_message: Callable[..., Any],
+) -> None:
+    """Bind concrete platform runners for built-in hook definitions."""
+
+    from latticeai.core.builtin_hooks import register_builtin_hook_runners
+
+    register_builtin_hook_runners(
+        registry,
+        append_audit_event=append_audit_event,
+        get_tool_permission=get_tool_permission,
+        classify_sensitive_message=classify_sensitive_message,
+    )

package/latticeai/runtime/lifespan_runtime.py

@@ -0,0 +1,138 @@
+"""FastAPI lifespan assembly for app startup and shutdown tasks."""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from typing import Any, Dict
+
+
+def build_lifespan_runtime(
+    *,
+    app_mode: str,
+    enable_telegram: bool,
+    autoload_models: bool,
+    is_public_mode: bool,
+    public_model: str,
+    allow_local_models: bool,
+    local_model: str,
+    local_draft_model: str,
+    model_idle_unload_seconds: int,
+    model_router: Any,
+    local_kg_watcher: Any,
+    local_server_processes: Dict[str, Any],
+    logger: Any,
+) -> Dict[str, Any]:
+    """Create lifespan and background task helpers for the FastAPI app."""
+
+    import asyncio
+    import os
+
+    async def autoload_default_model() -> None:
+        if not autoload_models:
+            print("⏭️ Model autoload disabled by LATTICEAI_AUTOLOAD_MODELS=false.")
+            return
+
+        if is_public_mode:
+            model_id = public_model
+            provider = model_id.split(":", 1)[0] if ":" in model_id else "openai"
+            env_by_provider = {
+                "openai": "OPENAI_API_KEY",
+                "openrouter": "OPENROUTER_API_KEY",
+                "groq": "GROQ_API_KEY",
+                "together": "TOGETHER_API_KEY",
+                "ollama": "OLLAMA_API_KEY",
+            }
+            required_env = env_by_provider.get(provider)
+            if required_env and not os.getenv(required_env) and provider != "ollama":
+                print(f"🌐 Public mode ready. Set {required_env} to autoload {model_id}.")
+                return
+            print(f"🌐 Public mode autoload: {model_id}")
+            try:
+                msg = await model_router.load_model(model_id)
+                print(f"✅ {msg}")
+            except Exception as e:  # pragma: no cover - startup diagnostics
+                print(f"⚠️ Public model autoload failed: {e}")
+            return
+
+        if not allow_local_models:
+            print("⏭️ Local model autoload skipped because LATTICEAI_ALLOW_LOCAL_MODELS=false.")
+            return
+
+        print("⏳ Auto-loading local model stack:")
+        print(f"   - Target: {local_model}")
+        if local_draft_model:
+            print(f"   - Draft:  {local_draft_model}")
+        else:
+            print("   - Draft:  disabled (set LATTICEAI_LOCAL_DRAFT_MODEL to enable)")
+        try:
+            await model_router.load_model(local_model, draft_model_id=local_draft_model or None)
+        except Exception as e:  # pragma: no cover - startup diagnostics
+            print(f"⚠️ Local model autoload failed: {e}")
+
+    async def unload_idle_models_loop() -> None:
+        if model_idle_unload_seconds <= 0:
+            print("⏭️ Model idle unload disabled.")
+            return
+        while True:
+            await asyncio.sleep(min(60, model_idle_unload_seconds))
+            try:
+                unloaded = model_router.unload_idle_models(model_idle_unload_seconds)
+                if unloaded:
+                    print(f"🧹 Idle model unload: {', '.join(unloaded)}")
+            except Exception as e:  # pragma: no cover - background diagnostics
+                logger.warning("Idle model unload failed: %s", e)
+
+    def spawn(coro, *, name: str):
+        """Fire-and-forget asyncio task that logs exceptions instead of swallowing them."""
+
+        task = asyncio.create_task(coro, name=name)
+
+        def _on_done(t: asyncio.Task) -> None:
+            if t.cancelled():
+                return
+            exc = t.exception()
+            if exc is not None:
+                logger.warning("background task '%s' failed: %s", name, exc)
+
+        task.add_done_callback(_on_done)
+        return task
+
+    @asynccontextmanager
+    async def lifespan(_app):
+        try:
+            print(f"🧭 Lattice AI mode: {app_mode}")
+            if enable_telegram:
+                from telegram_bot import run_bot
+
+                spawn(run_bot(), name="telegram_bot")
+                print("🚀 Telegram Bot Bridge activated!")
+            else:
+                print("⏭️ Telegram Bot Bridge disabled for this mode.")
+            spawn(unload_idle_models_loop(), name="unload_idle_models")
+            spawn(autoload_default_model(), name="autoload_default_model")
+            if local_kg_watcher:
+                restored = local_kg_watcher.restore_enabled_sources()
+                if restored.get("restored"):
+                    print(f"🕸️ Local knowledge watchers restored: {restored['restored']}")
+        except Exception as e:  # pragma: no cover - startup diagnostics
+            print(f"⚠️ Startup sequence failed: {e}")
+        try:
+            yield
+        finally:
+            if local_kg_watcher:
+                local_kg_watcher.stop_all()
+            model_router.unload_all()
+            for proc in local_server_processes.values():
+                try:
+                    if proc.poll() is None:
+                        proc.terminate()
+                        proc.wait(timeout=5)
+                except Exception:
+                    pass
+
+    return {
+        "autoload_default_model": autoload_default_model,
+        "unload_idle_models_loop": unload_idle_models_loop,
+        "_spawn": spawn,
+        "lifespan": lifespan,
+    }

package/latticeai/runtime/persistence_runtime.py

@@ -0,0 +1,87 @@
+"""Persistence/service assembly seams for app startup.
+
+This module owns the durable local stores and services that sit between the
+Brain runtime and API routers. Imports stay inside the function so importing
+``latticeai.app_factory`` remains side-effect free.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, Optional
+
+
+def build_persistence_runtime(
+    *,
+    data_dir: Any,
+    base_dir: Any,
+    enable_graph: bool,
+    knowledge_graph: Any,
+    hooks_registry: Any,
+    history_file: Any,
+    conversations: Any,
+    user_id_for_email: Callable[[Optional[str]], Optional[str]],
+    audit: Callable[[str, Dict[str, Any], Optional[str]], None],
+) -> Dict[str, Any]:
+    """Construct workspace, plugin, memory, ingestion, and portability services."""
+
+    import os
+    from pathlib import Path
+
+    from lattice_brain.identity import DeviceIdentity
+    from lattice_brain.ingestion import IngestionPipeline
+    from lattice_brain.portability import KGPortabilityService
+    from latticeai.core.agent_registry import AgentRegistry
+    from latticeai.core.invitations import InvitationStore
+    from latticeai.core.marketplace import TemplateCatalog
+    from latticeai.core.plugins import PluginRegistry
+    from latticeai.core.realtime import RealtimeBus
+    from latticeai.core.workspace_os import WorkspaceOSStore
+    from latticeai.services.memory_service import MemoryService
+    from latticeai.services.workspace_service import WorkspaceService
+
+    realtime_bus = RealtimeBus()
+    workspace_os = WorkspaceOSStore(data_dir, event_sink=realtime_bus)
+    workspace_service = WorkspaceService(workspace_os, resolve_user_id=user_id_for_email)
+    invitation_store = InvitationStore(data_dir / "invitations.json")
+
+    plugins_dir = Path(os.getenv("LATTICEAI_PLUGINS_DIR") or (base_dir / "plugins"))
+    plugin_registry = PluginRegistry(plugins_dir, store=workspace_os)
+    template_catalog = TemplateCatalog()
+    agent_registry = AgentRegistry(data_dir / "agent_registry.json")
+
+    memory_service = MemoryService(
+        store=workspace_os,
+        data_dir=data_dir,
+        knowledge_graph=knowledge_graph,
+        enable_graph=enable_graph,
+        history_file=history_file,
+        conversation_store=conversations,
+    )
+    ingestion_pipeline = IngestionPipeline(
+        knowledge_graph,
+        hooks=hooks_registry,
+        enable_graph=enable_graph,
+        audit=audit,
+    )
+    device_identity = DeviceIdentity(data_dir)
+    kg_portability = KGPortabilityService(
+        knowledge_graph=knowledge_graph,
+        data_dir=data_dir,
+        enable_graph=enable_graph,
+        device_identity=device_identity,
+    )
+
+    return {
+        "REALTIME_BUS": realtime_bus,
+        "WORKSPACE_OS": workspace_os,
+        "WORKSPACE_SERVICE": workspace_service,
+        "INVITATION_STORE": invitation_store,
+        "PLUGINS_DIR": plugins_dir,
+        "PLUGIN_REGISTRY": plugin_registry,
+        "TEMPLATE_CATALOG": template_catalog,
+        "AGENT_REGISTRY": agent_registry,
+        "MEMORY_SERVICE": memory_service,
+        "INGESTION_PIPELINE": ingestion_pipeline,
+        "DEVICE_IDENTITY": device_identity,
+        "KG_PORTABILITY": kg_portability,
+    }

package/latticeai/runtime/platform_services_runtime.py

@@ -0,0 +1,39 @@
+"""Small platform service construction seams for app startup."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def build_model_service(
+    *,
+    model_router: Any,
+    runtime_features: Any,
+    is_public: bool,
+) -> Any:
+    """Construct the health/model summary service."""
+
+    from latticeai.services.model_service import ModelService
+
+    return ModelService(
+        model_router=model_router,
+        runtime_features=runtime_features,
+        is_public=is_public,
+    )
+
+
+def build_brain_network(
+    *,
+    identity: Any,
+    portability: Any,
+    data_dir: Any,
+) -> Any:
+    """Construct peer sync/network service for brain portability routes."""
+
+    from lattice_brain.network import BrainNetwork
+
+    return BrainNetwork(
+        identity=identity,
+        portability=portability,
+        data_dir=data_dir,
+    )