ltcai 11.5.2 → 11.7.0

package/latticeai/runtime/build_phases/foundation.py

@@ -1,10 +1,23 @@
-"""Phases 1-4: platform, configuration, identity, brain.
+"""Phases 1-4 of the AI-Worker build: platform, configuration, identity, brain.
 
 The first half of the build order — everything that must exist before a
-FastAPI application object is worth creating. Split out of the single
-``build_phases`` module in v11.3.0; the package ``__init__`` still owns the
-:data:`~latticeai.runtime.build_phases.BUILD_PHASES` order and re-exports every
-phase under its historical name.
+FastAPI application object is worth creating.
+
+v11.6.0 (WP-P1) cut these four phases down to what a **pure compute worker**
+needs. The product application built users, sessions, SSO, VPC config, the MCP
+install state, the audit log, the knowledge-graph store, the conversation
+store, the workspace OS and a dozen platform services here; ``lattice-host``
+owns every one of those now, and Python owns no durable state at all. What
+survives is the smallest set that still lets this process infer, embed, parse,
+render and transcribe:
+
+* the MLX device (phase 1),
+* configuration and the data directory the model cache lives under (phase 2),
+* the **seam gate** — ``require_user``, the rate limiter, the loopback posture
+  (phase 3). It is a check, not a store: durable sessions and users are
+  ``lattice-auth``'s,
+* the resolved embedder and the multi-modal ports (phase 4), which is all the
+  compute seams need.
 
 Every heavy import lives *inside* a phase, never at module scope
 (``tests/unit/test_runtime_context.py`` enforces that for each submodule).
@@ -13,7 +26,7 @@ Every heavy import lives *inside* a phase, never at module scope
 from __future__ import annotations
 
 import logging
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
 from latticeai.core.quiet import quiet
 from latticeai.runtime.runtime_context import RuntimeContext
@@ -43,8 +56,7 @@ def phase_config(ctx: RuntimeContext) -> None:
     """Parse configuration once and lay out the data directory."""
     ctx.enter("config")
 
-    from latticeai.core.security import host_is_loopback as _host_is_loopback_impl
-    from latticeai.core.workspace_os import WORKSPACE_OS_VERSION
+    from latticeai import __version__ as app_version
     from latticeai.runtime.config_runtime import build_config_runtime
     from latticeai.runtime.security_runtime import build_security_runtime
 
@@ -62,7 +74,6 @@ def phase_config(ctx: RuntimeContext) -> None:
         "IS_PUBLIC_MODE",
         "DEFAULT_HOST",
         "DEFAULT_PORT",
-        "ENABLE_TELEGRAM",
         "ENABLE_GRAPH",
         "AUTOLOAD_MODELS",
         "MODEL_IDLE_UNLOAD_SECONDS",
@@ -78,29 +89,15 @@ def phase_config(ctx: RuntimeContext) -> None:
         "LOCAL_MODEL",
         "LOCAL_DRAFT_MODEL",
     )
-    ctx.set(
-        APP_VERSION=WORKSPACE_OS_VERSION,
-        OPEN_REGISTRATION=ctx.CONFIG.open_registration,
-        _RATE_LIMIT_ENABLED=ctx.CONFIG.rate_limit_enabled,
-        # Forwarded headers (X-Forwarded-For / CF-Connecting-IP) are honoured
-        # for IP rate limiting only when the direct peer is a trusted proxy.
-        _host_is_loopback=lambda host: _host_is_loopback_impl(host),
-    )
+    # The version the worker reports on ``/health``. It used to be read off the
+    # Workspace OS store's schema constant; that store is native platform state
+    # now, and the package version was always the same number.
+    ctx.set(APP_VERSION=app_version)
 
     security_runtime = build_security_runtime(ctx.CONFIG)
     ctx.set(security_runtime=security_runtime)
-    ctx.adopt(
-        security_runtime,
-        "SSO_DISCOVERY_URL",
-        "SSO_CLIENT_ID",
-        "SSO_CLIENT_SECRET",
-        "SSO_REDIRECT_URI",
-        "SSO_PROVIDER_NAME",
-        "INVITE_CODE",
-        "INVITE_COOKIE_SECRET",
-        "INVITE_GATE_ENABLED",
-        "SECURE_COOKIES",
-    )
+    ctx.adopt(security_runtime, "RATE_LIMIT_ENABLED")
+    ctx.set(_RATE_LIMIT_ENABLED=ctx.RATE_LIMIT_ENABLED)
 
     from pathlib import Path
 
@@ -117,148 +114,47 @@ def phase_config(ctx: RuntimeContext) -> None:
         DATA_DIR=data_dir,
         STATIC_DIR=ctx.CONFIG.static_dir,
         USERS_FILE=data_dir / "users.json",
-        HISTORY_FILE=data_dir / "chat_history.json",
-        VPC_FILE=data_dir / "vpc_config.json",
-        AUDIT_FILE=data_dir / "audit_log.json",
-        SSO_FILE=data_dir / "sso_config.json",
     )
 
 
-# ── phase 3: identity, audit, access ─────────────────────────────────────────
+# ── phase 3: the seam gate ───────────────────────────────────────────────────
 def phase_identity(ctx: RuntimeContext) -> None:
-    """Users, sessions, audit, access control, API keys, SSO/VPC config."""
+    """Who is calling, and may they call this often.
+
+    Not an identity *store*: the worker reads ``users.json`` to answer "what
+    role does this session's email have" and holds session tokens in memory for
+    the life of the process. Registration, password hashing, invitations, SSO
+    and the audit log went to ``lattice-auth`` and ``lattice-platform`` with
+    every route that used them.
+    """
     ctx.enter("identity")
 
     from fastapi import HTTPException, Request
 
-    from latticeai.core.mcp_registry import create_mcp_install_state
-    from latticeai.core.security import (
-        check_ip_rate_limit as _check_ip_rate_limit,
-    )
-    from latticeai.core.security import (
-        client_ip as _client_ip_impl,
-    )
-    from latticeai.core.security import (
-        enforce_rate_limit as _enforce_rate_limit,
-    )
-    from latticeai.core.security import hash_password, verify_password
-    from latticeai.core.security import (
-        redact_secret_text as _redact_secret_text,
-    )
-    from latticeai.core.users import (
-        ensure_user_identity,
-        load_users_file,
-        migrate_knowledge_graph_identity,
-        save_users_file,
-    )
+    from latticeai.core.security import enforce_rate_limit as _enforce_rate_limit
+    from latticeai.core.users import load_users_file
     from latticeai.core.users import user_id_for_email as _user_id_for_email
     from latticeai.runtime.access_runtime import build_access_runtime
-    from latticeai.runtime.audit_runtime import build_audit_runtime
     from latticeai.runtime.bootstrap import build_session_runtime
-    from latticeai.runtime.network_config_runtime import build_vpc_runtime
-    from latticeai.runtime.sso_config_runtime import build_sso_config_runtime
-    from latticeai.runtime.user_key_runtime import build_user_key_runtime
-
-    # MCP install state (extracted to mcp_registry during the server decomp).
-    mcp_state = create_mcp_install_state(ctx.DATA_DIR)
-    ctx.adopt(
-        mcp_state,
-        "load_mcp_installs",
-        "mcp_public_item",
-        "recommend_mcps",
-        "install_mcp",
-    )
 
     def load_users() -> Dict[str, Any]:
-        users = load_users_file(ctx.USERS_FILE)
-        email_to_id = {
-            email: str(user["id"])
-            for email, user in users.items()
-            if isinstance(user, dict) and user.get("id")
-        }
-        try:
-            migrate_knowledge_graph_identity(
-                ctx.DATA_DIR / "knowledge_graph.sqlite", email_to_id
-            )
-        except Exception as exc:
-            logging.warning("knowledge graph identity migration skipped: %s", exc)
-        try:
-            # WORKSPACE_OS is built in the `brain` phase; resolving it here (at
-            # call time) is why this stays a closure over ctx.
-            ctx.WORKSPACE_OS.migrate_workspace_identities(email_to_id)
-        except Exception as exc:
-            logging.warning("workspace identity migration skipped: %s", exc)
-        return users
-
-    def save_users(users: Dict[str, Any]) -> None:
-        save_users_file(ctx.USERS_FILE, users)
+        return load_users_file(ctx.USERS_FILE)
 
     def user_id_for_email(email: Optional[str]) -> Optional[str]:
         return _user_id_for_email(load_users(), email)
 
-    def verify_and_migrate_password(
-        email: str, plain: str, stored: str, users: Dict
-    ) -> bool:
-        """평문 비밀번호를 투명하게 해시로 마이그레이션. 마이그레이션 발생 시 audit log 남김."""
-        if ":" in stored and len(stored) > 64:
-            return verify_password(plain, stored)
-        if plain == stored:
-            users[email]["password"] = hash_password(plain)
-            save_users(users)
-            try:
-                ctx.append_audit_event(
-                    "password_migrated_from_plaintext", user_email=email
-                )
-            except Exception as exc:
-                logging.warning("audit log failed on password migration: %s", exc)
-            logging.info("Migrated plaintext password to scrypt hash for %s", email)
-            return True
-        return False
-
-    def redact_secret_text(text: str) -> str:
-        return _redact_secret_text(text)
-
-    def _check_rate_limit(
-        ip: str, action: str, max_calls: int, window_secs: float
-    ) -> None:
-        _check_ip_rate_limit(ip, action, max_calls=max_calls, window_secs=window_secs)
-
-    def _client_ip(request: Request) -> str:
-        return _client_ip_impl(request)
-
     def enforce_rate_limit(email: str, bucket_key: str) -> None:
         _enforce_rate_limit(email, bucket_key, enabled=ctx._RATE_LIMIT_ENABLED)
 
     ctx.set(
         load_users=load_users,
-        save_users=save_users,
         user_id_for_email=user_id_for_email,
-        verify_and_migrate_password=verify_and_migrate_password,
-        redact_secret_text=redact_secret_text,
-        _check_rate_limit=_check_rate_limit,
-        _client_ip=_client_ip,
         enforce_rate_limit=enforce_rate_limit,
     )
 
     # Session token lifecycle; user_id_for_email is the injected subject resolver.
     session_runtime = build_session_runtime(user_id_resolver=user_id_for_email)
-    ctx.adopt(
-        session_runtime,
-        "_SESSION_TTL",
-        "_session_store",
-        "create_session",
-        "get_session_email",
-        "invalidate_session",
-    )
-
-    # Audit is built after redaction is available, because every event is
-    # redacted before it is written.
-    audit_runtime = build_audit_runtime(
-        audit_file=ctx.AUDIT_FILE,
-        logging=logging,
-        redact_fn=redact_secret_text,
-    )
-    ctx.adopt(audit_runtime, "get_audit_log", "append_audit_event")
+    ctx.adopt(session_runtime, "_session_store", "get_session_email")
 
     access_runtime = build_access_runtime(
         config=ctx.CONFIG,
@@ -276,81 +172,34 @@ def phase_identity(ctx: RuntimeContext) -> None:
         "get_current_user",
         "require_user",
         "require_admin",
-        "public_user",
-    )
-
-    user_key_runtime = build_user_key_runtime(
-        load_users=load_users,
-        save_users=save_users,
-        ensure_user_identity=ensure_user_identity,
-        keyring=ctx.keyring,
-        allow_plaintext_api_keys=ctx.ALLOW_PLAINTEXT_API_KEYS,
-        logging=logging,
-        http_exception=HTTPException,
     )
-    ctx.adopt(
-        user_key_runtime, "get_history_user", "get_user_api_key", "set_user_api_key"
-    )
-
-    # SSO config + OIDC discovery share one closure, so saving the config
-    # actually invalidates the discovery cache.
-    sso_runtime = build_sso_config_runtime(
-        sso_file=ctx.SSO_FILE,
-        discovery_url=ctx.SSO_DISCOVERY_URL,
-        client_id=ctx.SSO_CLIENT_ID,
-        client_secret=ctx.SSO_CLIENT_SECRET,
-        redirect_uri=ctx.SSO_REDIRECT_URI,
-        provider_name=ctx.SSO_PROVIDER_NAME,
-        logging=logging,
-    )
-    ctx.adopt(
-        sso_runtime,
-        "_sso_env_defaults",
-        "get_sso_settings",
-        "public_sso_config",
-        "save_sso_config",
-        "_get_sso_discovery",
-        "_sso_states",
-    )
-
-    vpc_runtime = build_vpc_runtime(vpc_file=ctx.VPC_FILE, logging=logging)
-    ctx.adopt(vpc_runtime, "load_vpc_config", "save_vpc_config")
 
 
-# ── phase 4: brain and persistence ───────────────────────────────────────────
+# ── phase 4: the compute ports ───────────────────────────────────────────────
 def phase_brain(ctx: RuntimeContext) -> None:
-    """Embedder, knowledge graph, conversations, hooks, persistence, history."""
+    """The embedder and the multi-modal ports — the worker's whole Brain half.
+
+    ``phase_brain`` used to open ``knowledge_graph.sqlite``, build the
+    conversation store, the workspace OS, the plugin registry, the memory
+    service and the ingest write door. Rust owns every one of those writes
+    (§Wave 2.5), so what is left is the two *ports* the compute seams need:
+    something that turns text into a vector, and something that can look at a
+    picture or listen to a recording. Both degrade to an honest absence rather
+    than failing construction.
+    """
     ctx.enter("brain")
 
-    import os
-
-    from fastapi import HTTPException
-
-    from lattice_brain.graph.schema import set_embed_dim
-    from lattice_brain.ingestion import IngestionItem
-    from lattice_brain.storage import storage_from_env
-    from latticeai.core.audit import (
-        build_admin_audit_report as _build_admin_audit_report,
-    )
-    from latticeai.core.audit import (
-        build_sensitivity_report as _build_sensitivity_report,
-    )
-    from latticeai.core.audit import (
-        classify_sensitive_message as _classify_sensitive_message,
-    )
+    from lattice_brain.ingestion import IngestionPipeline
     from latticeai.core.embedding_providers import (
         resolve_embedder,
         resolve_embedding_profile,
     )
-    from latticeai.core.security import (
-        bytes_match_extension as _bytes_match_extension_impl,
+    from latticeai.runtime.brain_runtime import build_embedder_runtime
+    from latticeai.services.multimodal_ports import (
+        build_multimodal_ports,
+        multimodal_enabled,
     )
-    from latticeai.models.router import normalize_branding
-    from latticeai.runtime.brain_runtime import build_brain_runtime
-    from latticeai.runtime.history_runtime import build_history_query_runtime
-    from latticeai.runtime.history_writer import HistoryWriterDeps, write_chat_turn
-    from latticeai.runtime.hooks_runtime import build_hooks_runtime
-    from latticeai.runtime.persistence_runtime import build_persistence_runtime
+    from latticeai.services.voice_capture import VoiceCaptureService
 
     # Resolve the configured embedding provider once. Degrades to the offline
     # hash fallback when unavailable, recording requested-vs-active provider.
@@ -360,196 +209,37 @@ def phase_brain(ctx: RuntimeContext) -> None:
         logging.warning("Embedding profile ignored: %s", exc)
         embedding_profile = {}
 
-    provider = ctx.CONFIG.embedding_provider
-    model = ctx.CONFIG.embedding_model or str(embedding_profile.get("model") or "")
-    dim = ctx.CONFIG.embedding_dim or int(embedding_profile.get("dimensions") or 0)
-    if ctx.CONFIG.embedding_profile and provider in {"", "hash", "local", "fallback"}:
-        provider = str(embedding_profile.get("provider") or provider)
-
-    embedder = resolve_embedder(
-        provider,
-        model=model,
-        base_url=ctx.CONFIG.embedding_base_url,
-        api_key=ctx.CONFIG.embedding_api_key,
-        dim=dim,
-        timeout=ctx.CONFIG.embedding_timeout,
-        extra={"target": ctx.CONFIG.embedding_custom_target},
-        probe=provider not in {"", "hash", "local", "fallback"},
+    embedder = build_embedder_runtime(
+        config=ctx.CONFIG,
+        profile=embedding_profile,
+        resolve_embedder=resolve_embedder,
     )
-    set_embed_dim(int(getattr(embedder, "dim", None) or dim or 384))
     if embedder.fell_back:
         logging.warning(
             "Embedding provider %s unavailable: %s", embedder.requested, embedder.detail
         )
-
-    storage_engine = (
-        storage_from_env(os.environ, data_dir=ctx.DATA_DIR) if ctx.ENABLE_GRAPH else None
-    )
-    ctx.set(
-        EMBEDDING_PROFILE=embedding_profile,
-        EMBEDDER=embedder,
-        STORAGE_ENGINE=storage_engine,
-    )
-
-    brain_runtime = build_brain_runtime(
-        data_dir=ctx.DATA_DIR,
-        history_file=ctx.HISTORY_FILE,
-        enable_graph=ctx.ENABLE_GRAPH,
-        embedder=embedder,
-        storage_engine=storage_engine,
-    )
-    ctx.set(brain_runtime=brain_runtime)
-    # CONVERSATIONS is the v4 durable episodic store: unbounded history in the
-    # same SQLite file as the graph, so backup/restore covers it for free.
-    ctx.adopt(brain_runtime, "KNOWLEDGE_GRAPH", "CONVERSATIONS")
-
-    def save_to_history(
-        role: str,
-        message: str,
-        user_email: Optional[str] = None,
-        user_nickname: Optional[str] = None,
-        source: Optional[str] = None,
-        conversation_id: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-    ) -> None:
-        """Persist one chat turn. Logic lives in runtime/history_writer.py.
-
-        The dependency bundle is built per call because INGESTION_PIPELINE is
-        published later in this same phase; resolving through ctx keeps the
-        ordering contract in one place instead of reordering the phase.
-        """
-        write_chat_turn(
-            role,
-            message,
-            user_email=user_email,
-            user_nickname=user_nickname,
-            source=source,
-            conversation_id=conversation_id,
-            workspace_id=workspace_id,
-            deps=HistoryWriterDeps(
-                conversations=ctx.CONVERSATIONS,
-                append_audit_event=ctx.append_audit_event,
-                classify_sensitive_message=ctx.classify_sensitive_message,
-                redact_secret_text=ctx.redact_secret_text,
-                normalize_branding=normalize_branding,
-                ingestion_pipeline=ctx.INGESTION_PIPELINE,
-                ingestion_item_factory=IngestionItem,
-                enable_graph=ctx.ENABLE_GRAPH,
-                knowledge_graph=ctx.KNOWLEDGE_GRAPH,
-            ),
-        )
-
-    def classify_sensitive_message(item: Dict, index: int) -> Dict:
-        return _classify_sensitive_message(item, index)
-
-    def build_sensitivity_report(history: List[Dict]) -> Dict:
-        return _build_sensitivity_report(history)
-
-    def build_admin_audit_report(
-        users: Dict, audit_events: Optional[List[Dict]] = None
-    ) -> Dict:
-        graph_stats = None
-        try:
-            if ctx.ENABLE_GRAPH and ctx.KNOWLEDGE_GRAPH:
-                graph_stats = ctx.KNOWLEDGE_GRAPH.stats()
-        except Exception:
-            quiet()
-        return _build_admin_audit_report(
-            ctx.AUDIT_FILE,
-            users,
-            get_user_role=ctx.get_user_role,
-            graph_stats=graph_stats,
-            audit_events=audit_events,
-        )
-
-    def _bytes_match_extension(data: bytes, ext: str) -> bool:
-        return _bytes_match_extension_impl(data, ext)
-
-    def _require_graph() -> None:
-        if not ctx.ENABLE_GRAPH or ctx.KNOWLEDGE_GRAPH is None:
-            raise HTTPException(
-                status_code=404,
-                detail=(
-                    "지식 그래프가 비활성화되어 있습니다. "
-                    "LATTICEAI_ENABLE_GRAPH=true 설정 후 다시 시도해 주세요."
-                ),
-            )
-
-    def _workspace_graph() -> Any:
-        return (
-            ctx.KNOWLEDGE_GRAPH
-            if (ctx.ENABLE_GRAPH and ctx.KNOWLEDGE_GRAPH)
-            else None
-        )
-
+    ctx.set(EMBEDDING_PROFILE=embedding_profile, EMBEDDER=embedder)
+
+    # Multi-modal capture (v11.1.0): off unless the user turned it on, and even
+    # then only as capable as the models actually present. With nothing
+    # configured this resolves to an empty bundle without importing or loading
+    # anything.
+    #
+    # The transcriber is resolved through the voice service so a voice memo and
+    # a scanned ``.m4a`` are transcribed by the same thing — or, far more often,
+    # by the same nothing.
+    voice_capture = VoiceCaptureService(transcriber=None)
+    multimodal_ports = build_multimodal_ports(
+        transcriber=voice_capture.multimodal_ports().transcriber
+    )
+    # The capability probe behind ``GET /api/ingestion/multimodal``. It answers
+    # what this machine *could* do with a picture, a recording or a video; the
+    # ingest door itself is native (§W3b), which is why nothing is passed here
+    # for it to write through.
     ctx.set(
-        save_to_history=save_to_history,
-        classify_sensitive_message=classify_sensitive_message,
-        build_sensitivity_report=build_sensitivity_report,
-        build_admin_audit_report=build_admin_audit_report,
-        _bytes_match_extension=_bytes_match_extension,
-        _require_graph=_require_graph,
-        _workspace_graph=_workspace_graph,
-    )
-
-    # The hooks registry is built ahead of the watcher so folder-watch
-    # reindexes can fire the pre_index/post_index lifecycle hooks.
-    hooks_runtime = build_hooks_runtime(
-        data_dir=ctx.DATA_DIR,
-        enable_graph=ctx.ENABLE_GRAPH,
-        knowledge_graph_getter=lambda: ctx.KNOWLEDGE_GRAPH,
-    )
-    ctx.adopt(hooks_runtime, "HOOKS_REGISTRY", "LOCAL_KG_WATCHER")
-
-    persistence_runtime = build_persistence_runtime(
-        data_dir=ctx.DATA_DIR,
-        base_dir=ctx.BASE_DIR,
-        enable_graph=ctx.ENABLE_GRAPH,
-        knowledge_graph=ctx.KNOWLEDGE_GRAPH,
-        hooks_registry=ctx.HOOKS_REGISTRY,
-        history_file=ctx.HISTORY_FILE,
-        conversations=ctx.CONVERSATIONS,
-        user_id_for_email=ctx.user_id_for_email,
-        audit=lambda action, detail, user: ctx.append_audit_event(
-            action, user_email=user, **detail
+        VOICE_CAPTURE=voice_capture,
+        MULTIMODAL_PORTS=multimodal_ports,
+        INGESTION_PIPELINE=IngestionPipeline(
+            allow_multimodal=multimodal_enabled(), multimodal=multimodal_ports
         ),
     )
-    ctx.adopt(
-        persistence_runtime,
-        "REALTIME_BUS",
-        "WORKSPACE_OS",
-        "WORKSPACE_SERVICE",
-        "INVITATION_STORE",
-        "PLUGIN_REGISTRY",
-        "TEMPLATE_CATALOG",
-        "AGENT_REGISTRY",
-        "MEMORY_SERVICE",
-        "BRAIN_INTELLIGENCE",
-        "AUTOMATION_INTELLIGENCE",
-        "INGESTION_PIPELINE",
-        "MULTIMODAL_PORTS",
-        "DEVICE_IDENTITY",
-        "KG_PORTABILITY",
-        "FUNNEL_METRICS",
-    )
-
-    # History reads/clears. The write path (save_to_history above) stays a
-    # closure because it is bound to redaction/audit/ingestion.
-    history_query_runtime = build_history_query_runtime(
-        conversations=ctx.CONVERSATIONS,
-        workspace_service=ctx.WORKSPACE_SERVICE,
-        require_auth=ctx.REQUIRE_AUTH,
-        logging=logging,
-    )
-    ctx.adopt(
-        history_query_runtime,
-        "_history_allowed_workspaces_for",
-        "_history_include_legacy_global",
-        "get_history",
-        "conversation_title",
-        "group_history_conversations",
-        "get_conversation_messages",
-        "clear_history",
-        "clear_conversation",
-    )
-