ltcai 11.2.0 → 11.4.0

package/latticeai/models/router/loading.py

@@ -0,0 +1,291 @@
+"""The optional backends, and everything that loads a model through them.
+
+Two things live here together, deliberately. First, the module-level runtime
+bindings — ``mx`` / ``vlm_load`` / ``lm_load`` / ``VLM_AVAILABLE`` /
+``LM_AVAILABLE`` / ``AsyncOpenAI`` / ``executor`` — which are guarded imports
+that ``ensure_mlx_runtime`` **rebinds** after an installer has run. Second,
+every method that reads them: ``load_model``, ``_load_cloud_model`` and
+``_release_memory``.
+
+They are one module because a rebindable global is only ever correct where it
+is defined: a sibling module that did ``from .loading import mx`` would hold
+the import-time value forever, and ``ensure_mlx_runtime`` would appear to do
+nothing. For the same reason a test standing in for a backend patches
+``latticeai.models.router.loading``, not the package.
+"""
+
+import os
+
+# Default Gemma 4 assistant drafting to MTP without overriding an operator's
+# explicit MLX runtime choice.
+os.environ.setdefault("MLX_VLM_DRAFT_KIND", "mtp")
+
+import asyncio
+import gc
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any, Dict, List, Optional
+
+from latticeai.models.model_providers import (
+    OPENAI_COMPATIBLE_PROVIDERS,
+    PROVIDER_MODEL_CATALOG,
+)
+
+from ._contract import RouterCore as _Core
+from .catalog import CloudModel, parse_model_ref, source_metadata_for_model
+from .local_models import (
+    _is_gemma4_model_id,
+    _local_model_type,
+    _resolve_local_hf_model,
+)
+
+# Optional dependencies. Each is aliased on import and then re-exported as
+# `Any`, so "installed" and "absent" are the same declared type and every
+# call site keeps its historical name.
+try:
+    from openai import AsyncOpenAI as _AsyncOpenAI
+except Exception:
+    _AsyncOpenAI = None  # type: ignore[assignment,misc]
+AsyncOpenAI: Any = _AsyncOpenAI
+
+
+# 추론 전용 싱글 스레드 워커 (GPU 스트림 보호용)
+executor = ThreadPoolExecutor(max_workers=1)
+
+
+try:
+    import mlx.core as _mx
+except Exception as e:
+    _mx = None  # type: ignore[assignment]
+    print(f"⚠️ MLX core unavailable: {e}")
+mx: Any = _mx
+
+
+try:
+    from mlx_vlm import load as _vlm_load
+    VLM_AVAILABLE = True
+    print("✅ MLX-VLM is ready for multimodal models.")
+except Exception as e:
+    _vlm_load = None  # type: ignore[assignment]
+    VLM_AVAILABLE = False
+    print(f"⚠️ MLX-VLM unavailable: {e}")
+vlm_load: Any = _vlm_load
+
+
+try:
+    from mlx_lm import load as _lm_load
+    LM_AVAILABLE = True
+    print("✅ MLX-LM is ready for text fallback models.")
+except Exception as e:
+    _lm_load = None  # type: ignore[assignment]
+    LM_AVAILABLE = False
+    print(f"⚠️ MLX-LM unavailable: {e}")
+lm_load: Any = _lm_load
+
+
+def ensure_mlx_runtime() -> None:
+    global mx, vlm_load, lm_load, VLM_AVAILABLE, LM_AVAILABLE
+    if mx is not None and (vlm_load is not None or lm_load is not None):
+        return
+    errors = []
+    try:
+        import mlx.core as mlx_core
+        mx = mlx_core
+        mx.set_default_device(mx.gpu)
+    except Exception as e:
+        errors.append(f"mlx: {e}")
+        mx = None
+
+    try:
+        from mlx_vlm import load as mlx_vlm_load
+        vlm_load = mlx_vlm_load
+        VLM_AVAILABLE = True
+    except Exception as e:
+        vlm_load = None
+        VLM_AVAILABLE = False
+        errors.append(f"mlx-vlm: {e}")
+
+    try:
+        from mlx_lm import load as mlx_lm_load
+        lm_load = mlx_lm_load
+        LM_AVAILABLE = True
+    except Exception as e:
+        lm_load = None
+        LM_AVAILABLE = False
+        errors.append(f"mlx-lm: {e}")
+
+    if mx is None or (vlm_load is None and lm_load is None):
+        raise RuntimeError(f"MLX runtime is not available after install: {'; '.join(errors)}")
+
+
+def _mlx_sampler(temperature: float):
+    """Build an MLX sampler callable for the given temperature.
+
+    Lattice v2.2 keeps local execution on MLX-VLM only. Returning ``None`` lets
+    MLX-VLM use its bundled default sampler without pulling another generation
+    package into the runtime contract.
+    """
+    _ = temperature
+    return
+
+
+class _LoadingMixin(_Core):
+    """Loading, unloading memory, and enumerating what could be loaded."""
+
+    def _release_memory(self) -> None:
+        gc.collect()
+        if mx is not None and hasattr(mx, "clear_cache"):
+            try:
+                mx.clear_cache()
+            except Exception as e:
+                print(f"⚠️ MLX cache clear skipped: {e}")
+
+    async def load_model(
+        self,
+        model_id: str,
+        adapter_path: Optional[str] = None,
+        draft_model_id: Optional[str] = None,
+        api_key_override: Optional[str] = None,
+        owner: Optional[str] = None,
+    ) -> str:
+        provider, provider_model = parse_model_ref(model_id)
+        if provider != "local_mlx":
+            return self._load_cloud_model(provider, provider_model, api_key_override=api_key_override, owner=owner)
+
+        ensure_mlx_runtime()
+        if mx is None or (vlm_load is None and lm_load is None):
+            raise RuntimeError("MLX is not available in this process. Run on Apple Silicon with Metal access.")
+
+        cache_key = f"{model_id}_{draft_model_id}" if draft_model_id else model_id
+        with self._lock:
+            if cache_key in self._cache:
+                self._current = cache_key
+                self._touch(cache_key)
+                return f"Cached: {cache_key}"
+
+        self._enforce_local_model_limit(cache_key)
+        print(f"⏳ Loading local model stack: {cache_key}...")
+        loop = asyncio.get_event_loop()
+        target_model_id = _resolve_local_hf_model(model_id)
+        target_draft_model_id = _resolve_local_hf_model(draft_model_id) if draft_model_id else None
+        
+        def _load():
+            mx.set_default_device(mx.gpu)
+            is_gemma4 = _is_gemma4_model_id(model_id)
+            model_type = _local_model_type(target_model_id) or _local_model_type(model_id)
+            loader_kind = "mlx_vlm"
+
+            try:
+                if vlm_load is None:
+                    raise RuntimeError("MLX-VLM is not installed.")
+                print(f"🔄 Loading Target (VLM Mode): {target_model_id}...")
+                model, tokenizer = vlm_load(target_model_id)
+            except Exception as vlm_error:
+                if not (is_gemma4 and model_type != "gemma4_unified" and lm_load is not None):
+                    raise
+                print(f"⚠️ Gemma 4 MLX-VLM load failed; retrying MLX-LM text path: {vlm_error}")
+                print(f"🔄 Loading Target (LM Mode): {target_model_id}...")
+                model, tokenizer = lm_load(target_model_id)
+                loader_kind = "mlx_lm"
+
+            draft_model = None
+            if target_draft_model_id:
+                if loader_kind == "mlx_vlm":
+                    print(f"🔄 Loading Assistant (VLM Mode): {target_draft_model_id}...")
+                    draft_model, _ = vlm_load(target_draft_model_id)
+                elif lm_load is not None:
+                    print(f"🔄 Loading Assistant (LM Mode): {target_draft_model_id}...")
+                    draft_model, _ = lm_load(target_draft_model_id)
+                print("✅ Assistant Ready.")
+
+            return model, tokenizer, draft_model, loader_kind
+
+        try:
+            # Use the dedicated single-thread executor to ensure MLX GPU streams match during inference
+            model, tokenizer, draft_model, loader_kind = await loop.run_in_executor(executor, _load)
+            with self._lock:
+                self._cache[cache_key] = (model, tokenizer, draft_model, loader_kind)
+                self._current = cache_key
+                self._touch(cache_key)
+            print(f"✅ Fully Loaded: {cache_key} ({loader_kind})")
+            return f"Success: {cache_key} ({loader_kind})"
+        except Exception as e:
+            print(f"❌ Load Error: {e}")
+            raise e
+
+    def _load_cloud_model(self, provider: str, model: str, api_key_override: Optional[str] = None, owner: Optional[str] = None) -> str:
+        if AsyncOpenAI is None:
+            raise RuntimeError("openai package is not installed. Add it to requirements.txt and install dependencies.")
+        config = OPENAI_COMPATIBLE_PROVIDERS.get(provider)
+        if not config:
+            raise RuntimeError(f"Unsupported cloud provider: {provider}")
+
+        api_key = api_key_override or os.getenv(config["env_key"]) or config.get("api_key_fallback")
+        if not api_key:
+            raise RuntimeError(f"Missing API key env var: {config['env_key']}")
+
+        base_url = os.getenv(config.get("base_url_env", "")) if config.get("base_url_env") else None
+        base_url = base_url or config.get("base_url")
+        # base_url is passed only when configured: an explicit None is not
+        # the same as omitting the argument.
+        client = (
+            AsyncOpenAI(api_key=api_key, base_url=base_url)
+            if base_url
+            else AsyncOpenAI(api_key=api_key)
+        )
+
+        cache_owner = owner or "global"
+        cache_key = f"{provider}:{model}::{cache_owner}"
+        with self._lock:
+            self._cache[cache_key] = CloudModel(
+                provider=provider, model=model, client=client, cache_key=cache_key
+            )
+            self._current = cache_key
+            self._touch(cache_key)
+        return f"Cloud provider ready: {cache_key}"
+
+    def detected_cloud_models(self) -> List[Dict[str, str]]:
+        local_server_providers = {"ollama", "vllm", "lmstudio", "llamacpp"}
+        items: List[Dict[str, Any]] = []
+        for provider, config in OPENAI_COMPATIBLE_PROVIDERS.items():
+            has_key = bool(os.getenv(config["env_key"]) or config.get("api_key_fallback"))
+            provider_models = PROVIDER_MODEL_CATALOG.get(provider) or [{
+                "id": config["default_model"],
+                "name": f"{provider.title()} · {config['default_model']}",
+                "family": provider.title(),
+            }]
+            for model in provider_models:
+                model_id = model["id"]
+                local_server = provider in local_server_providers
+                items.append({
+                    "id": f"{provider}:{model_id}",
+                    "name": model.get("name") or f"{provider.title()} · {model_id}",
+                    "provider": provider,
+                    "family": model.get("family"),
+                    "tag": "local-server" if local_server else "cloud",
+                    "available": has_key,
+                    "requires": config["env_key"] if not has_key else None,
+                    **source_metadata_for_model(provider, model, local_server=local_server),
+                })
+        custom = os.getenv("LATTICEAI_CLOUD_MODELS") or ""
+        for raw in [item.strip() for item in custom.split(",") if item.strip()]:
+            provider, custom_model = parse_model_ref(raw)
+            if provider != "local_mlx" and provider in OPENAI_COMPATIBLE_PROVIDERS:
+                config = OPENAI_COMPATIBLE_PROVIDERS[provider]
+                items.append({
+                    "id": f"{provider}:{custom_model}",
+                    "name": f"{provider.title()} · {custom_model}",
+                    "provider": provider,
+                    "tag": "cloud",
+                    "available": bool(os.getenv(config["env_key"]) or config.get("api_key_fallback")),
+                    "requires": None,
+                    **source_metadata_for_model(
+                        provider,
+                        {
+                            "id": custom_model,
+                            "name": f"{provider.title()} · {custom_model}",
+                            "family": provider.title(),
+                        },
+                        local_server=provider in local_server_providers,
+                    ),
+                })
+        return items

package/latticeai/models/router/local_models.py

@@ -0,0 +1,85 @@
+"""Finding a locally downloaded model on disk.
+
+Three places a model may already be: an explicit path, this app's own
+``~/.ltcai/hf-models`` directory, or the shared Hugging Face cache. A directory
+only counts when it actually holds a config, weights and a tokenizer — a
+half-finished download must not be handed to the loader as if it were a model.
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import Optional
+
+HF_MODELS_ROOT = Path.home() / ".ltcai" / "hf-models"
+
+
+def hf_model_dir(repo_id: str) -> Path:
+    return HF_MODELS_ROOT / repo_id.replace("/", "__")
+
+
+def hf_cache_model_dir(repo_id: str) -> Optional[Path]:
+    """Return a usable Hugging Face cache snapshot for an already-downloaded model."""
+    cache_root = Path.home() / ".cache" / "huggingface" / "hub" / f"models--{repo_id.replace('/', '--')}"
+    snapshots = cache_root / "snapshots"
+    if not snapshots.exists():
+        return None
+    candidates = sorted(
+        (item for item in snapshots.iterdir() if item.is_dir()),
+        key=lambda item: item.stat().st_mtime,
+        reverse=True,
+    )
+    for snapshot in candidates:
+        if _looks_like_hf_model_dir(snapshot):
+            return snapshot
+    return None
+
+
+def _looks_like_hf_model_dir(path: Path) -> bool:
+    if not path.exists() or not path.is_dir():
+        return False
+    has_config = (path / "config.json").exists()
+    has_weights = any(path.glob("*.safetensors")) or any(path.glob("*.bin"))
+    has_tokenizer = (
+        (path / "tokenizer.json").exists()
+        or (path / "tokenizer.model").exists()
+        or (path / "tokenizer_config.json").exists()
+    )
+    return has_config and has_weights and has_tokenizer
+
+
+def _resolve_local_hf_model(model_id: str) -> str:
+    explicit_path = Path(model_id).expanduser()
+    if explicit_path.exists():
+        return str(explicit_path)
+    local_dir = hf_model_dir(model_id)
+    if _looks_like_hf_model_dir(local_dir):
+        return str(local_dir)
+    cached_dir = hf_cache_model_dir(model_id)
+    if cached_dir is not None:
+        return str(cached_dir)
+    return model_id
+
+
+def _is_gemma4_model_id(model_id: str) -> bool:
+    raw = str(model_id or "").lower()
+    return bool(re.search(r"gemma[-_/ ]?4|gemma4", raw))
+
+
+def _local_model_type(path_or_model_id: str) -> Optional[str]:
+    raw = str(path_or_model_id or "").strip()
+    candidates = []
+    explicit = Path(raw).expanduser()
+    if raw and explicit.exists():
+        candidates.append(explicit / "config.json")
+    candidates.append(hf_model_dir(raw) / "config.json")
+    for config_path in candidates:
+        try:
+            if config_path.exists():
+                data = json.loads(config_path.read_text(encoding="utf-8"))
+                model_type = str(data.get("model_type") or "").strip().lower()
+                if model_type:
+                    return model_type
+        except Exception as e:
+            print(f"⚠️ Model config read skipped for {config_path}: {e}")
+    return None

package/latticeai/models/router/registry.py

@@ -0,0 +1,147 @@
+"""The mutable model registry: what is loaded, what is current, what to evict.
+
+Everything here is guarded by one reentrant lock, because the eviction path
+nests (``_enforce_local_model_limit`` → ``unload_model`` → ``_release_memory``)
+and because generation must never change ``_current`` — that value is a
+UI/default preference shared by every request, so a request-scoped model is
+taken as an immutable snapshot instead.
+"""
+
+import os
+import threading
+import time
+from typing import Any, Dict, List, Optional, Tuple
+
+from latticeai.models.model_providers import OPENAI_COMPATIBLE_PROVIDERS
+
+from ._contract import RouterCore as _Core
+from .catalog import CloudModel
+
+
+class _RegistryMixin(_Core):
+    """The loaded-model registry half of :class:`LLMRouter`."""
+
+    def __init__(self):
+        # A local entry is (model, tokenizer, draft_model, loader_kind); a
+        # cloud entry is a CloudModel. `_unpack_local_cache` splits them.
+        self._cache: Dict[str, Any] = {}
+        self._current: Optional[str] = None
+        self._last_used: Dict[str, float] = {}
+        self._max_local_models = max(1, int(os.getenv("LATTICEAI_MAX_LOCAL_MODELS", "1")))
+        # Guards the mutable model registry (_cache/_current/_last_used).
+        # Reentrant because the eviction path nests: _enforce_local_model_limit
+        # → unload_model → _release_memory. Never held across the heavy
+        # ``run_in_executor`` load (only the sync insert/read is guarded), so a
+        # long model load can't block a concurrent switch/unload from acquiring.
+        self._lock = threading.RLock()
+
+    @property
+    def current_model_id(self) -> Optional[str]:
+        with self._lock:
+            return self._current
+
+    @property
+    def loaded_model_ids(self) -> List[str]:
+        with self._lock:
+            return list(self._cache.keys())
+
+    def switch_model(self, model_id: str) -> None:
+        with self._lock:
+            if model_id not in self._cache:
+                raise KeyError(model_id)
+            self._current = model_id
+            self._touch(model_id)
+
+    def unload_model(self, model_id: str) -> None:
+        with self._lock:
+            self._cache.pop(model_id, None)
+            self._last_used.pop(model_id, None)
+            if self._current == model_id:
+                self._current = next(iter(self._cache), None)
+            self._release_memory()
+
+    def unload_all(self) -> None:
+        with self._lock:
+            self._cache.clear()
+            self._last_used.clear()
+            self._current = None
+            self._release_memory()
+
+    def unload_idle_models(self, idle_seconds: int) -> List[str]:
+        if idle_seconds <= 0:
+            return []
+        now = time.monotonic()
+        unloaded = []
+        with self._lock:
+            for model_id, last_used in list(self._last_used.items()):
+                if now - last_used >= idle_seconds:
+                    self.unload_model(model_id)
+                    unloaded.append(model_id)
+        return unloaded
+
+    def model_memory_policy(self) -> Dict[str, object]:
+        with self._lock:
+            return {
+                "max_local_models": self._max_local_models,
+                "loaded_count": len(self._cache),
+                "last_used": dict(self._last_used),
+            }
+
+    def _touch(self, model_id: Optional[str] = None) -> None:
+        model_id = model_id or self._current
+        if model_id:
+            self._last_used[model_id] = time.monotonic()
+
+    def _is_local_model(self, model_id: str) -> bool:
+        cached = self._cache.get(model_id)
+        return cached is not None and not isinstance(cached, CloudModel)
+
+    def _enforce_local_model_limit(self, incoming_key: str) -> None:
+        with self._lock:
+            local_ids = [model_id for model_id in self._cache if self._is_local_model(model_id)]
+            while len(local_ids) >= self._max_local_models:
+                victim = min(local_ids, key=lambda model_id: self._last_used.get(model_id, 0))
+                if victim == incoming_key:
+                    break
+                print(f"🧹 Unloading local model to stay within memory policy: {victim}")
+                self.unload_model(victim)
+                local_ids = [model_id for model_id in self._cache if self._is_local_model(model_id)]
+
+    def _is_cloud_current(self) -> bool:
+        with self._lock:
+            return bool(self._current and isinstance(self._cache.get(self._current), CloudModel))
+
+    def _local_server_error_hint(self, cloud: CloudModel, error: Exception) -> str:
+        raw = str(error)
+        if cloud.provider == "lmstudio":
+            base_url = os.getenv("LMSTUDIO_BASE_URL") or OPENAI_COMPATIBLE_PROVIDERS["lmstudio"]["base_url"]
+            return (
+                f"LM Studio 연결 실패: {raw}\n\n"
+                f"- LM Studio의 Developer/Local Server를 켜고 모델을 로드했는지 확인하세요.\n"
+                f"- Lattice가 보는 주소는 {base_url} 입니다. 포트가 다르면 LMSTUDIO_BASE_URL을 맞춰주세요.\n"
+                f"- 모델 선택창에는 LM Studio /v1/models에서 감지된 모델만 표시됩니다."
+            )
+        return raw
+
+    def _unpack_local_cache(self, cached: Any) -> Tuple[Any, Any, Any, str]:
+        model, tokenizer, draft_model = cached[:3]
+        loader_kind = str(cached[3]) if len(cached) > 3 else "mlx_vlm"
+        return model, tokenizer, draft_model, loader_kind
+
+    def _model_snapshot(self, model_id: Optional[str] = None) -> tuple[Optional[str], object | None]:
+        """Return an immutable request-scoped view of a loaded model.
+
+        Generation must never change ``_current``: that value is a UI/default
+        preference shared by every request.  Capturing the cache entry while
+        holding the registry lock prevents concurrent requests from selecting
+        or restoring each other's models.
+        """
+        with self._lock:
+            selected = model_id or self._current
+            if not selected:
+                return None, None
+            cached = self._cache.get(selected)
+            if cached is None:
+                raise ValueError(f"Model '{selected}' is not loaded. Load it first via /models/load.")
+            self._touch(selected)
+            return selected, cached

package/latticeai/runtime/build_phases/__init__.py

@@ -0,0 +1,82 @@
+"""The ordered phases that build the Lattice AI application.
+
+Each phase reads what earlier phases published on the :class:`RuntimeContext`
+and publishes its own results. The order below *is* the dependency order and is
+fixed by ``tests/unit/test_runtime_context.py``:
+
+1.  ``platform``   — MLX/GPU device selection (the only step that touches hardware)
+2.  ``config``     — configuration, security settings, paths, filesystem layout
+3.  ``identity``   — users, sessions, audit, access control, API keys, SSO/VPC config
+4.  ``brain``      — embedder, knowledge graph, conversations, hooks, persistence, history
+5.  ``domain``     — model router, garden, chat service (needed by the web phase)
+6.  ``web``        — lifespan, the FastAPI app, model runtime, static + foundation routers
+7.  ``services``   — retrieval/context, chat agent runtime, the typed AppContext
+8.  ``foundation_routes`` — mount the foundation routers now that AppContext exists
+9.  ``platform_features`` — workspace platform, automation, review, command centre
+10. ``interaction`` — model/chat/search/tools routers and the brain tail routers
+
+Every heavy import lives *inside* a phase, never at module scope: importing
+this module must stay free of GPU init, singleton construction, and filesystem
+writes (``tests/unit/test_app_factory.py`` enforces that).
+
+Why closures still appear here: several handlers must resolve a dependency at
+call time rather than at construction time, because the dependency is built by
+a later phase. Those read through ``ctx``, which is exactly the late binding
+the original single function got from Python's closure rules.
+
+v11.3.0 split the 1,450-line module into three stage submodules — ``foundation``
+(1-4), ``web`` (5-8) and ``features`` (9-10). Nothing else moved: this package
+re-exports every phase under its historical name and :data:`BUILD_PHASES` stays
+defined here, in one place, so the order test and the factory read the same
+tuple rather than a copy that can drift.
+"""
+
+from __future__ import annotations
+
+from latticeai.runtime.build_phases.features import (
+    phase_interaction as phase_interaction,
+)
+from latticeai.runtime.build_phases.features import (
+    phase_platform_features as phase_platform_features,
+)
+from latticeai.runtime.build_phases.foundation import phase_brain as phase_brain
+from latticeai.runtime.build_phases.foundation import phase_config as phase_config
+from latticeai.runtime.build_phases.foundation import phase_identity as phase_identity
+from latticeai.runtime.build_phases.foundation import phase_platform as phase_platform
+from latticeai.runtime.build_phases.web import phase_domain as phase_domain
+from latticeai.runtime.build_phases.web import (
+    phase_foundation_routes as phase_foundation_routes,
+)
+from latticeai.runtime.build_phases.web import phase_services as phase_services
+from latticeai.runtime.build_phases.web import phase_web as phase_web
+from latticeai.runtime.build_phases.web import self_model_port as self_model_port
+
+#: The build order. Exported so the ordering test reads the same list the
+#: factory runs, rather than a copy that can drift.
+BUILD_PHASES = (
+    phase_platform,
+    phase_config,
+    phase_identity,
+    phase_brain,
+    phase_domain,
+    phase_web,
+    phase_services,
+    phase_foundation_routes,
+    phase_platform_features,
+    phase_interaction,
+)
+
+
+__all__ = [
+    "BUILD_PHASES",
+    "phase_brain",
+    "phase_domain",
+    "phase_config",
+    "phase_foundation_routes",
+    "phase_identity",
+    "phase_interaction",
+    "phase_platform",
+    "phase_platform_features",
+    "phase_services",
+    "phase_web",
+]