ltcai 11.2.0 → 11.4.0

package/latticeai/services/model_runtime/engines.py

@@ -0,0 +1,341 @@
+"""Local engine discovery, server hand-off, and the LM Studio HTTP client.
+
+Thin wrappers over ``latticeai.services.model_engines`` that keep the historical
+``model_runtime`` import path alive, plus the one piece of real logic that never
+belonged in the engine layer: the LM Studio native API (list / download / load)
+and its short-lived model cache.
+
+``_LMSTUDIO_MODELS_CACHE`` and ``_LMSTUDIO_MODELS_CACHE_TS`` are rebindable
+module state and therefore live here and nowhere else — the package
+``__init__`` deliberately does not re-export them, because a
+``from … import`` copy would freeze at import time and quietly disagree with
+the live value.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import shutil
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from latticeai.models.router import OPENAI_COMPATIBLE_PROVIDERS, AsyncOpenAI
+from latticeai.services.model_engines import (
+    LOCAL_SERVER_PROCESSES as _LOCAL_SERVER_PROCESSES,
+)
+from latticeai.services.model_engines import (
+    engine_install_plan as _engine_install_plan,
+)
+from latticeai.services.model_engines import (
+    engine_support_status as _engine_support_status,
+)
+from latticeai.services.model_engines import (
+    ensure_llamacpp_server as _ensure_llamacpp_server,
+)
+from latticeai.services.model_engines import (
+    ensure_lmstudio_server as _ensure_lmstudio_server,
+)
+from latticeai.services.model_engines import (
+    ensure_ollama_server as _ensure_ollama_server,
+)
+from latticeai.services.model_engines import (
+    ensure_vllm_server as _ensure_vllm_server,
+)
+from latticeai.services.model_engines import (
+    find_lmstudio_cli as _find_lmstudio_cli,
+)
+from latticeai.services.model_engines import (
+    get_ollama_pulled_models as _get_ollama_pulled_models,
+)
+from latticeai.services.model_engines import (
+    get_openai_compatible_server_models as _get_openai_compatible_server_models,
+)
+from latticeai.services.model_engines import (
+    local_binary as _local_binary,
+)
+from latticeai.services.model_engines import (
+    pull_ollama_model_with_progress as _pull_ollama_model_with_progress,
+)
+from latticeai.services.model_engines import (
+    vllm_executable as _vllm_executable,
+)
+from latticeai.services.model_engines import (
+    vllm_metal_python as _vllm_metal_python,
+)
+from latticeai.services.model_engines import (
+    wait_for_openai_compatible_server as _wait_for_openai_compatible_server,
+)
+from latticeai.services.model_engines import (
+    windows_binary_candidates as _windows_binary_candidates,
+)
+from latticeai.services.model_errors import ModelRuntimeError
+
+
+def _update_env_file(env_file: Path, key: str, value: str) -> None:
+    lines = []
+    found = False
+    if env_file.exists():
+        for line in env_file.read_text(encoding="utf-8").splitlines():
+            if line.startswith(f"{key}="):
+                lines.append(f"{key}={value}")
+                found = True
+            else:
+                lines.append(line)
+    if not found:
+        lines.append(f"{key}={value}")
+    env_file.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+LOCAL_SERVER_PROCESSES = _LOCAL_SERVER_PROCESSES
+VLLM_METAL_ENV = Path.home() / ".venv-vllm-metal"
+VLLM_METAL_BIN = VLLM_METAL_ENV / "bin" / "vllm"
+VLLM_METAL_PYTHON = VLLM_METAL_ENV / "bin" / "python"
+LMSTUDIO_BUNDLED_CLI = Path("/Applications/LM Studio.app/Contents/Resources/app/.webpack/lms")
+
+def windows_binary_candidates(binary: str) -> List[Path]:
+    return _windows_binary_candidates(binary)
+
+
+def local_binary(binary: str) -> Optional[str]:
+    return _local_binary(binary)
+
+
+def find_lmstudio_cli() -> Optional[str]:
+    return _find_lmstudio_cli()
+
+
+def vllm_executable() -> Optional[str]:
+    return _vllm_executable()
+
+
+def vllm_metal_python() -> Optional[str]:
+    return _vllm_metal_python()
+
+
+def _json_request(
+    url: str,
+    *,
+    method: str = "GET",
+    payload: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, str]] = None,
+    timeout: float = 10.0,
+) -> Dict[str, Any]:
+    data = None
+    req_headers = dict(headers or {})
+    if payload is not None:
+        data = json.dumps(payload).encode("utf-8")
+        req_headers.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
+    with urllib.request.urlopen(req, timeout=timeout) as res:
+        raw = res.read().decode("utf-8", errors="replace")
+    if not raw.strip():
+        return {}
+    return json.loads(raw)
+
+
+def lmstudio_api_base() -> str:
+    return (os.getenv("LMSTUDIO_BASE_URL") or OPENAI_COMPATIBLE_PROVIDERS["lmstudio"]["base_url"]).rstrip("/")
+
+
+def lmstudio_native_api_base() -> str:
+    base = lmstudio_api_base()
+    return base[:-3] if base.endswith("/v1") else base
+
+
+def ensure_lmstudio_server() -> None:
+    return _ensure_lmstudio_server()
+
+
+_LMSTUDIO_MODELS_CACHE: List[Dict[str, Any]] = []
+_LMSTUDIO_MODELS_CACHE_TS: float = 0.0
+_LMSTUDIO_MODELS_CACHE_TTL: float = 10.0
+
+
+def get_lmstudio_models(*, force: bool = False) -> List[Dict[str, Any]]:
+    global _LMSTUDIO_MODELS_CACHE, _LMSTUDIO_MODELS_CACHE_TS
+    if not force and time.monotonic() - _LMSTUDIO_MODELS_CACHE_TS < _LMSTUDIO_MODELS_CACHE_TTL:
+        return _LMSTUDIO_MODELS_CACHE
+    try:
+        payload = _json_request(
+            f"{lmstudio_native_api_base()}/api/v1/models",
+            headers={"Authorization": f"Bearer {os.getenv('LMSTUDIO_API_KEY') or 'lmstudio'}"},
+            timeout=2.5,
+        )
+    except Exception:
+        return _LMSTUDIO_MODELS_CACHE
+    models = payload.get("models")
+    _LMSTUDIO_MODELS_CACHE = models if isinstance(models, list) else []
+    _LMSTUDIO_MODELS_CACHE_TS = time.monotonic()
+    return _LMSTUDIO_MODELS_CACHE
+
+
+def _lmstudio_candidate_keys(model_name: str) -> List[str]:
+    raw = model_name.strip()
+    if not raw:
+        return []
+    slug = raw.split("/")[-1].lower()
+    slug = slug.replace("-gguf", "").replace("-awq", "")
+    parts = [p for p in slug.split("-") if p]
+    candidates = [raw.lower(), slug]
+    if parts:
+        candidates.append("-".join(parts[: min(4, len(parts))]))
+    return list(dict.fromkeys(candidates))
+
+
+def _find_lmstudio_model_key(model_name: str, models: List[Dict[str, Any]]) -> Optional[str]:
+    if not models:
+        return None
+    candidate_keys = _lmstudio_candidate_keys(model_name)
+    exact = []
+    fuzzy = []
+    for item in models:
+        key = str(item.get("key") or "").strip()
+        display_name = str(item.get("display_name") or "").strip()
+        haystacks = [key.lower(), display_name.lower()]
+        if any(raw == key.lower() for raw in candidate_keys):
+            exact.append(key)
+            continue
+        if any(token and token in hay for token in candidate_keys for hay in haystacks):
+            fuzzy.append(key)
+    return next(iter(exact or fuzzy), None)
+
+
+def ensure_lmstudio_model(model_name: str) -> Dict[str, Any]:
+    ensure_lmstudio_server()
+    auth_header = {"Authorization": f"Bearer {os.getenv('LMSTUDIO_API_KEY') or 'lmstudio'}"}
+    models = get_lmstudio_models()
+    found_key = _find_lmstudio_model_key(model_name, models)
+    model_key = found_key or model_name
+
+    if not found_key:
+        try:
+            job = _json_request(
+                f"{lmstudio_native_api_base()}/api/v1/models/download",
+                method="POST",
+                payload={"model": model_name},
+                headers=auth_header,
+                timeout=30,
+            )
+        except urllib.error.HTTPError as e:
+            detail = e.read().decode("utf-8", errors="replace")[-2000:]
+            raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {detail or e.reason}")
+        except Exception as e:
+            raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {e}")
+
+        status = str(job.get("status") or "")
+        job_id = str(job.get("job_id") or "")
+        if status not in {"completed", "already_downloaded"} and job_id:
+            deadline = time.time() + 3600
+            while time.time() < deadline:
+                polled = _json_request(
+                    f"{lmstudio_native_api_base()}/api/v1/models/download/status/{job_id}",
+                    headers=auth_header,
+                    timeout=30,
+                )
+                polled_status = str(polled.get("status") or "")
+                if polled_status == "completed":
+                    break
+                if polled_status == "failed":
+                    raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {polled}")
+                time.sleep(2)
+            else:
+                raise ModelRuntimeError(status_code=408, detail="LM Studio 모델 다운로드 시간이 초과되었습니다.")
+
+        models = get_lmstudio_models(force=True)
+        model_key = _find_lmstudio_model_key(model_name, models) or model_name
+
+    target = next((item for item in models if isinstance(item, dict) and item.get("key") == model_key), None)
+    loaded_instances = target.get("loaded_instances") if isinstance(target, dict) else None
+    if loaded_instances:
+        return {"provider": "lmstudio", "model": model_name, "resolved_model": model_key, "server_ready": True, "cached": True}
+
+    try:
+        loaded = _json_request(
+            f"{lmstudio_native_api_base()}/api/v1/models/load",
+            method="POST",
+            payload={"model": model_key, "context_length": 4096},
+            headers=auth_header,
+            timeout=120,
+        )
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace")[-2000:]
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {detail or e.reason}")
+    except Exception as e:
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {e}")
+
+    if str(loaded.get("status") or "") != "loaded":
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {loaded}")
+
+    return {
+        "provider": "lmstudio",
+        "model": model_name,
+        "resolved_model": model_key,
+        "instance_id": loaded.get("instance_id"),
+        "server_ready": True,
+        "cached": False,
+    }
+
+def engine_support_status(engine: str) -> Dict[str, Any]:
+    return _engine_support_status(engine)
+
+def pull_ollama_model_with_progress(model_name: str, progress_emit=None) -> Dict[str, Any]:
+    return _pull_ollama_model_with_progress(model_name, progress_emit)
+
+
+def get_ollama_pulled_models() -> set:
+    return _get_ollama_pulled_models()
+
+
+def get_openai_compatible_server_models(provider: str) -> List[str]:
+    return _get_openai_compatible_server_models(provider)
+
+
+def ensure_ollama_server() -> None:
+    return _ensure_ollama_server()
+
+
+def wait_for_openai_compatible_server(provider: str, model_name: Optional[str] = None, timeout: int = 45) -> bool:
+    return _wait_for_openai_compatible_server(provider, model_name=model_name, timeout=timeout)
+
+
+def ensure_vllm_server(model_name: str) -> None:
+    return _ensure_vllm_server(model_name)
+
+
+def ensure_llamacpp_server(model_name: str) -> None:
+    return _ensure_llamacpp_server(model_name)
+
+
+def _safe_engine_install_plan(
+    engine: str,
+    *,
+    base_dir: Path,
+) -> Optional[Dict[str, Any]]:
+    try:
+        return _engine_install_plan(engine, base_dir=base_dir)
+    except Exception:
+        return None
+
+
+def engine_installed(engine: str) -> bool:
+    if engine == "local_mlx":
+        return bool(
+            importlib.util.find_spec("mlx")
+            and (importlib.util.find_spec("mlx_vlm") or importlib.util.find_spec("mlx_lm"))
+        )
+    if engine == "ollama":
+        return local_binary("ollama") is not None
+    if engine == "vllm":
+        return vllm_metal_python() is not None or vllm_executable() is not None or importlib.util.find_spec("vllm") is not None
+    if engine == "lmstudio":
+        return find_lmstudio_cli() is not None or Path("/Applications/LM Studio.app").exists()
+    if engine == "llamacpp":
+        return shutil.which("llama-server") is not None
+    if engine in {"openai", "openrouter", "groq", "together", "xai"}:
+        return AsyncOpenAI is not None
+    return False

package/latticeai/services/model_runtime/loading.py

@@ -0,0 +1,178 @@
+"""Model identity, engine readiness, and the load entrypoints.
+
+The path from a string a user clicked to a loaded model: resolve engine
+aliases into one canonical identity, make sure the engine that identity needs
+is installed, then hand off to ``latticeai.services.model_loading`` for the
+load itself (blocking and streaming forms) and to
+``latticeai.services.model_engines`` for the post-load smoke test.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any, AsyncIterator, Dict, Optional
+
+from latticeai.core.model_resolution import ModelResolution as _ModelResolution
+from latticeai.models.router import OPENAI_COMPATIBLE_PROVIDERS, ensure_mlx_runtime
+from latticeai.services.model_catalog import ENGINE_INSTALLERS, MODEL_ENGINE_ALIASES
+from latticeai.services.model_errors import ModelRuntimeError
+from latticeai.services.model_runtime.engines import (
+    engine_installed,
+    engine_support_status,
+)
+from latticeai.services.model_runtime.state import ModelRuntimeState
+from latticeai.services.model_runtime.status import install_engine
+
+
+def _resolve_model_alias(model_id: str, engine: Optional[str] = None) -> str:
+    raw = model_id.strip()
+    engine_hint = (engine or "").strip().lower()
+    provider: Optional[str] = None
+    model_name = raw
+    if ":" in raw:
+        prefix, rest = raw.split(":", 1)
+        prefix = prefix.strip().lower()
+        if prefix in {"ollama", "vllm", "lmstudio", "llamacpp", "local_mlx", "mlx"}:
+            provider = "local_mlx" if prefix in {"local_mlx", "mlx"} else prefix
+            model_name = rest.strip()
+    provider = provider or ("local_mlx" if engine_hint in {"", "local_mlx", "mlx"} else engine_hint)
+    aliases = MODEL_ENGINE_ALIASES.get(model_name.lower())
+    if not aliases:
+        return raw
+    mapped = aliases.get(provider)
+    if not mapped:
+        return raw
+    return mapped if provider == "local_mlx" else f"{provider}:{mapped}"
+
+
+def normalize_local_model_request(model_id: str, engine: Optional[str] = None) -> str:
+    model_id = _resolve_model_alias(model_id, engine)
+    engine = (engine or "").strip().lower()
+    if engine in {"local_mlx", "mlx"} and model_id.startswith(("local_mlx:", "mlx:")):
+        return model_id.split(":", 1)[1].strip()
+    if engine and engine not in {"local_mlx", "mlx"} and ":" not in model_id:
+        return f"{engine}:{model_id}"
+    return model_id
+
+
+def ensure_engine_ready(engine: str, *, state: ModelRuntimeState) -> Dict[str, Any]:
+    engine = "local_mlx" if engine == "mlx" else engine
+    if engine not in ENGINE_INSTALLERS and engine not in OPENAI_COMPATIBLE_PROVIDERS:
+        raise ModelRuntimeError(status_code=400, detail=f"지원하지 않는 엔진입니다: {engine}")
+    support = engine_support_status(engine)
+    if not support["supported"]:
+        raise ModelRuntimeError(status_code=400, detail=str(support["reason"]))
+
+    if engine_installed(engine):
+        if engine == "local_mlx":
+            ensure_mlx_runtime()
+        return {"engine": engine, "installed": True, "installed_now": False}
+
+    if engine not in ENGINE_INSTALLERS:
+        raise ModelRuntimeError(status_code=400, detail=f"{engine} 엔진 설치 방법이 등록되어 있지 않습니다.")
+
+    result = install_engine(engine, state=state)
+    if result.get("returncode") not in (0, None) or not engine_installed(engine):
+        detail = result.get("stderr") or result.get("stdout") or f"{engine} 설치에 실패했습니다."
+        raise ModelRuntimeError(status_code=500, detail=str(detail)[-2000:])
+
+    if engine == "local_mlx":
+        ensure_mlx_runtime()
+    return {"engine": engine, "installed": True, "installed_now": True, "install": result}
+
+
+def build_model_resolution(
+    input_id: str,
+    engine: Optional[str],
+    *,
+    user_email: Optional[str] = None,
+    display_name: Optional[str] = None,
+) -> _ModelResolution:
+    """피드백 #1/#2 공용 ModelResolution 생성기.
+
+    사용자가 클릭한 input_id + engine 힌트를 받아 모든 단계가 공유할
+    canonical identity를 만든다.
+    """
+    normalized = normalize_local_model_request(input_id, engine)
+    return _ModelResolution.from_request(
+        normalized,
+        engine=engine,
+        user_email=user_email,
+        display_name=display_name or input_id,
+        engine_aliases=MODEL_ENGINE_ALIASES,
+    )
+
+
+_LOCAL_SMOKE_ENGINES = {"local_mlx", "ollama", "vllm", "lmstudio", "llamacpp"}
+
+
+async def _smoke_test_loaded_model(
+    resolution: _ModelResolution,
+    *,
+    api_key_override: Optional[str] = None,
+    state: ModelRuntimeState,
+) -> Dict[str, Any]:
+    # Delegated to model_engines for server decomp
+    # Absolute since v11.3.0: this file moved one level down into the
+    # model_runtime package, so ``.model_engines`` would resolve inside it.
+    from latticeai.services.model_engines import (
+        _smoke_test_loaded_model as _impl_smoke,
+    )
+    return await _impl_smoke(
+        resolution,
+        api_key_override=api_key_override,
+        model_router=state.router,
+    )
+
+
+async def prepare_and_load_model(
+    model_id: str,
+    request: Any,
+    engine: Optional[str] = None,
+    user_email: Optional[str] = None,
+    adapter_path: Optional[str] = None,
+    draft_model_id: Optional[str] = None,
+    allow_download: bool = False,
+    *,
+    state: ModelRuntimeState,
+) -> Dict[str, Any]:
+    from latticeai.services.model_loading import prepare_and_load_model as _impl
+
+    return await _impl(
+        model_id,
+        request,
+        engine=engine,
+        user_email=user_email,
+        adapter_path=adapter_path,
+        draft_model_id=draft_model_id,
+        allow_download=allow_download,
+        runtime_state=state,
+    )
+
+
+def sse_event(event: str, data: Dict[str, Any]) -> str:
+    return f"event: {event}\ndata: {json.dumps(data, ensure_ascii=False)}\n\n"
+
+
+async def prepare_and_load_model_stream(
+    model_id: str,
+    request: Any,
+    engine: Optional[str] = None,
+    user_email: Optional[str] = None,
+    allow_download: bool = False,
+    *,
+    state: ModelRuntimeState,
+) -> AsyncIterator[str]:
+    from latticeai.services.model_loading import (
+        prepare_and_load_model_stream as _impl,
+    )
+
+    async for event in _impl(
+        model_id,
+        request,
+        engine=engine,
+        user_email=user_email,
+        allow_download=allow_download,
+        runtime_state=state,
+    ):
+        yield event

package/latticeai/services/model_runtime/service.py

@@ -0,0 +1,129 @@
+"""The bound runtime service — one application's model operations.
+
+Everything above is a free function taking an explicit ``state``. This is the
+object the composition root holds: it carries that state plus the only piece of
+per-application operational data (the cloud verification cache), so a second
+ASGI app in the same process starts with an empty cache rather than inheriting
+probe results it never ran.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, AsyncIterator, Dict, List, Optional
+
+from latticeai.services.model_runtime.cloud import verify_cloud_models
+from latticeai.services.model_runtime.loading import (
+    prepare_and_load_model,
+    prepare_and_load_model_stream,
+)
+from latticeai.services.model_runtime.state import (
+    ModelRuntimeState,
+    create_model_runtime_state,
+)
+from latticeai.services.model_runtime.status import (
+    engine_status,
+    install_engine,
+    runtime_features,
+)
+
+
+def configure_model_runtime(**deps: Any) -> "ModelRuntimeService":
+    """Compatibility factory returning an isolated, bound runtime service.
+
+    The historical function mutated process-wide module globals.  Keeping the
+    import path while returning a service preserves practical construction
+    compatibility without ambient state or cross-application leakage.
+    """
+
+    return ModelRuntimeService(create_model_runtime_state(**deps))
+
+@dataclass(slots=True)
+class ModelRuntimeService:
+    """Bound model operations for one explicitly configured application.
+
+    All configuration and app-owned callables live on ``state``. Operational
+    verification cache data belongs to this service instance, so creating a
+    second ASGI app cannot inherit credentials, routers, or probe results from
+    the first one.
+    """
+
+    state: ModelRuntimeState
+    _cloud_verify_cache: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+
+    def runtime_features(self) -> Dict[str, Any]:
+        return runtime_features(state=self.state)
+
+    def engine_status(self) -> List[Dict[str, Any]]:
+        return engine_status(
+            state=self.state,
+            cloud_verify_cache=self._cloud_verify_cache,
+        )
+
+    def install_engine(
+        self,
+        engine: str,
+        confirmation_token: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        return install_engine(
+            engine,
+            confirmation_token=confirmation_token,
+            state=self.state,
+        )
+
+    async def verify_cloud_models(
+        self,
+        force: bool = False,
+        provider_filter: Optional[str] = None,
+    ) -> Dict[str, Dict[str, Any]]:
+        return await verify_cloud_models(
+            force=force,
+            provider_filter=provider_filter,
+            state=self.state,
+            cache=self._cloud_verify_cache,
+        )
+
+    async def prepare_and_load_model(
+        self,
+        model_id: str,
+        request: Any,
+        engine: Optional[str] = None,
+        user_email: Optional[str] = None,
+        adapter_path: Optional[str] = None,
+        draft_model_id: Optional[str] = None,
+        allow_download: bool = False,
+    ) -> Dict[str, Any]:
+        return await prepare_and_load_model(
+            model_id,
+            request,
+            engine=engine,
+            user_email=user_email,
+            adapter_path=adapter_path,
+            draft_model_id=draft_model_id,
+            allow_download=allow_download,
+            state=self.state,
+        )
+
+    async def prepare_and_load_model_stream(
+        self,
+        model_id: str,
+        request: Any,
+        engine: Optional[str] = None,
+        user_email: Optional[str] = None,
+        allow_download: bool = False,
+    ) -> AsyncIterator[str]:
+        async for event in prepare_and_load_model_stream(
+            model_id,
+            request,
+            engine=engine,
+            user_email=user_email,
+            allow_download=allow_download,
+            state=self.state,
+        ):
+            yield event
+
+
+def build_model_runtime(**deps: Any) -> ModelRuntimeService:
+    """Build the application's isolated model runtime service."""
+
+    return ModelRuntimeService(create_model_runtime_state(**deps))