ltcai 11.1.0 → 11.4.0

package/latticeai/services/model_runtime/download.py

@@ -0,0 +1,282 @@
+"""Hugging Face model presence, download, and download progress.
+
+Answers one question — "are the weights actually on this disk?" — and, when
+they are not and the caller has consent, fetches them while emitting a progress
+payload per file. The readiness check is deliberately format-aware: a GGUF
+repo, an MLX 4-bit repo and a vLLM bf16 repo each prove completeness
+differently.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import logging
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from latticeai.core.quiet import quiet
+from latticeai.models.router import hf_cache_model_dir, hf_model_dir
+from latticeai.services.model_errors import ModelRuntimeError
+
+
+def hf_model_ready(repo_id: str, provider: str = "local_mlx") -> bool:
+    model_dir = hf_model_dir(repo_id)
+    if provider in {"local_mlx", "vllm"} and (not model_dir.exists() or not model_dir.is_dir()):
+        hf_cache_repo = Path.home() / ".cache" / "huggingface" / "hub" / f"models--{repo_id.replace('/', '--')}"
+        if hf_cache_repo.exists() and any(hf_cache_repo.glob("snapshots/*")):
+            if provider == "vllm":
+                return True
+            return hf_cache_model_dir(repo_id) is not None
+        return False
+    if not model_dir.exists() or not model_dir.is_dir():
+        return False
+    if provider == "llamacpp":
+        return any(model_dir.rglob("*.gguf"))
+    has_config = (model_dir / "config.json").exists()
+    has_weights = any(model_dir.glob("*.safetensors")) or any(model_dir.glob("*.bin"))
+    has_tokenizer = (
+        (model_dir / "tokenizer.json").exists()
+        or (model_dir / "tokenizer.model").exists()
+        or (model_dir / "tokenizer_config.json").exists()
+    )
+    return has_config and has_weights and has_tokenizer
+
+
+def model_download_progress_payload(
+    stage: str,
+    message: str,
+    *,
+    percent: Optional[float] = None,
+    detail: Optional[str] = None,
+    downloaded_bytes: Optional[int] = None,
+    total_bytes: Optional[int] = None,
+    eta_seconds: Optional[float] = None,
+    file: Optional[str] = None,
+    indeterminate: bool = False,
+) -> Dict[str, Any]:
+    payload: Dict[str, Any] = {
+        "stage": stage,
+        "message": message,
+        "indeterminate": indeterminate,
+        "ts": time.time(),
+    }
+    if percent is not None:
+        payload["percent"] = max(0, min(100, round(float(percent), 1)))
+    if detail:
+        payload["detail"] = detail
+    if downloaded_bytes is not None:
+        payload["downloaded_bytes"] = max(0, int(downloaded_bytes))
+    if total_bytes is not None:
+        payload["total_bytes"] = max(0, int(total_bytes))
+    if eta_seconds is not None:
+        payload["eta_seconds"] = max(0, round(float(eta_seconds)))
+    if file:
+        payload["file"] = file
+    return payload
+
+
+def estimate_eta_seconds(started_at: float, percent: Optional[float]) -> Optional[float]:
+    if percent is None or percent <= 0 or percent >= 100:
+        return None
+    elapsed = max(0.0, time.time() - started_at)
+    return elapsed * (100.0 - percent) / percent
+
+
+def hf_repo_files_with_sizes(repo_id: str) -> List[Dict[str, Any]]:
+    from huggingface_hub import HfApi
+
+    api = HfApi()
+    try:
+        info = api.model_info(repo_id, files_metadata=True)
+        files = []
+        for sibling in getattr(info, "siblings", []) or []:
+            name = str(getattr(sibling, "rfilename", "") or "").strip()
+            if not name or name.endswith("/"):
+                continue
+            files.append({"name": name, "size": int(getattr(sibling, "size", 0) or 0)})
+        if files:
+            return files
+    except TypeError:
+        quiet()
+    except Exception as e:
+        logging.warning("huggingface model_info failed for %s: %s", repo_id, e)
+
+    return [{"name": str(name), "size": 0} for name in api.list_repo_files(repo_id) if str(name).strip()]
+
+
+def download_hf_model(
+    repo_id: str,
+    provider: str = "local_mlx",
+    progress_emit=None,
+) -> Dict[str, Any]:
+    if importlib.util.find_spec("huggingface_hub") is None:
+        raise ModelRuntimeError(status_code=400, detail="huggingface_hub가 없습니다. 먼저 MLX runtime 설치를 진행해 주세요.")
+
+    target_dir = hf_model_dir(repo_id)
+    if hf_model_ready(repo_id, provider):
+        cached_dir = hf_cache_model_dir(repo_id) if provider == "local_mlx" else None
+        resolved_dir = cached_dir or target_dir
+        if progress_emit:
+            progress_emit(model_download_progress_payload(
+                "download",
+                "이미 다운로드된 모델을 확인했습니다.",
+                percent=100,
+                downloaded_bytes=0,
+                total_bytes=0,
+                eta_seconds=0,
+            ))
+        return {"model": repo_id, "path": str(resolved_dir), "cached": True}
+
+    target_dir.mkdir(parents=True, exist_ok=True)
+    try:
+        from huggingface_hub import hf_hub_download
+
+        started_at = time.time()
+        all_files = hf_repo_files_with_sizes(repo_id)
+        if provider == "llamacpp":
+            ggufs = sorted(
+                [item for item in all_files if str(item["name"]).lower().endswith(".gguf")],
+                key=lambda item: str(item["name"]),
+            )
+            if not ggufs:
+                raise RuntimeError("GGUF 파일을 찾지 못했습니다.")
+            preference = ("q4_k_m", "q4_0", "q4_k_s", "q3_k_m", "q2_k")
+            selected_files = [
+                next(
+                    (item for pref in preference for item in ggufs if pref in str(item["name"]).lower()),
+                    ggufs[0],
+                )
+            ]
+        else:
+            selected_files = all_files
+
+        total_bytes = sum(int(item.get("size") or 0) for item in selected_files) or None
+        downloaded_bytes = 0
+        total_files = max(1, len(selected_files))
+        if progress_emit:
+            progress_emit(model_download_progress_payload(
+                "download",
+                "모델 파일 정보를 확인했습니다.",
+                percent=0,
+                downloaded_bytes=0,
+                total_bytes=total_bytes,
+                indeterminate=total_bytes is None,
+            ))
+
+        for index, item in enumerate(selected_files, start=1):
+            filename = str(item["name"])
+            size = int(item.get("size") or 0)
+            tqdm_class = None
+            if progress_emit:
+                current_percent = (
+                    (downloaded_bytes / total_bytes) * 100 if total_bytes else ((index - 1) / total_files) * 100
+                )
+                progress_emit(model_download_progress_payload(
+                    "download",
+                    "모델 다운로드 중입니다.",
+                    percent=current_percent,
+                    detail=filename,
+                    downloaded_bytes=downloaded_bytes,
+                    total_bytes=total_bytes,
+                    eta_seconds=estimate_eta_seconds(started_at, current_percent),
+                    file=filename,
+                    indeterminate=total_bytes is None and total_files <= 1,
+                ))
+                try:
+                    from tqdm.auto import tqdm as base_tqdm
+
+                    downloaded_before = downloaded_bytes
+                    last_emit = {"at": 0.0, "percent": -1.0}
+
+                    def emit_byte_progress(
+                        done_bytes: float,
+                        # Bound per iteration: this callback outlives the loop
+                        # body when a download runs long, and late binding
+                        # would report every file's progress against the last
+                        # file's offsets.
+                        downloaded_before: int = downloaded_before,
+                        size: Any = size,
+                        index: int = index,
+                        last_emit: dict = last_emit,
+                        filename: str = filename,
+                    ) -> None:
+                        done = max(0, int(done_bytes or 0))
+                        if total_bytes:
+                            aggregate = min(total_bytes, downloaded_before + done)
+                            percent = (aggregate / total_bytes) * 100
+                        else:
+                            file_total = size or done
+                            file_ratio = min(1.0, done / file_total) if file_total else 0.0
+                            aggregate = downloaded_before + done
+                            percent = ((index - 1) + file_ratio) / total_files * 100
+                        now = time.time()
+                        if percent < 100 and now - last_emit["at"] < 0.5 and percent - last_emit["percent"] < 0.3:
+                            return
+                        last_emit["at"] = now
+                        last_emit["percent"] = percent
+                        progress_emit(model_download_progress_payload(
+                            "download",
+                            "모델 다운로드 중입니다.",
+                            percent=percent,
+                            detail=filename,
+                            downloaded_bytes=aggregate,
+                            total_bytes=total_bytes,
+                            eta_seconds=estimate_eta_seconds(started_at, percent),
+                            file=filename,
+                            indeterminate=total_bytes is None and total_files <= 1,
+                        ))
+
+                    class ProgressTqdm(base_tqdm):
+                        def update(self, n=1):
+                            result = super().update(n)
+                            emit_byte_progress(float(getattr(self, "n", 0) or 0))
+                            return result
+
+                    tqdm_class = ProgressTqdm
+                except Exception:
+                    tqdm_class = None
+            local_path = hf_hub_download(
+                repo_id=repo_id,
+                filename=filename,
+                local_dir=str(target_dir),
+                tqdm_class=tqdm_class,
+            )
+            if size <= 0:
+                try:
+                    size = Path(local_path).stat().st_size
+                except OSError:
+                    size = 0
+            downloaded_bytes += size
+            if progress_emit:
+                current_percent = (
+                    (downloaded_bytes / total_bytes) * 100 if total_bytes else (index / total_files) * 100
+                )
+                progress_emit(model_download_progress_payload(
+                    "download",
+                    "모델 다운로드 중입니다.",
+                    percent=current_percent,
+                    detail=filename,
+                    downloaded_bytes=downloaded_bytes,
+                    total_bytes=total_bytes,
+                    eta_seconds=estimate_eta_seconds(started_at, current_percent),
+                    file=filename,
+                    indeterminate=False,
+                ))
+
+        if progress_emit:
+            progress_emit(model_download_progress_payload(
+                "download",
+                "모델 다운로드가 완료되었습니다.",
+                percent=100,
+                downloaded_bytes=downloaded_bytes,
+                total_bytes=total_bytes or downloaded_bytes,
+                eta_seconds=0,
+            ))
+    except Exception as e:
+        raise ModelRuntimeError(status_code=500, detail=f"{repo_id} 다운로드 실패: {str(e)[-2000:]}")
+
+    if not hf_model_ready(repo_id, provider):
+        raise ModelRuntimeError(status_code=500, detail=f"{repo_id} 다운로드가 완료되지 않았습니다. 모델 파일을 찾지 못했습니다.")
+
+    return {"model": repo_id, "path": str(target_dir), "cached": False}

package/latticeai/services/model_runtime/engines.py

@@ -0,0 +1,341 @@
+"""Local engine discovery, server hand-off, and the LM Studio HTTP client.
+
+Thin wrappers over ``latticeai.services.model_engines`` that keep the historical
+``model_runtime`` import path alive, plus the one piece of real logic that never
+belonged in the engine layer: the LM Studio native API (list / download / load)
+and its short-lived model cache.
+
+``_LMSTUDIO_MODELS_CACHE`` and ``_LMSTUDIO_MODELS_CACHE_TS`` are rebindable
+module state and therefore live here and nowhere else — the package
+``__init__`` deliberately does not re-export them, because a
+``from … import`` copy would freeze at import time and quietly disagree with
+the live value.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import shutil
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+from latticeai.models.router import OPENAI_COMPATIBLE_PROVIDERS, AsyncOpenAI
+from latticeai.services.model_engines import (
+    LOCAL_SERVER_PROCESSES as _LOCAL_SERVER_PROCESSES,
+)
+from latticeai.services.model_engines import (
+    engine_install_plan as _engine_install_plan,
+)
+from latticeai.services.model_engines import (
+    engine_support_status as _engine_support_status,
+)
+from latticeai.services.model_engines import (
+    ensure_llamacpp_server as _ensure_llamacpp_server,
+)
+from latticeai.services.model_engines import (
+    ensure_lmstudio_server as _ensure_lmstudio_server,
+)
+from latticeai.services.model_engines import (
+    ensure_ollama_server as _ensure_ollama_server,
+)
+from latticeai.services.model_engines import (
+    ensure_vllm_server as _ensure_vllm_server,
+)
+from latticeai.services.model_engines import (
+    find_lmstudio_cli as _find_lmstudio_cli,
+)
+from latticeai.services.model_engines import (
+    get_ollama_pulled_models as _get_ollama_pulled_models,
+)
+from latticeai.services.model_engines import (
+    get_openai_compatible_server_models as _get_openai_compatible_server_models,
+)
+from latticeai.services.model_engines import (
+    local_binary as _local_binary,
+)
+from latticeai.services.model_engines import (
+    pull_ollama_model_with_progress as _pull_ollama_model_with_progress,
+)
+from latticeai.services.model_engines import (
+    vllm_executable as _vllm_executable,
+)
+from latticeai.services.model_engines import (
+    vllm_metal_python as _vllm_metal_python,
+)
+from latticeai.services.model_engines import (
+    wait_for_openai_compatible_server as _wait_for_openai_compatible_server,
+)
+from latticeai.services.model_engines import (
+    windows_binary_candidates as _windows_binary_candidates,
+)
+from latticeai.services.model_errors import ModelRuntimeError
+
+
+def _update_env_file(env_file: Path, key: str, value: str) -> None:
+    lines = []
+    found = False
+    if env_file.exists():
+        for line in env_file.read_text(encoding="utf-8").splitlines():
+            if line.startswith(f"{key}="):
+                lines.append(f"{key}={value}")
+                found = True
+            else:
+                lines.append(line)
+    if not found:
+        lines.append(f"{key}={value}")
+    env_file.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+LOCAL_SERVER_PROCESSES = _LOCAL_SERVER_PROCESSES
+VLLM_METAL_ENV = Path.home() / ".venv-vllm-metal"
+VLLM_METAL_BIN = VLLM_METAL_ENV / "bin" / "vllm"
+VLLM_METAL_PYTHON = VLLM_METAL_ENV / "bin" / "python"
+LMSTUDIO_BUNDLED_CLI = Path("/Applications/LM Studio.app/Contents/Resources/app/.webpack/lms")
+
+def windows_binary_candidates(binary: str) -> List[Path]:
+    return _windows_binary_candidates(binary)
+
+
+def local_binary(binary: str) -> Optional[str]:
+    return _local_binary(binary)
+
+
+def find_lmstudio_cli() -> Optional[str]:
+    return _find_lmstudio_cli()
+
+
+def vllm_executable() -> Optional[str]:
+    return _vllm_executable()
+
+
+def vllm_metal_python() -> Optional[str]:
+    return _vllm_metal_python()
+
+
+def _json_request(
+    url: str,
+    *,
+    method: str = "GET",
+    payload: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, str]] = None,
+    timeout: float = 10.0,
+) -> Dict[str, Any]:
+    data = None
+    req_headers = dict(headers or {})
+    if payload is not None:
+        data = json.dumps(payload).encode("utf-8")
+        req_headers.setdefault("Content-Type", "application/json")
+    req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
+    with urllib.request.urlopen(req, timeout=timeout) as res:
+        raw = res.read().decode("utf-8", errors="replace")
+    if not raw.strip():
+        return {}
+    return json.loads(raw)
+
+
+def lmstudio_api_base() -> str:
+    return (os.getenv("LMSTUDIO_BASE_URL") or OPENAI_COMPATIBLE_PROVIDERS["lmstudio"]["base_url"]).rstrip("/")
+
+
+def lmstudio_native_api_base() -> str:
+    base = lmstudio_api_base()
+    return base[:-3] if base.endswith("/v1") else base
+
+
+def ensure_lmstudio_server() -> None:
+    return _ensure_lmstudio_server()
+
+
+_LMSTUDIO_MODELS_CACHE: List[Dict[str, Any]] = []
+_LMSTUDIO_MODELS_CACHE_TS: float = 0.0
+_LMSTUDIO_MODELS_CACHE_TTL: float = 10.0
+
+
+def get_lmstudio_models(*, force: bool = False) -> List[Dict[str, Any]]:
+    global _LMSTUDIO_MODELS_CACHE, _LMSTUDIO_MODELS_CACHE_TS
+    if not force and time.monotonic() - _LMSTUDIO_MODELS_CACHE_TS < _LMSTUDIO_MODELS_CACHE_TTL:
+        return _LMSTUDIO_MODELS_CACHE
+    try:
+        payload = _json_request(
+            f"{lmstudio_native_api_base()}/api/v1/models",
+            headers={"Authorization": f"Bearer {os.getenv('LMSTUDIO_API_KEY') or 'lmstudio'}"},
+            timeout=2.5,
+        )
+    except Exception:
+        return _LMSTUDIO_MODELS_CACHE
+    models = payload.get("models")
+    _LMSTUDIO_MODELS_CACHE = models if isinstance(models, list) else []
+    _LMSTUDIO_MODELS_CACHE_TS = time.monotonic()
+    return _LMSTUDIO_MODELS_CACHE
+
+
+def _lmstudio_candidate_keys(model_name: str) -> List[str]:
+    raw = model_name.strip()
+    if not raw:
+        return []
+    slug = raw.split("/")[-1].lower()
+    slug = slug.replace("-gguf", "").replace("-awq", "")
+    parts = [p for p in slug.split("-") if p]
+    candidates = [raw.lower(), slug]
+    if parts:
+        candidates.append("-".join(parts[: min(4, len(parts))]))
+    return list(dict.fromkeys(candidates))
+
+
+def _find_lmstudio_model_key(model_name: str, models: List[Dict[str, Any]]) -> Optional[str]:
+    if not models:
+        return None
+    candidate_keys = _lmstudio_candidate_keys(model_name)
+    exact = []
+    fuzzy = []
+    for item in models:
+        key = str(item.get("key") or "").strip()
+        display_name = str(item.get("display_name") or "").strip()
+        haystacks = [key.lower(), display_name.lower()]
+        if any(raw == key.lower() for raw in candidate_keys):
+            exact.append(key)
+            continue
+        if any(token and token in hay for token in candidate_keys for hay in haystacks):
+            fuzzy.append(key)
+    return next(iter(exact or fuzzy), None)
+
+
+def ensure_lmstudio_model(model_name: str) -> Dict[str, Any]:
+    ensure_lmstudio_server()
+    auth_header = {"Authorization": f"Bearer {os.getenv('LMSTUDIO_API_KEY') or 'lmstudio'}"}
+    models = get_lmstudio_models()
+    found_key = _find_lmstudio_model_key(model_name, models)
+    model_key = found_key or model_name
+
+    if not found_key:
+        try:
+            job = _json_request(
+                f"{lmstudio_native_api_base()}/api/v1/models/download",
+                method="POST",
+                payload={"model": model_name},
+                headers=auth_header,
+                timeout=30,
+            )
+        except urllib.error.HTTPError as e:
+            detail = e.read().decode("utf-8", errors="replace")[-2000:]
+            raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {detail or e.reason}")
+        except Exception as e:
+            raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {e}")
+
+        status = str(job.get("status") or "")
+        job_id = str(job.get("job_id") or "")
+        if status not in {"completed", "already_downloaded"} and job_id:
+            deadline = time.time() + 3600
+            while time.time() < deadline:
+                polled = _json_request(
+                    f"{lmstudio_native_api_base()}/api/v1/models/download/status/{job_id}",
+                    headers=auth_header,
+                    timeout=30,
+                )
+                polled_status = str(polled.get("status") or "")
+                if polled_status == "completed":
+                    break
+                if polled_status == "failed":
+                    raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 다운로드 실패: {polled}")
+                time.sleep(2)
+            else:
+                raise ModelRuntimeError(status_code=408, detail="LM Studio 모델 다운로드 시간이 초과되었습니다.")
+
+        models = get_lmstudio_models(force=True)
+        model_key = _find_lmstudio_model_key(model_name, models) or model_name
+
+    target = next((item for item in models if isinstance(item, dict) and item.get("key") == model_key), None)
+    loaded_instances = target.get("loaded_instances") if isinstance(target, dict) else None
+    if loaded_instances:
+        return {"provider": "lmstudio", "model": model_name, "resolved_model": model_key, "server_ready": True, "cached": True}
+
+    try:
+        loaded = _json_request(
+            f"{lmstudio_native_api_base()}/api/v1/models/load",
+            method="POST",
+            payload={"model": model_key, "context_length": 4096},
+            headers=auth_header,
+            timeout=120,
+        )
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace")[-2000:]
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {detail or e.reason}")
+    except Exception as e:
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {e}")
+
+    if str(loaded.get("status") or "") != "loaded":
+        raise ModelRuntimeError(status_code=500, detail=f"LM Studio 모델 로드 실패: {loaded}")
+
+    return {
+        "provider": "lmstudio",
+        "model": model_name,
+        "resolved_model": model_key,
+        "instance_id": loaded.get("instance_id"),
+        "server_ready": True,
+        "cached": False,
+    }
+
+def engine_support_status(engine: str) -> Dict[str, Any]:
+    return _engine_support_status(engine)
+
+def pull_ollama_model_with_progress(model_name: str, progress_emit=None) -> Dict[str, Any]:
+    return _pull_ollama_model_with_progress(model_name, progress_emit)
+
+
+def get_ollama_pulled_models() -> set:
+    return _get_ollama_pulled_models()
+
+
+def get_openai_compatible_server_models(provider: str) -> List[str]:
+    return _get_openai_compatible_server_models(provider)
+
+
+def ensure_ollama_server() -> None:
+    return _ensure_ollama_server()
+
+
+def wait_for_openai_compatible_server(provider: str, model_name: Optional[str] = None, timeout: int = 45) -> bool:
+    return _wait_for_openai_compatible_server(provider, model_name=model_name, timeout=timeout)
+
+
+def ensure_vllm_server(model_name: str) -> None:
+    return _ensure_vllm_server(model_name)
+
+
+def ensure_llamacpp_server(model_name: str) -> None:
+    return _ensure_llamacpp_server(model_name)
+
+
+def _safe_engine_install_plan(
+    engine: str,
+    *,
+    base_dir: Path,
+) -> Optional[Dict[str, Any]]:
+    try:
+        return _engine_install_plan(engine, base_dir=base_dir)
+    except Exception:
+        return None
+
+
+def engine_installed(engine: str) -> bool:
+    if engine == "local_mlx":
+        return bool(
+            importlib.util.find_spec("mlx")
+            and (importlib.util.find_spec("mlx_vlm") or importlib.util.find_spec("mlx_lm"))
+        )
+    if engine == "ollama":
+        return local_binary("ollama") is not None
+    if engine == "vllm":
+        return vllm_metal_python() is not None or vllm_executable() is not None or importlib.util.find_spec("vllm") is not None
+    if engine == "lmstudio":
+        return find_lmstudio_cli() is not None or Path("/Applications/LM Studio.app").exists()
+    if engine == "llamacpp":
+        return shutil.which("llama-server") is not None
+    if engine in {"openai", "openrouter", "groq", "together", "xai"}:
+        return AsyncOpenAI is not None
+    return False