ltcai 11.2.0 → 11.4.0

package/latticeai/core/embedding_providers/captions.py

@@ -0,0 +1,162 @@
+"""Descriptions of an image — what a vision-language model said, or nothing.
+
+Every "clever" fallback here would be a lie: ``Image IMG_2381.png (JPEG
+3024x4032)`` is metadata wearing a caption's clothes, and once it is in the
+graph nothing downstream can tell it from a model's actual description. So the
+base :class:`VisionCaptioner` returns ``None`` and reports itself unavailable,
+and ``vision_caption_port`` hands Brain Core ``None`` rather than a decoy.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Dict, Optional, Tuple
+
+from .base import EmbeddingUnavailable, _load_dotted
+
+VISION_CAPTION_TARGET_ENV = "LATTICEAI_VISION_CAPTION_TARGET"
+
+
+class VisionCaptioner:
+    """Describes an image — the null implementation, which describes nothing.
+
+    Every "clever" fallback here is a lie: ``Image IMG_2381.png (JPEG
+    3024x4032)`` is metadata wearing a caption's clothes, and once it is in the
+    graph nothing downstream can tell it from a model's actual description. So
+    the base class returns ``None`` and :meth:`available` says ``False``.
+    """
+
+    provider = "none"
+    model_id = ""
+
+    def available(self) -> bool:
+        return False
+
+    def caption(self, path: str, *, prompt: str = "") -> Optional[str]:
+        return None
+
+    def health(self) -> Dict[str, Any]:
+        return {"status": "unavailable", "detail": "no vision-language model is loaded"}
+
+    def metadata(self) -> Dict[str, Any]:
+        return {
+            "provider": self.provider,
+            "model": self.model_id,
+            "available": self.available(),
+        }
+
+
+#: Short, literal instruction — a caption is a description, not an essay.
+DEFAULT_CAPTION_PROMPT = "Describe this image in one factual sentence."
+
+
+class MLXVisionCaptioner(VisionCaptioner):
+    """Caption through a locally loaded ``mlx_vlm`` model (guarded import)."""
+
+    provider = "mlx-vlm"
+
+    def __init__(self, model: str, *, prompt: str = DEFAULT_CAPTION_PROMPT, max_tokens: int = 64):
+        self.model_id = str(model or "")
+        self._prompt = prompt or DEFAULT_CAPTION_PROMPT
+        self._max_tokens = max(1, int(max_tokens))
+        self._loaded: Optional[Tuple[Any, Any]] = None
+
+    def _load(self) -> Tuple[Any, Any]:
+        if self._loaded is not None:
+            return self._loaded
+        try:  # optional dependency (`pip install "ltcai[local]"`)
+            from mlx_vlm import load as vlm_load  # type: ignore
+
+            model, processor = vlm_load(self.model_id)
+            self._loaded = (model, processor)
+        except Exception as exc:
+            raise EmbeddingUnavailable(f"vision-language model unavailable: {exc}") from exc
+        return self._loaded
+
+    def available(self) -> bool:
+        try:
+            self._load()
+            return True
+        except EmbeddingUnavailable:
+            return False
+
+    def caption(self, path: str, *, prompt: str = "") -> Optional[str]:
+        try:
+            model, processor = self._load()
+            from mlx_vlm import generate as vlm_generate  # type: ignore
+
+            text = vlm_generate(
+                model,
+                processor,
+                str(path),
+                prompt or self._prompt,
+                max_tokens=self._max_tokens,
+            )
+        except Exception:
+            # A caption the model did not produce is not a caption. Absence is
+            # the honest answer, and every caller already handles it.
+            return None
+        cleaned = str(text or "").strip()
+        return cleaned or None
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            self._load()
+            return {"status": "ok", "detail": f"VLM {self.model_id} loaded"}
+        except EmbeddingUnavailable as exc:
+            return {"status": "unavailable", "detail": str(exc)}
+
+
+class CustomVisionCaptioner(VisionCaptioner):
+    """A user-supplied ``module:callable`` that captions an image path."""
+
+    provider = "custom-vlm"
+
+    def __init__(self, target: str = ""):
+        self._target_ref = str(target or os.getenv(VISION_CAPTION_TARGET_ENV, ""))
+        self.model_id = self._target_ref
+        self._fn: Optional[Callable[..., Any]] = None
+
+    def _load(self) -> Callable[..., Any]:
+        if self._fn is None:
+            self._fn = _load_dotted(self._target_ref, VISION_CAPTION_TARGET_ENV, "vision caption")
+        return self._fn
+
+    def available(self) -> bool:
+        try:
+            self._load()
+            return True
+        except EmbeddingUnavailable:
+            return False
+
+    def caption(self, path: str, *, prompt: str = "") -> Optional[str]:
+        try:
+            text = self._load()(str(path), prompt or DEFAULT_CAPTION_PROMPT)
+        except Exception:
+            return None
+        cleaned = str(text or "").strip()
+        return cleaned or None
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            self._load()
+            return {"status": "ok", "detail": f"custom captioner {self._target_ref} loaded"}
+        except EmbeddingUnavailable as exc:
+            return {"status": "unavailable", "detail": str(exc)}
+
+
+def resolve_vision_captioner(
+    provider: str = "", *, model: str = "", target: str = ""
+) -> VisionCaptioner:
+    """Build a captioner, or the null one that honestly captions nothing."""
+    kind = str(provider or "").strip().lower()
+    if kind in {"mlx", "mlx-vlm", "mlx_vlm"} and model:
+        return MLXVisionCaptioner(model)
+    if kind in {"custom", "custom-vlm"}:
+        return CustomVisionCaptioner(target)
+    return VisionCaptioner()
+
+
+def vision_caption_port(captioner: VisionCaptioner) -> Optional[Callable[[str], Optional[str]]]:
+    """The caption seam Brain Core injects — ``None`` when no VLM is loaded."""
+    return captioner.caption if captioner.available() else None

package/latticeai/core/embedding_providers/profiles.py

@@ -0,0 +1,126 @@
+"""The production embedding profiles the setup and admin surfaces offer.
+
+A literal table, deliberately: a profile is a *named, supported* combination of
+provider, model and dimensionality, so the UI can offer a short list of things
+known to work instead of asking the user to assemble one.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+PRODUCTION_PROVIDER_PROFILES: Dict[str, Dict[str, Any]] = {
+    "local:bge-m3": {
+        "id": "local:bge-m3",
+        "provider": "mlx",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "BGE-M3 local",
+        "detail": "Multilingual semantic embeddings for local retrieval.",
+    },
+    "local:nomic-embed-text": {
+        "id": "local:nomic-embed-text",
+        "provider": "ollama",
+        "model": "nomic-embed-text",
+        "dimensions": 768,
+        "grade": "production",
+        "family": "local",
+        "label": "Nomic Embed Text local",
+        "detail": "General-purpose local semantic embeddings.",
+    },
+    "local:e5-large": {
+        "id": "local:e5-large",
+        "provider": "mlx",
+        "model": "e5-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "E5 Large local",
+        "detail": "High-recall local retrieval profile.",
+    },
+    "local:gte-large": {
+        "id": "local:gte-large",
+        "provider": "mlx",
+        "model": "gte-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "local",
+        "label": "GTE Large local",
+        "detail": "Large local semantic embedding profile.",
+    },
+    "ollama:nomic-embed-text": {
+        "id": "ollama:nomic-embed-text",
+        "provider": "ollama",
+        "model": "nomic-embed-text",
+        "dimensions": 768,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama Nomic Embed Text",
+        "detail": "Production semantic embeddings through Ollama.",
+    },
+    "ollama:mxbai-embed-large": {
+        "id": "ollama:mxbai-embed-large",
+        "provider": "ollama",
+        "model": "mxbai-embed-large",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama MXBAI Embed Large",
+        "detail": "High-quality local semantic embeddings through Ollama.",
+    },
+    "ollama:bge-m3": {
+        "id": "ollama:bge-m3",
+        "provider": "ollama",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "ollama",
+        "label": "Ollama BGE-M3-compatible",
+        "detail": "BGE-M3-compatible providers exposed through Ollama.",
+    },
+    "mlx:bge-m3": {
+        "id": "mlx:bge-m3",
+        "provider": "mlx",
+        "model": "bge-m3",
+        "dimensions": 1024,
+        "grade": "production",
+        "family": "mlx",
+        "label": "MLX BGE-M3",
+        "detail": "Apple Silicon optimized local embeddings.",
+    },
+    "openai:text-embedding-3-small": {
+        "id": "openai:text-embedding-3-small",
+        "provider": "openai",
+        "model": "text-embedding-3-small",
+        "dimensions": 1536,
+        "grade": "production",
+        "family": "openai-compatible",
+        "label": "OpenAI-compatible small",
+        "detail": "OpenAI-compatible /v1/embeddings endpoint.",
+    },
+    "openai:text-embedding-3-large": {
+        "id": "openai:text-embedding-3-large",
+        "provider": "openai",
+        "model": "text-embedding-3-large",
+        "dimensions": 3072,
+        "grade": "production",
+        "family": "openai-compatible",
+        "label": "OpenAI-compatible large",
+        "detail": "Highest-dimensional OpenAI-compatible embedding profile.",
+    },
+}
+
+
+def embedding_provider_profiles() -> List[Dict[str, Any]]:
+    return [dict(PRODUCTION_PROVIDER_PROFILES[key]) for key in sorted(PRODUCTION_PROVIDER_PROFILES)]
+
+
+def resolve_embedding_profile(profile: str) -> Dict[str, Any]:
+    if not profile:
+        return {}
+    key = str(profile).strip().lower()
+    if key in PRODUCTION_PROVIDER_PROFILES:
+        return dict(PRODUCTION_PROVIDER_PROFILES[key])
+    raise ValueError(f"unknown embedding profile: {profile!r}")

package/latticeai/core/embedding_providers/text.py

@@ -0,0 +1,350 @@
+"""The five text embedders, their factory, and the resolution that never fails.
+
+Hash (offline fallback), MLX, Ollama, OpenAI-compatible and a user-supplied
+dotted callable. ``build_embedding_provider`` constructs one by name without
+ever making a network call; ``resolve_embedder`` probes it and degrades to the
+hash fallback while *reporting* requested vs. active, so a down provider is
+never quietly presented as live.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
+
+from latticeai.core.local_embeddings import DEFAULT_EMBEDDING_DIM, LocalEmbeddingModel
+
+from .base import (
+    EmbeddingProvider,
+    EmbeddingUnavailable,
+    _guess_dim,
+    _NetworkEmbeddingProvider,
+    _RemoteConfig,
+)
+
+
+class HashEmbeddingProvider(EmbeddingProvider):
+    """Deterministic feature-hashing embedder — no network, always available."""
+
+    provider = "hash"
+    grade = "fallback"
+
+    def __init__(self, dim: int = DEFAULT_EMBEDDING_DIM):
+        self._model = LocalEmbeddingModel(dim=dim)
+        self.dim = self._model.dim
+        self.model_id = self._model.model_id
+
+    def embed(self, text: str) -> List[float]:
+        return self._model.embed(text)  # already L2-normalized
+
+    def embed_batch(self, texts: Sequence[str]) -> List[List[float]]:
+        return [self._model.embed(t) for t in texts]
+
+    def health(self) -> Dict[str, Any]:
+        return {"status": "ok", "detail": "deterministic local fallback"}
+
+
+def _as_float_list(value: Any) -> List[Any]:
+    """A pooled embedding row as a flat list.
+
+    ``mx.array.tolist()`` is typed as scalar-or-nested; at this call site the
+    array is always 1-D, so a scalar would be a bug worth surfacing.
+    """
+    if isinstance(value, (int, float)):
+        raise EmbeddingUnavailable("MLX embedding produced a scalar, not a vector")
+    return list(value)
+
+
+class MLXEmbeddingProvider(_NetworkEmbeddingProvider):
+    provider = "mlx"
+
+    def __init__(self, cfg: _RemoteConfig):
+        super().__init__(cfg)
+        if not cfg.dim:
+            self.dim = _guess_dim(cfg.model, DEFAULT_EMBEDDING_DIM)
+        self.model_id = f"mlx:{cfg.model}:{self.dim}"
+        self._encoder: Optional[Tuple[str, Any, Any]] = None
+
+    def _load(self):
+        if self._encoder is not None:
+            return self._encoder
+        try:  # optional dependency; only imported when this provider is used
+            from mlx_embeddings.utils import load as mlx_load  # type: ignore
+
+            model, tokenizer = mlx_load(self._cfg.model)
+            self._encoder = ("mlx_embeddings", model, tokenizer)
+            return self._encoder
+        except Exception as exc:  # pragma: no cover - environment dependent
+            raise EmbeddingUnavailable(f"MLX embedding model unavailable: {exc}") from exc
+
+    def _embed_raw(self, texts: Sequence[str]) -> List[List[float]]:
+        kind, model, tokenizer = self._load()
+        try:
+            import mlx.core as mx  # type: ignore
+
+            out: List[List[float]] = []
+            for text in texts:
+                ids = tokenizer.encode(text)
+                tokens = mx.array([ids])
+                result = model(tokens)
+                pooled = result[0] if isinstance(result, (tuple, list)) else result
+                vec = mx.mean(pooled, axis=1)[0] if pooled.ndim == 3 else pooled[0]
+                out.append([float(x) for x in _as_float_list(vec.tolist())])
+            return out
+        except EmbeddingUnavailable:
+            raise
+        except Exception as exc:  # pragma: no cover - environment dependent
+            raise EmbeddingUnavailable(f"MLX embedding failed: {exc}") from exc
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            self._load()
+            return {"status": "ok", "detail": f"MLX model {self._cfg.model} loaded"}
+        except Exception as exc:
+            return {"status": "unavailable", "detail": str(exc)}
+
+
+class OllamaEmbeddingProvider(_NetworkEmbeddingProvider):
+    provider = "ollama"
+
+    def __init__(self, cfg: _RemoteConfig):
+        super().__init__(cfg)
+        self._base = (cfg.base_url or "http://127.0.0.1:11434").rstrip("/")
+        if not cfg.dim:
+            self.dim = _guess_dim(cfg.model, DEFAULT_EMBEDDING_DIM)
+        self.model_id = f"ollama:{cfg.model}:{self.dim}"
+
+    def _embed_raw(self, texts: Sequence[str]) -> List[List[float]]:
+        out: List[List[float]] = []
+        try:
+            import httpx
+
+            with httpx.Client(timeout=self._cfg.timeout) as client:
+                # /api/embed supports batching; fall back to /api/embeddings.
+                resp = client.post(
+                    f"{self._base}/api/embed",
+                    json={"model": self._cfg.model, "input": list(texts)},
+                )
+                if resp.status_code == 404:
+                    for text in texts:
+                        r = client.post(
+                            f"{self._base}/api/embeddings",
+                            json={"model": self._cfg.model, "prompt": text},
+                        )
+                        r.raise_for_status()
+                        out.append(r.json().get("embedding") or [])
+                    return out
+                resp.raise_for_status()
+                data = resp.json()
+                return data.get("embeddings") or [data.get("embedding") or []]
+        except Exception as exc:
+            raise EmbeddingUnavailable(f"Ollama embedding failed: {exc}") from exc
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            import httpx
+
+            with httpx.Client(timeout=min(self._cfg.timeout, 5.0)) as client:
+                r = client.get(f"{self._base}/api/tags")
+                r.raise_for_status()
+            return {"status": "ok", "detail": f"Ollama reachable at {self._base}"}
+        except Exception as exc:
+            return {"status": "unavailable", "detail": f"Ollama unreachable: {exc}"}
+
+
+class OpenAICompatibleEmbeddingProvider(_NetworkEmbeddingProvider):
+    provider = "openai"
+
+    def __init__(self, cfg: _RemoteConfig):
+        super().__init__(cfg)
+        self._base = (cfg.base_url or "https://api.openai.com/v1").rstrip("/")
+        if not cfg.dim:
+            self.dim = _guess_dim(cfg.model, DEFAULT_EMBEDDING_DIM)
+        self.model_id = f"openai:{cfg.model}:{self.dim}"
+
+    def _headers(self) -> Dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if self._cfg.api_key:
+            headers["Authorization"] = f"Bearer {self._cfg.api_key}"
+        return headers
+
+    def _embed_raw(self, texts: Sequence[str]) -> List[List[float]]:
+        try:
+            import httpx
+
+            with httpx.Client(timeout=self._cfg.timeout) as client:
+                r = client.post(
+                    f"{self._base}/embeddings",
+                    headers=self._headers(),
+                    json={"model": self._cfg.model, "input": list(texts)},
+                )
+                r.raise_for_status()
+                rows = sorted(r.json().get("data", []), key=lambda d: d.get("index", 0))
+                return [row.get("embedding") or [] for row in rows]
+        except Exception as exc:
+            raise EmbeddingUnavailable(f"OpenAI-compatible embedding failed: {exc}") from exc
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            self._embed_raw(["ping"])
+            return {"status": "ok", "detail": f"{self._base} reachable"}
+        except Exception as exc:
+            return {"status": "unavailable", "detail": str(exc)}
+
+
+class CustomEmbeddingProvider(_NetworkEmbeddingProvider):
+    """Loads a dotted ``module:callable`` (or ``module.callable``).
+
+    The callable receives ``List[str]`` and returns ``List[List[float]]``.
+    Configured via ``LATTICEAI_EMBEDDING_CUSTOM_TARGET``.
+    """
+
+    provider = "custom"
+
+    def __init__(self, cfg: _RemoteConfig):
+        super().__init__(cfg)
+        self._target_ref = str(cfg.extra.get("target") or os.getenv("LATTICEAI_EMBEDDING_CUSTOM_TARGET", ""))
+        self.model_id = f"custom:{cfg.model or self._target_ref or 'callable'}:{self.dim}"
+        self._fn: Optional[Callable[..., Any]] = None
+
+    def _load(self):
+        if self._fn is not None:
+            return self._fn
+        ref = self._target_ref
+        if not ref:
+            raise EmbeddingUnavailable("custom embedding target not configured (LATTICEAI_EMBEDDING_CUSTOM_TARGET)")
+        module_name, _, attr = ref.replace(":", ".").rpartition(".")
+        if not module_name:
+            raise EmbeddingUnavailable(f"invalid custom embedding target: {ref}")
+        try:
+            module = importlib.import_module(module_name)
+            self._fn = getattr(module, attr)
+            return self._fn
+        except Exception as exc:
+            raise EmbeddingUnavailable(f"custom embedding target unavailable: {exc}") from exc
+
+    def _embed_raw(self, texts: Sequence[str]) -> List[List[float]]:
+        fn = self._load()
+        try:
+            return list(fn(list(texts)))
+        except Exception as exc:
+            raise EmbeddingUnavailable(f"custom embedding failed: {exc}") from exc
+
+    def health(self) -> Dict[str, Any]:
+        try:
+            self._load()
+            return {"status": "ok", "detail": f"custom target {self._target_ref} loaded"}
+        except Exception as exc:
+            return {"status": "unavailable", "detail": str(exc)}
+
+
+# ── factory + resolution ──────────────────────────────────────────────────────
+PROVIDER_TYPES = ("hash", "mlx", "ollama", "openai", "custom")
+
+
+def build_embedding_provider(
+    provider: str,
+    *,
+    model: str = "",
+    base_url: str = "",
+    api_key: str = "",
+    dim: int = 0,
+    timeout: float = 30.0,
+    extra: Optional[Dict[str, Any]] = None,
+) -> EmbeddingProvider:
+    """Construct a provider by name. Never makes a network call."""
+    kind = str(provider or "hash").strip().lower()
+    if kind in {"", "hash", "local", "fallback"}:
+        return HashEmbeddingProvider(dim=int(dim or DEFAULT_EMBEDDING_DIM))
+    cfg = _RemoteConfig(
+        model=model,
+        base_url=base_url,
+        api_key=api_key,
+        dim=int(dim or 0),
+        timeout=float(timeout or 30.0),
+        extra=dict(extra or {}),
+    )
+    if kind == "mlx":
+        return MLXEmbeddingProvider(cfg)
+    if kind == "ollama":
+        return OllamaEmbeddingProvider(cfg)
+    if kind in {"openai", "openai-compatible", "openai_compatible"}:
+        return OpenAICompatibleEmbeddingProvider(cfg)
+    if kind == "custom":
+        return CustomEmbeddingProvider(cfg)
+    raise ValueError(f"unknown embedding provider: {provider!r} (expected one of {PROVIDER_TYPES})")
+
+
+@dataclass
+class ResolvedEmbedder:
+    provider: EmbeddingProvider
+    requested: str
+    active: str
+    fell_back: bool
+    health: Dict[str, Any]
+    detail: str = ""
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "requested_provider": self.requested,
+            "active_provider": self.active,
+            "fell_back": self.fell_back,
+            "health": self.health,
+            "detail": self.detail,
+            **self.provider.metadata(),
+        }
+
+
+def resolve_embedder(
+    provider: str = "",
+    *,
+    model: str = "",
+    base_url: str = "",
+    api_key: str = "",
+    dim: int = 0,
+    timeout: float = 30.0,
+    extra: Optional[Dict[str, Any]] = None,
+    probe: bool = True,
+) -> ResolvedEmbedder:
+    """Build the requested provider, degrading to hash if it is unavailable.
+
+    Local-first guarantee: the app always gets a working embedder. When the
+    requested provider is unreachable we return the hash fallback but record
+    ``fell_back=True`` and the failing health detail so the UI shows it as
+    *Unavailable* — the system never pretends a down provider is live.
+    """
+    requested = str(provider or "hash").strip().lower() or "hash"
+    if requested in {"hash", "local", "fallback", ""}:
+        hash_prov = HashEmbeddingProvider(dim=int(dim or DEFAULT_EMBEDDING_DIM))
+        return ResolvedEmbedder(
+            hash_prov, "hash", "hash", False, hash_prov.health(), "deterministic local fallback"
+        )
+
+    try:
+        prov = build_embedding_provider(
+            requested, model=model, base_url=base_url, api_key=api_key, dim=dim, timeout=timeout, extra=extra
+        )
+    except Exception as exc:
+        fallback = HashEmbeddingProvider(dim=int(dim or DEFAULT_EMBEDDING_DIM))
+        return ResolvedEmbedder(
+            fallback, requested, "hash", True,
+            {"status": "unavailable", "detail": str(exc)},
+            f"could not construct {requested}; using hash fallback",
+        )
+
+    if probe:
+        try:
+            health = prov.health()
+        except Exception as exc:  # provider health must never crash startup
+            health = {"status": "unavailable", "detail": str(exc)}
+    else:
+        health = {"status": "unknown", "detail": "not probed"}
+    if probe and health.get("status") != "ok":
+        fallback = HashEmbeddingProvider(dim=int(dim or DEFAULT_EMBEDDING_DIM))
+        return ResolvedEmbedder(
+            fallback, requested, "hash", True, health,
+            f"{requested} unavailable ({health.get('detail', '')}); using hash fallback",
+        )
+    return ResolvedEmbedder(prov, requested, prov.provider, False, health, "")