claude-smart 0.2.41 → 0.2.43

package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py

@@ -0,0 +1,255 @@
+"""Local in-process embedder using ``nomic-ai/nomic-embed-text-v1.5``.
+
+A higher-quality alternative to the chromadb-bundled MiniLM-L6-v2: 137M
+parameters, 768-dim native, supports Matryoshka representation (64–768
+dimensions without retraining), 8192-token context, Apache-2.0 licensed.
+Performs comparably to OpenAI's ``text-embedding-3-small`` on MTEB
+retrieval at a fraction of the latency cost when run locally on CPU or
+Apple Silicon.
+
+Activation
+----------
+
+- Set ``CLAUDE_SMART_USE_LOCAL_EMBEDDING=1`` in the process environment.
+- Pass model name ``local/nomic-embed-v1.5`` (or ``local/nomic-embed-text-v1.5``)
+  to :func:`LiteLLMClient.get_embedding`/``get_embeddings``.
+- Requires the ``sentence-transformers`` pip dependency.
+
+Storage compatibility
+---------------------
+
+Reflexio's vec0 tables expect 512-dim vectors (``EMBEDDING_DIMENSIONS``).
+Nomic's native 768 dim is reduced via Matryoshka — slice the first 512
+floats, then re-normalize to unit length so cosine similarity remains
+comparable. Quality on retrieval tasks at 512 dim is ~95% of the full
+768 (per Nomic's own evaluation).
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import logging
+import math
+import os
+import threading
+from typing import Any
+
+_LOGGER = logging.getLogger(__name__)
+
+_ENV_ENABLE = "CLAUDE_SMART_USE_LOCAL_EMBEDDING"
+_ENV_PROVIDER = "REFLEXIO_EMBEDDING_PROVIDER"
+_ENV_DAEMON = "REFLEXIO_EMBEDDING_DAEMON"
+_MODEL_KEYS = {"local/nomic-embed-v1.5", "local/nomic-embed-text-v1.5"}
+_HF_MODEL_NAME = "nomic-ai/nomic-embed-text-v1.5"
+
+# Reflexio's vec0 schema dim. Nomic v1.5 outputs 768 natively; we slice
+# to 512 (Matryoshka) and re-normalize.
+_TARGET_DIM = 512
+# Nomic v1.5 was trained with task-prefixed inputs; "search_document"
+# vs "search_query" prefixes give better asymmetric retrieval. Reflexio's
+# storage layer already passes a "search_document: " / "search_query: "
+# prefix when calling _get_embedding(purpose=...), so we don't add another
+# prefix here — the input arrives correctly tagged.
+# The model has a 8192 token context window; we still cap chars
+# defensively to avoid pathological multi-MB inputs.
+_MAX_CHARS = 32_000
+
+
+class NomicEmbedderError(RuntimeError):
+    """Raised when the Nomic embedder is requested but its deps are missing."""
+
+
+class NomicEmbedder:
+    """Lazily-loaded singleton wrapping a sentence-transformers model.
+
+    Loading the underlying ``nomic-embed-text-v1.5`` model takes ~5–10 s on
+    first call (downloads ~550 MB on cold start, then cached under
+    ``~/.cache/huggingface/``). After that, embedding latency on CPU is
+    ~30–60 ms per single text and ~200 ms per batch of 32 (Apple M-series).
+    """
+
+    _instance: NomicEmbedder | None = None
+    _lock = threading.Lock()
+
+    def __init__(self) -> None:
+        self._model: Any | None = None
+        self._model_lock = threading.Lock()
+
+    @classmethod
+    def get(cls) -> NomicEmbedder:
+        """Return the process-wide singleton, constructing it on first use."""
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = cls()
+        return cls._instance
+
+    def _load(self) -> Any:
+        """Lazy-import sentence-transformers and load the Nomic model."""
+        if self._model is not None:
+            return self._model
+        with self._model_lock:
+            if self._model is not None:
+                return self._model
+            try:
+                from sentence_transformers import (
+                    SentenceTransformer,  # type: ignore[import-not-found]
+                )
+            except ImportError as exc:
+                raise NomicEmbedderError(
+                    "sentence-transformers is required for the Nomic local "
+                    "embedder. Install with `uv add sentence-transformers`."
+                ) from exc
+            _LOGGER.info(
+                "Loading Nomic embedding model %s — first call may download "
+                "~550 MB to ~/.cache/huggingface/",
+                _HF_MODEL_NAME,
+            )
+            # Force CPU device — MPS init has been observed to hang on some
+            # Apple Silicon + macOS combos for several minutes during model
+            # load. CPU is fast enough for our use case (137M params) and
+            # behaves predictably. Set NOMIC_EMBED_DEVICE=mps|cuda|cpu to
+            # override.
+            device = os.environ.get("NOMIC_EMBED_DEVICE", "cpu")
+            self._model = SentenceTransformer(
+                _HF_MODEL_NAME,
+                trust_remote_code=True,  # Nomic v1.5 ships custom code
+                device=device,
+            )
+            _LOGGER.info(
+                "Nomic embedder ready (model=%s, target_dim=%d, native_dim=%d)",
+                _HF_MODEL_NAME,
+                _TARGET_DIM,
+                self._model.get_sentence_embedding_dimension(),
+            )
+            return self._model
+
+    def embed(self, texts: list[str]) -> list[list[float]]:
+        """Embed a batch of texts, returning ``_TARGET_DIM``-sized unit vectors.
+
+        Args:
+            texts: Inputs to encode. Each is char-truncated to ``_MAX_CHARS``
+                as a defensive cap; Nomic itself supports 8192 tokens.
+
+        Returns:
+            list[list[float]]: One vector per input, each exactly
+                ``_TARGET_DIM`` (512) floats and L2-normalised so cosine
+                similarity equals dot product.
+        """
+        model = self._load()
+        safe = [(t or "")[:_MAX_CHARS] for t in texts]
+        # show_progress_bar=False so server logs stay clean during ingest
+        # batches. convert_to_numpy=True returns a numpy ndarray; we slice
+        # and renormalise per-row before converting to plain Python lists.
+        raw = model.encode(safe, show_progress_bar=False, convert_to_numpy=True)
+        return [_truncate_and_renormalise(vec.tolist()) for vec in raw]
+
+
+def _truncate_and_renormalise(vec: list[float]) -> list[float]:
+    """Slice to ``_TARGET_DIM`` and L2-renormalise for valid Matryoshka use.
+
+    Args:
+        vec (list[float]): Native-dim Nomic embedding (typically 768 floats,
+            already L2-unit on the full 768).
+
+    Returns:
+        list[float]: Exactly ``_TARGET_DIM`` floats, L2-normalised in the
+            truncated subspace so cosine similarity remains a valid metric.
+            Zero-padded if the input is shorter than ``_TARGET_DIM``.
+    """
+    if len(vec) >= _TARGET_DIM:
+        sliced = vec[:_TARGET_DIM]
+    else:
+        sliced = vec + [0.0] * (_TARGET_DIM - len(vec))
+    norm = math.sqrt(sum(x * x for x in sliced))
+    if norm <= 0:
+        return sliced
+    return [x / norm for x in sliced]
+
+
+_REGISTERED = False
+
+
+def register_if_enabled() -> bool:
+    """Make the Nomic embedder available when env + deps allow it.
+
+    Idempotent. Returns ``True`` when the embedder is usable after this
+    call. Routing happens via prefix-match on the model name in
+    ``LiteLLMClient.get_embedding(s)``.
+
+    Eagerly pre-warms the model in a daemon thread so the first request
+    doesn't pay the ~30 s cold-start cost. The thread is fire-and-forget;
+    callers either land mid-load (and block briefly) or after-load (and
+    proceed immediately).
+    """
+    global _REGISTERED
+    if _REGISTERED:
+        return True
+    if os.environ.get(_ENV_ENABLE) != "1":
+        return False
+    provider = os.environ.get(_ENV_PROVIDER, "").strip().lower()
+    if provider in {"local_service", "internal_service", "off"}:
+        _LOGGER.info(
+            "Nomic in-process prewarm skipped because %s=%s",
+            _ENV_PROVIDER,
+            provider,
+        )
+        return False
+    if not provider and os.environ.get(_ENV_DAEMON) != "1":
+        _LOGGER.info(
+            "Nomic in-process prewarm skipped; %s=1 now defaults to the "
+            "shared embedding service.",
+            _ENV_ENABLE,
+        )
+        return False
+    if importlib.util.find_spec("sentence_transformers") is None:
+        _LOGGER.warning(
+            "%s=1 set but `sentence-transformers` not installed; the Nomic "
+            "local embedder will not be available.",
+            _ENV_ENABLE,
+        )
+        return False
+    _REGISTERED = True
+    _LOGGER.info(
+        "Nomic local embedding provider enabled (models=%s)", sorted(_MODEL_KEYS)
+    )
+
+    def _prewarm() -> None:
+        """Background load + dummy inference so the first real request is fast."""
+        try:
+            embedder = NomicEmbedder.get()
+            embedder.embed(["warmup"])
+            _LOGGER.info("Nomic embedder pre-warmed")
+        except Exception:  # noqa: BLE001
+            _LOGGER.exception(
+                "Nomic embedder pre-warm failed; first call will pay the cost"
+            )
+
+    threading.Thread(target=_prewarm, daemon=True, name="nomic-prewarm").start()
+    return True
+
+
+def is_enabled() -> bool:
+    """Return True after a successful :func:`register_if_enabled`."""
+    return _REGISTERED
+
+
+def is_nomic_model(model: str) -> bool:
+    """Predicate used by ``LiteLLMClient`` to route by model name.
+
+    Args:
+        model (str): The embedding model name passed by the caller.
+
+    Returns:
+        bool: True when the model resolves to the Nomic provider.
+    """
+    return model in _MODEL_KEYS
+
+
+__all__ = [
+    "NomicEmbedder",
+    "NomicEmbedderError",
+    "is_enabled",
+    "is_nomic_model",
+    "register_if_enabled",
+]

package/plugin/vendor/reflexio/reflexio/server/llm/rerank/__init__.py

@@ -0,0 +1,6 @@
+"""Reranking helpers — local cross-encoder + LLM relevance judge."""
+
+from reflexio.server.llm.rerank.cross_encoder_reranker import prewarm, score_pairs
+from reflexio.server.llm.rerank.llm_reranker import score_pairs_llm
+
+__all__ = ["prewarm", "score_pairs", "score_pairs_llm"]

package/plugin/vendor/reflexio/reflexio/server/llm/rerank/cross_encoder_reranker.py

@@ -0,0 +1,177 @@
+"""Local cross-encoder reranker for ``(query, document)`` pairs.
+
+Wraps ``cross-encoder/ms-marco-MiniLM-L-6-v2`` (~25M params) from
+``sentence-transformers``. The model is lazy-loaded on first call and
+held as a process-wide singleton — load takes ~3 s but only happens
+once per server start. Scoring K=30 pairs takes ~50 ms on CPU.
+
+Usage
+-----
+
+>>> from reflexio.server.llm.rerank import score_pairs
+>>> scores = score_pairs("italian food", ["pasta lover", "weather report"])
+>>> scores[0] > scores[1]
+True
+
+The helper is intentionally side-effect free at import time: building
+the singleton happens only when ``score_pairs`` is called, so importing
+this module never triggers a model download.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any
+
+_LOGGER = logging.getLogger(__name__)
+
+# HuggingFace identifier for the cross-encoder. Chosen for the
+# size/quality trade-off: 22M parameters, ~50 ms for K=30 on CPU,
+# well-known MS-MARCO benchmark performance.
+_MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+# Singleton state — never accessed directly outside ``_get_model``.
+_MODEL: Any | None = None
+_MODEL_LOCK = threading.Lock()
+
+
+class CrossEncoderUnavailableError(RuntimeError):
+    """Raised when the cross-encoder model cannot be loaded.
+
+    The most common cause is ``sentence-transformers`` being absent from
+    the runtime environment. Callers should treat this as a soft failure
+    (log + skip rerank) rather than a 500.
+    """
+
+
+def _import_cross_encoder() -> Any:
+    """Robustly import ``sentence_transformers.CrossEncoder``.
+
+    The ``sentence_transformers`` package is loaded both by the Nomic
+    local-embedding pre-warm thread (kicked off by ``LiteLLMClient.__init__``
+    when ``CLAUDE_SMART_USE_LOCAL_EMBEDDING=1``) and by this reranker. When
+    those concurrent imports race, one thread can see a half-loaded
+    ``sentence_transformers`` module in ``sys.modules`` whose ``CrossEncoder``
+    attribute was never bound — Python's import machinery hands back the
+    partial module without re-running ``__init__``. This helper detects that
+    case, drops the stale entry, and re-imports cleanly.
+
+    Returns:
+        Any: The ``CrossEncoder`` class.
+
+    Raises:
+        CrossEncoderUnavailableError: When the package genuinely isn't
+            installed, or every retry yields a partial module.
+    """
+    import sys
+
+    for _attempt in range(2):
+        try:
+            from sentence_transformers import CrossEncoder
+        except ImportError:
+            # Partial import — drop the stale entry and try once more.
+            sys.modules.pop("sentence_transformers", None)
+            continue
+        return CrossEncoder
+    try:
+        from sentence_transformers import (
+            CrossEncoder,  # noqa: F401 — final attempt for the error path
+        )
+    except ImportError as e:
+        raise CrossEncoderUnavailableError(
+            "sentence-transformers is not installed; cannot use the "
+            "cross-encoder reranker"
+        ) from e
+    return CrossEncoder
+
+
+def _get_model() -> Any:
+    """Return the lazy-loaded cross-encoder singleton.
+
+    The first caller pays the load cost (~3 s, weights cached under
+    ``~/.cache/huggingface/`` after first download). Subsequent callers
+    get the warm instance immediately.
+
+    Returns:
+        Any: A ``sentence_transformers.CrossEncoder`` instance.
+
+    Raises:
+        CrossEncoderUnavailableError: If ``sentence-transformers`` is not
+            importable, or if the underlying model fails to load.
+    """
+    global _MODEL  # noqa: PLW0603 — singleton-pattern intentional
+    if _MODEL is not None:
+        return _MODEL
+    with _MODEL_LOCK:
+        if _MODEL is not None:
+            return _MODEL
+        cross_encoder_cls = _import_cross_encoder()
+        try:
+            _MODEL = cross_encoder_cls(_MODEL_NAME)
+        except Exception as e:  # noqa: BLE001 — surface as a typed failure
+            raise CrossEncoderUnavailableError(
+                f"Failed to load cross-encoder model {_MODEL_NAME!r}: {e}"
+            ) from e
+        _LOGGER.info("Loaded cross-encoder model %s", _MODEL_NAME)
+        return _MODEL
+
+
+def score_pairs(query: str, docs: list[str]) -> list[float]:
+    """Score ``(query, doc)`` pairs with the cross-encoder.
+
+    Higher score means more relevant. Scores are not bounded to a fixed
+    range — they are raw model logits — so callers should treat them as
+    opaque relative-ranking signal, not as probabilities.
+
+    Args:
+        query (str): The reranking query.
+        docs (list[str]): Documents to score against ``query``.
+
+    Returns:
+        list[float]: One score per document, in the same order as
+            ``docs``. Empty list when ``docs`` is empty.
+
+    Raises:
+        CrossEncoderUnavailableError: If the cross-encoder cannot be
+            loaded (re-raised from :func:`_get_model`).
+    """
+    if not docs:
+        return []
+    model = _get_model()
+    pairs = [(query, doc) for doc in docs]
+    raw_scores = model.predict(pairs)
+    # ``predict`` returns a numpy array; convert to plain Python floats so
+    # the caller can serialise the result without numpy as a dependency.
+    return [float(s) for s in raw_scores]
+
+
+def prewarm() -> bool:
+    """Force the cross-encoder model to load at startup.
+
+    Call once during app startup so the ~3s model load never lands on a user
+    query (and never serializes a concurrent burst behind the model lock).
+    Never raises — startup must not crash if the cross-encoder is unavailable
+    or fails to score the smoke input.
+
+    Returns:
+        True if the model loaded and scored a smoke input; False otherwise
+        (callers should treat the floor as degraded, not crash).
+    """
+    try:
+        score_pairs("warmup", ["warmup"])
+    except CrossEncoderUnavailableError:
+        _LOGGER.warning(
+            "Cross-encoder unavailable at startup; relevance floor will degrade "
+            "to unfiltered results until the model is available."
+        )
+        return False
+    except Exception:  # noqa: BLE001 — pre-warm must never crash startup
+        _LOGGER.warning(
+            "Cross-encoder pre-warm failed unexpectedly; relevance floor will "
+            "degrade to unfiltered results until the model is available.",
+            exc_info=True,
+        )
+        return False
+    _LOGGER.info("Cross-encoder pre-warmed.")
+    return True

package/plugin/vendor/reflexio/reflexio/server/llm/rerank/llm_reranker.py

@@ -0,0 +1,148 @@
+"""LLM-based reranker for ``(query, document)`` pairs.
+
+Sends a single LLM call with the query and a numbered list of candidate
+facts, expects a JSON array of relevance scores (0-10) in the same order.
+Fills the gap that cross-encoders can't bridge: world knowledge / brand
+↔ category equivalence (e.g. "Thrive Market" = grocery service).
+
+Usage
+-----
+
+>>> from reflexio.server.llm.rerank.llm_reranker import score_pairs_llm
+>>> scores = score_pairs_llm(
+...     query="grocery store",
+...     docs=["Walmart $120 grocery", "Thrive Market organic"],
+...     llm_client=client, prompt_manager=pm,
+... )
+>>> # scores is a list[float] of length 2, or None on failure.
+
+The helper returns ``None`` on any failure (no LLM client, prompt error,
+LLM call error, malformed JSON, score-count mismatch). Callers should
+fall back to the existing rerank order.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any
+
+_LOGGER = logging.getLogger(__name__)
+
+_PROMPT_ID = "rerank_relevance"
+_LLM_TIMEOUT = 30.0
+_LLM_MAX_RETRIES = 1
+
+
+def _parse_scores(text: str, expected_n: int) -> list[float] | None:
+    """Extract a JSON array of N numbers from raw LLM output.
+
+    Tolerates surrounding prose by locating the outermost ``[ ... ]`` span
+    via the first ``[`` and last ``]``. Returns ``None`` when the array
+    can't be parsed or the count doesn't match ``expected_n``.
+
+    Args:
+        text (str): Raw LLM response.
+        expected_n (int): Number of scores expected.
+
+    Returns:
+        list[float] | None: Parsed scores, or ``None`` on any parse failure.
+    """
+    start = text.find("[")
+    end = text.rfind("]")
+    if start == -1 or end <= start:
+        return None
+    try:
+        parsed = json.loads(text[start : end + 1])
+    except json.JSONDecodeError:
+        return None
+    if not isinstance(parsed, list) or len(parsed) != expected_n:
+        return None
+    try:
+        return [float(s) for s in parsed]
+    except (TypeError, ValueError):
+        return None
+
+
+def _format_docs_block(docs: list[str]) -> str:
+    """Render docs as a numbered, single-line-per-fact block.
+
+    Newlines inside individual docs are replaced with spaces so the LLM
+    sees a clean ``i. <fact>`` structure that matches the prompt's
+    output rule (one score per numbered fact).
+
+    Args:
+        docs (list[str]): Candidate documents.
+
+    Returns:
+        str: One ``i. <doc>`` per line.
+    """
+    flattened = (re.sub(r"\s+", " ", d).strip() for d in docs)
+    return "\n".join(f"{i + 1}. {d}" for i, d in enumerate(flattened))
+
+
+def score_pairs_llm(
+    query: str,
+    docs: list[str],
+    llm_client: Any,
+    prompt_manager: Any,
+    timeout: float = _LLM_TIMEOUT,
+) -> list[float] | None:
+    """Score ``(query, doc)`` pairs via an LLM relevance-judge prompt.
+
+    A single LLM call is made for the entire batch; the model returns one
+    score per doc. The model is the LiteLLMClient's default — same model
+    that drives the search agent — chosen to keep configuration simple.
+
+    Args:
+        query (str): The reranking query.
+        docs (list[str]): Documents to score against ``query``.
+        llm_client (Any): A ``LiteLLMClient`` (must expose ``generate_response``).
+        prompt_manager (Any): A ``PromptManager`` (must expose ``render_prompt``).
+        timeout (float): Per-call timeout in seconds.
+
+    Returns:
+        list[float] | None: One score per doc on success (same order as
+            ``docs``); ``None`` on any failure so the caller can fall back.
+    """
+    if not docs:
+        return []
+    if llm_client is None or prompt_manager is None:
+        return None
+    try:
+        prompt = prompt_manager.render_prompt(
+            _PROMPT_ID,
+            {
+                "query": query,
+                "docs_block": _format_docs_block(docs),
+                "num_docs": str(len(docs)),
+            },
+        )
+    except Exception as e:  # noqa: BLE001 — prompt-render failure must not break search
+        _LOGGER.warning("LLM rerank: prompt render failed: %s", e)
+        return None
+
+    try:
+        result = llm_client.generate_response(
+            prompt,
+            timeout=timeout,
+            max_retries=_LLM_MAX_RETRIES,
+            temperature=0.0,
+        )
+    except Exception as e:  # noqa: BLE001 — LLM-call failure must not break search
+        _LOGGER.warning("LLM rerank: generate_response failed: %s", e)
+        return None
+
+    if not isinstance(result, str) or not result.strip():
+        _LOGGER.warning(
+            "LLM rerank: empty/non-string response (%r)", type(result).__name__
+        )
+        return None
+
+    scores = _parse_scores(result, expected_n=len(docs))
+    if scores is None:
+        _LOGGER.warning(
+            "LLM rerank: parse failure (n=%d, raw[:200]=%r)", len(docs), result[:200]
+        )
+    return scores