claude-smart 0.2.48 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py

@@ -197,6 +197,24 @@ def _truncate_for_embedding(
     return encoding.decode(tokens[:max_tokens])
 
 
+def _embedding_error(message: str, mode: str) -> LiteLLMClientError:
+    """Build a ``LiteLLMClientError`` and log it tagged with the resolved mode.
+
+    Attaching ``mode`` (daemon-path vs in-process-path vs cloud) to the log makes
+    embedding failures attributable to a routing decision. The exception message
+    itself is unchanged so existing callers/tests keep matching on it.
+
+    Args:
+        message (str): The exception message (also logged).
+        mode (str): The resolved embedding provider mode.
+
+    Returns:
+        LiteLLMClientError: The exception to raise (``raise ... from e``).
+    """
+    _LOGGER.warning("%s (mode=%s)", message, mode, extra={"mode": mode})
+    return LiteLLMClientError(message)
+
+
 class EmbeddingMixin:
     """Embedding dispatch (service / Nomic / local ONNX / litellm) for ``LiteLLMClient``.
 
@@ -251,6 +269,10 @@ class EmbeddingMixin:
         """
         Get embedding vector for the given text.
 
+        Thin wrapper over ``get_embeddings`` — the single- and batch-text paths
+        share one dispatch body so their routing, truncation, and error handling
+        cannot drift.
+
         Args:
             text: The text to get embedding for.
             model: Optional embedding model. When omitted, the model is
@@ -265,73 +287,7 @@ class EmbeddingMixin:
         Raises:
             LiteLLMClientError: If embedding generation fails.
         """
-        embedding_model = model or self._resolve_default_embedding_model()
-        mode = embedding_provider_mode(embedding_model)
-        if mode == "off":
-            raise EmbeddingUnavailableError("Embedding provider is disabled")
-        if should_use_embedding_service(embedding_model):
-            return get_service_embeddings(
-                [text], model=embedding_model, dimensions=dimensions
-            )[0]
-
-        # local/nomic-embed-* must stay on the Nomic provider (137M params,
-        # 768d Matryoshka-truncated to 512). Falling through to MiniLM would
-        # mix embedding models inside existing vector stores.
-        if _is_nomic_model(embedding_model):
-            _reject_cloud_mode(embedding_model, mode)
-            try:
-                return NomicEmbedder.get().embed([text])[0]
-            except Exception as e:
-                raise LiteLLMClientError(
-                    f"Nomic embedding generation failed: {str(e)}"
-                ) from e
-
-        # local/* models route through the in-process ONNX embedder — no
-        # network call, no litellm API, no tiktoken truncation (the embedder
-        # applies its own token cap). The dispatch is gated solely on
-        # ``chromadb`` being importable; the env-var opt-in (claude-smart's
-        # ``CLAUDE_SMART_USE_LOCAL_EMBEDDING``) is enforced earlier in the
-        # auto-detection layer (see ``model_defaults._auto_detect_model``).
-        if embedding_model.startswith("local/"):
-            _reject_cloud_mode(embedding_model, mode)
-            if not _is_chromadb_importable():
-                raise LiteLLMClientError(
-                    f"Embedding model {embedding_model!r} requires chromadb. "
-                    "Run `pip install chromadb`."
-                )
-            try:
-                return LocalEmbedder.get().embed([text])[0]
-            except Exception as e:
-                raise LiteLLMClientError(
-                    f"Local embedding generation failed: {str(e)}"
-                ) from e
-
-        text = _truncate_for_embedding(text, embedding_model)
-
-        try:
-            params = {"model": embedding_model, "input": [text]}
-            if dimensions:
-                params["dimensions"] = dimensions
-
-            # Resolve and add API key configuration if provided (overrides env vars)
-            api_key, api_base, api_version = self._resolve_api_key(
-                embedding_model, for_embedding=True
-            )
-            if api_key:
-                params["api_key"] = api_key
-            if api_base:
-                params["api_base"] = api_base
-            if api_version:
-                params["api_version"] = api_version
-
-            response = litellm.embedding(
-                **params,
-                timeout=self.config.timeout,
-                num_retries=self.config.max_retries,
-            )
-            return response.data[0]["embedding"]
-        except Exception as e:
-            raise LiteLLMClientError(f"Embedding generation failed: {str(e)}") from e
+        return self._embed_texts([text], model, dimensions, batch=False)[0]
 
     def get_embeddings(
         self,
@@ -358,44 +314,86 @@ class EmbeddingMixin:
         """
         if not texts:
             return []
+        return self._embed_texts(list(texts), model, dimensions, batch=True)
 
+    def _embed_texts(
+        self,
+        texts: list[str],
+        model: str | None,
+        dimensions: int | None,
+        *,
+        batch: bool,
+    ) -> list[list[float]]:
+        """Shared embedding dispatch for ``get_embedding`` and ``get_embeddings``.
+
+        ``get_embedding`` passes a single-element list and unwraps ``[0]``;
+        ``get_embeddings`` passes the whole list. Behavior is identical between
+        the two callers except the error-message wording, which ``batch``
+        selects ("batch" wording matches the historical batch-path messages).
+
+        Args:
+            texts: Non-empty list of texts to embed (caller guarantees non-empty).
+            model: Optional embedding model; auto-detected when ``None``.
+            dimensions: Optional embedding dimensions.
+            batch: Selects the batch vs single error-message wording.
+
+        Returns:
+            One embedding vector per input text, in input order.
+
+        Raises:
+            EmbeddingUnavailableError: If the embedding provider is disabled.
+            LiteLLMClientError: If embedding generation fails.
+        """
         embedding_model = model or self._resolve_default_embedding_model()
         mode = embedding_provider_mode(embedding_model)
         if mode == "off":
             raise EmbeddingUnavailableError("Embedding provider is disabled")
         if should_use_embedding_service(embedding_model):
             return get_service_embeddings(
-                list(texts), model=embedding_model, dimensions=dimensions
+                texts, model=embedding_model, dimensions=dimensions
             )
 
-        # See matching short-circuits in get_embedding above.
+        # local/nomic-embed-* must stay on the Nomic provider (137M params,
+        # 768d Matryoshka-truncated to 512). Falling through to MiniLM would
+        # mix embedding models inside existing vector stores.
         if _is_nomic_model(embedding_model):
             _reject_cloud_mode(embedding_model, mode)
             try:
-                return NomicEmbedder.get().embed(list(texts))
+                return NomicEmbedder.get().embed(texts)
             except Exception as e:
-                raise LiteLLMClientError(
-                    f"Nomic batch embedding generation failed: {str(e)}"
+                raise _embedding_error(
+                    f"Nomic{' batch' if batch else ''} embedding generation "
+                    f"failed: {str(e)}",
+                    mode,
                 ) from e
 
+        # local/* models route through the in-process ONNX embedder — no
+        # network call, no litellm API, no tiktoken truncation (the embedder
+        # applies its own token cap). The dispatch is gated solely on
+        # ``chromadb`` being importable; the env-var opt-in (claude-smart's
+        # ``CLAUDE_SMART_USE_LOCAL_EMBEDDING``) is enforced earlier in the
+        # auto-detection layer (see ``model_defaults._auto_detect_model``).
         if embedding_model.startswith("local/"):
             _reject_cloud_mode(embedding_model, mode)
             if not _is_chromadb_importable():
-                raise LiteLLMClientError(
+                raise _embedding_error(
                     f"Embedding model {embedding_model!r} requires chromadb. "
-                    "Run `pip install chromadb`."
+                    "Run `pip install chromadb`.",
+                    mode,
                 )
             try:
-                return LocalEmbedder.get().embed(list(texts))
+                return LocalEmbedder.get().embed(texts)
             except Exception as e:
-                raise LiteLLMClientError(
-                    f"Local batch embedding generation failed: {str(e)}"
+                raise _embedding_error(
+                    f"Local{' batch' if batch else ''} embedding generation "
+                    f"failed: {str(e)}",
+                    mode,
                 ) from e
 
-        texts = [_truncate_for_embedding(t, embedding_model) for t in texts]
+        truncated = [_truncate_for_embedding(t, embedding_model) for t in texts]
 
         try:
-            params = {"model": embedding_model, "input": texts}
+            params = {"model": embedding_model, "input": truncated}
             if dimensions:
                 params["dimensions"] = dimensions
 
@@ -419,6 +417,8 @@ class EmbeddingMixin:
             sorted_data = sorted(response.data, key=lambda x: x["index"])
             return [item["embedding"] for item in sorted_data]
         except Exception as e:
-            raise LiteLLMClientError(
-                f"Batch embedding generation failed: {str(e)}"
+            raise _embedding_error(
+                f"{'Batch embedding' if batch else 'Embedding'} generation "
+                f"failed: {str(e)}",
+                mode,
             ) from e

package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py

@@ -141,9 +141,37 @@ def _picklable_completion_result(response: Any) -> Any:
     return response
 
 
+def _reset_llm_client_state_after_fork() -> None:
+    """Drop litellm's module-level HTTP client cache in the forked CHILD.
+
+    ``_litellm_completion_worker`` runs in a ``multiprocessing`` fork child, which
+    inherits a *copy* of the parent's ``litellm.in_memory_llm_clients_cache`` —
+    including any warm keep-alive HTTPS connection the parent held. Reusing such
+    an inherited socket in the child shares one TLS/HTTP connection across two
+    processes and corrupts the child's first completion (surfaces as
+    ``[SSL] record layer failure`` / ``Server disconnected`` / a garbled request
+    the provider rejects as missing auth). Clearing the cache here forces the
+    child to build a fresh client (and socket) for its one completion. Best-effort
+    and fully guarded: a litellm build without these attributes is a no-op, never
+    a crash. Only the child's copy is mutated, so the parent cache is untouched.
+    """
+    try:
+        cache = getattr(litellm, "in_memory_llm_clients_cache", None)
+        if cache is not None:
+            flush = getattr(cache, "flush_cache", None)
+            if callable(flush):
+                flush()
+            cache_dict = getattr(cache, "cache_dict", None)
+            if isinstance(cache_dict, dict):
+                cache_dict.clear()
+    except Exception:  # noqa: BLE001, S110 — a reset failure must never break the call
+        pass
+
+
 def _litellm_completion_worker(
     params: dict[str, Any], result_queue: multiprocessing.Queue
 ) -> None:
+    _reset_llm_client_state_after_fork()
     try:
         result_queue.put(
             ("ok", _picklable_completion_result(litellm.completion(**params)))

package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py

@@ -53,7 +53,11 @@ from reflexio.server.llm.image_utils import (
     encode_image_to_base64 as _encode_image_to_base64,
 )
 from reflexio.server.llm.llm_utils import is_pydantic_model
-from reflexio.server.llm.model_defaults import ModelRole, resolve_model_name
+from reflexio.server.llm.model_defaults import (
+    ModelRole,
+    default_max_tokens_for_model,
+    resolve_model_name,
+)
 
 if TYPE_CHECKING:
     from reflexio.server.llm._litellm_types import LiteLLMConfig
@@ -74,6 +78,36 @@ _MODEL_TIMEOUT_FLOOR_SECONDS: dict[str, int] = {
 }
 
 
+# Upstream-provider errors that are EXPECTED and transient. By the time one of
+# these reaches the request handler the fallback ladder is already exhausted,
+# but the caller — not the client — owns fatality, and many callers degrade
+# gracefully (FTS fallback for document expansion, skip-dedup for profile
+# consolidation). Log these at WARNING so a flaky provider (e.g. minimax
+# timeouts / 529 overload) can't flood ERROR-level alerts for handled failures;
+# genuinely-unexpected errors (bugs, auth, malformed structured output) stay
+# ERROR. Classified by exception TYPE NAME to avoid importing the heavy
+# ``litellm``/``openai`` exception hierarchies at module import.
+_TRANSIENT_LLM_ERROR_NAMES: frozenset[str] = frozenset(
+    {
+        "Timeout",
+        "APITimeoutError",
+        "APIConnectionError",
+        "RateLimitError",
+        "InternalServerError",
+        "ServiceUnavailableError",
+    }
+)
+
+
+def _is_expected_transient_llm_error(exc: BaseException) -> bool:
+    """True for expected transient upstream failures (timeout / connection /
+    rate-limit / overload), including our own ``LLMHardTimeoutError`` (a
+    ``TimeoutError`` subclass raised when a provider hang is killed)."""
+    if isinstance(exc, TimeoutError):  # incl. LLMHardTimeoutError
+        return True
+    return type(exc).__name__ in _TRANSIENT_LLM_ERROR_NAMES
+
+
 class TextGenerationMixin:
     """Chat/response generation, completion-param build, hard-timeout, cost, multimodal.
 
@@ -293,7 +327,10 @@ class TextGenerationMixin:
         tool_choice = kwargs.pop("tool_choice", None)
         model_role: ModelRole | None = kwargs.pop("model_role", None)
 
-        actual_model = kwargs.pop("model", self.config.model)
+        # An explicit ``model=None`` means "use the config default" — callers
+        # like the eval judges forward an optional model straight through, and
+        # a literal None would crash on ``.lower()`` during key resolution.
+        actual_model = kwargs.pop("model", None) or self.config.model
 
         # model_role takes priority over the default model but falls through
         # to the custom_endpoint override below (highest priority).
@@ -322,8 +359,16 @@ class TextGenerationMixin:
         }
 
         # Drop any fallback entry that points back at the primary — sending the
-        # same broken endpoint twice never helps.
-        fallback_models = [m for m in fallback_models_raw if m != actual_model]
+        # same broken endpoint twice never helps. Also drop in-process ``local/*``
+        # embedding models: they have no litellm completion route (they are served
+        # in-process by ``_litellm_embedding.py``), so litellm would raise
+        # ``BadRequestError: LLM Provider NOT provided`` deep inside its fallback
+        # ladder if one ever landed in this list (Sentry PYTHON-FASTAPI-CV).
+        fallback_models = [
+            m
+            for m in fallback_models_raw
+            if m != actual_model and not m.startswith("local/")
+        ]
 
         temperature = kwargs.pop("temperature", self.config.temperature)
         if self._is_temperature_restricted_model(actual_model):
@@ -357,6 +402,10 @@ class TextGenerationMixin:
             params["temperature"] = 0.0
 
         max_tokens = kwargs.pop("max_tokens", self.config.max_tokens)
+        if max_tokens is None:
+            # Provider-level guard: some providers (MiniMax-M3) stall into the
+            # request timeout when max_tokens is omitted. See model_defaults.
+            max_tokens = default_max_tokens_for_model(actual_model)
         if max_tokens:
             params["max_tokens"] = max_tokens
         if self.config.top_p != 1.0:
@@ -523,7 +572,7 @@ class TextGenerationMixin:
 
             # The loop only exits with a drained result (every no-result path
             # above raises); the assert makes that invariant explicit for pyright.
-            assert result is not None
+            assert result is not None  # noqa: S101
             status, payload = result
             # The payload is drained, so the feeder is unblocked and the child can
             # exit. Reap it (terminating if it lingers) to avoid a zombie.
@@ -815,7 +864,15 @@ class TextGenerationMixin:
                 )
                 return _call_and_parse()
         except Exception as e:
-            self.logger.error(
+            # Expected transient upstream failures (provider timeout / connection
+            # / rate-limit / overload) log at WARNING — callers own fatality and
+            # most degrade gracefully; only genuinely-unexpected errors are ERROR.
+            log = (
+                self.logger.warning
+                if _is_expected_transient_llm_error(e)
+                else self.logger.error
+            )
+            log(
                 "event=llm_request_end model=%s elapsed_seconds=%.3f success=False error_type=%s error=%s",
                 params.get("model"),
                 time.perf_counter() - request_start,

package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py

@@ -34,29 +34,19 @@ _SUPPORTED_MODELS = {
 _ACTIVE_MODEL: str | None = None
 _ACTIVE_MODEL_LOCK = threading.Lock()
 
-# The underlying sentence-transformers / ONNX embedders share internal buffers
-# and are NOT thread-safe: concurrent ``.embed()`` calls interleave and corrupt
-# each other's padding/attention tensors (observed under concurrent publishes as
-# ``RuntimeError: The size of tensor a (62) must match the size of tensor b (61)
-# at non-singleton dimension 1``). The ``_ENCODE_SEMAPHORE`` below only caps how
-# many requests run at once (for memory), it does NOT serialize them, so this
-# lock guards the actual model inference. Throughput comes from micro-batch
-# coalescing (many texts per encode), not from parallel encodes on one model.
-_MODEL_ENCODE_LOCK = threading.Lock()
-
 DEFAULT_OSS_EMBEDDING_MODEL = MINILM_MODEL
 
-# Bound how many embed/encode calls run at once. The endpoint is a sync ``def``
-# served from Starlette's threadpool, so without a guard a burst of requests
-# would run that many model.encode() calls in parallel, stacking their
-# activation memory and OOM-killing the daemon. The semaphore caps simultaneous
-# encodes; excess requests block on acquire() and are picked up when a slot
-# frees — they queue, they are never rejected. Pair with a small encode
-# batch_size so each in-flight encode stays cheap.
+# Bound how many micro-batch processor threads run at once. The endpoint is a
+# sync ``def`` served from Starlette's threadpool, so without a cap a burst of
+# requests would spawn a processor per request and stack their activation
+# memory, OOM-killing the daemon. This caps concurrent processors; excess
+# requests queue on the micro-batch condition and are picked up when a slot
+# frees — they are never rejected. The embedder singletons (NomicEmbedder /
+# LocalEmbedder) serialize the actual model.encode() internally — the shared
+# models are not thread-safe — so no encode lock or semaphore is needed here.
+# Pair with a small encode batch_size so each in-flight encode stays cheap.
 _DEFAULT_MAX_CONCURRENCY = 4
 _ENV_MAX_CONCURRENCY = "REFLEXIO_EMBED_MAX_CONCURRENCY"
-_ENCODE_SEMAPHORE: threading.BoundedSemaphore | None = None
-_ENCODE_SEMAPHORE_LOCK = threading.Lock()
 
 # Opportunistically coalesce concurrent small requests before calling
 # model.encode(). This keeps request concurrency thread-based while giving the
@@ -111,16 +101,6 @@ def _micro_batch_max_texts() -> int:
     )
 
 
-def _encode_semaphore() -> threading.BoundedSemaphore:
-    """Return the process-wide encode semaphore, building it on first use."""
-    global _ENCODE_SEMAPHORE
-    if _ENCODE_SEMAPHORE is None:
-        with _ENCODE_SEMAPHORE_LOCK:
-            if _ENCODE_SEMAPHORE is None:
-                _ENCODE_SEMAPHORE = threading.BoundedSemaphore(_max_concurrency())
-    return _ENCODE_SEMAPHORE
-
-
 class EmbeddingRequest(BaseModel):
     model: str
     input: str | list[str]
@@ -313,29 +293,16 @@ def _process_micro_batch(jobs: list[_EmbeddingJob]) -> None:
 
 
 def _encode_texts_now(model: str, texts: list[str]) -> list[list[float]]:
-    semaphore = _encode_semaphore()
-    # Non-blocking probe purely for observability: if no slot is free, this
-    # request will have to wait. The real acquire below blocks until a slot
-    # opens, so the request queues and is picked up later — never rejected.
-    if not semaphore.acquire(blocking=False):
-        logger.info(
-            "Embedding request queued; all %d encode slots busy",
-            _max_concurrency(),
-        )
-        semaphore.acquire()
-    try:
-        # Serialize the actual inference: the shared embedder models are not
-        # thread-safe, so concurrent encodes race and produce tensor-shape
-        # errors. The semaphore (above) bounds memory; this lock bounds the
-        # model to one in-flight encode at a time.
-        with _MODEL_ENCODE_LOCK:
-            if is_nomic_model(model):
-                return NomicEmbedder.get().embed(texts)
-            if model == MINILM_MODEL:
-                return LocalEmbedder.get().embed(texts)
-        raise HTTPException(status_code=400, detail=f"Unsupported model: {model}")
-    finally:
-        semaphore.release()
+    # The embedder singletons serialize their own model.encode() internally
+    # (the shared sentence-transformers / ONNX models are not thread-safe — see
+    # NomicEmbedder / LocalEmbedder). Concurrency is already bounded by the
+    # micro-batch processor cap (``_max_concurrency``), so no extra encode
+    # lock/semaphore is needed here.
+    if is_nomic_model(model):
+        return NomicEmbedder.get().embed(texts)
+    if model == MINILM_MODEL:
+        return LocalEmbedder.get().embed(texts)
+    raise HTTPException(status_code=400, detail=f"Unsupported model: {model}")
 
 
 def _activate_model(model: str) -> None:

package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py

@@ -110,6 +110,7 @@ __all__ = [
     "create_litellm_client",
 ]
 
+
 class LiteLLMClient(TextGenerationMixin, EmbeddingMixin, StructuredOutputMixin):
     """
     Unified LLM client using LiteLLM for multi-provider support.

package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py

@@ -267,6 +267,34 @@ _PROVIDER_DEFAULTS: dict[str, ProviderDefaults] = {
 }
 
 
+# Output-token cap applied when neither the call site nor the client config
+# sets max_tokens. MiniMax-M3 misbehaves with unbounded output: omitting
+# max_tokens (especially combined with a strict json_schema response_format)
+# deterministically stalls generation into litellm's 120s timeout (reproduced
+# 2026-07; observed in prod as consolidator/document-expansion timeouts).
+# Sizing (measured in prod, 2026-07-14): M3's reasoning tokens count against
+# this budget, so too small a cap starves the visible output — at 4096 the
+# model regularly spent the whole budget thinking and returned empty/truncated
+# content (structured-output parse failures ran ~10-20x the 8192-era rate,
+# breaking extraction). 8192 was the healthiest measured setting. The 120s
+# provider stalls occur at every cap value (provider-side; mitigate with
+# fallback models, not here). Providers absent from this map stay unbounded.
+_PROVIDER_DEFAULT_MAX_TOKENS: dict[str, int] = {"minimax": 8192}
+
+
+def default_max_tokens_for_model(model: str) -> int | None:
+    """Return the provider-level default output-token cap for ``model``.
+
+    Args:
+        model (str): Full model name, e.g. ``"minimax/MiniMax-M3"``.
+
+    Returns:
+        int | None: Cap to apply when the caller set none, or None (no cap).
+    """
+    provider = model.split("/", 1)[0] if "/" in model else ""
+    return _PROVIDER_DEFAULT_MAX_TOKENS.get(provider)
+
+
 EMBEDDING_CAPABLE_PROVIDERS: frozenset[str] = frozenset(
     p for p, d in _PROVIDER_DEFAULTS.items() if d.embedding is not None
 )

package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py

@@ -89,6 +89,12 @@ class ClaudeCodeCLIError(RuntimeError):
     """Raised when the claude CLI subprocess fails in a way we cannot recover from."""
 
 
+def _diagnostic_excerpt(text: str, limit: int = 500) -> str:
+    """Return a bounded single-line excerpt for local CLI diagnostics."""
+    compact = " ".join((text or "").split())
+    return compact[:limit]
+
+
 def _env_enabled() -> bool:
     """Return True when ``CLAUDE_SMART_USE_LOCAL_CLI`` is set to a truthy value.
 
@@ -1037,7 +1043,9 @@ class ClaudeCodeLLM(CustomLLM):
         self._record_stall_safely(result)
         raise ClaudeCodeCLIError(
             f"claude -p stream failed; retry_errors={result.retry_errors}; "
-            f"stderr={result.stderr_text[:200]!r}"
+            f"stderr={_diagnostic_excerpt(result.stderr_text)!r}; "
+            f"stdout={_diagnostic_excerpt(result.terminal_text)!r}; "
+            f"parsed={result.raw_lines_parsed}; failed={result.raw_lines_failed}"
         )
 
     def _record_stall_safely(self, result: ParseResult) -> None: