flowent 0.1.4 → 0.2.0

package/backend/src/flowent/llm.py

@@ -1,11 +1,17 @@
 import logging
+import re
 from collections.abc import AsyncIterator, Awaitable, Mapping, Sequence
 from enum import StrEnum
-from typing import Any, Literal, Protocol
+from typing import Any, Literal, Protocol, cast
+from urllib.parse import urlsplit, urlunsplit
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from flowent.logging import TRACE_LEVEL, configure_litellm_logging
+from flowent.logging import (
+    TRACE_LEVEL,
+    configure_litellm_logging,
+    write_llm_request_diagnostic,
+)
 
 
 class ProviderFormat(StrEnum):
@@ -62,12 +68,26 @@ class ModelListCallable(Protocol):
 logger = logging.getLogger("flowent.llm")
 
 
+class LLMStreamError(RuntimeError):
+    pass
+
+
 MODEL_PREFIXES: dict[ProviderFormat, str] = {
     ProviderFormat.OPENAI: "openai",
     ProviderFormat.OPENAI_RESPONSES: "openai",
     ProviderFormat.ANTHROPIC: "anthropic",
     ProviderFormat.GEMINI: "gemini",
 }
+_litellm_stream_error_patch_installed = False
+
+PROVIDER_API_VERSIONS: dict[ProviderFormat, str] = {
+    ProviderFormat.OPENAI: "v1",
+    ProviderFormat.OPENAI_RESPONSES: "v1",
+    ProviderFormat.ANTHROPIC: "v1",
+    ProviderFormat.GEMINI: "v1beta",
+}
+
+VERSION_PATH_SEGMENT = re.compile(r"^v\d+(?:[a-z]+)?$", re.IGNORECASE)
 
 
 def provider_model_name(connection: ProviderConnection) -> str:
@@ -78,6 +98,40 @@ def provider_litellm_name(provider: ProviderFormat) -> str:
     return MODEL_PREFIXES[provider]
 
 
+def normalize_provider_base_url(
+    provider: ProviderFormat, base_url: str | None
+) -> str | None:
+    if base_url is None:
+        return None
+    raw_base_url = base_url.strip()
+    if not raw_base_url:
+        return None
+    if raw_base_url.endswith("#"):
+        return raw_base_url[:-1].rstrip("/") or None
+
+    trimmed_base_url = raw_base_url.rstrip("/")
+    parsed_base_url = urlsplit(trimmed_base_url)
+    path_segments = [segment for segment in parsed_base_url.path.split("/") if segment]
+    if any(VERSION_PATH_SEGMENT.fullmatch(segment) for segment in path_segments):
+        return trimmed_base_url
+
+    version = PROVIDER_API_VERSIONS[provider]
+    if parsed_base_url.scheme and parsed_base_url.netloc:
+        path = parsed_base_url.path.rstrip("/")
+        normalized_path = f"{path}/{version}" if path else f"/{version}"
+        return urlunsplit(
+            (
+                parsed_base_url.scheme,
+                parsed_base_url.netloc,
+                normalized_path,
+                parsed_base_url.query,
+                parsed_base_url.fragment,
+            )
+        )
+
+    return f"{trimmed_base_url}/{version}"
+
+
 def normalize_provider_model_name(provider: ProviderFormat, model: str) -> str:
     prefix = f"{provider_litellm_name(provider)}/"
     if model.startswith(prefix):
@@ -85,6 +139,71 @@ def normalize_provider_model_name(provider: ProviderFormat, model: str) -> str:
     return model
 
 
+def stream_failure_message(chunk: Any) -> str:
+    if isinstance(chunk, BaseModel):
+        chunk = chunk.model_dump()
+    if not isinstance(chunk, Mapping):
+        return ""
+
+    event_type = getattr(chunk.get("type"), "value", chunk.get("type"))
+    event_type = str(event_type or "")
+    if event_type == "error":
+        error = chunk.get("error", {})
+    elif event_type == "response.failed":
+        response = chunk.get("response", {})
+        error = value_at(response, "error", {})
+    else:
+        return ""
+
+    message = value_at(error, "message", "")
+    if isinstance(message, str) and message:
+        return message
+    code = value_at(error, "code", "")
+    if isinstance(code, str) and code:
+        return code
+    return "Upstream request failed"
+
+
+def raise_for_stream_failure(chunk: Any) -> None:
+    message = stream_failure_message(chunk)
+    if message:
+        raise LLMStreamError(message)
+
+
+def configure_litellm_stream_error_handling() -> None:
+    global _litellm_stream_error_patch_installed
+
+    if _litellm_stream_error_patch_installed:
+        return
+    try:
+        from litellm.completion_extras.litellm_responses_transformation.transformation import (
+            OpenAiResponsesToChatCompletionStreamIterator,
+        )
+    except Exception:
+        return
+
+    if getattr(
+        OpenAiResponsesToChatCompletionStreamIterator,
+        "_flowent_stream_error_patch_installed",
+        False,
+    ):
+        _litellm_stream_error_patch_installed = True
+        return
+
+    transformer = cast(Any, OpenAiResponsesToChatCompletionStreamIterator)
+    original = transformer.translate_responses_chunk_to_openai_stream
+
+    def translate_responses_chunk_to_openai_stream(parsed_chunk: Any) -> Any:
+        raise_for_stream_failure(parsed_chunk)
+        return original(parsed_chunk)
+
+    transformer.translate_responses_chunk_to_openai_stream = staticmethod(
+        translate_responses_chunk_to_openai_stream
+    )
+    transformer._flowent_stream_error_patch_installed = True
+    _litellm_stream_error_patch_installed = True
+
+
 def unique_model_names(provider: ProviderFormat, models: Sequence[str]) -> list[str]:
     seen: set[str] = set()
     normalized_models: list[str] = []
@@ -111,7 +230,7 @@ def list_provider_models(
         model_lister = get_valid_models
 
     models = model_lister(
-        api_base=base_url,
+        api_base=normalize_provider_base_url(provider, base_url),
         api_key=secret_reference,
         check_provider_endpoint=True,
         custom_llm_provider=provider_litellm_name(provider),
@@ -119,6 +238,21 @@ def list_provider_models(
     return unique_model_names(provider, models)
 
 
+def normalize_system_messages(
+    messages: Sequence[Mapping[str, Any]],
+    provider: ProviderFormat,
+) -> list[dict[str, Any]]:
+    normalized_messages = [dict(message) for message in messages]
+    if provider in {ProviderFormat.ANTHROPIC, ProviderFormat.GEMINI}:
+        return [
+            {**message, "role": "user"}
+            if message.get("role") == "system" and index > 0
+            else message
+            for index, message in enumerate(normalized_messages)
+        ]
+    return normalized_messages
+
+
 def build_litellm_request(
     connection: ProviderConnection,
     messages: Sequence[ChatMessage | Mapping[str, Any]],
@@ -126,10 +260,13 @@ def build_litellm_request(
     stream: bool = False,
     tools: Sequence[Mapping[str, Any]] | None = None,
 ) -> dict[str, Any]:
-    request_messages = [
-        message.model_dump() if isinstance(message, ChatMessage) else dict(message)
-        for message in messages
-    ]
+    request_messages = normalize_system_messages(
+        [
+            message.model_dump() if isinstance(message, ChatMessage) else dict(message)
+            for message in messages
+        ],
+        connection.provider,
+    )
     request: dict[str, Any] = {
         "api_key": connection.secret_reference,
         "messages": request_messages,
@@ -139,8 +276,11 @@ def build_litellm_request(
         request["tools"] = list(tools)
     if stream:
         request["stream"] = True
-    if connection.base_url:
-        request["api_base"] = connection.base_url
+    normalized_base_url = normalize_provider_base_url(
+        connection.provider, connection.base_url
+    )
+    if normalized_base_url:
+        request["api_base"] = normalized_base_url
     if connection.reasoning_effort != ReasoningEffort.DEFAULT:
         request["reasoning_effort"] = connection.reasoning_effort.value
     logger.log(
@@ -148,7 +288,7 @@ def build_litellm_request(
         "Built LiteLLM request provider=%s model=%s base_url=%s stream=%s tools=%s reasoning_effort=%s messages=%r",
         connection.provider,
         connection.model,
-        connection.base_url or "",
+        normalized_base_url or "",
         stream,
         bool(tools),
         connection.reasoning_effort,
@@ -157,6 +297,24 @@ def build_litellm_request(
     return request
 
 
+def record_litellm_request_diagnostic(
+    connection: ProviderConnection,
+    request: Mapping[str, Any],
+) -> None:
+    write_llm_request_diagnostic(
+        {
+            "base_url": request.get("api_base"),
+            "litellm_model": request["model"],
+            "messages": request["messages"],
+            "model": connection.model,
+            "provider": connection.provider.value,
+            "reasoning_effort": connection.reasoning_effort.value,
+            "stream": request.get("stream", False),
+            "tools": request.get("tools", []),
+        }
+    )
+
+
 async def complete_chat(
     connection: ProviderConnection,
     messages: Sequence[ChatMessage | Mapping[str, Any]],
@@ -175,9 +333,9 @@ async def complete_chat(
         connection.provider,
         connection.model,
     )
-    response = await completion(
-        **build_litellm_request(connection, messages, tools=tools)
-    )
+    request = build_litellm_request(connection, messages, tools=tools)
+    record_litellm_request_diagnostic(connection, request)
+    response = await completion(**request)
     logger.log(TRACE_LEVEL, "LLM completion response=%r", response)
     choice = response["choices"][0]["message"]
     return ChatMessage(role=choice.get("role", "assistant"), content=choice["content"])
@@ -277,6 +435,7 @@ async def stream_chat_chunks(
         from litellm import acompletion
 
         configure_litellm_logging()
+        configure_litellm_stream_error_handling()
         completion = acompletion
 
     logger.debug(
@@ -284,10 +443,11 @@ async def stream_chat_chunks(
         connection.provider,
         connection.model,
     )
-    response = await completion(
-        **build_litellm_request(connection, messages, stream=True, tools=tools)
-    )
+    request = build_litellm_request(connection, messages, stream=True, tools=tools)
+    record_litellm_request_diagnostic(connection, request)
+    response = await completion(**request)
     async for chunk in response:
+        raise_for_stream_failure(chunk)
         logger.log(TRACE_LEVEL, "LLM stream chunk=%r", chunk)
         yield chunk
 

package/backend/src/flowent/logging.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
 
+import json
 import logging
 import os
 import re
 import sys
 from datetime import datetime
 from pathlib import Path
+from typing import Any
 
 from flowent.paths import data_directory
 
@@ -14,6 +16,7 @@ DEFAULT_LOG_RETENTION = 5
 LITELLM_LOGGER_NAMES = ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy")
 _configured_log_file: Path | None = None
 _configured_log_process_id: int | None = None
+_llm_request_counter = 0
 _SECRET_PATTERNS = (
     re.compile(r"(?i)\b(bearer)\s+([^\s,}]+)"),
     re.compile(
@@ -36,10 +39,21 @@ def redact_log_value(value: object) -> str:
     return text
 
 
+def redact_diagnostic_value(value: object) -> str:
+    text = str(value)
+    for pattern in _SECRET_PATTERNS:
+        text = pattern.sub("[REDACTED]", text)
+    return text
+
+
 def log_directory(directory: Path | None = None) -> Path:
     return (directory or data_directory()) / "logs"
 
 
+def llm_request_log_directory(directory: Path | None = None) -> Path:
+    return log_directory(directory) / "llm-requests"
+
+
 def parse_log_level(value: str | None, default: int) -> int:
     if not value:
         return default
@@ -112,6 +126,52 @@ def new_log_file_path(directory: Path | None = None) -> Path:
     return logs / f"flowent-{timestamp}-{os.getpid()}.log"
 
 
+def sanitize_diagnostic_value(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {
+            key: sanitize_diagnostic_value(item)
+            for key, item in value.items()
+            if not secret_field_name(str(key))
+        }
+    if isinstance(value, list | tuple):
+        return [sanitize_diagnostic_value(item) for item in value]
+    if isinstance(value, str):
+        return redact_diagnostic_value(value)
+    return value
+
+
+def secret_field_name(name: str) -> bool:
+    normalized = re.sub(r"[^a-z0-9]", "", name.lower())
+    return "secret" in normalized or normalized in {
+        "accesstoken",
+        "apikey",
+        "authorization",
+        "password",
+        "refreshtoken",
+        "token",
+    }
+
+
+def write_llm_request_diagnostic(payload: dict[str, Any]) -> Path | None:
+    global _llm_request_counter
+
+    if not development_mode():
+        return None
+
+    _llm_request_counter += 1
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S-%f")
+    path = llm_request_log_directory() / (
+        f"llm-request-{timestamp}-{os.getpid()}-{_llm_request_counter:06d}.json"
+    )
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    path.write_text(
+        json.dumps(sanitize_diagnostic_value(payload), ensure_ascii=False, indent=2)
+        + "\n",
+        encoding="utf-8",
+    )
+    return path
+
+
 def prune_old_logs(logs: Path, *, keep: int = DEFAULT_LOG_RETENTION) -> None:
     files = sorted(logs.glob("flowent-*.log"), key=lambda item: item.name)
     for old_log in files[:-keep]: