lex-llm 0.4.9 → 0.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2b7839d1fcb47e176aa62970f3ec40d04b9e25095418285d3cf6a5d492ccb8d
-  data.tar.gz: f444cd5054325a007f05749d6d4a2ca0933bf1f5b7004b2585daaa97745eb337
+  metadata.gz: 6d60f78c459fb43344897e6fdba10730b881f698229058a50a1c1be2564539cf
+  data.tar.gz: d7fcedadb69266af972caf1a51d1153bd5270f1fd5e9b45f65d51076fafa07aa
 SHA512:
-  metadata.gz: 7412bc0234b379941ae045fa826f267d97b1577dba15ee7268957d62c999808e27392d9acf65d0bf464c31aa7365e3912642a4b0041eb92294f108c4203d8f38
-  data.tar.gz: a90e99a7c61f6fda2ffc4d2e4fd6b2fcba6f69bf75d5da526cacef550c345aadd157ea1ee6efeae59e3f69160a8e6ad92237a31997e9ef259c69dc29a6db01e6
+  metadata.gz: c60726bfac3eff11cf37d8035ad78c7437b627f465bad31efdac1be3061fe410dc176d805bd168389859fa94773cd994578d265a6534a8e3feed1d37db517988
+  data.tar.gz: 40439ec46e06530b9e5d287fe8d5980d57b87c2700343b3282c30deb9cd1b241862812e4264a9842d6a1fea20aa9bcb4f580cf08ed31cc61b73c00c2c753c9ce

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.4.13 - 2026-05-15
+
+- Strip provider thinking from OpenAI-compatible responses when local models emit `<thinking>` tags or untagged initial reasoning preambles, and keep those hidden from live streaming content deltas.
+
+## 0.4.12 - 2026-05-15
+
+- Preserve streamed provider error bodies in a custom Faraday env key so Faraday Net::HTTP finalization cannot replace the buffered body with an empty string before `ErrorMiddleware` parses it.
+
+## 0.4.11 - 2026-05-15
+
+- Fix `handle_failed_response` to preserve non-200 streaming error bodies across chunks instead of swallowing `ParseError` and falling through to a generic "An unknown error occurred". Complete JSON error bodies still raise typed provider errors immediately; incomplete bodies are buffered onto the Faraday response env for final middleware parsing, with regex fallback extraction for vLLM-style partial `message` fields when the env cannot carry the buffered body.
+
+## 0.4.10 - 2026-05-13
+
+- Add cache-backed `model_detail` lookup with 24-hour TTL; nil results are not cached; `fetch_model_detail` hook for subclasses to override with live API calls.
+- Build `model_detail_cache_key` from tier, slug, instance, and credential fingerprint so remote providers never share model detail entries across credentials.
+- Add `credential_cache_fragment` — includes an 8-char SHA-256 credential fingerprint in cache keys for non-local providers.
+- Add `source_tag`, `credential_fingerprint`, and `config_fingerprint` to `CredentialSources` for provenance tracking across discovered instances.
+- Suppress Faraday raw stacktrace dumps on connection failures by setting `errors: false` on the response logger middleware.
+- Rescue `Faraday::ConnectionFailed` in `discover_offerings` and return an empty list with a concise warning instead of propagating the exception.
+- Wire `model_allowed?` filtering into `discover_offerings` so whitelist/blacklist settings are enforced during live discovery (was dead code before).
+- Check instance config first for `model_whitelist`/`model_blacklist` before falling back to provider settings, enabling per-instance override.
+- Add `legion-cache >= 1.3.0` as a runtime dependency and include `Legion::Cache::Helper` in the base `Provider` class.
+
 ## 0.4.9 - 2026-05-13
 
 - Route provider, tool, streaming, model, attachment, connection, credential, and fleet diagnostics through `Legion::Logging::Helper`.

data/lex-llm.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'faraday-multipart', '>= 1'
   spec.add_dependency 'faraday-net_http', '>= 1'
   spec.add_dependency 'faraday-retry', '>= 1'
+  spec.add_dependency 'legion-cache', '>= 1.3.0'
   spec.add_dependency 'legion-crypt', '>= 1.5.1'
   spec.add_dependency 'legion-json', '>= 1.2.1'
   spec.add_dependency 'legion-logging', '>= 1.3.2'

data/lib/legion/extensions/llm/connection.rb

@@ -79,7 +79,7 @@ module Legion
           faraday.response :logger,
                            logger,
                            bodies: debug_logger?(logger),
-                           errors: true,
+                           errors: false,
                            headers: false,
                            log_level: :debug do |logger|
             logger.filter(logging_regexp('[A-Za-z0-9+/=]{100,}'), '[BASE64 DATA]')

data/lib/legion/extensions/llm/credential_sources.rb

@@ -167,6 +167,30 @@ module Legion
           Digest::SHA256.hexdigest(val.to_s)
         end
 
+        # Build a human-readable source tag describing where a credential was found.
+        # Format: "type:location:key" e.g. "env:ANTHROPIC_API_KEY", "file:~/.claude/settings.json:anthropicApiKey"
+        def source_tag(type, location, key = nil)
+          parts = [type.to_s, location.to_s]
+          parts << key.to_s if key && !key.to_s.empty?
+          parts.join(':')
+        end
+
+        # Generate a short fingerprint (first 8 chars of SHA-256) for a credential value.
+        # Stable for the lifetime of the credential; safe to log and include in audit events.
+        def credential_fingerprint(value)
+          return nil if value.nil? || value.to_s.strip.empty?
+
+          Digest::SHA256.hexdigest(value.to_s)[0, 8]
+        end
+
+        # Extract fingerprint from a config hash by finding the first credential field.
+        def config_fingerprint(config)
+          val = config[:api_key] || config['api_key'] ||
+                config[:bearer_token] || config['bearer_token'] ||
+                config[:access_token] || config['access_token']
+          credential_fingerprint(val)
+        end
+
         # Returns true when the URL points to localhost / 127.0.0.1 / ::1.
         def localhost?(url)
           return false if url.nil?
@@ -185,7 +209,9 @@ module Legion
         module_function :env, :claude_config, :claude_config_value,
                         :claude_env_value, :codex_token, :codex_openai_key,
                         :setting, :socket_open?, :http_ok?,
-                        :dedup_credentials, :credential_hash, :localhost?
+                        :dedup_credentials, :credential_hash,
+                        :source_tag, :credential_fingerprint, :config_fingerprint,
+                        :localhost?
 
         # --- private helpers -----------------------------------------------
 

data/lib/legion/extensions/llm/error.rb

@@ -54,6 +54,8 @@ module Legion
 
       # Faraday middleware that maps provider-specific API errors to Legion::Extensions::Llm errors.
       class ErrorMiddleware < Faraday::Middleware
+        STREAM_ERROR_BODY_KEY = :legion_llm_stream_error_body
+
         def initialize(app, options = {})
           super(app)
           @provider = options[:provider]
@@ -79,6 +81,7 @@ module Legion
           ].freeze
 
           def parse_error(provider:, response:) # rubocop:disable Metrics/PerceivedComplexity
+            response = response_with_stream_error_body(response)
             message = provider&.parse_error(response)
 
             case response.status
@@ -116,12 +119,51 @@ module Legion
 
           private
 
+          def response_with_stream_error_body(response)
+            return response unless empty_body?(response)
+
+            stream_body = preserved_stream_error_body(response)
+            return response if stream_body.to_s.empty?
+
+            ResponseWithBody.new(response, stream_body)
+          end
+
+          def empty_body?(response)
+            !response.respond_to?(:body) || response.body.to_s.empty?
+          end
+
+          def preserved_stream_error_body(response)
+            return unless response.respond_to?(:[])
+
+            response[STREAM_ERROR_BODY_KEY]
+          rescue StandardError
+            nil
+          end
+
           def context_length_exceeded?(message)
             return false if message.to_s.empty?
 
             CONTEXT_LENGTH_PATTERNS.any? { |pattern| message.match?(pattern) }
           end
         end
+
+        ResponseWithBody = Struct.new(:response, :body) do
+          def status = response.status
+
+          def [](key)
+            response[key] if response.respond_to?(:[])
+          end
+
+          def method_missing(method_name, ...)
+            return response.public_send(method_name, ...) if response.respond_to?(method_name)
+
+            super
+          end
+
+          def respond_to_missing?(method_name, include_private = false)
+            response.respond_to?(method_name, include_private) || super
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/llm/provider.rb

@@ -28,6 +28,7 @@ module Legion
       class Provider
         include Streaming
         include Legion::Logging::Helper
+        include Legion::Cache::Helper
 
         attr_reader :config, :connection
 
@@ -123,10 +124,14 @@ module Legion
           provider_health = health(live:)
           @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
             next unless model_matches_filters?(model, filters)
+            next unless model_allowed?(model.id)
 
             offering_from_model(model, health: provider_health)
           end
           @cached_offerings
+        rescue Faraday::ConnectionFailed => e
+          log.warn("[#{slug}] instance=#{provider_instance_id} unreachable: #{e.message}")
+          []
         end
 
         def health(live: false)
@@ -259,6 +264,8 @@ module Legion
               error = part['error']
               error.is_a?(String) ? error : part.dig('error', 'message')
             end.join('. ')
+          when String
+            body[/"message"\s*:\s*"([^"]{1,500})/, 1] || body
           else
             body
           end
@@ -284,12 +291,14 @@ module Legion
         # ── Model allow-list / deny-list filtering ────────────────────────
 
         def model_whitelist
-          wl = settings[:model_whitelist] if respond_to?(:settings)
+          wl = config.model_whitelist if config.respond_to?(:model_whitelist)
+          wl ||= settings[:model_whitelist] if respond_to?(:settings)
           Array(wl).map { |p| p.to_s.downcase }
         end
 
         def model_blacklist
-          bl = settings[:model_blacklist] if respond_to?(:settings)
+          bl = config.model_blacklist if config.respond_to?(:model_blacklist)
+          bl ||= settings[:model_blacklist] if respond_to?(:settings)
           Array(bl).map { |p| p.to_s.downcase }
         end
 
@@ -371,21 +380,24 @@ module Legion
           nil
         end
 
-        def model_cache_set(key, value, ttl:)
-          return unless defined?(Legion::Cache)
+        def model_detail(model_name)
+          key = model_detail_cache_key(model_name)
+          cached = cache_get(key)
+          return cached if cached
 
-          cache_local_instance? ? local_cache_set(key, value, ttl: ttl) : cache_set(key, value, ttl: ttl)
+          result = fetch_model_detail(model_name)
+          cache_set(key, result, ttl: 86_400) if result
+          result
         rescue StandardError => e
-          handle_exception(e, level: :debug, handled: true, operation: 'lex.provider.model_cache_set')
+          handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.model_detail',
+                              model: model_name)
+          nil
         end
 
-        def model_cache_fetch(key, ttl:, &)
-          return yield unless defined?(Legion::Cache)
-
-          cache_local_instance? ? local_cache_fetch(key, ttl: ttl, &) : cache_fetch(key, ttl: ttl, &)
-        rescue StandardError => e
-          handle_exception(e, level: :debug, handled: true, operation: 'llm.provider.model_cache_fetch', key:)
-          yield
+        # Override in subclasses to make a live API call for model detail.
+        # Must return a Hash with symbol keys (e.g. { context_window: 128000 }).
+        def fetch_model_detail(_model_name)
+          nil
         end
 
         def cache_instance_key
@@ -448,6 +460,26 @@ module Legion
 
         private
 
+        def model_detail_cache_key(model_name)
+          tier = offering_tier
+          instance_key = cache_instance_key
+          cred_fp = credential_cache_fragment
+          key_parts = ['model_info', tier, slug, instance_key, cred_fp, model_name].compact
+          key_parts.join('.')
+        end
+
+        def credential_cache_fragment
+          return nil if cache_local_instance?
+
+          cred = config.respond_to?(:bearer_token) && config.bearer_token
+          cred ||= config.respond_to?(:api_key) && config.api_key
+          cred ||= config.respond_to?(:bedrock_access_key_id) && config.bedrock_access_key_id
+          return nil unless cred
+
+          require 'digest'
+          Digest::SHA256.hexdigest(cred.to_s)[0, 8]
+        end
+
         def validate_paint_inputs!(with:, mask:)
           return if with.nil? && mask.nil?
 

data/lib/legion/extensions/llm/responses/thinking_extractor.rb

@@ -8,9 +8,39 @@ module Legion
         module ThinkingExtractor
           Extraction = Struct.new(:content, :thinking, :signature, :metadata, keyword_init: true)
 
-          THINK_OPEN = '<think>'
-          THINK_CLOSE = '</think>'
-          THINK_PATTERN = %r{<think>(.*?)</think>}m
+          THINK_TAG_PAIRS = [
+            ['<thinking>', '</thinking>'],
+            ['<think>',    '</think>']
+          ].freeze
+          UNTAGGED_PREAMBLE_MAX_LENGTH = 4_000
+          UNTAGGED_PREAMBLE_STARTS = [
+            'the user',
+            'the request',
+            'the prompt',
+            'the question',
+            'i need',
+            'i should',
+            'i will',
+            "i'll",
+            'i can',
+            'we need',
+            'we should',
+            'we will',
+            "we'll",
+            'we can',
+            'let me'
+          ].freeze
+          UNTAGGED_PREAMBLE_PATTERNS = [
+            /
+              \AThe\s+(?:user|request|prompt|question)\b.*\b
+              (?:let\s+me|i'll|i\s+will|i\s+should|i\s+need|i\s+can|respond|answer|reply)\b
+            /imx,
+            /
+              \A(?:I|We)\s+(?:need|should|will|can)\s+(?:to\s+)?
+              (?:answer|respond|reply|confirm|provide|explain|help)\b
+            /imx,
+            /\ALet me\s+(?:answer|respond|reply|confirm|provide|explain|help)\b/im
+          ].freeze
           THINKING_METADATA_KEYS = %i[
             reasoning_content reasoning thinking thinking_text thinking_signature reasoning_signature thought_signature
           ].freeze
@@ -42,20 +72,45 @@ module Legion
             remaining = content.dup
 
             remaining = consume_next_segment(remaining, clean, thinking_parts) until remaining.empty?
+            clean, untagged_thinking = extract_untagged_preamble(clean.strip)
+            thinking_parts << untagged_thinking
 
-            [clean.strip, compact_thinking(thinking_parts)]
+            [clean, compact_thinking(thinking_parts)]
           end
           private_class_method :extract_from_content
 
+          def extract_untagged_preamble(content)
+            return [content, nil] unless content.is_a?(String)
+
+            match = content.match(/\A(?<preamble>.+?)\n{2,}(?<visible>.+)\z/m)
+            return [content, nil] unless match
+
+            preamble = match[:preamble].strip
+            return [content, nil] unless untagged_reasoning_preamble?(preamble)
+
+            [match[:visible].sub(/\A[[:space:]]+/, '').strip, preamble]
+          end
+
+          def untagged_reasoning_preamble_candidate?(content)
+            return false unless content.is_a?(String)
+
+            text = content.lstrip.downcase
+            return false if text.empty?
+
+            UNTAGGED_PREAMBLE_STARTS.any? do |start|
+              start.start_with?(text) || text.start_with?(start)
+            end
+          end
+
           def consume_next_segment(remaining, clean, thinking_parts)
-            close_index = remaining.index(THINK_CLOSE)
-            open_index = remaining.index(THINK_OPEN)
-
-            if close_index && (open_index.nil? || close_index < open_index)
-              thinking_parts << remaining.slice(0, close_index)
-              remaining.slice((close_index + THINK_CLOSE.length)..).to_s.sub(/\A[[:space:]]+/, '')
-            elsif open_index
-              consume_open_think_segment(remaining, open_index, clean, thinking_parts)
+            close_match = next_tag_match(remaining, :close)
+            open_match = next_tag_match(remaining, :open)
+
+            if close_match && (open_match.nil? || close_match[:index] < open_match[:index])
+              thinking_parts << remaining.slice(0, close_match[:index])
+              remaining.slice((close_match[:index] + close_match[:tag].length)..).to_s.sub(/\A[[:space:]]+/, '')
+            elsif open_match
+              consume_open_think_segment(remaining, open_match, clean, thinking_parts)
             else
               clean << remaining
               +''
@@ -63,20 +118,37 @@ module Legion
           end
           private_class_method :consume_next_segment
 
-          def consume_open_think_segment(remaining, open_index, clean, thinking_parts)
-            clean << remaining.slice(0, open_index)
-            after_open = remaining.slice((open_index + THINK_OPEN.length)..).to_s
-            close_index = after_open.index(THINK_CLOSE)
+          def consume_open_think_segment(remaining, open_match, clean, thinking_parts)
+            clean << remaining.slice(0, open_match[:index])
+            after_open = remaining.slice((open_match[:index] + open_match[:tag].length)..).to_s
+            close_index = after_open.index(open_match[:close_tag])
             unless close_index
               thinking_parts << after_open
               return +''
             end
 
             thinking_parts << after_open.slice(0, close_index)
-            after_open.slice((close_index + THINK_CLOSE.length)..).to_s
+            after_open.slice((close_index + open_match[:close_tag].length)..).to_s
           end
           private_class_method :consume_open_think_segment
 
+          def next_tag_match(text, type)
+            matches = THINK_TAG_PAIRS.filter_map do |open_tag, close_tag|
+              tag = type == :open ? open_tag : close_tag
+              index = text.index(tag)
+              { index: index, tag: tag, close_tag: close_tag } if index
+            end
+            matches.min_by { |match| match[:index] }
+          end
+          private_class_method :next_tag_match
+
+          def untagged_reasoning_preamble?(preamble)
+            return false if preamble.length > UNTAGGED_PREAMBLE_MAX_LENGTH
+
+            UNTAGGED_PREAMBLE_PATTERNS.any? { |pattern| preamble.match?(pattern) }
+          end
+          private_class_method :untagged_reasoning_preamble?
+
           def extract_metadata_thinking(metadata)
             compact_thinking(
               [

data/lib/legion/extensions/llm/stream_accumulator.rb

@@ -21,6 +21,9 @@ module Legion
           @thinking_tokens = nil
           @inside_think_tag = false
           @pending_think_tag = +''
+          @active_think_close_tag = nil
+          @untagged_preamble_pending = true
+          @untagged_preamble_buffer = +''
           @latest_tool_call_id = nil
         end
 
@@ -55,6 +58,8 @@ module Legion
         end
 
         def to_message(response)
+          flush_pending_untagged_preamble
+
           Message.new(
             role: :assistant,
             content: content.empty? ? nil : content,
@@ -171,14 +176,63 @@ module Legion
 
         def append_text_with_thinking(text)
           content_chunk, thinking_chunk = extract_think_tags(text)
+          content_chunk, untagged_thinking = extract_untagged_preamble(content_chunk)
           @content << content_chunk
           @last_content_delta << content_chunk
+          if untagged_thinking
+            @thinking_text << untagged_thinking
+            @last_thinking_delta << untagged_thinking
+          end
           return unless thinking_chunk
 
           @thinking_text << thinking_chunk
           @last_thinking_delta << thinking_chunk
         end
 
+        def extract_untagged_preamble(content_chunk)
+          return [content_chunk, nil] unless @untagged_preamble_pending
+          return [content_chunk, nil] unless @content.empty? && @thinking_text.empty?
+          return [content_chunk, nil] if content_chunk.empty?
+
+          candidate = @untagged_preamble_buffer + content_chunk
+          return release_untagged_preamble(candidate) unless candidate_untagged_preamble?(candidate)
+
+          content, thinking = Responses::ThinkingExtractor.extract_untagged_preamble(candidate)
+          return release_untagged_preamble(content, thinking) if thinking
+          return release_untagged_preamble(candidate) if complete_untagged_preamble_candidate?(candidate)
+
+          @untagged_preamble_buffer = candidate
+          ['', nil]
+        end
+
+        def candidate_untagged_preamble?(candidate)
+          Responses::ThinkingExtractor.untagged_reasoning_preamble_candidate?(candidate)
+        end
+
+        def complete_untagged_preamble_candidate?(candidate)
+          candidate.match?(/\n{2,}/) || candidate.length > Responses::ThinkingExtractor::UNTAGGED_PREAMBLE_MAX_LENGTH
+        end
+
+        def release_untagged_preamble(content, thinking = nil)
+          @untagged_preamble_pending = false
+          @untagged_preamble_buffer = +''
+          [content, thinking]
+        end
+
+        def flush_pending_untagged_preamble
+          return if @untagged_preamble_buffer.empty?
+
+          content, thinking = Responses::ThinkingExtractor.extract_untagged_preamble(@untagged_preamble_buffer)
+          if thinking
+            @content << content
+            @thinking_text << thinking
+          else
+            @content << @untagged_preamble_buffer
+          end
+          @untagged_preamble_buffer = +''
+          @untagged_preamble_pending = false
+        end
+
         def append_thinking_from_chunk(chunk)
           thinking = chunk.thinking
           return unless thinking
@@ -191,8 +245,6 @@ module Legion
         end
 
         def extract_think_tags(text)
-          start_tag = '<think>'
-          end_tag = '</think>'
           remaining = @pending_think_tag + text
           @pending_think_tag = +''
 
@@ -201,9 +253,9 @@ module Legion
 
           until remaining.empty?
             remaining = if @inside_think_tag
-                          consume_think_content(remaining, end_tag, thinking)
+                          consume_think_content(remaining, @active_think_close_tag, thinking)
                         else
-                          consume_non_think_content(remaining, start_tag, output)
+                          consume_non_think_content(remaining, output)
                         end
           end
 
@@ -215,41 +267,59 @@ module Legion
           if end_index
             thinking << remaining.slice(0, end_index)
             @inside_think_tag = false
+            @active_think_close_tag = nil
             remaining.slice((end_index + end_tag.length)..) || +''
           else
-            suffix_len = longest_suffix_prefix(remaining, end_tag)
+            suffix_len = longest_suffix_prefix(remaining, [end_tag])
             thinking << remaining.slice(0, remaining.length - suffix_len)
             @pending_think_tag = remaining.slice(-suffix_len, suffix_len)
             +''
           end
         end
 
-        def consume_non_think_content(remaining, start_tag, output)
-          unmatched_close = remaining.index('</think>')
-          start_index = remaining.index(start_tag)
-          if unmatched_close && (start_index.nil? || unmatched_close < start_index)
+        def consume_non_think_content(remaining, output)
+          unmatched_close = next_stream_tag_match(remaining, :close)
+          start_match = next_stream_tag_match(remaining, :open)
+          if unmatched_close && (start_match.nil? || unmatched_close[:index] < start_match[:index])
             consume_unmatched_think_close(remaining, unmatched_close)
-          elsif start_index
-            output << remaining.slice(0, start_index)
+          elsif start_match
+            output << remaining.slice(0, start_match[:index])
             @inside_think_tag = true
-            remaining.slice((start_index + start_tag.length)..) || +''
+            @active_think_close_tag = start_match[:close_tag]
+            remaining.slice((start_match[:index] + start_match[:tag].length)..) || +''
           else
-            suffix_len = longest_suffix_prefix(remaining, start_tag)
+            suffix_len = longest_suffix_prefix(remaining, stream_tag_tokens)
             output << remaining.slice(0, remaining.length - suffix_len)
             @pending_think_tag = remaining.slice(-suffix_len, suffix_len)
             +''
           end
         end
 
-        def consume_unmatched_think_close(remaining, close_index)
-          end_tag = '</think>'
-          thinking = remaining.slice(0, close_index)
+        def consume_unmatched_think_close(remaining, close_match)
+          thinking = remaining.slice(0, close_match[:index])
           @thinking_text << thinking
           @last_thinking_delta << thinking
-          remaining.slice((close_index + end_tag.length)..).to_s.sub(/\A[[:space:]]+/, '')
+          remaining.slice((close_match[:index] + close_match[:tag].length)..).to_s.sub(/\A[[:space:]]+/, '')
+        end
+
+        def next_stream_tag_match(text, type)
+          matches = Responses::ThinkingExtractor::THINK_TAG_PAIRS.filter_map do |open_tag, close_tag|
+            tag = type == :open ? open_tag : close_tag
+            index = text.index(tag)
+            { index: index, tag: tag, close_tag: close_tag } if index
+          end
+          matches.min_by { |match| match[:index] }
+        end
+
+        def stream_tag_tokens
+          Responses::ThinkingExtractor::THINK_TAG_PAIRS.flat_map { |open_tag, close_tag| [open_tag, close_tag] }
+        end
+
+        def longest_suffix_prefix(text, tags)
+          tags.map { |tag| longest_suffix_prefix_for_tag(text, tag) }.max || 0
         end
 
-        def longest_suffix_prefix(text, tag)
+        def longest_suffix_prefix_for_tag(text, tag)
           max = [text.length, tag.length - 1].min
           max.downto(1) do |len|
             return len if text.end_with?(tag[0, len])

data/lib/legion/extensions/llm/streaming.rb

@@ -93,10 +93,48 @@ module Legion
 
         def handle_failed_response(chunk, buffer, env)
           buffer << chunk
+          body_persisted = persist_failed_response_body(buffer, env)
           error_data = Legion::JSON.parse(buffer, symbolize_names: false)
           handle_parsed_error(error_data, env)
-        rescue Legion::JSON::ParseError => e
-          handle_exception(e, level: :warn, handled: true, operation: 'llm.streaming.handle_failed_response')
+        rescue Legion::JSON::ParseError
+          return if body_persisted
+
+          raise_partial_streaming_error(buffer, env)
+        end
+
+        def persist_failed_response_body(buffer, env)
+          custom_persisted = persist_failed_response_custom_body?(buffer, env)
+          body_persisted = persist_failed_response_env_body?(buffer, env)
+          custom_persisted || body_persisted
+        end
+
+        def persist_failed_response_env_body?(buffer, env)
+          return false unless env.respond_to?(:body=)
+
+          env.body = buffer.dup
+          true
+        end
+
+        def persist_failed_response_custom_body?(buffer, env)
+          return false unless env.respond_to?(:[]=)
+
+          env[ErrorMiddleware::STREAM_ERROR_BODY_KEY] = buffer.dup
+          true
+        rescue StandardError
+          false
+        end
+
+        def raise_partial_streaming_error(buffer, env)
+          partial = buffer[/"message"\s*:\s*"([^"]{1,200})/, 1]
+          status  = env&.status || 0
+          msg     = if partial
+                      "Provider error (status #{status}): #{partial}"
+                    else
+                      "Provider error (status #{status}) - response body incomplete"
+                    end
+          log.warn "[llm][streaming] action=handle_failed_response status=#{status} " \
+                   "partial_body=#{buffer.length}b msg=#{partial.inspect}"
+          raise Legion::Extensions::Llm::ServerError, msg
         end
 
         def handle_sse(chunk, parser, env, &)

data/lib/legion/extensions/llm/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Llm
-      VERSION = '0.4.9'
+      VERSION = '0.4.13'
     end
   end
 end

data/lib/legion/extensions/llm.rb

@@ -9,6 +9,15 @@ require 'faraday/multipart'
 require 'faraday/retry'
 require 'legion/json'
 require 'legion/logging'
+# legion/cache writes DEBUG lines to $stdout on first load; suppress them here
+# so callers that capture our stdout (e.g. Open3-based integration tests) are unaffected.
+begin
+  old_stdout = $stdout
+  $stdout = File.open(File::NULL, 'w')
+  require 'legion/cache'
+ensure
+  $stdout = old_stdout
+end
 require 'logger'
 require 'marcel'
 require 'ruby_llm/schema'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm
 version: !ruby/object:Gem::Version
-  version: 0.4.9
+  version: 0.4.13
 platform: ruby
 authors:
 - LegionIO
@@ -108,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: legion-cache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: legion-crypt
   requirement: !ruby/object:Gem::Requirement