legion-llm 0.8.26 → 0.8.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 942f34663b8d915ee982996b5b2e63e26a7edf79a7aac17f8ce71ed1829dff01
-  data.tar.gz: dd78dd3bd79c9f1cf19d170f4ee2905fc92865dd3e21b107856c973eaf752fb5
+  metadata.gz: 523afac32d76644a92db4f6af5228c9ff9856521ccffb7cff0a0e8194570a432
+  data.tar.gz: b58073ec104d42eb18436fd708a33bea881931386dfab1e578f0570d891a6f55
 SHA512:
-  metadata.gz: bfc1f55dce2a3eda78b5b6ab2405b6ce5d4e58fa841a81bb304af3bbe9a5b52851023c845d898713cfa87d9e292cd5fd1545464a7e0937eadde6f8668595ccc2
-  data.tar.gz: 4cad8eb9c6b6cfc79c1ffce687b7fddbb7b47d4e22ec9bca424f2dbb061ed83fff97d4ab2bbec441d3b319922316a238b057752210f1d7908e0d7169380485e9
+  metadata.gz: 205d3a1ef6f1c9e8712bc61e2d88382b88a91560343ab6be7e5c863f2b839ea3d384f5a5642240f7a62fed73ed96aff7b653855c15c1cb5517a43d342306a54b
+  data.tar.gz: 8847c3be8580a5c1c62bd61b83c72ef53db974b19a8567fd102827b748e9113bfc3dc0af7aa14e23d25b524453b2465dd8053e55f64ee2798f9ae9b387cac264

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Legion LLM Changelog
 
+## [0.8.28] - 2026-04-24
+
+### Fixed
+- Model/provider mismatch when clients send a model name (e.g., `qwen3.5:latest`) without an explicit provider. The fallback paths blindly paired it with `default_provider` (typically `bedrock`), causing `RubyLLM::ModelNotFoundError`. Now infers the correct provider from model naming patterns before falling back to the global default.
+- `arbitrage_fallback` hardcoded `:cloud` tier and `:bedrock` provider when inference failed. Now uses `PROVIDER_TIER` to resolve the correct tier for the inferred provider.
+
+### Added
+- `Router.infer_provider_for_model(model)` — public method that maps model naming patterns to providers. Recognizes Ollama-style models (`:` or `/` in name), Bedrock (`us.*`), OpenAI (`gpt-*`, `o1-*`/`o3-*`/`o4-*`), Anthropic (`claude-*`), and Gemini (`gemini-*`).
+
+## [0.8.27] - 2026-04-24
+
+### Fixed
+- vLLM provider sent `developer` message role (OpenAI convention) which Qwen's chat template rejects. Added `Vllm::Chat` module that overrides `format_messages` and `format_role` to always send `system`.
+- vLLM provider called `OpenAI::Chat.render_payload` as a module function without provider instance context, causing `NoMethodError` on `openai_use_system_role`. Rewrote to use `super` with instance method overrides.
+- Audit events included the full conversation history in every message — quadratic payload growth. Now caps at the last 20 messages (configurable via `compliance.audit_max_messages`). Full conversation reconstructable via `conversation_id`.
+
+### Added
+- vLLM `chat_template_kwargs` with `enable_thinking` sent on every request so vLLM separates reasoning into the `reasoning` response field instead of inline `<think>` tags.
+- `providers.vllm.enable_thinking` setting (default: `true`). Controls whether thinking is enabled for vLLM requests. Per-request `thinking` param overrides.
+
 ## [0.8.26] - 2026-04-24
 
 ### Added

data/lib/legion/llm/inference/audit_publisher.rb

@@ -40,7 +40,7 @@ module Legion
             timeline:         compact_timeline(response.timeline),
             classification:   response.classification,
             tracing:          response.tracing,
-            messages:         request.messages,
+            messages:         current_turn_messages(request.messages),
             response_content: msg_content,
             tools_used:       tools_data,
             timestamp:        Time.now,
@@ -109,6 +109,23 @@ module Legion
           end
         end
 
+        def current_turn_messages(messages)
+          return messages unless messages.is_a?(Array)
+
+          max = audit_max_messages
+          return messages if messages.size <= max
+
+          messages.last(max)
+        end
+
+        def audit_max_messages
+          return 20 unless defined?(Legion::Settings)
+
+          Legion::Settings[:llm].dig(:compliance, :audit_max_messages) || 20
+        rescue StandardError
+          20
+        end
+
         def build_message_context(response:, **)
           {
             request_id:      response.request_id,

data/lib/legion/llm/inference/executor.rb

@@ -328,7 +328,9 @@ module Legion
             end
           end
 
-          @resolved_provider = provider || Legion::LLM.settings[:default_provider]
+          @resolved_provider = provider ||
+                               (model && Router.infer_provider_for_model(model)) ||
+                               Legion::LLM.settings[:default_provider]
           @resolved_model = model || Legion::LLM.settings[:default_model]
 
           log.info "[llm][inference] resolved provider=#{@resolved_provider} model=#{@resolved_model}"
@@ -846,6 +848,8 @@ module Legion
           duration_ms = started_at ? ((finished_at - started_at) * 1000).round : nil
 
           result_str = (raw.is_a?(String) ? raw : raw.to_s)
+          result_str = result_str.encode('UTF-8', invalid: :replace, undef: :replace, replace: '�') unless result_str.valid_encoding?
+          result_str = result_str.delete("\x00")
           is_error = raw.is_a?(Hash) && (raw[:error] || raw['error']) ? true : false
 
           @pending_tool_history_mutex.synchronize do

data/lib/legion/llm/inference.rb

@@ -496,7 +496,8 @@ module Legion
         end
 
         model ||= Legion::LLM.settings[:default_model]
-        provider ||= Legion::LLM.settings[:default_provider]
+        provider ||= (model && Router.infer_provider_for_model(model)) ||
+                     Legion::LLM.settings[:default_provider]
 
         opts = {}
         opts[:model] = model if model

data/lib/legion/llm/patches/ruby_llm_vllm.rb

@@ -3,6 +3,47 @@
 module RubyLLM
   module Providers
     class Vllm < OpenAI
+      module Chat
+        def format_role(role)
+          role.to_s
+        end
+
+        def format_messages(messages)
+          messages.map do |msg|
+            {
+              role:         format_role(msg.role),
+              content:      OpenAI::Media.format_content(msg.content),
+              tool_calls:   format_tool_calls(msg.tool_calls),
+              tool_call_id: msg.tool_call_id
+            }.compact.merge(OpenAI::Chat.format_thinking(msg))
+          end
+        end
+
+        def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil,
+                           thinking: nil, tool_prefs: nil)
+          payload = super
+          enable = if thinking.nil?
+                     vllm_thinking_default
+                   else
+                     thinking ? true : false
+                   end
+          payload[:chat_template_kwargs] = { enable_thinking: enable }
+          payload
+        end
+
+        private
+
+        def vllm_thinking_default
+          return true unless defined?(Legion::Settings)
+
+          Legion::Settings[:llm].dig(:providers, :vllm, :enable_thinking) != false
+        rescue StandardError
+          true
+        end
+      end
+
+      include Vllm::Chat
+
       def api_base
         @config.vllm_api_base
       end

data/lib/legion/llm/router.rb

@@ -18,7 +18,22 @@ module Legion
                         gemini: :cloud, azure: :cloud, ollama: :local, vllm: :local }.freeze
       PROVIDER_ORDER = %i[ollama vllm bedrock azure gemini anthropic openai].freeze
 
+      OLLAMA_MODEL_PATTERN = %r{[:/]}
+
       class << self
+        def infer_provider_for_model(model)
+          return nil if model.nil? || model.to_s.empty?
+
+          model_s = model.to_s
+          return :bedrock if model_s.start_with?('us.')
+          return :openai if model_s.match?(/\Agpt-|\Ao[134]-/)
+          return :anthropic if model_s.start_with?('claude-')
+          return :gemini if model_s.start_with?('gemini-')
+          return :ollama if model_s.match?(OLLAMA_MODEL_PATTERN)
+
+          nil
+        end
+
         # Resolve an LLM routing intent to a tier/provider/model decision.
         #
         # @param intent   [Hash, nil] routing intent (capability, privacy, etc.)
@@ -95,18 +110,12 @@ module Legion
           model = Arbitrage.cheapest_for(capability: capability)
           return nil unless model
 
-          provider = Arbitrage.cost_table[model] ? infer_provider(model) : nil
-          log.debug("Router: arbitrage fallback selected model=#{model}")
-          Resolution.new(tier: :cloud, provider: provider || :bedrock, model: model, rule: 'arbitrage_fallback')
-        end
-
-        def infer_provider(model)
-          return :ollama if model.include?('llama')
-          return :bedrock if model.start_with?('us.')
-          return :openai if model.start_with?('gpt')
-          return :google if model.start_with?('gemini')
+          provider = infer_provider_for_model(model)
+          return nil unless provider
 
-          :anthropic if model.start_with?('claude')
+          tier = PROVIDER_TIER.fetch(provider, :cloud)
+          log.debug("Router: arbitrage fallback selected model=#{model} provider=#{provider} tier=#{tier}")
+          Resolution.new(tier: tier, provider: provider, model: model, rule: 'arbitrage_fallback')
         end
 
         def explicit_resolution(tier, provider, model)

data/lib/legion/llm/settings.rb

@@ -377,10 +377,11 @@ module Legion
             base_url:      'http://localhost:11434'
           },
           vllm:      {
-            enabled:       false,
-            default_model: 'qwen3.6-27b',
-            base_url:      'http://localhost:8000/v1',
-            api_key:       nil
+            enabled:         false,
+            default_model:   'qwen3.6-27b',
+            base_url:        'http://localhost:8000/v1',
+            api_key:         nil,
+            enable_thinking: true
           }
         }
       end

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.8.26'
+    VERSION = '0.8.28'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.8.26
+  version: 0.8.28
 platform: ruby
 authors:
 - Esity