legion-llm 0.8.25 → 0.8.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b37f926f357a862c036b2e3d4676b579ae505ba6f76b7b7da332ba3aeeed8ac
-  data.tar.gz: 59ab5db376ac8be2a45cda642dec1d22c8eda3e76782b9f2cd5689233bcc2063
+  metadata.gz: 2cccf9351fd9f4db59b1548197bf7b78c5947e85183535e86ede5c3359d71b89
+  data.tar.gz: 14e3a1b5b6648bea618941f84473e63aeccee7edc9520366d09dde8d27b00a7b
 SHA512:
-  metadata.gz: 6994e774e6a0551c720ef0a2c7a5aa4681f324c9e8c09a624ca2e227565cd87ca26f1457ed108967ae3e17593e6a5ee8df7fdf20515aa0ccdd277f2eb16db827
-  data.tar.gz: 815d14a99f379c2d655341776507e96be1625582b4fd4989597c4c299a1086be394dbad785b4b33e2d1423419a37ea37392ee617fa75fbb88208dac6d6864bce
+  metadata.gz: 31ec279fcb498e5cc3308bcefcb6adc94915c36867967fd08aa3f0422d4c583f83bc0ace1db9a47ecd45371a0ce0c82542fea7015c1474f5b7386409d789e5e0
+  data.tar.gz: '083bd8e581399a574b313a31784eacca4424fadb131f82cd17a5a7e840420da14ec3e5f589efa88b7be6be8a76916439c88bbf936a6d13c9a9435bd8fd04245c'

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Legion LLM Changelog
 
+## [0.8.27] - 2026-04-24
+
+### Fixed
+- vLLM provider sent `developer` message role (OpenAI convention) which Qwen's chat template rejects. Added `Vllm::Chat` module that overrides `format_messages` and `format_role` to always send `system`.
+- vLLM provider called `OpenAI::Chat.render_payload` as a module function without provider instance context, causing `NoMethodError` on `openai_use_system_role`. Rewrote to use `super` with instance method overrides.
+- Audit events included the full conversation history in every message — quadratic payload growth. Now caps at the last 20 messages (configurable via `compliance.audit_max_messages`). Full conversation reconstructable via `conversation_id`.
+
+### Added
+- vLLM `chat_template_kwargs` with `enable_thinking` sent on every request so vLLM separates reasoning into the `reasoning` response field instead of inline `<think>` tags.
+- `providers.vllm.enable_thinking` setting (default: `true`). Controls whether thinking is enabled for vLLM requests. Per-request `thinking` param overrides.
+
+## [0.8.26] - 2026-04-24
+
+### Added
+- First-class vLLM provider support. vLLM exposes an OpenAI-compatible API and is registered as a new RubyLLM provider (`:vllm`). Configured via `providers.vllm.base_url` in settings. Mapped to `:fleet` tier in the router.
+- vLLM discovery via `/v1/models` endpoint. Caches model list with `max_model_len` (context window size) using the same TTL as Ollama discovery. Health checks via `/health` endpoint.
+- Context overflow escalation: when vLLM rejects a request due to context length limits (32k on V100 hardware), the executor automatically falls back to cloud/frontier providers.
+
+### Changed
+- `find_fallback_provider` in `Executor` now skips all local providers (`:ollama` and `:vllm`) when searching for fallbacks, not just `:ollama`. Ensures context overflow escalates to cloud/frontier.
+- `Router::PROVIDER_ORDER` updated: `:vllm` inserted after `:ollama` and before `:bedrock`.
+- `default_provider_for_tier(:fleet)` returns `:vllm` when vLLM is enabled, falls back to `:ollama`.
+
 ## [0.8.25] - 2026-04-24
 
 ### Fixed

data/lib/legion/llm/call/providers.rb

@@ -76,6 +76,8 @@ module Legion
             config[:api_base] && (usable_setting?(config[:api_key]) || usable_setting?(config[:auth_token]))
           when :ollama
             ollama_running?(config)
+          when :vllm
+            vllm_running?(config)
           else
             usable_setting?(config[:api_key])
           end
@@ -106,6 +108,22 @@ module Legion
           false
         end
 
+        def vllm_running?(config)
+          require 'faraday'
+          url = config[:base_url] || 'http://localhost:8000/v1'
+          base = url.sub(%r{/+\z}, '').sub(%r{/v1\z}, '')
+          log.debug "[llm][providers] vllm_running? url=#{base}/health"
+          response = Faraday.new(url: base) do |f|
+            f.options.timeout = 2
+            f.options.open_timeout = 2
+            f.adapter Faraday.default_adapter
+          end.get('/health')
+          response.success?
+        rescue StandardError => e
+          handle_exception(e, level: :debug, operation: 'llm.providers.vllm_running', base_url: url)
+          false
+        end
+
         def apply_provider_config(provider, config)
           case provider
           when :bedrock   then configure_bedrock(config)
@@ -114,6 +132,7 @@ module Legion
           when :gemini    then configure_gemini(config)
           when :azure     then configure_azure(config)
           when :ollama    then configure_ollama(config)
+          when :vllm then configure_vllm(config)
           else
             log.warn "[llm][providers] unknown provider=#{provider}"
           end
@@ -214,6 +233,15 @@ module Legion
           log.info "[llm][providers] configured ollama base_url=#{config[:base_url].inspect}"
         end
 
+        def configure_vllm(config)
+          base_url = config[:base_url] || 'http://localhost:8000/v1'
+          RubyLLM.configure do |c|
+            c.vllm_api_base = base_url
+            c.vllm_api_key = config[:api_key] if config[:api_key]
+          end
+          log.info "[llm][providers] configured vllm base_url=#{base_url.inspect}"
+        end
+
         SAAS_PROVIDERS = %i[bedrock anthropic openai gemini azure].freeze
 
         def verify_providers

data/lib/legion/llm/discovery/vllm.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'faraday'
+
+require 'legion/logging/helper'
+require 'legion/json'
+
+module Legion
+  module LLM
+    module Discovery
+      module Vllm
+        extend Legion::Logging::Helper
+
+        class << self
+          def models
+            ensure_fresh
+            @models || []
+          end
+
+          def model_names
+            models.map { |m| m[:id] }
+          end
+
+          def model_available?(name)
+            model_names.any? { |n| n == name }
+          end
+
+          def max_context(name)
+            model = models.find { |m| m[:id] == name }
+            model&.dig(:max_model_len)
+          end
+
+          def healthy?
+            response = health_connection.get('/health')
+            response.success?
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'llm.discovery.vllm.healthy')
+            false
+          end
+
+          def refresh!
+            response = connection.get('/v1/models')
+            if response.success?
+              parsed = Legion::JSON.load(response.body)
+              @models = parsed[:data] || []
+              log.debug "[llm][discovery][vllm] model list refreshed count=#{@models.size}"
+            else
+              log.warn "[llm][discovery][vllm] HTTP failure status=#{response.status}"
+              @models ||= []
+            end
+          rescue StandardError => e
+            handle_exception(e, level: :warn, operation: 'llm.discovery.vllm.refresh')
+            @models ||= []
+          ensure
+            @last_refreshed_at = Time.now
+          end
+
+          def reset!
+            @models = nil
+            @last_refreshed_at = nil
+          end
+
+          def stale?
+            return true if @last_refreshed_at.nil?
+
+            ttl = discovery_settings[:refresh_seconds] || 60
+            Time.now - @last_refreshed_at > ttl
+          end
+
+          private
+
+          def ensure_fresh
+            refresh! if stale?
+          end
+
+          def connection
+            Faraday.new(url: vllm_base_url) do |f|
+              f.options.timeout = 3
+              f.options.open_timeout = 2
+              f.adapter Faraday.default_adapter
+            end
+          end
+
+          def health_connection
+            base = vllm_base_url.sub(%r{/+\z}, '').sub(%r{/v1\z}, '')
+            Faraday.new(url: base) do |f|
+              f.options.timeout = 2
+              f.options.open_timeout = 2
+              f.adapter Faraday.default_adapter
+            end
+          end
+
+          def vllm_base_url
+            return 'http://localhost:8000/v1' unless Legion.const_defined?('Settings', false)
+
+            Legion::Settings[:llm].dig(:providers, :vllm, :base_url) || 'http://localhost:8000/v1'
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'llm.discovery.vllm.base_url')
+            'http://localhost:8000/v1'
+          end
+
+          def discovery_settings
+            return {} unless Legion.const_defined?('Settings', false)
+
+            Legion::Settings[:llm][:discovery] || {}
+          rescue StandardError => e
+            handle_exception(e, level: :debug, operation: 'llm.discovery.vllm.settings')
+            {}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/discovery.rb

@@ -2,6 +2,7 @@
 
 require 'legion/logging/helper'
 require_relative 'discovery/ollama'
+require_relative 'discovery/vllm'
 require_relative 'discovery/system'
 
 module Legion
@@ -23,15 +24,21 @@ module Legion
 
         def run
           log.debug '[llm][discovery] run.enter'
-          return unless Legion::LLM.settings.dig(:providers, :ollama, :enabled)
 
-          Ollama.refresh!
-          System.refresh!
+          if Legion::LLM.settings.dig(:providers, :ollama, :enabled)
+            Ollama.refresh!
+            System.refresh!
+            names = Ollama.model_names
+            log.info "[llm][discovery] ollama model_count=#{names.size} models=#{names.join(', ')}"
+            log.info "[llm][discovery] system total_mb=#{System.total_memory_mb} available_mb=#{System.available_memory_mb}"
+          end
 
-          names = Ollama.model_names
-          count = names.size
-          log.info "[llm][discovery] ollama model_count=#{count} models=#{names.join(', ')}"
-          log.info "[llm][discovery] system total_mb=#{System.total_memory_mb} available_mb=#{System.available_memory_mb}"
+          if Legion::LLM.settings.dig(:providers, :vllm, :enabled)
+            Vllm.refresh!
+            names = Vllm.model_names
+            contexts = names.map { |n| "#{n}(#{Vllm.max_context(n)})" }
+            log.info "[llm][discovery] vllm model_count=#{names.size} models=#{contexts.join(', ')}"
+          end
         rescue StandardError => e
           handle_exception(e, level: :warn, operation: 'llm.discovery.run')
         end

data/lib/legion/llm/inference/audit_publisher.rb

@@ -40,7 +40,7 @@ module Legion
             timeline:         compact_timeline(response.timeline),
             classification:   response.classification,
             tracing:          response.tracing,
-            messages:         request.messages,
+            messages:         current_turn_messages(request.messages),
             response_content: msg_content,
             tools_used:       tools_data,
             timestamp:        Time.now,
@@ -109,6 +109,23 @@ module Legion
           end
         end
 
+        def current_turn_messages(messages)
+          return messages unless messages.is_a?(Array)
+
+          max = audit_max_messages
+          return messages if messages.size <= max
+
+          messages.last(max)
+        end
+
+        def audit_max_messages
+          return 20 unless defined?(Legion::Settings)
+
+          Legion::Settings[:llm].dig(:compliance, :audit_max_messages) || 20
+        rescue StandardError
+          20
+        end
+
         def build_message_context(response:, **)
           {
             request_id:      response.request_id,

data/lib/legion/llm/inference/executor.rb

@@ -1030,7 +1030,7 @@ module Legion
           providers.each do |name, config|
             next unless config.is_a?(Hash) && config[:enabled]
             next if exclude.include?(name) || exclude.include?(name.to_s)
-            next if name == :ollama
+            next if %i[ollama vllm].include?(name)
             next unless config[:default_model]
 
             return { provider: name, model: config[:default_model] }

data/lib/legion/llm/patches/ruby_llm_vllm.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Vllm < OpenAI
+      module Chat
+        def format_role(role)
+          role.to_s
+        end
+
+        def format_messages(messages)
+          messages.map do |msg|
+            {
+              role:         format_role(msg.role),
+              content:      OpenAI::Media.format_content(msg.content),
+              tool_calls:   format_tool_calls(msg.tool_calls),
+              tool_call_id: msg.tool_call_id
+            }.compact.merge(OpenAI::Chat.format_thinking(msg))
+          end
+        end
+
+        def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil,
+                           thinking: nil, tool_prefs: nil)
+          payload = super
+          enable = if thinking.nil?
+                     vllm_thinking_default
+                   else
+                     thinking ? true : false
+                   end
+          payload[:chat_template_kwargs] = { enable_thinking: enable }
+          payload
+        end
+
+        private
+
+        def vllm_thinking_default
+          return true unless defined?(Legion::Settings)
+
+          Legion::Settings[:llm].dig(:providers, :vllm, :enable_thinking) != false
+        rescue StandardError
+          true
+        end
+      end
+
+      include Vllm::Chat
+
+      def api_base
+        @config.vllm_api_base
+      end
+
+      def headers
+        return {} unless @config.vllm_api_key
+
+        { 'Authorization' => "Bearer #{@config.vllm_api_key}" }
+      end
+
+      class << self
+        def configuration_options
+          %i[vllm_api_base vllm_api_key]
+        end
+
+        def configuration_requirements
+          %i[vllm_api_base]
+        end
+
+        def local?
+          true
+        end
+
+        def capabilities
+          nil
+        end
+      end
+    end
+  end
+end
+
+RubyLLM::Provider.register :vllm, RubyLLM::Providers::Vllm

data/lib/legion/llm/router.rb

@@ -15,8 +15,8 @@ module Legion
       extend Legion::Logging::Helper
 
       PROVIDER_TIER = { bedrock: :cloud, anthropic: :frontier, openai: :frontier,
-                        gemini: :cloud, azure: :cloud, ollama: :local }.freeze
-      PROVIDER_ORDER = %i[ollama bedrock azure gemini anthropic openai].freeze
+                        gemini: :cloud, azure: :cloud, ollama: :local, vllm: :local }.freeze
+      PROVIDER_ORDER = %i[ollama vllm bedrock azure gemini anthropic openai].freeze
 
       class << self
         # Resolve an LLM routing intent to a tier/provider/model decision.
@@ -296,8 +296,11 @@ module Legion
 
         def default_provider_for_tier(tier)
           case tier.to_sym
-          when :local, :fleet
+          when :local
             :ollama
+          when :fleet
+            vllm_config = Legion::Settings[:llm].dig(:providers, :vllm)
+            vllm_config.is_a?(Hash) && vllm_config[:enabled] ? :vllm : :ollama
           when :openai_compat
             :openai
           when :cloud
@@ -316,7 +319,13 @@ module Legion
             ollama = Legion::Settings[:llm].dig(:providers, :ollama) || {}
             ollama[:default_model] || 'llama3'
           when :fleet
-            'llama4:70b'
+            vllm_config = Legion::Settings[:llm].dig(:providers, :vllm) || {}
+            if vllm_config[:enabled]
+              vllm_config[:default_model] || 'qwen3.6-27b'
+            else
+              ollama = Legion::Settings[:llm].dig(:providers, :ollama) || {}
+              ollama[:default_model] || 'llama3'
+            end
           when :openai_compat
             'gpt-4o'
           when :cloud

data/lib/legion/llm/settings.rb

@@ -375,6 +375,13 @@ module Legion
             enabled:       false,
             default_model: 'qwen3.5:latest',
             base_url:      'http://localhost:11434'
+          },
+          vllm:      {
+            enabled:         false,
+            default_model:   'qwen3.6-27b',
+            base_url:        'http://localhost:8000/v1',
+            api_key:         nil,
+            enable_thinking: true
           }
         }
       end

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.8.25'
+    VERSION = '0.8.27'
   end
 end

data/lib/legion/llm.rb

@@ -4,6 +4,7 @@ require 'legion/logging/helper'
 
 require 'ruby_llm'
 require_relative 'llm/patches/ruby_llm_parallel_tools'
+require_relative 'llm/patches/ruby_llm_vllm'
 require_relative 'llm/version'
 require_relative 'llm/errors'
 require_relative 'llm/settings'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.8.25
+  version: 0.8.27
 platform: ruby
 authors:
 - Esity
@@ -262,6 +262,7 @@ files:
 - lib/legion/llm/discovery.rb
 - lib/legion/llm/discovery/ollama.rb
 - lib/legion/llm/discovery/system.rb
+- lib/legion/llm/discovery/vllm.rb
 - lib/legion/llm/errors.rb
 - lib/legion/llm/fleet.rb
 - lib/legion/llm/fleet/dispatcher.rb
@@ -323,6 +324,7 @@ files:
 - lib/legion/llm/metering/tracker.rb
 - lib/legion/llm/metering/usage.rb
 - lib/legion/llm/patches/ruby_llm_parallel_tools.rb
+- lib/legion/llm/patches/ruby_llm_vllm.rb
 - lib/legion/llm/quality.rb
 - lib/legion/llm/quality/checker.rb
 - lib/legion/llm/quality/confidence/score.rb