lex-llm-vllm 0.2.8 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0ec67bb352cf0bd9dc3ae3f0102e7d26a27ebc8c64c56d98811742c4b75e4ae0
-  data.tar.gz: de092940c2c186e982b0d453b4f1f94157a868626e2a7b4b16cd6c0c1596b28c
+  metadata.gz: a3082ce3d1b3d61f220aed833a0b87138ad199cce52cfb455683c04d57db4f10
+  data.tar.gz: a542a059e10c1a12a58fa68f611706e3c6007c0642ba57777ddf2e7b31829e5e
 SHA512:
-  metadata.gz: aad15995696865b3cc0a6e42b6e627e2d4a5dbdb7768ec676d3867964fad107709ce10bfe908128a6575d8bc81d0a8ee9b97b45c554db093b12dbea5b67300c0
-  data.tar.gz: ecfe08d54a3113188c09f642e1b457af39eca29542315d529c477f3f2cac4096a7a1485fd2977fb6acdb7a4c6885f904510b9a3653eb9a966da5ffb6b60ac5ba
+  metadata.gz: 864d11b0394b30b9df44c5e5ffb97dcba939311c30b87c9588ca8dea73b92f2087c5a6af6c052606f97c63e93201e964d5a358b95ca6b32905f9a087d860bf80
+  data.tar.gz: 22df5b5ea5c9dedabe193cd2caa83944eef6f0b107f98605216587eda77ce53efb9fe4fea596fd796415d9c1e5ea7fda2d4079890ef805fd054ba78d7f398d9f

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.2.10 - 2026-05-13
+
+- Add `fetch_model_detail` to re-fetch `/v1/models` for `context_window` on a cache miss.
+- Pre-warm the model detail cache during offering discovery via `cache_set` using `model_detail_cache_key`.
+
+## 0.2.9 - 2026-05-12
+
+- Route fleet actor load failures through `Legion::Logging::Helper` instead of direct warnings.
+- Add debug logging around vLLM instance discovery, fleet worker dispatch, offering construction, payload rendering, and management endpoints.
+
 ## 0.2.8 - 2026-05-07
 
 - Read vLLM thinking defaults from the active provider instance config so per-instance `enable_thinking` settings affect chat payloads.

data/lib/legion/extensions/llm/vllm/actors/fleet_worker.rb

@@ -3,7 +3,11 @@
 begin
   require 'legion/extensions/actors/subscription'
 rescue LoadError => e
-  warn(e.message) if $VERBOSE
+  require 'legion/extensions/llm/vllm'
+  unless defined?(Legion::Extensions::Actors::Subscription)
+    Legion::Extensions::Llm::Vllm.handle_exception(e, level: :warn, handled: false,
+                                                      operation: 'vllm.fleet_worker.load_actor_runtime')
+  end
 end
 
 unless defined?(Legion::Extensions::Actors::Subscription)
@@ -12,6 +16,7 @@ end
 
 require 'legion/extensions/llm/vllm'
 require 'legion/extensions/llm/fleet/provider_responder'
+require 'legion/logging'
 
 module Legion
   module Extensions
@@ -20,6 +25,8 @@ module Legion
         module Actor
           # Subscription actor for vLLM fleet request consumption.
           class FleetWorker < Legion::Extensions::Actors::Subscription
+            include Legion::Logging::Helper
+
             def runner_class
               'Legion::Extensions::Llm::Vllm::Runners::FleetWorker'
             end
@@ -33,7 +40,9 @@ module Legion
             end
 
             def enabled?
-              Legion::Extensions::Llm::Fleet::ProviderResponder.enabled_for?(Vllm.discover_instances)
+              Legion::Extensions::Llm::Fleet::ProviderResponder.enabled_for?(Vllm.discover_instances).tap do |enabled|
+                log.debug { "vLLM fleet worker enabled=#{enabled}" }
+              end
             end
           end
         end

data/lib/legion/extensions/llm/vllm/provider.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/extensions/llm'
+require 'legion/logging'
 require 'uri'
 
 module Legion
@@ -94,7 +95,9 @@ module Legion
                      else
                        Array(@cached_models)
                      end
-            models.map { |model_info| offering_from_model(model_info) }
+            models.map { |model_info| offering_from_model(model_info) }.tap do |offerings|
+              log.debug { "built #{offerings.size} vLLM offering(s) live=#{live}" }
+            end
           rescue StandardError => e
             handle_exception(e, level: :warn, handled: true, operation: 'vllm.discover_offerings')
             []
@@ -106,25 +109,51 @@ module Legion
           end
 
           def reset_prefix_cache(reset_running_requests: nil, reset_external: nil)
+            log.debug do
+              "resetting vLLM prefix cache reset_running_requests=#{reset_running_requests.inspect} " \
+                "reset_external=#{reset_external.inspect}"
+            end
             connection.post(with_query(reset_prefix_cache_url, reset_running_requests:, reset_external:), {}).body
           end
 
           def reset_mm_cache
+            log.debug { 'resetting vLLM multimodal cache' }
             connection.post(reset_mm_cache_url, {}).body
           end
 
           def sleep(level: 1)
+            log.debug { "putting vLLM worker to sleep level=#{level.inspect}" }
             connection.post(with_query(sleep_url, level:), {}).body
           end
 
           def wake_up(tags: nil)
+            log.debug { "waking vLLM worker tags=#{Array(tags).inspect}" }
             query = Array(tags).map { |tag| ['tags', tag] }
             connection.post(with_query(wake_up_url, query), {}).body
           end
 
+          def fetch_model_detail(model_name)
+            # vLLM provides context_length via /v1/models during discovery.
+            # Re-fetch from the models endpoint if we need it outside discovery.
+            response = @connection.get(models_url)
+            models = response.body.fetch('data', [])
+            entry = models.find { |m| m['id'] == model_name.to_s }
+            return nil unless entry
+
+            ctx = entry['max_model_len']
+            ctx ? { context_window: ctx } : nil
+          rescue StandardError => e
+            handle_exception(e, level: :warn, handled: true, operation: 'vllm.fetch_model_detail',
+                                model: model_name)
+            nil
+          end
+
           private
 
           def offering_from_model(model_info)
+            ctx = model_info.context_length
+            cache_set(model_detail_cache_key(model_info.id), { context_window: ctx }, ttl: 86_400) if ctx
+
             Legion::Extensions::Llm::Routing::ModelOffering.new(
               provider_family: :vllm,
               instance_id: config.respond_to?(:instance_id) ? config.instance_id : :default,
@@ -133,8 +162,8 @@ module Legion
               model: model_info.id,
               usage_type: model_info.embedding? ? :embedding : :inference,
               capabilities: model_info.capabilities.map(&:to_s),
-              limits: { context_window: model_info.context_length }.compact,
-              metadata: { context_length: model_info.context_length }
+              limits: { context_window: ctx }.compact,
+              metadata: { context_length: ctx }
             )
           end
 
@@ -150,6 +179,10 @@ module Legion
             payload = super
             payload.delete(:reasoning_effort)
             payload[:chat_template_kwargs] = { enable_thinking: true } if thinking_enabled?(thinking)
+            log.debug do
+              "rendered vLLM payload model=#{model.respond_to?(:id) ? model.id : model} stream=#{stream} " \
+                "tools=#{tools.respond_to?(:size) ? tools.size : 0} thinking=#{payload.key?(:chat_template_kwargs)}"
+            end
             payload
           end
 

data/lib/legion/extensions/llm/vllm/runners/fleet_worker.rb

@@ -2,6 +2,7 @@
 
 require 'legion/extensions/llm/fleet/provider_responder'
 require 'legion/extensions/llm/vllm'
+require 'legion/logging'
 
 module Legion
   module Extensions
@@ -10,9 +11,17 @@ module Legion
         module Runners
           # Runner entrypoint for vLLM fleet request execution.
           module FleetWorker
+            include Legion::Logging::Helper
+            extend Legion::Logging::Helper
+
             module_function
 
             def handle_fleet_request(payload, delivery: nil, properties: nil)
+              log.debug do
+                "handling vLLM fleet request request_id=#{payload_field(payload, :request_id).inspect} " \
+                  "provider_instance=#{payload_field(payload, :provider_instance).inspect} " \
+                  "operation=#{payload_field(payload, :operation).inspect}"
+              end
               Legion::Extensions::Llm::Fleet::ProviderResponder.call(
                 payload: payload,
                 provider_family: Vllm::PROVIDER_FAMILY,
@@ -22,6 +31,16 @@ module Legion
                 properties: properties
               )
             end
+
+            def payload_field(payload, key)
+              return unless payload.respond_to?(:[])
+
+              payload[key] || payload[key.to_s]
+            rescue StandardError => e
+              handle_exception(e, level: :debug, handled: true, operation: 'vllm.fleet_worker.payload_field',
+                                  field: key)
+              nil
+            end
           end
         end
       end

data/lib/legion/extensions/llm/vllm/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Vllm
-        VERSION = '0.2.8'
+        VERSION = '0.2.10'
       end
     end
   end

data/lib/legion/extensions/llm/vllm.rb

@@ -3,6 +3,7 @@
 require 'legion/extensions/llm'
 require 'legion/extensions/llm/vllm/provider'
 require 'legion/extensions/llm/vllm/version'
+require 'legion/logging'
 
 module Legion
   module Extensions
@@ -65,6 +66,7 @@ module Legion
             end
           end
 
+          log.debug { "discovered #{instances.size} vLLM instance(s): #{instances.keys.join(', ')}" }
           instances
         end
 
@@ -92,7 +94,8 @@ module Legion
           require 'uri'
           host = URI.parse(url.to_s).host.to_s.downcase
           %w[localhost 127.0.0.1 ::1].include?(host) ? :local : :direct
-        rescue URI::InvalidURIError
+        rescue URI::InvalidURIError => e
+          handle_exception(e, level: :debug, handled: true, operation: 'vllm.infer_tier_from_endpoint')
           :direct
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-vllm
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.10
 platform: ruby
 authors:
 - LegionIO