lex-llm-vllm 0.3.5 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20aa357958da5d294b132bdb5d3bea065d0192b42051deae734a3e1b8af592e5
-  data.tar.gz: 203a81b2aa087bc2cdabbe26cd71ea88e1bd7b1aaa9e44a95a5b43c35ab2c344
+  metadata.gz: 66c6939650ded871419a3833d9d0a89e5c31a104f3acccbb065af86af17709cb
+  data.tar.gz: e639e39a60ff4c03838100bd361048997fecfe38e92fdd9f1763809958a43f8e
 SHA512:
-  metadata.gz: 4ab5021f00c1f6652147297c4f9fc56a2f501eaa1468c0f098aff78d83426706069e4cca7184c96647135ff951dc4965d871352144826d4cf6c8961e60616862
-  data.tar.gz: 1e52f17e28c52ddae6317c3fd6ea6fd58b54a71a46d77c80f00544fb35333bf0340c1498b2f9db6419c811765c9926e0bd20228c7bc44badae04dd68a9fd178c
+  metadata.gz: cf346327446a5e3c7b8666c8ea04269d7fea32b7bbe89b9a0c11b7578fe3ca1a8f948cf4fc639265bfcfa6f0a65bca7007f24ce90146f6234f1589fcb6fe5c31
+  data.tar.gz: 1e324da7463bd1a63b14b5fde9b3f6030be0f8cdd933d802d350066beaf40fe8208998a298a848b00126bf5d1d68868b8384c2ed719fb863c825179aa8c92ced

data/CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## [0.3.11] - 2026-06-20
+
+### Fixed
+- Stub shared registry publishing through `RegistryPublisher#schedule` in specs so async availability-event coverage stays stable after the shared publisher moved off raw `Thread.new`.
+
+## [0.3.10] - 2026-06-20
+
+### Fixed
+- Stop bulk-publishing vLLM model availability from `list_models`; discovery now emits one registry event per seen model from the shared `lex-llm` policy-filter path so blocked models stay observable without duplicate publishes.
+
+## [0.3.9] - 2026-06-20
+
+### Changed
+- Slow the live discovery refresh cadence from 60 seconds to 300 seconds for vLLM instances; `extensions.llm.vllm.discovery_interval` still overrides the default.
+
+## [0.3.8] - 2026-06-20
+
+### Fixed
+- Use the shared `lex-llm` capability override contract for provider, instance, and model settings, with canonical capability normalization for embedding/tool/thinking routing.
+
+## [0.3.7] - 2026-06-19
+
+### Changed
+- Adopt `Legion::Extensions::Llm::Inventory::ScopedRefresher` mixin (lex-llm 0.6.0). Discovery
+  refresh actors now write directly to the live `Inventory` catalog via `Inventory.write_lane`.
+- Pin `lex-llm >= 0.6.0` and `legion-llm >= 0.14.0` in gemspec.
+- Standard `weight: 100` default added to provider instance settings schema.
+
+## 0.3.6 - 2026-06-18
+
+- **Streaming token usage** — request `stream_options: { include_usage: true }` on streaming chat so
+  vLLM emits the final usage-only chunk. Streaming responses now carry input/output token counts;
+  previously every streamed response reported zero tokens, which blinded metering/cost. Overridable
+  per-instance via `config[:stream_token_usage] = false` for a non-conforming OpenAI-compatible
+  backend that rejects the field. The chunk parser already handled the trailing `choices: []` usage
+  chunk; the gap was only that the request never asked for it.
+
 ## 0.3.5 - 2026-06-16
 
 - Extract `vllm_api_key` from `credentials: { api_key: ... }` in instance settings so Bearer auth works with the standard settings layout.

data/lex-llm-vllm.gemspec

@@ -27,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'legion-logging', '>= 1.3.2'
   spec.add_dependency 'legion-settings', '>= 1.3.14'
   spec.add_dependency 'legion-transport', '>= 1.4.14'
-  spec.add_dependency 'lex-llm', '>= 0.5.0'
+  spec.add_dependency 'lex-llm', '>= 0.6.0'
 end

data/lib/legion/extensions/llm/vllm/actors/discovery_refresh.rb

@@ -1,11 +1,19 @@
 # frozen_string_literal: true
 
+require 'digest'
+
 begin
   require 'legion/extensions/actors/every'
 rescue LoadError => e
   warn(e.message) if $VERBOSE
 end
 
+begin
+  require 'legion/extensions/llm/inventory/scoped_refresher'
+rescue LoadError => e
+  warn(e.message) if $VERBOSE
+end
+
 return unless defined?(Legion::Extensions::Actors::Every)
 
 module Legion
@@ -17,7 +25,11 @@ module Legion
           class DiscoveryRefresh < Legion::Extensions::Actors::Every
             include Legion::Logging::Helper
 
-            REFRESH_INTERVAL = 1800
+            if defined?(Legion::Extensions::Llm::Inventory::ScopedRefresher)
+              include Legion::Extensions::Llm::Inventory::ScopedRefresher
+            end
+
+            def self.every_seconds = 300
 
             def runner_class    = self.class
             def runner_function = 'manual'
@@ -27,24 +39,117 @@ module Legion
             def generate_task?  = false
 
             def time
-              return REFRESH_INTERVAL unless defined?(Legion::Settings)
+              return self.class.every_seconds unless defined?(Legion::Settings)
+
+              Legion::Settings.dig(:extensions, :llm, :vllm, :discovery_interval) || self.class.every_seconds
+            end
 
-              Legion::Settings.dig(:extensions, :llm, :vllm, :discovery_interval) || REFRESH_INTERVAL
+            def scope_key                = { provider: :vllm }
+            def offering_type(raw_type)  = %i[embed embedding].include?(raw_type) ? :embedding : :inference
+
+            def vllm_cfg
+              return unless defined?(Legion::Settings)
+
+              Legion::Settings.dig(:extensions, :llm,
+                                   :vllm)
+            end
+
+            def compute_lanes_for_scope(**)
+              return [] unless defined?(Legion::LLM::Call::Registry)
+
+              vllm_instances.flat_map { |entry| lanes_from_instance(entry) }
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: 'vllm.actor.compute_lanes_for_scope')
+              []
+            end
+
+            def credential_hash(**)
+              cfg = vllm_cfg
+              Digest::SHA256.hexdigest(cfg&.dig(:api_key).to_s + cfg&.dig(:instances).to_s)[0, 16]
+            rescue StandardError
+              'unknown'
             end
 
             def manual
-              log.debug('[vllm][discovery_refresh] refreshing model list')
-              return unless defined?(Legion::LLM::Discovery)
+              run_scoped_tick
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: 'vllm.actor.discovery_refresh')
+            end
+
+            private
+
+            def run_scoped_tick
+              return unless defined?(Legion::Extensions::Llm::Inventory::ScopedRefresher)
+              return unless self.class.ancestors.include?(Legion::Extensions::Llm::Inventory::ScopedRefresher)
+
+              tick
+            end
 
-              Legion::LLM::Discovery.refresh_discovered_models!(provider: :vllm)
-              if defined?(Legion::LLM::Router) && Legion::LLM::Router.respond_to?(:populate_auto_rules)
-                Legion::LLM::Router.populate_auto_rules(Legion::LLM::Discovery.discovered_instances)
+            def vllm_instances
+              Legion::LLM::Call::Registry.all_instances.select { |e| (e[:provider] || '').to_sym == :vllm }
+            end
+
+            def lanes_from_instance(instance_entry)
+              adapter = instance_entry[:adapter]
+              return [] unless adapter.respond_to?(:discover_offerings)
+
+              Array(adapter.discover_offerings(live: true)).flat_map do |offering|
+                raw  = offering_to_hash(offering)
+                lane = build_lane(raw, instance_entry)
+                fleet = maybe_fleet_lane(lane)
+                fleet ? [lane, fleet] : [lane]
               end
-              if defined?(Legion::LLM::Inventory) && Legion::LLM::Inventory.respond_to?(:invalidate_offerings_cache!)
-                Legion::LLM::Inventory.invalidate_offerings_cache!
+            end
+
+            # ModelOffering objects do not implement `[]`; normalize to a Hash so the
+            # rest of the writer stays Hash-shaped. Hash inputs pass through untouched.
+            def offering_to_hash(offering)
+              return offering if offering.is_a?(Hash)
+
+              hash = offering.to_h
+              hash[:type] ||= hash[:usage_type]
+              hash[:enabled] = offering.respond_to?(:enabled?) ? offering.enabled? : true
+              hash
+            end
+
+            def build_lane(offering, instance_entry) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity
+              tier            = offering[:tier] || :direct
+              type            = offering_type(offering[:type])
+              instance_id     = offering[:instance_id] ||
+                                instance_entry[:instance] ||
+                                instance_entry[:instance_id] ||
+                                instance_entry[:id]
+              provider_family = offering[:provider_family] || :vllm
+              model           = offering[:model]
+              lane_id = Legion::Extensions::Llm::Inventory::ScopedRefresher.compose_id(
+                tier: tier, provider_family: provider_family, instance_id: instance_id, type: type, model: model
+              )
+              { id: lane_id, tier: tier, provider_family: provider_family, instance_id: instance_id,
+                model: model, canonical_model_alias: offering[:canonical_model_alias], type: type,
+                capabilities: normalize_caps(offering[:capabilities]),
+                limits: offering[:limits] || {}, enabled: offering.fetch(:enabled, true), cost: offering[:cost] || {} }
+            end
+
+            def maybe_fleet_lane(lane)
+              return unless lane[:type] == :inference && vllm_cfg&.dig(:fleet, :dispatch, :enabled)
+
+              fleet_id = Legion::Extensions::Llm::Inventory::ScopedRefresher.compose_id(
+                tier: :fleet, provider_family: lane[:provider_family],
+                instance_id: lane[:instance_id], type: lane[:type], model: lane[:model]
+              )
+              lane.merge(id: fleet_id, tier: :fleet)
+            end
+
+            def normalize_caps(caps)
+              # Inventory::Capabilities lives in lex-llm; the previous fallback (`return []
+              # unless defined?(...)`) silently swallowed every capability the operator
+              # declared via enable_thinking/enable_tools when the constant wasn't loaded.
+              # Always normalize through the shared vocabulary so aliases collapse.
+              if defined?(Legion::Extensions::Llm::Inventory::Capabilities)
+                Legion::Extensions::Llm::Inventory::Capabilities.normalize(caps)
+              else
+                Array(caps).compact.map(&:to_sym).uniq
               end
-            rescue StandardError => e
-              handle_exception(e, level: :warn, handled: true, operation: 'vllm.actor.discovery_refresh')
             end
           end
         end

data/lib/legion/extensions/llm/vllm/provider.rb

@@ -33,9 +33,9 @@ module Legion
 
             def chat?(_model) = true
             def streaming?(_model) = true
-            def vision?(_model) = true
-            def functions?(_model) = true
-            def embeddings?(_model) = true
+            def vision?(_model) = false
+            def functions?(_model) = false
+            def embeddings?(_model) = false
 
             def critical_capabilities_for(model)
               [
@@ -78,7 +78,7 @@ module Legion
 
           def health(live: false)
             log.info { "checking health live=#{live} at #{api_base}#{health_url}" }
-            connection.get(health_url).body
+            super
           end
 
           def readiness(live: false)
@@ -88,27 +88,20 @@ module Legion
             end
           end
 
-          def list_models
+          def list_models(live: false, **filters)
             log.info { "discovering models from #{api_base}#{models_url}" }
             super.tap do |models|
               log.info { "discovered #{models.size} model(s) from vLLM" }
-              self.class.registry_publisher.publish_models_async(models, readiness: readiness(live: false))
             end
           end
 
-          def discover_offerings(live: false, **)
-            models = if live
-                       @cached_models = list_models
-                     else
-                       Array(@cached_models)
-                     end
-            offerings = models.filter_map do |model_info|
-              next unless model_allowed?(model_info.id)
+          def discover_offerings(live: false, **filters)
+            return filter_cached_offerings(Array(@cached_offerings), filters) unless live
 
-              offering_from_model(model_info)
-            end
-            log.debug { "built #{offerings.size} vLLM offering(s) live=#{live}" }
-            offerings
+            provider_health = health(live:)
+            @cached_offerings = discover_live_offerings(filters, provider_health, live:)
+            log_discover_complete(@cached_offerings)
+            @cached_offerings
           rescue StandardError => e
             handle_exception(e, level: :warn, handled: true, operation: 'vllm.discover_offerings')
             []
@@ -161,13 +154,49 @@ module Legion
 
           private
 
-          def offering_from_model(model_info)
+          def discovery_registry_readiness(provider_health, live:)
+            {
+              provider: slug.to_sym,
+              configured: configured?,
+              ready: provider_health[:ready] == true,
+              live: live,
+              health: provider_health
+            }
+          end
+
+          def discover_live_offerings(filters, provider_health, live:)
+            readiness = discovery_registry_readiness(provider_health, live:)
+            Array(list_models(live:, **filters)).filter_map do |model|
+              self.class.registry_publisher.publish_models_async([model], readiness:)
+              next unless model_matches_filters?(model, filters)
+              next unless model_allowed?(model.id)
+
+              log_model_discovered(model)
+              offering_from_model(model, health: provider_health)
+            end
+          end
+
+          def log_model_discovered(model)
+            log.debug(
+              "[#{slug}] instance=#{provider_instance_id} action=model_discovered " \
+              "model=#{model.id} family=#{model.family}"
+            )
+          end
+
+          def log_discover_complete(offerings)
+            log.info(
+              "[#{slug}] instance=#{provider_instance_id} action=discover_complete " \
+              "model_count=#{Array(offerings).size}"
+            )
+          end
+
+          def offering_from_model(model_info, health: {})
             ctx = model_info.context_length
             if ctx
               begin
                 cache_set(model_detail_cache_key(model_info.id), { context_window: ctx }, ttl: 86_400)
               rescue StandardError => e
-                handle_exception(e, level: :debug, handled: true, operation: 'vllm.cache_model_detail')
+                handle_exception(e, level: :warn, handled: true, operation: 'vllm.cache_model_detail')
               end
             end
 
@@ -181,11 +210,12 @@ module Legion
               model_config: model_capability_config(model_info.id)
             )
 
-            build_offering(model_info, policy, ctx)
+            build_offering(model_info, policy, ctx, health)
           end
 
-          def build_offering(model_info, policy, ctx) # rubocop:disable Metrics/AbcSize
+          def build_offering(model_info, policy, ctx, health) # rubocop:disable Metrics/AbcSize
             max_out = model_info.respond_to?(:max_output_tokens) ? model_info.max_output_tokens : nil
+            usage_type = policy[:capabilities].include?(:embedding) ? :embedding : :inference
 
             Legion::Extensions::Llm::Routing::ModelOffering.new(
               provider_family: :vllm,
@@ -195,10 +225,11 @@ module Legion
               model: model_info.id,
               canonical_model_alias: model_info.respond_to?(:name) ? model_info.name : nil,
               model_family: model_info.respond_to?(:family) ? model_info.family : nil,
-              usage_type: model_info.embedding? ? :embedding : :inference,
+              usage_type: usage_type,
               capabilities: policy[:capabilities],
               capability_sources: policy[:sources],
               limits: { context_window: ctx, max_output_tokens: max_out }.compact,
+              health: health,
               metadata: offering_metadata_for(model_info).merge(capability_sources: policy[:sources])
             )
           end
@@ -212,51 +243,7 @@ module Legion
           end
 
           def provider_envelope_capabilities
-            { streaming: true }
-          end
-
-          def provider_capability_config
-            return {} unless defined?(Legion::Extensions::Llm::CredentialSources)
-
-            conf = Legion::Extensions::Llm::CredentialSources.setting(:extensions, :llm, :vllm)
-            conf.is_a?(Hash) ? conf.to_h.except(:instances, 'instances') : {}
-          rescue StandardError => e
-            handle_exception(e, level: :debug, handled: true, operation: 'vllm.provider_capability_config')
-            {}
-          end
-
-          def instance_capability_config
-            cfg = config
-            result = {}
-            %i[capabilities enable_thinking enable_tools enable_streaming enable_vision enable_embeddings
-               thinking_flag tools_flag streaming_flag vision_flag embedding_flag embeddings_flag
-               tool_flag images_flag image_flag].each do |key|
-              next unless cfg.respond_to?(key)
-
-              val = cfg.send(key)
-              result[key] = val unless val.nil?
-            rescue StandardError
-              next
-            end
-            result
-          end
-
-          def model_capability_config(model_id)
-            models_conf = resolve_models_config
-            return {} unless models_conf.respond_to?(:to_h)
-
-            hash = models_conf.to_h
-            hash[model_id.to_s] || hash[model_id.to_sym] || {}
-          rescue StandardError => e
-            handle_exception(e, level: :debug, handled: true, operation: 'vllm.model_capability_config')
-            {}
-          end
-
-          def resolve_models_config
-            return config.models if config.respond_to?(:models)
-            return config[:models] if config.respond_to?(:[])
-
-            nil
+            { completion: true, streaming: true }
           end
 
           def offering_metadata_for(model_info)
@@ -453,7 +440,7 @@ module Legion
           def vllm_thinking_setting
             instance_thinking_enabled? || global_thinking_enabled?
           rescue StandardError => e
-            handle_exception(e, level: :debug, handled: true, operation: 'vllm.thinking_setting')
+            handle_exception(e, level: :warn, handled: true, operation: 'vllm.thinking_setting')
             false
           end
 

data/lib/legion/extensions/llm/vllm/runners/fleet_worker.rb

@@ -37,7 +37,7 @@ module Legion
 
               payload[key] || payload[key.to_s]
             rescue StandardError => e
-              handle_exception(e, level: :debug, handled: true, operation: 'vllm.fleet_worker.payload_field',
+              handle_exception(e, level: :warn, handled: true, operation: 'vllm.fleet_worker.payload_field',
                                   field: key)
               nil
             end

data/lib/legion/extensions/llm/vllm/translator.rb

@@ -70,6 +70,7 @@ module Legion
             payload[:tools] = format_tools(request.tools) unless request.tools.to_h.empty?
             payload[:tool_choice] = format_tool_choice(request.tool_choice) if request.tool_choice
             payload.merge!(map_params_to_wire(request.params)) if request.params
+            payload[:stream_options] = { include_usage: true } if request.stream && stream_token_usage?
             apply_thinking_config(payload, request)
             if formatted_response_format?(request.params)
               payload[:response_format] =
@@ -208,6 +209,18 @@ module Legion
 
           attr_reader :config
 
+          # OpenAI `stream_options.include_usage` asks the server to emit a final
+          # usage-only chunk (choices:[]) so streaming responses carry token counts.
+          # vLLM supports it (capability streaming_token_usage); defaults on, but a
+          # non-conforming OpenAI-compatible backend that rejects the field can opt
+          # out per-instance via config[:stream_token_usage] = false.
+          def stream_token_usage?
+            override = config.respond_to?(:[]) ? config[:stream_token_usage] : nil
+            return override != false unless override.nil?
+
+            capabilities[:streaming_token_usage] == true
+          end
+
           # ── Message formatting ──
 
           def format_messages(request)

data/lib/legion/extensions/llm/vllm/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Vllm
-        VERSION = '0.3.5'
+        VERSION = '0.3.11'
       end
     end
   end

data/lib/legion/extensions/llm/vllm.rb

@@ -103,7 +103,7 @@ module Legion
           host = URI.parse(url.to_s).host.to_s.downcase
           %w[localhost 127.0.0.1 ::1].include?(host) ? :local : :direct
         rescue URI::InvalidURIError => e
-          handle_exception(e, level: :debug, handled: true, operation: 'vllm.infer_tier_from_endpoint')
+          handle_exception(e, level: :warn, handled: true, operation: 'vllm.infer_tier_from_endpoint')
           :direct
         end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-vllm
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.11
 platform: ruby
 authors:
 - LegionIO
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: 0.6.0
 description: vLLM provider integration for the LegionIO LLM routing framework.
 email:
 - matthewdiverson@gmail.com