lex-llm-vllm 0.2.13 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6adc86b9d3286821c0efa59e4c820f3d99ee0acb5327f133a96010383d154505
-  data.tar.gz: 73ecff7ccc309eb0469a79edc3970fdf3d766199a6df31edde4cbaf2016dc970
+  metadata.gz: 20aa357958da5d294b132bdb5d3bea065d0192b42051deae734a3e1b8af592e5
+  data.tar.gz: 203a81b2aa087bc2cdabbe26cd71ea88e1bd7b1aaa9e44a95a5b43c35ab2c344
 SHA512:
-  metadata.gz: 4b2c498e26f09fa27edfa7abf08bf6fae656313cf6e2ce625772a9ce809ff1fcfae55a8746261b943b6b41111a46784fa97d9d5004f4be69f58761de05c6383d
-  data.tar.gz: 94b867bd099f8e062f23aee550be30d4549bbc3d4f10d40b6e4e9b3dcabca7c8837e246198a6ea9a208035da3dcfaaf41be626709de9c8e642e3a4035f6681b0
+  metadata.gz: 4ab5021f00c1f6652147297c4f9fc56a2f501eaa1468c0f098aff78d83426706069e4cca7184c96647135ff951dc4965d871352144826d4cf6c8961e60616862
+  data.tar.gz: 1e52f17e28c52ddae6317c3fd6ea6fd58b54a71a46d77c80f00544fb35333bf0340c1498b2f9db6419c811765c9926e0bd20228c7bc44badae04dd68a9fd178c

data/.rubocop.yml

@@ -21,3 +21,12 @@ RSpec/ExampleLength:
   Max: 8
 RSpec/MultipleExpectations:
   Enabled: false
+
+RSpec/ExampleLength:
+  Max: 10
+
+
+Layout/LineLength:
+  Exclude:
+    - spec/**/*
+

data/CHANGELOG.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## 0.3.5 - 2026-06-16
+
+- Extract `vllm_api_key` from `credentials: { api_key: ... }` in instance settings so Bearer auth works with the standard settings layout.
+- Fix `Dalli::RingError` crash in `offering_from_model` when cache server is unavailable; cache write is now best-effort.
+
+## 0.3.3 - 2026-06-16
+
+- Dependency updates (concurrent-ruby 1.3.7, faraday 2.14.3, rubocop 1.88.0) and code quality improvements.
+
+## 0.3.2 - 2026-06-15
+
+- **CapabilityPolicy integration** — Optional capabilities default false; use `CapabilityPolicy.resolve` for offerings. Static all-true predicates no longer used for routing truth. Settings overrides at provider/instance/model level supported.
+
+## 0.3.1 - 2026-06-13
+
+- **Gemfile cleanup** — Remove local path overrides; dependencies resolve from gemspec via rubygems.
+- **Bug fix** — Restore vLLM streaming; private `ThinkingExtractor` call was killing every text delta.
+- **Canonical tool normalization** — Use canonical normalization for tool parameter schemas.
+- 155 examples, 0 failures; 17 files, 0 rubocop offenses.
+
+## 0.3.0 - 2026-06-10
+
+- Add canonical provider translator (`Translator`) implementing `render_request`,
+  `parse_response`, `parse_chunk`, and `capabilities` per N×N routing design
+- Wire provider `render_payload`, `parse_completion_response`, `build_chunk` to
+  delegate to translator with legacy Message/Chunk bridge for backward compat
+- Declare vLLM quirks: `tool_calls_as_text`, `forced_tool_choice`, `thinking_tags`,
+  `streaming_token_usage`
+- G18 parameter mapping: max_tokens, temperature, top_p, top_k, stop_sequences,
+  seed, frequency_penalty, presence_penalty, response_format
+- Qwen-style </think> tag extraction and tool-call synthesis from content text
+- Adopt conformance kit (`it_behaves_like 'a canonical provider translator'`)
+- Bump lex-llm dependency floor to >= 0.5.0
+
 ## 0.2.13 - 2026-06-05
 
 - Fix missing documentation comment on `DiscoveryRefresh` actor (RuboCop Style/Documentation)

data/Gemfile

@@ -2,13 +2,6 @@
 
 source 'https://rubygems.org'
 
-group :test do
-  llm_base_path = ENV.fetch('LEX_LLM_PATH', File.expand_path('../lex-llm', __dir__))
-  transport_path = ENV.fetch('LEGION_TRANSPORT_PATH', File.expand_path('../../legion-transport', __dir__))
-  gem 'legion-transport', path: transport_path if File.directory?(transport_path)
-  gem 'lex-llm', path: llm_base_path if File.directory?(llm_base_path)
-end
-
 gemspec
 
 group :development do

data/lex-llm-vllm.gemspec

@@ -27,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'legion-logging', '>= 1.3.2'
   spec.add_dependency 'legion-settings', '>= 1.3.14'
   spec.add_dependency 'legion-transport', '>= 1.4.14'
-  spec.add_dependency 'lex-llm', '>= 0.4.3'
+  spec.add_dependency 'lex-llm', '>= 0.5.0'
 end

data/lib/legion/extensions/llm/vllm/actors/discovery_refresh.rb

@@ -37,6 +37,12 @@ module Legion
               return unless defined?(Legion::LLM::Discovery)
 
               Legion::LLM::Discovery.refresh_discovered_models!(provider: :vllm)
+              if defined?(Legion::LLM::Router) && Legion::LLM::Router.respond_to?(:populate_auto_rules)
+                Legion::LLM::Router.populate_auto_rules(Legion::LLM::Discovery.discovered_instances)
+              end
+              if defined?(Legion::LLM::Inventory) && Legion::LLM::Inventory.respond_to?(:invalidate_offerings_cache!)
+                Legion::LLM::Inventory.invalidate_offerings_cache!
+              end
             rescue StandardError => e
               handle_exception(e, level: :warn, handled: true, operation: 'vllm.actor.discovery_refresh')
             end

data/lib/legion/extensions/llm/vllm/provider.rb

@@ -53,6 +53,11 @@ module Legion
             Vllm.default_settings
           end
 
+          # Canonical translator instance — renders requests, parses responses/chunks.
+          def translator
+            @translator ||= Translator.new(config: config)
+          end
+
           def api_base
             normalize_url(config.vllm_api_base || settings[:endpoint] || 'http://localhost:8000')
           end
@@ -158,7 +163,29 @@ module Legion
 
           def offering_from_model(model_info)
             ctx = model_info.context_length
-            cache_set(model_detail_cache_key(model_info.id), { context_window: ctx }, ttl: 86_400) if ctx
+            if ctx
+              begin
+                cache_set(model_detail_cache_key(model_info.id), { context_window: ctx }, ttl: 86_400)
+              rescue StandardError => e
+                handle_exception(e, level: :debug, handled: true, operation: 'vllm.cache_model_detail')
+              end
+            end
+
+            policy = Legion::Extensions::Llm::CapabilityPolicy.resolve(
+              real: extract_real_capabilities(model_info),
+              provider_catalog: {},
+              probe: {},
+              provider_envelope: provider_envelope_capabilities,
+              provider_config: provider_capability_config,
+              instance_config: instance_capability_config,
+              model_config: model_capability_config(model_info.id)
+            )
+
+            build_offering(model_info, policy, ctx)
+          end
+
+          def build_offering(model_info, policy, ctx) # rubocop:disable Metrics/AbcSize
+            max_out = model_info.respond_to?(:max_output_tokens) ? model_info.max_output_tokens : nil
 
             Legion::Extensions::Llm::Routing::ModelOffering.new(
               provider_family: :vllm,
@@ -166,22 +193,253 @@ module Legion
               transport: offering_transport,
               tier: offering_tier,
               model: model_info.id,
+              canonical_model_alias: model_info.respond_to?(:name) ? model_info.name : nil,
+              model_family: model_info.respond_to?(:family) ? model_info.family : nil,
               usage_type: model_info.embedding? ? :embedding : :inference,
-              capabilities: model_info.capabilities.map(&:to_s),
-              limits: { context_window: ctx }.compact,
-              metadata: { context_length: ctx }
+              capabilities: policy[:capabilities],
+              capability_sources: policy[:sources],
+              limits: { context_window: ctx, max_output_tokens: max_out }.compact,
+              metadata: offering_metadata_for(model_info).merge(capability_sources: policy[:sources])
+            )
+          end
+
+          def extract_real_capabilities(model_info)
+            return {} unless model_info.respond_to?(:metadata)
+
+            meta = model_info.metadata
+            meta_caps = meta.is_a?(Hash) ? meta[:capabilities] : nil
+            meta_caps.is_a?(Hash) ? meta_caps : {}
+          end
+
+          def provider_envelope_capabilities
+            { streaming: true }
+          end
+
+          def provider_capability_config
+            return {} unless defined?(Legion::Extensions::Llm::CredentialSources)
+
+            conf = Legion::Extensions::Llm::CredentialSources.setting(:extensions, :llm, :vllm)
+            conf.is_a?(Hash) ? conf.to_h.except(:instances, 'instances') : {}
+          rescue StandardError => e
+            handle_exception(e, level: :debug, handled: true, operation: 'vllm.provider_capability_config')
+            {}
+          end
+
+          def instance_capability_config
+            cfg = config
+            result = {}
+            %i[capabilities enable_thinking enable_tools enable_streaming enable_vision enable_embeddings
+               thinking_flag tools_flag streaming_flag vision_flag embedding_flag embeddings_flag
+               tool_flag images_flag image_flag].each do |key|
+              next unless cfg.respond_to?(key)
+
+              val = cfg.send(key)
+              result[key] = val unless val.nil?
+            rescue StandardError
+              next
+            end
+            result
+          end
+
+          def model_capability_config(model_id)
+            models_conf = resolve_models_config
+            return {} unless models_conf.respond_to?(:to_h)
+
+            hash = models_conf.to_h
+            hash[model_id.to_s] || hash[model_id.to_sym] || {}
+          rescue StandardError => e
+            handle_exception(e, level: :debug, handled: true, operation: 'vllm.model_capability_config')
+            {}
+          end
+
+          def resolve_models_config
+            return config.models if config.respond_to?(:models)
+            return config[:models] if config.respond_to?(:[])
+
+            nil
+          end
+
+          def offering_metadata_for(model_info)
+            {
+              raw_model: model_info.id,
+              parameter_count: model_info.respond_to?(:parameter_count) ? model_info.parameter_count : nil,
+              parameter_size: model_info.respond_to?(:parameter_size) ? model_info.parameter_size : nil,
+              quantization: model_info.respond_to?(:quantization) ? model_info.quantization : nil,
+              size_bytes: model_info.respond_to?(:size_bytes) ? model_info.size_bytes : nil
+            }.compact
+          end
+
+          # ── Canonical bridge: legacy provider API → Canonical::Request ──
+
+          # rubocop:disable Metrics/ParameterLists, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- bridge method can be complex
+          def build_canonical_request(
+            messages:, tools:, temperature:, model:, stream:, schema:, thinking:, tool_prefs:
+          )
+            model_id = model.respond_to?(:id) ? model.id : model.to_s
+
+            canonical_messages = messages.filter_map do |msg|
+              Canonical::Message.from_hash(msg.to_h) if msg.respond_to?(:to_h)
+            end
+
+            canonical_tools = tools.to_h.transform_values do |tool|
+              if tool.is_a?(Canonical::ToolDefinition)
+                tool
+              else
+                Canonical::ToolDefinition.from_hash(tool.respond_to?(:to_h) ? tool.to_h : tool)
+              end
+            end
+
+            params_hash = { temperature: temperature }
+            params_hash[:response_format] = schema if schema
+            canonical_params = Canonical::Params.from_hash(params_hash)
+
+            canonical_thinking = if thinking.respond_to?(:enabled?) && thinking.enabled?
+                                   Canonical::Thinking::Config.new(
+                                     effort: thinking.respond_to?(:effort) ? thinking.effort : nil
+                                   )
+                                 elsif thinking.is_a?(Hash)
+                                   Canonical::Thinking::Config.new(
+                                     effort: thinking[:effort] || thinking['effort'],
+                                     budget: thinking[:budget] || thinking['budget']
+                                   )
+                                 end
+
+            # Tool choice from tool_prefs
+            tool_choice = format_tool_choice_from_prefs(tool_prefs)
+
+            Canonical::Request.build(
+              messages: canonical_messages,
+              system: extract_system_prompt(messages),
+              tools: canonical_tools,
+              tool_choice: tool_choice,
+              params: canonical_params,
+              thinking: canonical_thinking,
+              stream: stream,
+              metadata: { model: model_id }
             )
           end
+          # rubocop:enable Metrics/ParameterLists, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+          # ── Canonical bridge: Canonical→legacy Message/Chunk ──
+
+          # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- verbose bridge
+          def to_legacy_message(canonical, raw_body, _raw_response)
+            thinking = nil
+            if canonical.thinking
+              thinking = Thinking.build(
+                text: canonical.thinking.content,
+                signature: canonical.thinking.signature
+              )
+            end
+
+            tool_calls = {}
+            canonical.tool_calls.each do |tc|
+              key = (tc.name || tc.id).to_s.to_sym
+              tool_calls[key] = Legion::Extensions::Llm::ToolCall.new(
+                id: tc.id,
+                name: tc.name,
+                arguments: tc.arguments
+              )
+            end
+
+            usage = canonical.usage || {}
+
+            Legion::Extensions::Llm::Message.new(
+              role: :assistant,
+              content: canonical.text,
+              model_id: canonical.model,
+              tool_calls: tool_calls.empty? ? nil : tool_calls,
+              thinking: thinking,
+              input_tokens: usage.respond_to?(:input_tokens) ? usage.input_tokens : nil,
+              output_tokens: usage.respond_to?(:output_tokens) ? usage.output_tokens : nil,
+              reasoning_tokens: usage.respond_to?(:thinking_tokens) ? usage.thinking_tokens : nil,
+              raw: raw_body
+            )
+          end
+          # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+          def to_legacy_chunk(canonical, raw_data)
+            usage = canonical&.usage || {}
+
+            content = canonical.delta
+            thinking = nil
+            if canonical.type == :thinking_delta
+              thinking = Thinking.build(text: canonical.delta)
+              content = nil
+            end
+
+            Legion::Extensions::Llm::Chunk.new(
+              role: :assistant,
+              content: content,
+              model_id: raw_data['model'],
+              tool_calls: legacy_chunk_tool_calls(canonical),
+              thinking: thinking,
+              input_tokens: usage.respond_to?(:input_tokens) ? usage.input_tokens : nil,
+              output_tokens: usage.respond_to?(:output_tokens) ? usage.output_tokens : nil,
+              raw: raw_data
+            )
+          end
+
+          # Map a canonical tool_call_delta onto the legacy chunk tool_calls hash.
+          # Fragment semantics matter: an entry with a non-nil id starts a new tool
+          # call in the StreamAccumulator; a nil id appends the raw arguments
+          # fragment to the most recently started call.
+          def legacy_chunk_tool_calls(canonical)
+            return nil unless canonical.type == :tool_call_delta && canonical.tool_call
+
+            tc = canonical.tool_call
+            key = (tc.id || tc.name || :fragment).to_s.to_sym
+            {
+              key => Legion::Extensions::Llm::ToolCall.new(
+                id: tc.id,
+                name: tc.name,
+                arguments: tc.arguments
+              )
+            }
+          end
+
+          # ── Tool choice helpers ──
+
+          def format_tool_choice_from_prefs(tool_prefs)
+            return nil unless tool_prefs
+
+            choice = tool_prefs[:choice] || tool_prefs['choice']
+            return nil unless choice
+            return choice.to_sym if %w[auto none required].include?(choice.to_s)
+
+            { name: choice.to_s }
+          end
+
+          # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- multibranch guard chain for system parsing
+          def extract_system_prompt(messages)
+            return nil unless messages.is_a?(Array)
+            return nil if messages.empty?
+
+            first = messages.first
+            return nil unless first
+
+            role = first.respond_to?(:role) ? first.role.to_sym : (first[:role] || first['role'])
+            return nil unless [:system, 'system'].include?(role)
+
+            content = first.respond_to?(:content) ? first.content : (first[:content] || first['content'])
+            content.is_a?(String) ? content : nil
+          end
+          # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
           def render_payload(messages, tools:, temperature:, model:, stream:, schema:, thinking:, tool_prefs:) # rubocop:disable Metrics/ParameterLists
-            payload = super
-            payload.delete(:reasoning_effort)
-            payload[:chat_template_kwargs] = { enable_thinking: true } if thinking_enabled?(thinking)
+            # Build a canonical request from provider call parameters,
+            # then delegate to the translator for wire-format rendering.
+            canonical_req = build_canonical_request(
+              messages:, tools:, temperature:, model:, stream:,
+              schema:, thinking:, tool_prefs:
+            )
+            wire = translator.render_request(canonical_req)
+
             log.debug do
-              "rendered vLLM payload model=#{model.respond_to?(:id) ? model.id : model} stream=#{stream} " \
-                "tools=#{tools.respond_to?(:size) ? tools.size : 0} thinking=#{payload.key?(:chat_template_kwargs)}"
+              "vLLM provider rendered wire payload model=#{wire[:model]} stream=#{wire[:stream]} " \
+                "messages=#{(wire[:messages] || []).size} keys=#{wire.keys.join(', ')}"
             end
-            payload
+            wire
           end
 
           def thinking_enabled?(thinking)
@@ -214,6 +472,24 @@ module Legion
             vllm[:enable_thinking] == true || vllm['enable_thinking'] == true
           end
 
+          # Override: delegate completion response parsing to the canonical translator.
+          def parse_completion_response(response)
+            body = response.body
+            canonical = translator.parse_response(body)
+
+            # Convert Canonical::Response back to the legacy Message/Chunk shape
+            # that the Provider base class expects (backward compat with existing callers).
+            to_legacy_message(canonical, body, response)
+          end
+
+          # Override: delegate SSE chunk parsing to the canonical translator.
+          def build_chunk(data)
+            canonical_chunk = translator.parse_chunk(data)
+            return nil if canonical_chunk.nil?
+
+            to_legacy_chunk(canonical_chunk, data)
+          end
+
           def parse_list_models_response(response, provider, capabilities)
             response.body.fetch('data', []).map do |model|
               critical_capabilities = critical_capabilities_for(capabilities, model)

data/lib/legion/extensions/llm/vllm/translator.rb

@@ -0,0 +1,703 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/llm/canonical'
+require 'legion/extensions/llm/responses/thinking_extractor'
+require 'legion/json'
+require 'legion/logging'
+
+module Legion
+  module Extensions
+    module Llm
+      module Vllm
+        # Canonical provider translator for vLLM (OpenAI-compatible wire format).
+        #
+        # Implements render_request, parse_response, parse_chunk, and capabilities.
+        # Extracted from existing format_openai_*/parse_* methods in OpenAICompatible mixin
+        # and vLLM-specific render_payload override in Provider.
+        #
+        # vLLM quirks (declared in capabilities):
+        # - tool_calls_as_text: true — some model configurations output tool calls
+        #   as JSON text in the content field rather than structured tool_calls.
+        # - forced_tool_choice: true — vLLM's tool_choice handling is strict;
+        #   named tool choices must be explicit function references.
+        # - thinking_tags: ['think', 'thinking'] — Qwen-style models emit reasoning
+        #   in <think> or <thinking> tags within content text.
+        # rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- translator implementation
+        class Translator
+          include Legion::Logging::Helper
+
+          # vLLM-specific stop_reason mapping (per conformance fixture stop_reason_matrix).
+          VLLM_STOP_REASON_MAP = {
+            'stop' => :end_turn,
+            'tool_use' => :tool_use,
+            'length' => :max_tokens
+          }.freeze
+          FALLBACK_STOP_REASON = :end_turn
+
+          # G18 parameter mapping: supported canonical params.
+          SUPPORTED_PARAMS = %i[
+            max_tokens temperature top_p top_k stop_sequences
+            seed frequency_penalty presence_penalty response_format
+          ].freeze
+
+          # vLLM wire keys for supported params (most are 1:1 with canonical names).
+          PARAM_WIRE_KEYS = {
+            max_tokens: :max_tokens,
+            temperature: :temperature,
+            top_p: :top_p,
+            top_k: :top_k,
+            stop_sequences: :stop,
+            seed: :seed,
+            frequency_penalty: :frequency_penalty,
+            presence_penalty: :presence_penalty,
+            response_format: :response_format
+          }.freeze
+
+          def initialize(config: nil)
+            @config = config
+          end
+
+          # Render a canonical request into an OpenAI-compatible wire payload for vLLM.
+          def render_request(request)
+            model = request.metadata&.dig(:model) || 'default'
+            messages = format_messages(request)
+            payload = {
+              model: model,
+              messages: messages,
+              stream: request.stream
+            }
+
+            payload[:tools] = format_tools(request.tools) unless request.tools.to_h.empty?
+            payload[:tool_choice] = format_tool_choice(request.tool_choice) if request.tool_choice
+            payload.merge!(map_params_to_wire(request.params)) if request.params
+            apply_thinking_config(payload, request)
+            if formatted_response_format?(request.params)
+              payload[:response_format] =
+                format_response_format(request.params)
+            end
+
+            log.debug do
+              "vLLM translator rendered request model=#{model} stream=#{request.stream} " \
+                "messages=#{messages.size} tools=#{request.tools&.size || 0} params=#{payload.keys.size}"
+            end
+
+            payload
+          end
+
+          # Parse a vLLM/OpenAI-compatible completion response into a Canonical::Response.
+          def parse_response(wire)
+            return canonical_error_response(wire) unless wire.is_a?(Hash)
+            # Canonical-form response (from conformance kit): already in canonical shape
+            return Canonical::Response.from_hash(wire) if canonical_response?(wire)
+
+            choice = Array(wire['choices']).first || {}
+            message = choice['message'] || {}
+            usage = wire['usage'] || {}
+            finish_reason = choice['finish_reason']
+            model = wire['model']
+
+            content = message['content'] || ''
+            thinking_meta = extract_thinking_metadata(message)
+            extraction = Responses::ThinkingExtractor.extract(content, metadata: thinking_meta)
+
+            text = extraction.content || ''
+            thinking = build_canonical_thinking(extraction)
+
+            tool_calls = parse_tool_calls(message['tool_calls'])
+
+            # vLLM quirk: tool_calls_as_text — synthesize from content if none found.
+            if tool_calls.empty?
+              synthesized = synthesize_tool_calls_from_content(extraction.content, message)
+              tool_calls.concat(synthesized) unless synthesized.empty?
+            end
+
+            stop_reason = map_stop_reason(finish_reason)
+
+            Canonical::Response.build(
+              text: text.to_s,
+              thinking: thinking,
+              tool_calls: tool_calls,
+              usage: Canonical::Usage.from_hash(usage),
+              stop_reason: stop_reason,
+              model: model,
+              metadata: wire_metadata(wire, message, thinking_meta)
+            )
+          rescue Legion::JSON::ParseError => e
+            handle_exception(e, level: :warn, handled: true, operation: 'vllm.translator.parse_response')
+            canonical_error_response(wire)
+          rescue StandardError => e
+            handle_exception(e, level: :error, handled: false, operation: 'vllm.translator.parse_response')
+            raise
+          end
+
+          # Parse a single SSE chunk into a Canonical::Chunk or nil.
+          def parse_chunk(raw)
+            return nil if raw.nil?
+            return nil if raw.is_a?(String) && (raw == '[DONE]' || raw.strip.empty?)
+
+            data = raw.is_a?(Hash) ? raw : parse_json_safely(raw)
+            return nil if data.nil?
+
+            # Handle canonical-form chunks (from conformance fixtures or other translators)
+            return handle_canonical_chunk(data) if data['type']
+
+            if data['error']
+              return Canonical::Chunk.error_chunk(
+                error: data['error'],
+                request_id: data['id']
+              )
+            end
+
+            choice = Array(data['choices']).first
+            return build_done_chunk(data) if choice.nil? && data['usage']
+            return nil unless choice
+
+            delta = choice['delta'] || {}
+            finish_reason = choice['finish_reason']
+            request_id = data['request_id'] || data['id']
+
+            if finish_reason && empty_delta?(delta)
+              return Canonical::Chunk.done(
+                request_id: request_id,
+                usage: Canonical::Usage.from_hash(data['usage']),
+                stop_reason: map_stop_reason(finish_reason)
+              )
+            end
+
+            tool_calls = Array(delta['tool_calls'])
+            return build_tool_call_delta_chunk(tool_calls.first, request_id) unless tool_calls.empty?
+
+            # Thinking delta from reasoning_content
+            reasoning_content = delta['reasoning_content'] || delta['reasoning']
+            unless reasoning_content.to_s.empty?
+              return Canonical::Chunk.thinking_delta(
+                delta: reasoning_content,
+                request_id: request_id,
+                block_index: delta.dig('content_block', 'index'),
+                item_id: delta['content_block_start']&.dig('id')
+              )
+            end
+
+            # Text delta — check for embedded think tags
+            content = delta['content']
+            return parse_text_delta_with_thinking(content, request_id, data) unless content.to_s.empty?
+
+            nil
+          rescue Legion::JSON::ParseError => e
+            handle_exception(e, level: :warn, handled: true, operation: 'vllm.translator.parse_chunk')
+            nil
+          rescue StandardError => e
+            handle_exception(e, level: :error, handled: false, operation: 'vllm.translator.parse_chunk')
+            raise
+          end
+
+          # Declared capabilities for the vLLM provider.
+          def capabilities
+            {
+              provider: 'vllm',
+              wire_format: 'openai_compatible',
+              tool_calls_as_text: true,
+              forced_tool_choice: true,
+              thinking_tags: %w[think thinking],
+              stop_reason_map: VLLM_STOP_REASON_MAP,
+              streaming_token_usage: true
+            }.freeze
+          end
+
+          private
+
+          attr_reader :config
+
+          # ── Message formatting ──
+
+          def format_messages(request)
+            non_system = request.messages&.reject { |m| m.role.to_s == 'system' } || []
+            messages = format_request_messages(non_system)
+
+            if request.system.to_s.strip.empty?
+              messages
+            else
+              [{ role: 'system', content: request.system.strip }] + messages
+            end
+          end
+
+          def format_request_messages(messages)
+            return [] if messages.nil? || messages.empty?
+
+            messages.map { |msg| format_message(msg) }
+          end
+
+          def format_message(msg)
+            role = msg.role.to_s
+            content = format_message_content(msg)
+            tool_calls = format_message_tool_calls(msg.tool_calls) if msg.tool_calls&.any?
+            tool_call_id = msg.tool_call_id
+            name = msg.name
+
+            {
+              role: role,
+              content: content,
+              tool_call_id: tool_call_id,
+              tool_calls: tool_calls,
+              name: name
+            }.compact.reject { |k, v| k == :name && (v.nil? || v.to_s.empty?) }
+          end
+
+          def format_message_content(msg)
+            content = msg.content
+            return content if content.is_a?(String) && !content.empty?
+
+            case content
+            when Array
+              format_content_blocks(content)
+            when Canonical::ContentBlock
+              format_content_blocks([content])
+            when Hash
+              format_content_blocks_from_hash(content)
+            else
+              content&.to_s
+            end
+          end
+
+          def format_content_blocks(blocks)
+            parts = blocks.map do |block|
+              if block.is_a?(Canonical::ContentBlock)
+                format_content_block(block)
+              elsif block.is_a?(Hash)
+                format_content_block_from_hash(block)
+              else
+                { type: 'text', text: block.to_s }
+              end
+            end
+            parts.empty? ? '' : parts
+          end
+
+          # rubocop:disable Lint/DuplicateBranch -- multiple block types intentionally normalize to text in OpenAI wire format
+          def format_content_block(block)
+            case block.type
+            when :text, :thinking, :tool_result
+              { type: 'text', text: block.text.to_s }
+            when :tool_use
+              { type: 'text', text: Legion::JSON.generate(block.input || {}) }
+            when :image
+              build_image_block(block)
+            else
+              { type: 'text', text: block.text.to_s }
+            end
+          end
+          # rubocop:enable Lint/DuplicateBranch
+
+          def format_content_blocks_from_hash(hash_input)
+            case hash_input
+            when Hash
+              [format_content_block_from_hash(hash_input)]
+            when Array
+              hash_input.map { |h| format_content_block_from_hash(h) }
+            else
+              []
+            end
+          end
+
+          # rubocop:disable Lint/DuplicateBranch -- multiple block types intentionally normalize to text in OpenAI wire format
+          def format_content_block_from_hash(block_hash)
+            h = block_hash.transform_keys(&:to_sym)
+            type = (h[:type] || :text).to_sym
+
+            case type
+            when :text, :thinking, :tool_result
+              { type: 'text', text: h[:text].to_s }
+            when :tool_use
+              { type: 'text', text: Legion::JSON.generate(h[:input] || {}) }
+            when :image, :image_url
+              { type: 'image_url', image_url: { url: h[:data] || h[:url] || '' } }
+            else
+              { type: 'text', text: h[:text].to_s }
+            end
+          end
+          # rubocop:enable Lint/DuplicateBranch
+
+          def build_image_block(block)
+            return {} unless block.data || block.source_type
+
+            url = if block.source_type == :base64 && block.media_type
+                    "data:#{block.media_type};base64,#{block.data}"
+                  else
+                    block.data
+                  end
+            { type: 'image_url', image_url: { url: url } }
+          end
+
+          def format_message_tool_calls(tool_calls)
+            return [] if tool_calls.empty?
+
+            tc_array = tool_calls.is_a?(Hash) ? tool_calls.values : Array(tool_calls)
+            tc_array.map { |tc| format_tool_call_for_history(tc) }
+          end
+
+          def format_tool_call_for_history(tool_call_entry)
+            tc_hash = case tool_call_entry
+                      when Canonical::ToolCall
+                        { name: tool_call_entry&.name&.to_s, id: tool_call_entry&.id&.to_s,
+                          arguments: tool_call_entry&.arguments || {} }
+                      when Hash
+                        tool_call_entry.transform_keys(&:to_sym)
+                      else
+                        tool_call_entry
+                      end
+
+            name = tc_hash[:name] || tc_hash['name']
+            id = tc_hash[:id] || tc_hash['id']
+            args = tc_hash[:arguments] || tc_hash['arguments'] || {}
+            args = args.is_a?(Hash) ? Legion::JSON.generate(args) : args.to_s
+
+            {
+              id: id.to_s,
+              type: 'function',
+              function: { name: name.to_s, arguments: args }
+            }
+          end
+
+          # ── Tool formatting ──
+
+          def format_tools(tools)
+            return [] if tools.to_h.empty?
+
+            tools.to_h.values.map do |tool|
+              tool_hash = if tool.is_a?(Canonical::ToolDefinition)
+                            { name: tool.name, description: tool.description, parameters: tool.parameters }
+                          elsif tool.is_a?(Hash)
+                            tool.transform_keys(&:to_sym)
+                          else
+                            tool
+                          end
+
+              name = tool_hash[:name] || tool_hash['name']
+              description = (tool_hash[:description] || tool_hash['description'] || '').to_s
+              raw_params = tool_hash[:parameters] || tool_hash[:input_schema]
+              raw_params = raw_params.to_h if raw_params.respond_to?(:to_h) && !raw_params.is_a?(Hash)
+              parameters = Legion::Extensions::Llm::Canonical::ToolDefinition.normalize_parameters(raw_params)
+
+              {
+                type: 'function',
+                function: {
+                  name: name.to_s,
+                  description: description,
+                  parameters: parameters
+                }
+              }
+            end
+          end
+
+          def format_tool_choice(choice)
+            return nil unless choice
+
+            case choice
+            when :auto, 'auto'
+              'auto'
+            when :none, 'none'
+              'none'
+            when :required, 'required'
+              'required'
+            when Hash
+              name = choice[:name] || choice['name']
+              { type: 'function', function: { name: name.to_s } }
+            when Symbol, String
+              { type: 'function', function: { name: choice.to_s } }
+            end
+          end
+
+          # ── Parameter mapping (G18) ──
+
+          def map_params_to_wire(params)
+            return {} unless params.is_a?(Canonical::Params)
+
+            wire = {}
+            SUPPORTED_PARAMS.each do |param_key|
+              value = params.public_send(param_key)
+              next if value.nil?
+
+              wire_key = PARAM_WIRE_KEYS[param_key]
+              wire[wire_key] = case param_key
+                               when :stop_sequences
+                                 format_stop_sequences(value)
+                               when :response_format
+                                 format_response_format_value(value)
+                               else
+                                 value
+                               end
+            end
+
+            unsupported = {}
+            unsupported[:max_thinking_tokens] = params.max_thinking_tokens if params.max_thinking_tokens
+
+            unless unsupported.empty?
+              log.debug do
+                "vLLM translator dropping unsupported params: #{unsupported.keys.join(', ')} " \
+                  '(handled via vLLM-specific render paths)'
+              end
+            end
+
+            wire
+          end
+
+          def format_stop_sequences(sequences)
+            sequences.is_a?(Array) ? sequences : [sequences]
+          end
+
+          def format_response_format(params)
+            return nil unless formatted_response_format?(params)
+
+            format_response_format_value(params.response_format)
+          end
+
+          def formatted_response_format?(params)
+            params.is_a?(Canonical::Params) && params.response_format
+          end
+
+          def format_response_format_value(value)
+            return value if value.is_a?(String)
+
+            val_hash = value.is_a?(Hash) ? value.transform_keys(&:to_sym) : {}
+            type = val_hash[:type] || val_hash['type']
+
+            case type&.to_s
+            when 'json_schema'
+              schema = val_hash[:schema] || val_hash['schema'] || val_hash[:json_schema] || val_hash['json_schema']
+              { type: 'json_schema', json_schema: schema }
+            when 'json_object'
+              { type: 'json_object' }
+            else
+              value
+            end
+          end
+
+          # ── Thinking configuration ──
+
+          def apply_thinking_config(payload, request)
+            return unless enable_thinking?(request)
+
+            payload[:chat_template_kwargs] = { enable_thinking: true }
+            budget = request.params&.max_thinking_tokens
+            return unless budget&.positive?
+
+            log.debug { "vLLM translator thinking max_thinking_tokens=#{budget} via chat template" }
+          end
+
+          def enable_thinking?(request)
+            return true if request.thinking.is_a?(Canonical::Thinking::Config) && request.thinking.enabled?
+            return true if request.thinking.is_a?(Hash) && (request.thinking[:enabled] != false)
+
+            if request.thinking.nil? && config
+              config_thinking = if config.respond_to?(:enable_thinking)
+                                  config.enable_thinking
+                                else
+                                  config.respond_to?(:[]) ? config[:enable_thinking] : nil
+                                end
+              return true if config_thinking == true
+            end
+
+            false
+          end
+
+          # ── Response parsing ──
+
+          def canonical_error_response(wire)
+            body = wire.is_a?(Hash) ? wire : {}
+            error_info = body['error'] || { type: 'parse_error', message: 'Failed to parse response' }
+
+            Canonical::Response.build(
+              text: '',
+              tool_calls: [],
+              usage: Canonical::Usage.from_hash(body['usage'] || {}),
+              stop_reason: :error,
+              model: body['model'],
+              metadata: { error: error_info }
+            )
+          end
+
+          def extract_thinking_metadata(message)
+            {
+              reasoning_content: message['reasoning_content'],
+              reasoning: message['reasoning'],
+              thinking: message['thinking'],
+              thinking_text: message['thinking_text'],
+              thinking_signature: message['thinking_signature'],
+              reasoning_signature: message['reasoning_signature']
+            }.compact
+          end
+
+          def build_canonical_thinking(extraction)
+            return nil unless extraction.thinking || extraction.signature
+
+            Canonical::Thinking.new(
+              content: extraction.thinking,
+              signature: extraction.signature
+            )
+          end
+
+          def parse_tool_calls(tool_calls)
+            return [] unless tool_calls.is_a?(Array) && !tool_calls.empty?
+
+            tool_calls.filter_map do |call|
+              function = call.fetch('function', {})
+              name = function['name']
+              id = call['id'] || name || call['index']
+              args = parse_tool_arguments(function['arguments'])
+
+              Canonical::ToolCall.build(
+                id: id.to_s,
+                name: name.to_s,
+                arguments: args,
+                source: :client
+              )
+            rescue StandardError => e
+              handle_exception(e, level: :warn, handled: true, operation: 'vllm.translator.parse_tool_call')
+              nil
+            end
+          end
+
+          def parse_tool_arguments(arguments)
+            return {} if arguments.nil? || arguments == ''
+            return arguments if arguments.is_a?(Hash)
+
+            Legion::JSON.load(arguments)
+          rescue Legion::JSON::ParseError
+            {}
+          end
+
+          # vLLM quirk: synthesize tool calls from content text JSON.
+          def synthesize_tool_calls_from_content(content, _message)
+            return [] unless content.is_a?(String) && !content.empty?
+
+            tool_call = try_parse_tool_call_from_text(content)
+            return [tool_call] if tool_call
+
+            json_match = content.match(/\{[^{}]*(?:tool|function|name|arguments)[^{}]*\}/m)
+            return [] unless json_match
+
+            tool_call = try_parse_tool_call_from_text(json_match[0])
+            tool_call ? [tool_call] : []
+          end
+
+          def try_parse_tool_call_from_text(text)
+            parsed = Legion::JSON.load(text)
+            return nil unless parsed.is_a?(Hash)
+
+            name = parsed[:name] || parsed[:function_name]
+            args = parsed[:arguments] || parsed[:parameters] || parsed[:input] || {}
+            args = Legion::JSON.load(args) if args.is_a?(String)
+
+            return nil if name.nil? || name.to_s.empty?
+
+            Canonical::ToolCall.build(
+              name: name.to_s,
+              arguments: args.is_a?(Hash) ? args : {},
+              source: :client
+            )
+          rescue Legion::JSON::ParseError
+            nil
+          end
+
+          def wire_metadata(wire, message, _thinking_meta)
+            meta = {}
+            meta[:reasoning_content] = message['reasoning_content'] if message['reasoning_content']
+            raw_usage = wire['usage']
+            if raw_usage.is_a?(Hash) && raw_usage['completion_tokens_details']
+              meta[:completion_tokens_details] = raw_usage['completion_tokens_details']
+            end
+            meta
+          end
+
+          # ── Chunk helpers ──
+
+          def build_done_chunk(data)
+            Canonical::Chunk.done(
+              request_id: data['request_id'] || data['id'],
+              usage: Canonical::Usage.from_hash(data['usage']),
+              stop_reason: nil
+            )
+          end
+
+          # Build a tool_call_delta chunk preserving OpenAI streaming fragment
+          # semantics: the opening fragment carries id + name; continuation
+          # fragments carry id: nil and a raw partial-JSON arguments string.
+          # The StreamAccumulator keys off a nil id to append fragments to the
+          # current tool call, so the id must NOT be synthesized here.
+          def build_tool_call_delta_chunk(first_call, request_id)
+            function = first_call.fetch('function', {})
+
+            tc = Canonical::ToolCall.new(
+              id: first_call['id'], exchange_id: nil,
+              name: function['name'], arguments: function['arguments'].to_s,
+              source: :client, status: nil, duration_ms: nil, result: nil,
+              error: nil, started_at: nil, finished_at: nil, category: nil,
+              data_handling_classification: nil, policy_decision: nil
+            )
+
+            Canonical::Chunk.tool_call_delta(
+              tool_call: tc,
+              request_id: request_id,
+              block_index: first_call['index']
+            )
+          end
+
+          def empty_delta?(delta)
+            (delta['content'].nil? || delta['content'].to_s.empty?) &&
+              (delta['tool_calls'].nil? || Array(delta['tool_calls']).empty?) &&
+              (delta['reasoning_content'].nil? || delta['reasoning_content'].to_s.empty?)
+          end
+
+          # Per-chunk think-tag extraction is structurally impossible while streaming:
+          # tags arrive split across SSE chunks, and ThinkingExtractor strips per-chunk
+          # whitespace, corrupting reassembled text. Emit the raw delta unmodified —
+          # the StreamAccumulator extracts think tags statefully across deltas.
+          # (Previously called ThinkingExtractor.extract_from_content, which is
+          # private_class_method in lex-llm >= 0.5.0 and raised NoMethodError on
+          # every streamed text delta, silently killing all vLLM streaming.)
+          def parse_text_delta_with_thinking(content, request_id, data)
+            Canonical::Chunk.text_delta(
+              delta: content,
+              request_id: request_id,
+              index: data['index']
+            )
+          end
+
+          # Parse a canonical-form chunk (from conformance kit fixtures).
+
+          # Detect canonical-form response (from conformance fixtures).
+          def canonical_response?(wire)
+            wire.key?('text') || wire['text'] || wire.key?(:stop_reason) || wire.key?('stop_reason')
+          end
+
+          def handle_canonical_chunk(data)
+            Canonical::Chunk.from_hash(data)
+          rescue StandardError => e
+            log.debug { "vLLM translator canonical chunk parse error: #{e.message}" }
+            nil
+          end
+
+          # ── Stop reason mapping ──
+
+          def map_stop_reason(raw)
+            return FALLBACK_STOP_REASON if raw.nil? || raw.to_s.empty?
+
+            VLLM_STOP_REASON_MAP.fetch(raw.to_s, FALLBACK_STOP_REASON)
+          end
+
+          # ── JSON helpers ──
+          # Never use bare ::JSON inside the Legion namespace.
+
+          def parse_json_safely(raw)
+            return nil unless raw.is_a?(String)
+
+            Legion::JSON.load(raw)
+          rescue Legion::JSON::ParseError => e
+            log.debug { "vLLM translator chunk parse error: #{e.message}" }
+            nil
+          end
+        end
+        # rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/vllm/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Vllm
-        VERSION = '0.2.13'
+        VERSION = '0.3.5'
       end
     end
   end

data/lib/legion/extensions/llm/vllm.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/extensions/llm'
+require 'legion/extensions/llm/vllm/translator'
 require 'legion/extensions/llm/vllm/provider'
 require 'legion/extensions/llm/vllm/version'
 require 'legion/logging'
@@ -15,7 +16,7 @@ module Legion
         extend Legion::Extensions::Llm::AutoRegistration
 
         PROVIDER_FAMILY = :vllm
-        DEFAULT_INSTANCE_TIER = { tier: :direct, capabilities: %i[completion streaming vision tools] }.freeze
+        DEFAULT_INSTANCE_TIER = { tier: :direct, capabilities: {}, provider_capabilities: { streaming: true } }.freeze
 
         def self.default_settings
           ::Legion::Extensions::Llm.provider_settings(
@@ -31,10 +32,7 @@ module Legion
               fleet: {
                 enabled: false,
                 respond_to_requests: false,
-                capabilities: %i[chat stream_chat embed],
-                lanes: [],
-                concurrency: 1,
-                queue_suffix: nil
+                capabilities: %i[chat stream_chat embed]
               }
             }
           )
@@ -73,10 +71,19 @@ module Legion
         def self.normalize_instance_config(config)
           normalized = config.to_h.transform_keys(&:to_sym)
           resolve_api_base_aliases(normalized)
+          resolve_credentials(normalized)
           normalized[:tier] ||= infer_tier_from_endpoint(normalized[:vllm_api_base])
           normalized
         end
 
+        def self.resolve_credentials(normalized)
+          creds = normalized.delete(:credentials)
+          return unless creds.is_a?(Hash)
+
+          creds = creds.transform_keys(&:to_sym)
+          normalized[:vllm_api_key] ||= creds[:api_key]
+        end
+
         def self.resolve_api_base_aliases(normalized)
           normalized[:vllm_api_base] ||= normalized.delete(:base_url)
           normalized[:vllm_api_base] ||= normalized.delete(:api_base)
@@ -92,12 +99,15 @@ module Legion
           return :direct if url.nil? || url.to_s.empty?
 
           require 'uri'
+          require_relative 'vllm/actors/discovery_refresh'
           host = URI.parse(url.to_s).host.to_s.downcase
           %w[localhost 127.0.0.1 ::1].include?(host) ? :local : :direct
         rescue URI::InvalidURIError => e
           handle_exception(e, level: :debug, handled: true, operation: 'vllm.infer_tier_from_endpoint')
           :direct
         end
+
+        Legion::Extensions::Llm::Configuration.register_provider_options(Provider.configuration_options)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-vllm
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.3.5
 platform: ruby
 authors:
 - LegionIO
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: 0.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.3
+        version: 0.5.0
 description: vLLM provider integration for the LegionIO LLM routing framework.
 email:
 - matthewdiverson@gmail.com
@@ -101,6 +101,7 @@ files:
 - lib/legion/extensions/llm/vllm/actors/fleet_worker.rb
 - lib/legion/extensions/llm/vllm/provider.rb
 - lib/legion/extensions/llm/vllm/runners/fleet_worker.rb
+- lib/legion/extensions/llm/vllm/translator.rb
 - lib/legion/extensions/llm/vllm/version.rb
 homepage: https://github.com/LegionIO/lex-llm-vllm
 licenses: