lex-llm-vllm 0.3.13 → 0.3.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a62d4fc4e5b1020ccf9a4bcb5ed9c03096923075c7111357648cc1b360dd804b
-  data.tar.gz: 1cb2fe7f84f9fcd44b0c04407e62310e5fa4fad59bdfd2bc65889bfa8b9bd213
+  metadata.gz: 8a53b31d5fe4efcc8b9451b0934c52bcbf71a4af70e493cfe82d7ba75f40ce68
+  data.tar.gz: a567a34e2f14e2cf0497b98926527fa9895040981e19e0340006bdf143640ef4
 SHA512:
-  metadata.gz: 0a68362c7660f1474b943d8d67948e71b3d2b23840b39835937839bf5d2542d0baaa05b569764f77e5268cde1848dfca20e2e16274de6f112756a8eaa1b79ccc
-  data.tar.gz: de8011c0d1468c8d3f35215f01b7ee3ffa182d47c56dc89ac5fd983dcbf6f9de52ef167fc04d7514b984423081b65f4a8a6f5b463dc6d4372bbcbcdd90d7aa75
+  metadata.gz: 0a64470ff262c97f5cd99b02c066cf24ffb2872ae7dcde10597e9869f3cfc2be89ffc4d07bf9bb43bd83b83d161cfc8d987f25121f87e9cf4f8770abe66fb358
+  data.tar.gz: cd4b89c6a0297f70232cf882480b24f903517caec5ed9f7129a61e0dc0d65bb599126cc1fa7cceccd388c8411035db74d737ff15351d7c3ad4c9433823fd5e37

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## [0.3.16] - 2026-07-31
+
+### Fixed
+- **`parse_chunk` now handles multiple tool_calls batched in a single SSE delta.** vLLM can batch several parallel tool_calls into one `choices[0].delta.tool_calls` array. Previously only `tool_calls.first` was processed — the 2nd+ tool calls were silently dropped, causing parallel tool invocations to lose calls. Now returns an array of `tool_call_delta` chunks (one per tool_call), which the streaming handler already iterates (per lex-llm 0.6.13). Single tool_call deltas still return a single chunk (not wrapped in an array) for backward compatibility.
+
+## [0.3.15] - 2026-07-31
+
+### Fixed
+- **`to_legacy_chunk` now propagates `stop_reason` from canonical chunks.** The canonical translator correctly parsed vLLM's `finish_reason` into `Canonical::Chunk.stop_reason`, but `to_legacy_chunk` never passed it into the legacy `Legion::Extensions::Llm::Chunk` constructor. The `StreamAccumulator` already reads `chunk.stop_reason` (line 40), so the field was silently nil on every streamed chunk — downstream always defaulted to `:end_turn` regardless of what the provider actually said. Truncated responses (`:max_tokens`) and content-filtered responses (`:content_filter`) were indistinguishable from clean completions. Now the real finish_reason flows through: canonical chunk → legacy chunk → accumulator → assembled Message.
+
+## [0.3.14] - 2026-07-24
+
+### Fixed
+- **Translator passes `finish_reason` and `usage` through on all streaming chunk types.** vLLM sends `finish_reason` on the same chunk as the last content token. Previously the translator dropped it because the early-return pattern only emitted finish_reason when the delta was empty. Now `text_delta`, `tool_call_delta`, and `thinking_delta` chunks carry `stop_reason` and `usage` when present on the SSE event, enabling the accumulator to capture the real provider signal.
+- **Translator returns both thinking and content when vLLM sends both on the same SSE chunk.** vLLM emits `reasoning` and `content` simultaneously on the boundary between thinking and visible output. Previously the first-branch-wins pattern dropped content when reasoning was present, causing visible characters to disappear from responses (e.g. numbered list items losing their leading digit). `parse_chunk` now returns an array `[thinking_delta, text_delta]` when both fields are present, and the streaming handler iterates them.
+
 ## [0.3.13] - 2026-07-14
 
 ### Fixed

data/lex-llm-vllm.gemspec

@@ -27,5 +27,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'legion-logging', '>= 1.3.2'
   spec.add_dependency 'legion-settings', '>= 1.3.14'
   spec.add_dependency 'legion-transport', '>= 1.4.14'
-  spec.add_dependency 'lex-llm', '>= 0.6.9'
+  spec.add_dependency 'lex-llm', '>= 0.6.13'
 end

data/lib/legion/extensions/llm/vllm/provider.rb

@@ -363,6 +363,7 @@ module Legion
               thinking: thinking,
               input_tokens: usage.respond_to?(:input_tokens) ? usage.input_tokens : nil,
               output_tokens: usage.respond_to?(:output_tokens) ? usage.output_tokens : nil,
+              stop_reason: canonical.stop_reason,
               raw: raw_data
             )
           end
@@ -471,10 +472,14 @@ module Legion
 
           # Override: delegate SSE chunk parsing to the canonical translator.
           def build_chunk(data)
-            canonical_chunk = translator.parse_chunk(data)
-            return nil if canonical_chunk.nil?
+            result = translator.parse_chunk(data)
+            return nil if result.nil?
 
-            to_legacy_chunk(canonical_chunk, data)
+            if result.is_a?(Array)
+              result.map { |c| to_legacy_chunk(c, data) }
+            else
+              to_legacy_chunk(result, data)
+            end
           end
 
           def parse_list_models_response(response, provider, capabilities)

data/lib/legion/extensions/llm/vllm/translator.rb

@@ -163,23 +163,46 @@ module Legion
               )
             end
 
+            chunk_stop_reason = finish_reason ? map_stop_reason(finish_reason) : nil
+            chunk_usage = finish_reason && data['usage'] ? Canonical::Usage.from_hash(data['usage']) : nil
+
             tool_calls = Array(delta['tool_calls'])
-            return build_tool_call_delta_chunk(tool_calls.first, request_id) unless tool_calls.empty?
+            unless tool_calls.empty?
+              if delta['content'] && !delta['content'].to_s.empty?
+                log.debug '[vllm][translator] action=content_dropped_with_tool_call ' \
+                          "content=#{delta['content'][0, 100].inspect} request_id=#{request_id}"
+              end
+              chunks = tool_calls.map do |tc|
+                build_tool_call_delta_chunk(tc, request_id,
+                                            stop_reason: chunk_stop_reason, usage: chunk_usage)
+              end
+              return chunks.size == 1 ? chunks.first : chunks
+            end
 
             # Thinking delta from reasoning_content
             reasoning_content = delta['reasoning_content'] || delta['reasoning']
+            content = delta['content']
             unless reasoning_content.to_s.empty?
-              return Canonical::Chunk.thinking_delta(
+              thinking_chunk = Canonical::Chunk.thinking_delta(
                 delta: reasoning_content,
                 request_id: request_id,
                 block_index: delta.dig('content_block', 'index'),
-                item_id: delta['content_block_start']&.dig('id')
+                item_id: delta['content_block_start']&.dig('id'),
+                stop_reason: content.to_s.empty? ? chunk_stop_reason : nil,
+                usage: content.to_s.empty? ? chunk_usage : nil
               )
+              return thinking_chunk if content.to_s.empty?
+
+              content_chunk = parse_text_delta_with_thinking(content, request_id, data,
+                                                             stop_reason: chunk_stop_reason, usage: chunk_usage)
+              return [thinking_chunk, content_chunk]
             end
 
             # Text delta — check for embedded think tags
-            content = delta['content']
-            return parse_text_delta_with_thinking(content, request_id, data) unless content.to_s.empty?
+            unless content.to_s.empty?
+              return parse_text_delta_with_thinking(content, request_id, data,
+                                                    stop_reason: chunk_stop_reason, usage: chunk_usage)
+            end
 
             nil
           rescue Legion::JSON::ParseError => e
@@ -635,7 +658,7 @@ module Legion
           # fragments carry id: nil and a raw partial-JSON arguments string.
           # The StreamAccumulator keys off a nil id to append fragments to the
           # current tool call, so the id must NOT be synthesized here.
-          def build_tool_call_delta_chunk(first_call, request_id)
+          def build_tool_call_delta_chunk(first_call, request_id, stop_reason: nil, usage: nil)
             function = first_call.fetch('function', {})
 
             tc = Canonical::ToolCall.new(
@@ -649,7 +672,9 @@ module Legion
             Canonical::Chunk.tool_call_delta(
               tool_call: tc,
               request_id: request_id,
-              block_index: first_call['index']
+              block_index: first_call['index'],
+              stop_reason: stop_reason,
+              usage: usage
             )
           end
 
@@ -667,11 +692,13 @@ module Legion
           # (Previously called ThinkingExtractor.extract_from_content, which is
           # private_class_method in lex-llm >= 0.5.0 and raised NoMethodError on
           # every streamed text delta, silently killing all vLLM streaming.)
-          def parse_text_delta_with_thinking(content, request_id, data)
+          def parse_text_delta_with_thinking(content, request_id, data, stop_reason: nil, usage: nil)
             Canonical::Chunk.text_delta(
               delta: content,
               request_id: request_id,
-              index: data['index']
+              index: data['index'],
+              stop_reason: stop_reason,
+              usage: usage
             )
           end
 

data/lib/legion/extensions/llm/vllm/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Llm
       module Vllm
-        VERSION = '0.3.13'
+        VERSION = '0.3.16'
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-vllm
 version: !ruby/object:Gem::Version
-  version: 0.3.13
+  version: 0.3.16
 platform: ruby
 authors:
 - LegionIO
@@ -71,14 +71,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.9
+        version: 0.6.13
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.9
+        version: 0.6.13
 description: vLLM provider integration for the LegionIO LLM routing framework.
 email:
 - matthewdiverson@gmail.com