lex-llm 0.3.1 → 0.4.3

data/lib/legion/extensions/llm/responses/thinking_extractor.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Llm
+      module Responses
+        # Separates provider thinking markup from caller-visible text.
+        module ThinkingExtractor
+          Extraction = Struct.new(:content, :thinking, :signature, :metadata, keyword_init: true)
+
+          THINK_OPEN = '<think>'
+          THINK_CLOSE = '</think>'
+          THINK_PATTERN = %r{<think>(.*?)</think>}m
+          THINKING_METADATA_KEYS = %i[
+            reasoning_content reasoning thinking thinking_text thinking_signature reasoning_signature thought_signature
+          ].freeze
+          RAW_METADATA_KEYS = %i[
+            raw raw_response response_body provider_body provider_response
+          ].freeze
+
+          module_function
+
+          def extract(content, metadata: {})
+            metadata = normalized_metadata(metadata)
+            content, extracted_thinking = extract_from_content(content)
+            metadata_thinking = extract_metadata_thinking(metadata)
+            metadata_signature = extract_metadata_signature(metadata)
+
+            Extraction.new(
+              content: content,
+              thinking: compact_thinking([metadata_thinking, extracted_thinking]),
+              signature: metadata_signature,
+              metadata: scrub_metadata(metadata)
+            )
+          end
+
+          def extract_from_content(content)
+            return [content, nil] unless content.is_a?(String)
+
+            clean = +''
+            thinking_parts = []
+            remaining = content.dup
+
+            remaining = consume_next_segment(remaining, clean, thinking_parts) until remaining.empty?
+
+            [clean.strip, compact_thinking(thinking_parts)]
+          end
+          private_class_method :extract_from_content
+
+          def consume_next_segment(remaining, clean, thinking_parts)
+            close_index = remaining.index(THINK_CLOSE)
+            open_index = remaining.index(THINK_OPEN)
+
+            if close_index && (open_index.nil? || close_index < open_index)
+              thinking_parts << remaining.slice(0, close_index)
+              remaining.slice((close_index + THINK_CLOSE.length)..).to_s.sub(/\A[[:space:]]+/, '')
+            elsif open_index
+              consume_open_think_segment(remaining, open_index, clean, thinking_parts)
+            else
+              clean << remaining
+              +''
+            end
+          end
+          private_class_method :consume_next_segment
+
+          def consume_open_think_segment(remaining, open_index, clean, thinking_parts)
+            clean << remaining.slice(0, open_index)
+            after_open = remaining.slice((open_index + THINK_OPEN.length)..).to_s
+            close_index = after_open.index(THINK_CLOSE)
+            unless close_index
+              thinking_parts << after_open
+              return +''
+            end
+
+            thinking_parts << after_open.slice(0, close_index)
+            after_open.slice((close_index + THINK_CLOSE.length)..).to_s
+          end
+          private_class_method :consume_open_think_segment
+
+          def extract_metadata_thinking(metadata)
+            compact_thinking(
+              [
+                metadata[:reasoning_content],
+                metadata[:reasoning],
+                metadata[:thinking],
+                metadata[:thinking_text]
+              ]
+            )
+          end
+          private_class_method :extract_metadata_thinking
+
+          def extract_metadata_signature(metadata)
+            [
+              metadata[:thinking_signature],
+              metadata[:reasoning_signature],
+              metadata[:thought_signature]
+            ].compact.map { |signature| signature.to_s.strip }.find { |signature| !signature.empty? }
+          end
+          private_class_method :extract_metadata_signature
+
+          def scrub_metadata(metadata)
+            metadata.each_with_object({}) do |(key, value), scrubbed|
+              normalized_key = normalize_metadata_key(key)
+              next if THINKING_METADATA_KEYS.include?(normalized_key) || RAW_METADATA_KEYS.include?(normalized_key)
+
+              scrubbed[normalized_key] = scrub_metadata_value(value)
+            end
+          end
+          private_class_method :scrub_metadata
+
+          def normalize_metadata_key(key)
+            key.to_s
+               .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+               .tr('-', '_')
+               .downcase
+               .to_sym
+          end
+          private_class_method :normalize_metadata_key
+
+          def scrub_metadata_value(value)
+            case value
+            when Hash
+              scrub_metadata(normalized_metadata(value))
+            when Array
+              value.map { |item| scrub_metadata_value(item) }
+            when String
+              extract_from_content(value).first
+            else
+              value
+            end
+          end
+          private_class_method :scrub_metadata_value
+
+          def normalized_metadata(metadata)
+            return {} if metadata.nil?
+
+            metadata.to_h.transform_keys { |key| normalize_metadata_key(key) }
+          end
+          private_class_method :normalized_metadata
+
+          def compact_thinking(parts)
+            text = parts.compact.map { |part| part.to_s.strip }.reject(&:empty?).join
+            blank_to_nil(text)
+          end
+          private_class_method :compact_thinking
+
+          def blank_to_nil(value)
+            value.nil? || value.empty? ? nil : value
+          end
+          private_class_method :blank_to_nil
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/stream_accumulator.rb

@@ -210,8 +210,11 @@ module Legion
         end
 
         def consume_non_think_content(remaining, start_tag, output)
+          unmatched_close = remaining.index('</think>')
           start_index = remaining.index(start_tag)
-          if start_index
+          if unmatched_close && (start_index.nil? || unmatched_close < start_index)
+            consume_unmatched_think_close(remaining, unmatched_close)
+          elsif start_index
             output << remaining.slice(0, start_index)
             @inside_think_tag = true
             remaining.slice((start_index + start_tag.length)..) || +''
@@ -223,6 +226,14 @@ module Legion
           end
         end
 
+        def consume_unmatched_think_close(remaining, close_index)
+          end_tag = '</think>'
+          thinking = remaining.slice(0, close_index)
+          @thinking_text << thinking
+          @last_thinking_delta << thinking
+          remaining.slice((close_index + end_tag.length)..).to_s.sub(/\A[[:space:]]+/, '')
+        end
+
         def longest_suffix_prefix(text, tag)
           max = [text.length, tag.length - 1].min
           max.downto(1) do |len|

data/lib/legion/extensions/llm/transport/exchanges/fleet.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'legion/transport'
+
+module Legion
+  module Extensions
+    module Llm
+      module Transport
+        module Exchanges
+          # Shared topic exchange for live LLM fleet requests and replies.
+          class Fleet < ::Legion::Transport::Exchange
+            def exchange_name
+              'llm.fleet'
+            end
+
+            def default_type
+              'topic'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/transport/messages/fleet_error.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative '../../fleet/default_exchange_reply'
+require_relative '../../fleet/envelope_validation'
+require_relative '../../fleet/protocol'
+require_relative '../exchanges/fleet'
+
+module Legion
+  module Extensions
+    module Llm
+      module Transport
+        module Messages
+          # Correlated protocol-v2 error envelope for fleet reply queues.
+          class FleetError < ::Legion::Transport::Message
+            include Fleet::DefaultExchangeReply
+            include Fleet::EnvelopeValidation
+
+            def type = Fleet::Protocol::ERROR_TYPE
+            def app_id = @options[:app_id] || 'lex-llm'
+            def reply_to = @options[:reply_to]
+            def correlation_id = @options[:correlation_id]
+            def message_id = @options[:message_id] ||= "llm_fleet_err_#{SecureRandom.uuid}"
+
+            def routing_key
+              @options[:reply_to] || raise(ArgumentError, 'reply_to is required')
+            end
+
+            def validate
+              reject_legacy_options!
+              require_option!(:request_id)
+              require_option!(:correlation_id)
+              require_option!(:reply_to)
+              require_option!(:code)
+              require_protocol_version!
+              @valid = true
+            end
+
+            def message
+              super.merge(
+                protocol_version: @options[:protocol_version] || Fleet::Protocol::VERSION,
+                request_id: @options[:request_id],
+                correlation_id: correlation_id,
+                idempotency_key: @options[:idempotency_key],
+                operation: @options[:operation],
+                provider: @options[:provider],
+                provider_instance: @options[:provider_instance] || @options[:instance],
+                model: @options[:model],
+                reply_to: reply_to,
+                message_context: @options[:message_context],
+                trace_context: @options[:trace_context],
+                code: @options[:code],
+                message: @options[:message],
+                error_class: @options[:error_class],
+                retryable: @options[:retryable],
+                metadata: @options[:metadata] || {}
+              ).compact
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/transport/messages/fleet_request.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative '../../fleet/envelope_validation'
+require_relative '../../fleet/publish_safety'
+require_relative '../../fleet/protocol'
+require_relative '../exchanges/fleet'
+
+module Legion
+  module Extensions
+    module Llm
+      module Transport
+        module Messages
+          # Strict protocol-v2 request envelope for outbound fleet work.
+          class FleetRequest < ::Legion::Transport::Message
+            include Fleet::EnvelopeValidation
+            include Fleet::PublishSafety
+
+            PRIORITY_MAP = { critical: 9, high: 7, normal: 5, low: 2 }.freeze
+            DEFAULT_PUBLISH_OPTIONS = {
+              mandatory: true,
+              publisher_confirm: true,
+              spool: false,
+              return_result: true
+            }.freeze
+            REQUIRED_OPTIONS = %i[
+              request_id correlation_id operation provider provider_instance model params reply_to
+              message_context caller trace_context signed_token timeout_seconds expires_at protocol_version
+              idempotency_key
+            ].freeze
+
+            def exchange = Exchanges::Fleet
+            def type = Fleet::Protocol::REQUEST_TYPE
+            def app_id = @options[:app_id] || 'lex-llm'
+            def reply_to = @options[:reply_to]
+            def correlation_id = @options[:correlation_id]
+            def message_id = @options[:message_id] ||= "llm_fleet_req_#{SecureRandom.uuid}"
+
+            def priority
+              PRIORITY_MAP.fetch(@options[:priority].to_sym, 5) if @options[:priority]
+            end
+
+            def routing_key
+              @options[:routing_key] || raise(ArgumentError, 'routing_key is required')
+            end
+
+            def expiration
+              ttl = @options[:ttl] || @options[:timeout_seconds]
+              return super unless ttl
+
+              (Float(ttl) * 1000).ceil.to_s
+            rescue ArgumentError, TypeError
+              super
+            end
+
+            def publish(options = nil)
+              raise unless @valid
+
+              requested_options = DEFAULT_PUBLISH_OPTIONS.merge(@options).merge(options || {})
+              return_result = return_publish_result?(requested_options)
+              publish_options = request_publish_options(requested_options)
+              validate_payload_size
+              exchange_dest = fleet_exchange
+              return_state = {}
+              install_return_listener(exchange_dest, requested_options, return_state)
+              prepare_publisher_confirms(exchange_dest, requested_options)
+              exchange_dest.publish(encode_message, **publish_options)
+              return nil unless return_result
+
+              publish_result(exchange_dest, requested_options.merge(publish_options), return_state)
+            rescue Bunny::ConnectionClosedError, Bunny::ChannelAlreadyClosed, Bunny::ChannelError,
+                   Bunny::NetworkErrorWrapper, IOError, Timeout::Error => e
+              handle_exception(e, level: :warn, handled: true, operation: 'llm.fleet.request.publish')
+              publish_failure_result(:failed, e, publish_options || requested_options || @options)
+            end
+
+            def validate
+              reject_legacy_options!
+              require_option!(:routing_key)
+              REQUIRED_OPTIONS.each { |key| require_option!(key) }
+              require_protocol_version!
+              @valid = true
+            end
+
+            def message
+              super.merge(
+                protocol_version: @options[:protocol_version],
+                request_id: @options[:request_id],
+                correlation_id: correlation_id,
+                idempotency_key: @options[:idempotency_key],
+                operation: @options[:operation],
+                provider: @options[:provider],
+                provider_instance: @options[:provider_instance],
+                model: @options[:model],
+                params: @options[:params] || {},
+                reply_to: reply_to,
+                message_context: @options[:message_context],
+                caller: @options[:caller],
+                trace_context: @options[:trace_context],
+                signed_token: @options[:signed_token],
+                timeout_seconds: @options[:timeout_seconds],
+                expires_at: @options[:expires_at]
+              ).compact
+            end
+
+            private
+
+            def fleet_exchange
+              exchange_class = exchange
+              if exchange_class.respond_to?(:cached_instance)
+                exchange_class.cached_instance || exchange_class.new
+              elsif exchange_class.respond_to?(:new)
+                exchange_class.new
+              else
+                exchange_class
+              end
+            end
+
+            def request_publish_options(options)
+              request_publish_envelope(options).tap do |envelope|
+                envelope[:mandatory] = true if options[:mandatory] == true
+              end.compact
+            end
+
+            def request_publish_envelope(options)
+              {
+                routing_key: routing_key || '',
+                content_type: options[:content_type] || content_type,
+                content_encoding: options[:content_encoding] || content_encoding,
+                type: options[:type] || type,
+                priority: options[:priority] || priority,
+                expiration: options[:expiration] || expiration,
+                headers: request_headers(options),
+                persistent: request_persistent(options),
+                message_id: message_id,
+                correlation_id: correlation_id,
+                reply_to: reply_to,
+                app_id: options[:app_id] || app_id,
+                timestamp: timestamp
+              }
+            end
+
+            def request_headers(options)
+              options[:headers] ? headers.merge(options[:headers]) : headers
+            end
+
+            def request_persistent(options)
+              options.key?(:persistent) ? options[:persistent] : persistent
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/transport/messages/fleet_response.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative '../../fleet/default_exchange_reply'
+require_relative '../../fleet/envelope_validation'
+require_relative '../../fleet/protocol'
+require_relative '../exchanges/fleet'
+
+module Legion
+  module Extensions
+    module Llm
+      module Transport
+        module Messages
+          # Correlated protocol-v2 response envelope for fleet reply queues.
+          class FleetResponse < ::Legion::Transport::Message
+            include Fleet::DefaultExchangeReply
+            include Fleet::EnvelopeValidation
+
+            def type = Fleet::Protocol::RESPONSE_TYPE
+            def app_id = @options[:app_id] || 'lex-llm'
+            def reply_to = @options[:reply_to]
+            def correlation_id = @options[:correlation_id]
+            def message_id = @options[:message_id] ||= "llm_fleet_res_#{SecureRandom.uuid}"
+
+            def routing_key
+              @options[:reply_to] || raise(ArgumentError, 'reply_to is required')
+            end
+
+            def validate
+              reject_legacy_options!
+              require_option!(:request_id)
+              require_option!(:correlation_id)
+              require_option!(:reply_to)
+              require_protocol_version!
+              @valid = true
+            end
+
+            def message
+              super.except(:thinking).merge(
+                protocol_version: @options[:protocol_version] || Fleet::Protocol::VERSION,
+                request_id: @options[:request_id],
+                correlation_id: correlation_id,
+                idempotency_key: @options[:idempotency_key],
+                operation: @options[:operation],
+                provider: @options[:provider],
+                provider_instance: @options[:provider_instance] || @options[:instance],
+                model: @options[:model],
+                reply_to: reply_to,
+                message_context: @options[:message_context],
+                trace_context: @options[:trace_context],
+                content: @options[:content],
+                tool_calls: @options[:tool_calls],
+                usage: @options[:usage] || {},
+                finish_reason: @options[:finish_reason],
+                metadata: @options[:metadata] || {}
+              ).compact
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Llm
-      VERSION = '0.3.1'
+      VERSION = '0.4.3'
     end
   end
 end

data/lib/legion/extensions/llm.rb

@@ -110,20 +110,33 @@ module Legion
       def self.default_settings
         {
           fleet: {
-            enabled: false,
-            scheduler: :basic_get,
-            consumer_priority: 0,
-            queue_expires_ms: 60_000,
-            message_ttl_ms: 120_000,
-            queue_max_length: 100,
-            delivery_limit: 3,
-            consumer_ack_timeout_ms: 300_000,
-            endpoint: {
+            consumer: {
               enabled: false,
+              scheduler: :basic_get,
+              consumer_priority: 0,
+              queue_expires_ms: 60_000,
+              message_ttl_ms: 120_000,
+              queue_max_length: 100,
+              delivery_limit: 3,
+              consumer_ack_timeout_ms: 90_000,
               empty_lane_backoff_ms: 250,
               idle_backoff_ms: 1_000,
-              max_consecutive_pulls_per_lane: 0,
-              accept_when: []
+              max_consecutive_pulls_per_lane: 0
+            },
+            auth: {
+              require_signed_token: true,
+              issuer: 'legion-llm',
+              audience: 'lex-llm-fleet-worker',
+              algorithm: 'HS256',
+              accepted_issuers: ['legion-llm'],
+              max_clock_skew_seconds: 30,
+              replay_ttl_seconds: 600
+            },
+            responder: {
+              require_auth: nil,
+              require_policy: false,
+              require_idempotency: true,
+              idempotency_ttl_seconds: 600
             }
           }
         }
@@ -136,6 +149,20 @@ module Legion
       require_relative 'llm/auto_registration'
       require_relative 'llm/credential_sources'
       loader.eager_load
+
+      module Transport
+        # Local autoloads for fleet exchange classes that depend on legion-transport.
+        module Exchanges
+          autoload :Fleet, File.expand_path('llm/transport/exchanges/fleet', __dir__)
+        end
+
+        # Local autoloads for fleet message classes that depend on legion-transport.
+        module Messages
+          autoload :FleetRequest, File.expand_path('llm/transport/messages/fleet_request', __dir__)
+          autoload :FleetResponse, File.expand_path('llm/transport/messages/fleet_response', __dir__)
+          autoload :FleetError, File.expand_path('llm/transport/messages/fleet_error', __dir__)
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.3
 platform: ruby
 authors:
 - LegionIO
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: event_stream_parser
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: legion-crypt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.1
 - !ruby/object:Gem::Dependency
   name: legion-json
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.14
+- !ruby/object:Gem::Dependency
+  name: legion-transport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.14
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.14
 - !ruby/object:Gem::Dependency
   name: marcel
   requirement: !ruby/object:Gem::Requirement
@@ -211,6 +253,16 @@ files:
 - lib/legion/extensions/llm/credential_sources.rb
 - lib/legion/extensions/llm/embedding.rb
 - lib/legion/extensions/llm/error.rb
+- lib/legion/extensions/llm/errors/unsupported_capability.rb
+- lib/legion/extensions/llm/fleet/default_exchange_reply.rb
+- lib/legion/extensions/llm/fleet/envelope_validation.rb
+- lib/legion/extensions/llm/fleet/protocol.rb
+- lib/legion/extensions/llm/fleet/provider_responder.rb
+- lib/legion/extensions/llm/fleet/publish_safety.rb
+- lib/legion/extensions/llm/fleet/settings.rb
+- lib/legion/extensions/llm/fleet/token_error.rb
+- lib/legion/extensions/llm/fleet/token_validator.rb
+- lib/legion/extensions/llm/fleet/worker_execution.rb
 - lib/legion/extensions/llm/image.rb
 - lib/legion/extensions/llm/message.rb
 - lib/legion/extensions/llm/mime_type.rb
@@ -226,9 +278,14 @@ files:
 - lib/legion/extensions/llm/moderation.rb
 - lib/legion/extensions/llm/provider.rb
 - lib/legion/extensions/llm/provider/open_ai_compatible.rb
+- lib/legion/extensions/llm/provider_contract.rb
 - lib/legion/extensions/llm/provider_settings.rb
 - lib/legion/extensions/llm/registry_event_builder.rb
 - lib/legion/extensions/llm/registry_publisher.rb
+- lib/legion/extensions/llm/responses/chat_response.rb
+- lib/legion/extensions/llm/responses/embedding_response.rb
+- lib/legion/extensions/llm/responses/stream_chunk.rb
+- lib/legion/extensions/llm/responses/thinking_extractor.rb
 - lib/legion/extensions/llm/routing.rb
 - lib/legion/extensions/llm/routing/lane_key.rb
 - lib/legion/extensions/llm/routing/model_offering.rb
@@ -241,8 +298,12 @@ files:
 - lib/legion/extensions/llm/tool.rb
 - lib/legion/extensions/llm/tool_call.rb
 - lib/legion/extensions/llm/transcription.rb
+- lib/legion/extensions/llm/transport/exchanges/fleet.rb
 - lib/legion/extensions/llm/transport/exchanges/llm_registry.rb
 - lib/legion/extensions/llm/transport/fleet_lane.rb
+- lib/legion/extensions/llm/transport/messages/fleet_error.rb
+- lib/legion/extensions/llm/transport/messages/fleet_request.rb
+- lib/legion/extensions/llm/transport/messages/fleet_response.rb
 - lib/legion/extensions/llm/transport/messages/registry_event.rb
 - lib/legion/extensions/llm/utils.rb
 - lib/legion/extensions/llm/version.rb