lex-llm 0.3.1 → 0.4.3

data/lib/legion/extensions/llm/provider/open_ai_compatible.rb

@@ -39,17 +39,17 @@ module Legion
             messages.map do |message|
               {
                 role: message.role.to_s,
-                content: openai_content(message.content),
+                content: openai_content(message.content, role: message.role),
                 tool_call_id: message.tool_call_id,
                 tool_calls: format_openai_tool_calls(message.tool_calls)
               }.compact
             end
           end
 
-          def openai_content(content)
+          def openai_content(content, role:)
             return content.format if content.is_a?(Legion::Extensions::Llm::Content::Raw)
-            return content unless content.respond_to?(:attachments)
-            return content.text.to_s if content.attachments.empty?
+            return sanitize_openai_text(content, role:) unless content.respond_to?(:attachments)
+            return sanitize_openai_text(content.text.to_s, role:) if content.attachments.empty?
 
             openai_content_parts(content)
           end
@@ -63,6 +63,12 @@ module Legion
             parts
           end
 
+          def sanitize_openai_text(text, role:)
+            return text unless role.to_sym == :assistant && text.is_a?(String)
+
+            Responses::ThinkingExtractor.extract(text).content
+          end
+
           def format_openai_tool_calls(tool_calls)
             return nil unless tool_calls&.any?
 
@@ -135,18 +141,29 @@ module Legion
           end
 
           def extract_thinking_from_completion(message)
-            reasoning = message['reasoning_content'] || message['reasoning']
-            content = message['content']
+            extraction = Responses::ThinkingExtractor.extract(
+              message['content'],
+              metadata: thinking_metadata(message)
+            )
 
-            if reasoning
-              [content, Thinking.build(text: reasoning)]
-            elsif content.is_a?(String) && content.include?('<think>')
-              think_text = content[%r{<think>(.*?)</think>}m, 1]
-              clean = content.gsub(%r{<think>.*?</think>}m, '').strip
-              [clean, Thinking.build(text: think_text)]
-            else
-              [content, nil]
-            end
+            [
+              extraction.content,
+              Thinking.build(
+                text: extraction.thinking,
+                signature: extraction.signature
+              )
+            ]
+          end
+
+          def thinking_metadata(message)
+            {
+              reasoning_content: message['reasoning_content'],
+              reasoning: message['reasoning'],
+              thinking: message['thinking'],
+              thinking_text: message['thinking_text'],
+              thinking_signature: message['thinking_signature'],
+              reasoning_signature: message['reasoning_signature']
+            }.compact
           end
 
           def build_chunk(data)
@@ -173,39 +190,23 @@ module Legion
 
             if reasoning
               [content, Thinking.build(text: reasoning)]
-            elsif content.is_a?(String) && content.include?('<think>')
-              clean, think_text = split_think_tags(content)
-              [clean, Thinking.build(text: think_text)]
             else
               [content, nil]
             end
           end
 
-          def split_think_tags(text) # rubocop:disable Metrics/PerceivedComplexity
-            if text.match?(%r{<think>.*</think>}m)
-              thinking = text[%r{<think>(.*?)</think>}m, 1]
-              clean = text.gsub(%r{<think>.*?</think>}m, '').strip
-              [clean.empty? ? nil : clean, thinking]
-            elsif text.start_with?('<think>')
-              [nil, text.delete_prefix('<think>')]
-            elsif text.include?('</think>')
-              parts = text.split('</think>', 2)
-              [parts[1]&.strip.then { |s| s&.empty? ? nil : s }, parts[0]]
-            else
-              [text, nil]
-            end
-          end
-
           def parse_tool_calls(tool_calls)
             return nil unless tool_calls&.any?
 
             tool_calls.to_h do |call|
               function = call.fetch('function', {})
-              name = function.fetch('name')
+              name = function['name']
+              id = call['id'] || name || call['index']
+              key = name || id
               [
-                name.to_sym,
+                key.to_s.to_sym,
                 Legion::Extensions::Llm::ToolCall.new(
-                  id: call['id'] || name,
+                  id: id&.to_s,
                   name: name,
                   arguments: parse_tool_arguments(function['arguments'])
                 )

data/lib/legion/extensions/llm/provider.rb

@@ -62,6 +62,16 @@ module Legion
         end
 
         # rubocop:disable Metrics/ParameterLists
+        def chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil, thinking: nil,
+                 tool_prefs: nil)
+          complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:)
+        end
+
+        def stream_chat(messages:, model:, tools: [], temperature: nil, params: {}, headers: {}, schema: nil,
+                        thinking: nil, tool_prefs: nil, &)
+          complete(messages, tools:, temperature:, model:, params:, headers:, schema:, thinking:, tool_prefs:, &)
+        end
+
         def complete(messages, tools:, temperature:, model:, params: {}, headers: {}, schema: nil, thinking: nil,
                      tool_prefs: nil, &)
           normalized_temperature = maybe_normalize_temperature(temperature, model)
@@ -88,14 +98,55 @@ module Legion
         end
         # rubocop:enable Metrics/ParameterLists
 
-        def list_models
+        def list_models(live: false, **filters)
+          _ = [live, filters]
           response = @connection.get models_url
           parse_list_models_response response, slug, capabilities
         end
 
-        def embed(text, model:, dimensions:)
-          payload = render_embedding_payload(text, model:, dimensions:)
-          response = @connection.post(embedding_url(model:), payload)
+        def discover_offerings(live: false, **filters)
+          return filter_cached_offerings(Array(@cached_offerings), filters) unless live
+
+          provider_health = health(live:)
+          @cached_offerings = Array(list_models(live:, **filters)).filter_map do |model|
+            next unless model_matches_filters?(model, filters)
+
+            offering_from_model(model, health: provider_health)
+          end
+          @cached_offerings
+        end
+
+        def health(live: false)
+          readiness_data = readiness(live:)
+          raw_health = readiness_data[:health] || readiness_data['health'] || {}
+          status = health_status(readiness_data, raw_health)
+          {
+            provider: slug.to_sym,
+            instance_id: provider_instance_id,
+            status:,
+            ready: readiness_data[:ready] == true || readiness_data['ready'] == true,
+            circuit_state: status == 'healthy' ? 'closed' : 'open',
+            latency_ms: raw_health[:latency_ms] || raw_health['latency_ms'],
+            raw: raw_health
+          }.compact
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: 'llm.provider.health')
+          {
+            provider: slug.to_sym,
+            instance_id: provider_instance_id,
+            status: 'unhealthy',
+            ready: false,
+            circuit_state: 'open',
+            error: e.class.name,
+            message: e.message
+          }
+        end
+
+        def embed(text:, model:, dimensions: nil, params: {}, headers: {})
+          payload = Utils.deep_merge(render_embedding_payload(text, model:, dimensions:), params)
+          response = @connection.post(embedding_url(model:), payload) do |req|
+            req.headers = headers.merge(req.headers) unless headers.empty?
+          end
           parse_embedding_response(response, model:, text:)
         end
 
@@ -112,6 +163,18 @@ module Legion
           parse_image_response(response, model:)
         end
 
+        def image(prompt:, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Metrics/ParameterLists
+          paint(prompt, model:, size:, with:, mask:, params:)
+        end
+
+        def count_tokens(messages:, model:, params: {})
+          _ = [model, params]
+          Array(messages).sum do |message|
+            content = message.respond_to?(:content) ? message.content : message[:content] || message['content']
+            estimate_text_tokens(content)
+          end
+        end
+
         def transcribe(audio_file, model:, language:, **)
           file_part = build_audio_file_part(audio_file)
           payload = render_transcription_payload(file_part, model:, language:, **)
@@ -319,6 +382,12 @@ module Legion
           end
         end
 
+        def provider_instance_id
+          return config.instance_id.to_sym if config.respond_to?(:instance_id) && config.instance_id
+
+          :default
+        end
+
         class << self
           def name
             to_s.split('::').last
@@ -369,6 +438,131 @@ module Legion
           raise UnsupportedAttachmentError, "#{name} does not support image references in paint"
         end
 
+        def offering_from_model(model, health: {})
+          Routing::ModelOffering.new(
+            provider_family: slug.to_sym,
+            provider_instance: model.instance || provider_instance_id,
+            transport: offering_transport,
+            tier: offering_tier,
+            model: model.id,
+            canonical_model_alias: model.name,
+            model_family: model.family,
+            usage_type: offering_usage_type(model),
+            capabilities: model.capabilities,
+            limits: offering_limits(model),
+            health:,
+            metadata: offering_metadata(model)
+          )
+        end
+
+        def offering_transport
+          local? ? :local : :http
+        end
+
+        def offering_tier
+          local? ? :local : :direct
+        end
+
+        def offering_usage_type(model)
+          model.embedding? ? :embedding : :inference
+        end
+
+        def offering_limits(model)
+          {
+            context_window: model.context_length,
+            max_output_tokens: model.max_output_tokens
+          }.compact
+        end
+
+        def offering_metadata(model)
+          {
+            raw_model: model.id,
+            parameter_count: model.parameter_count,
+            parameter_size: model.parameter_size,
+            quantization: model.quantization,
+            size_bytes: model.size_bytes,
+            modalities_input: model.modalities_input,
+            modalities_output: model.modalities_output
+          }.merge(model.metadata || {}).compact
+        end
+
+        def model_matches_filters?(model, filters)
+          return true if filters.empty?
+
+          filters.all? do |key, value|
+            blank_filter_value?(value) || model_matches_filter?(model, key, value)
+          end
+        end
+
+        def blank_filter_value?(value)
+          value.nil? || (value.respond_to?(:empty?) && value.empty?)
+        end
+
+        def model_matches_filter?(model, key, value)
+          case key.to_sym
+          when :capability, :capabilities
+            Array(value).all? { |capability| model.supports?(capability) }
+          when :type, :usage_type, :purpose
+            offering_usage_type(model).to_s == value.to_s || model.type.to_s == value.to_s
+          when :model, :id, :name
+            [model.id, model.name].map(&:to_s).include?(value.to_s)
+          when :instance, :instance_id, :provider_instance
+            provider_instance_id.to_s == value.to_s || model.instance.to_s == value.to_s
+          else
+            true
+          end
+        end
+
+        def filter_cached_offerings(offerings, filters)
+          return offerings if filters.empty?
+
+          offerings.select do |offering|
+            filters.all? do |key, value|
+              blank_filter_value?(value) || offering_matches_filter?(offering, key, value)
+            end
+          end
+        end
+
+        def offering_matches_filter?(offering, key, value)
+          case key.to_sym
+          when :provider, :provider_family
+            offering.provider_family.to_s == value.to_s
+          when :capability, :capabilities
+            Array(value).all? { |capability| offering.supports?(capability) }
+          when :type, :usage_type, :purpose
+            offering.usage_type.to_s == value.to_s
+          when :model, :id, :name
+            [offering.model, offering.canonical_model_alias].compact.map(&:to_s).include?(value.to_s)
+          when :instance, :instance_id, :provider_instance
+            [offering.provider_instance, offering.instance_id].compact.map(&:to_s).include?(value.to_s)
+          else
+            true
+          end
+        end
+
+        def health_status(readiness_data, raw_health)
+          return 'healthy' if readiness_data[:ready] == true || readiness_data['ready'] == true
+
+          status = raw_health[:status] || raw_health['status'] || raw_health[:state] || raw_health['state']
+          return 'healthy' if %w[ok ready healthy running].include?(status.to_s.downcase)
+
+          'unhealthy'
+        end
+
+        def estimate_text_tokens(content)
+          text = case content
+                 when Content
+                   [content.text, *content.attachments.map(&:to_s)].compact.join(' ')
+                 when Array
+                   content.map do |part|
+                     part.respond_to?(:[]) ? part[:text] || part['text'] || part.to_s : part.to_s
+                   end.join(' ')
+                 else
+                   content.to_s
+                 end
+          [(text.length / 4.0).ceil, 1].max
+        end
+
         def build_audio_file_part(file_path)
           expanded_path = File.expand_path(file_path)
           mime_type = Marcel::MimeType.for(Pathname.new(expanded_path))

data/lib/legion/extensions/llm/provider_contract.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Llm
+      # Documents the canonical public provider method signatures shared by provider gems.
+      module ProviderContract
+        REQUIRED_SIGNATURES = {
+          chat: [%i[keyreq messages], %i[keyreq model]],
+          stream_chat: [%i[keyreq messages], %i[keyreq model]],
+          embed: [%i[keyreq text], %i[keyreq model]],
+          image: [%i[keyreq prompt], %i[keyreq model]],
+          list_models: [%i[key live], %i[keyrest filters]],
+          discover_offerings: [%i[key live], %i[keyrest filters]],
+          health: [%i[key live]],
+          count_tokens: [%i[keyreq messages], %i[keyreq model], %i[key params]]
+        }.freeze
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/provider_settings.rb

@@ -7,7 +7,10 @@ module Legion
       module ProviderSettings
         module_function
 
-        def build(family:, instance: {}, enabled: true, discovery: {}, instances: {})
+        def build(family:, instance: {}, enabled: true, discovery: {}, instances: {}, fleet: nil, gateways: nil, # rubocop:disable Metrics/ParameterLists
+                  **legacy_settings)
+          validate_provider_defaults!(fleet:, gateways:, legacy_settings:)
+
           deep_merge(
             Legion::Extensions::Llm.default_settings,
             {
@@ -27,6 +30,20 @@ module Legion
           )
         end
 
+        def validate_provider_defaults!(fleet:, gateways:, legacy_settings:)
+          if fleet
+            raise ArgumentError,
+                  'Provider fleet defaults must be nested under an instance, for example instances.default.fleet'
+          end
+
+          raise ArgumentError, 'Provider gateways settings are no longer supported; use instances instead' if gateways
+
+          return if legacy_settings.empty?
+
+          invalid_keys = legacy_settings.keys.map(&:to_sym)
+          raise ArgumentError, "Unsupported top-level provider settings: #{invalid_keys.join(', ')}"
+        end
+
         def deep_dup(value)
           case value
           when Hash

data/lib/legion/extensions/llm/responses/chat_response.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Llm
+      module Responses
+        # Normalized non-streaming chat provider response.
+        class ChatResponse
+          attr_reader :content, :thinking, :metadata, :model, :tool_calls, :tokens, :raw, :internal_metadata
+
+          def initialize(content:, thinking: nil, metadata: {}, model: nil, tool_calls: nil, tokens: nil, raw: nil) # rubocop:disable Metrics/ParameterLists
+            extraction = ThinkingExtractor.extract(content, metadata: metadata)
+
+            @content = extraction.content
+            @thinking = thinking || extraction.thinking
+            @metadata = extraction.metadata
+            @internal_metadata = metadata.to_h
+            @model = model
+            @tool_calls = tool_calls
+            @tokens = tokens
+            @raw = raw
+
+            freeze
+          end
+
+          def to_h
+            {
+              content: content,
+              metadata: metadata,
+              model: model,
+              tool_calls: tool_calls,
+              tokens: tokens
+            }.compact
+          end
+
+          def to_internal_h
+            to_h.merge(thinking: thinking, metadata: internal_metadata, raw: raw).compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/responses/embedding_response.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Llm
+      module Responses
+        # Normalized embedding provider response.
+        class EmbeddingResponse
+          attr_reader :vectors, :model, :tokens, :metadata, :raw
+
+          def initialize(vectors:, model:, tokens: nil, metadata: {}, raw: nil)
+            @vectors = vectors
+            @model = model
+            @tokens = tokens
+            @metadata = ThinkingExtractor.extract(nil, metadata: metadata).metadata
+            @internal_metadata = metadata.to_h
+            @raw = raw
+
+            freeze
+          end
+
+          def to_h
+            {
+              vectors: vectors,
+              model: model,
+              tokens: tokens,
+              metadata: metadata
+            }.compact
+          end
+
+          def to_internal_h
+            to_h.merge(metadata: @internal_metadata, raw: raw).compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/responses/stream_chunk.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Llm
+      module Responses
+        # Normalized streaming provider response chunk.
+        class StreamChunk
+          attr_reader :content, :thinking, :metadata, :model, :tool_calls, :tokens, :raw, :internal_metadata
+
+          def initialize(content: nil, thinking: nil, metadata: {}, model: nil, tool_calls: nil, tokens: nil, raw: nil) # rubocop:disable Metrics/ParameterLists
+            extraction = ThinkingExtractor.extract(content, metadata: metadata)
+
+            @content = extraction.content
+            @thinking = thinking || extraction.thinking
+            @metadata = extraction.metadata
+            @internal_metadata = metadata.to_h
+            @model = model
+            @tool_calls = tool_calls
+            @tokens = tokens
+            @raw = raw
+
+            freeze
+          end
+
+          def to_h
+            {
+              content: content,
+              metadata: metadata,
+              model: model,
+              tool_calls: tool_calls,
+              tokens: tokens
+            }.compact
+          end
+
+          def to_internal_h
+            to_h.merge(thinking: thinking, metadata: internal_metadata, raw: raw).compact
+          end
+        end
+      end
+    end
+  end
+end