ruby_llm 1.15.0 → 2.0.0.rc1

data/lib/ruby_llm/providers/gpustack/speech.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # GPUStack's vLLM speech endpoint requires an explicit stream flag.
+      module Speech
+        def stream_speech(payload, model:, voice:, format:, &)
+          format ||= 'pcm'
+          super(payload.merge(stream: true, response_format: format), model:, voice:, format:, &)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack/transcription.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # GPUStack's vLLM transcription stream uses Chat Completions deltas.
+      module Transcription
+        module_function
+
+        def stream_transcription(payload, model:, &)
+          super({ stream_include_usage: 'true' }.merge(payload), model:, &)
+        end
+
+        def build_transcription_chunk(data)
+          return super unless data.key?('choices')
+
+          choice = data['choices'].first || {}
+          finished = choice['finish_reason'] || data['choices'].empty?
+
+          RubyLLM::TranscriptionChunk.new(
+            type: finished ? RubyLLM::TranscriptionChunk::DONE : RubyLLM::TranscriptionChunk::DELTA,
+            delta: choice.dig('delta', 'content'),
+            raw: data
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack.rb

@@ -2,16 +2,44 @@
 
 module RubyLLM
   module Providers
-    # GPUStack API integration based on Ollama.
-    class GPUStack < OpenAI
-      include GPUStack::Chat
-      include GPUStack::Models
-      include GPUStack::Media
+    # GPUStack API integration.
+    class GPUStack < Provider
+      # GPUStack's dialect of the Chat Completions API.
+      class ChatCompletions < Protocols::ChatCompletions
+        include GPUStack::Chat
+        include GPUStack::Embeddings
+        include GPUStack::Media
+        include GPUStack::Models
+        include GPUStack::Speech
+        include GPUStack::Transcription
+        include Protocols::ChatCompletions::Rerank
+        include Protocols::GPUStack::Tokenization
+        include Protocols::GPUStack::Videos
+      end
+
+      protocol :chat_completions, ChatCompletions
+      protocol :responses, Protocols::GPUStack::Responses
+
+      def resolve_protocol(name, model, **request)
+        return fetch_protocol(:chat_completions) if !name && request[:operation]
+
+        super
+      end
 
       def api_base
         @config.gpustack_api_base
       end
 
+      def backend_api_base # :nodoc:
+        uri = URI(api_base)
+        unless uri.path.match?(%r{/model/proxy/\d+/v1/?\z})
+          raise Error, 'This GPUStack operation requires gpustack_api_base to end in /model/proxy/ROUTE_ID/v1'
+        end
+
+        uri.path = uri.path.sub(%r{/v1/?\z}, '')
+        uri.to_s
+      end
+
       def headers
         return {} unless @config.gpustack_api_key
 
@@ -25,16 +53,12 @@ module RubyLLM
           %i[gpustack_api_base gpustack_api_key]
         end
 
-        def local?
-          true
-        end
-
         def configuration_requirements
           %i[gpustack_api_base]
         end
 
-        def capabilities
-          GPUStack::Capabilities
+        def local?
+          true
         end
       end
     end

data/lib/ruby_llm/providers/mistral/capabilities.rb

@@ -3,164 +3,16 @@
 module RubyLLM
   module Providers
     class Mistral
-      # Determines capabilities for Mistral models
+      # Feature capability gaps not represented in upstream model catalogs.
       module Capabilities
-        module_function
+        TOOL_CAPABILITIES = %w[tool_choice parallel_tool_calls].freeze
+        STRUCTURED_OUTPUT_MODELS = %w[mistral-small-2603 mistral-small-latest].freeze
 
-        def supports_streaming?(model_id)
-          !model_id.match?(/embed|moderation|ocr|transcriptions/)
-        end
-
-        def supports_tools?(model_id)
-          !model_id.match?(/embed|moderation|ocr|voxtral|transcriptions|mistral-(tiny|small)-(2312|2402)/)
-        end
-
-        def supports_tool_choice?(_model_id)
-          true
-        end
-
-        def supports_tool_parallel_control?(_model_id)
-          true
-        end
-
-        def supports_vision?(model_id)
-          model_id.match?(/pixtral|mistral-small-(2503|2506)|mistral-medium/)
-        end
-
-        def supports_json_mode?(model_id)
-          !model_id.match?(/embed|moderation|ocr|voxtral|transcriptions/) && supports_tools?(model_id)
-        end
-
-        def supports_reasoning?(model_id)
-          model_id.match?(/magistral/) ||
-            model_id.match?(/\Amistral-(?:small-latest|medium-(?:3(?:[.-]5)?|latest))\z/)
-        end
-
-        def format_display_name(model_id)
-          case model_id
-          when /mistral-large/ then 'Mistral Large'
-          when /mistral-medium/ then 'Mistral Medium'
-          when /mistral-small/ then 'Mistral Small'
-          when /ministral-3b/ then 'Ministral 3B'
-          when /ministral-8b/ then 'Ministral 8B'
-          when /codestral/ then 'Codestral'
-          when /pixtral-large/ then 'Pixtral Large'
-          when /pixtral-12b/ then 'Pixtral 12B'
-          when /mistral-embed/ then 'Mistral Embed'
-          when /mistral-moderation/ then 'Mistral Moderation'
-          else model_id.split('-').map(&:capitalize).join(' ')
-          end
-        end
-
-        def model_family(model_id)
-          case model_id
-          when /mistral-large/ then 'mistral-large'
-          when /mistral-medium/ then 'mistral-medium'
-          when /mistral-small/ then 'mistral-small'
-          when /ministral/ then 'ministral'
-          when /codestral/ then 'codestral'
-          when /pixtral/ then 'pixtral'
-          when /mistral-embed/ then 'mistral-embed'
-          when /mistral-moderation/ then 'mistral-moderation'
-          else 'mistral'
-          end
-        end
-
-        def context_window_for(_model_id)
-          32_768
-        end
-
-        def max_tokens_for(_model_id)
-          8192
-        end
-
-        def modalities_for(model_id)
-          case model_id
-          when /pixtral/
-            {
-              input: %w[text image],
-              output: ['text']
-            }
-          when /embed/
-            {
-              input: ['text'],
-              output: ['embeddings']
-            }
-          else
-            {
-              input: ['text'],
-              output: ['text']
-            }
-          end
-        end
-
-        def capabilities_for(model_id) # rubocop:disable Metrics/PerceivedComplexity
-          case model_id
-          when /moderation/ then ['moderation']
-          when /voxtral.*transcribe/ then ['transcription']
-          when /ocr/ then ['vision']
-          else
-            capabilities = []
-            capabilities << 'streaming' if supports_streaming?(model_id)
-            capabilities << 'function_calling' if supports_tools?(model_id)
-            capabilities << 'structured_output' if supports_json_mode?(model_id)
-            capabilities << 'vision' if supports_vision?(model_id)
-
-            capabilities << 'reasoning' if supports_reasoning?(model_id)
-            capabilities << 'batch' unless model_id.match?(/voxtral|ocr|embed|moderation/)
-            capabilities << 'fine_tuning' if model_id.match?(/mistral-(small|medium|large)|devstral/)
-            capabilities << 'distillation' if model_id.match?(/ministral/)
-            capabilities << 'predicted_outputs' if model_id.match?(/codestral/)
-
-            capabilities.uniq
-          end
-        end
-
-        def pricing_for(_model_id)
-          {
-            input: 0.0,
-            output: 0.0
-          }
-        end
-
-        def release_date_for(model_id)
-          case model_id
-          when 'open-mistral-7b', 'mistral-tiny' then '2023-09-27'
-          when 'mistral-medium-2312', 'mistral-small-2312', 'mistral-small',
-               'open-mixtral-8x7b', 'mistral-tiny-2312' then '2023-12-11'
-
-          when 'mistral-embed' then '2024-01-11'
-          when 'mistral-large-2402', 'mistral-small-2402' then '2024-02-26'
-          when 'open-mixtral-8x22b', 'open-mixtral-8x22b-2404' then '2024-04-17'
-          when 'codestral-2405' then '2024-05-22'
-          when 'codestral-mamba-2407', 'codestral-mamba-latest', 'open-codestral-mamba' then '2024-07-16'
-          when 'open-mistral-nemo', 'open-mistral-nemo-2407', 'mistral-tiny-2407',
-               'mistral-tiny-latest' then '2024-07-18'
-          when 'mistral-large-2407' then '2024-07-24'
-          when 'pixtral-12b-2409', 'pixtral-12b-latest', 'pixtral-12b' then '2024-09-17'
-          when 'mistral-small-2409' then '2024-09-18'
-          when 'ministral-3b-2410', 'ministral-3b-latest', 'ministral-8b-2410',
-               'ministral-8b-latest' then '2024-10-16'
-          when 'pixtral-large-2411', 'pixtral-large-latest', 'mistral-large-pixtral-2411' then '2024-11-12'
-          when 'mistral-large-2411', 'mistral-large-latest', 'mistral-large' then '2024-11-20'
-          when 'codestral-2411-rc5', 'mistral-moderation-2411', 'mistral-moderation-latest' then '2024-11-26'
-          when 'codestral-2412' then '2024-12-17'
+        def self.augment(capabilities, model_id:, **)
+          capabilities |= TOOL_CAPABILITIES if capabilities.include?('function_calling')
+          capabilities |= ['structured_output'] if STRUCTURED_OUTPUT_MODELS.include?(model_id)
 
-          when 'mistral-small-2501' then '2025-01-13'
-          when 'codestral-2501' then '2025-01-14'
-          when 'mistral-saba-2502', 'mistral-saba-latest' then '2025-02-18'
-          when 'mistral-small-2503' then '2025-03-03'
-          when 'mistral-ocr-2503' then '2025-03-21'
-          when 'mistral-medium', 'mistral-medium-latest', 'mistral-medium-2505' then '2025-05-06'
-          when 'codestral-embed', 'codestral-embed-2505' then '2025-05-21'
-          when 'mistral-ocr-2505', 'mistral-ocr-latest' then '2025-05-23'
-          when 'devstral-small-2505' then '2025-05-28'
-          when 'mistral-small-2506', 'mistral-small-latest', 'magistral-medium-2506',
-               'magistral-medium-latest' then '2025-06-10'
-          when 'devstral-small-2507', 'devstral-small-latest', 'devstral-medium-2507',
-               'devstral-medium-latest' then '2025-07-09'
-          when 'codestral-2508', 'codestral-latest' then '2025-08-30'
-          end
+          capabilities
         end
       end
     end

data/lib/ruby_llm/providers/mistral/chat.rb

@@ -5,38 +5,51 @@ module RubyLLM
     class Mistral
       # Chat methods for Mistral API
       module Chat
+        PROMPT_CACHE_OPTIONS = %i[key].freeze
+
         module_function
 
         def format_role(role)
           role.to_s
         end
 
-        def format_messages(messages)
-          messages.map do |msg|
-            {
-              role: format_role(msg.role),
-              content: format_content_with_thinking(msg),
-              tool_calls: OpenAI::Tools.format_tool_calls(msg.tool_calls),
-              tool_call_id: msg.tool_call_id
-            }.compact
-          end
-        end
-
-        # rubocop:disable Metrics/ParameterLists
-        def render_payload(messages, tools:, temperature:, model:, stream: false,
-                           schema: nil, thinking: nil, tool_prefs: nil)
+        def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil,
+                           schema: nil, thinking: nil, citations: false, caching: nil, tool_prefs: nil)
           payload = super
           payload.delete(:stream_options)
-          configure_thinking_payload(payload, model, thinking)
           normalize_required_tool_choice(payload)
+          payload.merge!(prompt_cache_params(caching)) if caching
           payload
         end
-        # rubocop:enable Metrics/ParameterLists
+
+        def openai_prompt_caching?
+          false
+        end
+
+        def prompt_cache_params(caching)
+          options = prompt_cache_options(caching)
+
+          {}.tap do |params|
+            params[:prompt_cache_key] = options[:key] if options[:key]
+          end
+        end
+
+        def prompt_cache_options(caching)
+          options = caching.to_h.transform_keys(&:to_sym)
+          unsupported = options.keys - PROMPT_CACHE_OPTIONS
+          return options if unsupported.empty?
+
+          raise ArgumentError, "Mistral prompt caching accepts :key, got #{format_cache_option_keys(unsupported)}"
+        end
+
+        def format_cache_option_keys(keys)
+          keys.map { |key| ":#{key}" }.join(', ')
+        end
 
         def build_tool_choice(tool_choice)
           return 'any' if tool_choice == :required
 
-          OpenAI::Tools.build_tool_choice(tool_choice)
+          Protocols::ChatCompletions::Tools.build_tool_choice(tool_choice)
         end
 
         def normalize_required_tool_choice(payload)
@@ -51,8 +64,10 @@ module RubyLLM
           }
         end
 
-        def format_content_with_thinking(msg)
-          formatted_content = OpenAI::Media.format_content(msg.content)
+        # Mistral carries reasoning in content blocks, not in the top-level
+        # reasoning fields the rest of the Chat Completions family uses.
+        def format_message_content(msg, **)
+          formatted_content = super
           return formatted_content unless msg.role == :assistant && msg.thinking
 
           content_blocks = build_thinking_blocks(msg.thinking)
@@ -61,47 +76,8 @@ module RubyLLM
           content_blocks
         end
 
-        def warn_on_unsupported_thinking(model, thinking)
-          return unless thinking&.enabled?
-          return if native_reasoning_model?(model.id) || adjustable_reasoning_model?(model.id)
-
-          RubyLLM.logger.warn(
-            'Mistral thinking is only supported on Magistral and adjustable-reasoning models. ' \
-            "Ignoring thinking settings for #{model.id}."
-          )
-        end
-
-        def configure_thinking_payload(payload, model, thinking)
-          return unless thinking&.enabled?
-
-          if native_reasoning_model?(model.id)
-            configure_native_reasoning_payload(payload, thinking)
-          elsif adjustable_reasoning_model?(model.id)
-            payload[:reasoning_effort] = reasoning_effort_for(thinking)
-          else
-            payload.delete(:reasoning_effort)
-            warn_on_unsupported_thinking(model, thinking)
-          end
-        end
-
-        def configure_native_reasoning_payload(payload, thinking)
-          payload.delete(:reasoning_effort)
-          payload[:prompt_mode] = thinking.effort == 'none' ? nil : 'reasoning'
-        end
-
-        def reasoning_effort_for(thinking)
-          effort = thinking.respond_to?(:effort) ? thinking.effort : nil
-          return effort if %w[high none].include?(effort)
-
-          'high'
-        end
-
-        def native_reasoning_model?(model_id)
-          model_id.to_s.include?('magistral')
-        end
-
-        def adjustable_reasoning_model?(model_id)
-          model_id.to_s.match?(/\Amistral-(?:small-latest|medium-(?:3(?:[.-]5)?|latest))\z/)
+        def format_thinking(_msg)
+          {}
         end
 
         def build_thinking_blocks(thinking)
@@ -123,7 +99,7 @@ module RubyLLM
         def append_formatted_content(content_blocks, formatted_content)
           if formatted_content.is_a?(Array)
             content_blocks.concat(formatted_content)
-          elsif formatted_content
+          elsif formatted_content && !formatted_content.empty?
             content_blocks << { type: 'text', text: formatted_content }
           end
         end

data/lib/ruby_llm/providers/mistral/chat_completions/batches.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Mistral
+      class ChatCompletions
+        # Mistral batch jobs for chat completions and embeddings.
+        module Batches
+          include RubyLLM::Batch::Helpers
+
+          TERMINAL_STATUSES = %w[SUCCESS FAILED TIMEOUT_EXCEEDED CANCELLED].freeze
+          Response = Struct.new(:body)
+          private_constant :TERMINAL_STATUSES, :Response
+
+          def create_batch(requests)
+            model = single_batch_model!(requests, 'mistral')
+            response = @connection.post('batch/jobs', {
+                                          endpoint: mistral_batch_endpoint(requests),
+                                          model: model,
+                                          requests: requests.map { |request| mistral_batch_request(request) }
+                                        }, idempotent: false)
+
+            parse_batch_response(response.body)
+          end
+
+          def find_batch(id)
+            attempts = 0
+            begin
+              parse_batch_response @connection.get(batch_url(id)).body
+            rescue Error => e
+              attempts += 1
+              raise unless e.response&.status == 404 && attempts < 3
+
+              sleep(0.5 * attempts)
+              retry
+            end
+          end
+
+          def cancel_batch(id)
+            parse_batch_response @connection.post("#{batch_url(id)}/cancel", {}).body
+          end
+
+          def batch_results(id)
+            response = @connection.get(batch_url(id)) { |request| request.params[:inline] = true }
+            Array(response.body['outputs']).filter_map { |line| parse_batch_result(line) }
+          end
+
+          private
+
+          def batch_url(id)
+            "batch/jobs/#{id}"
+          end
+
+          def mistral_batch_request(request)
+            body = batch_payload(request, except: :model)
+            custom_id = request[:custom_id]
+            custom_id = "#{custom_id}:array" if (body[:input] || body['input']).is_a?(Array)
+
+            {
+              custom_id: custom_id,
+              body: body
+            }
+          end
+
+          def mistral_batch_endpoint(requests)
+            endpoints = requests.map do |request|
+              payload = request.fetch(:payload)
+              payload.key?(:input) || payload.key?('input') ? '/v1/embeddings' : '/v1/chat/completions'
+            end.uniq
+            return endpoints.first if endpoints.one?
+
+            raise Error, 'Mistral batches cannot mix chat and embedding requests'
+          end
+
+          def parse_batch_response(data)
+            {
+              id: data['id'],
+              raw_status: data['status'],
+              completed: TERMINAL_STATUSES.include?(data['status']),
+              request_count: data['total_requests'],
+              request_counts: {
+                'total' => data['total_requests'],
+                'completed' => data['completed_requests'],
+                'succeeded' => data['succeeded_requests'],
+                'failed' => data['failed_requests']
+              }.compact
+            }
+          end
+
+          def parse_batch_status(raw_status, completed:)
+            return :pending unless completed
+            return :succeeded if raw_status == 'SUCCESS'
+            return :cancelled if raw_status == 'CANCELLED'
+
+            :failed
+          end
+
+          def parse_batch_result(line)
+            custom_id, shape = line['custom_id'].split(':', 2)
+            index = batch_result_index(custom_id)
+            response = line['response']
+
+            if response && response['body']
+              body = response['body']
+              [index, parse_mistral_batch_body(body, shape:)]
+            else
+              [index, nil, batch_failure(line['custom_id'], batch_error_message(line))]
+            end
+          end
+
+          def parse_mistral_batch_body(body, shape:)
+            return parse_completion_body(body, raw: body) unless body['data'].is_a?(Array)
+
+            parse_embedding_response(Response.new(body), model: body['model'], text: shape == 'array' ? [] : nil)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/mistral/chat_completions.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Mistral
+      # Mistral's dialect of the Chat Completions API.
+      class ChatCompletions < Protocols::ChatCompletions
+        include Mistral::Chat
+        include Mistral::Embeddings
+        include Mistral::Media
+        include Mistral::Models
+        include Mistral::OCR
+        include Mistral::Speech
+        include Mistral::Transcription
+        include Protocols::Mistral::MultiCompletion
+
+        public :server_tool_aliases
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/mistral/conversations.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Mistral
+      # Mistral Conversations with its document and image attachment formats.
+      class Conversations < Protocols::Mistral::Conversations
+        include Mistral::Media
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/mistral/embeddings.rb

@@ -11,16 +11,18 @@ module RubyLLM
           'embeddings'
         end
 
-        def render_embedding_payload(text, model:, dimensions:) # rubocop:disable Lint/UnusedMethodArgument
+        # rubocop:disable-next Lint/UnusedMethodArgument
+        def render_embedding_payload(text, model:, dimensions:, task_type: nil, title: nil, provider_options: {})
           {
             model: model,
-            input: text
-          }
+            input: text,
+            output_dimension: dimensions
+          }.compact.merge(provider_options)
         end
 
         def parse_embedding_response(response, model:, text:)
           data = response.body
-          input_tokens = data.dig('usage', 'prompt_tokens') || 0
+          input_tokens = data.dig('usage', 'prompt_tokens')
           vectors = data['data'].map { |d| d['embedding'] }
 
           vectors = vectors.first if vectors.length == 1 && !text.is_a?(Array)

data/lib/ruby_llm/providers/mistral/media.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Mistral
+      # Handles media content for Mistral Chat Completions.
+      module Media
+        module_function
+
+        def format_content(content, attachments = [])
+          Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
+            case attachment.type
+            when :image
+              format_image(attachment)
+            when :audio
+              Protocols::ChatCompletions::Media.format_audio(attachment)
+            when :pdf, :document
+              format_document(attachment)
+            when :text
+              Protocols::ChatCompletions::Media.format_text_file(attachment)
+            else
+              raise UnsupportedAttachmentError, attachment.mime_type
+            end
+          end
+        end
+
+        def format_image(image)
+          {
+            type: 'image_url',
+            image_url: image.url? ? image.source.to_s : image.for_llm
+          }
+        end
+
+        def format_document(document)
+          {
+            type: 'document_url',
+            document_url: document.url? ? document.source.to_s : document.for_llm
+          }
+        end
+      end
+    end
+  end
+end