ruby_llm 1.15.0 → 1.16.0

data/lib/ruby_llm/providers/gemini/chat.rb

@@ -72,8 +72,7 @@ module RubyLLM
         def format_role(role)
           case role
           when :assistant then 'model'
-          when :system then 'user'
-          when :tool then 'function'
+          when :system, :tool then 'user'
           else role.to_s
           end
         end
@@ -314,7 +313,7 @@ module RubyLLM
           end
 
           def build_tool_response(parts)
-            { role: 'function', parts: parts }
+            { role: 'user', parts: parts }
           end
 
           def remember_tool_calls

data/lib/ruby_llm/providers/gemini/media.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'base64'
+require 'stringio'
+
 module RubyLLM
   module Providers
     class Gemini # rubocop:disable Style/Documentation
@@ -16,19 +19,23 @@ module RubyLLM
           parts << format_text(content.text) if content.text
 
           content.attachments.each do |attachment|
-            case attachment.type
-            when :text
-              parts << format_text_file(attachment)
-            when :unknown
-              raise UnsupportedAttachmentError, attachment.mime_type
-            else
-              parts << format_attachment(attachment)
-            end
+            parts << format_content_attachment(attachment)
           end
 
           parts
         end
 
+        def format_content_attachment(attachment)
+          case attachment.type
+          when :text
+            format_text_file(attachment)
+          when :document, :unknown
+            raise UnsupportedAttachmentError, attachment.mime_type
+          else
+            format_attachment(attachment)
+          end
+        end
+
         def format_attachment(attachment)
           {
             inline_data: {
@@ -71,7 +78,7 @@ module RubyLLM
         text = nil if text.empty?
         return text if attachments.empty?
 
-        Content.new(text:, attachments:)
+        Content.new(text, attachments)
       end
 
       def build_inline_attachment(inline_data, index)

data/lib/ruby_llm/providers/gemini/streaming.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module RubyLLM
   module Providers
     class Gemini

data/lib/ruby_llm/providers/gemini/tools.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'securerandom'
+
 module RubyLLM
   module Providers
     class Gemini

data/lib/ruby_llm/providers/gpustack/chat.rb

@@ -11,13 +11,20 @@ module RubyLLM
           messages.map do |msg|
             {
               role: format_role(msg.role),
-              content: GPUStack::Media.format_content(msg.content),
+              content: format_message_content(msg),
               tool_calls: format_tool_calls(msg.tool_calls),
               tool_call_id: msg.tool_call_id
             }.compact.merge(OpenAI::Chat.format_thinking(msg))
           end
         end
 
+        def format_message_content(msg)
+          content = GPUStack::Media.format_content(msg.content)
+          return '' if content.nil? && OpenAI::Chat.thinking_only_assistant_message?(msg)
+
+          content
+        end
+
         def format_role(role)
           role.to_s
         end

data/lib/ruby_llm/providers/gpustack/models.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module RubyLLM
   module Providers
     class GPUStack

data/lib/ruby_llm/providers/mistral/capabilities.rb

@@ -123,7 +123,7 @@ module RubyLLM
           }
         end
 
-        def release_date_for(model_id)
+        def release_date_for(model_id) # rubocop:disable Metrics/CyclomaticComplexity
           case model_id
           when 'open-mistral-7b', 'mistral-tiny' then '2023-09-27'
           when 'mistral-medium-2312', 'mistral-small-2312', 'mistral-small',

data/lib/ruby_llm/providers/mistral/chat.rb

@@ -52,7 +52,7 @@ module RubyLLM
         end
 
         def format_content_with_thinking(msg)
-          formatted_content = OpenAI::Media.format_content(msg.content)
+          formatted_content = Mistral::Media.format_content(msg.content)
           return formatted_content unless msg.role == :assistant && msg.thinking
 
           content_blocks = build_thinking_blocks(msg.thinking)

data/lib/ruby_llm/providers/mistral/media.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Mistral
+      # Handles media content for Mistral Chat Completions.
+      module Media
+        module_function
+
+        def format_content(content) # rubocop:disable Metrics/PerceivedComplexity
+          if content.is_a?(RubyLLM::Content::Raw)
+            value = content.value
+            return value.is_a?(Hash) ? value.to_json : value
+          end
+          return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
+          return content unless content.is_a?(Content)
+
+          parts = []
+          parts << OpenAI::Media.format_text(content.text) if content.text
+
+          content.attachments.each do |attachment|
+            case attachment.type
+            when :image
+              parts << format_image(attachment)
+            when :audio
+              parts << OpenAI::Media.format_audio(attachment)
+            when :pdf, :document
+              parts << format_document(attachment)
+            when :text
+              parts << OpenAI::Media.format_text_file(attachment)
+            else
+              raise UnsupportedAttachmentError, attachment.mime_type
+            end
+          end
+
+          parts
+        end
+
+        def format_image(image)
+          {
+            type: 'image_url',
+            image_url: image.url? ? image.source.to_s : image.for_llm
+          }
+        end
+
+        def format_document(document)
+          {
+            type: 'document_url',
+            document_url: document.url? ? document.source.to_s : document.for_llm
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/mistral/models.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module RubyLLM
   module Providers
     class Mistral

data/lib/ruby_llm/providers/mistral.rb

@@ -9,7 +9,7 @@ module RubyLLM
       include Mistral::Embeddings
 
       def api_base
-        'https://api.mistral.ai/v1'
+        @config.mistral_api_base || 'https://api.mistral.ai/v1'
       end
 
       def headers
@@ -24,7 +24,7 @@ module RubyLLM
         end
 
         def configuration_options
-          %i[mistral_api_key]
+          %i[mistral_api_key mistral_api_base]
         end
 
         def configuration_requirements

data/lib/ruby_llm/providers/ollama/chat.rb

@@ -11,13 +11,20 @@ module RubyLLM
           messages.map do |msg|
             {
               role: format_role(msg.role),
-              content: Ollama::Media.format_content(msg.content),
+              content: format_message_content(msg),
               tool_calls: format_tool_calls(msg.tool_calls),
               tool_call_id: msg.tool_call_id
             }.compact.merge(OpenAI::Chat.format_thinking(msg))
           end
         end
 
+        def format_message_content(msg)
+          content = Ollama::Media.format_content(msg.content)
+          return '' if content.nil? && OpenAI::Chat.thinking_only_assistant_message?(msg)
+
+          content
+        end
+
         def format_role(role)
           role.to_s
         end

data/lib/ruby_llm/providers/openai/chat.rb

@@ -125,13 +125,28 @@ module RubyLLM
           messages.map do |msg|
             {
               role: format_role(msg.role),
-              content: Media.format_content(msg.content),
+              content: format_message_content(msg),
               tool_calls: format_tool_calls(msg.tool_calls),
               tool_call_id: msg.tool_call_id
             }.compact.merge(format_thinking(msg))
           end
         end
 
+        def format_message_content(msg)
+          content = format_content(msg.content)
+          return '' if content.nil? && thinking_only_assistant_message?(msg)
+
+          content
+        end
+
+        def thinking_only_assistant_message?(msg)
+          msg.role == :assistant && msg.thinking && !msg.tool_call?
+        end
+
+        def format_content(content)
+          Media.format_content(content)
+        end
+
         def format_role(role)
           case role
           when :system

data/lib/ruby_llm/providers/openai/images.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'faraday'
+require 'stringio'
+
 module RubyLLM
   module Providers
     class OpenAI
@@ -48,27 +51,24 @@ module RubyLLM
           payload = params.merge(
             model: model,
             prompt: prompt,
-            image: build_upload_parts(with, label: 'images'),
+            image: build_upload_parts(with),
             n: 1
           )
-          payload[:mask] = build_upload_part(mask, label: 'mask') if mask
+          payload[:mask] = build_upload_part(mask) if mask
           payload
         end
 
-        def build_upload_parts(sources, label:)
+        def build_upload_parts(sources)
           Array(sources).filter_map do |source|
             next if blank_attachment?(source)
 
-            build_upload_part(source, label:)
+            build_upload_part(source)
           end
         end
 
-        def build_upload_part(source, label:)
+        def build_upload_part(source)
           attachment = Attachment.new(source)
-          unless attachment.image?
-            raise UnsupportedAttachmentError,
-                  "OpenAI image editing only supports image attachments for #{label}"
-          end
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
 
           Faraday::UploadIO.new(StringIO.new(attachment.content), attachment.mime_type, attachment.filename)
         end

data/lib/ruby_llm/providers/openai/media.rb

@@ -7,7 +7,7 @@ module RubyLLM
       module Media
         module_function
 
-        def format_content(content) # rubocop:disable Metrics/PerceivedComplexity
+        def format_content(content, document_attachments: :pdf, image_attachments: true, audio_attachments: true)
           if content.is_a?(RubyLLM::Content::Raw)
             value = content.value
             return value.is_a?(Hash) ? value.to_json : value
@@ -19,23 +19,36 @@ module RubyLLM
           parts << format_text(content.text) if content.text
 
           content.attachments.each do |attachment|
-            case attachment.type
-            when :image
-              parts << format_image(attachment)
-            when :pdf
-              parts << format_pdf(attachment)
-            when :audio
-              parts << format_audio(attachment)
-            when :text
-              parts << format_text_file(attachment)
-            else
-              raise UnsupportedAttachmentError, attachment.type
-            end
+            parts << format_attachment(
+              attachment,
+              document_attachments:,
+              image_attachments:,
+              audio_attachments:
+            )
           end
 
           parts
         end
 
+        def format_attachment(attachment, document_attachments:, image_attachments:, audio_attachments:)
+          case attachment.type
+          when :image
+            raise UnsupportedAttachmentError, attachment.mime_type unless image_attachments
+
+            format_image(attachment)
+          when :audio
+            raise UnsupportedAttachmentError, attachment.mime_type unless audio_attachments
+
+            format_audio(attachment)
+          when :pdf, :document
+            format_document_attachment(attachment, document_attachments)
+          when :text
+            format_text_file(attachment)
+          else
+            raise UnsupportedAttachmentError, attachment.mime_type
+          end
+        end
+
         def format_image(image)
           {
             type: 'image_url',
@@ -45,16 +58,20 @@ module RubyLLM
           }
         end
 
-        def format_pdf(pdf)
+        def format_document(document)
           {
             type: 'file',
             file: {
-              filename: pdf.filename,
-              file_data: pdf.for_llm
+              filename: document.filename,
+              file_data: document.for_llm
             }
           }
         end
 
+        def format_pdf(pdf)
+          format_document(pdf)
+        end
+
         def format_text_file(text_file)
           {
             type: 'text',
@@ -78,6 +95,13 @@ module RubyLLM
             text: text
           }
         end
+
+        def format_document_attachment(attachment, strategy)
+          return format_document(attachment) if strategy == :all
+          return format_document(attachment) if strategy == :pdf && attachment.pdf?
+
+          raise UnsupportedAttachmentError, attachment.mime_type
+        end
       end
     end
   end

data/lib/ruby_llm/providers/openai/streaming.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module RubyLLM
   module Providers
     class OpenAI

data/lib/ruby_llm/providers/openai/tools.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module RubyLLM
   module Providers
     class OpenAI

data/lib/ruby_llm/providers/openai/transcription.rb

@@ -60,6 +60,7 @@ module RubyLLM
             language: data['language'],
             duration: data['duration'],
             segments: data['segments'],
+            words: data['words'],
             input_tokens: usage['input_tokens'] || usage['prompt_tokens'],
             output_tokens: usage['output_tokens'] || usage['completion_tokens']
           )

data/lib/ruby_llm/providers/openrouter/chat.rb

@@ -52,7 +52,7 @@ module RubyLLM
 
         def parse_completion_response(response)
           data = response.body
-          return if data.empty?
+          return if data.nil? || data.empty?
 
           raise Error.new(response, data.dig('error', 'message')) if data.dig('error', 'message')
 
@@ -108,13 +108,17 @@ module RubyLLM
           messages.map do |msg|
             {
               role: format_role(msg.role),
-              content: OpenAI::Media.format_content(msg.content),
+              content: format_content(msg.content),
               tool_calls: OpenAI::Tools.format_tool_calls(msg.tool_calls),
               tool_call_id: msg.tool_call_id
             }.compact.merge(format_thinking(msg))
           end
         end
 
+        def format_content(content)
+          OpenAI::Media.format_content(content)
+        end
+
         def format_role(role)
           case role
           when :system

data/lib/ruby_llm/providers/perplexity/chat.rb

@@ -10,6 +10,17 @@ module RubyLLM
         def format_role(role)
           role.to_s
         end
+
+        def format_messages(messages)
+          messages.map do |msg|
+            {
+              role: format_role(msg.role),
+              content: Perplexity::Media.format_content(msg.content),
+              tool_calls: OpenAI::Tools.format_tool_calls(msg.tool_calls),
+              tool_call_id: msg.tool_call_id
+            }.compact.merge(OpenAI::Chat.format_thinking(msg))
+          end
+        end
       end
     end
   end

data/lib/ruby_llm/providers/perplexity/media.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Perplexity
+      # Handles Perplexity Sonar media content.
+      module Media
+        module_function
+
+        SUPPORTED_DOCUMENT_EXTENSIONS = %w[pdf doc docx txt rtf].freeze
+
+        def format_content(content) # rubocop:disable Metrics/PerceivedComplexity
+          if content.is_a?(RubyLLM::Content::Raw)
+            value = content.value
+            return value.is_a?(Hash) ? value.to_json : value
+          end
+          return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
+          return content unless content.is_a?(Content)
+
+          parts = []
+          parts << OpenAI::Media.format_text(content.text) if content.text
+
+          content.attachments.each do |attachment|
+            case attachment.type
+            when :image
+              parts << OpenAI::Media.format_image(attachment)
+            when :pdf, :document
+              parts << format_document(attachment)
+            when :text
+              parts << format_text_attachment(attachment)
+            else
+              raise UnsupportedAttachmentError, attachment.mime_type
+            end
+          end
+
+          parts
+        end
+
+        def format_document(attachment)
+          raise UnsupportedAttachmentError, attachment.mime_type unless supported_file?(attachment)
+
+          {
+            type: 'file_url',
+            file_url: {
+              url: attachment.url? ? attachment.source.to_s : attachment.encoded
+            }
+          }
+        end
+
+        def format_text_attachment(attachment)
+          OpenAI::Media.format_text_file(attachment)
+        end
+
+        def supported_file?(attachment)
+          return true if attachment.pdf?
+
+          SUPPORTED_DOCUMENT_EXTENSIONS.include?(attachment.extension)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/perplexity.rb

@@ -8,7 +8,7 @@ module RubyLLM
       include Perplexity::Models
 
       def api_base
-        'https://api.perplexity.ai'
+        @config.perplexity_api_base || 'https://api.perplexity.ai'
       end
 
       def headers
@@ -24,7 +24,7 @@ module RubyLLM
         end
 
         def configuration_options
-          %i[perplexity_api_key]
+          %i[perplexity_api_key perplexity_api_base]
         end
 
         def configuration_requirements

data/lib/ruby_llm/providers/vertexai.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'stringio'
+
 module RubyLLM
   module Providers
     # Google Vertex AI implementation
@@ -21,6 +23,8 @@ module RubyLLM
       end
 
       def api_base
+        return @config.vertexai_api_base if @config.vertexai_api_base
+
         if @config.vertexai_location.to_s == 'global'
           'https://aiplatform.googleapis.com/v1beta1'
         else
@@ -41,7 +45,7 @@ module RubyLLM
 
       class << self
         def configuration_options
-          %i[vertexai_project_id vertexai_location vertexai_service_account_key]
+          %i[vertexai_project_id vertexai_location vertexai_service_account_key vertexai_api_base]
         end
 
         def configuration_requirements

data/lib/ruby_llm/providers/xai/chat.rb

@@ -9,6 +9,15 @@ module RubyLLM
         def format_role(role)
           role.to_s
         end
+
+        def format_content(content)
+          OpenAI::Media.format_content(
+            content,
+            document_attachments: :none,
+            image_attachments: true,
+            audio_attachments: false
+          )
+        end
       end
     end
   end

data/lib/ruby_llm/providers/xai/models.rb

@@ -7,23 +7,6 @@ module RubyLLM
       module Models
         module_function
 
-        IMAGE_MODELS = %w[grok-2-image-1212].freeze
-        VISION_MODELS = %w[
-          grok-2-vision-1212
-          grok-4-0709
-          grok-4-fast-non-reasoning
-          grok-4-fast-reasoning
-          grok-4-1-fast-non-reasoning
-          grok-4-1-fast-reasoning
-        ].freeze
-        REASONING_MODELS = %w[
-          grok-3-mini
-          grok-4-0709
-          grok-4-fast-reasoning
-          grok-4-1-fast-reasoning
-          grok-code-fast-1
-        ].freeze
-
         def parse_list_models_response(response, slug, _capabilities)
           Array(response.body['data']).map do |model_data|
             model_id = model_data['id']
@@ -48,27 +31,32 @@ module RubyLLM
         end
 
         def modalities_for(model_id)
-          if IMAGE_MODELS.include?(model_id)
-            { input: ['text'], output: ['image'] }
+          if image_model?(model_id)
+            { input: %w[text image], output: ['image'] }
+          elsif video_model?(model_id)
+            { input: %w[text image video], output: ['video'] }
           else
-            input = ['text']
-            input << 'image' if VISION_MODELS.include?(model_id)
-            { input: input, output: ['text'] }
+            { input: ['text'], output: ['text'] }
           end
         end
 
         def capabilities_for(model_id)
-          return [] if IMAGE_MODELS.include?(model_id)
+          return ['vision'] if image_model?(model_id) || video_model?(model_id)
 
-          capabilities = %w[streaming function_calling structured_output]
-          capabilities << 'reasoning' if REASONING_MODELS.include?(model_id)
-          capabilities << 'vision' if VISION_MODELS.include?(model_id)
-          capabilities
+          ['streaming']
         end
 
         def format_display_name(model_id)
           model_id.tr('-', ' ').split.map(&:capitalize).join(' ')
         end
+
+        def image_model?(model_id)
+          model_id.match?(/\Agrok-(?:2-)?imagine-image/) || model_id == 'grok-2-image-1212'
+        end
+
+        def video_model?(model_id)
+          model_id.match?(/\Agrok-(?:2-)?imagine-video/)
+        end
       end
     end
   end