ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/providers/deepseek/capabilities.rb

@@ -3,59 +3,12 @@
 module RubyLLM
   module Providers
     class DeepSeek
-      # Provider-level capability checks used outside the model registry.
+      # Feature capability gaps not represented in upstream model catalogs.
       module Capabilities
-        module_function
+        def self.augment(capabilities, **)
+          return capabilities unless capabilities.include?('function_calling')
 
-        DEFAULT_CONTEXT_WINDOW = 1_000_000
-        DEFAULT_MAX_OUTPUT_TOKENS = 384_000
-        DEFAULT_PRICES = {
-          input: 0.14,
-          output: 0.28,
-          cache_read: 0.0028
-        }.freeze
-        PRO_PRICES = {
-          input: 0.435,
-          output: 0.87,
-          cache_read: 0.003625
-        }.freeze
-
-        def supports_tool_choice?(_model_id)
-          true
-        end
-
-        def supports_tool_parallel_control?(_model_id)
-          false
-        end
-
-        def context_window_for(_model_id)
-          DEFAULT_CONTEXT_WINDOW
-        end
-
-        def max_tokens_for(_model_id)
-          DEFAULT_MAX_OUTPUT_TOKENS
-        end
-
-        def critical_capabilities_for(model_id)
-          v4_model = model_id.start_with?('deepseek-v4-')
-          capabilities = ['function_calling']
-          capabilities << 'structured_output' if v4_model
-          capabilities << 'reasoning' if model_id == 'deepseek-reasoner' || v4_model
-          capabilities
-        end
-
-        def pricing_for(model_id)
-          prices = model_id == 'deepseek-v4-pro' ? PRO_PRICES : DEFAULT_PRICES
-
-          {
-            text_tokens: {
-              standard: {
-                input_per_million: prices[:input],
-                output_per_million: prices[:output],
-                cache_read_input_per_million: prices[:cache_read]
-              }
-            }
-          }
+          capabilities | ['tool_choice']
         end
       end
     end

data/lib/ruby_llm/providers/deepseek/chat.rb

@@ -11,9 +11,56 @@ module RubyLLM
           role.to_s
         end
 
-        def format_content(content)
-          OpenAI::Media.format_content(
+        def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil,
+                           schema: nil, thinking: nil, citations: false, caching: nil, tool_prefs: nil)
+          payload = super
+          configure_thinking_payload(payload, thinking)
+          degrade_schema_payload(payload) if schema
+          payload
+        end
+
+        def apply_end_user(payload, identifier)
+          payload.merge(user_id: identifier)
+        end
+
+        def configure_thinking_payload(payload, thinking)
+          return unless thinking&.enabled?
+
+          if thinking.budget
+            RubyLLM.logger.debug { "DeepSeek has no thinking budgets; ignoring budget #{thinking.budget}" }
+          end
+
+          if thinking.disabled?
+            payload.delete(:reasoning_effort)
+            payload[:thinking] = { type: 'disabled' }
+          else
+            payload[:thinking] = { type: 'enabled' }
+          end
+        end
+
+        def degrade_schema_payload(payload)
+          RubyLLM.logger.warn(
+            'DeepSeek Chat Completions does not support json_schema response formats. ' \
+            'Use protocol: :responses to enforce the schema. Falling back to json_object mode.'
+          )
+          payload[:response_format] = { type: 'json_object' }
+        end
+
+        def format_thinking(msg)
+          return {} unless msg.role == :assistant
+
+          thinking = msg.thinking
+          text = thinking&.text.to_s
+          payload = { reasoning_content: text }
+          payload[:reasoning] = text unless text.empty?
+          payload[:reasoning_signature] = thinking.signature if thinking&.signature
+          payload
+        end
+
+        def format_content(content, attachments = [])
+          Protocols::ChatCompletions::Media.format_content(
             content,
+            attachments,
             document_attachments: :none,
             image_attachments: false,
             audio_attachments: false

data/lib/ruby_llm/providers/deepseek/responses.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class DeepSeek
+      # DeepSeek's dialect of the OpenAI Responses API.
+      # Reasoning arrives as reasoning_text content parts
+      # instead of summaries, both in responses and in the stream.
+      class Responses < Protocols::Responses
+        SERVER_TOOL_ALIASES = {
+          web_search: { tool: { type: 'web_search' } },
+          apply_patch: { tool: { type: 'custom', name: 'apply_patch' } }
+        }.freeze
+
+        def server_tool_aliases
+          SERVER_TOOL_ALIASES
+        end
+
+        def apply_end_user(payload, identifier)
+          payload.merge(user_id: identifier)
+        end
+
+        def build_chunk(data)
+          return chunk(thinking: Thinking.build(text: data['delta'])) if data['type'] == 'response.reasoning_text.delta'
+
+          super
+        end
+
+        def parse_reasoning_summary(output)
+          texts = output.select { |item| item['type'] == 'reasoning' }.flat_map do |item|
+            Array(item['content']).filter_map { |part| part['text'] if part['type'] == 'reasoning_text' }
+          end
+
+          texts.empty? ? super : texts.join("\n")
+        end
+
+        def format_assistant_items(message)
+          items = super
+          return items if message.raw_content || message.thinking&.text.to_s.empty? || message.thinking.signature
+
+          items.unshift(format_reasoning_item(message.thinking))
+        end
+
+        def format_reasoning_item(thinking)
+          { type: 'reasoning', content: [{ type: 'reasoning_text', text: thinking.text }] }
+        end
+
+        def format_tool_items(message)
+          [{
+            type: 'function_call_output',
+            call_id: message.tool_call_id,
+            output: format_content(message.content, message.attachments)
+          }]
+        end
+
+        def format_provider_file(file)
+          raise UnsupportedAttachmentError, file.mime_type unless file.image?
+
+          { type: 'input_image', file_id: file.provider_file_id }
+        end
+
+        def format_document(document)
+          raise UnsupportedAttachmentError, document.mime_type
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/deepseek.rb

@@ -3,8 +3,15 @@
 module RubyLLM
   module Providers
     # DeepSeek API integration.
-    class DeepSeek < OpenAI
-      include DeepSeek::Chat
+    class DeepSeek < Provider
+      # DeepSeek's dialect of the Chat Completions API.
+      class ChatCompletions < Protocols::ChatCompletions
+        include DeepSeek::Chat
+      end
+
+      protocol :chat_completions, ChatCompletions
+      protocol :responses, DeepSeek::Responses
+      protocol :files, Protocols::DeepSeek::Files
 
       def api_base
         @config.deepseek_api_base || 'https://api.deepseek.com'

data/lib/ruby_llm/providers/elevenlabs.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    # Integrates ElevenLabs speech, transcription, ElevenAgents, and Image & Video APIs.
+    class ElevenLabs < Provider
+      protocol :elevenlabs, Protocols::ElevenLabs
+      protocol :flows, Protocols::ElevenLabs::Flows
+      protocol :files, Protocols::ElevenLabs::Assets
+
+      def resolve_protocol(name, model, operation: nil, **options)
+        name ||= :flows if %i[paint animate].include?(operation)
+        super
+      end
+
+      def api_base
+        @config.elevenlabs_api_base || 'https://api.elevenlabs.io'
+      end
+
+      def headers
+        { 'xi-api-key' => @config.elevenlabs_api_key }
+      end
+
+      class << self
+        def configuration_options
+          %i[elevenlabs_api_key elevenlabs_api_base]
+        end
+
+        def configuration_requirements
+          %i[elevenlabs_api_key]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gemini/capabilities.rb

@@ -3,116 +3,17 @@
 module RubyLLM
   module Providers
     class Gemini
-      # Provider-level capability checks and narrow registry fallbacks.
+      # Feature capability gaps not represented in upstream model catalogs.
       module Capabilities
-        module_function
-
-        PRICES = {
-          flash_2: { input: 0.10, output: 0.40 }, # rubocop:disable Naming/VariableNumber
-          flash_lite_2: { input: 0.075, output: 0.30 }, # rubocop:disable Naming/VariableNumber
-          flash: { input: 0.075, output: 0.30 },
-          flash_8b: { input: 0.0375, output: 0.15 },
-          pro: { input: 1.25, output: 5.0 },
-          pro_2_5: { input: 0.12, output: 0.50 }, # rubocop:disable Naming/VariableNumber
-          gemini_embedding: { input: 0.002, output: 0.004 },
-          embedding: { input: 0.00, output: 0.00 },
-          imagen: { price: 0.03 },
-          aqa: { input: 0.00, output: 0.00 }
-        }.freeze
-
-        def supports_tool_choice?(_model_id)
-          true
-        end
-
-        def supports_tool_parallel_control?(_model_id)
-          false
-        end
-
-        def context_window_for(model_id)
-          case model_id
-          when /gemini-2\.5-pro-exp-03-25/, /gemini-2\.0-flash/, /gemini-2\.0-flash-lite/, /gemini-1\.5-flash/,
-               /gemini-1\.5-flash-8b/
-            1_048_576
-          when /gemini-1\.5-pro/ then 2_097_152
-          when /gemini-embedding-exp/ then 8_192
-          when /text-embedding-004/, /embedding-001/ then 2_048
-          when /aqa/ then 7_168
-          when /imagen-3/ then nil
-          else 32_768
-          end
-        end
-
-        def max_tokens_for(model_id)
-          case model_id
-          when /gemini-2\.5-pro-exp-03-25/ then 64_000
-          when /gemini-2\.0-flash/, /gemini-2\.0-flash-lite/, /gemini-1\.5-flash/, /gemini-1\.5-flash-8b/,
-               /gemini-1\.5-pro/
-            8_192
-          when /gemini-embedding-exp/ then nil
-          when /text-embedding-004/, /embedding-001/ then 768
-          when /imagen-3/ then 4
-          else 4_096
+        def self.augment(capabilities, model_id:, modalities:)
+          additions = []
+          additions << 'tool_choice' if capabilities.include?('function_calling')
+          if !model_id.include?('embedding') &&
+             modalities[:input].include?('audio') && modalities[:output].include?('text')
+            additions << 'transcription'
           end
+          capabilities | additions
         end
-
-        def critical_capabilities_for(model_id)
-          capabilities = []
-          capabilities << 'function_calling' if supports_functions?(model_id)
-          capabilities << 'structured_output' if supports_structured_output?(model_id)
-          capabilities << 'vision' if supports_vision?(model_id)
-          capabilities
-        end
-
-        def pricing_for(model_id)
-          prices = PRICES.fetch(pricing_family(model_id), { input: 0.075, output: 0.30 })
-          {
-            text_tokens: {
-              standard: {
-                input_per_million: prices[:input] || prices[:price] || 0.075,
-                output_per_million: prices[:output] || prices[:price] || 0.30
-              }
-            }
-          }
-        end
-
-        def supports_vision?(model_id)
-          return false if model_id.match?(/text-embedding|embedding-001|aqa/)
-
-          model_id.match?(/gemini|flash|pro|imagen/)
-        end
-
-        def supports_functions?(model_id)
-          return false if model_id.match?(/text-embedding|embedding-001|aqa|flash-lite|imagen|gemini-2\.0-flash-lite/)
-
-          model_id.match?(/gemini|pro|flash/)
-        end
-
-        def supports_structured_output?(model_id)
-          if model_id.match?(/text-embedding|embedding-001|aqa|imagen|gemini-2\.0-flash-lite|gemini-2\.5-pro-exp-03-25/)
-            return false
-          end
-
-          model_id.match?(/gemini|pro|flash/)
-        end
-
-        def pricing_family(model_id)
-          case model_id
-          when /gemini-2\.5-pro-exp-03-25/ then :pro_2_5 # rubocop:disable Naming/VariableNumber
-          when /gemini-2\.0-flash-lite/ then :flash_lite_2 # rubocop:disable Naming/VariableNumber
-          when /gemini-2\.0-flash/ then :flash_2 # rubocop:disable Naming/VariableNumber
-          when /gemini-1\.5-flash-8b/ then :flash_8b
-          when /gemini-1\.5-flash/ then :flash
-          when /gemini-1\.5-pro/ then :pro
-          when /gemini-embedding-exp/ then :gemini_embedding
-          when /text-embedding|embedding/ then :embedding
-          when /imagen/ then :imagen
-          when /aqa/ then :aqa
-          else :base
-          end
-        end
-
-        module_function :context_window_for, :max_tokens_for, :critical_capabilities_for, :pricing_for,
-                        :supports_vision?, :supports_functions?, :supports_structured_output?, :pricing_family
       end
     end
   end

data/lib/ruby_llm/providers/gemini.rb

@@ -4,14 +4,17 @@ module RubyLLM
   module Providers
     # Native Gemini API implementation
     class Gemini < Provider
-      include Gemini::Chat
-      include Gemini::Embeddings
-      include Gemini::Images
-      include Gemini::Models
-      include Gemini::Transcription
-      include Gemini::Streaming
-      include Gemini::Tools
-      include Gemini::Media
+      protocol :gemini, Protocols::Gemini, batches: Protocols::Gemini::Batches
+      protocol :interactions, Protocols::Interactions
+      protocol :live_transcription, Protocols::Gemini::LiveTranscription
+      protocol :files, Protocols::Gemini::Files
+
+      def protocol_for(model, operation: nil, **)
+        return protocols[:interactions] if operation == :transcribe && model.id == 'gemini-3.5-transcribe'
+        return protocols[:live_transcription] if operation == :transcribe && model.id == 'gemini-3.5-transcribe-live'
+
+        super
+      end
 
       def api_base
         @config.gemini_api_base || 'https://generativelanguage.googleapis.com/v1beta'
@@ -23,6 +26,10 @@ module RubyLLM
         }
       end
 
+      def batch_cost_multiplier(component:, **)
+        %i[cache_read cache_write].include?(component) ? 1 : 0.5
+      end
+
       class << self
         def capabilities
           Gemini::Capabilities

data/lib/ruby_llm/providers/gpustack/chat.rb

@@ -7,22 +7,8 @@ module RubyLLM
       module Chat
         module_function
 
-        def format_messages(messages)
-          messages.map do |msg|
-            {
-              role: format_role(msg.role),
-              content: format_message_content(msg),
-              tool_calls: format_tool_calls(msg.tool_calls),
-              tool_call_id: msg.tool_call_id
-            }.compact.merge(OpenAI::Chat.format_thinking(msg))
-          end
-        end
-
-        def format_message_content(msg)
-          content = GPUStack::Media.format_content(msg.content)
-          return '' if content.nil? && OpenAI::Chat.thinking_only_assistant_message?(msg)
-
-          content
+        def format_content(content, attachments = [])
+          GPUStack::Media.format_content(content, attachments)
         end
 
         def format_role(role)

data/lib/ruby_llm/providers/gpustack/embeddings.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class GPUStack
+      # GPUStack's vLLM backend accepts multimodal embeddings as messages.
+      module Embeddings
+        module_function
+
+        def supports_embedding_media?
+          true
+        end
+
+        def render_embedding_payload(text, model:, dimensions:, task_type: nil, title: nil, with: [],
+                                     provider_options: {})
+          payload = super(text, model:, dimensions:, task_type:, title:, provider_options: {})
+          if with.any?
+            raise ArgumentError, 'embed one text at a time when embedding attachments' if text.is_a?(Array)
+
+            payload.delete(:input)
+            payload[:messages] = [{ role: 'user', content: GPUStack::Media.format_content(text, with) }]
+          end
+          payload.merge(provider_options)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/gpustack/media.rb

@@ -3,32 +3,26 @@
 module RubyLLM
   module Providers
     class GPUStack
-      # Handles formatting of media content (images, audio) for GPUStack APIs
+      # Handles formatting of media content for GPUStack. Images ride inline
+      # as base64 data URLs so the cluster never has to fetch remote sources.
       module Media
-        extend OpenAI::Media
-
         module_function
 
-        def format_content(content)
-          return content.value if content.is_a?(RubyLLM::Content::Raw)
-          return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
-          return content unless content.is_a?(Content)
-
-          parts = []
-          parts << format_text(content.text) if content.text
-
-          content.attachments.each do |attachment|
+        def format_content(content, attachments = [])
+          Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
             case attachment.type
             when :image
-              parts << GPUStack::Media.format_image(attachment)
+              format_image(attachment)
+            when :audio
+              Protocols::ChatCompletions::Media.format_audio(attachment)
+            when :video
+              format_video(attachment)
             when :text
-              parts << format_text_file(attachment)
+              Protocols::ChatCompletions::Media.format_text_file(attachment)
             else
               raise UnsupportedAttachmentError, attachment.mime_type
             end
           end
-
-          parts
         end
 
         def format_image(image)
@@ -40,6 +34,13 @@ module RubyLLM
             }
           }
         end
+
+        def format_video(video)
+          {
+            type: 'video_url',
+            video_url: { url: video.url? ? video.source.to_s : video.for_llm }
+          }
+        end
       end
     end
   end