ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/providers/mistral/chat.rb

@@ -46,6 +46,13 @@ module RubyLLM
           keys.map { |key| ":#{key}" }.join(', ')
         end
 
+        # extra_content carries Gemini's thought signatures on Google's
+        # OpenAI-compatible endpoints, and Mistral refuses any field it does
+        # not define.
+        def format_tool_calls(tool_calls)
+          super&.map { |call| call.except(:extra_content) }
+        end
+
         def build_tool_choice(tool_choice)
           return 'any' if tool_choice == :required
 
@@ -70,7 +77,7 @@ module RubyLLM
           formatted_content = super
           return formatted_content unless msg.role == :assistant && msg.thinking
 
-          content_blocks = build_thinking_blocks(msg.thinking)
+          content_blocks = build_thinking_blocks(msg)
           append_formatted_content(content_blocks, formatted_content)
 
           content_blocks
@@ -80,17 +87,14 @@ module RubyLLM
           {}
         end
 
-        def build_thinking_blocks(thinking)
-          return [] unless thinking
-
-          if thinking.text
-            [{
-              type: 'thinking',
-              thinking: [{ type: 'text', text: thinking.text }],
-              signature: thinking.signature
-            }.compact]
-          elsif thinking.signature
-            [{ type: 'thinking', signature: thinking.signature }]
+        def build_thinking_blocks(msg)
+          text = msg.thinking&.text
+          signature = own_signature(msg)
+
+          if text
+            [{ type: 'thinking', thinking: [{ type: 'text', text: }], signature: }.compact]
+          elsif signature
+            [{ type: 'thinking', signature: }]
           else
             []
           end

data/lib/ruby_llm/providers/mistral/media.rb

@@ -7,8 +7,14 @@ module RubyLLM
       module Media
         module_function
 
+        def supported_attachment?(attachment)
+          %i[image audio pdf document text].include?(attachment.type)
+        end
+
         def format_content(content, attachments = [])
           Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless supported_attachment?(attachment)
+
             case attachment.type
             when :image
               format_image(attachment)
@@ -18,8 +24,6 @@ module RubyLLM
               format_document(attachment)
             when :text
               Protocols::ChatCompletions::Media.format_text_file(attachment)
-            else
-              raise UnsupportedAttachmentError, attachment.mime_type
             end
           end
         end
@@ -27,14 +31,14 @@ module RubyLLM
         def format_image(image)
           {
             type: 'image_url',
-            image_url: image.url? ? image.source.to_s : image.for_llm
+            image_url: image.url_or_data_uri
           }
         end
 
         def format_document(document)
           {
             type: 'document_url',
-            document_url: document.url? ? document.source.to_s : document.for_llm
+            document_url: document.url_or_data_uri
           }
         end
       end

data/lib/ruby_llm/providers/mistral/ocr.rb

@@ -22,7 +22,7 @@ module RubyLLM
         end
 
         def ocr_document_part(attachment)
-          reference = attachment.url? ? attachment.source.to_s : ocr_data_uri(attachment)
+          reference = attachment.url_or_data_uri
 
           if attachment.image?
             { type: 'image_url', image_url: reference }
@@ -31,10 +31,6 @@ module RubyLLM
           end
         end
 
-        def ocr_data_uri(attachment)
-          "data:#{attachment.mime_type};base64,#{attachment.encoded}"
-        end
-
         def parse_ocr_response(response, model:)
           data = response.body
 

data/lib/ruby_llm/providers/ollama/judgments.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class Ollama
+      # Ollama's dialect of the System One API, which accepts images for multimodal decision models.
+      module Judgments
+        module_function
+
+        def judgment_url
+          'systemone'
+        end
+
+        def render_judgment_payload(input, questions:, model:, with: [], provider_options: {})
+          images = with.map do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+            attachment.encoded
+          end
+          payload = super(input, questions:, model:, provider_options:)
+          images.empty? ? payload : payload.merge(images:)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/ollama/media.rb

@@ -7,8 +7,14 @@ module RubyLLM
       module Media
         module_function
 
+        def supported_attachment?(attachment)
+          %i[image audio text].include?(attachment.type)
+        end
+
         def format_content(content, attachments = [])
           Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless supported_attachment?(attachment)
+
             case attachment.type
             when :image
               format_image(attachment)
@@ -16,8 +22,6 @@ module RubyLLM
               Protocols::ChatCompletions::Media.format_audio(attachment)
             when :text
               Protocols::ChatCompletions::Media.format_text_file(attachment)
-            else
-              raise UnsupportedAttachmentError, attachment.mime_type
             end
           end
         end

data/lib/ruby_llm/providers/ollama/models.rb

@@ -59,6 +59,7 @@ module RubyLLM
 
         def build_capabilities(reported)
           return [] if reported.include?('embedding')
+          return %w[judgment] if reported.include?('decision')
 
           derived = CAPABILITY_MAP.filter_map { |native, capability| capability if reported.include?(native) }
           model_capabilities + derived
@@ -66,6 +67,7 @@ module RubyLLM
 
         def build_modalities(reported)
           return { input: %w[text], output: %w[embeddings] } if reported.include?('embedding')
+          return { input: %w[text], output: %w[judgment] } if reported.include?('decision')
 
           input = %w[text]
           input << 'image' if reported.include?('vision')

data/lib/ruby_llm/providers/ollama.rb

@@ -11,12 +11,23 @@ module RubyLLM
         include Ollama::Models
       end
 
+      # Ollama's dialect of the System One API for decision models such as Clef.
+      class SystemOne < Protocols::SystemOne
+        include Ollama::Judgments
+      end
+
       protocol :chat_completions, ChatCompletions
+      protocol :system_one, SystemOne
 
       def api_base
         @config.ollama_api_base
       end
 
+      def resolve_protocol(name, model, operation: nil, **options)
+        name ||= :system_one if operation == :judge
+        super
+      end
+
       def headers
         return {} unless @config.ollama_api_key
 

data/lib/ruby_llm/providers/openai/capabilities.rb

@@ -46,13 +46,15 @@ module RubyLLM
           gpt-4o-transcribe-diarize
           whisper-1
         ].freeze
+        JUDGMENT_MODELS = %w[gpt-6-luna].freeze
         EXPLICIT_CAPABILITIES = {
           'function_calling' => (CHAT_MODELS + CODEX_MODELS).freeze,
           'structured_output' => (CHAT_MODELS + CODEX_MODELS + SEARCH_MODELS).freeze,
           'vision' => (CHAT_MODELS + CODEX_MODELS + DEEP_RESEARCH_MODELS + MODERATION_MODELS).freeze,
           'reasoning' => (CODEX_MODELS + DEEP_RESEARCH_MODELS).freeze,
           'transcription' => TRANSCRIPTION_MODELS,
-          'citations' => SEARCH_MODELS
+          'citations' => SEARCH_MODELS,
+          'judgment' => JUDGMENT_MODELS
         }.freeze
 
         def self.augment(capabilities, model_id:, **)

data/lib/ruby_llm/providers/openai.rb

@@ -9,14 +9,24 @@ module RubyLLM
       protocol :embeddings, Protocols::ChatCompletions,
                batches: Protocols::ChatCompletions::EmbeddingBatches
       protocol :files, Protocols::OpenAI::Files
+      protocol :decisions, Protocols::OpenAI::Decisions
 
       RATE_LIMIT_RESET_HEADERS = %w[x-ratelimit-reset-requests x-ratelimit-reset-tokens].freeze
       RESET_DURATION_UNITS = { 'h' => 3600, 'm' => 60, 's' => 1, 'ms' => 0.001 }.freeze
 
+      def original_image_detail? # :nodoc:
+        true
+      end
+
       def api_base
         @config.openai_api_base || 'https://api.openai.com/v1'
       end
 
+      def resolve_protocol(name, model, operation: nil, **options)
+        name ||= :decisions if operation == :judge
+        super
+      end
+
       # Audio, realtime, and dedicated search models only exist on Chat Completions.
       def protocol_for(model, **)
         return protocols[:chat_completions] if Capabilities::SEARCH_MODELS.include?(model.id)
@@ -32,6 +42,10 @@ module RubyLLM
         }.compact
       end
 
+      def account_identity
+        account_digest(api_base, @config.openai_api_key, @config.openai_organization_id, @config.openai_project_id)
+      end
+
       # OpenAI reports when each rate limit resets in its own headers,
       # as durations like "6m0s", "7.66s" or "76ms".
       def retry_delay(response)

data/lib/ruby_llm/providers/openrouter/chat.rb

@@ -6,7 +6,6 @@ module RubyLLM
       # Chat methods of the OpenRouter API integration
       module Chat
         OPENROUTER_INLINE_FILE_THRESHOLD = 50 * 1024 * 1024
-        OPENROUTER_FILE_UPLOAD_LIMIT = 100 * 1024 * 1024
         CACHE_CONTROL_TYPE = 'ephemeral'
         PROMPT_CACHE_OPTIONS = %i[ttl].freeze
         COMPACTION_PLUGIN_ID = 'context-compression'
@@ -35,6 +34,10 @@ module RubyLLM
           end
         end
 
+        def supported_attachment?(attachment)
+          OpenRouter::Media.supported_attachment?(attachment)
+        end
+
         def format_content(content, attachments = [])
           OpenRouter::Media.format_content(content, attachments)
         end
@@ -86,32 +89,28 @@ module RubyLLM
           return {} unless msg.role == :assistant
           return { reasoning_details: msg.raw_reasoning } if msg.raw_reasoning.is_a?(Array)
 
-          thinking = msg.thinking
-          return {} unless thinking
+          return {} unless msg.thinking && producer_slug(msg) == @provider.slug
+
+          detail = reasoning_detail(msg.thinking)
+          detail ? { reasoning_details: [detail] } : {}
+        end
 
-          details = []
+        def reasoning_detail(thinking)
           if thinking.text
-            details << {
-              type: 'reasoning.text',
-              text: thinking.text,
-              signature: thinking.signature
-            }.compact
+            { type: 'reasoning.text', text: thinking.text, signature: thinking.signature }.compact
           elsif thinking.signature
-            details << {
-              type: 'reasoning.encrypted',
-              data: thinking.signature
-            }
+            { type: 'reasoning.encrypted', data: thinking.signature }
           end
-
-          details.empty? ? {} : { reasoning_details: details }
         end
 
         def format_message_content(msg, caching: nil)
           content = super
-          caching != false && msg.cache_until_here? ? inject_cache_control(content, caching:) : content
+          return content unless caching != false && msg.cache_until_here?
+
+          inject_cache_control(content, caching:, ttl: msg.cache_ttl)
         end
 
-        def inject_cache_control(content, caching: nil)
+        def inject_cache_control(content, caching: nil, ttl: nil)
           blocks = content.is_a?(Array) ? content.dup : [{ type: 'text', text: content }]
           return blocks if blocks.empty?
 
@@ -119,15 +118,16 @@ module RubyLLM
           return blocks unless last.is_a?(Hash)
           return blocks if last[:cache_control] || last['cache_control']
 
-          blocks[-1] = last.merge(cache_control: prompt_cache_control(caching))
+          blocks[-1] = last.merge(cache_control: prompt_cache_control(caching, ttl:))
           blocks
         end
 
-        def prompt_cache_control(caching = nil)
+        def prompt_cache_control(caching = nil, ttl: nil)
           options = prompt_cache_options(caching)
+          ttl ||= options[:ttl]
 
           { type: CACHE_CONTROL_TYPE }.tap do |control|
-            control[:ttl] = options[:ttl] if options[:ttl]
+            control[:ttl] = ttl if ttl
           end
         end
 
@@ -166,10 +166,6 @@ module RubyLLM
           OPENROUTER_INLINE_FILE_THRESHOLD
         end
 
-        def provider_file_upload_limit
-          OPENROUTER_FILE_UPLOAD_LIMIT
-        end
-
         def provider_file_attachable?(attachment)
           attachment.pdf?
         end

data/lib/ruby_llm/providers/openrouter/embeddings.rb

@@ -39,6 +39,8 @@ module RubyLLM
 
           if (type = EMBEDDING_MEDIA_TYPES[attachment.type])
             { type: type.to_s, type => { data: attachment.for_llm, format: attachment.format } }
+          elsif attachment.type == :image
+            Protocols::ChatCompletions::Media.format_image(attachment)
           else
             Protocols::ChatCompletions::Media.format_attachment(
               attachment, document_attachments: :none, image_attachments: true, audio_attachments: false

data/lib/ruby_llm/providers/openrouter/media.rb

@@ -8,6 +8,10 @@ module RubyLLM
       module Media
         module_function
 
+        def supported_attachment?(attachment)
+          attachment.video? || Protocols::ChatCompletions::Media.supported_attachment?(attachment)
+        end
+
         def format_content(content, attachments = [])
           Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
             if attachment.type == :video
@@ -24,7 +28,7 @@ module RubyLLM
           {
             type: 'video_url',
             video_url: {
-              url: video.url? ? video.source.to_s : video.for_llm
+              url: video.url_or_data_uri
             }
           }
         end

data/lib/ruby_llm/providers/openrouter/streaming.rb

@@ -17,6 +17,7 @@ module RubyLLM
             role: :assistant,
             model: data['model'],
             content: delta['content'],
+            citations: extract_chunk_citations(delta, data),
             thinking: Thinking.build(
               text: extract_thinking_text(delta),
               signature: extract_thinking_signature(delta)
@@ -28,6 +29,7 @@ module RubyLLM
             cache_read_tokens: cache_read_tokens(usage),
             cache_write_tokens: cache_write_tokens(usage),
             thinking_tokens: thinking_tokens(usage),
+            server_tool_use: server_tool_use(usage),
             reported_cost: reported_cost(usage),
             finish_reason: normalize_finish_reason(data.dig('choices', 0, 'finish_reason'))
           )
@@ -47,7 +49,8 @@ module RubyLLM
 
           detail.each do |key, value|
             if value.is_a?(String) && target[key].is_a?(String) && ACCUMULATED_REASONING_KEYS.include?(key)
-              target[key] += value
+              target[key] = +target[key]
+              target[key] << value
             elsif target[key].nil?
               target[key] = value
             end

data/lib/ruby_llm/providers/openrouter/videos.rb

@@ -20,11 +20,16 @@ module RubyLLM
           payload.merge(provider_options)
         end
 
-        def parse_video_job(response, model:)
+        def parse_video_job(response, model:, **request)
           id = response.body['id']
           raise Error.new('OpenRouter did not return a video job', response:) unless id
 
-          VideoJob.new(id: id, protocol: self, model: model, status: job_status(response.body), raw: response.body)
+          VideoJob.new(id: id, protocol: self, model: model, status: job_status(response.body), raw: response.body,
+                       **request)
+        end
+
+        def video_request_settings(request)
+          { duration: request[:duration], resolution: request[:resolution] || request[:size] }
         end
 
         def video_job_url(job)
@@ -34,9 +39,9 @@ module RubyLLM
         def parse_video_job_status(response, job:) # rubocop:disable Lint/UnusedMethodArgument
           body = response.body
           status = job_status(body)
-          state = { status: status, raw: body }
+          state = { status: status, raw: body, reported_cost: reported_cost(body['usage'] || {}) }
           state[:error] = body['error'] || body['status'] if status == :failed
-          state
+          state.compact
         end
 
         def download_video(job)
@@ -71,7 +76,7 @@ module RubyLLM
 
         def frame_images(attachments)
           attachments.zip(FRAME_TYPES).map do |attachment, frame_type|
-            url = attachment.url? ? attachment.source.to_s : attachment.for_llm
+            url = attachment.url_or_data_uri
             { type: 'image_url', image_url: { url: url }, frame_type: frame_type }
           end
         end

data/lib/ruby_llm/providers/openrouter.rb

@@ -56,6 +56,10 @@ module RubyLLM
         }
       end
 
+      def account_identity
+        account_digest(api_base, @config.openrouter_api_key)
+      end
+
       def parse_error(response)
         body = parse_error_body(response)
         return unless body

data/lib/ruby_llm/providers/perplexity/chat.rb

@@ -11,6 +11,10 @@ module RubyLLM
           role.to_s
         end
 
+        def supported_attachment?(attachment)
+          Perplexity::Media.supported_attachment?(attachment)
+        end
+
         def format_content(content, attachments = [])
           Perplexity::Media.format_content(content, attachments)
         end

data/lib/ruby_llm/providers/perplexity/media.rb

@@ -7,10 +7,14 @@ module RubyLLM
       module Media
         module_function
 
-        SUPPORTED_DOCUMENT_EXTENSIONS = %w[pdf doc docx txt rtf].freeze
+        def supported_attachment?(attachment)
+          %i[image pdf document text].include?(attachment.type)
+        end
 
         def format_content(content, attachments = [])
           Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless supported_attachment?(attachment)
+
             case attachment.type
             when :image
               Protocols::ChatCompletions::Media.format_image(attachment)
@@ -18,15 +22,11 @@ module RubyLLM
               format_document(attachment)
             when :text
               Protocols::ChatCompletions::Media.format_text_file(attachment)
-            else
-              raise UnsupportedAttachmentError, attachment.mime_type
             end
           end
         end
 
         def format_document(attachment)
-          raise UnsupportedAttachmentError, attachment.mime_type unless supported_file?(attachment)
-
           {
             type: 'file_url',
             file_url: {
@@ -34,12 +34,6 @@ module RubyLLM
             }
           }
         end
-
-        def supported_file?(attachment)
-          return true if attachment.pdf?
-
-          SUPPORTED_DOCUMENT_EXTENSIONS.include?(attachment.extension)
-        end
       end
     end
   end

data/lib/ruby_llm/providers/perplexity/models.rb

@@ -11,12 +11,16 @@ module RubyLLM
           sonar-reasoning-pro
           sonar-deep-research
         ].freeze
+        PRESET_MODEL_IDS = Protocols::Perplexity::Agent::PRESETS
         EMBEDDING_MODEL_IDS = %w[
           pplx-embed-v1-0.6b
           pplx-embed-v1-4b
         ].freeze
-        STATIC_MODEL_IDS = (SEARCH_MODEL_IDS + EMBEDDING_MODEL_IDS).freeze
-        STATIC_MODEL_DATA = {
+        STATIC_MODEL_IDS = (SEARCH_MODEL_IDS + PRESET_MODEL_IDS + EMBEDDING_MODEL_IDS).freeze
+        PRESET_MODEL_DATA = PRESET_MODEL_IDS.to_h do |id|
+          [id, { capabilities: %w[streaming structured_output citations function_calling] }]
+        end
+        STATIC_MODEL_DATA = PRESET_MODEL_DATA.merge(
           'sonar' => {
             context_window: 128_000, input_price: 1.0, output_price: 1.0,
             capabilities: %w[streaming structured_output citations]
@@ -35,7 +39,7 @@ module RubyLLM
           },
           'pplx-embed-v1-0.6b' => { context_window: 32_768, input_price: 0.004 },
           'pplx-embed-v1-4b' => { context_window: 32_768, input_price: 0.03 }
-        }.freeze
+        ).freeze
 
         def models_url
           'v1/models'
@@ -49,7 +53,7 @@ module RubyLLM
         end
 
         # The models endpoint lists the multi-model catalog but not the search
-        # or embedding models, so those ride along statically.
+        # models, presets, or embedding models, so those ride along statically.
         def parse_list_models_response(response, slug)
           listed = Array(response.body['data']).map do |model_data|
             create_model_info(model_data['id'], slug, pricing: endpoint_pricing(model_data['pricing']))

data/lib/ruby_llm/providers/perplexity.rb

@@ -14,11 +14,22 @@ module RubyLLM
       protocol :chat_completions, ChatCompletions
       protocol :router_chat_completions, Protocols::Perplexity::Router
       protocol :files, Protocols::Perplexity::Files
+      protocol :agent_responses, Protocols::Perplexity::Agent
 
       def api_base
         @config.perplexity_api_base || 'https://api.perplexity.ai'
       end
 
+      # Chat runs on the Agent API. Embeddings and the model listing stay on
+      # the endpoints the Chat Completions dialect knows.
+      def protocol_for(model, operation: nil, **)
+        operation ? super : protocols[:agent_responses]
+      end
+
+      def agent_url # :nodoc:
+        "#{api_base.delete_suffix('/').delete_suffix('/v1')}/v1/agent"
+      end
+
       def router_url(operation) # :nodoc:
         "#{api_base.delete_suffix('/').delete_suffix('/router/v1')}/router/v1/#{operation}"
       end

data/lib/ruby_llm/providers/typesafe.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    # Connects TypeSafe's judgment models to RubyLLM.
+    class TypeSafe < Provider
+      protocol :system_one, Protocols::SystemOne
+
+      def api_base
+        @config.typesafe_api_base || 'https://api.typesafe.ai'
+      end
+
+      def headers
+        { 'Authorization' => "Bearer #{@config.typesafe_api_key}" }
+      end
+
+      def parse_error(response)
+        protocols.fetch(:system_one).new(self).parse_error_response(response) || super
+      end
+
+      class << self
+        def configuration_options
+          %i[typesafe_api_key typesafe_api_base]
+        end
+
+        def configuration_requirements
+          %i[typesafe_api_key]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/vertexai/credentials.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'stringio'
+
+module RubyLLM
+  module Providers
+    class VertexAI
+      # Google credentials shared by every Vertex AI provider in the process.
+      # googleauth keeps the access token on the credentials object and
+      # refreshes it a minute before it expires, so sharing the object shares
+      # the token.
+      class Credentials # :nodoc:
+        CACHE = Support::ProcessCache.new
+
+        attr_reader :authorizer
+
+        def self.for(config)
+          key = config.vertexai_service_account_key
+          CACHE.fetch([source(key), SCOPES]) { new(key) }
+        end
+
+        def self.cache
+          CACHE
+        end
+
+        def self.source(key)
+          return Digest::SHA256.hexdigest(key) if key
+
+          [:application_default, ENV.fetch('GOOGLE_APPLICATION_CREDENTIALS', nil)]
+        end
+        private_class_method :source
+
+        def initialize(service_account_key)
+          @authorizer = build(service_account_key)
+          @lock = Mutex.new
+        end
+
+        # One caller fetches an expiring token while the others wait for it.
+        def headers
+          @lock.synchronize { @authorizer.apply({}) }
+        end
+
+        private
+
+        def build(service_account_key)
+          require 'googleauth'
+          if service_account_key
+            ::Google::Auth::ServiceAccountCredentials.make_creds(
+              json_key_io: StringIO.new(service_account_key),
+              scope: SCOPES
+            )
+          else
+            ::Google::Auth.get_application_default(SCOPES)
+          end
+        rescue LoadError
+          raise Error,
+                'The googleauth gem ~> 1.15 is required for Vertex AI. Please add it to your Gemfile: gem "googleauth"'
+        end
+      end
+    end
+  end
+end