ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/providers/openrouter/chat.rb

@@ -5,142 +5,85 @@ module RubyLLM
     class OpenRouter
       # Chat methods of the OpenRouter API integration
       module Chat
-        module_function
-
-        # rubocop:disable Metrics/ParameterLists,Metrics/PerceivedComplexity
-        def render_payload(messages, tools:, temperature:, model:, stream: false, schema: nil,
-                           thinking: nil, tool_prefs: nil)
-          tool_prefs ||= {}
-          payload = {
-            model: model.id,
-            messages: format_messages(messages),
-            stream: stream
-          }
-
-          payload[:temperature] = temperature unless temperature.nil?
-          if tools.any?
-            payload[:tools] = tools.map { |_, tool| OpenAI::Tools.tool_for(tool) }
-            payload[:tool_choice] = OpenAI::Tools.build_tool_choice(tool_prefs[:choice]) unless tool_prefs[:choice].nil?
-            payload[:parallel_tool_calls] = tool_prefs[:calls] == :many unless tool_prefs[:calls].nil?
-          end
+        OPENROUTER_INLINE_FILE_THRESHOLD = 50 * 1024 * 1024
+        OPENROUTER_FILE_UPLOAD_LIMIT = 100 * 1024 * 1024
+        CACHE_CONTROL_TYPE = 'ephemeral'
+        PROMPT_CACHE_OPTIONS = %i[ttl].freeze
+        COMPACTION_PLUGIN_ID = 'context-compression'
 
-          if schema
-            schema_name = schema[:name]
-            schema_def = RubyLLM::Utils.deep_dup(schema[:schema])
-            if schema_def.is_a?(Hash)
-              schema_def.delete(:strict)
-              schema_def.delete('strict')
-            end
-            strict = schema[:strict]
-            payload[:response_format] = {
-              type: 'json_schema',
-              json_schema: {
-                name: schema_name,
-                schema: schema_def,
-                strict: strict
-              }
-            }
-          end
-
-          reasoning = build_reasoning(thinking)
-          payload[:reasoning] = reasoning if reasoning
+        module_function
 
-          payload[:stream_options] = { include_usage: true } if stream
-          payload
+        def apply_end_user(payload, identifier)
+          payload.merge(user: identifier)
         end
-        # rubocop:enable Metrics/ParameterLists,Metrics/PerceivedComplexity
-
-        def parse_completion_response(response)
-          data = response.body
-          return if data.nil? || data.empty?
-
-          raise Error.new(response, data.dig('error', 'message')) if data.dig('error', 'message')
 
-          message_data = data.dig('choices', 0, 'message')
-          return unless message_data
-
-          usage = data['usage'] || {}
-          thinking_tokens = thinking_tokens(usage)
-          thinking_text = extract_thinking_text(message_data)
-          thinking_signature = extract_thinking_signature(message_data)
-
-          Message.new(
-            role: :assistant,
-            content: message_data['content'],
-            thinking: Thinking.build(text: thinking_text, signature: thinking_signature),
-            tool_calls: OpenAI::Tools.parse_tool_calls(message_data['tool_calls']),
-            input_tokens: input_tokens(usage),
-            output_tokens: output_tokens(usage),
-            cached_tokens: cache_read_tokens(usage),
-            cache_creation_tokens: cache_write_tokens(usage),
-            thinking_tokens: thinking_tokens,
-            model_id: data['model'],
-            raw: response
-          )
+        # OpenRouter compacts through its context-compression plugin, which
+        # drops messages from the middle of the conversation once the prompt
+        # would overflow the model's context window. It summarizes nothing
+        # and takes no threshold of its own, so the portable options have
+        # nowhere to go.
+        def apply_compaction(payload, compaction)
+          log_ignored_compaction_options(compaction)
+          payload.merge(plugins: Array(payload[:plugins]) + [{ id: COMPACTION_PLUGIN_ID }])
         end
 
-        def input_tokens(usage)
-          return usage['prompt_cache_miss_tokens'] if usage['prompt_cache_miss_tokens']
+        def log_ignored_compaction_options(compaction)
+          return if compaction.empty?
 
-          prompt_tokens = usage['prompt_tokens']
-          return unless prompt_tokens
-
-          [prompt_tokens.to_i - cache_read_tokens(usage).to_i - cache_write_tokens(usage).to_i, 0].max
-        end
-
-        def output_tokens(usage)
-          OpenAI::Chat.output_tokens(usage)
-        end
-
-        def cache_read_tokens(usage)
-          usage.dig('prompt_tokens_details', 'cached_tokens') || usage['prompt_cache_hit_tokens']
+          RubyLLM.logger.debug do
+            "#{@provider.name} compresses context at the model's own limit, dropping #{compaction.inspect}"
+          end
         end
 
-        def cache_write_tokens(usage)
-          usage.dig('prompt_tokens_details', 'cache_write_tokens') || 0
+        def format_content(content, attachments = [])
+          OpenRouter::Media.format_content(content, attachments)
         end
 
-        def thinking_tokens(usage)
-          OpenAI::Chat.thinking_tokens(usage)
-        end
+        def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil, schema: nil,
+                           thinking: nil, citations: false, caching: nil, tool_prefs: nil)
+          payload = super
+          payload.delete(:reasoning_effort)
+          strip_schema_strict(payload)
 
-        def format_messages(messages)
-          messages.map do |msg|
-            {
-              role: format_role(msg.role),
-              content: format_content(msg.content),
-              tool_calls: OpenAI::Tools.format_tool_calls(msg.tool_calls),
-              tool_call_id: msg.tool_call_id
-            }.compact.merge(format_thinking(msg))
-          end
+          reasoning = build_reasoning(thinking)
+          payload[:reasoning] = reasoning if reasoning
+          payload[:cache_control] = prompt_cache_control(caching) if caching && !cache_boundaries?(messages)
+          payload
         end
 
-        def format_content(content)
-          OpenAI::Media.format_content(content)
-        end
+        def strip_schema_strict(payload)
+          schema_def = payload.dig(:response_format, :json_schema, :schema)
+          return unless schema_def.is_a?(Hash)
 
-        def format_role(role)
-          case role
-          when :system
-            @config.openai_use_system_role ? 'system' : 'developer'
-          else
-            role.to_s
-          end
+          schema_def = RubyLLM::Support::Utils.deep_dup(schema_def)
+          schema_def.delete(:strict)
+          schema_def.delete('strict')
+          payload[:response_format][:json_schema][:schema] = schema_def
         end
 
         def build_reasoning(thinking)
           return nil unless thinking&.enabled?
 
           reasoning = {}
-          reasoning[:effort] = thinking.effort if thinking.respond_to?(:effort) && thinking.effort
+          reasoning[:effort] = thinking.effort.to_s if thinking.respond_to?(:effort) && thinking.effort
           reasoning[:max_tokens] = thinking.budget if thinking.respond_to?(:budget) && thinking.budget
+          add_reasoning_toggle(reasoning, thinking)
           reasoning[:enabled] = true if reasoning.empty?
           reasoning
         end
 
+        def add_reasoning_toggle(reasoning, thinking)
+          return unless thinking.respond_to?(:enabled) && !thinking.enabled.nil?
+
+          reasoning[:enabled] = thinking.enabled
+        end
+
         def format_thinking(msg)
+          return {} unless msg.role == :assistant
+          return { reasoning_details: msg.raw_reasoning } if msg.raw_reasoning.is_a?(Array)
+
           thinking = msg.thinking
-          return {} unless thinking && msg.role == :assistant
+          return {} unless thinking
 
           details = []
           if thinking.text
@@ -159,6 +102,78 @@ module RubyLLM
           details.empty? ? {} : { reasoning_details: details }
         end
 
+        def format_message_content(msg, caching: nil)
+          content = super
+          caching != false && msg.cache_until_here? ? inject_cache_control(content, caching:) : content
+        end
+
+        def inject_cache_control(content, caching: nil)
+          blocks = content.is_a?(Array) ? content.dup : [{ type: 'text', text: content }]
+          return blocks if blocks.empty?
+
+          last = blocks.last
+          return blocks unless last.is_a?(Hash)
+          return blocks if last[:cache_control] || last['cache_control']
+
+          blocks[-1] = last.merge(cache_control: prompt_cache_control(caching))
+          blocks
+        end
+
+        def prompt_cache_control(caching = nil)
+          options = prompt_cache_options(caching)
+
+          { type: CACHE_CONTROL_TYPE }.tap do |control|
+            control[:ttl] = options[:ttl] if options[:ttl]
+          end
+        end
+
+        def prompt_cache_options(caching)
+          return {} unless caching
+
+          options = caching.to_h.transform_keys(&:to_sym)
+          unsupported = options.keys - PROMPT_CACHE_OPTIONS
+          return options if unsupported.empty?
+
+          raise ArgumentError,
+                "OpenRouter prompt caching accepts :ttl, got #{format_cache_option_keys(unsupported)}"
+        end
+
+        def format_cache_option_keys(keys)
+          keys.map { |key| ":#{key}" }.join(', ')
+        end
+
+        def cache_boundaries?(messages)
+          messages.any?(&:cache_until_here?)
+        end
+
+        def openai_prompt_caching?
+          false
+        end
+
+        def reported_cost(usage)
+          cost = usage['cost']
+          return nil unless cost
+
+          cost += usage.dig('cost_details', 'upstream_inference_cost').to_f if usage['is_byok']
+          cost
+        end
+
+        def supports_provider_file_references?
+          true
+        end
+
+        def default_large_file_upload_threshold
+          OPENROUTER_INLINE_FILE_THRESHOLD
+        end
+
+        def provider_file_upload_limit
+          OPENROUTER_FILE_UPLOAD_LIMIT
+        end
+
+        def provider_file_attachable?(attachment)
+          attachment.pdf?
+        end
+
         def extract_thinking_text(message_data)
           candidate = message_data['reasoning']
           return candidate if candidate.is_a?(String)
@@ -190,6 +205,13 @@ module RubyLLM
           encrypted = details.find { |detail| detail['type'] == 'reasoning.encrypted' && detail['data'].is_a?(String) }
           encrypted&.dig('data')
         end
+
+        # OpenRouter requires the reasoning_details array back untouched for
+        # signed reasoning to survive multi-turn tool calls.
+        def extract_raw_reasoning(message_data)
+          details = message_data['reasoning_details']
+          details if details.is_a?(Array) && !details.empty?
+        end
       end
     end
   end

data/lib/ruby_llm/providers/openrouter/embeddings.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class OpenRouter
+      # Text and multimodal embedding requests for OpenRouter.
+      module Embeddings
+        EMBEDDING_MEDIA_TYPES = { audio: :input_audio, video: :input_video, pdf: :input_file }.freeze
+
+        module_function
+
+        def supports_embedding_media?
+          true
+        end
+
+        def render_embedding_payload(text, model:, dimensions:, task_type: nil, title: nil, with: [],
+                                     provider_options: {})
+          input = if with.any?
+                    raise ArgumentError, 'embed one text at a time when embedding attachments' if text.is_a?(Array)
+
+                    [{ content: format_embedding_content(text, with) }]
+                  else
+                    text
+                  end
+
+          payload = super(input, model:, dimensions:, task_type:, title:, provider_options: {})
+          payload[:input_type] = task_type if task_type
+          payload.merge(provider_options)
+        end
+
+        def format_embedding_content(text, attachments)
+          Protocols::ChatCompletions::Media.format_parts(text, attachments) do |attachment|
+            format_embedding_attachment(attachment)
+          end
+        end
+
+        def format_embedding_attachment(attachment)
+          raise UnsupportedAttachmentError, attachment.mime_type if attachment.provider_file?
+
+          if (type = EMBEDDING_MEDIA_TYPES[attachment.type])
+            { type: type.to_s, type => { data: attachment.for_llm, format: attachment.format } }
+          else
+            Protocols::ChatCompletions::Media.format_attachment(
+              attachment, document_attachments: :none, image_attachments: true, audio_attachments: false
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/openrouter/images.rb

@@ -4,64 +4,65 @@ module RubyLLM
   module Providers
     class OpenRouter
       # Image generation methods for the OpenRouter API integration.
-      # OpenRouter uses the chat completions endpoint for image generation
-      # instead of a dedicated images endpoint.
+      # OpenRouter has a unified images endpoint that generates and edits
+      # images across providers and reports the exact cost of each call.
       module Images
         module_function
 
         def images_url(with: nil, mask: nil) # rubocop:disable Lint/UnusedMethodArgument
-          'chat/completions'
+          'images'
         end
 
-        def render_image_payload(prompt, model:, size:, with: nil, mask: nil, params: {}) # rubocop:disable Lint/UnusedMethodArgument,Metrics/ParameterLists
-          RubyLLM.logger.debug { "Ignoring size #{size}. OpenRouter image generation does not support size parameter." }
-          {
-            model: model,
-            messages: [
-              {
-                role: 'user',
-                content: prompt
-              }
-            ],
-            modalities: %w[image text]
-          }
+        def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {}) # rubocop:disable Lint/UnusedMethodArgument
+          RubyLLM.logger.debug { "Ignoring size #{size}. Use aspect_ratio/resolution provider options instead." }
+          if count && count > 1
+            RubyLLM.logger.debug do
+              "Ignoring count #{count}. OpenRouter generates one image per request."
+            end
+          end
+          payload = { model: model, prompt: prompt }
+          references = build_input_references(with)
+          payload[:input_references] = references if references.any?
+          payload.merge(provider_options)
         end
 
-        def parse_image_response(response, model:)
-          data = response.body
-          message = data.dig('choices', 0, 'message')
+        def build_input_references(with)
+          Attachment.wrap(with, config: @config).map do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
 
-          unless message&.key?('images') && message['images']&.any?
-            raise Error.new(nil, 'Unexpected response format from OpenRouter image generation API')
+            Protocols::ChatCompletions::Media.format_image(attachment)
           end
+        end
 
-          image_data = message['images'].first
-          image_url = image_data.dig('image_url', 'url') || image_data['url']
-
-          raise Error.new(nil, 'No image URL found in OpenRouter response') unless image_url
+        def validate_paint_inputs!(with:, mask:) # rubocop:disable Lint/UnusedMethodArgument
+          raise UnsupportedAttachmentError, 'image mask' unless mask.nil?
+        end
 
-          build_image_from_url(image_url, model)
+        # OpenRouter's images endpoint returns one image per request.
+        def parse_image_responses(response, model:)
+          [parse_image_response(response, model:)]
         end
 
-        def build_image_from_url(image_url, model)
-          if image_url.start_with?('data:')
-            # Parse data URL format: data:image/png;base64,<data>
-            match = image_url.match(/^data:([^;]+);base64,(.+)$/)
-            raise Error.new(nil, 'Invalid data URL format from OpenRouter') unless match
+        def parse_image_response(response, model:)
+          data = response.body
+          image_data = Array(data['data']).first
 
-            Image.new(
-              data: match[2],
-              mime_type: match[1],
-              model_id: model
-            )
-          else
-            # Regular URL
-            Image.new(
-              url: image_url,
-              mime_type: 'image/png',
-              model_id: model
-            )
-          end
+          raise Error, 'Unexpected response format from OpenRouter image API' unless image_data
+
+          Image.new(
+            data: image_data['b64_json'],
+            mime_type: image_data['media_type'] || 'image/png',
+            model: model,
+            usage: image_usage(data['usage'] || {})
+          )
+        end
+
+        def image_usage(usage)
+          {
+            'input_tokens' => usage['prompt_tokens'],
+            'output_tokens' => usage['completion_tokens'],
+            'cost' => reported_cost(usage)
+          }.compact
         end
       end
     end

data/lib/ruby_llm/providers/openrouter/media.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class OpenRouter
+      # Handles media content for OpenRouter, which adds video input parts
+      # to the standard Chat Completions vocabulary.
+      module Media
+        module_function
+
+        def format_content(content, attachments = [])
+          Protocols::ChatCompletions::Media.format_parts(content, attachments) do |attachment|
+            if attachment.type == :video
+              format_video(attachment)
+            else
+              Protocols::ChatCompletions::Media.format_attachment(
+                attachment, document_attachments: :pdf, image_attachments: true, audio_attachments: true
+              )
+            end
+          end
+        end
+
+        def format_video(video)
+          {
+            type: 'video_url',
+            video_url: {
+              url: video.url? ? video.source.to_s : video.for_llm
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/providers/openrouter/models.rb

@@ -3,19 +3,56 @@
 module RubyLLM
   module Providers
     class OpenRouter
-      # Models methods of the OpenRouter API integration
+      # Models methods of the OpenRouter API integration. Chat models come
+      # from the main catalog; embedding, speech, transcription, and image
+      # models live in separate catalogs that are merged in.
       module Models
+        CATALOG_URLS = [
+          'models',
+          'embeddings/models',
+          'models?output_modalities=speech',
+          'models?output_modalities=transcription',
+          'models?output_modalities=rerank',
+          'images/models'
+        ].freeze
+
+        OUTPUT_MODALITY_MAP = {
+          'speech' => 'audio',
+          'transcription' => 'text'
+        }.freeze
+
+        CAPABILITY_BY_OUTPUT_MODALITY = {
+          'speech' => 'speech_generation',
+          'transcription' => 'transcription',
+          'image' => 'image_generation'
+        }.freeze
+
+        CAPABILITY_PARAMETERS = {
+          'function_calling' => %w[tools tool_choice],
+          'tool_choice' => %w[tool_choice],
+          'parallel_tool_calls' => %w[parallel_tool_calls],
+          'structured_output' => %w[response_format structured_outputs],
+          'batch' => %w[batch]
+        }.freeze
+
+        def list_models
+          CATALOG_URLS.flat_map do |url|
+            parse_list_models_response @connection.get(url), @provider.slug
+          end.uniq(&:id)
+        end
+
         module_function
 
         def models_url
           'models'
         end
 
-        def parse_list_models_response(response, slug, _capabilities)
+        def parse_list_models_response(response, slug)
           Array(response.body['data']).map do |model_data| # rubocop:disable Metrics/BlockLength
+            output_modalities = Array(model_data.dig('architecture', 'output_modalities'))
             modalities = {
               input: Array(model_data.dig('architecture', 'input_modalities')),
-              output: Array(model_data.dig('architecture', 'output_modalities'))
+              output: output_modalities.map { |modality| OUTPUT_MODALITY_MAP.fetch(modality, modality) }.uniq
             }
 
             pricing = { text_tokens: { standard: {} } }
@@ -24,6 +61,7 @@ module RubyLLM
               prompt: :input_per_million,
               completion: :output_per_million,
               input_cache_read: :cache_read_input_per_million,
+              input_cache_write: :cache_write_input_per_million,
               internal_reasoning: :reasoning_output_per_million
             }
 
@@ -32,9 +70,10 @@ module RubyLLM
               pricing[:text_tokens][:standard][target_key] = value * 1_000_000 if value.positive?
             end
 
-            capabilities = supported_parameters_to_capabilities(model_data['supported_parameters'])
+            capabilities = supported_parameters_to_capabilities(model_data['supported_parameters']) |
+                           output_modalities.filter_map { |modality| CAPABILITY_BY_OUTPUT_MODALITY[modality] }
 
-            Model::Info.new(
+            Model.new(
               id: model_data['id'],
               name: model_data['name'],
               provider: slug,
@@ -42,6 +81,7 @@ module RubyLLM
               created_at: model_data['created'] ? Time.at(model_data['created']) : nil,
               context_window: model_data['context_length'],
               max_output_tokens: model_data.dig('top_provider', 'max_completion_tokens'),
+              knowledge_cutoff: model_data['knowledge_cutoff'],
               modalities: modalities,
               capabilities: capabilities,
               pricing: pricing,
@@ -50,7 +90,8 @@ module RubyLLM
                 architecture: model_data['architecture'],
                 top_provider: model_data['top_provider'],
                 per_request_limits: model_data['per_request_limits'],
-                supported_parameters: model_data['supported_parameters']
+                supported_parameters: model_data['supported_parameters'],
+                expiration_date: model_data['expiration_date']
               }
             )
           end
@@ -59,11 +100,9 @@ module RubyLLM
         def supported_parameters_to_capabilities(params)
           return [] unless params
 
-          capabilities = []
-          capabilities << 'streaming'
-          capabilities << 'function_calling' if params.include?('tools') || params.include?('tool_choice')
-          capabilities << 'structured_output' if params.include?('response_format')
-          capabilities << 'batch' if params.include?('batch')
+          capabilities = ['streaming'] + CAPABILITY_PARAMETERS.filter_map do |capability, parameters|
+            capability if parameters.any? { |parameter| params.include?(parameter) }
+          end
           capabilities << 'predicted_outputs' if params.include?('logit_bias') && params.include?('top_k')
           capabilities
         end

data/lib/ruby_llm/providers/openrouter/speech.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Providers
+    class OpenRouter
+      # Speech generation methods for the OpenRouter API integration.
+      # Voices are model-specific across OpenRouter's TTS catalog, so no
+      # default voice is assumed.
+      module Speech
+        module_function
+
+        def render_speech_payload(input, model:, voice:, format:, provider_options: {})
+          {
+            model: model,
+            input: input,
+            voice: voice,
+            response_format: format || 'mp3'
+          }.compact.merge(provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice,
+            format: (format || 'mp3').to_s
+          )
+        end
+      end
+    end
+  end
+end