ruby_llm 1.15.0 → 2.0.0.rc1

data/lib/ruby_llm/protocols/chat_completions/images.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'stringio'
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Image generation methods for the OpenAI API integration
+      module Images
+        module_function
+
+        def images_url(with: nil, mask: nil)
+          editing?(with, mask) ? 'images/edits' : 'images/generations'
+        end
+
+        def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
+          return render_edit_payload(prompt, model:, size:, count:, with:, mask:, provider_options:) if editing?(with,
+                                                                                                                 mask)
+
+          {
+            model: model,
+            prompt: prompt,
+            n: count || 1,
+            size: size
+          }.merge(provider_options)
+        end
+
+        def parse_image_response(response, model:)
+          parse_image_responses(response, model:).first
+        end
+
+        def parse_image_responses(response, model:)
+          data = response.body
+          entries = Array(data['data'])
+
+          raise Error, 'Unexpected response format from OpenAI image API' if entries.empty?
+
+          entries.map.with_index do |image_data, index|
+            Image.new(
+              url: image_data['url'],
+              mime_type: 'image/png', # DALL-E typically returns PNGs
+              revised_prompt: image_data['revised_prompt'],
+              model: model,
+              data: image_data['b64_json'],
+              usage: index.zero? ? (data['usage'] || {}) : {}
+            )
+          end
+        end
+
+        def validate_paint_inputs!(with:, mask:)
+          return unless editing?(with, mask)
+
+          raise ArgumentError, 'with: is required when mask: is provided' if mask && !attachments?(with)
+        end
+
+        def render_edit_payload(prompt, model:, size:, with:, mask:, provider_options:, count: nil)
+          payload = { model: model, prompt: prompt, n: count || 1 }
+          if json_image_references?(model)
+            payload[:images] = build_image_references(with)
+            payload[:mask] = build_image_reference(mask) if mask
+            payload[:size] = size if size
+          else
+            payload[:image] = single_upload_part(build_upload_parts(with))
+            payload[:mask] = build_upload_part(mask) if mask
+          end
+          payload.merge(provider_options)
+        end
+
+        def json_image_references?(model)
+          model.match?(/\A(gpt-image|chatgpt-image)/)
+        end
+
+        def build_image_references(sources)
+          Array(sources).filter_map do |source|
+            next if blank_attachment?(source)
+
+            build_image_reference(source)
+          end
+        end
+
+        def build_image_reference(source)
+          attachment = Attachment.new(source, config: @config)
+          return { file_id: attachment.provider_file_id } if attachment.provider_file?
+          return { image_url: attachment.source.to_s } if attachment.url?
+
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          { image_url: attachment.for_llm }
+        end
+
+        def build_upload_parts(sources)
+          Array(sources).filter_map do |source|
+            next if blank_attachment?(source)
+
+            build_upload_part(source)
+          end
+        end
+
+        # Retries only rewind upload parts at the top level of the payload,
+        # so a lone image rides there instead of inside an array.
+        def single_upload_part(parts)
+          parts.one? ? parts.first : parts
+        end
+
+        def build_upload_part(source)
+          attachment = Attachment.new(source, config: @config)
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          Faraday::UploadIO.new(StringIO.new(attachment.content), attachment.mime_type, attachment.filename)
+        end
+
+        def editing?(with, mask)
+          attachments?(with) || !mask.nil?
+        end
+
+        def attachments?(value)
+          Array(value).any? { |item| !blank_attachment?(item) }
+        end
+
+        def blank_attachment?(value)
+          value.nil? || (value.is_a?(String) && value.strip.empty?)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/media.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Handles formatting of media content (images, audio) for OpenAI APIs
+      module Media
+        module_function
+
+        def format_content(content, attachments = [], document_attachments: :pdf, image_attachments: true,
+                           audio_attachments: true)
+          format_parts(content, attachments) do |attachment|
+            format_attachment(
+              attachment,
+              document_attachments:,
+              image_attachments:,
+              audio_attachments:
+            )
+          end
+        end
+
+        # Shared preamble and attachment loop for OpenAI-compatible providers.
+        # The block formats a single attachment in the provider's dialect.
+        def format_parts(content, attachments = [])
+          return content if attachments.empty?
+
+          parts = []
+          parts << format_text(content) if content
+
+          attachments.each do |attachment|
+            parts << yield(attachment)
+          end
+
+          parts
+        end
+
+        def format_attachment(attachment, document_attachments:, image_attachments:, audio_attachments:)
+          return format_provider_file(attachment, document_attachments:) if attachment.provider_file?
+
+          case attachment.type
+          when :image
+            raise UnsupportedAttachmentError, attachment.mime_type unless image_attachments
+
+            format_image(attachment)
+          when :audio
+            raise UnsupportedAttachmentError, attachment.mime_type unless audio_attachments
+
+            format_audio(attachment)
+          when :pdf, :document
+            format_document_attachment(attachment, document_attachments)
+          when :text
+            format_text_file(attachment)
+          else
+            raise UnsupportedAttachmentError, attachment.mime_type
+          end
+        end
+
+        def format_image(image)
+          {
+            type: 'image_url',
+            image_url: {
+              url: image.url? ? image.source.to_s : image.for_llm
+            }
+          }
+        end
+
+        def format_document(document)
+          {
+            type: 'file',
+            file: {
+              filename: document.filename,
+              file_data: document.for_llm
+            }
+          }
+        end
+
+        def format_provider_file(file, document_attachments:)
+          raise UnsupportedAttachmentError, file.mime_type if document_attachments == :none
+
+          {
+            type: 'file',
+            file: {
+              file_id: file.provider_file_id
+            }
+          }
+        end
+
+        def format_text_file(text_file)
+          {
+            type: 'text',
+            text: text_file.for_llm
+          }
+        end
+
+        def format_audio(audio)
+          {
+            type: 'input_audio',
+            input_audio: {
+              data: audio.encoded,
+              format: audio.format
+            }
+          }
+        end
+
+        def format_text(text)
+          {
+            type: 'text',
+            text: text
+          }
+        end
+
+        def format_document_attachment(attachment, strategy)
+          return format_document(attachment) if strategy == :all
+          return format_document(attachment) if strategy == :pdf && attachment.pdf?
+
+          raise UnsupportedAttachmentError, attachment.mime_type
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/models.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Models methods of the OpenAI API integration
+      module Models
+        module_function
+
+        def models_url
+          'models'
+        end
+
+        def parse_list_models_response(response, slug)
+          Array(response.body['data']).map do |model_data|
+            model_id = model_data['id']
+
+            Model.new(
+              id: model_id,
+              name: model_id,
+              provider: slug,
+              created_at: model_data['created'] ? Time.at(model_data['created']) : nil,
+              metadata: model_metadata(model_data)
+            )
+          end
+        end
+
+        def model_metadata(model_data)
+          metadata = {
+            object: model_data['object'],
+            owned_by: model_data['owned_by']
+          }
+          metadata[:shutdown_date] = model_data['shutdown_date'] if model_data['shutdown_date']
+          metadata
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/moderation.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Moderation methods of the OpenAI API integration
+      module Moderation
+        module_function
+
+        def moderation_url
+          'moderations'
+        end
+
+        def render_moderation_payload(input, model:, with: [], provider_options: {})
+          attachments = Attachment.wrap(with)
+
+          {
+            model: model,
+            input: moderation_input(input, attachments)
+          }.merge(provider_options)
+        end
+
+        def parse_moderation_response(response, model:)
+          data = response.body
+          raise Error.new(data.dig('error', 'message'), response:) if data.dig('error', 'message')
+
+          RubyLLM::Moderation.new(
+            id: data['id'],
+            model: model,
+            results: Array(data['results']).map { |result| RubyLLM::Moderation::Result.from_h(result) },
+            raw: data
+          )
+        end
+
+        def moderation_input(input, attachments)
+          return input if attachments.empty?
+
+          parts = []
+          parts << Media.format_text(input) if input
+          parts.concat(attachments.map { |attachment| format_moderation_attachment(attachment) })
+          parts
+        end
+
+        def format_moderation_attachment(attachment)
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          Media.format_image(attachment)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/rerank.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # The Jina-style rerank endpoint several OpenAI-compatible providers
+      # serve: OpenRouter at /api/v1/rerank and GPUStack at /v1/rerank,
+      # with identical request and response shapes.
+      module Rerank
+        module_function
+
+        def rerank_url
+          'rerank'
+        end
+
+        def render_rerank_payload(query, documents, model:, top_n: nil, provider_options: {})
+          {
+            model: model,
+            query: query,
+            documents: documents,
+            top_n: top_n
+          }.compact.merge(provider_options)
+        end
+
+        def parse_rerank_response(response, model:, documents: [])
+          data = response.body
+          usage = data['usage'] || {}
+
+          RubyLLM::Rerank.new(
+            results: parse_rerank_results(data, documents),
+            model: data['model'] || model,
+            raw: data,
+            input_tokens: usage['total_tokens'],
+            reported_cost: usage['cost']
+          )
+        end
+
+        def parse_rerank_results(data, documents = [])
+          Array(data['results']).map do |result|
+            index = result['index']
+
+            RubyLLM::Rerank::Result.new(
+              index: index,
+              document: rerank_document(result['document']) || (index && documents[index]),
+              score: result['relevance_score']
+            )
+          end
+        end
+
+        def rerank_document(document)
+          document.is_a?(Hash) ? document['text'] : document
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/speech.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Speech generation methods for the OpenAI API integration
+      module Speech
+        module_function
+
+        def speech_url(model:) # rubocop:disable Lint/UnusedMethodArgument
+          'audio/speech'
+        end
+
+        def stream_speech(payload, model:, voice:, format:, &)
+          stream_speech_response(speech_url(model:), payload, model:, voice:, format:, &)
+        end
+
+        def render_speech_payload(input, model:, voice:, format:, provider_options: {})
+          {
+            model: model,
+            input: input,
+            voice: voice || 'alloy',
+            response_format: format
+          }.compact.merge(provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          resolved_format = (format || 'mp3').to_s
+
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice || 'alloy',
+            format: resolved_format
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/streaming.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Streaming methods of the OpenAI API integration
+      module Streaming
+        module_function
+
+        def stream_url
+          completion_url
+        end
+
+        def build_chunk(data)
+          usage = data['usage'] || {}
+          delta = data.dig('choices', 0, 'delta') || {}
+          content_source = delta['content'] || data.dig('choices', 0, 'message', 'content')
+          content, thinking_from_blocks = extract_content_and_thinking(content_source)
+
+          Chunk.new(
+            role: :assistant,
+            model: data['model'],
+            content: content,
+            citations: extract_chunk_citations(delta, data),
+            thinking: Thinking.build(
+              text: thinking_from_blocks || delta['reasoning_content'] || delta['reasoning'],
+              signature: delta['reasoning_signature']
+            ),
+            tool_calls: parse_tool_calls(delta['tool_calls'], parse_arguments: false, stream_keys: true),
+            input_tokens: input_tokens(usage),
+            output_tokens: output_tokens(usage),
+            cache_read_tokens: cache_read_tokens(usage),
+            cache_write_tokens: cache_write_tokens(usage),
+            thinking_tokens: thinking_tokens(usage),
+            server_tool_use: server_tool_use(usage),
+            reported_cost: reported_cost(usage),
+            finish_reason: normalize_finish_reason(data.dig('choices', 0, 'finish_reason'))
+          )
+        end
+
+        def extract_chunk_citations(delta, data)
+          annotations = parse_annotations(delta['annotations'], nil)
+          return annotations if annotations.any?
+
+          parse_root_citations(data)
+        end
+
+        def parse_streaming_error(data)
+          error_data = JSON.parse(data)
+          return [nil, error_data.to_s] unless error_data.is_a?(Hash)
+
+          error = error_data['error']
+          return [nil, error.to_s] unless error.is_a?(Hash)
+
+          case error['type']
+          when 'server_error'
+            [500, error['message']]
+          when 'rate_limit_exceeded', 'insufficient_quota'
+            [429, error['message']]
+          else
+            [400, error['message']]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/{providers/openai → protocols/chat_completions}/tools.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require 'json'
+
 module RubyLLM
-  module Providers
-    class OpenAI
+  module Protocols
+    class ChatCompletions
       # Tools methods of the OpenAI API integration
       module Tools
         module_function
@@ -16,8 +18,8 @@ module RubyLLM
         }.freeze
 
         def parameters_schema_for(tool)
-          tool.params_schema ||
-            schema_from_parameters(tool.parameters)
+          tool.parameters_schema ||
+            schema_from_parameters(tool.declared_parameters)
         end
 
         def schema_from_parameters(parameters)
@@ -37,16 +39,9 @@ module RubyLLM
             }
           }
 
-          return definition if tool.provider_params.empty?
-
-          RubyLLM::Utils.deep_merge(definition, tool.provider_params)
-        end
+          return definition if tool.provider_options.empty?
 
-        def param_schema(param)
-          {
-            type: param.type,
-            description: param.description
-          }.compact
+          RubyLLM::Support::Utils.deep_merge(definition, tool.provider_options)
         end
 
         def format_tool_calls(tool_calls)
@@ -70,7 +65,7 @@ module RubyLLM
           end
         end
 
-        def parse_tool_call_arguments(tool_call)
+        def parse_tool_call_arguments(tool_call, response: nil, finish_reason: nil)
           arguments = tool_call.dig('function', 'arguments')
 
           if arguments.nil? || arguments.empty?
@@ -78,19 +73,24 @@ module RubyLLM
           else
             JSON.parse(arguments)
           end
+        rescue JSON::ParserError => e
+          raise ToolCallParseError.new(response: response, finish_reason: finish_reason), cause: e
         end
 
-        def parse_tool_calls(tool_calls, parse_arguments: true)
+        # Streaming deltas key by index so id-less argument fragments find
+        # their call even when a delta batches or interleaves several calls;
+        # complete responses key by id, which tool results join on.
+        def parse_tool_calls(tool_calls, parse_arguments: true, response: nil, finish_reason: nil, stream_keys: false)
           return nil unless tool_calls&.any?
 
           tool_calls.to_h do |tc|
             [
-              tc['id'],
+              stream_keys ? tc['index'] || tc['id'] : tc['id'],
               ToolCall.new(
                 id: tc['id'],
                 name: tc.dig('function', 'name'),
                 arguments: if parse_arguments
-                             parse_tool_call_arguments(tc)
+                             parse_tool_call_arguments(tc, response: response, finish_reason: finish_reason)
                            else
                              tc.dig('function', 'arguments')
                            end,