ruby_llm 1.16.0 → 2.0.0.rc2

data/lib/ruby_llm/protocols/chat_completions/images.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'stringio'
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Image generation methods for the OpenAI API integration
+      module Images
+        module_function
+
+        def images_url(with: nil, mask: nil)
+          editing?(with, mask) ? 'images/edits' : 'images/generations'
+        end
+
+        def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
+          return render_edit_payload(prompt, model:, size:, count:, with:, mask:, provider_options:) if editing?(with,
+                                                                                                                 mask)
+
+          {
+            model: model,
+            prompt: prompt,
+            n: count || 1,
+            size: size
+          }.merge(provider_options)
+        end
+
+        def parse_image_response(response, model:)
+          parse_image_responses(response, model:).first
+        end
+
+        def parse_image_responses(response, model:)
+          data = response.body
+          entries = Array(data['data'])
+
+          raise Error, 'Unexpected response format from OpenAI image API' if entries.empty?
+
+          entries.map.with_index do |image_data, index|
+            Image.new(
+              url: image_data['url'],
+              mime_type: 'image/png', # DALL-E typically returns PNGs
+              revised_prompt: image_data['revised_prompt'],
+              model: model,
+              data: image_data['b64_json'],
+              usage: index.zero? ? (data['usage'] || {}) : {}
+            )
+          end
+        end
+
+        def validate_paint_inputs!(with:, mask:)
+          return unless editing?(with, mask)
+
+          raise ArgumentError, 'with: is required when mask: is provided' if mask && !attachments?(with)
+        end
+
+        def render_edit_payload(prompt, model:, size:, with:, mask:, provider_options:, count: nil)
+          payload = { model: model, prompt: prompt, n: count || 1 }
+          if json_image_references?(model)
+            payload[:images] = build_image_references(with)
+            payload[:mask] = build_image_reference(mask) if mask
+            payload[:size] = size if size
+          else
+            payload[:image] = single_upload_part(build_upload_parts(with))
+            payload[:mask] = build_upload_part(mask) if mask
+          end
+          payload.merge(provider_options)
+        end
+
+        def json_image_references?(model)
+          model.match?(/\A(gpt-image|chatgpt-image)/)
+        end
+
+        def build_image_references(sources)
+          Array(sources).filter_map do |source|
+            next if blank_attachment?(source)
+
+            build_image_reference(source)
+          end
+        end
+
+        def build_image_reference(source)
+          attachment = Attachment.new(source, config: @config)
+          return { file_id: attachment.provider_file_id } if attachment.provider_file?
+          return { image_url: attachment.source.to_s } if attachment.url?
+
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          { image_url: attachment.for_llm }
+        end
+
+        def build_upload_parts(sources)
+          Array(sources).filter_map do |source|
+            next if blank_attachment?(source)
+
+            build_upload_part(source)
+          end
+        end
+
+        # Retries only rewind upload parts at the top level of the payload,
+        # so a lone image rides there instead of inside an array.
+        def single_upload_part(parts)
+          parts.one? ? parts.first : parts
+        end
+
+        def build_upload_part(source)
+          attachment = Attachment.new(source, config: @config)
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          Faraday::UploadIO.new(StringIO.new(attachment.content), attachment.mime_type, attachment.filename)
+        end
+
+        def editing?(with, mask)
+          attachments?(with) || !mask.nil?
+        end
+
+        def attachments?(value)
+          Array(value).any? { |item| !blank_attachment?(item) }
+        end
+
+        def blank_attachment?(value)
+          value.nil? || (value.is_a?(String) && value.strip.empty?)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/{providers/openai → protocols/chat_completions}/media.rb

@@ -1,36 +1,42 @@
 # frozen_string_literal: true
 
 module RubyLLM
-  module Providers
-    class OpenAI
+  module Protocols
+    class ChatCompletions
       # Handles formatting of media content (images, audio) for OpenAI APIs
       module Media
         module_function
 
-        def format_content(content, document_attachments: :pdf, image_attachments: true, audio_attachments: true)
-          if content.is_a?(RubyLLM::Content::Raw)
-            value = content.value
-            return value.is_a?(Hash) ? value.to_json : value
-          end
-          return content.to_json if content.is_a?(Hash) || content.is_a?(Array)
-          return content unless content.is_a?(Content)
-
-          parts = []
-          parts << format_text(content.text) if content.text
-
-          content.attachments.each do |attachment|
-            parts << format_attachment(
+        def format_content(content, attachments = [], document_attachments: :pdf, image_attachments: true,
+                           audio_attachments: true)
+          format_parts(content, attachments) do |attachment|
+            format_attachment(
               attachment,
               document_attachments:,
               image_attachments:,
               audio_attachments:
             )
           end
+        end
+
+        # Shared preamble and attachment loop for OpenAI-compatible providers.
+        # The block formats a single attachment in the provider's dialect.
+        def format_parts(content, attachments = [])
+          return content if attachments.empty?
+
+          parts = []
+          parts << format_text(content) if content
+
+          attachments.each do |attachment|
+            parts << yield(attachment)
+          end
 
           parts
         end
 
         def format_attachment(attachment, document_attachments:, image_attachments:, audio_attachments:)
+          return format_provider_file(attachment, document_attachments:) if attachment.provider_file?
+
           case attachment.type
           when :image
             raise UnsupportedAttachmentError, attachment.mime_type unless image_attachments
@@ -68,8 +74,15 @@ module RubyLLM
           }
         end
 
-        def format_pdf(pdf)
-          format_document(pdf)
+        def format_provider_file(file, document_attachments:)
+          raise UnsupportedAttachmentError, file.mime_type if document_attachments == :none
+
+          {
+            type: 'file',
+            file: {
+              file_id: file.provider_file_id
+            }
+          }
         end
 
         def format_text_file(text_file)

data/lib/ruby_llm/protocols/chat_completions/models.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Models methods of the OpenAI API integration
+      module Models
+        module_function
+
+        def models_url
+          'models'
+        end
+
+        def parse_list_models_response(response, slug)
+          Array(response.body['data']).map do |model_data|
+            model_id = model_data['id']
+
+            Model.new(
+              id: model_id,
+              name: model_id,
+              provider: slug,
+              created_at: model_data['created'] ? Time.at(model_data['created']) : nil,
+              metadata: model_metadata(model_data)
+            )
+          end
+        end
+
+        def model_metadata(model_data)
+          metadata = {
+            object: model_data['object'],
+            owned_by: model_data['owned_by']
+          }
+          metadata[:shutdown_date] = model_data['shutdown_date'] if model_data['shutdown_date']
+          metadata
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/moderation.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Moderation methods of the OpenAI API integration
+      module Moderation
+        module_function
+
+        def moderation_url
+          'moderations'
+        end
+
+        def render_moderation_payload(input, model:, with: [], provider_options: {})
+          attachments = Attachment.wrap(with)
+
+          {
+            model: model,
+            input: moderation_input(input, attachments)
+          }.merge(provider_options)
+        end
+
+        def parse_moderation_response(response, model:)
+          data = response.body
+          raise Error.new(data.dig('error', 'message'), response:) if data.dig('error', 'message')
+
+          RubyLLM::Moderation.new(
+            id: data['id'],
+            model: model,
+            results: Array(data['results']).map { |result| RubyLLM::Moderation::Result.from_h(result) },
+            raw: data
+          )
+        end
+
+        def moderation_input(input, attachments)
+          return input if attachments.empty?
+
+          parts = []
+          parts << Media.format_text(input) if input
+          parts.concat(attachments.map { |attachment| format_moderation_attachment(attachment) })
+          parts
+        end
+
+        def format_moderation_attachment(attachment)
+          raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
+
+          Media.format_image(attachment)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/rerank.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # The Jina-style rerank endpoint several OpenAI-compatible providers
+      # serve: OpenRouter at /api/v1/rerank and GPUStack at /v1/rerank,
+      # with identical request and response shapes.
+      module Rerank
+        module_function
+
+        def rerank_url
+          'rerank'
+        end
+
+        def render_rerank_payload(query, documents, model:, top_n: nil, provider_options: {})
+          {
+            model: model,
+            query: query,
+            documents: documents,
+            top_n: top_n
+          }.compact.merge(provider_options)
+        end
+
+        def parse_rerank_response(response, model:, documents: [])
+          data = response.body
+          usage = data['usage'] || {}
+
+          RubyLLM::Rerank.new(
+            results: parse_rerank_results(data, documents),
+            model: data['model'] || model,
+            raw: data,
+            input_tokens: usage['total_tokens'],
+            reported_cost: usage['cost']
+          )
+        end
+
+        def parse_rerank_results(data, documents = [])
+          Array(data['results']).map do |result|
+            index = result['index']
+
+            RubyLLM::Rerank::Result.new(
+              index: index,
+              document: rerank_document(result['document']) || (index && documents[index]),
+              score: result['relevance_score']
+            )
+          end
+        end
+
+        def rerank_document(document)
+          document.is_a?(Hash) ? document['text'] : document
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/speech.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Speech generation methods for the OpenAI API integration
+      module Speech
+        module_function
+
+        def speech_url(model:) # rubocop:disable Lint/UnusedMethodArgument
+          'audio/speech'
+        end
+
+        def stream_speech(payload, model:, voice:, format:, &)
+          stream_speech_response(speech_url(model:), payload, model:, voice:, format:, &)
+        end
+
+        def render_speech_payload(input, model:, voice:, format:, provider_options: {})
+          {
+            model: model,
+            input: input,
+            voice: voice || 'alloy',
+            response_format: format
+          }.compact.merge(provider_options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          resolved_format = (format || 'mp3').to_s
+
+          RubyLLM::Speech.new(
+            data: response.body,
+            model: model,
+            voice: voice || 'alloy',
+            format: resolved_format
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/streaming.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Streaming methods of the OpenAI API integration
+      module Streaming
+        module_function
+
+        def stream_url
+          completion_url
+        end
+
+        def build_chunk(data)
+          usage = data['usage'] || {}
+          delta = data.dig('choices', 0, 'delta') || {}
+          content_source = delta['content'] || data.dig('choices', 0, 'message', 'content')
+          content, thinking_from_blocks = extract_content_and_thinking(content_source)
+
+          Chunk.new(
+            role: :assistant,
+            model: data['model'],
+            content: content,
+            citations: extract_chunk_citations(delta, data),
+            thinking: Thinking.build(
+              text: thinking_from_blocks || delta['reasoning_content'] || delta['reasoning'],
+              signature: delta['reasoning_signature']
+            ),
+            tool_calls: parse_tool_calls(delta['tool_calls'], parse_arguments: false, stream_keys: true),
+            input_tokens: input_tokens(usage),
+            output_tokens: output_tokens(usage),
+            cache_read_tokens: cache_read_tokens(usage),
+            cache_write_tokens: cache_write_tokens(usage),
+            thinking_tokens: thinking_tokens(usage),
+            server_tool_use: server_tool_use(usage),
+            reported_cost: reported_cost(usage),
+            finish_reason: normalize_finish_reason(data.dig('choices', 0, 'finish_reason'))
+          )
+        end
+
+        def extract_chunk_citations(delta, data)
+          annotations = parse_annotations(delta['annotations'], nil)
+          return annotations if annotations.any?
+
+          parse_root_citations(data)
+        end
+
+        def parse_streaming_error(data)
+          error_data = JSON.parse(data)
+          return [nil, error_data.to_s] unless error_data.is_a?(Hash)
+
+          error = error_data['error']
+          return [nil, error.to_s] unless error.is_a?(Hash)
+
+          case error['type']
+          when 'server_error'
+            [500, error['message']]
+          when 'rate_limit_exceeded', 'insufficient_quota'
+            [429, error['message']]
+          else
+            [400, error['message']]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/{providers/openai → protocols/chat_completions}/tools.rb

@@ -3,8 +3,8 @@
 require 'json'
 
 module RubyLLM
-  module Providers
-    class OpenAI
+  module Protocols
+    class ChatCompletions
       # Tools methods of the OpenAI API integration
       module Tools
         module_function
@@ -18,8 +18,8 @@ module RubyLLM
         }.freeze
 
         def parameters_schema_for(tool)
-          tool.params_schema ||
-            schema_from_parameters(tool.parameters)
+          tool.parameters_schema ||
+            schema_from_parameters(tool.declared_parameters)
         end
 
         def schema_from_parameters(parameters)
@@ -39,16 +39,9 @@ module RubyLLM
             }
           }
 
-          return definition if tool.provider_params.empty?
+          return definition if tool.provider_options.empty?
 
-          RubyLLM::Utils.deep_merge(definition, tool.provider_params)
-        end
-
-        def param_schema(param)
-          {
-            type: param.type,
-            description: param.description
-          }.compact
+          RubyLLM::Support::Utils.deep_merge(definition, tool.provider_options)
         end
 
         def format_tool_calls(tool_calls)
@@ -72,7 +65,7 @@ module RubyLLM
           end
         end
 
-        def parse_tool_call_arguments(tool_call)
+        def parse_tool_call_arguments(tool_call, response: nil, finish_reason: nil)
           arguments = tool_call.dig('function', 'arguments')
 
           if arguments.nil? || arguments.empty?
@@ -80,19 +73,24 @@ module RubyLLM
           else
             JSON.parse(arguments)
           end
+        rescue JSON::ParserError => e
+          raise ToolCallParseError.new(response: response, finish_reason: finish_reason), cause: e
         end
 
-        def parse_tool_calls(tool_calls, parse_arguments: true)
+        # Streaming deltas key by index so id-less argument fragments find
+        # their call even when a delta batches or interleaves several calls;
+        # complete responses key by id, which tool results join on.
+        def parse_tool_calls(tool_calls, parse_arguments: true, response: nil, finish_reason: nil, stream_keys: false)
           return nil unless tool_calls&.any?
 
           tool_calls.to_h do |tc|
             [
-              tc['id'],
+              stream_keys ? tc['index'] || tc['id'] : tc['id'],
               ToolCall.new(
                 id: tc['id'],
                 name: tc.dig('function', 'name'),
                 arguments: if parse_arguments
-                             parse_tool_call_arguments(tc)
+                             parse_tool_call_arguments(tc, response: response, finish_reason: finish_reason)
                            else
                              tc.dig('function', 'arguments')
                            end,

data/lib/ruby_llm/protocols/chat_completions/transcription.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Audio transcription methods for the OpenAI API integration
+      module Transcription
+        module_function
+
+        def transcription_url
+          'audio/transcriptions'
+        end
+
+        def render_transcription_options(timestamps:, format:, streaming:)
+          return {} if timestamps.nil?
+
+          values = Array(timestamps).map(&:to_s)
+          unless values.any? && (values - %w[word segment]).empty?
+            raise ArgumentError, 'Transcription timestamps must be word or segment'
+          end
+          if streaming || (format && format != 'verbose_json')
+            raise ArgumentError, 'Transcription timestamps require a non-streaming verbose_json response'
+          end
+
+          { response_format: 'verbose_json', timestamp_granularities: values }
+        end
+
+        def render_transcription_payload(file_part, model:, language:, format: nil, speaker_names: nil,
+                                         speaker_references: nil, provider_options: {}, prompt: nil,
+                                         temperature: nil)
+          {
+            model: model,
+            file: file_part,
+            language: language,
+            response_format: format || default_response_format(model),
+            prompt: prompt,
+            temperature: temperature,
+            known_speaker_names: speaker_names,
+            known_speaker_references: encode_speaker_references(speaker_references)
+          }.compact.merge(provider_options)
+        end
+
+        def encode_speaker_references(references)
+          return nil unless references
+
+          references.map do |ref|
+            Attachment.new(ref, config: @config).for_llm
+          end
+        end
+
+        def reported_cost(_usage)
+          nil
+        end
+
+        # Diarization models return plain text with no segments unless the
+        # response format asks for them.
+        def default_response_format(model)
+          'diarized_json' if model.include?('diarize')
+        end
+
+        # OpenAI streams transcriptions as server-sent events carrying text
+        # deltas, completed segments on diarization models, and a final
+        # event with the whole transcript and its usage.
+        def stream_transcription(payload, model:, &block)
+          chunks = []
+
+          stream_events(transcription_url, payload.merge(stream: 'true')) do |data|
+            chunk = build_transcription_chunk(data)
+            chunks << chunk
+            block.call chunk
+          end
+
+          build_streamed_transcription(chunks, model: model)
+        end
+
+        def build_transcription_chunk(data)
+          type = data['type']
+
+          RubyLLM::TranscriptionChunk.new(
+            type: type,
+            delta: data['delta'],
+            text: (data['text'] if type == RubyLLM::TranscriptionChunk::DONE),
+            segment: (data.except('type') if type == RubyLLM::TranscriptionChunk::SEGMENT),
+            raw: data
+          )
+        end
+
+        def build_streamed_transcription(chunks, model:)
+          final = chunks.reverse.find(&:done?)
+          data = final&.raw || {}
+          usage = data['usage'] || {}
+
+          RubyLLM::Transcription.new(
+            text: final&.text || streamed_transcript_text(chunks),
+            model: model,
+            language: data['language'],
+            duration: transcription_duration(usage),
+            segments: streamed_transcription_segments(chunks, data),
+            reported_cost: reported_cost(usage),
+            **transcription_tokens(usage)
+          )
+        end
+
+        # Diarization models stream segments instead of deltas, so the
+        # transcript is rebuilt from whichever the provider sent.
+        def streamed_transcript_text(chunks)
+          deltas = chunks.filter_map(&:delta)
+          return deltas.join if deltas.any?
+
+          chunks.filter_map { |chunk| chunk.segment&.fetch('text', nil) }.join(' ')
+        end
+
+        def streamed_transcription_segments(chunks, data)
+          segments = data['segments'] || chunks.filter_map(&:segment)
+          segments.empty? ? nil : segments
+        end
+
+        def parse_transcription_response(response, model:)
+          data = response.body
+
+          return RubyLLM::Transcription.new(text: data, model: model) if data.is_a?(String)
+
+          usage = data['usage'] || {}
+
+          RubyLLM::Transcription.new(
+            text: data['text'],
+            model: model,
+            language: data['language'],
+            duration: data['duration'] || transcription_duration(usage),
+            segments: data['segments'],
+            words: data['words'],
+            reported_cost: reported_cost(usage),
+            **transcription_tokens(usage)
+          )
+        end
+
+        def transcription_tokens(usage)
+          {
+            input_tokens: usage['input_tokens'] || usage['prompt_tokens'],
+            output_tokens: usage['output_tokens'] || usage['completion_tokens']
+          }
+        end
+
+        def transcription_duration(usage)
+          usage['seconds']
+        end
+      end
+    end
+  end
+end