ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/protocols/chat_completions/content_filters.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Content filter verdicts as Azure reports them beside a completion:
+      # prompt_filter_results and each choice's content_filter_results on
+      # Chat Completions, a content_filters list on the Responses API.
+      # Each category is an object with filtered and either a severity or
+      # detected.
+      module ContentFilters
+        module_function
+
+        def parse_content_filters(data)
+          prompt = Array(data['prompt_filter_results']).flat_map do |result|
+            parse_filter_results(result['content_filter_results'], :prompt)
+          end
+          completion = parse_filter_results(data.dig('choices', 0, 'content_filter_results'), :completion)
+          listed = Array(data['content_filters']).flat_map do |result|
+            parse_filter_results(result['content_filter_results'], result['source_type'])
+          end
+          prompt + completion + listed
+        end
+
+        def parse_filter_results(results, source)
+          return [] unless results.is_a?(Hash) && ContentFilter::SOURCES.include?(source&.to_sym)
+
+          results.flat_map do |category, verdicts|
+            (verdicts.is_a?(Array) ? verdicts : [verdicts]).filter_map do |verdict|
+              parse_verdict(verdict, category, source)
+            end
+          end
+        end
+
+        # Custom blocklists come as a list, one verdict for each blocklist.
+        def parse_verdict(verdict, category, source)
+          return unless verdict.is_a?(Hash) && verdict.key?('filtered')
+
+          ContentFilter.new(source:, category:, severity: verdict['severity'], filtered: verdict['filtered'],
+                            detected: verdict['detected'])
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/chat_completions/embedding_batches.rb

@@ -26,29 +26,29 @@ module RubyLLM
           payload.key?(:input) || payload.key?('input')
         end
 
-        def parse_batch_completion_response(body)
-          parse_embedding_response(Response.new(body), model: body['model'], text: nil)
+        def batch_request_line(request)
+          line = super
+          line[:custom_id] = "#{line[:custom_id]}:array" if request[:text].is_a?(Array)
+          line
+        end
+
+        def parse_batch_completion_response(body, shape: nil)
+          parse_embedding_response(Response.new(body), model: body['model'], text: shape == 'array' ? [] : nil)
         end
 
         def parse_batch_result(line)
+          custom_id, shape = line['custom_id'].split(':', 2)
+          line = line.merge('custom_id' => custom_id)
           body = line.dig('response', 'body')
           return super unless body.is_a?(Hash) && body['data'].is_a?(Array)
+
+          index = batch_result_index(custom_id)
           unless valid_embedding_positions?(body['data'])
-            return [batch_result_index(line['custom_id']), nil,
-                    batch_failure(line['custom_id'], 'Invalid or duplicate embedding record positions')]
+            return [index, nil, batch_failure(custom_id, 'Invalid or duplicate embedding record positions')]
           end
+          return super unless batch_response_succeeded?(line['response'])
 
-          ordered = body.merge('data' => body['data'].sort_by { |row| row['index'] })
-          super(line.merge('response' => line['response'].merge('body' => ordered)))
-        end
-
-        # A row's index names the input its vector belongs to, so rows are put
-        # back in that order and positions that are not exactly 0...N fail the
-        # request rather than pairing vectors with the wrong texts.
-        def valid_embedding_positions?(rows)
-          positions = rows.map { |row| row.is_a?(Hash) ? row['index'] : nil }
-
-          positions.all?(Integer) && positions.sort == (0...positions.size).to_a
+          parsed_batch_result(index, custom_id) { parse_batch_completion_response(body, shape:) }
         end
       end
     end

data/lib/ruby_llm/protocols/chat_completions/embeddings.rb

@@ -5,14 +5,17 @@ module RubyLLM
     class ChatCompletions
       # Embeddings methods of the OpenAI API integration
       module Embeddings
+        def rejected_options(operation)
+          operation == :embed ? %i[task_type title] : super
+        end
+
         module_function
 
         def embedding_url(...)
           'embeddings'
         end
 
-        # rubocop:disable-next Lint/UnusedMethodArgument
-        def render_embedding_payload(text, model:, dimensions:, task_type: nil, title: nil, provider_options: {})
+        def render_embedding_payload(text, model:, dimensions:, provider_options: {}, **)
           {
             model: model,
             input: text,
@@ -22,8 +25,11 @@ module RubyLLM
 
         def parse_embedding_response(response, model:, text:)
           data = response.body
+          raise_stream_error(JSON.generate(data), data, nil) if data.dig('error', 'message')
+          raise Error.new('The provider returned no embeddings', response:) unless data['data'].is_a?(Array)
+
           input_tokens = data.dig('usage', 'prompt_tokens')
-          rows = data['data']
+          rows = order_embedding_rows(data['data'])
           single = rows.length == 1 && !text.is_a?(Array)
 
           vectors = rows.map { |row| row['embedding'] }
@@ -34,6 +40,23 @@ module RubyLLM
                         reported_cost: reported_cost(data['usage'] || {}))
         end
 
+        # A row's index names the input its vector belongs to, so rows are put
+        # back in that order and positions that are not exactly 0...N raise
+        # rather than pairing vectors with the wrong texts. Servers that send
+        # no index keep the order of the response.
+        def order_embedding_rows(rows)
+          return rows if rows.none? { |row| row.key?('index') }
+          raise Error, 'Invalid or duplicate embedding record positions' unless valid_embedding_positions?(rows)
+
+          rows.sort_by { |row| row['index'] }
+        end
+
+        def valid_embedding_positions?(rows)
+          positions = rows.map { |row| row.is_a?(Hash) ? row['index'] : nil }
+
+          positions.all?(Integer) && positions.sort == (0...positions.size).to_a
+        end
+
         # Sparse-capable models return a token-to-weight map beside the dense
         # vector, under lexical_weights on BGE-M3 and sparse_embedding
         # elsewhere. It is an extension: dense-only servers send neither, and

data/lib/ruby_llm/protocols/chat_completions/images.rb

@@ -39,7 +39,7 @@ module RubyLLM
           entries.map.with_index do |image_data, index|
             Image.new(
               url: image_data['url'],
-              mime_type: 'image/png', # DALL-E typically returns PNGs
+              mime_type: "image/#{data['output_format'] || 'png'}",
               revised_prompt: image_data['revised_prompt'],
               model: model,
               data: image_data['b64_json'],
@@ -59,11 +59,11 @@ module RubyLLM
           if json_image_references?(model)
             payload[:images] = build_image_references(with)
             payload[:mask] = build_image_reference(mask) if mask
-            payload[:size] = size if size
           else
             payload[:image] = single_upload_part(build_upload_parts(with))
             payload[:mask] = build_upload_part(mask) if mask
           end
+          payload[:size] = size if size
           payload.merge(provider_options)
         end
 
@@ -72,7 +72,7 @@ module RubyLLM
         end
 
         def build_image_references(sources)
-          Array(sources).filter_map do |source|
+          Support::Utils.to_safe_array(sources).filter_map do |source|
             next if blank_attachment?(source)
 
             build_image_reference(source)
@@ -80,7 +80,7 @@ module RubyLLM
         end
 
         def build_image_reference(source)
-          attachment = Attachment.new(source, config: @config)
+          attachment = source.is_a?(Attachment) ? source : Attachment.new(source, config: @config)
           return { file_id: attachment.provider_file_id } if attachment.provider_file?
           return { image_url: attachment.source.to_s } if attachment.url?
 
@@ -90,21 +90,19 @@ module RubyLLM
         end
 
         def build_upload_parts(sources)
-          Array(sources).filter_map do |source|
+          Support::Utils.to_safe_array(sources).filter_map do |source|
             next if blank_attachment?(source)
 
             build_upload_part(source)
           end
         end
 
-        # Retries only rewind upload parts at the top level of the payload,
-        # so a lone image rides there instead of inside an array.
         def single_upload_part(parts)
           parts.one? ? parts.first : parts
         end
 
         def build_upload_part(source)
-          attachment = Attachment.new(source, config: @config)
+          attachment = source.is_a?(Attachment) ? source : Attachment.new(source, config: @config)
           raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
 
           Faraday::UploadIO.new(StringIO.new(attachment.content), attachment.mime_type, attachment.filename)
@@ -115,7 +113,7 @@ module RubyLLM
         end
 
         def attachments?(value)
-          Array(value).any? { |item| !blank_attachment?(item) }
+          Support::Utils.to_safe_array(value).any? { |item| !blank_attachment?(item) }
         end
 
         def blank_attachment?(value)

data/lib/ruby_llm/protocols/chat_completions/media.rb

@@ -21,10 +21,10 @@ module RubyLLM
           end
         end
 
-        def format_content(content, attachments = [], document_attachments: :pdf, image_attachments: true,
+        def render_content(content, attachments = [], document_attachments: :pdf, image_attachments: true,
                            audio_attachments: true)
-          format_parts(content, attachments) do |attachment|
-            format_attachment(
+          render_parts(content, attachments) do |attachment|
+            render_attachment(
               attachment,
               document_attachments:,
               image_attachments:,
@@ -35,11 +35,11 @@ module RubyLLM
 
         # Shared preamble and attachment loop for OpenAI-compatible providers.
         # The block formats a single attachment in the provider's dialect.
-        def format_parts(content, attachments = [])
+        def render_parts(content, attachments = [])
           return content if attachments.empty?
 
           parts = []
-          parts << format_text(content) if content
+          parts << render_text(content) if content
 
           attachments.each do |attachment|
             parts << yield(attachment)
@@ -48,26 +48,26 @@ module RubyLLM
           parts
         end
 
-        def format_attachment(attachment, document_attachments:, image_attachments:, audio_attachments:)
+        def render_attachment(attachment, document_attachments:, image_attachments:, audio_attachments:)
           unless supported_attachment?(attachment, document_attachments:, image_attachments:, audio_attachments:)
             raise UnsupportedAttachmentError, attachment.mime_type
           end
 
-          return format_provider_file(attachment, document_attachments:) if attachment.provider_file?
+          return render_provider_file(attachment, document_attachments:) if attachment.provider_file?
 
           case attachment.type
           when :image
-            with_image_detail(format_image(attachment), attachment)
+            with_image_detail(render_image(attachment), attachment)
           when :audio
-            format_audio(attachment)
+            render_audio(attachment)
           when :pdf, :document
-            format_document_attachment(attachment, document_attachments)
+            render_document_attachment(attachment, document_attachments)
           when :text
-            format_text_file(attachment)
+            render_text_file(attachment)
           end
         end
 
-        def format_image(image)
+        def render_image(image)
           {
             type: 'image_url',
             image_url: {
@@ -90,7 +90,7 @@ module RubyLLM
           'high'
         end
 
-        def format_document(document)
+        def render_document(document)
           {
             type: 'file',
             file: {
@@ -100,7 +100,7 @@ module RubyLLM
           }
         end
 
-        def format_provider_file(file, document_attachments:)
+        def render_provider_file(file, document_attachments:)
           raise UnsupportedAttachmentError, file.mime_type if document_attachments == :none
 
           {
@@ -111,14 +111,14 @@ module RubyLLM
           }
         end
 
-        def format_text_file(text_file)
+        def render_text_file(text_file)
           {
             type: 'text',
             text: text_file.for_llm
           }
         end
 
-        def format_audio(audio)
+        def render_audio(audio)
           {
             type: 'input_audio',
             input_audio: {
@@ -128,16 +128,16 @@ module RubyLLM
           }
         end
 
-        def format_text(text)
+        def render_text(text)
           {
             type: 'text',
             text: text
           }
         end
 
-        def format_document_attachment(attachment, strategy)
-          return format_document(attachment) if strategy == :all
-          return format_document(attachment) if strategy == :pdf && attachment.pdf?
+        def render_document_attachment(attachment, strategy)
+          return render_document(attachment) if strategy == :all
+          return render_document(attachment) if strategy == :pdf && attachment.pdf?
 
           raise UnsupportedAttachmentError, attachment.mime_type
         end

data/lib/ruby_llm/protocols/chat_completions/moderation.rb

@@ -22,7 +22,10 @@ module RubyLLM
 
         def parse_moderation_response(response, model:)
           data = response.body
-          raise Error.new(data.dig('error', 'message'), response:) if data.dig('error', 'message')
+          if data.dig('error', 'message')
+            raise Error.new(provider_message: data.dig('error', 'message'), error_code: data.dig('error', 'code'),
+                            response:, redact: @config&.redact_error_messages)
+          end
 
           RubyLLM::Moderation.new(
             id: data['id'],
@@ -36,15 +39,15 @@ module RubyLLM
           return input if attachments.empty?
 
           parts = []
-          parts << Media.format_text(input) if input
-          parts.concat(attachments.map { |attachment| format_moderation_attachment(attachment) })
+          parts << Media.render_text(input) if input
+          parts.concat(attachments.map { |attachment| render_moderation_attachment(attachment) })
           parts
         end
 
-        def format_moderation_attachment(attachment)
+        def render_moderation_attachment(attachment)
           raise UnsupportedAttachmentError, attachment.mime_type unless attachment.image?
 
-          Media.format_image(attachment)
+          Media.render_image(attachment)
         end
       end
     end

data/lib/ruby_llm/protocols/chat_completions/speech.rb

@@ -5,9 +5,14 @@ module RubyLLM
     class ChatCompletions
       # Speech generation methods for the OpenAI API integration
       module Speech
+        def speak(input, **, &)
+          @speech_characters = input.to_s.length
+          super
+        end
+
         module_function
 
-        def speech_url(model:) # rubocop:disable Lint/UnusedMethodArgument
+        def speech_url(**)
           'audio/speech'
         end
 
@@ -31,7 +36,8 @@ module RubyLLM
             data: response.body,
             model: model,
             voice: voice || 'alloy',
-            format: resolved_format
+            format: resolved_format,
+            characters: @speech_characters
           )
         end
       end

data/lib/ruby_llm/protocols/chat_completions/streaming.rb

@@ -7,27 +7,42 @@ module RubyLLM
     class ChatCompletions
       # Streaming methods of the OpenAI API integration
       module Streaming
+        ERROR_STATUSES = {
+          'authentication_error' => 401,
+          'invalid_api_key' => 401,
+          'rate_limit_exceeded' => 429,
+          'insufficient_quota' => 429,
+          'server_error' => 500,
+          'service_unavailable' => 503,
+          'overloaded_error' => 529
+        }.freeze
+
         module_function
 
         def stream_url
           completion_url
         end
 
+        def stream_response(...)
+          @think_tags = split_think_tags? ? ThinkTags.new : nil
+          super
+        end
+
         def build_chunk(data)
           usage = data['usage'] || {}
           delta = data.dig('choices', 0, 'delta') || {}
-          content_source = delta['content'] || data.dig('choices', 0, 'message', 'content')
-          content, thinking_from_blocks = extract_content_and_thinking(content_source)
+          content_source = delta['content'] || delta['refusal'] || data.dig('choices', 0, 'message', 'content')
+          content, thinking_from_blocks = parse_content_and_thinking(content_source)
+          thinking_text = thinking_from_blocks || delta['reasoning_content'] || delta['reasoning']
+          citations = parse_chunk_citations(delta, data)
+          content, thinking_text, citations = split_chunk_think_tags(content, thinking_text, citations) if @think_tags
 
           Chunk.new(
             role: :assistant,
             model: data['model'],
             content: content,
-            citations: extract_chunk_citations(delta, data),
-            thinking: Thinking.build(
-              text: thinking_from_blocks || delta['reasoning_content'] || delta['reasoning'],
-              signature: delta['reasoning_signature']
-            ),
+            citations: citations,
+            thinking: Thinking.build(text: thinking_text, signature: delta['reasoning_signature']),
             tool_calls: parse_tool_calls(delta['tool_calls'], parse_arguments: false, stream_keys: true),
             input_tokens: input_tokens(usage),
             output_tokens: output_tokens(usage),
@@ -36,11 +51,24 @@ module RubyLLM
             thinking_tokens: thinking_tokens(usage),
             server_tool_use: server_tool_use(usage),
             reported_cost: reported_cost(usage),
-            finish_reason: normalize_finish_reason(data.dig('choices', 0, 'finish_reason'))
+            finish_reason: normalize_finish_reason(data.dig('choices', 0, 'finish_reason')),
+            content_filters: ContentFilters.parse_content_filters(data)
           )
         end
 
-        def extract_chunk_citations(delta, data)
+        def split_chunk_think_tags(content, thinking_text, citations)
+          content, thinking_text = @think_tags.feed(content, thinking_text)
+          [content, thinking_text, offset_citations(citations, @think_tags.offset)]
+        end
+
+        def stream_remainder
+          content, thinking = @think_tags&.finish
+          return unless content || thinking
+
+          Chunk.new(role: :assistant, content: content, thinking: Thinking.build(text: thinking))
+        end
+
+        def parse_chunk_citations(delta, data)
           annotations = parse_annotations(delta['annotations'], nil)
           return annotations if annotations.any?
 
@@ -57,14 +85,7 @@ module RubyLLM
           code = error['code']
           return [code, error['message']] if code.is_a?(Integer) && code.between?(400, 599)
 
-          case error['type']
-          when 'server_error'
-            [500, error['message']]
-          when 'rate_limit_exceeded', 'insufficient_quota'
-            [429, error['message']]
-          else
-            [400, error['message']]
-          end
+          [ERROR_STATUSES[code] || ERROR_STATUSES.fetch(error['type'], 400), error['message']]
         end
       end
     end

data/lib/ruby_llm/protocols/chat_completions/think_tags.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    class ChatCompletions
+      # Splits a <think>...</think> block that opens the content into
+      # thinking. Model output is untrusted: each byte is scanned once,
+      # without regular expressions, and only a partial tag and a little
+      # leading whitespace are ever held back.
+      class ThinkTags # :nodoc:
+        OPEN = '<think>'
+        CLOSE = '</think>'
+        MAX_LEADING_WHITESPACE = 64
+        WHITESPACE = [9, 10, 11, 12, 13, 32].freeze
+
+        def self.split(text)
+          splitter = new
+          content, thinking = splitter.feed(text)
+          rest_content, rest_thinking = splitter.finish
+          [join(content, rest_content), join(thinking, rest_thinking), splitter.offset]
+        end
+
+        def self.join(first, second)
+          return second if first.nil? || first.empty?
+          return first if second.nil? || second.empty?
+
+          first + second
+        end
+
+        def initialize
+          @state = :start
+          @held = +''
+          @seen = 0
+        end
+
+        def offset
+          @offset || (@state == :start ? 0 : @seen)
+        end
+
+        def feed(text, explicit_thinking = nil)
+          return stop(text, explicit_thinking) if explicit_thinking
+          return [text, nil] if text.nil? || text.empty? || @state == :off
+
+          @seen += text.length
+          case @state
+          when :start then feed_start(text)
+          when :thinking then feed_thinking(text)
+          else feed_answer(text)
+          end
+        end
+
+        def finish
+          stop(nil, nil)
+        end
+
+        private
+
+        def stop(text, thinking)
+          held = take_held
+          state = @state
+          @offset ||= offset
+          @state = :off
+          return [presence(self.class.join(held, text)), thinking] if state == :start
+
+          [text, presence(self.class.join(held, thinking))]
+        end
+
+        def feed_start(text)
+          buffer = with_held(text)
+          leading = whitespace_length(buffer, MAX_LEADING_WHITESPACE + 1)
+          if leading <= MAX_LEADING_WHITESPACE
+            body = buffer.byteslice(leading..)
+            return open_block(body) if body.start_with?(OPEN)
+            return hold(buffer) if OPEN.start_with?(body)
+          end
+
+          @state = :off
+          @offset = 0
+          [buffer, nil]
+        end
+
+        def open_block(body)
+          @state = :thinking
+          feed_thinking(body.byteslice(OPEN.bytesize..))
+        end
+
+        def feed_thinking(text)
+          buffer = with_held(text)
+          close = buffer.byteindex(CLOSE)
+          return close_block(buffer, close) if close
+
+          kept = partial_close_length(buffer)
+          @held = buffer.byteslice(buffer.bytesize - kept, kept)
+          [nil, presence(buffer.byteslice(0, buffer.bytesize - kept))]
+        end
+
+        def close_block(buffer, close)
+          @state = :answer
+          content, = feed_answer(buffer.byteslice((close + CLOSE.bytesize)..))
+          [content, presence(buffer.byteslice(0, close))]
+        end
+
+        def feed_answer(text)
+          leading = whitespace_length(text, text.bytesize)
+          return [nil, nil] if leading == text.bytesize
+
+          answer = text.byteslice(leading..)
+          @state = :off
+          @offset = @seen - answer.length
+          [answer, nil]
+        end
+
+        def whitespace_length(text, limit)
+          length = 0
+          length += 1 while length < limit && WHITESPACE.include?(text.getbyte(length))
+          length
+        end
+
+        def partial_close_length(buffer)
+          (CLOSE.bytesize - 1).downto(1).find { |length| buffer.end_with?(CLOSE.byteslice(0, length)) } || 0
+        end
+
+        def hold(buffer)
+          @held = buffer
+          [nil, nil]
+        end
+
+        def with_held(text)
+          return text if @held.empty?
+
+          take_held + text
+        end
+
+        def take_held
+          held = @held
+          @held = +''
+          held
+        end
+
+        def presence(text)
+          return if text.nil? || text.empty?
+
+          text
+        end
+      end
+    end
+  end
+end