ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocols/responses/media.rb

@@ -11,31 +11,31 @@ module RubyLLM
           attachment.provider_file? || %i[image pdf document text].include?(attachment.type)
         end
 
-        def format_content(content, attachments = [])
+        def render_content(content, attachments = [])
           return content if attachments.empty?
 
           parts = []
           parts << { type: 'input_text', text: content } if content
-          attachments.each { |attachment| parts << format_attachment(attachment) }
+          attachments.each { |attachment| parts << render_attachment(attachment) }
           parts
         end
 
-        def format_attachment(attachment)
+        def render_attachment(attachment)
           raise UnsupportedAttachmentError, attachment.mime_type unless supported_attachment?(attachment)
 
-          return format_provider_file(attachment) if attachment.provider_file?
+          return render_provider_file(attachment) if attachment.provider_file?
 
           case attachment.type
           when :image
-            format_image(attachment)
+            render_image(attachment)
           when :pdf, :document
-            format_document(attachment)
+            render_document(attachment)
           when :text
             { type: 'input_text', text: attachment.for_llm }
           end
         end
 
-        def format_image(image, image_url: image.url_or_data_uri, original_detail: @provider&.original_image_detail?)
+        def render_image(image, image_url: image.url_or_data_uri, original_detail: @provider&.original_image_detail?)
           part = { type: 'input_image', image_url: }
           return part unless image.resolution
 
@@ -45,7 +45,9 @@ module RubyLLM
         # The Responses API extracts text from documents, presentations, and
         # spreadsheets as well as PDFs, so every document attachment rides
         # along as a native input_file.
-        def format_document(document)
+        def render_document(document)
+          return { type: 'input_file', file_url: document.source.to_s } if document.url? && document_urls?
+
           {
             type: 'input_file',
             filename: document.filename,
@@ -53,7 +55,13 @@ module RubyLLM
           }
         end
 
-        def format_provider_file(file)
+        # The provider fetches the URL itself. Dialects opt in once a live
+        # request shows their endpoint downloads input_file URLs.
+        def document_urls?
+          false
+        end
+
+        def render_provider_file(file)
           {
             type: 'input_file',
             file_id: file.provider_file_id

data/lib/ruby_llm/protocols/responses/streaming.rb

@@ -6,17 +6,15 @@ module RubyLLM
       # Streaming methods of the OpenAI Responses API. Events are semantic:
       # each SSE data frame carries a `type` describing what changed.
       module Streaming
-        ERROR_STATUSES = {
-          'server_error' => 500,
-          'rate_limit_exceeded' => 429,
-          'too_many_requests' => 429,
-          'insufficient_quota' => 429
-        }.freeze
+        ERROR_STATUSES = ChatCompletions::Streaming::ERROR_STATUSES.merge('too_many_requests' => 429).freeze
 
         module_function
 
         def stream_response(...)
           @citation_content_lengths = Hash.new(0)
+          @citation_markers = {}
+          @reasoning_output_index = nil
+          @text_output_index = nil
           super
         end
 
@@ -28,12 +26,16 @@ module RubyLLM
             chunk thinking: Thinking.build(text: data['delta'])
           when 'response.reasoning_summary_part.added'
             build_reasoning_summary_part_chunk(data)
+          when 'response.output_text.done', 'response.refusal.done'
+            build_text_done_chunk(data)
           when 'response.output_text.annotation.added'
             build_annotation_chunk(data)
+          when 'response.content_part.added'
+            build_content_part_added_chunk(data)
           when 'response.output_item.added'
             build_item_added_chunk(data)
           when 'response.function_call_arguments.delta'
-            chunk tool_calls: { data['output_index'] => ToolCall.new(id: nil, name: nil, arguments: data['delta']) }
+            chunk tool_calls: { data['output_index'] => ToolCall.new(name: nil, arguments: data['delta']) }
           when 'response.output_item.done'
             build_item_done_chunk(data)
           when 'response.completed', 'response.incomplete'
@@ -46,9 +48,17 @@ module RubyLLM
         end
 
         def build_text_chunk(data)
+          text_content_chunk(data, citation_markers(data).feed(data['delta']))
+        end
+
+        def build_text_done_chunk(data)
+          text_content_chunk(data, citation_markers(data).finish)
+        end
+
+        def text_content_chunk(data, content)
           @citation_content_lengths ||= Hash.new(0)
-          @citation_content_lengths[citation_content_position(data)] += data['delta'].to_s.length
-          chunk content: data['delta']
+          @citation_content_lengths[citation_content_position(data)] += content.to_s.length
+          chunk content:
         end
 
         def build_annotation_chunk(data)
@@ -57,7 +67,18 @@ module RubyLLM
             (key <=> position).negative? ? length : 0
           end
 
-          chunk citations: offset_citations(parse_annotations([data['annotation']], nil), offset, nil)
+          chunk citations: offset_citations(parse_annotations([data['annotation']], nil), offset, nil,
+                                            citation_markers(data))
+        end
+
+        def citation_markers(data)
+          @citation_markers ||= {}
+          @citation_markers[citation_content_position(data)] ||= CitationMarkers.new
+        end
+
+        def stream_remainder
+          content = (@citation_markers || {}).values.filter_map(&:finish).join
+          chunk content: content unless content.empty?
         end
 
         def citation_content_position(data)
@@ -65,7 +86,9 @@ module RubyLLM
         end
 
         def build_reasoning_summary_part_chunk(data)
-          return chunk unless data['summary_index'].positive?
+          earlier_item = @reasoning_output_index && @reasoning_output_index != data['output_index']
+          @reasoning_output_index = data['output_index']
+          return chunk unless earlier_item || data['summary_index'].positive?
 
           chunk thinking: Thinking.build(text: "\n\n")
         end
@@ -75,17 +98,51 @@ module RubyLLM
           return chunk unless item['type'] == 'function_call'
 
           chunk tool_calls: {
-            data['output_index'] => ToolCall.new(id: item['call_id'], name: item['name'], arguments: +'')
+            data['output_index'] => ToolCall.new(provider_call_id: item['call_id'], name: item['name'],
+                                                 namespace: item['namespace'], arguments: +'')
           }
         end
 
+        def build_content_part_added_chunk(data)
+          return chunk unless Chat::MESSAGE_TEXT_KEYS.key?(data.dig('part', 'type'))
+
+          output_index = data['output_index']
+          earlier_item = @text_output_index && @text_output_index != output_index
+          @text_output_index = output_index
+          return chunk unless earlier_item
+
+          @citation_content_lengths[[output_index, -1]] += Chat::MESSAGE_SEPARATOR.length
+          chunk content: Chat::MESSAGE_SEPARATOR
+        end
+
         def build_item_done_chunk(data)
           item = data['item']
+          return build_function_call_done_chunk(data) if item['type'] == 'function_call'
           return chunk unless item['type'] == 'reasoning' && item['encrypted_content']
 
           chunk thinking: Thinking.build(text: nil, signature: item['encrypted_content'])
         end
 
+        def build_function_call_done_chunk(data)
+          item = data['item']
+          arguments = completed_function_call_arguments(item['arguments'])
+          return chunk unless arguments
+
+          chunk tool_calls: {
+            data['output_index'] => ToolCall.new(provider_call_id: item['call_id'], name: item['name'],
+                                                 namespace: item['namespace'], arguments:)
+          }
+        end
+
+        def completed_function_call_arguments(arguments)
+          return unless arguments.is_a?(String)
+
+          parsed = JSON.parse(arguments)
+          parsed if parsed.is_a?(Hash)
+        rescue JSON::ParserError
+          nil
+        end
+
         def build_final_chunk(data)
           response = data['response'] || {}
           output = response['output'] || []
@@ -94,9 +151,11 @@ module RubyLLM
           chunk model: response['model'],
                 tool_calls: parse_tool_approvals(output, finish_reason: parse_finish_reason(response)),
                 finish_reason: parse_finish_reason(response),
+                content_filters: ChatCompletions::ContentFilters.parse_content_filters(response),
                 citations: parse_citations(response, output, nil),
                 server_tool_calls: server_tool_calls,
                 raw_content: keep_raw_output?(output, server_tool_calls) ? output : nil,
+                phase: parse_phase(output),
                 **parse_usage(response['usage'] || {}),
                 server_tool_use: parse_server_tool_use(response)
         end

data/lib/ruby_llm/protocols/responses/token_counting.rb

@@ -14,7 +14,7 @@ module RubyLLM
         end
 
         def render_count_tokens_payload(messages, model:, **)
-          render_payload(messages, model: model, temperature: nil, **).slice(*COUNT_TOKENS_KEYS)
+          apply_tool_search(render_payload(messages, model: model, temperature: nil, **)).slice(*COUNT_TOKENS_KEYS)
         end
 
         def parse_count_tokens_response(response)

data/lib/ruby_llm/protocols/responses/tools.rb

@@ -9,6 +9,7 @@ module RubyLLM
         module_function
 
         NATIVE_TOOL_SEARCH = { type: 'tool_search' }.freeze
+        RESERVED_NAMESPACES = %w[computer browser].freeze
 
         def tool_for(tool)
           definition = {
@@ -25,10 +26,36 @@ module RubyLLM
           RubyLLM::Support::Utils.deep_merge(definition, tool.provider_options)
         end
 
+        def render_tools(tools)
+          ordinary, namespaced = tools.values.partition { |tool| tool.namespace.nil? }
+          groups = namespaced.group_by(&:namespace).map do |namespace, members|
+            if RESERVED_NAMESPACES.include?(namespace)
+              raise Error, "The namespace \"#{namespace}\" is reserved by the Responses API, which drops its " \
+                           'functions; choose another namespace'
+            end
+
+            { type: 'namespace', name: namespace, description: members.filter_map(&:namespace_description).first,
+              tools: members.map { |tool| tool_for(tool) } }.compact
+          end
+          ordinary.map { |tool| tool_for(tool) } + groups
+        end
+
         def deferred?(tool)
           tool.is_a?(RubyLLM::Tool::Deferred)
         end
 
+        # Tool search loads a deferred function declared outside any
+        # namespace under a namespace of its own name, and the API refuses a
+        # call to it that lacks the namespace.
+        def namespace_searched_calls(input, tools)
+          searched = tools.each_value.filter_map { |tool| tool.name.to_s if deferred?(tool) && tool.namespace.nil? }
+          input.map { |item| searched_call?(item, searched) ? item.merge(namespace: item[:name]) : item }
+        end
+
+        def searched_call?(item, searched)
+          item.is_a?(Hash) && item[:type] == 'function_call' && item[:namespace].nil? && searched.include?(item[:name])
+        end
+
         # The API requires a search tool next to deferred tools, and rejects
         # search items in the input of a request that declares none.
         def apply_tool_search(payload)
@@ -41,8 +68,28 @@ module RubyLLM
           end)
         end
 
+        # A web search reports the pages it found only when the request asks
+        # for them, and services on this wire format that lack this include
+        # value reject the whole request, so each one opts in.
+        WEB_SEARCH_TOOL_TYPES = %w[web_search web_search_preview].freeze
+        WEB_SEARCH_SOURCES = 'web_search_call.action.sources'
+
+        def apply_provider_tools(*)
+          payload = super
+          return payload unless web_search_sources?
+          return payload unless Array(payload[:tools]).any? { |tool| web_search_tool?(tool) }
+
+          payload.merge(include: Array(payload[:include]) | [WEB_SEARCH_SOURCES])
+        end
+
+        def web_search_tool?(tool)
+          WEB_SEARCH_TOOL_TYPES.include?(tool[:type] || tool['type'])
+        end
+
         def defers_loading?(tools)
-          tools.any? { |tool| tool[:defer_loading] || tool['defer_loading'] }
+          tools.any? do |tool|
+            tool[:defer_loading] || tool['defer_loading'] || defers_loading?(Array(tool[:tools] || tool['tools']))
+          end
         end
 
         def search_tool?(tool)

data/lib/ruby_llm/protocols/responses.rb

@@ -28,9 +28,30 @@ module RubyLLM
         end
       }.freeze
 
+      # Returns the text of +message+ that holds its answer when its turn
+      # marks other items as intermediate, or +nil+ when all of its content
+      # is the answer. Reads only what the message keeps, so a restored
+      # message answers the same.
+      def self.parse_answer_text(message)
+        Chat.parse_answer_text(message)
+      end
+
+      # Returns the pages the message's web searches returned, as WebSource objects.
+      def self.parse_web_sources(message)
+        Chat.parse_web_sources(message)
+      end
+
       def server_tool_aliases
         SERVER_TOOL_ALIASES
       end
+
+      def tool_search?
+        true
+      end
+
+      def web_search_sources? # :nodoc:
+        false
+      end
     end
   end
 end

data/lib/ruby_llm/protocols/system_one/responses.rb

@@ -8,15 +8,22 @@ module RubyLLM
 
         def parse_judgment_response(response, questions:)
           body = response.body
+          usage = body['usage'] || {}
           answers = questions.values.to_h do |question|
             [question.name, parse_answer(body.fetch('answers').fetch(question.name.to_s), question)]
           end
           RubyLLM::Judgment.new(
             answers:, model: body['model'], raw: response, model_info: @model,
-            tokens: Tokens.new(input: body.dig('usage', 'input_tokens'), output: body.dig('usage', 'output_tokens'))
+            tokens: Tokens.new(input: usage['input_tokens'], output: usage['output_tokens'],
+                               reported_cost: reported_cost(usage))
           )
         rescue KeyError, NoMethodError, TypeError => e
-          raise Error.new("System One returned an invalid judgment: #{e.message}", response:)
+          raise Error.new('System One returned an invalid judgment', provider_message: e.message, response:,
+                                                                     redact: @config&.redact_error_messages)
+        end
+
+        def reported_cost(_usage)
+          nil
         end
 
         def parse_answer(answer, question)

data/lib/ruby_llm/protocols/vertexai/files.rb

@@ -10,8 +10,8 @@ module RubyLLM
 
         STORAGE_CLIENTS = Support::ProcessCache.new
 
-        def self.storage_clients
-          STORAGE_CLIENTS
+        def rejected_options(operation)
+          operation == :upload ? %i[purpose expires_in] : super
         end
 
         # rubocop:disable-next Lint/UnusedMethodArgument
@@ -22,7 +22,8 @@ module RubyLLM
           bucket_name, key = parse_gcs_uri(target_uri)
 
           with_file_body(attachment) do |body|
-            bucket(bucket_name).create_file(body, key, content_type: content_type || file_content_type(attachment))
+            bucket(bucket_name).create_file(body, key, **provider_options.transform_keys(&:to_sym),
+                                            content_type: content_type || file_content_type(attachment))
           end
 
           uploaded_file(

data/lib/ruby_llm/protocols/vertexai/text_to_speech.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'base64'
+
+module RubyLLM
+  module Protocols
+    module VertexAI
+      # Google Cloud Text-to-Speech, served from the endpoint of the
+      # configured location. A voice name is locale, model, and voice:
+      # de-DE-Chirp3-HD-Leda speaks German, and a bare Leda on Chirp3-HD is
+      # en-US-Chirp3-HD-Leda.
+      class TextToSpeech < Protocol
+        MODELS = %w[Chirp3-HD].freeze
+
+        DEFAULT_VOICE = 'Charon'
+        DEFAULT_LOCALE = 'en-US'
+        DEFAULT_FORMAT = 'mp3'
+
+        # RubyLLM format names in Cloud Text-to-Speech's audioEncoding
+        # vocabulary. LINEAR16, MULAW, and ALAW come back in a WAV container.
+        ENCODINGS = {
+          'mp3' => 'MP3',
+          'wav' => 'LINEAR16',
+          'opus' => 'OGG_OPUS',
+          'mulaw' => 'MULAW',
+          'alaw' => 'ALAW'
+        }.freeze
+
+        # Request fields that choose the text, voice, or encoding RubyLLM
+        # reports, which provider_options may not replace in either JSON
+        # spelling (languageCode or language_code).
+        RESERVED_OPTIONS = {
+          'input' => %w[text ssml markup multispeakermarkup],
+          'voice' => %w[name languagecode customvoice voiceclone],
+          'audioconfig' => %w[audioencoding]
+        }.freeze
+
+        MIME_TYPES = {
+          'opus' => 'audio/ogg',
+          'mulaw' => 'audio/wav',
+          'alaw' => 'audio/wav'
+        }.freeze
+
+        def initialize(provider, model = nil)
+          super
+          @connection = provider.text_to_speech_connection
+        end
+
+        def speech_url(**)
+          'text:synthesize'
+        end
+
+        def speak(input, **, &)
+          @speech_characters = input.to_s.length
+          super
+        end
+
+        def render_speech_payload(input, model:, voice:, format:, provider_options: {})
+          options = Support::Utils.deep_stringify_keys(provider_options || {})
+          reject_reserved_options(options)
+
+          name = voice_name(voice, model)
+          payload = {
+            'input' => { 'text' => input },
+            'voice' => { 'languageCode' => name.split('-').first(2).join('-'), 'name' => name },
+            'audioConfig' => { 'audioEncoding' => encoding_for(format) }
+          }
+          Support::Utils.deep_merge(payload, options)
+        end
+
+        def parse_speech_response(response, model:, voice:, format:)
+          audio = response.body['audioContent'] if response.body.is_a?(Hash)
+          raise Error.new('Cloud Text-to-Speech returned no audio', response:) unless audio
+
+          format = format_for(format)
+          RubyLLM::Speech.new(data: Base64.decode64(audio), model:, voice: voice_name(voice, model), format:,
+                              mime_type: MIME_TYPES[format], characters: @speech_characters)
+        end
+
+        private
+
+        def reject_reserved_options(options)
+          reserved = options.flat_map do |section, fields|
+            keys = RESERVED_OPTIONS[field_key(section)]
+            next [] unless keys && fields.is_a?(Hash)
+
+            fields.keys.select { |key| keys.include?(field_key(key)) }.map { |key| "#{section}.#{key}" }
+          end
+          return if reserved.empty?
+
+          raise ArgumentError, 'Use the input, voice:, and format: arguments instead of provider_options for ' \
+                               "#{reserved.join(', ')}"
+        end
+
+        def field_key(key)
+          key.delete('_').downcase
+        end
+
+        def voice_name(voice, model)
+          voice = (voice || DEFAULT_VOICE).to_s
+          voice.include?('-') ? voice : "#{DEFAULT_LOCALE}-#{model}-#{voice}"
+        end
+
+        def encoding_for(format)
+          format = format_for(format)
+          ENCODINGS.fetch(format, format.upcase)
+        end
+
+        def format_for(format)
+          format = (format || DEFAULT_FORMAT).to_s
+          ENCODINGS.key(format.upcase) || format.downcase
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/xai/files.rb

@@ -5,15 +5,21 @@ module RubyLLM
     module XAI
       # xAI Files API.
       class Files < Protocols::Files
+        def rejected_options(operation)
+          operation == :upload ? %i[uri] : super
+        end
+
         private
 
-        # rubocop:disable-next Lint/UnusedMethodArgument
-        def render_upload_payload(attachment, purpose: nil, expires_in: nil, visibility: nil,
-                                  display_name: nil, uri: nil, content_type: nil)
-          multipart_payload(attachment, expires_after: expires_in, purpose:)
+        def render_upload_fields(_attachment, purpose: nil, expires_in: nil)
+          { expires_after: expires_in, purpose: }
+        end
+
+        def parse_upload_response(data, attachment)
+          parse_file_response(data, mime_type: file_content_type(attachment))
         end
 
-        def parse_file_response(data)
+        def parse_file_response(data, mime_type: nil)
           uploaded_file(
             data,
             id: data['id'],
@@ -21,7 +27,8 @@ module RubyLLM
             byte_size: data['bytes'],
             created_at: timestamp(data['created_at']),
             expires_at: timestamp(data['expires_at']),
-            purpose: data['purpose']
+            purpose: data['purpose'],
+            mime_type:
           )
         end
       end

data/lib/ruby_llm/protocols/zai/files.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module ZAI
+      # Z.ai file storage. Files go up and can be referenced by id, but Z.ai
+      # has no route that reads one back, and no expiry option.
+      class Files < Protocols::OpenAI::Files
+        def rejected_options(operation)
+          operation == :upload ? %i[expires_in uri] : super
+        end
+
+        def find(_file_id)
+          raise Error, 'Z.ai does not support retrieving uploaded files'
+        end
+
+        def download(_file_id)
+          raise Error, 'Z.ai does not support downloading uploaded files'
+        end
+
+        private
+
+        def render_upload_fields(_attachment, purpose: nil, **)
+          { purpose: purpose || 'user_data' }
+        end
+
+        def timestamp(value)
+          Time.at(value.fdiv(1000)) if value
+        end
+
+        def uploaded_file(data, **attributes)
+          super(data, **attributes, downloadable: false)
+        end
+      end
+    end
+  end
+end