ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocols/anthropic/streaming.rb

@@ -38,27 +38,27 @@ module RubyLLM
 
           Chunk.new(
             role: :assistant,
-            model: extract_model_id(data),
-            content: extract_content_delta(data, delta_type),
-            citations: extract_citations_delta(data, delta_type),
+            model: parse_model_id(data),
+            content: parse_content_delta(data, delta_type),
+            citations: parse_block_citations(data),
             thinking: Thinking.build(
-              text: extract_thinking_delta(data, delta_type),
-              signature: extract_signature_delta(data, delta_type)
+              text: parse_thinking_delta(data, delta_type),
+              signature: parse_signature_delta(data, delta_type)
             ),
-            input_tokens: extract_input_tokens(data),
-            output_tokens: extract_output_tokens(data),
-            thinking_tokens: extract_thinking_tokens(data),
-            cache_read_tokens: extract_cache_read_tokens(data),
-            cache_write_tokens: extract_cache_write_tokens(data),
-            cache_write_tokens_by_ttl: extract_cache_write_by_ttl(data),
-            server_tool_use: extract_server_tool_use(data),
+            input_tokens: parse_input_tokens(data),
+            output_tokens: parse_output_tokens(data),
+            thinking_tokens: parse_thinking_tokens(data),
+            cache_read_tokens: parse_cache_read_tokens(data),
+            cache_write_tokens: parse_cache_write_tokens(data),
+            cache_write_tokens_by_ttl: parse_cache_write_by_ttl(data),
+            server_tool_use: parse_server_tool_use(data),
             tool_calls: extract_tool_calls(data),
             finish_reason: normalize_finish_reason(data.dig('delta', 'stop_reason')),
             **stream_end_fields(data)
           )
         end
 
-        def extract_server_tool_use(data)
+        def parse_server_tool_use(data)
           data.dig('message', 'usage', 'server_tool_use') || data.dig('usage', 'server_tool_use')
         end
 
@@ -129,35 +129,56 @@ module RubyLLM
         def stream_end_fields(data)
           return {} unless data['type'] == 'message_stop'
 
-          blocks = (@stream_blocks || {}).sort.map { |_index, block| block }
+          blocks = stream_block_list
           {
-            server_tool_calls: extract_server_tool_calls(blocks),
+            server_tool_calls: parse_server_tool_calls(blocks),
             raw_content: @saw_server_block ? blocks : nil,
             raw_reasoning: parse_thinking_blocks(blocks)
           }
         end
 
-        def extract_content_delta(data, delta_type)
+        def stream_block_list
+          (@stream_blocks || {}).sort.map { |_index, block| block }
+        end
+
+        def parse_content_delta(data, delta_type)
           return data.dig('delta', 'text') if delta_type == 'text_delta'
 
           nil
         end
 
-        def extract_citations_delta(data, delta_type)
-          return nil unless delta_type == 'citations_delta'
+        # A citation delta arrives before the text it supports, so a block's
+        # citations wait for the block to stop and take its span of the response.
+        def parse_block_citations(data)
+          return nil unless data['type'] == 'content_block_stop'
+
+          block = @stream_blocks&.dig(data['index'])
+          citations = Array(block && block['citations']).compact
+          return nil if citations.empty?
+
+          text = block['text'].to_s
+          start_index = stream_text_length_before(data['index'])
+          citations.map do |citation|
+            Chat.parse_citation(citation, text:, start_index:, end_index: start_index + text.length)
+          end
+        end
 
-          citation = data.dig('delta', 'citation')
-          citation ? [Chat.parse_citation(citation)] : nil
+        def stream_text_length_before(index)
+          @stream_blocks.sum do |block_index, block|
+            block_index < index && block['type'] == 'text' ? block['text'].to_s.length : 0
+          end
         end
 
-        def extract_thinking_delta(data, delta_type)
+        def parse_thinking_delta(data, delta_type)
           return data.dig('delta', 'thinking') if delta_type == 'thinking_delta'
+          return '' if delta_type == 'signature_delta'
 
           nil
         end
 
-        def extract_signature_delta(data, delta_type)
+        def parse_signature_delta(data, delta_type)
           return data.dig('delta', 'signature') if delta_type == 'signature_delta'
+          return parse_thinking_signature(stream_block_list) if data['type'] == 'message_stop'
 
           nil
         end

data/lib/ruby_llm/protocols/anthropic/tools.rb

@@ -22,42 +22,58 @@ module RubyLLM
           block['type'] == 'server_tool_use' && block['name'].to_s.start_with?('tool_search_tool_')
         end
 
-        def format_tool_result(msg)
+        def render_tool_result(msg, citations: false)
           {
             role: 'user',
-            content: [format_tool_result_block(msg)]
+            content: [render_tool_result_block(msg, citations:)]
           }
         end
 
-        def format_tool_result_block(msg)
-          {
+        def render_tool_result_block(msg, citations: false, tool_call: nil)
+          block = {
             type: 'tool_result',
-            tool_use_id: msg.tool_call_id,
-            content: format_tool_result_content(msg)
+            tool_use_id: msg.provider_call_id,
+            content: render_tool_result_content(msg, citations:)
           }
+          return block unless tool_call&.namespace
+
+          block[:toolset_name] = tool_call.namespace
+          error = msg.tool_error_message
+          return block unless error
+
+          block.merge(is_error: true, content: [{ type: 'text', text: error.to_s }])
         end
 
-        def format_tool_result_content(msg)
+        def render_tool_result_content(msg, citations: false)
           search_results = RubyLLM::SearchResults.from_content(msg.content)
           return search_results.results.map { |result| search_result_block(result) } if search_results
 
           content = msg.content
-          content = nil if content && content.empty?
+          content = nil if content&.strip&.empty?
           content = '(no output)' if content.nil? && msg.attachments.empty?
 
-          Media.format_content(content, msg.attachments)
+          Media.render_content(content, msg.attachments, citations:)
         end
 
         def search_result_block(result)
           {
             type: 'search_result',
-            source: result[:url] || result[:title],
+            source: result[:source_id] || result[:url] || result[:title],
             title: result[:title],
             content: [{ type: 'text', text: result[:text] }],
             citations: { enabled: true }
           }
         end
 
+        CLIENT_TOOLSETS = {
+          'computer' => 'Not executed: an earlier computer action in this turn failed.',
+          'browser' => 'Not executed: an earlier action in this turn failed.'
+        }.freeze
+
+        def render_tools(tools)
+          tools.values.reject(&:namespace).map { |tool| function_for(tool) }
+        end
+
         def function_for(tool)
           input_schema = tool.parameters_schema ||
                          RubyLLM::Tool::SchemaDefinition.from_parameters(tool.declared_parameters)&.json_schema
@@ -69,8 +85,6 @@ module RubyLLM
           }
           declaration[:defer_loading] = true if deferred?(tool)
 
-          return declaration if tool.provider_options.empty?
-
           RubyLLM::Support::Utils.deep_merge(declaration, tool.provider_options)
         end
 
@@ -107,19 +121,19 @@ module RubyLLM
 
         def extract_tool_calls(data)
           if json_delta?(data)
-            extract_tool_call_delta(data)
+            parse_tool_call_delta(data)
           elsif content_block_start?(data)
-            extract_tool_call_start(data)
+            parse_tool_call_start(data)
           else
             parse_tool_calls(data['content_block'])
           end
         end
 
-        def extract_tool_call_delta(data)
-          { data['index'] => ToolCall.new(id: nil, name: nil, arguments: data.dig('delta', 'partial_json')) }
+        def parse_tool_call_delta(data)
+          { data['index'] => ToolCall.new(name: nil, arguments: data.dig('delta', 'partial_json')) }
         end
 
-        def extract_tool_call_start(data)
+        def parse_tool_call_start(data)
           tool_calls = parse_tool_calls(data['content_block'])
           return tool_calls if tool_calls.nil? || data['index'].nil?
 
@@ -131,22 +145,20 @@ module RubyLLM
         end
 
         def parse_tool_calls(content_blocks)
-          return nil if content_blocks.nil?
-
           content_blocks = [content_blocks] unless content_blocks.is_a?(Array)
 
-          tool_calls = {}
-          content_blocks.each do |block|
+          calls = content_blocks.filter_map do |block|
             next unless block && block['type'] == 'tool_use'
 
-            tool_calls[block['id']] = ToolCall.new(
-              id: block['id'],
+            ToolCall.new(
+              provider_call_id: block['id'],
               name: block['name'],
+              namespace: block['toolset_name'],
               arguments: block['input']
             )
           end
 
-          tool_calls.empty? ? nil : tool_calls
+          ToolCall.index(calls, by: :provider_call_id) unless calls.empty?
         end
 
         def default_input_schema

data/lib/ruby_llm/protocols/anthropic.rb

@@ -21,6 +21,7 @@ module RubyLLM
       # web_search/web_fetch pin allowed_callers to direct invocation: their
       # 2026 tool versions otherwise default to requiring programmatic tool
       # calling, which not every model supports.
+      MCP_BETA = 'mcp-client-2025-11-20'
       SERVER_TOOL_ALIASES = {
         web_search: { tool: { type: 'web_search_20260318', name: 'web_search', allowed_callers: ['direct'] } },
         web_fetch: { tool: { type: 'web_fetch_20260318', name: 'web_fetch', allowed_callers: ['direct'] } },
@@ -35,11 +36,16 @@ module RubyLLM
               .merge(Anthropic.mcp_tool_filter(options))
               .merge(options.slice(:default_config, :configs)),
             payload: { mcp_servers: [server] },
-            headers: { 'anthropic-beta' => 'mcp-client-2025-11-20' }
+            headers: { 'anthropic-beta' => MCP_BETA }
           }
         end
       }.freeze
 
+      # Returns the pages the message's web searches returned, as WebSource objects.
+      def self.parse_web_sources(message)
+        Chat.parse_web_sources(message)
+      end
+
       def self.mcp_tool_filter(options) # :nodoc:
         approval = options[:require_approval]
         unless approval.nil? || approval.to_s == 'never'
@@ -58,6 +64,14 @@ module RubyLLM
         SERVER_TOOL_ALIASES
       end
 
+      def tool_search?
+        true
+      end
+
+      def skipped_tool_call_error(tool_call)
+        Tools::CLIENT_TOOLSETS[tool_call.namespace]
+      end
+
       # Server tools can pause a long turn with stop_reason pause_turn; the
       # API expects the conversation back verbatim to continue it. Segments
       # are merged into the single Message the caller sees.
@@ -79,11 +93,26 @@ module RubyLLM
       private
 
       def sync_response(payload, additional_headers = {})
-        super(payload, apply_files_beta(additional_headers, payload))
+        super(payload, beta_headers(additional_headers, payload))
       end
 
       def stream_response(payload, additional_headers = {}, &)
-        super(payload, apply_files_beta(additional_headers, payload), &)
+        super(payload, beta_headers(additional_headers, payload), &)
+      end
+
+      def post_count_tokens(payload)
+        @connection.post(count_tokens_url, payload) { |request| request.headers.merge!(beta_headers({}, payload)) }
+      end
+
+      def beta_headers(additional_headers, payload)
+        apply_binding_beta(apply_files_beta(canonical_beta_headers(additional_headers), payload), payload)
+      end
+
+      def merge_tool_headers(tool_headers, headers)
+        headers = canonical_beta_headers(headers)
+        return super unless tool_headers[BETA_HEADER] && headers[BETA_HEADER]
+
+        super.merge(BETA_HEADER => join_betas(tool_headers[BETA_HEADER], headers[BETA_HEADER]))
       end
 
       def merge_turn_segments(segments)
@@ -93,10 +122,10 @@ module RubyLLM
         Message.new(
           role: :assistant,
           content: segments.filter_map(&:content).join,
-          citations: segments.flat_map(&:citations),
+          citations: merge_citations(segments),
           server_tool_calls: segments.flat_map(&:server_tool_calls),
           raw_content: merge_raw_content(segments),
-          thinking: final.thinking,
+          thinking: merge_thinking(segments),
           tool_calls: merge_tool_calls(segments),
           tokens: Tokens.aggregate(segments.map(&:tokens)),
           finish_reason: final.finish_reason,
@@ -105,10 +134,36 @@ module RubyLLM
         )
       end
 
+      # A signature covers one segment's thinking, so joined text carries
+      # none; replay sends each segment's own blocks from raw_content.
+      def merge_thinking(segments)
+        thoughts = segments.filter_map(&:thinking)
+        return thoughts.first if thoughts.size <= 1
+
+        texts = thoughts.filter_map(&:text)
+        Thinking.build(text: (texts.join unless texts.empty?))
+      end
+
+      def merge_citations(segments)
+        offset = 0
+        segments.flat_map do |segment|
+          citations = segment.citations.map { |citation| offset_citation(citation, offset) }
+          offset += segment.content.to_s.length
+          citations
+        end
+      end
+
+      def offset_citation(citation, offset)
+        return citation unless citation.start_index && citation.end_index
+
+        Citation.new(citation.to_h.merge(start_index: citation.start_index + offset,
+                                         end_index: citation.end_index + offset))
+      end
+
       def merge_raw_content(segments)
         return unless segments.any?(&:raw_content)
 
-        segments.flat_map { |segment| segment.raw_content || format_message(segment)[:content] }
+        segments.flat_map { |segment| segment.raw_content || render_message(segment)[:content] }
       end
 
       def merge_tool_calls(segments)

data/lib/ruby_llm/protocols/azure/speech/transcription.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module RubyLLM
+  module Protocols
+    module Azure
+      class Speech
+        # Fast transcription on Azure AI Speech. The request is multipart:
+        # the audio, and a JSON definition carrying the options. Several
+        # languages make a candidate list Azure identifies the spoken one
+        # from, a prompt becomes a phrase list, and asking for speakers
+        # turns on diarization.
+        module Transcription
+          # Azure masks profanity unless told otherwise. Turned off so the
+          # transcript reads like the transcripts every other provider
+          # returns; pass <tt>provider_options: { profanityFilterMode: "Masked" }</tt>
+          # to mask it.
+          DEFAULT_DEFINITION = { profanityFilterMode: 'None' }.freeze
+
+          def render_transcription_options(timestamps:, **)
+            return {} if timestamps.nil? || timestamps == :word
+
+            raise ArgumentError, 'Azure Speech transcription timestamps must be word'
+          end
+
+          def rejected_options(operation)
+            operation == :transcribe ? %i[format temperature speaker_references named_speakers] : super
+          end
+
+          def transcription_url
+            @provider.azure_speech_transcription_url
+          end
+
+          def render_transcription_payload(file_part, language: nil, speaker_names: nil, prompt: nil,
+                                           provider_options: {}, **)
+            {
+              audio: file_part,
+              definition: JSON.generate(render_transcription_definition(language:, speaker_names:, prompt:,
+                                                                        provider_options:))
+            }
+          end
+
+          def render_transcription_definition(language: nil, speaker_names: nil, prompt: nil, provider_options: {})
+            definition = DEFAULT_DEFINITION.dup
+            definition[:locales] = Array(language) if language
+            definition[:diarization] = { enabled: true } if speaker_names
+            definition[:phraseList] = { phrases: Array(prompt) } if prompt
+            Support::Utils.deep_merge(definition, provider_options)
+          end
+
+          def transcribe(audio_file, model:, language:, speaker_names: nil, provider_options: {}, prompt: nil, **)
+            raise_transcription_streaming_unsupported if block_given?
+
+            track_usage(:transcription) do
+              file_part = build_audio_file_part(audio_file)
+              payload = render_transcription_payload(file_part, language:, speaker_names:, prompt:, provider_options:)
+              parse_transcription_response(post_transcription(payload), model:)
+            rescue RubyLLM::Error => e
+              raise unless no_language_identified?(e.response)
+
+              RubyLLM::Transcription.new(text: '', model:)
+            end
+          end
+
+          def parse_transcription_response(response, model:)
+            data = response.body || {}
+            phrases = Array(data['phrases'])
+            duration = data['durationMilliseconds'] && (data['durationMilliseconds'] / 1000.0)
+
+            RubyLLM::Transcription.new(
+              text: Array(data['combinedPhrases']).map { |phrase| phrase['text'] }.join("\n"),
+              model: model,
+              language: phrases.filter_map { |phrase| phrase['locale'] }.first,
+              duration:,
+              audio_seconds: billed_seconds(response) || duration,
+              segments: phrases.map { |phrase| timed(phrase.except('words')) },
+              words: words_in(phrases)
+            )
+          end
+
+          private
+
+          def post_transcription(payload)
+            @connection.post(transcription_url, payload, usage: @usage_tracker) do |request|
+              request.headers.merge!(@provider.azure_speech_headers)
+            end
+          end
+
+          # Given candidate languages, Azure answers silence with a 422 that
+          # bills nothing, where every other provider returns an empty
+          # transcript.
+          def no_language_identified?(response)
+            return false unless response&.status == 422
+
+            body = response.body.is_a?(String) ? JSON.parse(response.body) : response.body
+            body.is_a?(Hash) && body.dig('innerError', 'code') == 'NoLanguageIdentified'
+          rescue JSON::ParserError
+            false
+          end
+
+          # Azure bills whole seconds and reports them in a header, as
+          # <tt>CognitiveServices.SpeechServices.BatchSpeechtoTextSync=4</tt>.
+          def billed_seconds(response)
+            response.headers['csp-billing-usage'].to_s.scan(/=(\d+(?:\.\d+)?)/).flatten.map(&:to_f).max
+          end
+
+          def words_in(phrases)
+            phrases.flat_map do |phrase|
+              Array(phrase['words']).map { |word| timed(word.merge(phrase.slice('speaker'))) }
+            end
+          end
+
+          def timed(part)
+            offset = part['offsetMilliseconds']
+            return part unless offset
+
+            part.merge('start' => offset / 1000.0,
+                       'end' => (offset + part['durationMilliseconds'].to_i) / 1000.0)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/azure/speech.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Protocols
+    module Azure
+      # The Azure AI Speech API on an Azure AI Services resource. Only fast
+      # transcription is implemented; chat, embeddings, and images go through
+      # the resource's OpenAI-compatible protocols.
+      class Speech < Protocol
+        include Speech::Transcription
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/bedrock/batches.rb

@@ -45,7 +45,8 @@ module RubyLLM
 
         def find_batch(id)
           data = @provider.signed_get(@provider.control_api_base, bedrock_job_url(id)).body
-          protocol = @provider.embedding_batch_protocol(data['modelId']) if data['modelInvocationType'] == 'InvokeModel'
+          invoke_model = data['modelInvocationType'] == 'InvokeModel'
+          protocol = @provider.invoke_model_batch_protocol(data['modelId']) if invoke_model
           parser = protocol ? protocol.new(@provider) : self
           parser.send(:parse_batch_response, data).tap do |result|
             result[:batch_protocol] = protocol if protocol
@@ -104,8 +105,7 @@ module RubyLLM
           {
             id: data['jobArn'] || data['jobIdentifier'],
             raw_status: status,
-            completed: TERMINAL.include?(status),
-            request_counts: bedrock_request_counts(data)
+            completed: TERMINAL.include?(status)
           }
         end
 
@@ -117,11 +117,29 @@ module RubyLLM
           :failed
         end
 
-        def bedrock_request_counts(data)
-          {
-            'submitted' => data['submitTime'],
-            'completed' => data['endTime']
-          }.compact
+        def bedrock_batch_jsonl(requests)
+          requests.map do |request|
+            JSON.generate(recordId: request[:custom_id], modelInput: bedrock_model_input(request))
+          end.join("\n")
+        end
+
+        def bedrock_model_input(request)
+          batch_payload(request)
+        end
+
+        def parse_bedrock_outputs(outputs, model:)
+          outputs.flat_map do |body|
+            body.to_s.each_line.filter_map { |line| parse_bedrock_record(JSON.parse(line), model:) }
+          end
+        end
+
+        def parse_bedrock_record(line, model:)
+          record_id = line['recordId'] || line['record_id']
+          index = batch_result_index(record_id)
+          output = line['modelOutput']
+          return [index, parse_bedrock_model_output(output, model:)] if output
+
+          [index, nil, batch_failure(record_id, line.dig('error', 'message') || line['errorMessage'])]
         end
       end
     end