ruby_llm 2.1.0 → 2.2.0

data/lib/ruby_llm/protocols/cohere/transcription.rb

@@ -7,6 +7,10 @@ module RubyLLM
       module Transcription
         DEFAULT_LANGUAGE = 'en'
 
+        def rejected_options(operation)
+          operation == :transcribe ? %i[format prompt speaker_names speaker_references] : super
+        end
+
         module_function
 
         def transcription_url
@@ -16,10 +20,7 @@ module RubyLLM
         # Cohere parses the multipart body as it streams, so every form field
         # has to be written before the file part or the request is rejected as
         # missing them. The file goes in last, after provider options.
-        # rubocop:disable-next Lint/UnusedMethodArgument
-        def render_transcription_payload(file_part, model:, language:, format: nil, speaker_names: nil,
-                                         speaker_references: nil, provider_options: {}, prompt: nil,
-                                         temperature: nil)
+        def render_transcription_payload(file_part, model:, language:, provider_options: {}, temperature: nil, **)
           {
             model: model,
             language: language || DEFAULT_LANGUAGE,

data/lib/ruby_llm/protocols/converse/batches.rb

@@ -13,12 +13,6 @@ module RubyLLM
           'Converse'
         end
 
-        def bedrock_batch_jsonl(requests)
-          requests.map do |request|
-            JSON.generate(recordId: request[:custom_id], modelInput: batch_payload(request))
-          end.join("\n")
-        end
-
         def validate_bedrock_batch_requests!(requests)
           unsupported = requests.any? do |request|
             payload = request.fetch(:payload)
@@ -30,24 +24,8 @@ module RubyLLM
           raise Error, 'bedrock batch requests do not support tools or structured output'
         end
 
-        def parse_bedrock_outputs(outputs, **)
-          outputs.flat_map { |body| parse_bedrock_output(body) }
-        end
-
-        def parse_bedrock_output(body)
-          body.to_s.each_line.filter_map { |line| parse_bedrock_record(JSON.parse(line)) }
-        end
-
-        def parse_bedrock_record(line)
-          record_id = line['recordId'] || line['record_id']
-          index = batch_result_index(record_id)
-
-          if line['modelOutput']
-            body = line['modelOutput']
-            [index, parse_completion_body(body, raw: body)]
-          else
-            [index, nil, batch_failure(record_id, line.dig('error', 'message') || line['errorMessage'])]
-          end
+        def parse_bedrock_model_output(body, model:)
+          parse_completion_body({ 'modelId' => model }.merge(body), raw: body)
         end
       end
     end

data/lib/ruby_llm/protocols/converse/chat.rb

@@ -15,6 +15,22 @@ module RubyLLM
 
         BEDROCK_INLINE_DOCUMENT_LIMIT = 4_500_000
         PROMPT_CACHE_OPTIONS = %i[ttl].freeze
+        # Bedrock accepts at most four cache points in one request, and
+        # Converse takes them only for Claude and Nova models; Nova takes none
+        # in toolConfig.tools. See "Supported models, Regions, and explicit
+        # caching limits" in
+        # https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
+        MAX_CACHE_POINTS = 4
+        CACHE_POINT_MODELS = %w[anthropic. amazon.nova].freeze
+        TOOL_CACHE_POINT_MODELS = %w[anthropic.].freeze
+        # Converse refuses searchResult blocks from every model but Claude, so an
+        # application inference profile, which doesn't name its model, gets text.
+        SEARCH_RESULT_MODELS = %w[anthropic.].freeze
+        # Bedrock forwards an S3 source to other models untranslated, and
+        # Claude rejects it.
+        S3_SOURCE_MODELS = %w[amazon.nova].freeze
+        OFFICE_DOCUMENT_FORMATS = %w[doc docx xls xlsx].freeze
+        CACHE_LIFETIME_UNITS = { 's' => 1, 'm' => 60, 'h' => 3600 }.freeze
         COUNT_TOKENS_KEYS = %i[messages system toolConfig additionalModelRequestFields].freeze
 
         module_function
@@ -24,7 +40,7 @@ module RubyLLM
         def normalize_finish_reason(reason)
           return nil if reason.nil?
 
-          finish_reasons.fetch(reason.to_s) { reason.to_s.to_sym }
+          finish_reasons.fetch(reason.to_s) { reason.to_s.downcase.to_sym }
         end
 
         def completion_url
@@ -41,28 +57,31 @@ module RubyLLM
         def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil,
                            schema: nil, thinking: nil, citations: false, caching: nil, tool_prefs: nil)
           tool_prefs ||= {}
+          max_output_tokens ||= model.max_output_tokens
           @used_document_names = {}
           system_messages, chat_messages = messages.partition { |msg| msg.role == :system }
           prompt_cache_options(caching)
-          automatic_cache_target = automatic_cache_target(system_messages, chat_messages, caching)
           payload = {
-            messages: format_messages(chat_messages, caching:, automatic_cache_target:, citations:)
+            messages: render_messages(chat_messages, caching:, citations:, search_results: search_results?(model),
+                                                     tool_text: tools.empty?)
           }
 
-          system_blocks = format_system(system_messages, caching:, automatic_cache_target:)
+          system_blocks = render_system(system_messages, caching:)
           payload[:system] = system_blocks unless system_blocks.empty?
 
-          payload[:inferenceConfig] = format_inference_config(model, temperature, max_output_tokens)
+          payload[:inferenceConfig] = render_inference_config(model, temperature, max_output_tokens)
 
-          tool_config = format_tool_config(tools, tool_prefs)
+          tool_config = render_tool_config(tools, tool_prefs)
           payload[:toolConfig] = tool_config if tool_config
 
-          additional_fields = format_additional_model_request_fields(thinking, model, max_output_tokens)
+          additional_fields = render_additional_model_request_fields(thinking, model, max_output_tokens)
+          additional_fields = bind_thinking(additional_fields, model)
           payload[:additionalModelRequestFields] = additional_fields if additional_fields
 
           output_config = build_output_config(schema)
           payload[:outputConfig] = output_config if output_config
 
+          add_automatic_cache_points(payload, model, caching)
           payload
         end
 
@@ -91,8 +110,8 @@ module RubyLLM
           response.body['inputTokens']
         end
 
-        def supports_provider_file_references?
-          true
+        def provider_file_references?
+          S3_SOURCE_MODELS.any? { |prefix| foundation_model_id(@model&.id).start_with?(prefix) }
         end
 
         def default_large_file_upload_threshold
@@ -108,7 +127,7 @@ module RubyLLM
           content_blocks = data.dig('output', 'message', 'content') || []
           usage = data['usage'] || {}
           thinking_text, thinking_signature = parse_thinking(content_blocks)
-          text_content, citations = extract_text_and_citations(content_blocks)
+          text_content, citations = parse_text_and_citations(content_blocks)
 
           Message.new(
             role: :assistant,
@@ -117,7 +136,7 @@ module RubyLLM
             thinking: Thinking.build(text: thinking_text, signature: thinking_signature),
             raw_reasoning: parse_thinking_blocks(content_blocks),
             tool_calls: parse_tool_calls(content_blocks),
-            server_tool_calls: extract_server_tool_calls(content_blocks),
+            server_tool_calls: parse_server_tool_calls(content_blocks),
             input_tokens: input_tokens(usage),
             output_tokens: usage['outputTokens'],
             cache_read_tokens: usage['cacheReadInputTokens'],
@@ -151,22 +170,31 @@ module RubyLLM
           end
         end
 
-        def format_messages(messages, caching: nil, automatic_cache_target: nil, citations: false)
+        # Converse refuses toolUse and toolResult blocks in a request without
+        # a toolConfig, and has no tool choice of none, so a request that
+        # declares no tools, such as a summary of a chat that called them,
+        # gets its tool rounds as text (+tool_text+).
+        def render_messages(messages, caching: nil, citations: false, search_results: false, tool_text: false)
           rendered = []
           tool_result_blocks = []
+          calls = {}
 
           messages.each do |msg|
+            calls.merge!(msg.tool_calls.to_h)
             if msg.tool_result?
-              tool_result_blocks << format_tool_result_block(msg)
+              call = calls[msg.tool_call_id] if tool_text
+              tool_result_blocks.concat(render_tool_result_blocks(msg, caching:, citations:, search_results:,
+                                                                       tool_text:, call:))
               next
             end
 
+            message = render_non_tool_message(msg, caching:, citations:, tool_text:)
+            next tool_result_blocks.concat(message[:content]) if joins_tool_results?(message, tool_result_blocks)
+
             unless tool_result_blocks.empty?
               rendered << { role: 'user', content: tool_result_blocks }
               tool_result_blocks = []
             end
-
-            message = format_non_tool_message(msg, caching:, automatic_cache_target:, citations:)
             rendered << message if message
           end
 
@@ -174,42 +202,50 @@ module RubyLLM
           rendered
         end
 
-        def format_non_tool_message(msg, caching: nil, automatic_cache_target: nil, citations: false)
-          content = format_message_content(msg, caching:, automatic_cache_target:, citations:)
+        def joins_tool_results?(message, tool_result_blocks)
+          message && message[:role] == 'user' && !tool_result_blocks.empty?
+        end
+
+        def render_non_tool_message(msg, caching: nil, citations: false, tool_text: false)
+          content = render_message_content(msg, caching:, citations:, tool_text:)
           return nil if content.empty?
 
           {
-            role: format_role(msg.role),
+            role: render_role(msg.role),
             content: content
           }
         end
 
-        def format_message_content(msg, caching: nil, automatic_cache_target: nil, citations: false)
-          blocks = format_structured_message_content(msg, citations:)
+        def render_message_content(msg, caching: nil, citations: false, tool_text: false)
+          blocks = render_structured_message_content(msg, citations:)
 
           if msg.tool_call?
-            msg.tool_calls.each_value do |tool_call|
+            msg.tool_calls.each do |tool_call|
+              next blocks << tool_call_text_block(tool_call) if tool_text
+
               blocks << {
                 toolUse: {
-                  toolUseId: tool_call.id,
+                  toolUseId: tool_call.provider_call_id,
                   name: tool_call.name,
                   input: tool_call.arguments
                 }
               }
             end
           end
-          if cache_boundary?(msg, automatic_cache_target, caching:)
-            blocks << converse_cache_block_for(caching, ttl: msg.cache_ttl)
-          end
+          blocks << converse_cache_block_for(caching, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
 
           blocks
         end
 
-        def format_structured_message_content(msg, citations: false)
-          blocks = msg.role == :assistant ? format_thinking_blocks(msg) : []
+        def tool_call_text_block(tool_call)
+          { text: "Called the #{tool_call.name} tool with #{tool_call.arguments.to_json}" }
+        end
+
+        def render_structured_message_content(msg, citations: false)
+          blocks = msg.role == :assistant ? render_thinking_blocks(msg) : []
 
           blocks.concat(
-            Media.format_content(msg.content, msg.attachments, used_document_names: @used_document_names, citations:)
+            Media.render_content(msg.content, msg.attachments, used_document_names: @used_document_names, citations:)
           )
 
           blocks
@@ -218,35 +254,55 @@ module RubyLLM
         # Reasoning rebuilt from thinking text goes back only to the provider
         # that produced it: Claude refuses another provider's signature, and
         # a block without one.
-        def format_thinking_blocks(msg)
+        def render_thinking_blocks(msg)
           blocks = msg.raw_reasoning['converse'] if msg.raw_reasoning.is_a?(Hash)
           return Support::Utils.deep_dup(blocks) if blocks
           return [] unless msg.thinking && producer_slug(msg) == @provider.slug
 
-          [format_thinking_block(msg.thinking)].compact
+          [render_thinking_block(msg.thinking)].compact
+        end
+
+        def render_tool_result_blocks(msg, caching: nil, citations: false, search_results: false, tool_text: false,
+                                      call: nil)
+          blocks = if tool_text
+                     render_tool_result_text(msg, call, citations:)
+                   else
+                     [render_tool_result_block(msg, citations:, search_results:)]
+                   end
+          blocks << converse_cache_block_for(caching, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
+          blocks
         end
 
-        def format_tool_result_block(msg)
+        def render_tool_result_block(msg, citations: false, search_results: false)
           {
             toolResult: {
-              toolUseId: msg.tool_call_id,
-              content: format_tool_result_content(msg)
+              toolUseId: msg.provider_call_id,
+              content: render_tool_result_content(msg, citations:, search_results:)
             }
           }
         end
 
-        def format_tool_result_content(msg)
-          search_results = RubyLLM::SearchResults.from_content(msg.content)
-          return search_results.results.map { |result| search_result_block(result) } if search_results
+        def render_tool_result_text(msg, call, citations: false)
+          label = call ? "The #{call.name} tool returned:" : 'The tool returned:'
+          first, *rest = render_tool_result_content(msg, citations:)
+          return [{ text: "#{label}\n#{first[:text]}" }, *rest] if first.key?(:text)
 
-          blocks = Media.format_content(msg.content, msg.attachments, used_document_names: @used_document_names)
+          [{ text: label }, first, *rest]
+        end
+
+        def render_tool_result_content(msg, citations: false, search_results: false)
+          results = RubyLLM::SearchResults.from_content(msg.content) if search_results
+          return results.results.map { |result| search_result_block(result) } if results
+
+          content = msg.content unless msg.content&.strip&.empty?
+          blocks = Media.render_content(content, msg.attachments, used_document_names: @used_document_names, citations:)
           blocks.empty? ? [text_tool_result_block(nil)] : blocks
         end
 
         def search_result_block(result)
           {
             searchResult: {
-              source: result[:url] || result[:title],
+              source: result[:source_id] || result[:url] || result[:title],
               title: result[:title],
               content: [{ text: result[:text] }],
               citations: { enabled: true }
@@ -260,43 +316,114 @@ module RubyLLM
           { text: text }
         end
 
-        def format_role(role)
+        def render_role(role)
           case role
           when :assistant then 'assistant'
           else 'user'
           end
         end
 
-        def format_system(messages, caching: nil, automatic_cache_target: nil)
+        def render_system(messages, caching: nil)
           messages.flat_map do |msg|
-            blocks = Media.format_content(msg.content, msg.attachments, used_document_names: @used_document_names)
-            if cache_boundary?(msg, automatic_cache_target, caching:)
-              blocks + [converse_cache_block_for(caching, ttl: msg.cache_ttl)]
-            else
-              blocks
-            end
+            blocks = Media.render_content(msg.content, msg.attachments, used_document_names: @used_document_names)
+            cache_boundary?(msg, caching:) ? blocks + [converse_cache_block_for(caching, ttl: msg.cache_ttl)] : blocks
           end
         end
 
-        def automatic_cache_target(system_messages, chat_messages, caching)
-          return unless caching
+        def cache_boundary?(message, caching: nil)
+          caching != false && message.cache_until_here?
+        end
+
+        # Bedrock's own caching of Claude and Nova prompts is best effort, so
+        # with caching on RubyLLM places explicit points: at the end of the
+        # conversation, so the next tool round or question can read what was
+        # sent so far; after the system prompt (or the tools without one),
+        # which stays cached when a later message changes; and after the
+        # previous assistant message, because Bedrock looks for an earlier
+        # cache entry only about 20 content blocks back from each point, and
+        # a round of parallel tool calls can be longer than that. Points the
+        # messages already mark count toward the limit of four, and an
+        # automatic point never puts a shorter lifetime before a longer one,
+        # which Bedrock refuses.
+        def add_automatic_cache_points(payload, model, caching)
+          return unless caching && automatic_cache_points?(model)
+
+          budget = MAX_CACHE_POINTS - cache_points(payload).size
+          automatic_cache_targets(payload, model).each do |blocks|
+            index = automatic_cache_point_index(blocks)
+            next unless index
+            break if budget <= 0
 
-          (chat_messages.reverse + system_messages.reverse).find { |msg| cacheable_message?(msg) }
+            blocks.insert(index, converse_cache_block_for(caching))
+            next blocks.delete_at(index) unless cache_lifetimes_descend?(payload)
+
+            budget -= 1
+          end
         end
 
-        def cacheable_message?(message)
-          !message.tool_result?
+        # Claude refuses a cache point right after an Office document, so the
+        # point goes after the last block that takes one.
+        def automatic_cache_point_index(blocks)
+          last = blocks.rindex { |block| !office_document?(block) }
+          last + 1 if last && !cache_point?(blocks[last])
         end
 
-        def cache_boundary?(message, automatic_cache_target, caching: nil)
-          caching != false && (message.cache_until_here? || message.equal?(automatic_cache_target))
+        def office_document?(block)
+          block.is_a?(Hash) && OFFICE_DOCUMENT_FORMATS.include?(block.dig(:document, :format))
+        end
+
+        def search_results?(model)
+          model_id = foundation_model_id(model&.id)
+          SEARCH_RESULT_MODELS.any? { |prefix| model_id.start_with?(prefix) }
+        end
+
+        def automatic_cache_points?(model)
+          model_id = foundation_model_id(model&.id)
+          # An application inference profile ARN does not name its model.
+          CACHE_POINT_MODELS.any? { |prefix| model_id.start_with?(prefix) } || !model_id.include?('.')
+        end
+
+        def automatic_cache_targets(payload, model)
+          messages = payload[:messages]
+          earlier_answer = messages[0...-1].reverse.find { |message| message[:role] == 'assistant' }
+
+          [messages.last&.dig(:content), stable_prefix_blocks(payload, model), earlier_answer&.dig(:content)].compact
+        end
+
+        def stable_prefix_blocks(payload, model)
+          return payload[:system] if payload[:system]
+
+          tools = payload.dig(:toolConfig, :tools)
+          tools if tools && TOOL_CACHE_POINT_MODELS.any? { |prefix| foundation_model_id(model&.id).start_with?(prefix) }
+        end
+
+        # In the order Bedrock reads them: tools, system, then messages.
+        def cache_points(payload)
+          blocks = Array(payload.dig(:toolConfig, :tools)) + Array(payload[:system]) +
+                   payload[:messages].flat_map { |message| message[:content] }
+          blocks.select { |block| cache_point?(block) }
+        end
+
+        def cache_lifetimes_descend?(payload)
+          lifetimes = cache_points(payload).map { |block| cache_lifetime(block[:cachePoint] || block['cachePoint']) }
+          lifetimes.each_cons(2).all? { |earlier, later| earlier >= later }
+        end
+
+        def cache_lifetime(point)
+          ttl = point.is_a?(Hash) ? (point[:ttl] || point['ttl']) : nil
+          amount, unit = ttl.to_s.match(/\A(\d+)([smh])\z/)&.captures
+          amount ? amount.to_i * CACHE_LIFETIME_UNITS.fetch(unit) : 300
+        end
+
+        def cache_point?(block)
+          block.is_a?(Hash) && (block.key?(:cachePoint) || block.key?('cachePoint'))
         end
 
         def converse_cache_block_for(caching, ttl: nil)
           options = prompt_cache_options(caching)
           ttl ||= options[:ttl]
           point = { type: 'default' }
-          point[:ttl] = ttl if ttl
+          point[:ttl] = Protocol::CacheLifetime.render(ttl) if ttl
           { cachePoint: point }
         end
 
@@ -311,37 +438,37 @@ module RubyLLM
                 "Bedrock Converse prompt caching accepts :ttl, got #{format_cache_option_keys(unsupported)}"
         end
 
+        def cache_option_names = PROMPT_CACHE_OPTIONS
+
         def format_cache_option_keys(keys)
           keys.map { |key| ":#{key}" }.join(', ')
         end
 
-        def format_inference_config(_model, temperature, max_output_tokens = nil)
+        def render_inference_config(_model, temperature, max_output_tokens = nil)
           config = {}
           config[:temperature] = temperature unless temperature.nil?
           config[:maxTokens] = max_output_tokens unless max_output_tokens.nil?
           config
         end
 
-        def format_tool_config(tools, tool_prefs)
+        def render_tool_config(tools, tool_prefs)
           return nil if tools.empty?
 
           config = {
-            tools: tools.values.map { |tool| format_tool(tool) }
+            tools: tools.values.map { |tool| render_tool(tool) }
           }
 
           return config if tool_prefs.nil? || tool_prefs[:choice].nil?
 
-          tool_choice = format_tool_choice(tool_prefs[:choice])
-          config[:toolChoice] = tool_choice if tool_choice
-          config
+          config.merge(toolChoice: render_tool_choice(tool_prefs[:choice]))
         end
 
-        def format_tool_choice(choice)
+        def render_tool_choice(choice)
           case choice
           when :auto
             { auto: {} }
           when :none
-            nil
+            raise ArgumentError, 'Bedrock Converse has no tool choice of none; remove the tools from this request'
           when :required
             { any: {} }
           else
@@ -349,7 +476,7 @@ module RubyLLM
           end
         end
 
-        def format_tool(tool)
+        def render_tool(tool)
           input_schema = tool.parameters_schema ||
                          RubyLLM::Tool::SchemaDefinition.from_parameters(tool.declared_parameters)&.json_schema
 
@@ -368,15 +495,25 @@ module RubyLLM
           RubyLLM::Support::Utils.deep_merge(tool_spec, tool.provider_options)
         end
 
-        def format_additional_model_request_fields(thinking, model, max_output_tokens = nil)
+        def render_additional_model_request_fields(thinking, model, max_output_tokens = nil)
           fields = {}
 
-          reasoning_fields = Reasoning.format_reasoning_fields(thinking, model, max_output_tokens)
+          reasoning_fields = Reasoning.render_reasoning_fields(thinking, model, max_output_tokens)
           fields = RubyLLM::Support::Utils.deep_merge(fields, reasoning_fields) if reasoning_fields
 
           fields.empty? ? nil : fields
         end
 
+        # Bound from the first request, as on the Anthropic API, so the
+        # thinking setting never changes mid-conversation.
+        def bind_thinking(fields, model)
+          return fields unless Anthropic::BlockBinding.binds?(foundation_model_id(model&.id))
+          return fields if fields&.key?(:reasoning_config)
+
+          bound = Anthropic::BlockBinding.apply(fields&.dig(:thinking))
+          bound ? { **fields.to_h, thinking: bound, anthropic_beta: [Anthropic::BlockBinding::BETA] } : fields
+        end
+
         def build_output_config(schema)
           return nil unless schema
 
@@ -403,7 +540,7 @@ module RubyLLM
           model_id.to_s.rpartition('/').last.sub(/\A(?:#{prefixes})\./, '')
         end
 
-        def format_thinking_block(thinking)
+        def render_thinking_block(thinking)
           return nil unless thinking
 
           if thinking.text
@@ -424,7 +561,7 @@ module RubyLLM
           end
         end
 
-        def extract_text_and_citations(content_blocks)
+        def parse_text_and_citations(content_blocks)
           text = +''
           citations = []
 
@@ -468,6 +605,7 @@ module RubyLLM
             text: text,
             start_index: start_index,
             end_index: end_index,
+            source_id: data['source'],
             source_index: citation_source_index(location),
             start_page: page['start'],
             end_page: end_page && (end_page - 1)
@@ -526,22 +664,19 @@ module RubyLLM
         end
 
         def parse_tool_calls(content_blocks)
-          tool_calls = {}
-
-          content_blocks.each do |block|
+          calls = content_blocks.filter_map do |block|
             tool_use = block['toolUse']
             next unless tool_use
             next if server_tool_use?(tool_use)
 
-            tool_call_id = tool_use['toolUseId']
-            tool_calls[tool_call_id] = ToolCall.new(
-              id: tool_call_id,
+            ToolCall.new(
+              provider_call_id: tool_use['toolUseId'],
               name: tool_use['name'],
               arguments: tool_use['input'] || {}
             )
           end
 
-          tool_calls.empty? ? nil : tool_calls
+          ToolCall.index(calls, by: :provider_call_id) unless calls.empty?
         end
 
         # Provider-executed tool steps come back with a distinguishing type,
@@ -557,7 +692,7 @@ module RubyLLM
           !type.nil? && type != 'tool_result'
         end
 
-        def extract_server_tool_calls(content_blocks)
+        def parse_server_tool_calls(content_blocks)
           content_blocks.filter_map do |block|
             tool_use = block['toolUse']
             tool_result = block['toolResult']