ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/protocols/bedrock/event_stream.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'json'
+
+module RubyLLM
+  module Protocols
+    module Bedrock
+      # AWS event stream framing for Bedrock's streaming APIs, ConverseStream
+      # and InvokeModelWithResponseStream. Each frame carries one JSON event,
+      # bare or base64-encoded under "bytes", or an error.
+      module EventStream
+        ErrorResponse = Struct.new(:body, :status)
+
+        EXCEPTION_STATUSES = {
+          'throttlingException' => 429,
+          'validationException' => 400,
+          'accessDeniedException' => 401,
+          'unrecognizedClientException' => 401,
+          'serviceUnavailableException' => 503
+        }.freeze
+
+        private
+
+        # Posts +body+ as an event stream request, yielding each decoded
+        # event Hash and raising the errors the stream reports. Returns the
+        # Faraday response.
+        def stream_events(url, body, additional_headers = {}, &handler)
+          decoder = event_stream_decoder
+          progress = {}
+
+          on_data = RubyLLM::Protocol::Streaming::FaradayHandlers.build(
+            faraday_v1: faraday_1?,
+            on_chunk: ->(chunk, _env) { parse_stream_chunk(decoder, chunk, progress, &handler) },
+            on_failed_response: ->(chunk, env) { handle_failed_stream(chunk, env) }
+          )
+
+          on_reset = -> { decoder = event_stream_decoder }
+          post_stream(url, body, on_data:, progress:, on_reset:) do |req|
+            req.headers.merge!(additional_headers) unless additional_headers.empty?
+            req.headers['Accept'] = 'application/vnd.amazon.eventstream'
+          end
+        end
+
+        def event_stream_decoder
+          require 'aws-eventstream'
+          Aws::EventStream::Decoder.new
+        rescue LoadError
+          raise Error,
+                'The aws-eventstream gem is required for Bedrock streaming. ' \
+                'Please add it to your Gemfile: gem "aws-eventstream"'
+        end
+
+        # The error body arrives in as many reads as the adapter hands out, so
+        # it accumulates on the env, which ErrorMiddleware clears per attempt.
+        def handle_failed_stream(chunk, env)
+          buffer = failed_stream_buffer(env)
+          buffer << chunk
+          error_response = env.merge(body: JSON.parse(buffer))
+          Transport::ErrorMiddleware.parse_error(provider: self, response: error_response)
+        rescue JSON::ParserError
+          RubyLLM.logger.debug { "Accumulating Bedrock stream error chunk: #{chunk}" }
+        end
+
+        def failed_stream_buffer(env)
+          (env[:streaming_state] ||= RubyLLM::Protocol::Streaming::StreamState.new).buffer
+        end
+
+        def parse_stream_chunk(decoder, raw_chunk, progress)
+          handle_non_eventstream_error_chunk(raw_chunk)
+
+          decode_events(decoder, raw_chunk).each do |event|
+            raise_stream_error(event) if stream_error_event?(event)
+
+            progress[:started] = true
+            yield event
+          end
+        end
+
+        def handle_non_eventstream_error_chunk(raw_chunk)
+          text = raw_chunk.to_s
+
+          if text.start_with?('event: error')
+            payload = text.lines.find { |line| line.start_with?('data:') }&.delete_prefix('data:')&.strip
+            raise_streaming_chunk_error(payload) if payload
+            return
+          end
+
+          return unless text.lstrip.start_with?('{') && text.include?('"error"')
+
+          raise_streaming_chunk_error(text)
+        end
+
+        def raise_streaming_chunk_error(payload)
+          message = chunk_error_message(JSON.parse(payload)) || 'Bedrock streaming error'
+          response = ErrorResponse.new({ 'message' => message }, 500)
+          Transport::ErrorMiddleware.parse_error(provider: self, response: response)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def chunk_error_message(parsed)
+          return unless parsed.is_a?(Hash)
+
+          error = parsed['error']
+          (error.is_a?(Hash) ? error['message'] : error) || parsed['message']
+        end
+
+        def decode_events(decoder, raw_chunk)
+          events = []
+          message, eof = decoder.decode_chunk(raw_chunk)
+
+          while message
+            event = decode_event(message)
+            if event && RubyLLM.config.log_stream_debug
+              RubyLLM.logger.debug do
+                "Bedrock stream event keys: #{event.keys}"
+              end
+            end
+            events << event if event
+            break if eof
+
+            message, eof = decoder.decode_chunk
+          end
+
+          events
+        rescue Aws::EventStream::Errors::PreludeChecksumError
+          raise_streaming_chunk_error(raw_chunk)
+          raise Error, 'Bedrock returned an unreadable event stream'
+        end
+
+        # Modeled errors arrive as a frame after the 200, with the type in the
+        # frame headers and a bare message as the payload. Rewrap it under the
+        # exception name so raise_stream_error classifies it instead of the
+        # frame passing for an empty event.
+        def decode_event(message)
+          return decode_error_frame(message.headers) if stream_error_frame?(message)
+
+          exception_type = stream_exception_type(message)
+          payload = message.payload.read
+          event = decode_event_payload(payload)
+          return event unless exception_type
+
+          { exception_type => event || { 'message' => payload.to_s } }
+        end
+
+        # Failures the API does not model arrive as an error frame instead,
+        # with the code and message in the headers and nothing in the payload.
+        def stream_error_frame?(message)
+          event_header(message.headers, ':message-type') == 'error'
+        end
+
+        def decode_error_frame(headers)
+          detail = [event_header(headers, ':error-code'), event_header(headers, ':error-message')].compact.join(': ')
+          event = { 'type' => 'error' }
+          event['error'] = { 'message' => detail } unless detail.empty?
+          event
+        end
+
+        def stream_exception_type(message)
+          headers = message.headers
+          return nil unless event_header(headers, ':message-type') == 'exception'
+
+          event_header(headers, ':exception-type') || 'streamingException'
+        end
+
+        def event_header(headers, name)
+          header = headers[name]
+          header.respond_to?(:value) ? header.value : header
+        end
+
+        def decode_event_payload(payload)
+          outer = JSON.parse(payload)
+
+          if outer['bytes'].is_a?(String)
+            JSON.parse(Base64.decode64(outer['bytes']))
+          else
+            outer
+          end
+        rescue JSON::ParserError => e
+          RubyLLM.logger.debug { "Failed to decode Bedrock stream event payload: #{e.message}" }
+          nil
+        end
+
+        def stream_error_event?(event)
+          event.keys.any? { |key| key.end_with?('Exception') } || event['type'] == 'error'
+        end
+
+        def raise_stream_error(event)
+          if event['type'] == 'error'
+            message = event.dig('error', 'message') || 'Bedrock streaming error'
+            response = ErrorResponse.new({ 'message' => message }, 500)
+            Transport::ErrorMiddleware.parse_error(provider: self, response: response)
+            return
+          end
+
+          key = event.keys.find { |candidate| candidate.end_with?('Exception') }
+          message = event[key]['message'] || key
+          response = ErrorResponse.new({ 'message' => message }, EXCEPTION_STATUSES.fetch(key, 500))
+          Transport::ErrorMiddleware.parse_error(provider: self, response: response)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/protocols/bedrock/files.rb

@@ -5,16 +5,25 @@ module RubyLLM
     module Bedrock
       # S3-backed file storage for Bedrock batch input and output.
       class Files < Protocols::Files
+        # S3 object keys allow spaces and other characters URI() rejects or
+        # reads as a query or fragment.
+        S3_URI = %r{\As3://([^/]+)/?(.*)\z}m
+
+        def rejected_options(operation)
+          operation == :upload ? %i[purpose expires_in] : super
+        end
+
         # rubocop:disable-next Lint/UnusedMethodArgument
         def upload(file, filename: nil, purpose: nil, expires_in: nil, uri: nil, content_type: nil,
                    provider_options: {})
           attachment = file_attachment(file, filename:)
           target_uri = uri || storage_uri_for(attachment)
           bucket, key = parse_s3_uri(target_uri)
+          mime_type = content_type || file_content_type(attachment)
 
           with_file_body(attachment) do |body|
-            s3_client.put_object(bucket: bucket, key: key, body: body,
-                                 content_type: content_type || file_content_type(attachment))
+            s3_client.put_object(**provider_options.transform_keys(&:to_sym), bucket:, key:, body:,
+                                                                              content_type: mime_type)
           end
 
           uploaded_file(
@@ -23,7 +32,7 @@ module RubyLLM
             uri: target_uri,
             filename: attachment.filename,
             byte_size: file_size(attachment),
-            mime_type: content_type || file_content_type(attachment)
+            mime_type:
           )
         end
 
@@ -99,10 +108,10 @@ module RubyLLM
         end
 
         def parse_s3_uri(uri)
-          parsed = URI(uri)
-          raise ArgumentError, "Expected an s3:// URI, got: #{uri}" unless parsed.scheme == 's3'
+          match = S3_URI.match(uri.to_s)
+          raise ArgumentError, "Expected an s3:// URI, got: #{uri}" unless match
 
-          [parsed.host, parsed.path.delete_prefix('/')]
+          [match[1], match[2]]
         end
       end
     end

data/lib/ruby_llm/protocols/bedrock/guardrails.rb

@@ -46,7 +46,7 @@ module RubyLLM
         def parse_moderation_responses(responses)
           RubyLLM::Moderation.new(
             id: responses.one? ? responses.first.first.headers['x-amzn-requestid'] : nil,
-            model: nil,
+            model: guardrail_model_id,
             results: responses.map(&:last),
             raw: responses.one? ? responses.first.first.body : responses.map { |response, _result| response.body }
           )

data/lib/ruby_llm/protocols/bedrock/rerank.rb

@@ -71,6 +71,8 @@ module RubyLLM
               RubyLLM::Rerank::Result.new(index:, document: documents[index], score: item['relevanceScore'])
             end
           end
+          raise Error, 'Bedrock reranking returned a duplicate document index' unless results.uniq(&:index) == results
+
           RubyLLM::Rerank.new(results:, model:, raw: pages.length == 1 ? pages.first : pages)
         end
       end

data/lib/ruby_llm/protocols/chat_completions/chat.rb

@@ -13,10 +13,41 @@ module RubyLLM
         OPENAI_INLINE_FILE_LIMIT = 50 * 1024 * 1024
         PROMPT_CACHE_OPTIONS = %i[key ttl mode retention].freeze
 
+        UNSUPPORTED_PARAMETER_CODES = %w[unsupported_parameter unsupported_value unknown_parameter].freeze
+        # The Responses API reports some of these, such as a temperature,
+        # with a +param+ and no +code+.
+        UNSUPPORTED_PARAMETER_MESSAGE = /\AUnsupported (?:parameter|value): /
+        PARAMETERS = {
+          'temperature' => :temperature, 'reasoning_effort' => :thinking,
+          'max_tokens' => :max_output_tokens, 'max_completion_tokens' => :max_output_tokens
+        }.freeze
+
         def completion_url
           'chat/completions'
         end
 
+        # OpenAI names the rejected parameter in +param+ and why in +code+.
+        def parse_unsupported_parameter(response)
+          body = response.body
+          body = JSON.parse(body) if body.is_a?(String)
+          error = body['error'] if body.is_a?(Hash)
+          return unless error.is_a?(Hash) && unsupported_parameter_error?(error)
+          return unless error['param'].is_a?(String) && !error['param'].empty?
+
+          wire_parameters.fetch(error['param'], error['param'])
+        rescue JSON::ParserError
+          nil
+        end
+
+        def unsupported_parameter_error?(error)
+          return UNSUPPORTED_PARAMETER_CODES.include?(error['code']) if error['code']
+
+          error['message'].to_s.match?(UNSUPPORTED_PARAMETER_MESSAGE)
+        end
+
+        # The wire parameters that carry RubyLLM settings, by name.
+        def wire_parameters = PARAMETERS
+
         module_function
 
         def finish_reasons = FINISH_REASONS
@@ -24,7 +55,7 @@ module RubyLLM
         def normalize_finish_reason(reason)
           return nil if reason.nil?
 
-          finish_reasons.fetch(reason.to_s) { reason.to_s.to_sym }
+          finish_reasons.fetch(reason.to_s) { reason.to_s.downcase.to_sym }
         end
 
         # OpenAI strict mode rejects an object whose properties are not all
@@ -35,14 +66,15 @@ module RubyLLM
         end
 
         def strict_schema?(node)
-          case node
-          when Hash
-            return false if optional_properties?(node)
+          return true unless node.is_a?(Hash)
+          return false if optional_properties?(node)
 
-            node.values.all? { |value| strict_schema?(value) }
-          when Array then node.all? { |value| strict_schema?(value) }
-          else true
-          end
+          subschemas(node).all? { |schema| strict_schema?(schema) }
+        end
+
+        def subschemas(node)
+          keyed = node.values_at(:properties, :$defs, :definitions).grep(Hash).flat_map(&:values)
+          keyed + node.values_at(:items, :prefixItems, :anyOf, :oneOf, :allOf, :additionalProperties).flatten(1)
         end
 
         def optional_properties?(node)
@@ -59,7 +91,7 @@ module RubyLLM
           tool_prefs ||= {}
           payload = {
             model: model.id,
-            messages: format_messages(messages, caching: caching),
+            messages: render_messages(messages, caching: caching),
             stream: stream
           }
 
@@ -110,32 +142,41 @@ module RubyLLM
           message_data = data.dig('choices', 0, 'message')
           raise no_completion_message_error(data, raw) unless message_data
 
-          usage = data['usage'] || {}
-          thinking_tokens = thinking_tokens(usage)
-          content, thinking_from_blocks = extract_content_and_thinking(message_data['content'])
-          thinking_text = thinking_from_blocks || extract_thinking_text(message_data)
-          thinking_signature = extract_thinking_signature(message_data)
+          content, thinking_from_blocks =
+            parse_content_and_thinking(message_data['content'] || message_data['refusal'])
+          thinking_text = thinking_from_blocks || parse_thinking_text(message_data)
+          thinking_signature = parse_thinking_signature(message_data)
+          answer, thinking_text, offset = parse_think_tags(content, thinking_text)
 
           finish_reason = normalize_finish_reason(data.dig('choices', 0, 'finish_reason'))
 
           Message.new(
             role: :assistant,
-            content: content,
-            citations: extract_citations(message_data, data, content),
+            content: answer,
+            citations: offset_citations(extract_citations(message_data, data, content), offset),
             thinking: Thinking.build(text: thinking_text, signature: thinking_signature),
-            raw_reasoning: extract_raw_reasoning(message_data),
+            raw_reasoning: parse_raw_reasoning(message_data),
             tool_calls: parse_tool_calls(message_data['tool_calls'], response: raw, finish_reason: finish_reason),
+            finish_reason: finish_reason,
+            content_filters: ContentFilters.parse_content_filters(data),
+            model: data['model'],
+            raw: raw,
+            **parse_completion_usage(data)
+          )
+        end
+
+        def parse_completion_usage(data)
+          usage = data['usage'] || {}
+
+          {
             input_tokens: input_tokens(usage),
             output_tokens: output_tokens(usage),
             cache_read_tokens: cache_read_tokens(usage),
             cache_write_tokens: cache_write_tokens(usage),
-            thinking_tokens: thinking_tokens,
+            thinking_tokens: thinking_tokens(usage),
             server_tool_use: server_tool_use(usage),
-            reported_cost: reported_cost(usage),
-            finish_reason: finish_reason,
-            model: data['model'],
-            raw: raw
-          )
+            reported_cost: reported_cost(usage)
+          }
         end
 
         def reported_cost(_usage)
@@ -189,7 +230,10 @@ module RubyLLM
         end
 
         def cache_write_tokens(usage)
+          return if usage.empty?
+
           usage.dig('prompt_tokens_details', 'cache_write_tokens') ||
+            usage.dig('prompt_tokens_details', 'created_cache_tokens') ||
             usage.dig('input_tokens_details', 'cache_write_tokens') ||
             0
         end
@@ -277,10 +321,14 @@ module RubyLLM
 
           {}.tap do |cache_options|
             cache_options[:mode] = options[:mode] if options[:mode]
-            cache_options[:ttl] = ttl if ttl
+            cache_options[:ttl] = render_cache_lifetime(ttl) if ttl
           end
         end
 
+        def render_cache_lifetime(ttl)
+          Protocol::CacheLifetime.render(ttl)
+        end
+
         def retention_ttl(retention)
           return unless retention
 
@@ -301,39 +349,66 @@ module RubyLLM
                 "got #{format_cache_option_keys(unsupported)}"
         end
 
+        def cache_option_names = PROMPT_CACHE_OPTIONS
+
         def format_cache_option_keys(keys)
           keys.map { |key| ":#{key}" }.join(', ')
         end
 
-        def format_messages(messages, caching: nil)
+        def render_messages(messages, caching: nil)
           messages_for_provider(messages)
-            .chunk_while { |previous, current| previous.tool_result? && current.tool_result? }
-            .flat_map { |group| format_message_group(group, caching: caching) }
+            .chunk_while { |previous, current| current.tool_result? && (previous.tool_result? || carrier?(previous)) }
+            .flat_map { |group| render_message_group(group, caching: caching) }
         end
 
         # An assistant turn's tool results must stay consecutive, so the
-        # attachment carriers of a parallel round follow the whole run.
-        def format_message_group(group, caching: nil)
-          formatted = group.map { |msg| format_message(msg, caching: caching) }
-          carriers = group.select { |msg| msg.tool_result? && msg.attachments.any? }
+        # attachment carriers of a round follow the whole run.
+        def render_message_group(group, caching: nil)
+          formatted = group.filter_map do |msg|
+            next if attachments_only_assistant_message?(msg)
+
+            render_message(msg.role == :assistant ? msg.with_attachments([]) : msg, caching: caching)
+          end
+          carriers = group.select { |msg| carrier?(msg) }.map do |msg|
+            msg.tool_result? ? tool_attachment_message(msg) : assistant_attachment_message(msg)
+          end
 
-          formatted + carriers.map { |msg| tool_attachment_message(msg) }
+          formatted + carriers
         end
 
-        def format_message(msg, caching: nil)
+        def carrier?(msg)
+          (msg.tool_result? || msg.role == :assistant) && msg.attachments.any?
+        end
+
+        def attachments_only_assistant_message?(msg)
+          msg.role == :assistant && msg.attachments.any? && !msg.tool_call? && msg.thinking.nil? &&
+            msg.content.to_s.empty?
+        end
+
+        def render_message(msg, caching: nil)
           {
-            role: format_role(msg.role),
-            content: format_message_content(msg, caching: caching),
-            tool_calls: format_tool_calls(msg.tool_calls),
-            tool_call_id: msg.tool_call_id
-          }.compact.merge(format_thinking(msg))
+            role: render_role(msg.role),
+            content: render_message_content(msg, caching: caching),
+            tool_calls: render_tool_calls(msg.tool_calls),
+            tool_call_id: msg.provider_call_id
+          }.compact.merge(render_thinking(msg))
         end
 
         # Chat Completions tool messages are text-only on the wire, so tool
         # attachments ride a user message spliced in after the results.
         def tool_attachment_message(msg)
-          parts = [Media.format_text("Attachments from tool call #{msg.tool_call_id}:")]
-          parts.concat(format_content(nil, msg.attachments))
+          attachment_message("Attachments from tool call #{msg.provider_call_id}:", msg.attachments)
+        end
+
+        # Assistant messages take only text, so a replayed turn's attachments
+        # ride a user message after it, as on Responses.
+        def assistant_attachment_message(msg)
+          attachment_message('Attachments from the assistant:', msg.attachments)
+        end
+
+        def attachment_message(label, attachments)
+          parts = [Media.render_text(label)]
+          parts.concat(render_content(nil, attachments))
           { role: 'user', content: parts }
         end
 
@@ -342,8 +417,8 @@ module RubyLLM
           system_messages + other_messages
         end
 
-        def format_message_content(msg, caching: nil, **)
-          content = format_content(msg.content, msg.tool_result? ? [] : msg.attachments)
+        def render_message_content(msg, caching: nil, **)
+          content = render_content(msg.content, msg.tool_result? ? [] : msg.attachments)
           return '' if content.nil? && thinking_only_assistant_message?(msg)
           return inject_cache_breakpoint(content) if caching != false && msg.cache_until_here? && openai_prompt_caching?
 
@@ -361,7 +436,7 @@ module RubyLLM
         def cache_breakpoint_parts(content)
           case content
           when Array then content.dup
-          when String then [Media.format_text(content)] unless content.empty?
+          when String then [Media.render_text(content)] unless content.empty?
           end
         end
 
@@ -369,11 +444,11 @@ module RubyLLM
           msg.role == :assistant && msg.thinking && !msg.tool_call?
         end
 
-        def format_content(content, attachments = [])
-          Media.format_content(content, attachments)
+        def render_content(content, attachments = [])
+          Media.render_content(content, attachments)
         end
 
-        def format_role(role)
+        def render_role(role)
           case role
           when :system
             @config.openai_use_system_role ? 'system' : 'developer'
@@ -398,7 +473,7 @@ module RubyLLM
           payload.merge(safety_identifier: identifier)
         end
 
-        def supports_provider_file_references?
+        def provider_file_references?
           @provider.slug == 'openai'
         end
 
@@ -414,7 +489,7 @@ module RubyLLM
           { purpose: 'user_data' }
         end
 
-        def format_thinking(msg)
+        def render_thinking(msg)
           return {} unless msg.role == :assistant
 
           thinking = msg.thinking
@@ -429,44 +504,69 @@ module RubyLLM
           payload
         end
 
-        def extract_thinking_text(message_data)
+        def parse_thinking_text(message_data)
           candidate = message_data['reasoning_content'] || message_data['reasoning'] || message_data['thinking']
           candidate.is_a?(String) ? candidate : nil
         end
 
-        def extract_thinking_signature(message_data)
+        def parse_thinking_signature(message_data)
           candidate = message_data['reasoning_signature'] || message_data['signature']
           candidate.is_a?(String) ? candidate : nil
         end
 
-        def extract_raw_reasoning(_message_data)
+        def parse_think_tags(content, thinking_text)
+          return [content, thinking_text, 0] unless split_think_tags? && thinking_text.nil?
+          return [content, nil, 0] unless content.is_a?(String) && !content.empty?
+
+          ThinkTags.split(content)
+        end
+
+        # Annotation indices count from the start of the content the server
+        # sent, before the think block was removed.
+        def offset_citations(citations, offset)
+          return citations if offset.zero?
+
+          citations.map do |citation|
+            next citation unless citation.start_index && citation.end_index
+
+            start_index = citation.start_index - offset
+            end_index = citation.end_index - offset if start_index >= 0
+            Citation.new(**citation.to_h, start_index: end_index && start_index, end_index:)
+          end
+        end
+
+        def split_think_tags?
+          @config&.split_think_tags
+        end
+
+        def parse_raw_reasoning(_message_data)
           nil
         end
 
-        def extract_content_and_thinking(content)
+        def parse_content_and_thinking(content)
           return [content, nil] unless content.is_a?(Array)
 
-          text = extract_text_from_blocks(content)
-          thinking = extract_thinking_from_blocks(content)
+          text = parse_text_from_blocks(content)
+          thinking = parse_thinking_from_blocks(content)
 
           [text.empty? ? nil : text, thinking.empty? ? nil : thinking]
         end
 
-        def extract_text_from_blocks(blocks)
+        def parse_text_from_blocks(blocks)
           blocks.filter_map do |block|
             block['text'] if block['type'] == 'text' && block['text'].is_a?(String)
           end.join
         end
 
-        def extract_thinking_from_blocks(blocks)
+        def parse_thinking_from_blocks(blocks)
           blocks.filter_map do |block|
             next unless block['type'] == 'thinking'
 
-            extract_thinking_text_from_block(block)
+            parse_thinking_text_from_block(block)
           end.join
         end
 
-        def extract_thinking_text_from_block(block)
+        def parse_thinking_text_from_block(block)
           thinking_block = block['thinking']
           return thinking_block if thinking_block.is_a?(String)