ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/protocol.rb

@@ -37,6 +37,7 @@ module RubyLLM
   class Protocol
     include Streaming
     include BinaryStreaming
+    include RequestShapes
 
     # The Provider this protocol talks through.
     attr_reader :provider
@@ -77,6 +78,7 @@ module RubyLLM
     abstract :render_transcription_payload, :transcription_url, :parse_transcription_response
     abstract :render_ocr_payload, :ocr_url, :parse_ocr_response
     abstract :render_rerank_payload, :rerank_url, :parse_rerank_response
+    abstract :render_judgment_payload, :judgment_url, :parse_judgment_response
     abstract :render_count_tokens_payload, :count_tokens_url, :parse_count_tokens_response
     abstract :render_tokenization_payload, :tokenization_url, :parse_tokenization_response
     abstract :render_compaction_payload, :compaction_url, :parse_compaction_response
@@ -92,6 +94,8 @@ module RubyLLM
     def tool_approval_response(tool_call, approved:)
       Message.new(role: :tool, content: approved ? 'Approved' : 'Denied', tool_call_id: tool_call.id,
                   raw_content: render_tool_approval_response(tool_call, approved:))
+    rescue NotImplementedError
+      raise Error, "#{@provider.name} doesn't support remote tool approvals"
     end
 
     def complete(messages, tools:, temperature:, provider_options: {}, headers: {}, schema: nil, thinking: nil,
@@ -106,11 +110,15 @@ module RubyLLM
                   stream: block_given?
       )
 
+      # The caller's block can raise the error of another operation it ran.
+      yielding = false
       track_usage(:chat, on_finish: usage_recorder) do
         if block_given?
           stream_response(payload, headers) do |chunk|
             @usage_tracker.observe(chunk)
+            yielding = true
             yield chunk
+            yielding = false
           end
         else
           sync_response payload, headers
@@ -118,6 +126,9 @@ module RubyLLM
       end
     rescue NotImplementedError
       raise Error, "#{@provider.name} doesn't support chat"
+    rescue Error => e
+      claim_error(e, payload) unless yielding
+      raise
     end
 
     def render(messages, tools:, temperature:, provider_options: {}, schema: nil, thinking: nil,
@@ -140,6 +151,7 @@ module RubyLLM
       payload = apply_compaction(payload, compaction) if compaction
       payload = Support::Utils.deep_merge(payload, provider_options)
       payload = apply_provider_tools(payload, provider_tools)
+      payload = apply_tool_search(payload)
       apply_before_request_hooks(payload, before_request)
     rescue NotImplementedError
       raise Error, "#{@provider.name} doesn't support chat"
@@ -208,6 +220,9 @@ module RubyLLM
       parse_count_tokens_response post_count_tokens(payload)
     rescue NotImplementedError
       raise Error, "#{@provider.name} doesn't support token counting"
+    rescue Error => e
+      claim_error(e, payload)
+      raise
     end
 
     def list_models
@@ -233,6 +248,9 @@ module RubyLLM
       end
     rescue NotImplementedError
       raise Error, "#{@provider.name} doesn't support manual compaction"
+    rescue Error => e
+      claim_error(e, payload)
+      raise
     end
 
     def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
@@ -310,11 +328,30 @@ module RubyLLM
         url = video_request_url(payload)
       end
       response = post_video(url, payload)
-      parse_video_job(response, model:)
+      parse_video_job(response, model:, **parse_video_request(payload))
     rescue NotImplementedError
       raise Error, "#{@provider.name} doesn't support video generation"
     end
 
+    # Returns the duration in seconds and the resolution a rendered video
+    # request asked for, so the VideoJob keeps them. Protocols whose API
+    # names them differently override video_request_settings.
+    def parse_video_request(payload)
+      settings = video_request_settings(Support::Utils.deep_symbolize_keys(payload))
+      { duration: parse_video_seconds(settings[:duration]), resolution: settings[:resolution] }
+    end
+
+    def video_request_settings(request)
+      request.slice(:duration, :resolution)
+    end
+
+    def parse_video_seconds(value)
+      return value unless value.is_a?(String)
+
+      seconds = value.delete_suffix('s')
+      Integer(seconds, exception: false) || Float(seconds, exception: false)
+    end
+
     def post_video(url, payload)
       @connection.post url, payload, idempotent: false
     end
@@ -445,6 +482,16 @@ module RubyLLM
       raise Error, "#{@provider.name} doesn't support explicit content caching"
     end
 
+    def judge(input, questions:, model:, with: [], provider_options: {}) # :nodoc:
+      track_usage(:judgment) do
+        payload = render_judgment_payload(input, questions:, model:, with:, provider_options:)
+        response = @connection.post judgment_url, payload, usage: @usage_tracker
+        parse_judgment_response(response, questions:)
+      end
+    rescue NotImplementedError
+      raise Error, "#{@provider.name} doesn't support judgments"
+    end
+
     def find_cache(name)
       response = @connection.get cache_url(name)
       parse_cache_response(response.body)
@@ -470,32 +517,108 @@ module RubyLLM
       @provider.parse_error(response)
     end
 
-    def preprocess_message(message)
-      return message.without_thinking if foreign_thinking?(message)
-      return message unless auto_upload_large_files?
-      return message unless message.role == :user
+    # Returns a RequestShape describing +payload+, a conversation request
+    # this protocol rendered, as JSON parsed back with String keys.
+    # Error#request_shape asks for it when the request fails. The default
+    # returns +nil+; protocols that render conversations override it,
+    # naming each turn's role and measuring its parts:
+    #
+    #   def parse_request_shape(payload)
+    #     turns = payload['messages'].each_with_index.map do |message, index|
+    #       text = RubyLLM::RequestShape::Part.new(kind: :text, size: message['content'].length)
+    #       RubyLLM::RequestShape::Turn.new(index:, role: message['role'], parts: [text])
+    #     end
+    #     RubyLLM::RequestShape.new(turns:)
+    #   end
+    #
+    def parse_request_shape(_payload)
+      nil
+    end
+
+    # A description must not raise inside the rescue that asks for it.
+    def request_shape(payload) # :nodoc:
+      payload = Support::Utils.deep_stringify_keys(payload)
+      parse_request_shape(payload) if payload.is_a?(Hash)
+    rescue StandardError => e
+      RubyLLM.logger.debug { "#{self.class} could not describe its request: #{e.class}" }
+      nil
+    end
+
+    def preprocess_message(message, unsupported_attachment: nil)
+      message = without_foreign_native_content(message) if foreign_native_content?(message)
       return message if message.attachments.empty?
 
-      uploaded = message.attachments.map { |attachment| preprocess_attachment(attachment) }
-      return message if uploaded == message.attachments
+      upload = message.role == :user && auto_upload_large_files?
+      prepared = message.attachments.map do |attachment|
+        replacement = replace_unsupported_attachment(attachment, unsupported_attachment, message)
+        upload ? preprocess_attachment(replacement) : replacement
+      end
+      return message if prepared == message.attachments
+
+      message.with_attachments(prepared)
+    end
+
+    def supported_attachment?(_attachment) # :nodoc:
+      true
+    end
 
-      message.with_attachments(uploaded)
+    # A provider can delete a file RubyLLM uploaded for an attachment. When
+    # a request fails naming such a file, or with a 404 that names none, the
+    # attachments forget those uploads so the next request uploads them
+    # again. Returns the uploads it forgot.
+    def discard_missing_uploads(messages, error)
+      scope = provider_upload_scope
+      missing_uploads(messages, error, scope).map do |attachment, upload|
+        attachment.provider_uploads.delete(scope)
+        StoredUploads.new(@provider, attachment.provider_file_store).forget(upload)
+        upload
+      end
     end
 
     private
 
-    # A thinking signature is opaque to every provider but the one that
-    # issued it, so a message another provider produced replays without
-    # its thinking. A message with no known producer replays as it is.
-    def foreign_thinking?(message)
-      return false unless message.role == :assistant && carries_thinking?(message)
+    # An error raised before the request went out answers no request.
+    def claim_error(error, payload)
+      return if error.request_protocol || error.response.nil? || payload.nil?
+
+      error.request_protocol = self
+      error.request_payload = payload
+    end
+
+    # A thinking signature or a provider-shaped content block belongs to the
+    # model that produced it, and one provider can serve models that do not
+    # read each other's, so a message another model produced replays
+    # without them. A message with no known producer replays as it is.
+    def foreign_native_content?(message)
+      return false unless message.role == :assistant && carries_native_content?(message)
+
+      producer = producing_entry(message)
+      !producer.nil? && (producer.provider != @provider.slug || !current_model?(producer.model))
+    end
+
+    # Usage records the id a chat resolved its model to, and a message built
+    # by hand may name the model by an alias of it.
+    def current_model?(model_id)
+      return true if @model.nil? || model_id == @model.id
+
+      RubyLLM.models.find(model_id, provider: @provider.slug, config: @config).id == @model.id
+    rescue ModelNotFoundError
+      false
+    end
 
-      producer = producer_slug(message)
-      !producer.nil? && producer != @provider.slug
+    # Another model of this provider gets the raw content any of its models
+    # reads, such as a tool search, and none of the producing model's own.
+    def without_foreign_native_content(message)
+      portable = portable_raw_content(message) if producer_slug(message) == @provider.slug
+      message.without_native_content(raw_content: portable)
     end
 
-    def carries_thinking?(message)
-      return true if message.thinking || message.raw_reasoning
+    def portable_raw_content(_message)
+      nil
+    end
+
+    def carries_native_content?(message)
+      return true if message.thinking || message.raw_reasoning || message.raw_content
 
       message.tool_call? && message.tool_calls.each_value.any?(&:thought_signature)
     end
@@ -503,7 +626,19 @@ module RubyLLM
     # Only a usage entry names the producer: a model id alone can belong
     # to several providers.
     def producer_slug(message)
-      message.ruby_llm_usage_entries.reverse.find(&:succeeded?)&.provider
+      producing_entry(message)&.provider
+    end
+
+    def producing_entry(message)
+      message.ruby_llm_usage_entries.reverse.find(&:succeeded?)
+    end
+
+    # A thinking signature can be another provider's opaque blob, and a
+    # message restored without its usage could come from anyone, so a
+    # protocol that verifies signatures gets back only its own.
+    def own_signature(message)
+      signature = message.thinking&.signature
+      signature if signature && producer_slug(message) == @provider.slug
     end
 
     def resolve_provider_tools_for_request(entries)
@@ -534,11 +669,16 @@ module RubyLLM
       payload[:tools] = Array(payload[:tools]) + entries
     end
 
-    def track_usage(operation, on_finish: nil)
+    def apply_tool_search(payload)
+      payload
+    end
+
+    def track_usage(operation, on_finish: nil, model_id: nil)
       @usage_tracker = Accounting::Usage::Tracker.new(
         operation:,
         provider: @provider,
         model: @model,
+        model_id:,
         config: @config,
         on_finish:
       )
@@ -569,8 +709,24 @@ module RubyLLM
       return attachment if attachment.provider_file?
       return attachment unless upload_large_attachment?(attachment)
 
-      ensure_provider_file_size!(attachment)
-      Attachment.new(provider_upload(attachment), config: @config)
+      Attachment.new(provider_upload(attachment), resolution: attachment.resolution, config: @config)
+    end
+
+    def supported_message_attachment?(_message, attachment)
+      supported_attachment?(attachment)
+    end
+
+    def replace_unsupported_attachment(attachment, handler, message)
+      return attachment if handler.nil? || supported_message_attachment?(message, attachment)
+
+      replacement = handler.call(attachment)
+      return attachment if replacement.nil?
+      unless replacement.is_a?(Attachment)
+        raise ArgumentError, 'convert_unsupported_attachments must return a RubyLLM::Attachment or nil'
+      end
+      raise UnsupportedAttachmentError, replacement.mime_type unless supported_message_attachment?(message, replacement)
+
+      replacement
     end
 
     # Uploads are memoized per provider on the attachment itself, so a chat
@@ -582,8 +738,9 @@ module RubyLLM
       upload = attachment.provider_uploads[scope]
       return upload if upload && !upload.expired?
 
-      attachment.provider_uploads[scope] =
+      attachment.provider_uploads[scope] = StoredUploads.new(@provider, attachment.provider_file_store).fetch do
         @provider.upload_file(attachment, **provider_file_upload_options(attachment))
+      end
     end
 
     # A file id belongs to the account that uploaded it, so the memo is keyed
@@ -595,6 +752,19 @@ module RubyLLM
       "#{@provider.slug}:#{Digest::SHA256.hexdigest(credentials.join("\0"))}"
     end
 
+    def missing_uploads(messages, error, scope)
+      uploads = messages.flat_map(&:attachments).filter_map do |attachment|
+        upload = attachment.provider_uploads[scope]
+        [attachment, upload] if upload
+      end
+      named = uploads.select { |_, upload| names_file?(error, upload.id) }
+      named.empty? && error.response&.status == 404 ? uploads : named
+    end
+
+    def names_file?(error, id)
+      [error.message, error.response&.body].any? { |text| text.to_s.include?(id) }
+    end
+
     def upload_large_attachment?(attachment)
       size = attachment.byte_size
       size && size > default_large_file_upload_threshold && provider_file_attachable?(attachment)
@@ -604,10 +774,6 @@ module RubyLLM
       Float::INFINITY
     end
 
-    def provider_file_upload_limit
-      nil
-    end
-
     def provider_file_attachable?(_attachment)
       false
     end
@@ -616,20 +782,6 @@ module RubyLLM
       {}
     end
 
-    def ensure_provider_file_size!(attachment)
-      limit = provider_file_upload_limit
-      return unless limit && attachment.byte_size.to_i > limit
-
-      raise Error, "#{@provider.name} file uploads support files up to #{format_bytes(limit)}; " \
-                   "#{attachment.filename} is #{format_bytes(attachment.byte_size)}"
-    end
-
-    def format_bytes(bytes)
-      return 'unknown size' unless bytes
-
-      "#{(bytes.to_f / (1024 * 1024)).round(1)} MB"
-    end
-
     def validate_paint_inputs!(with:, mask:)
       return if with.nil? && mask.nil?
 

data/lib/ruby_llm/protocols/anthropic/chat.rb

@@ -11,7 +11,6 @@ module RubyLLM
         }.freeze
 
         ANTHROPIC_INLINE_REQUEST_LIMIT = 24 * 1024 * 1024
-        ANTHROPIC_FILE_UPLOAD_LIMIT = 500 * 1024 * 1024
         CACHE_CONTROL_TYPE = 'ephemeral'
         PROMPT_CACHE_OPTIONS = %i[ttl].freeze
         BETA_HEADER = 'anthropic-beta'
@@ -19,6 +18,7 @@ module RubyLLM
         COMPACTION_EDIT_TYPE = 'compact_20260112'
         COUNT_TOKENS_KEYS = %i[model messages system tools tool_choice thinking].freeze
         THINKING_BLOCK_TYPES = %w[thinking redacted_thinking].freeze
+        PORTABLE_BLOCK_TYPES = %w[text tool_use].freeze
 
         module_function
 
@@ -64,7 +64,7 @@ module RubyLLM
             thinking: thinking,
             citations: citations,
             caching: caching
-          ).slice(*COUNT_TOKENS_KEYS)
+          ).then { |payload| Tools.apply_tool_search(payload) }.slice(*COUNT_TOKENS_KEYS)
         end
 
         def parse_count_tokens_response(response)
@@ -90,7 +90,7 @@ module RubyLLM
           # own block in the resulting array.
           system_messages.flat_map do |msg|
             blocks = Media.format_content(msg.content, msg.attachments).dup
-            cache_boundary?(msg, caching:) ? inject_cache_control(blocks, caching:) : blocks
+            cache_boundary?(msg, caching:) ? inject_cache_control(blocks, caching:, ttl: msg.cache_ttl) : blocks
           end
         end
 
@@ -100,7 +100,7 @@ module RubyLLM
             model: model.id,
             messages: format_messages(chat_messages, thinking:, citations:, caching:),
             stream: stream,
-            max_tokens: max_output_tokens || model.max_output_tokens || 4096
+            max_tokens: max_output_tokens || model.max_output_tokens || DEFAULT_MAX_OUTPUT_TOKENS
           }
 
           add_thinking_fields(payload, thinking, model)
@@ -115,7 +115,7 @@ module RubyLLM
           messages.each do |msg|
             if msg.tool_result?
               tool_result_blocks << Tools.format_tool_result_block(msg)
-              inject_cache_control(tool_result_blocks, caching:) if cache_boundary?(msg, caching:)
+              inject_cache_control(tool_result_blocks, caching:, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
               next
             end
 
@@ -196,10 +196,6 @@ module RubyLLM
           ANTHROPIC_INLINE_REQUEST_LIMIT
         end
 
-        def provider_file_upload_limit
-          ANTHROPIC_FILE_UPLOAD_LIMIT
-        end
-
         def provider_file_attachable?(attachment)
           attachment.image? || attachment.pdf? || attachment.text?
         end
@@ -329,6 +325,7 @@ module RubyLLM
             output_tokens: usage['output_tokens'],
             cache_read_tokens: extract_cache_read_tokens(data),
             cache_write_tokens: extract_cache_write_tokens(data),
+            cache_write_tokens_by_ttl: extract_cache_write_by_ttl(data),
             thinking_tokens: thinking_tokens,
             server_tool_use: usage['server_tool_use'],
             finish_reason: normalize_finish_reason(data['stop_reason']),
@@ -352,12 +349,20 @@ module RubyLLM
           end
         end
 
+        def portable_raw_content(message)
+          blocks = message.raw_content
+          return unless blocks.is_a?(Array) && blocks.any? { |block| Tools.tool_search_block?(block) }
+
+          kept = blocks.reject { |block| THINKING_BLOCK_TYPES.include?(block['type']) }
+          kept if kept.all? { |block| PORTABLE_BLOCK_TYPES.include?(block['type']) || Tools.tool_search_block?(block) }
+        end
+
         # Turns that used server tools replay their provider-shaped blocks
         # verbatim: the API requires the tool_use/result blocks and their
         # citations back exactly as returned.
         def format_raw_assistant_message(msg, caching: nil)
           blocks = msg.raw_content.dup
-          inject_cache_control(blocks, caching:) if cache_boundary?(msg, caching:)
+          inject_cache_control(blocks, caching:, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
 
           { role: 'assistant', content: blocks }
         end
@@ -366,7 +371,7 @@ module RubyLLM
           content_blocks = msg.role == :assistant ? format_thinking_blocks(msg) : []
 
           append_formatted_content(content_blocks, msg, citations: citations)
-          inject_cache_control(content_blocks, caching:) if cache_boundary?(msg, caching:)
+          inject_cache_control(content_blocks, caching:, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
 
           {
             role: convert_role(msg.role),
@@ -386,7 +391,7 @@ module RubyLLM
               input: tool_call.arguments
             }
           end
-          inject_cache_control(content_blocks, caching:) if cache_boundary?(msg, caching:)
+          inject_cache_control(content_blocks, caching:, ttl: msg.cache_ttl) if cache_boundary?(msg, caching:)
 
           {
             role: 'assistant',
@@ -404,24 +409,16 @@ module RubyLLM
           blocks = msg.raw_reasoning['anthropic'] if msg.raw_reasoning.is_a?(Hash)
           return Support::Utils.deep_dup(blocks) if blocks
 
-          [build_thinking_block(msg.thinking)].compact
+          [build_thinking_block(msg)].compact
         end
 
-        def build_thinking_block(thinking)
-          return nil unless thinking
+        # Claude takes a thinking block back only with the signature it issued.
+        def build_thinking_block(msg)
+          signature = own_signature(msg) if msg.thinking
+          return unless signature
 
-          if thinking.text
-            {
-              type: 'thinking',
-              thinking: thinking.text,
-              signature: thinking.signature
-            }.compact
-          elsif thinking.signature
-            {
-              type: 'redacted_thinking',
-              data: thinking.signature
-            }
-          end
+          text = msg.thinking.text
+          text ? { type: 'thinking', thinking: text, signature: } : { type: 'redacted_thinking', data: signature }
         end
 
         def append_formatted_content(content_blocks, msg, citations: false)
@@ -432,22 +429,23 @@ module RubyLLM
           caching != false && message.cache_until_here?
         end
 
-        def inject_cache_control(blocks, caching: nil)
+        def inject_cache_control(blocks, caching: nil, ttl: nil)
           return blocks if blocks.empty?
 
           last = blocks.last
           return blocks if last.is_a?(Hash) && (last[:cache_control] || last['cache_control'])
           return blocks unless last.is_a?(Hash)
 
-          blocks[-1] = last.merge(cache_control: prompt_cache_control(caching))
+          blocks[-1] = last.merge(cache_control: prompt_cache_control(caching, ttl:))
           blocks
         end
 
-        def prompt_cache_control(caching = nil)
+        def prompt_cache_control(caching = nil, ttl: nil)
           options = prompt_cache_options(caching)
+          ttl ||= options[:ttl]
 
           { type: CACHE_CONTROL_TYPE }.tap do |control|
-            control[:ttl] = options[:ttl] if options[:ttl]
+            control[:ttl] = ttl if ttl
           end
         end
 
@@ -485,7 +483,8 @@ module RubyLLM
 
         def build_thinking_payload(thinking, model, max_tokens)
           return nil unless thinking&.enabled?
-          return { thinking: { type: 'disabled' } } if thinking.enabled == false
+
+          return { thinking: { type: thinking_off_type(model) } } if thinking.enabled == false
 
           effort = resolve_effort(thinking)
           return nil if effort == 'none'
@@ -533,6 +532,14 @@ module RubyLLM
           budget.clamp(minimum, [max_tokens - 1, minimum].max)
         end
 
+        def thinking_off_type(model)
+          between_tools_off?(model.id) ? 'between_tools' : 'disabled'
+        end
+
+        def between_tools_off?(model_id)
+          @provider.class.respond_to?(:between_tools_off?) && @provider.class.between_tools_off?(model_id)
+        end
+
         def resolve_effort(thinking)
           effort = thinking.effort.to_s
           effort.empty? ? nil : effort

data/lib/ruby_llm/protocols/anthropic/media.rb

@@ -7,11 +7,17 @@ module RubyLLM
       module Media
         module_function
 
+        def supported_attachment?(attachment)
+          attachment.provider_file? || %i[image pdf text].include?(attachment.type)
+        end
+
         def format_content(content, attachments = [], citations: false)
           parts = []
           parts << format_text(content) unless content.nil? || content.empty?
 
           attachments.each do |attachment|
+            raise UnsupportedAttachmentError, attachment.mime_type unless supported_attachment?(attachment)
+
             if attachment.provider_file?
               parts << format_provider_file(attachment, citations: citations)
               next
@@ -24,8 +30,6 @@ module RubyLLM
               parts << format_pdf(attachment, citations: citations)
             when :text
               parts << (citations ? format_text_document(attachment) : format_text_file(attachment))
-            else
-              raise UnsupportedAttachmentError, attachment.mime_type
             end
           end
 

data/lib/ruby_llm/protocols/anthropic/models.rb

@@ -102,6 +102,16 @@ module RubyLLM
           breakdown.values.compact.sum
         end
 
+        def extract_cache_write_by_ttl(data)
+          breakdown = message_usage(data)['cache_creation'] || delta_usage(data)['cache_creation']
+          return unless breakdown.is_a?(Hash)
+
+          breakdown.filter_map do |key, count|
+            ttl = key[/\Aephemeral_(.+)_input_tokens\z/, 1]
+            [ttl, count] if ttl
+          end.to_h
+        end
+
         def message_usage(data)
           aggregate_usage(data.dig('message', 'usage'))
         end
@@ -121,7 +131,12 @@ module RubyLLM
 
           usage.merge(ITERATION_TOKEN_KEYS.to_h do |key|
             [key, iterations.sum { |iteration| iteration[key].to_i }]
-          end)
+          end, { 'cache_creation' => iteration_cache_creation(iterations) }.compact)
+        end
+
+        def iteration_cache_creation(iterations)
+          breakdowns = iterations.map { |iteration| iteration['cache_creation'] }.grep(Hash)
+          breakdowns.reduce { |total, breakdown| total.merge(breakdown) { |_, sum, count| sum.to_i + count.to_i } }
         end
       end
     end