ruby_llm 2.1.1 → 2.2.0

data/lib/ruby_llm/error.rb

@@ -4,9 +4,8 @@ module RubyLLM
   # Error is the base class for provider-operation errors raised by
   # RubyLLM, including API, network, capability, and provider response-shape
   # failures. When an HTTP response is available, it wraps that response and
-  # normalizes the message across providers. When the provider refuses a
-  # conversation, #request_shape describes what it was sent. Subclasses map
-  # common HTTP status codes: BadRequestError (400), UnauthorizedError (401),
+  # normalizes the message across providers. Subclasses map common HTTP
+  # status codes: BadRequestError (400), UnauthorizedError (401),
   # PaymentRequiredError (402), ForbiddenError (403), RateLimitError (429),
   # ServerError (500), ServiceUnavailableError (502 to 504), and
   # OverloadedError (529).
@@ -20,6 +19,11 @@ module RubyLLM
   #     puts e.response&.status
   #   end
   #
+  # Providers often quote the request back in their error, so with
+  # Configuration#redact_error_messages the message carries only the HTTP
+  # status, the provider, and the provider's error code. What the provider
+  # wrote stays on #provider_message and its raw body on #response_body.
+  #
   # Local setup and programming errors, such as ConfigurationError and
   # ModelNotFoundError, inherit from StandardError directly and are not
   # caught by rescuing Error.
@@ -28,6 +32,20 @@ module RubyLLM
     # available. Its +status+ and +body+ carry the provider's reply.
     attr_reader :response
 
+    # The slug of the provider that answered, such as <tt>"openai"</tt>, or
+    # +nil+ when it is not known.
+    attr_reader :provider
+
+    # The provider's code for the error, such as
+    # <tt>"invalid_request_error"</tt> or <tt>"INVALID_ARGUMENT"</tt>, or
+    # +nil+ when it gave none that reads like a code.
+    attr_reader :error_code
+
+    # What the provider wrote about the error, or +nil+ when it wrote
+    # nothing RubyLLM could read. It can quote the request, so
+    # Configuration#redact_error_messages keeps it out of #message.
+    attr_reader :provider_message
+
     # The Protocol that rendered the request the error answered, which
     # reads its payload back for #request_shape.
     attr_accessor :request_protocol # :nodoc:
@@ -39,11 +57,28 @@ module RubyLLM
       nil
     end
 
+    ERROR_CODE = /\A[A-Za-z][\w.:-]{0,63}\z/ # :nodoc:
+
     # Creates an error with +message+. Pass +response:+ to attach the HTTP
-    # response; its body supplies the message when +message+ is +nil+.
-    def initialize(message = nil, response: nil)
+    # response, +provider:+ to name the provider's slug, +error_code:+ for
+    # the provider's code, kept only when it reads like one, and
+    # +provider_message:+ for what the provider wrote, which follows
+    # +message+, or stands in for it when +message+ is +nil+. Without
+    # either, the response body supplies the message.
+    def initialize(message = nil, response: nil, provider: nil, provider_message: nil, error_code: nil, redact: nil)
       @response = response
-      super(message || response&.body || self.class.default_message)
+      @provider = provider
+      @provider_message = provider_message
+      @error_code = error_code if error_code.is_a?(Integer) || error_code.to_s.match?(ERROR_CODE)
+      redact = RubyLLM.config.redact_error_messages if redact.nil?
+      super(full_message_for(message, redact:) || self.class.default_message)
+    end
+
+    # Returns the body of #response, or +nil+ without one. It can quote the
+    # request, so Configuration#redact_error_messages keeps it out of
+    # #message.
+    def response_body
+      response&.body
     end
 
     # Returns a RequestShape describing the conversation request the error
@@ -57,12 +92,40 @@ module RubyLLM
     #     Rails.logger.error("#{e.message}\n#{e.request_shape}")
     #     Rails.error.report(e, context: { request_shape: e.request_shape&.to_h })
     #
-    def request_shape
+    def request_shape # :nodoc:
       return unless request_protocol
       return @request_shape if defined?(@request_shape)
 
       @request_shape = request_protocol.request_shape(request_payload)
     end
+
+    private
+
+    def full_message_for(message, redact:)
+      return message || (redacted_message if remote?) if redact
+      return "#{message}: #{provider_message}" if message && provider_message
+
+      message || provider_message || response_body_message
+    end
+
+    def remote?
+      !response.nil? || !provider_message.nil?
+    end
+
+    def redacted_message
+      status = response.status if response.respond_to?(:status)
+      text = "#{provider || redacted_source} returned #{status ? "HTTP #{status}" : 'an error'}"
+      error_code ? "#{text} (#{error_code})" : text
+    end
+
+    def redacted_source
+      'The provider'
+    end
+
+    def response_body_message
+      body = response_body
+      body unless body.respond_to?(:empty?) && body.empty?
+    end
   end
 
   # Raised when a deprecated API is used and
@@ -75,12 +138,28 @@ module RubyLLM
   class DeprecationError < StandardError; end
 
   # Raised when required configuration, such as a provider API key, is
-  # missing.
+  # missing, or when a setup can't do what a chat asks of it, such as a
+  # subagent added with +tools+ that stops to wait (see AgentTool).
   class ConfigurationError < StandardError; end
 
   # Raised when RubyLLM.render_prompt cannot find the named prompt file.
   class PromptNotFoundError < StandardError; end
 
+  # Raised when a skill named by Chat#with_skills or Skill.find is missing.
+  class SkillNotFoundError < StandardError; end
+
+  # Raised when a skill's +SKILL.md+ breaks the Agent Skills specification:
+  # missing or malformed front matter, or a name or description it rejects.
+  class InvalidSkillError < StandardError; end
+
+  # Raised when a FileStore has no file at the path asked for.
+  class FileNotFoundError < StandardError; end
+
+  # Raised by a FileStore when a write would take it past its limit. Its
+  # message is what the model reads, so say what to do. The file tools and
+  # +run_code+ answer the model with it instead of failing the reply.
+  class FileStoreFullError < StandardError; end
+
   # Raised when a message role outside +:system+, +:user+, +:assistant+, or
   # +:tool+ is used.
   class InvalidRoleError < StandardError; end
@@ -92,9 +171,20 @@ module RubyLLM
   # Raised when a user message is staged while the last response still has
   # unanswered tool calls. Providers reject such a transcript, so the chat
   # refuses it up front: finish the round with #complete, recording
-  # #approve or #deny decisions for calls that require approval.
+  # #approve or #decline decisions for calls that require approval.
   class PendingToolCallsError < StandardError; end
 
+  # Raised by Chat#complete when the limit set with
+  # <tt>Chat#with_tool_options(rounds:)</tt> is reached. Every tool call of
+  # the rounds that ran has its result, so the conversation stays valid and
+  # another #complete continues with a fresh count. Not a RubyLLM::Error,
+  # which a record takes for a failed request and a job may retry.
+  class ToolRoundLimitError < StandardError; end
+
+  # Raised when a persisted tool call is declined after it started running.
+  # The call keeps running and the model receives its result.
+  class ToolCallRunningError < StandardError; end
+
   # Raised when a requested model id is not in the model registry.
   class ModelNotFoundError < StandardError; end
 
@@ -133,6 +223,24 @@ module RubyLLM
     end
   end
 
+  # Raised when the provider rejects a request because the model does not
+  # take one of its parameters, such as a temperature on a reasoning model.
+  # Chat#with_fallbacks can retry without it with <tt>drop_unsupported: true</tt>.
+  class UnsupportedParameterError < BadRequestError
+    # The parameter the provider rejected: a Symbol naming the RubyLLM
+    # setting, such as +:temperature+, +:thinking+, or +:max_output_tokens+,
+    # or the provider's own name as a String for anything else, such as a
+    # key of Chat#with_provider_options.
+    attr_reader :parameter
+
+    # Creates the error for +parameter+. Takes the other arguments of
+    # Error.new.
+    def initialize(message = nil, parameter: nil, **)
+      @parameter = parameter
+      super(message, **)
+    end
+  end
+
   # Raised for HTTP 403 responses when the API key lacks permission for the
   # requested resource.
   class ForbiddenError < Error
@@ -149,17 +257,39 @@ module RubyLLM
     end
   end
 
-  # Raised when a provider returns tool-call arguments that are not valid
-  # JSON, often because the response was truncated mid-tool-call.
+  # The parent of ToolCallTruncatedError, the error RubyLLM raises for
+  # tool-call arguments it cannot read. Other arguments that are not a JSON
+  # object do not raise: the chat answers the call with an error result
+  # and the model calls the tool again.
   class ToolCallParseError < Error
     # Returns the normalized reason generation stopped, or +nil+.
     attr_reader :finish_reason
 
     def initialize(message = nil, response: nil, finish_reason: nil) # :nodoc:
       @finish_reason = finish_reason
-      message ||= 'Provider returned malformed tool call arguments'
-      message = "#{message} (finish_reason: #{finish_reason})" if finish_reason
-      super(message, response: response)
+      super(message || default_message, response: response)
+    end
+
+    private
+
+    def default_message
+      message = 'Provider returned malformed tool call arguments'
+      finish_reason ? "#{message} (finish_reason: #{finish_reason})" : message
+    end
+  end
+
+  # Raised when the response reached its output token limit while the model
+  # was writing a tool call, so the call's arguments are incomplete. Raise
+  # the limit with Chat#with_max_output_tokens, or ask for less at once.
+  class ToolCallTruncatedError < ToolCallParseError
+    def initialize(message = nil, response: nil) # :nodoc:
+      super(message, response:, finish_reason: :max_tokens)
+    end
+
+    private
+
+    def default_message
+      'The response reached its output token limit while writing a tool call (finish_reason: max_tokens)'
     end
   end
 
@@ -207,6 +337,29 @@ module RubyLLM
     end
   end
 
+  # Raised when a request never gets an answer from the provider or an
+  # MCP server: the connection is refused or drops, TLS fails, or the
+  # request times out. RubyLLM retries provider requests first, following
+  # Configuration#max_retries. The original Faraday error is its +cause+.
+  #
+  #   rescue RubyLLM::ConnectionError => e
+  #     retry_later if e.timeout?
+  class ConnectionError < Error
+    # Returns whether the request timed out, including while opening the
+    # connection or waiting for the next bytes of a stream, rather than
+    # failing to connect.
+    def timeout?
+      return true if cause.is_a?(Faraday::TimeoutError)
+
+      cause.respond_to?(:wrapped_exception) && TIMEOUTS.any? { |timeout| cause.wrapped_exception.is_a?(timeout) }
+    end
+
+    # Some adapters, such as async-http-faraday, report a read that waited
+    # too long as a failed connection wrapping IO::TimeoutError.
+    TIMEOUTS = [Timeout::Error, IO::TimeoutError].freeze # :nodoc:
+    private_constant :TIMEOUTS
+  end
+
   # Raised for HTTP 401 responses when the API key is missing or invalid.
   class UnauthorizedError < Error
     def self.default_message # :nodoc:

data/lib/ruby_llm/evaluation/dataset.rb

@@ -42,8 +42,7 @@ module RubyLLM
       def discover(name)
         raise ArgumentError, 'An anonymous evaluation needs an explicit dataset' unless name
 
-        filename = name.gsub('::', '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
-                       .gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase
+        filename = Support::Utils.file_path(name)
         root = Prompt.root.parent.join('evals')
         paths = %w[yml yaml json jsonl].map { |extension| root.join("#{filename}.#{extension}") }.select(&:file?)
         raise ArgumentError, "Dataset not found: #{root.join(filename)}.{yml,yaml,json,jsonl}" if paths.empty?
@@ -55,8 +54,10 @@ module RubyLLM
       def read(path)
         case File.extname(path)
         when '.yml', '.yaml' then YAML.safe_load_file(path, aliases: false)
-        when '.json' then JSON.parse(File.read(path))
-        when '.jsonl' then File.foreach(path).reject { |line| line.strip.empty? }.map { |line| JSON.parse(line) }
+        when '.json' then JSON.parse(File.read(path, encoding: Encoding::UTF_8))
+        when '.jsonl'
+          File.foreach(path, encoding: Encoding::UTF_8).reject { |line| line.strip.empty? }
+              .map { |line| JSON.parse(line) }
         else raise ArgumentError, "Unsupported dataset format: #{path}"
         end
       end

data/lib/ruby_llm/evaluation/evidence.rb

@@ -26,7 +26,7 @@ module RubyLLM
                     when Message then [conversation]
                     else []
                     end.freeze
-        @tool_calls = messages.flat_map { |message| Array(message.tool_calls&.values) }
+        @tool_calls = messages.flat_map { |message| message.tool_calls.to_a }
         @tool_calls << result if result.is_a?(ToolCall)
         @tool_calls.freeze
         @output = primary_output(conversation, result)
@@ -57,12 +57,13 @@ module RubyLLM
         when Attachment then attachment_data(value)
         when Hash then serialize_hash(value, parents)
         when Array then value.map { |item| serialize(item, parents) }
+        when Numeric then value.to_f
         else serialize_object(value, parents)
         end
       end
 
       def scalar?(value)
-        [String, Symbol, Numeric, TrueClass, FalseClass, NilClass].any? { |type| value.is_a?(type) }
+        [String, Symbol, Integer, Float, TrueClass, FalseClass, NilClass].any? { |type| value.is_a?(type) }
       end
 
       def serialize_hash(value, parents)
@@ -95,7 +96,8 @@ module RubyLLM
       def conversation_data(chat, parents)
         answer = chat.messages.reverse.find { |message| message.role == :assistant }&.content
         { output: answer, messages: serialize(chat.messages, parents), complete: chat.complete?, waiting: chat.waiting?,
-          pending_approvals: chat.pending_approvals.map(&:id), cancelled: chat.cancelled?, model: chat.model.id }
+          pending_questions: chat.pending_questions.map(&:id), cancelled: chat.cancelled?,
+          model: chat.model.id }
       end
 
       def message_data(message, parents)

data/lib/ruby_llm/evaluation/report.rb

@@ -68,14 +68,18 @@ module RubyLLM
 
       # Writes a JSON report and returns the supplied path.
       def save(path)
-        File.write(path, JSON.pretty_generate(to_h))
+        File.write(File.expand_path(path), JSON.pretty_generate(to_h))
         path
       end
 
-      # Returns a readable per-case report followed by counts.
+      # Returns a readable per-case report followed by counts, and the total
+      # cost when every amount is known.
       def to_s
         rows = trials.map { |trial| trial_description(trial) }
-        [name, *rows, counts.map { |status, count| "#{count} #{status}" }.join(', ')].join("\n")
+        total = cost.total
+        summary = counts.map { |status, count| "#{count} #{status}" }.join(', ')
+        summary += format(', cost $%.4f', total) if total
+        [name, *rows, summary].join("\n")
       end
 
       def inspect_attributes # :nodoc:

data/lib/ruby_llm/evaluation/result.rb

@@ -62,7 +62,7 @@ module RubyLLM
 
       # Returns the measurement, policy, and outcome as a Hash.
       def to_h
-        { name:, status:, value: value.respond_to?(:to_h) ? value.to_h : value,
+        { name:, status:, value: value.nil? || !value.respond_to?(:to_h) ? value : value.to_h,
           reason:, minimum: @minimum, model:, evidence:,
           error: error && { class: error.class.name, message: error.message } }
       end

data/lib/ruby_llm/evaluation.rb

@@ -63,7 +63,8 @@ module RubyLLM
 
         @declared_names << name
         backend = Evaluator.new(evaluator) if evaluator
-        definitions[name] = { name:, instructions: instructions&.dup&.freeze, minimum:, evaluator: backend }.freeze
+        definitions[name] = { name:, instructions: instructions&.dup&.freeze, minimum: minimum&.to_f,
+                              evaluator: backend }.freeze
       end
 
       # Converts a custom result into evaluation evidence. The original stays available as result.

data/lib/ruby_llm/fallback.rb

@@ -5,7 +5,10 @@ module RubyLLM
   # When the active model fails with a matching error, the chat retries the
   # request with each fallback in order. Instances are yielded to
   # Chat#before_fallback and Chat#after_fallback callbacks, enriched with the
-  # details of that attempt.
+  # details of that attempt. With <tt>drop_unsupported: true</tt>, a retry
+  # of the same model without a parameter it rejected is an attempt too:
+  # its #from and #to are that model, and #dropped_parameter names what
+  # it left out.
   #
   #   chat = RubyLLM.chat(model: "gpt-4.1")
   #                 .with_fallbacks("gpt-4.1-mini", "claude-haiku-4-5")
@@ -27,11 +30,11 @@ module RubyLLM
       ServerError,
       ServiceUnavailableError,
       OverloadedError,
-      Faraday::TimeoutError,
-      Faraday::ConnectionFailed
+      ConnectionError
     ].freeze
 
-    # The model id of the fallback target.
+    # The model id of the fallback target, or of the model a retry without a
+    # rejected parameter runs on.
     attr_reader :id
 
     # The provider slug of the fallback target as a String, or +nil+.
@@ -56,6 +59,11 @@ module RubyLLM
     # The fallback attempt number, starting at 1.
     attr_reader :attempt
 
+    # The parameter a retry on the same model left out, as
+    # UnsupportedParameterError#parameter names it, or +nil+ for a fallback
+    # to another model. See Chat#with_fallbacks.
+    attr_reader :dropped_parameter
+
     # The response Message from a successful fallback attempt, or +nil+.
     attr_reader :response
 
@@ -78,6 +86,25 @@ module RubyLLM
       end
     end
 
+    # Returns whether +error+ is one of +errors+ and may move on to a
+    # fallback. A cancellation never does.
+    def self.error?(error, errors) # :nodoc:
+      !error.is_a?(CancelledError) && errors.any? { |error_class| error.is_a?(error_class) }
+    end
+
+    # Takes fallbacks off +queue+ until the block accepts one, skipping
+    # those whose model can't be resolved. Returns the fallback and the
+    # block's value, or +nil+ when none is left.
+    def self.next_usable(queue) # :nodoc:
+      while (fallback = queue.shift)
+        begin
+          return [fallback, yield(fallback)]
+        rescue ModelNotFoundError, ConfigurationError => e
+          RubyLLM.logger.warn("RubyLLM skipped fallback #{fallback.id}: #{e.message}")
+        end
+      end
+    end
+
     def inspect_attributes # :nodoc:
       { id: id, provider: provider, attempt: attempt }
     end
@@ -95,6 +122,7 @@ module RubyLLM
       @chunks_yielded = attributes.fetch(:chunks_yielded, false)
       @response = attributes[:response]
       @fallback_error = attributes[:fallback_error]
+      @dropped_parameter = attributes[:dropped_parameter]
     end
 
     def with_attempt(**attributes) # :nodoc:

data/lib/ruby_llm/file_store/converters/ocr.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Converters
+      # Reads scans, photos, and PDFs with RubyLLM.ocr. Each file costs one
+      # OCR request the first time the model reads or searches it, and is
+      # kept after that. Register it in Configuration#file_converters for
+      # the types it should read:
+      #
+      #   config.file_converters["image/*"] = RubyLLM::FileStore::Converters::OCR.new
+      #   config.file_converters["application/pdf"] = OCR.new(model: "mistral-ocr-latest")
+      #
+      class OCR
+        # Creates a converter that calls RubyLLM.ocr with +options+, such as
+        # +model:+, +provider:+, or +context:+.
+        def initialize(**options)
+          @options = options
+        end
+
+        # Returns the Markdown RubyLLM.ocr reads from +attachment+.
+        def call(attachment)
+          RubyLLM.ocr(attachment, **@options).markdown
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/file_store/converters/office/markup.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  class FileStore
+    module Converters
+      module Office
+        # Walks the tags of an Office XML part from left to right, so its
+        # time grows with the part's size whatever the part holds: nothing
+        # backtracks over unclosed tags. Names lose their prefix, since
+        # generators pick their own. Parts are binary Strings.
+        class Markup # :nodoc:
+          ENTITIES = { 'amp' => '&', 'lt' => '<', 'gt' => '>', 'quot' => '"', 'apos' => "'" }.freeze
+          ENTITY = /&(?:#(\d+)|#x(\h+)|(amp|lt|gt|quot|apos));/
+          NAME = %r{\A(?:[^\s:/]+:)?([^\s/]+)}
+          ATTRIBUTES = Hash.new do |patterns, name|
+            patterns[name] = /\s#{Regexp.escape(name)}=(?:"([^"]*)"|'([^']*)')/
+          end
+          PREFIXED = Hash.new do |patterns, name|
+            patterns[name] = /\s(?:[\w.-]+:)?#{Regexp.escape(name)}=(?:"([^"]*)"|'([^']*)')/
+          end
+          RELATIONSHIP_ID = /\s[\w.-]+:id=(?:"([^"]*)"|'([^']*)')/
+          SLASH = 47
+          QUESTION = 63
+          BANG = 33
+          private_constant :ENTITIES, :ENTITY, :NAME, :ATTRIBUTES, :PREFIXED, :RELATIONSHIP_ID, :SLASH, :QUESTION, :BANG
+
+          class << self
+            # Yields <tt>:open</tt>, <tt>:empty</tt>, or <tt>:close</tt> with
+            # each tag's name and its source, and <tt>:text</tt> with nil and
+            # the text between tags, leaving out the elements named in +skip+.
+            def scan(xml, skip: [], &)
+              new(xml.to_s.b, skip, &).run
+            end
+
+            # Returns the source of each tag named +name+.
+            def tags(xml, name)
+              found = []
+              scan(xml) { |kind, tag_name, source| found << source if tag_name == name && kind != :close }
+              found
+            end
+
+            # Returns the value of an attribute in a tag's source, unescaped.
+            def attribute(tag, name)
+              match = ATTRIBUTES[name].match(tag.to_s)
+              unescape(match[1] || match[2]) if match
+            end
+
+            # Returns the value of an attribute under any prefix, or none.
+            def prefixed(tag, name)
+              match = PREFIXED[name].match(tag.to_s)
+              unescape(match[1] || match[2]) if match
+            end
+
+            # Returns the prefixed +id+ attribute of a tag's source, whatever
+            # its prefix, which is how a part refers to a relationship.
+            def relationship_id(tag)
+              match = RELATIONSHIP_ID.match(tag.to_s)
+              unescape(match[1] || match[2]) if match
+            end
+
+            # Returns +text+ as UTF-8, with the entities XML defines replaced.
+            def unescape(text)
+              text = text.to_s.dup.force_encoding(Encoding::UTF_8).scrub
+              return text unless text.include?('&')
+
+              text.gsub(ENTITY) do
+                named = Regexp.last_match(3)
+                next ENTITIES.fetch(named) if named
+
+                character(Regexp.last_match(1)&.to_i || Regexp.last_match(2).to_i(16))
+              end
+            end
+
+            private
+
+            def character(code)
+              [code].pack('U').then { |text| text.valid_encoding? ? text : '' }
+            rescue RangeError
+              ''
+            end
+          end
+
+          def initialize(xml, skip, &block)
+            @xml = xml
+            @skip = skip
+            @block = block
+            @position = 0
+            @skipping = nil
+            @depth = 0
+          end
+
+          def run
+            while (start = @xml.index('<', @position))
+              finish = @xml.index('>', start)
+              break unless finish
+
+              @block.call(:text, nil, @xml.byteslice(@position, start - @position)) if start > @position && !@skipping
+              @position = finish + 1
+              tag(start, finish)
+            end
+          end
+
+          private
+
+          def tag(start, finish)
+            first = @xml.getbyte(start + 1)
+            return if [QUESTION, BANG].include?(first)
+
+            if first == SLASH
+              closing(name_of(@xml.byteslice(start + 2, finish - start - 2)))
+            else
+              opening(@xml.byteslice(start + 1, finish - start - 1))
+            end
+          end
+
+          def opening(source)
+            name = name_of(source)
+            empty = source.end_with?('/')
+            return count(name, empty) if @skipping
+            return start_skipping(name) if !empty && @skip.include?(name)
+
+            @block.call(empty ? :empty : :open, name, source)
+          end
+
+          def closing(name)
+            return @block.call(:close, name, nil) unless @skipping
+
+            @depth -= 1 if name == @skipping
+            @skipping = nil if @depth.zero?
+          end
+
+          def count(name, empty)
+            @depth += 1 if name == @skipping && !empty
+          end
+
+          def start_skipping(name)
+            @skipping = name
+            @depth = 1
+          end
+
+          def name_of(source)
+            source[NAME, 1].to_s
+          end
+        end
+      end
+    end
+  end
+end