ruby_llm 2.0.0 → 2.1.0

data/lib/ruby_llm/tokens.rb

@@ -15,6 +15,10 @@ module RubyLLM
   class Tokens
     include Support::Inspectable
 
+    READERS = %i[
+      input output cache_read cache_write thinking cache_write_by_ttl server_tool_use reported_cost
+    ].freeze # :nodoc:
+
     # The number of standard (non-cached) input tokens, or +nil+ if the
     # provider did not report it.
     attr_reader :input
@@ -32,14 +36,30 @@ module RubyLLM
     # +nil+ if the provider did not report it.
     attr_reader :cache_write
 
+    # The tokens in #cache_write split by cache lifetime, as a Hash of the
+    # TTL Strings that <tt>with_caching(ttl:)</tt> accepts and Integer
+    # counts, or +nil+ if the provider did not report a split. Lifetimes
+    # with no writes are left out. Providers that price a longer lifetime
+    # higher, such as Anthropic's one-hour cache, use it to price #cache_write.
+    #
+    #   response.tokens.cache_write_by_ttl # => {"5m" => 1024, "1h" => 2048}
+    #
+    attr_reader :cache_write_by_ttl
+
     # The number of thinking (reasoning) tokens, or +nil+ if the provider
     # does not report them.
     attr_reader :thinking
 
-    # The provider's server-tool usage counters as a Hash, such as
-    # <tt>{"web_search_requests" => 2}</tt>, or +nil+ if the provider did
-    # not report any. Counters are provider-shaped and billed per use, not
-    # in tokens.
+    # How many times each provider-executed tool ran, as a Hash of String
+    # keys and Integer counts, or +nil+ if the provider reported none.
+    # Web searches are <tt>"web_search_requests"</tt> on every provider
+    # that reports them. Other tools follow the same form, such as
+    # <tt>"web_fetch_requests"</tt> and <tt>"code_execution_requests"</tt>.
+    # Tools that did not run are left out. Providers bill these per use,
+    # not in tokens.
+    #
+    #   response.tokens.server_tool_use # => {"web_search_requests" => 2}
+    #
     attr_reader :server_tool_use
 
     # The exact cost of the call in US dollars as reported by the
@@ -54,16 +74,20 @@ module RubyLLM
     #   tokens = RubyLLM::Tokens.new(input: 100, output: 20)
     #   RubyLLM.models.find("gpt-5.6").cost_for(tokens).total
     #
-    # +server_tool_use:+ holds provider tool-usage counters;
+    # +cache_write_by_ttl:+ splits the cache writes by lifetime, keyed as
+    # #cache_write_by_ttl describes;
+    # +server_tool_use:+ counts the provider-executed tools that ran, keyed
+    # as #server_tool_use describes;
     # +reported_cost:+ holds the provider's total price in US dollars.
     def initialize(input: nil, output: nil, cache_read: nil, cache_write: nil, thinking: nil,
-                   server_tool_use: nil, reported_cost: nil)
+                   cache_write_by_ttl: nil, server_tool_use: nil, reported_cost: nil)
       @input = input
       @output = output
       @cache_read = cache_read
       @cache_write = cache_write
       @thinking = thinking
-      @server_tool_use = server_tool_use
+      @cache_write_by_ttl = positive_counts(cache_write_by_ttl)
+      @server_tool_use = positive_counts(server_tool_use)
       @reported_cost = reported_cost
     end
 
@@ -78,23 +102,33 @@ module RubyLLM
         reported = tokens.filter_map { |usage| usage.public_send(component) }
         [component, reported.empty? ? nil : reported.sum]
       end
-      new(**values, server_tool_use: aggregate_server_tool_use(tokens))
+      new(**values, cache_write_by_ttl: aggregate_counts(tokens, :cache_write_by_ttl),
+                    server_tool_use: aggregate_counts(tokens, :server_tool_use))
     end
 
-    def self.aggregate_server_tool_use(tokens) # :nodoc:
-      reported = tokens.filter_map(&:server_tool_use)
+    # Returns Tokens with each count +other+ reported replacing this one.
+    def merge(other) # :nodoc:
+      self.class.new(**READERS.to_h do |reader|
+        value = other.public_send(reader)
+        [reader, value.nil? ? public_send(reader) : value]
+      end)
+    end
+
+    def self.aggregate_counts(tokens, reader) # :nodoc:
+      reported = tokens.filter_map(&reader)
       return nil if reported.empty?
 
       reported.each_with_object({}) do |counters, total|
-        counters.each { |tool, count| total[tool] = total.fetch(tool, 0) + count.to_i }
+        counters.each { |key, count| total[key] = total.fetch(key, 0) + count.to_i }
       end
     end
-    private_class_method :aggregate_server_tool_use
+    private_class_method :aggregate_counts
 
     # Returns the counts as a hash with keys +:input_tokens+,
     # +:output_tokens+, +:cache_read_tokens+, +:cache_write_tokens+, and
-    # +:thinking_tokens+, omitting +nil+ counts. Includes +:server_tool_use+
-    # when reported. The provider's reported cost is available separately
+    # +:thinking_tokens+, omitting +nil+ counts. Includes
+    # +:cache_write_tokens_by_ttl+ and +:server_tool_use+ when reported.
+    # The provider's reported cost is available separately
     # through #reported_cost.
     #
     #   response.tokens.to_h
@@ -107,6 +141,7 @@ module RubyLLM
         cache_read_tokens: cache_read,
         cache_write_tokens: cache_write,
         thinking_tokens: thinking,
+        cache_write_tokens_by_ttl: cache_write_by_ttl,
         server_tool_use: server_tool_use
       }.compact
     end
@@ -118,8 +153,16 @@ module RubyLLM
         cache_read: cache_read,
         cache_write: cache_write,
         thinking: thinking,
+        cache_write_by_ttl: cache_write_by_ttl,
         server_tool_use: server_tool_use
       }.compact
     end
+
+    private
+
+    def positive_counts(counts)
+      used = counts.to_h.to_h { |key, count| [key.to_s, count.to_i] }.select { |_, count| count.positive? }
+      used unless used.empty?
+    end
   end
 end

data/lib/ruby_llm/tool.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'delegate'
 require 'schematist'
 
 module RubyLLM
@@ -49,6 +50,7 @@ module RubyLLM
       @parameters_schema_definition
       @requires_approval
       @approval_resolver
+      @deferred
     ].freeze
     private_constant :DUPED_INHERITED_CONFIG, :COPIED_INHERITED_CONFIG
 
@@ -74,9 +76,13 @@ module RubyLLM
       #   WeatherLookup.tool_name  # => "weather_lookup"
       #
       def tool_name
+        return @tool_name if @tool_name
+
         normalized = name.to_s.dup.force_encoding('UTF-8').unicode_normalize(:nfkd)
         ascii_name = normalized.encode('ASCII', replace: '').gsub(/[^a-zA-Z0-9_-]/, '-')
-        Support::Utils.underscore(ascii_name).delete_suffix('_tool')
+        derived = Support::Utils.underscore(ascii_name).delete_suffix('_tool')
+        @tool_name = derived if name
+        derived
       end
 
       # :call-seq:
@@ -107,8 +113,8 @@ module RubyLLM
       #     parameter :units, type: :string, description: "metric or imperial", required: false
       #   end
       #
-      def parameter(name, **options)
-        declared_parameters[name] = Parameter.new(name, **options)
+      def parameter(name, **)
+        declared_parameters[name] = Parameter.new(name, **)
       end
 
       def declared_parameters # :nodoc:
@@ -195,10 +201,29 @@ module RubyLLM
         self
       end
 
+      # Keeps this tool's definition out of the model's context until the
+      # provider's tool search loads it. Pass +defer: false+ to
+      # Chat#with_tools to offer it up front in one chat. See
+      # Chat#deferred_tools.
+      #
+      #   class DeepResearch < RubyLLM::Tool
+      #     description "Runs a multi-step web investigation"
+      #     defer
+      #   end
+      #
+      def defer
+        @deferred = true
+      end
+
+      def deferred? # :nodoc:
+        @deferred || false
+      end
+
       def split_result(result) # :nodoc:
         case result
         when Attachment then ['', [result]]
         when Array then split_array_result(result)
+        when MCP::Result then split_result(result.content)
         else [result_content(result), []]
         end
       end
@@ -258,6 +283,9 @@ module RubyLLM
       self.class.provider_options
     end
 
+    class Deferred < SimpleDelegator # :nodoc:
+    end
+
     # Returns the JSON Schema for the tool's arguments, whether declared
     # explicitly or inferred from the +execute+ signature.
     def parameters_schema
@@ -313,6 +341,25 @@ module RubyLLM
       raise NotImplementedError, 'Subclasses must implement #execute'
     end
 
+    # Reports what the tool is doing while #execute runs, so an application
+    # can show it before the result arrives. Chat#after_tool_progress
+    # callbacks receive it as a Progress. Pass +value:+ and +total:+ when
+    # you can count the work. Outside a chat, the report goes nowhere.
+    #
+    #   def execute(url:)
+    #     progress "Downloading #{File.basename(url)}"
+    #     pages = Scanner.pages(url)
+    #     pages.each_with_index.map do |page, index|
+    #       progress "Reading page #{index + 1} of #{pages.size}", value: index + 1, total: pages.size
+    #       page.text
+    #     end.join("\n")
+    #   end
+    #
+    def progress(message = nil, value: nil, total: nil)
+      Support::ProgressReporter.report(Progress.new(value:, total:, message:))
+      nil
+    end
+
     protected
 
     def validate_keyword_arguments(arguments) # :nodoc:

data/lib/ruby_llm/tool_call.rb

@@ -21,7 +21,10 @@ module RubyLLM
     # The name of the tool the model wants to invoke.
     attr_reader :name
 
-    # The arguments the model supplied for the invocation, as a Hash.
+    # The arguments the model supplied for the invocation, as a Hash. Keys
+    # are ordered the way PostgreSQL +jsonb+ and MySQL +json+ store them,
+    # shorter keys first, so a call reloaded from the database renders the
+    # same request bytes and keeps the provider's prompt cache.
     attr_reader :arguments
 
     # Returns +true+ if the provider executes this call after approval.
@@ -35,7 +38,7 @@ module RubyLLM
     def initialize(id:, name:, arguments: {}, thought_signature: nil, remote: false) # :nodoc:
       @id = id
       @name = name
-      @arguments = arguments
+      @arguments = canonical_arguments(arguments)
       @thought_signature = thought_signature
       @remote = remote
     end
@@ -54,5 +57,17 @@ module RubyLLM
         thought_signature: @thought_signature
       }.compact
     end
+
+    private
+
+    def canonical_arguments(value)
+      case value
+      when Hash
+        value.sort_by { |key, _| [key.to_s.bytesize, key.to_s] }
+             .to_h { |key, nested| [key, canonical_arguments(nested)] }
+      when Array then value.map { |item| canonical_arguments(item) }
+      else value
+      end
+    end
   end
 end

data/lib/ruby_llm/transcription.rb

@@ -61,6 +61,8 @@ module RubyLLM
       Cost.new(tokens:, model: model_info, category: :audio_tokens)
     end
 
+    attr_writer :model_info # :nodoc:
+
     def model_info # :nodoc:
       @model_info ||= RubyLLM.models.find(model)
     rescue ModelNotFoundError
@@ -96,6 +98,8 @@ module RubyLLM
     # +:segment+ or +:character+; unsupported granularities raise ArgumentError.
     # +provider_options:+ takes options in the provider's request vocabulary
     # and merges them into the rendered request as-is.
+    # +owner:+ attributes the usage to a record, such as a user, and wins
+    # over RubyLLM.with_usage_owner.
     #
     # Given a block, the transcript streams: each TranscriptionChunk is
     # yielded as it arrives and the completed Transcription is still
@@ -124,6 +128,7 @@ module RubyLLM
                         speaker_references: nil,
                         provider_options: {},
                         metadata: nil,
+                        owner: nil,
                         &block)
       config = context&.config || RubyLLM.config
       model ||= config.default_transcription_model
@@ -143,9 +148,11 @@ module RubyLLM
       }
 
       RubyLLM.instrument('transcription.ruby_llm', payload, config: config) do |event|
-        result = provider_instance.transcribe(audio_file, model:, language:, format:, timestamps:, speaker_names:,
-                                                          speaker_references:, provider_options:, prompt:,
-                                                          temperature:, &block)
+        result = Accounting::Usage.owned_by(owner) do
+          provider_instance.transcribe(audio_file, model:, language:, format:, timestamps:, speaker_names:,
+                                                   speaker_references:, provider_options:, prompt:,
+                                                   temperature:, &block)
+        end
         event[:result] = result
         event[:response_model] = result.model
         event[:tokens] = result.tokens

data/lib/ruby_llm/transport/connection.rb

@@ -4,6 +4,7 @@ require 'faraday'
 require 'faraday/multipart'
 require 'faraday/retry'
 require 'ruby_llm/transport/error_middleware'
+require 'ruby_llm/transport/signing_middleware'
 require 'ruby_llm/transport/usage_middleware'
 require 'timeout'
 
@@ -15,7 +16,17 @@ module RubyLLM
       IDEMPOTENT_KEY = :ruby_llm_idempotent
       STREAM_PROGRESS_KEY = :ruby_llm_stream_progress
 
-      attr_reader :provider, :connection, :config
+      # The key a shared Faraday connection is cached under: the provider,
+      # whether it streams, and everything ::build reads.
+      Settings = Struct.new(
+        :provider, :stream, :api_base, :adapter, :timeout, :proxy, :logger, :log_bodies, :log_regexp_timeout,
+        :max_retries, :retry_interval, :retry_max_interval, :retry_interval_randomness, :retry_backoff_factor,
+        keyword_init: true
+      )
+
+      CACHE = Support::ProcessCache.new
+
+      attr_reader :provider, :config
 
       def self.basic(config = RubyLLM.config, &)
         Faraday.new do |f|
@@ -32,34 +43,61 @@ module RubyLLM
         end
       end
 
-      def initialize(provider, config, api_base: nil)
+      def self.cache
+        CACHE
+      end
+
+      # The result is shared across threads and contexts, so the middleware
+      # stack is built now rather than lazily, without a lock, on the first
+      # request, and the defaults every request copies are frozen.
+      def self.build(settings)
+        connection = Faraday.new(settings.api_base) do |faraday|
+          setup_timeout(faraday, settings)
+          setup_logging(faraday, settings)
+          setup_retry(faraday, settings)
+          setup_middleware(faraday, settings)
+          setup_http_proxy(faraday, settings)
+        end
+        [connection.headers, connection.params, connection.options].each(&:freeze)
+        connection.app
+        connection
+      end
+
+      def initialize(provider, config, api_base: nil, headers: {}, signer: nil)
         @provider = provider
         @config = config
+        @headers = headers
+        @signer = signer
+        @settings = settings_for(api_base || provider.api_base)
+        @stream_settings = @settings.dup.tap { |settings| settings.stream = true }.freeze
+        connection
+      end
 
-        @connection = Faraday.new(api_base || provider.api_base) do |faraday|
-          setup_timeout(faraday)
-          setup_logging(faraday)
-          setup_retry(faraday)
-          setup_middleware(faraday)
-          setup_http_proxy(faraday)
-        end
+      # The Faraday connection shared by every request with these settings in
+      # this process. Looked up per request, so an object that outlives a fork
+      # never reaches the parent's sockets. Streaming requests get their own:
+      # some adapters, such as httpx, only stream if their first request did.
+      def connection(stream: false)
+        settings = stream ? @stream_settings : @settings
+        CACHE.fetch(settings) { self.class.build(settings) }
       end
 
-      def post(url, payload, usage: nil, idempotent: true, &)
+      def post(url, payload, usage: nil, idempotent: true, stream: false, &)
         instrument_request(:post, url) do
-          @connection.post url, payload do |req|
-            req.headers.merge! @provider.headers
+          response = connection(stream:).post url, payload do |req|
+            prepare(req)
             set_usage_tracker(req, usage) if usage
             mark_non_idempotent(req) unless idempotent
             yield req if block_given?
           end
+          release_request(response)
         end
       end
 
       def get(url, &)
         instrument_request(:get, url) do
-          @connection.get url do |req|
-            req.headers.merge! @provider.headers
+          connection.get url do |req|
+            prepare(req)
             yield req if block_given?
           end
         end
@@ -67,17 +105,18 @@ module RubyLLM
 
       def patch(url, payload, &)
         instrument_request(:patch, url) do
-          @connection.patch url, payload do |req|
-            req.headers.merge! @provider.headers
+          response = connection.patch url, payload do |req|
+            prepare(req)
             yield req if block_given?
           end
+          release_request(response)
         end
       end
 
       def delete(url, &)
         instrument_request(:delete, url) do
-          @connection.delete url do |req|
-            req.headers.merge! @provider.headers
+          connection.delete url do |req|
+            prepare(req)
             yield req if block_given?
           end
         end
@@ -99,35 +138,74 @@ module RubyLLM
         end
       end
 
-      def setup_timeout(faraday)
-        faraday.options.timeout = @config.request_timeout
+      # The response outlives the call as Message#raw and other results. A chat
+      # request body is the whole serialized conversation, and a streaming
+      # callback closes over the payload it was built from.
+      def release_request(response)
+        response.env.request_body = nil
+        response.env.request.on_data = nil
+        response.env.request.context&.delete(ErrorMiddleware::STREAM_RESET_KEY)
+        response
       end
 
-      def setup_logging(faraday)
+      def settings_for(api_base)
+        Settings.new(
+          provider: @provider.class,
+          stream: false,
+          api_base: api_base,
+          adapter: @config.faraday_adapter,
+          timeout: @config.request_timeout,
+          proxy: @config.http_proxy,
+          logger: RubyLLM.logger,
+          log_bodies: RubyLLM.logger.debug?,
+          log_regexp_timeout: @config.log_regexp_timeout,
+          max_retries: @config.max_retries,
+          retry_interval: @config.retry_interval,
+          retry_max_interval: @config.retry_max_interval,
+          retry_interval_randomness: @config.retry_interval_randomness,
+          retry_backoff_factor: @config.retry_backoff_factor
+        ).freeze
+      end
+
+      # Credentials, the signer, and the provider that parses errors travel
+      # with each request, because the Faraday connection is shared across
+      # contexts.
+      def prepare(request)
+        request.headers.merge!(@headers).merge!(@provider.headers)
+        context = request.options.context ||= {}
+        context[ErrorMiddleware::PROVIDER_KEY] = @provider
+        context[SigningMiddleware::CONTEXT_KEY] = @signer if @signer
+      end
+
+      def self.setup_timeout(faraday, settings)
+        faraday.options.timeout = settings.timeout
+      end
+
+      def self.setup_logging(faraday, settings)
         faraday.response :logger,
-                         RubyLLM.logger,
-                         bodies: RubyLLM.logger.debug?,
+                         settings.logger,
+                         bodies: settings.log_bodies,
                          errors: true,
                          headers: false,
                          log_level: :debug do |logger|
-          logger.filter(logging_regexp('[A-Za-z0-9+/=]{100,}'), '[BASE64 DATA]')
-          logger.filter(logging_regexp('[-\\d.e,\\s]{100,}'), '[EMBEDDINGS ARRAY]')
+          logger.filter(logging_regexp('[A-Za-z0-9+/=]{100,}', settings), '[BASE64 DATA]')
+          logger.filter(logging_regexp('[-\\d.e,\\s]{100,}', settings), '[EMBEDDINGS ARRAY]')
         end
       end
 
-      def logging_regexp(pattern)
-        return Regexp.new(pattern) if @config.log_regexp_timeout.nil? || !Regexp.respond_to?(:timeout)
+      def self.logging_regexp(pattern, settings)
+        return Regexp.new(pattern) if settings.log_regexp_timeout.nil?
 
-        Regexp.new(pattern, timeout: @config.log_regexp_timeout)
+        Regexp.new(pattern, timeout: settings.log_regexp_timeout)
       end
 
-      def setup_retry(faraday)
+      def self.setup_retry(faraday, settings)
         faraday.request :retry, {
-          max: @config.max_retries,
-          interval: @config.retry_interval,
-          max_interval: @config.retry_max_interval,
-          interval_randomness: @config.retry_interval_randomness,
-          backoff_factor: @config.retry_backoff_factor,
+          max: settings.max_retries,
+          interval: settings.retry_interval,
+          max_interval: settings.retry_max_interval,
+          interval_randomness: settings.retry_interval_randomness,
+          backoff_factor: settings.retry_backoff_factor,
           methods: Faraday::Retry::Middleware::IDEMPOTENT_METHODS,
           retry_if: lambda { |env, _exception|
             env[:method] == :post && idempotent?(env) && !stream_delivered?(env)
@@ -137,34 +215,36 @@ module RubyLLM
         faraday.use :llm_usage
       end
 
-      def stream_delivered?(env)
+      def self.stream_delivered?(env)
         env[:request]&.context&.dig(STREAM_PROGRESS_KEY, :started)
       end
 
-      def idempotent?(env)
+      def self.idempotent?(env)
         env[:request]&.context&.dig(IDEMPOTENT_KEY) != false
       end
 
-      def setup_middleware(faraday)
+      def self.setup_middleware(faraday, settings)
         faraday.request :multipart
         faraday.request :json
-        faraday.response :json
-        faraday.adapter(@config.faraday_adapter)
-        faraday.use :llm_errors, provider: @provider
+        faraday.use :llm_signing
+        faraday.use JsonResponse
+        faraday.adapter(settings.adapter)
+        faraday.use :llm_errors
       end
 
-      def setup_http_proxy(faraday)
-        return unless @config.http_proxy
+      def self.setup_http_proxy(faraday, settings)
+        return unless settings.proxy
 
-        faraday.proxy = @config.http_proxy
+        faraday.proxy = settings.proxy
       end
 
-      def retry_exceptions
+      def self.retry_exceptions
         [
           Errno::ETIMEDOUT,
           Timeout::Error,
           Faraday::TimeoutError,
           Faraday::ConnectionFailed,
+          Faraday::SSLError,
           Faraday::RetriableResponse,
           RubyLLM::RateLimitError,
           RubyLLM::ServerError,
@@ -173,6 +253,9 @@ module RubyLLM
         ]
       end
 
+      private_class_method :setup_timeout, :setup_logging, :logging_regexp, :setup_retry, :stream_delivered?,
+                           :idempotent?, :setup_middleware, :setup_http_proxy, :retry_exceptions
+
       def set_usage_tracker(request, tracker)
         context = request.options.context ||= {}
         context[UsageMiddleware::CONTEXT_KEY] = tracker