ruby_llm-agents 3.14.1 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4734a9b91b3dc228b78661cdbe4e336b9053beb6440601482709f99745c6e918
-  data.tar.gz: de29b3452e781a363c3330137323364edca7e80afd55680f200161c464d3f027
+  metadata.gz: 1d76f68609f8511674712633294484523bcedebd79b729796e3bee5504e2ed4c
+  data.tar.gz: 84ff7562baa359759a0d64a23030fc537addcb5860e0d823cb3a500f62c07a6e
 SHA512:
-  metadata.gz: 180db730cb98f426ce7e774cad42ad146b33cbb09b74340b772a2306cc0249152adbee79da9de71d372eb5f925ebf894db8305ef9df2ca58c60104280fec03e3
-  data.tar.gz: 2b2183165d6e2271ffda0b673ed173abde8cd77d8adc6e85b168de24b9744bc6c51ebe2194fcc45ad7d3e3fb35b82603f7830260927b2fabe763c552b93691cd
+  metadata.gz: 0cccf36a2f461c10fb0db268590e4d495a8a35ff0c06e4965051a669e49d10431daa9832dd4c1886100db401fe1dc1f8005d51cdb5f576d701a55973c4212017
+  data.tar.gz: 1ca7e6c26bc6973625228efe1bdb36033035d37f91ecc070b82eba31aad0fa527f6ad5b925bdb514bf7f8f4a224dae2c8d0919d1c9497af31406cece1b42e360

data/app/models/ruby_llm/agents/execution/analytics.rb

@@ -534,16 +534,22 @@ module RubyLLM
           # @param scope [ActiveRecord::Relation] Pre-filtered scope
           # @return [Array<Hash>] Model stats sorted by total cost descending
           def model_stats(scope: all)
+            # Attribute spend to the model that actually ran, not the one that
+            # was configured — otherwise a fallback's cost is billed to the
+            # primary in the comparison table. Older rows predate
+            # chosen_model_id, so fall back to model_id for those.
+            billed_model = Arel.sql("COALESCE(chosen_model_id, model_id)")
+
             rows = scope.where.not(model_id: nil)
               .select(
-                :model_id,
+                Arel.sql("#{billed_model} AS billed_model_id"),
                 Arel.sql("COUNT(*) AS exec_count"),
                 Arel.sql("COALESCE(SUM(total_cost), 0) AS sum_cost"),
                 Arel.sql("COALESCE(SUM(total_tokens), 0) AS sum_tokens"),
                 Arel.sql("AVG(duration_ms) AS avg_dur"),
                 Arel.sql("SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS success_cnt")
               )
-              .group(:model_id)
+              .group(billed_model)
 
             total_cost = rows.sum { |r| r["sum_cost"].to_f }
 
@@ -554,7 +560,7 @@ module RubyLLM
               successful = row["success_cnt"].to_i
 
               {
-                model_id: row.model_id,
+                model_id: row["billed_model_id"],
                 executions: count,
                 total_cost: model_cost,
                 total_tokens: model_tokens,
@@ -596,20 +602,31 @@ module RubyLLM
           # @return [Hash] Cache savings data
           def cache_savings(scope: all)
             cond = cache_hit_condition
-            total_count, cache_count, cache_cost = scope.pick(
+            total_count, cache_count, miss_count, miss_cost = scope.pick(
               Arel.sql("COUNT(*)"),
               Arel.sql("SUM(CASE WHEN #{cond} THEN 1 ELSE 0 END)"),
-              Arel.sql("COALESCE(SUM(CASE WHEN #{cond} THEN total_cost ELSE 0 END), 0)")
+              Arel.sql("SUM(CASE WHEN #{cond} THEN 0 ELSE 1 END)"),
+              Arel.sql("COALESCE(SUM(CASE WHEN #{cond} THEN 0 ELSE total_cost END), 0)")
             )
 
             total_count = total_count.to_i
             cache_count = cache_count.to_i
+            miss_count = miss_count.to_i
 
             return {count: 0, estimated_savings: 0, hit_rate: 0, total_executions: 0} if total_count.zero?
 
+            # Savings are the cost AVOIDED, not the cost recorded: a cache hit
+            # makes no API call, so its own total_cost is always 0 and summing
+            # those rows reported $0.00 saved forever. Estimate each hit at the
+            # mean cost of the misses in this same scope.
+            #
+            # ponytail: scope-wide mean, not per-agent. Group by agent_type if
+            # a mixed dashboard scope ever skews this enough to matter.
+            avg_miss_cost = miss_count.positive? ? (miss_cost.to_f / miss_count) : 0.0
+
             {
               count: cache_count,
-              estimated_savings: cache_cost.to_f,
+              estimated_savings: (cache_count * avg_miss_cost).round(6),
               hit_rate: (cache_count.to_f / total_count * 100).round(1),
               total_executions: total_count
             }

data/app/models/ruby_llm/agents/execution.rb

@@ -120,6 +120,13 @@ module RubyLLM
       # Used for multi-attempt executions (retries/fallbacks) where different models
       # may have been used. Calculates total cost by summing individual attempt costs.
       #
+      # Prompt-cache reads and writes are billed alongside the plain input: every
+      # provider reports input_tokens NET of the cache, so the cached portion is
+      # additive and pricing text tokens alone silently under-bills a cached
+      # request — the same defect the pipeline's cost path carried. Cache reads
+      # fall back to the full input rate when the registry publishes no cached
+      # price, so an unpriced model is never billed at zero.
+      #
       # @return [void]
       def aggregate_attempt_costs!
         return if attempts.blank?
@@ -134,13 +141,15 @@ module RubyLLM
           model_info = resolve_model_info(attempt["model_id"])
           next unless model_info&.pricing
 
-          input_price = model_info.pricing.text_tokens&.input || 0
-          output_price = model_info.pricing.text_tokens&.output || 0
+          tier = model_info.pricing.text_tokens
+          input_price = tier&.input || 0
+          output_price = tier&.output || 0
 
           input_tokens = attempt["input_tokens"] || 0
           output_tokens = attempt["output_tokens"] || 0
 
           total_input_cost += (input_tokens / 1_000_000.0) * input_price
+          total_input_cost += attempt_cache_cost(attempt, tier, input_price)
           total_output_cost += (output_tokens / 1_000_000.0) * output_price
         end
 
@@ -509,6 +518,31 @@ module RubyLLM
         Rails.logger.debug("[RubyLLM::Agents] Model lookup failed for #{lookup_model_id}: #{e.message}") if defined?(Rails) && Rails.logger
         nil
       end
+
+      # Prompt-cache charge for a single attempt, in USD.
+      #
+      # Reads bill at the registry's cached rate, falling back to the full input
+      # rate when it publishes none — never free. Writes bill at the cache-write
+      # rate, and are skipped entirely when unpriced, since no provider charges
+      # a write at the plain input rate.
+      #
+      # @param attempt [Hash] One entry from the attempts JSON
+      # @param tier [RubyLLM::Model::PricingCategory, nil] Text-token pricing
+      # @param input_price [Float] Per-million input rate, used as the read fallback
+      # @return [Float] Cache cost for this attempt
+      def attempt_cache_cost(attempt, tier, input_price)
+        read_tokens = attempt["cached_tokens"].to_i
+        write_tokens = attempt["cache_creation_tokens"].to_i
+        return 0.0 if read_tokens.zero? && write_tokens.zero?
+
+        read_price = (tier.respond_to?(:cache_read_input) ? tier.cache_read_input : nil) || input_price
+        write_price = (tier.respond_to?(:cache_write_input) ? tier.cache_write_input : nil) || 0
+
+        (read_tokens / 1_000_000.0) * read_price +
+          (write_tokens / 1_000_000.0) * write_price
+      rescue
+        0.0
+      end
     end
   end
 end

data/lib/ruby_llm/agents/audio/speech_pricing.rb

@@ -121,6 +121,9 @@ module RubyLLM
             return (data[:output_cost_per_character] * 1000).round(6)
           end
 
+          # Estimate only: audio-token-priced models publish no per-character
+          # rate, so convert at ~4 characters per audio token (1000 / 4 = 250).
+          # Set config.tts_model_pricing for a model where this matters.
           if data[:output_cost_per_audio_token]
             return (data[:output_cost_per_audio_token] * 250).round(6)
           end

data/lib/ruby_llm/agents/base_agent.rb

@@ -1008,7 +1008,12 @@ module RubyLLM
         input_price = model_info.pricing&.text_tokens&.input || 0
         output_price = model_info.pricing&.text_tokens&.output || 0
 
-        context.input_cost = input_cost_for(input_tokens, input_price, model_info, context)
+        # input_tokens is always the UNCACHED input: every RubyLLM provider
+        # reports it net of the cache reads (Anthropic natively; OpenAI, Gemini,
+        # Bedrock and OpenRouter subtract them when parsing usage). So the cache
+        # is purely additive and #extra_token_costs prices it on top — see
+        # #cache_read_cost.
+        context.input_cost = (input_tokens / 1_000_000.0) * input_price
 
         # Price cache/reasoning extras first so we know whether reasoning was
         # actually billed at the reasoning rate. Only then exclude those tokens
@@ -1021,73 +1026,31 @@ module RubyLLM
         context.total_cost = (context.input_cost + context.output_cost + extra).round(6)
       end
 
-      # Input charge, with cache reads billed at the cached rate rather than the
-      # full one.
+      # Charge for tokens served from the provider's prompt cache.
       #
-      # Providers disagree on whether the reported input_tokens ALREADY contains
-      # the cache reads, and the difference decides who has a bug:
+      # Cache reads are ALWAYS additive to input_tokens. Every provider RubyLLM
+      # supports reports input_tokens net of them — Anthropic natively, and
+      # OpenAI/Gemini/Bedrock/OpenRouter by subtracting the cache counters while
+      # parsing usage. So the read is never already paid for by the input
+      # charge, and must be added here or it is billed at zero.
       #
-      #   OpenAI    prompt_tokens       INCLUDES prompt_tokens_details.cached_tokens
-      #   Gemini    promptTokenCount    INCLUDES cachedContentTokenCount
-      #   Anthropic input_tokens        EXCLUDES cache_read_input_tokens
+      # Prefers RubyLLM::Cost's own cache_read (registry cache-read rate). When
+      # the registry publishes no cache-read price for the model, RubyLLM::Cost
+      # returns nil rather than guessing — bill those tokens at the full input
+      # rate instead, so an unpriced model is never under-billed.
       #
-      # On Anthropic the cache read is genuinely additive, so charging the full
-      # input_tokens at full rate and letting #extra_token_costs add cache_read
-      # on top is already correct — that path is left untouched.
-      #
-      # On OpenAI and Gemini the cached tokens are sitting inside input_tokens,
-      # so charging them at the full rate overstates spend by 2-3x on exactly
-      # the workload caching exists for (a long stable system prompt replayed
-      # every turn) — and would double-charge outright once the provider also
-      # reports a cache_read cost component. Those get split here instead.
-      #
-      # Falls back to the full input rate whenever the split can't be made
-      # safely (no cache reported, or no cached price in the registry), so an
-      # unknown model is never under-billed.
-      #
-      # @return [Float] The input cost in USD
-      def input_cost_for(input_tokens, input_price, model_info, context)
-        full_rate_charge = (input_tokens / 1_000_000.0) * input_price
+      # @param cost [RubyLLM::Cost] The response's cost helper
+      # @param model_info [RubyLLM::Model::Info] Resolved pricing source
+      # @param context [Pipeline::Context] The execution context
+      # @return [Float, nil] Cache-read cost in USD, or nil when nothing to bill
+      def cache_read_cost(cost, model_info, context)
+        priced = cost.cache_read
+        return priced if priced
 
         cached_tokens = context.cached_tokens.to_i
-        return full_rate_charge unless cached_tokens.positive?
-        return full_rate_charge unless cached_tokens_included_in_input?(model_info)
-
-        cached_price = cached_input_price(model_info)
-        return full_rate_charge unless cached_price
-
-        # Tells extra_token_costs not to charge these same reads a second time.
-        context[:cache_read_priced] = true
-
-        uncached_tokens = [input_tokens - cached_tokens, 0].max
-        (uncached_tokens / 1_000_000.0) * input_price +
-          (cached_tokens / 1_000_000.0) * cached_price
-      end
+        return nil unless cached_tokens.positive?
 
-      # Per-million price for a cache read, or nil when the model registry
-      # doesn't publish one. Pricing shapes vary by source (and are stubbed with
-      # partial doubles in tests), so an absent field degrades to "unknown" —
-      # callers then charge the full input rate rather than guessing a discount.
-      #
-      # @return [Float, nil]
-      def cached_input_price(model_info)
-        tier = model_info.pricing&.text_tokens
-        return nil unless tier.respond_to?(:cached_input)
-
-        tier.cached_input
-      rescue
-        nil
-      end
-
-      # Whether this provider's reported input_tokens already contains the cache
-      # reads. See #input_cost_for for the per-provider breakdown. Anthropic is
-      # the exception; defaulting the unknown case to "included" matches every
-      # other provider RubyLLM supports.
-      #
-      # @return [Boolean]
-      def cached_tokens_included_in_input?(model_info)
-        provider = model_info.respond_to?(:provider) ? model_info.provider.to_s : ""
-        !provider.include?("anthropic")
+        (cached_tokens / 1_000_000.0) * (model_info.pricing&.text_tokens&.input || 0)
       end
 
       # Number of reasoning (thinking) tokens that were actually charged at the
@@ -1126,12 +1089,10 @@ module RubyLLM
         cost = response_cost(response, model_info)
         return 0.0 unless cost
 
-        # Skip cache_read when #input_cost_for already billed the cache at the
-        # cached rate — otherwise the same tokens are charged twice (once inside
-        # input_tokens at full rate, once here). Cache WRITES are always extra:
-        # no provider folds them into input_tokens.
+        # Cache reads and writes are both additive: no provider folds either
+        # into the input_tokens RubyLLM reports. See #cache_read_cost.
         components = {
-          cache_read: (cost.cache_read unless context[:cache_read_priced]),
+          cache_read: cache_read_cost(cost, model_info, context),
           cache_write: cost.cache_write,
           thinking: cost.thinking
         }.compact.reject { |_, value| value.zero? }

data/lib/ruby_llm/agents/core/configuration.rb

@@ -142,9 +142,14 @@ module RubyLLM
       #   @return [Array<String>] List of model identifiers to try on failure
 
       # @!attribute [rw] default_total_timeout
-      #   Default total timeout across all retry attempts.
-      #   Can be overridden per-agent using the `total_timeout` DSL method.
-      #   @return [Integer, nil] Total timeout in seconds, or nil for no limit
+      #   Wall-clock budget for ALL attempts combined, across every model, for
+      #   agents that configure reliability without naming their own
+      #   `total_timeout`. Retries and fallbacks multiply into an attempt matrix
+      #   that is easy to under-estimate — `retries times: 5` with three
+      #   fallbacks is 24 attempts, which at the default 60s per-call timeout
+      #   plus backoff is over 20 minutes of one caller waiting.
+      #   Set to nil (or `total_timeout false` on an agent) for no limit.
+      #   @return [Integer, nil] Total timeout in seconds (default: 300)
 
       # @!attribute [rw] default_streaming
       #   Whether streaming mode is enabled by default for all agents.
@@ -201,6 +206,13 @@ module RubyLLM
       # @!attribute [rw] persist_prompts
       #   Whether to persist system and user prompts in execution records.
       #   Set to false to reduce storage or for privacy compliance.
+      #
+      #   Note that prompts are stored as RENDERED — parameter redaction covers
+      #   execution_details.parameters, not the prompt those parameters were
+      #   interpolated into. A secret passed as `api_key:` shows up as
+      #   "[REDACTED]" in parameters while still appearing verbatim in
+      #   user_prompt. Do not interpolate credentials into prompts, or disable
+      #   this.
       #   @return [Boolean] Enable prompt persistence (default: true)
 
       # @!attribute [rw] persist_responses
@@ -695,7 +707,7 @@ module RubyLLM
         # Reliability defaults (all disabled by default for backward compatibility)
         @default_retries = {max: 0, backoff: :exponential, base: 0.4, max_delay: 3.0, on: []}
         @default_fallback_models = []
-        @default_total_timeout = nil
+        @default_total_timeout = 300
         @default_retryable_patterns = {
           rate_limiting: ["rate limit", "rate_limit", "too many requests", "429", "quota"],
           server_errors: ["500", "502", "503", "504", "service unavailable",

data/lib/ruby_llm/agents/core/version.rb

@@ -4,6 +4,6 @@ module RubyLLM
   module Agents
     # Current version of the RubyLLM::Agents gem
     # @return [String] Semantic version string
-    VERSION = "3.14.1"
+    VERSION = "3.15.0"
   end
 end

data/lib/ruby_llm/agents/dsl/reliability.rb

@@ -51,7 +51,7 @@ module RubyLLM
           @retries_config = builder.retries_config if builder.retries_config
           @fallback_models = builder.fallback_models_list if builder.fallback_models_list.any?
           @fallback_providers = builder.fallback_providers_list if builder.fallback_providers_list.any?
-          @total_timeout = builder.total_timeout_value if builder.total_timeout_value
+          @total_timeout = builder.total_timeout_value unless builder.total_timeout_value.nil?
           @circuit_breaker_config = builder.circuit_breaker_config if builder.circuit_breaker_config
           @retryable_patterns = builder.retryable_patterns_list if builder.retryable_patterns_list
           @non_fallback_errors = builder.non_fallback_errors_list if builder.non_fallback_errors_list
@@ -80,7 +80,7 @@ module RubyLLM
           @retries_config = builder.retries_config if builder.retries_config
           @fallback_models = builder.fallback_models_list if builder.fallback_models_list.any?
           @fallback_providers = builder.fallback_providers_list if builder.fallback_providers_list.any?
-          @total_timeout = builder.total_timeout_value if builder.total_timeout_value
+          @total_timeout = builder.total_timeout_value unless builder.total_timeout_value.nil?
           @circuit_breaker_config = builder.circuit_breaker_config if builder.circuit_breaker_config
           @retryable_patterns = builder.retryable_patterns_list if builder.retryable_patterns_list
           @non_fallback_errors = builder.non_fallback_errors_list if builder.non_fallback_errors_list
@@ -186,9 +186,25 @@ module RubyLLM
         # @return [Integer, nil] The current total timeout
         # @example
         #   total_timeout 30
+        # Wall-clock budget for ALL attempts combined, across every model.
+        #
+        # Falls back to config.default_total_timeout rather than "unbounded":
+        # retries and fallbacks multiply, and the product is easy to miss.
+        # `retries times: 5` with three fallbacks is 24 attempts — at the
+        # default 60s per-call timeout plus backoff that is over 20 minutes of
+        # one caller waiting. Pass `total_timeout false` to opt out.
+        #
+        # @param seconds [Integer, Float, false, nil] Budget in seconds, or
+        #   false for no bound
+        # @return [Integer, Float, nil] The effective budget, nil when unbounded
         def total_timeout(seconds = nil)
-          @total_timeout = seconds if seconds
-          @total_timeout || inherited_total_timeout
+          @total_timeout = seconds unless seconds.nil?
+          configured = defined?(@total_timeout) ? @total_timeout : nil
+          effective = configured.nil? ? inherited_total_timeout : configured
+
+          return nil if effective == false
+
+          effective || default_total_timeout_from_config
         end
 
         # Configures circuit breaker for this agent
@@ -266,6 +282,12 @@ module RubyLLM
           superclass.total_timeout
         end
 
+        def default_total_timeout_from_config
+          RubyLLM::Agents.configuration.default_total_timeout
+        rescue
+          nil
+        end
+
         def inherited_circuit_breaker_config
           return nil unless superclass.respond_to?(:circuit_breaker_config)
 
@@ -436,7 +458,11 @@ module RubyLLM
           #
           def timeout(duration)
             # Handle ActiveSupport::Duration
-            @total_timeout_value = duration.respond_to?(:to_i) ? duration.to_i : duration
+            @total_timeout_value = if duration == false
+              false
+            else
+              duration.respond_to?(:to_i) ? duration.to_i : duration
+            end
           end
 
           # Also support total_timeout for compatibility

data/lib/ruby_llm/agents/pipeline/builder.rb

@@ -228,29 +228,20 @@ module RubyLLM
 
           # Check if reliability features are enabled for an agent
           #
-          # An agent has reliability enabled if it has:
-          # - retries > 0, OR
-          # - fallback_models configured
+          # Delegates to the same predicate the Reliability middleware itself
+          # uses (#reliability_config), so the builder gate and the middleware
+          # gate can never disagree. They previously did: this method tested
+          # `agent_class.retries` for a positive Integer, but that DSL reader
+          # returns the retry config Hash — so `on_failure { retries times: 3 }`
+          # never installed the middleware and retries silently never ran.
           #
           # @param agent_class [Class] The agent class
           # @return [Boolean]
           def reliability_enabled?(agent_class)
             return false unless agent_class
+            return false unless agent_class.respond_to?(:reliability_config)
 
-            retries = if agent_class.respond_to?(:retries)
-              agent_class.retries
-            else
-              0
-            end
-
-            fallbacks = if agent_class.respond_to?(:fallback_models)
-              agent_class.fallback_models
-            else
-              []
-            end
-
-            (retries.is_a?(Integer) && retries.positive?) ||
-              (fallbacks.is_a?(Array) && fallbacks.any?)
+            agent_class.reliability_config.present?
           rescue => e
             Rails.logger.debug("[RubyLLM::Agents::Pipeline] Failed to check reliability_enabled: #{e.message}") if defined?(Rails) && Rails.logger
             false

data/lib/ruby_llm/agents/pipeline/context.rb

@@ -48,6 +48,16 @@ module RubyLLM
         # a generic bag entry would be silently droppable (it was).
         attr_accessor :cached_tokens, :cache_creation_tokens
 
+        # Per-tenant provider API keys, set by the Tenant middleware.
+        #
+        # A first-class accessor for the opposite reason to the above: the
+        # metadata bag is copied wholesale onto the persisted execution record
+        # and rendered by the dashboard, so a credential stored there leaked
+        # into the database in plaintext and onto the execution page (it did).
+        # Keeping it off the bag makes that leak structurally impossible rather
+        # than dependent on a redaction list.
+        attr_accessor :tenant_api_keys
+
         # Response metadata
         attr_accessor :model_used, :finish_reason, :time_to_first_token_ms
 
@@ -185,7 +195,7 @@ module RubyLLM
         #
         # @return [RubyLLM::Context, RubyLLM] Scoped context or global module
         def llm
-          api_keys = self[:tenant_api_keys]
+          api_keys = @tenant_api_keys
           return RubyLLM if api_keys.nil? || api_keys.empty?
 
           @llm_context ||= build_llm_context(api_keys)

data/lib/ruby_llm/agents/pipeline/middleware/cache.rb

@@ -39,7 +39,8 @@ module RubyLLM
               unless context.skip_cache
                 # Try to read from cache
                 if (cached = cache_read(cache_key))
-                  context.output = cached
+                  context.output = mark_cached(cached, context)
+                  replay_to_stream(context)
                   context.cached = true
                   context[:cache_key] = cache_key
                   cache_action = "hit"
@@ -75,6 +76,61 @@ module RubyLLM
 
           private
 
+          # Marks a cache-read result as cached and re-points its execution_id
+          # at the row being written for THIS call.
+          #
+          # Without this, a cache hit returned the original result verbatim: it
+          # reported no way to tell it was cached, carried the original's
+          # execution_id (so it could not be correlated with the execution just
+          # created), and replayed the original's cost — double-counting for
+          # anyone summing #total_cost across calls.
+          #
+          # Results that predate #as_cache_hit (older cache entries mid-deploy)
+          # are returned untouched rather than raising.
+          #
+          # @param cached [Object] The value read from the cache store
+          # @param context [Context] The execution context
+          # @return [Object] The marked result, or the original value
+          def mark_cached(cached, context)
+            return cached unless cached.respond_to?(:as_cache_hit)
+
+            cached.as_cache_hit(
+              execution_id: context.execution_id,
+              cached_at: cached.try(:completed_at) || Time.current
+            )
+          rescue => e
+            debug("Failed to mark result as cached: #{e.message}", context)
+            cached
+          end
+
+          # Stand-in for a provider chunk when replaying a cached response.
+          ReplayChunk = Struct.new(:content)
+
+          # Feeds a cached response through the caller's stream block.
+          #
+          # A streaming agent with `cache_for` used to go silent on a hit: the
+          # block never fired and the caller saw an empty stream, even though
+          # the content was sitting in the return value.
+          #
+          # ponytail: replays as a single chunk, not token-by-token. The cache
+          # stores the finished response, not the chunk boundaries — recreating
+          # them would mean caching the stream itself.
+          #
+          # @param context [Context] The execution context
+          def replay_to_stream(context)
+            block = context.stream_block
+            return unless block
+
+            content = context.output.try(:content)
+            return if content.nil?
+
+            block.call(
+              context.stream_events? ? StreamEvent.new(:chunk, {content: content}) : ReplayChunk.new(content)
+            )
+          rescue => e
+            debug("Failed to replay cached response to stream: #{e.message}", context)
+          end
+
           # Emits an AS::Notification for cache events
           #
           # @param event [String] The notification event name
@@ -113,6 +169,14 @@ module RubyLLM
             config(:cache_ttl)
           end
 
+          # Execution options that change the response and must therefore change
+          # the cache key. Anything omitted here is a silent stale-result bug:
+          # attachments especially, since two calls with the same text prompt
+          # and different images used to collide.
+          KEYED_OPTIONS = %i[
+            system_prompt assistant_prefill temperature schema messages thinking attachments
+          ].freeze
+
           # Generates a cache key for the context
           #
           # Cache keys are content-based, including:
@@ -120,7 +184,9 @@ module RubyLLM
           # - Agent type
           # - Agent class name
           # - Model
+          # - Tenant (caches are never shared across tenants)
           # - SHA256 hash of input
+          # - SHA256 hash of the response-affecting execution options
           #
           # This means caches automatically invalidate when inputs change.
           #
@@ -132,12 +198,34 @@ module RubyLLM
               context.agent_type,
               context.agent_class&.name,
               context.model,
-              hash_input(context.input)
+              context.tenant_id,
+              hash_input(context.input),
+              hash_options(context)
             ].compact
 
             components.join("/")
           end
 
+          # Hashes the response-affecting execution options.
+          #
+          # Tools are reduced to their names: they arrive as classes or
+          # instances whose default JSON encoding is neither stable nor
+          # meaningful, and the name is what actually shapes the request.
+          #
+          # @param context [Context] The execution context
+          # @return [String, nil] SHA256 hash, or nil when there are no options
+          def hash_options(context)
+            opts = context.options[:options]
+            return nil unless opts.is_a?(Hash)
+
+            keyed = opts.slice(*KEYED_OPTIONS)
+            tools = Array(opts[:tools]).map { |t| t.respond_to?(:name) ? t.name : t.class.name }
+            keyed[:tools] = tools if tools.any?
+            return nil if keyed.empty?
+
+            hash_input(keyed)
+          end
+
           # Hashes the input for cache key
           #
           # @param input [Object] The input to hash

data/lib/ruby_llm/agents/pipeline/middleware/instrumentation.rb

@@ -388,6 +388,7 @@ module RubyLLM
               cache_hit: context.cached?,
               input_tokens: context.input_tokens || 0,
               output_tokens: context.output_tokens || 0,
+              cached_tokens: context.cached_tokens || 0,
               input_cost: context.input_cost,
               output_cost: context.output_cost,
               total_cost: context.total_cost || 0,
@@ -412,6 +413,7 @@ module RubyLLM
             if context.cached? && context[:cache_key]
               merged_metadata["response_cache_key"] = context[:cache_key]
             end
+            merged_metadata = redact_secrets(merged_metadata)
             data[:metadata] = merged_metadata if merged_metadata.any?
 
             # Error class on execution (error_message goes to detail)
@@ -457,6 +459,10 @@ module RubyLLM
               detail_data[:attempts] = context[:reliability_attempts]
             end
 
+            if context.cache_creation_tokens.to_i.positive?
+              detail_data[:cache_creation_tokens] = context.cache_creation_tokens
+            end
+
             if global_config.persist_responses && context.output.respond_to?(:content)
               detail_data[:response] = serialize_response(context)
             end
@@ -642,6 +648,30 @@ module RubyLLM
             access_token refresh_token private_key secret_key
           ].freeze
 
+          # Credential-shaped metadata keys. The execution's metadata column is
+          # rendered verbatim (with a copy button) on the execution page, so
+          # anything secret-looking is redacted before it is written.
+          #
+          # Deliberately word-bounded on `token` so the legitimate token
+          # counters — cached_tokens, token_count, thinking_tokens — survive.
+          SECRET_METADATA_PATTERN = /
+            api_?keys? | secret | password | credential |
+            private_key | access_token | refresh_token | auth_token | \btokens?\b
+          /xi
+
+          # Redacts credential-shaped entries from metadata before persistence.
+          #
+          # @param metadata [Hash] Metadata about to be written
+          # @return [Hash] Metadata with secret-shaped values replaced
+          def redact_secrets(metadata)
+            metadata.to_h do |key, value|
+              [key, key.to_s.match?(SECRET_METADATA_PATTERN) ? "[REDACTED]" : value]
+            end
+          rescue => e
+            debug("Failed to redact metadata: #{e.message}")
+            metadata
+          end
+
           # Internal keys that should be stripped from persisted parameters
           INTERNAL_KEYS = %w[
             _replay_source_id _ask_message _parent_execution_id _root_execution_id

data/lib/ruby_llm/agents/pipeline/middleware/reliability.rb

@@ -143,6 +143,22 @@ module RubyLLM
             # Store attempts even on total failure
             context[:reliability_attempts] = tracker.to_json_array
 
+            # Nothing was ever attempted — every model was short-circuited by an
+            # open breaker. That is a distinct condition from "we tried and they
+            # all failed", and it has a distinct error class; raising
+            # AllModelsExhaustedError here handed back a nil last_error and left
+            # CircuitBreakerOpenError impossible to rescue.
+            if context.attempts_made.zero?
+              emit_reliability_notification(
+                "ruby_llm_agents.reliability.circuit_open",
+                models_tried: models_to_try
+              )
+
+              raise Agents::Reliability::CircuitBreakerOpenError.new(
+                @agent_class&.name, models_to_try.join(", ")
+              )
+            end
+
             # All models exhausted
             emit_reliability_notification(
               "ruby_llm_agents.reliability.all_models_exhausted",
@@ -183,6 +199,13 @@ module RubyLLM
                 original_model = context.model
                 context.model = model
 
+                # Clear any error left by a previous attempt. Context#success?
+                # requires #error to be nil, and every middleware outside this
+                # one keys off it — a stale error made a recovered retry look
+                # like a failure, so Budget skipped recording the spend and
+                # Cache skipped writing the result.
+                context.error = nil
+
                 @app.call(context)
 
                 # Success - record in circuit breaker and tracker
@@ -238,9 +261,11 @@ module RubyLLM
           def should_retry?(error, config, attempt_index, max_retries, total_deadline)
             return false if attempt_index >= max_retries
             return false if total_deadline && Time.current > total_deadline
-            # Don't retry if fallback models are available — move to next model instead
-            return false if fallback_models?(config)
 
+            # Configuring fallbacks does NOT cancel the configured retries: each
+            # model exhausts its own retries before the next one is tried. A
+            # transient 429 on the primary should not immediately burn the
+            # fallback. `total_timeout` is the bound on total attempts.
             retryable_error?(error, config)
           end
 
@@ -254,15 +279,6 @@ module RubyLLM
             Agents::Reliability.non_fallback_error?(error, custom_errors: custom_errors)
           end
 
-          # Returns whether fallback models are configured
-          #
-          # @param config [Hash] The reliability configuration
-          # @return [Boolean]
-          def fallback_models?(config)
-            fallbacks = config[:fallback_models]
-            fallbacks.is_a?(Array) && fallbacks.any?
-          end
-
           # Checks if an error is retryable
           #
           # @param error [Exception] The error to check

data/lib/ruby_llm/agents/pipeline/middleware/tenant.rb

@@ -199,6 +199,10 @@ module RubyLLM
           # thread-safe), keys are stored on the context. The Pipeline::Context#llm
           # method creates a scoped RubyLLM::Context with these keys when needed.
           #
+          # Written to the dedicated accessor, never to context[...]: the
+          # metadata bag is persisted onto the execution record and rendered by
+          # the dashboard, so a credential placed there leaks.
+          #
           # @param context [Context] The execution context
           def apply_tenant_object_api_keys!(context)
             tenant_object = context.tenant_object
@@ -207,7 +211,7 @@ module RubyLLM
             api_keys = tenant_object.llm_api_keys
             return if api_keys.blank?
 
-            context[:tenant_api_keys] = api_keys
+            context.tenant_api_keys = api_keys
           rescue => e
             # Log but don't fail if API key extraction fails
             warn_api_key_error("tenant object", e)

data/lib/ruby_llm/agents/results/base.rb

@@ -20,6 +20,7 @@ module RubyLLM
     # @api public
     class Result
       extend ActiveSupport::Delegation
+      include Trackable
 
       # @!attribute [r] content
       #   @return [Hash, String] The processed response content
@@ -175,6 +176,9 @@ module RubyLLM
         # Execution record
         @execution_id = options[:execution_id]
 
+        # Caching
+        @cached_at = options[:cached_at]
+
         # Tracking
         @agent_class_name = options[:agent_class_name]
 
@@ -184,9 +188,10 @@ module RubyLLM
         # Debug trace
         @trace = options[:trace]
 
-        # Register with active tracker
-        tracker = Thread.current[:ruby_llm_agents_tracker]
-        tracker << self if tracker
+        # Wrapper results (RoutingResult) opt out: the results they wrap have
+        # already registered, and registering the wrapper too would report its
+        # combined cost on top of its own parts.
+        register_with_tracker unless options[:register_with_tracker] == false
       end
 
       # Loads the associated Execution record from the database
@@ -272,6 +277,7 @@ module RubyLLM
       # @return [Hash] All result data as a hash
       def to_h
         {
+          cached_at: cached_at,
           content: content,
           input_tokens: input_tokens,
           output_tokens: output_tokens,

data/lib/ruby_llm/agents/results/trackable.rb

@@ -2,15 +2,54 @@
 
 module RubyLLM
   module Agents
-    # Mixin that registers a result object with the active Tracker.
+    # Shared result lifecycle: registering with the active Tracker, and marking
+    # a result that was replayed from the response cache.
     #
-    # Included in every result class so that RubyLLM::Agents.track
-    # can collect results automatically.
+    # Both concerns fire at the same moment — when a result is handed back to
+    # the caller — and both must apply to every result type, since any agent
+    # with `cache_for` can have its result served from cache.
     #
     # @api private
     module Trackable
       def self.included(base)
         base.attr_reader :agent_class_name unless base.method_defined?(:agent_class_name)
+        base.attr_reader :cached_at unless base.method_defined?(:cached_at)
+      end
+
+      # Whether this response was replayed from the response cache rather than
+      # generated by a fresh LLM call.
+      #
+      # The token and cost fields describe the call that ORIGINALLY produced the
+      # response — no API call was made this time, and the execution record for
+      # this call is written with zero cost. Aggregate spend from the execution
+      # records, or exclude cached results, rather than summing #total_cost
+      # across every result. RubyLLM::Agents.track does this for you.
+      #
+      # @return [Boolean]
+      def cached?
+        !@cached_at.nil?
+      end
+
+      # Returns a copy of this result marked as served from the response cache.
+      #
+      # Used by the Cache middleware on a hit. Two things the raw cached object
+      # gets wrong on its own:
+      #
+      # - execution_id still points at the ORIGINAL call's row, so the result
+      #   cannot be correlated with the execution written for this call.
+      # - the copy never passes through #initialize (it is deserialized, then
+      #   duped), so it would never register with an active Tracker — making
+      #   TrackReport#call_count silently under-report.
+      #
+      # @param execution_id [Integer, nil] The current call's execution record id
+      # @param cached_at [Time] When the cached response was first produced
+      # @return [Object] A marked copy of this result
+      def as_cache_hit(execution_id:, cached_at:)
+        dup.tap do |copy|
+          copy.instance_variable_set(:@cached_at, cached_at)
+          copy.instance_variable_set(:@execution_id, execution_id)
+          copy.send(:register_with_tracker)
+        end
       end
 
       private

data/lib/ruby_llm/agents/routing/result.rb

@@ -47,13 +47,14 @@ module RubyLLM
         def initialize(base_result:, route_data:)
           @delegated_result = route_data[:delegated_result]
           @routing_cost = base_result.total_cost
+          @routing_tokens = base_result.total_tokens
 
-          # When delegated, merge costs from both classification and delegation
-          total = if @delegated_result
-            (base_result.total_cost || 0) + (@delegated_result.respond_to?(:total_cost) ? @delegated_result.total_cost || 0 : 0)
-          else
-            base_result.total_cost
-          end
+          # A routed call is two LLM calls: classify, then delegate. Every
+          # figure on this result covers BOTH, so they stay consistent with each
+          # other — total_cost used to include the delegation while the token
+          # counts did not, which made cost-per-token nonsense. Classification
+          # alone is still available via #routing_cost / #routing_tokens.
+          total = combine(base_result, :total_cost)
 
           # Use delegated content when available
           effective_content = if @delegated_result
@@ -64,11 +65,15 @@ module RubyLLM
 
           super(
             content: effective_content,
-            input_tokens: base_result.input_tokens,
-            output_tokens: base_result.output_tokens,
-            input_cost: base_result.input_cost,
-            output_cost: base_result.output_cost,
+            input_tokens: combine(base_result, :input_tokens),
+            output_tokens: combine(base_result, :output_tokens),
+            input_cost: combine(base_result, :input_cost),
+            output_cost: combine(base_result, :output_cost),
             total_cost: total,
+            # The wrapped results already registered with the active Tracker;
+            # registering this wrapper too reported the combined figures on top
+            # of the parts, doubling the cost of every routed call.
+            register_with_tracker: false,
             model_id: base_result.model_id,
             chosen_model_id: base_result.chosen_model_id,
             temperature: base_result.temperature,
@@ -108,6 +113,13 @@ module RubyLLM
           @routing_cost || 0
         end
 
+        # Tokens used by the classification step only (excluding delegation)
+        #
+        # @return [Integer]
+        def routing_tokens
+          @routing_tokens || 0
+        end
+
         # Converts the result to a hash including routing fields.
         #
         # @return [Hash] All result data plus route, agent_class, raw_response
@@ -119,6 +131,20 @@ module RubyLLM
             delegated: delegated?
           )
         end
+
+        private
+
+        # Sums a numeric field across the classification and the delegated call.
+        #
+        # @param base_result [Result] The classification result
+        # @param field [Symbol] The field to read from both
+        # @return [Numeric, nil] The combined value
+        def combine(base_result, field)
+          base = base_result.public_send(field)
+          return base unless @delegated_result&.respond_to?(field)
+
+          (base || 0) + (@delegated_result.public_send(field) || 0)
+        end
       end
     end
   end

data/lib/ruby_llm/agents/track_report.rb

@@ -37,32 +37,63 @@ module RubyLLM
         !successful?
       end
 
+      # Every agent call made in the block, including ones served from cache.
       def call_count
         @results.size
       end
 
+      # Calls served from the response cache — no API call, no spend.
+      def cached_count
+        cached_results.size
+      end
+
+      # Calls that actually hit a provider.
+      def billable_count
+        billable_results.size
+      end
+
+      # Costs and token counts cover only the calls that actually hit a
+      # provider. A cached result replays the ORIGINAL call's figures, so
+      # including them would report spend that was never incurred — the
+      # execution records for cache hits are written with zero cost, and these
+      # totals reconcile with them.
       def total_cost
-        @results.sum { |r| r.total_cost || 0 }
+        billable_results.sum { |r| r.total_cost || 0 }
       end
 
       def input_cost
-        @results.sum { |r| r.input_cost || 0 }
+        billable_results.sum { |r| r.input_cost || 0 }
       end
 
       def output_cost
-        @results.sum { |r| r.output_cost || 0 }
+        billable_results.sum { |r| r.output_cost || 0 }
       end
 
       def total_tokens
-        @results.sum { |r| r.total_tokens }
+        billable_results.sum { |r| r.total_tokens || 0 }
       end
 
       def input_tokens
-        @results.sum { |r| r.input_tokens || 0 }
+        billable_results.sum { |r| r.input_tokens || 0 }
       end
 
       def output_tokens
-        @results.sum { |r| r.output_tokens || 0 }
+        billable_results.sum { |r| r.output_tokens || 0 }
+      end
+
+      # What the cached calls would have cost at the original calls' prices.
+      def cache_savings
+        cached_results.sum { |r| r.total_cost || 0 }
+      end
+
+      # Results that hit a provider (everything except cache replays).
+      def billable_results
+        @billable_results ||= @results.reject { |r| cached_result?(r) }
+      end
+
+      # Results replayed from the response cache.
+      def cached_results
+        @cached_results ||= @results.select { |r| cached_result?(r) }
       end
 
       def duration_ms
@@ -95,8 +126,9 @@ module RubyLLM
           {
             agent: r.respond_to?(:agent_class_name) ? r.agent_class_name : nil,
             model: r.chosen_model_id,
-            cost: r.total_cost || 0,
-            tokens: r.total_tokens,
+            cost: cached_result?(r) ? 0 : (r.total_cost || 0),
+            tokens: cached_result?(r) ? 0 : (r.total_tokens || 0),
+            cached: cached_result?(r),
             duration_ms: r.duration_ms
           }
         end
@@ -109,7 +141,10 @@ module RubyLLM
           error: error&.message,
           request_id: request_id,
           call_count: call_count,
+          cached_count: cached_count,
+          billable_count: billable_count,
           total_cost: total_cost,
+          cache_savings: cache_savings,
           input_cost: input_cost,
           output_cost: output_cost,
           total_tokens: total_tokens,
@@ -122,6 +157,13 @@ module RubyLLM
           cost_breakdown: cost_breakdown
         }
       end
+
+      private
+
+      # Older result types may predate #cached?; treat those as billable.
+      def cached_result?(result)
+        result.respond_to?(:cached?) && result.cached?
+      end
     end
   end
 end

data/lib/tasks/ruby_llm_agents.rake

@@ -14,6 +14,41 @@ namespace :ruby_llm_agents do
     puts "Hard purged: #{result[:hard_purged]} executions (rows destroyed)"
   end
 
+  desc "Redact credential-shaped keys from historical execution metadata. Usage: rake ruby_llm_agents:redact_metadata_secrets [DRY_RUN=1]"
+  task redact_metadata_secrets: :environment do
+    # Versions before the tenant-API-key fix wrote per-tenant provider keys into
+    # executions.metadata in plaintext, and the execution page renders that
+    # column verbatim. New rows are safe; this scrubs the ones already written.
+    #
+    # Rotate the affected keys as well — redacting the copy in the database does
+    # not un-expose a key that was already displayed.
+    dry_run = ENV["DRY_RUN"].present?
+    pattern = RubyLLM::Agents::Pipeline::Middleware::Instrumentation::SECRET_METADATA_PATTERN
+    scanned = 0
+    redacted = 0
+
+    RubyLLM::Agents::Execution.where.not(metadata: nil).find_each do |execution|
+      scanned += 1
+      metadata = execution.metadata
+      next unless metadata.is_a?(Hash)
+
+      offending = metadata.keys.select { |key| key.to_s.match?(pattern) }
+      next if offending.empty?
+
+      redacted += 1
+      puts "execution ##{execution.id}: #{offending.join(", ")}"
+      next if dry_run
+
+      execution.update_column(
+        :metadata,
+        metadata.merge(offending.to_h { |key| [key, "[REDACTED]"] })
+      )
+    end
+
+    puts dry_run ? "DRY RUN — #{redacted} of #{scanned} executions would be redacted" : "Redacted #{redacted} of #{scanned} executions"
+    puts "Rotate any exposed credential: redaction here does not un-expose it." if redacted.positive?
+  end
+
   desc "Rename an agent type in execution records. Usage: rake ruby_llm_agents:rename_agent FROM=OldName TO=NewName [DRY_RUN=1]"
   task rename_agent: :environment do
     from = ENV["FROM"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-agents
 version: !ruby/object:Gem::Version
-  version: 3.14.1
+  version: 3.15.0
 platform: ruby
 authors:
 - adham90