ruby_llm-resilience 0.6.0

data/lib/generators/resilience/templates/initializer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# ruby_llm-resilience — circuit breakers and fallback chains for LLM calls.
+# Docs: https://github.com/danielstpaul/ruby_llm-resilience
+#
+# Everything below is optional; the gem works with defaults out of the box
+# (per-process memory store, threshold 5, cooldown 120s). The two settings
+# most apps DO want are cache_store (multi-process correctness) and
+# fallback_models (the actual routing).
+RubyLLM::Resilience.configure do |config|
+  # --- Store -----------------------------------------------------------
+  # REQUIRED for multi-process apps: the default MemoryStore is per-process,
+  # so a breaker tripped in one worker stays closed in the others. Any store
+  # with read / write(expires_in:, unless_exist:) / increment(expires_in:) /
+  # delete / delete_multi works — Redis is the usual choice:
+  #
+  # config.cache_store = ActiveSupport::Cache::RedisCacheStore.new(
+  #   url: ENV["REDIS_URL"], namespace: "circuit_breaker"
+  # )
+
+  # --- Breaker knobs ----------------------------------------------------
+  # config.failure_threshold       = 5    # consecutive failures before trip
+  # config.cooldown_seconds        = 120  # open duration before a probe
+  # config.failures_window_seconds = 3600 # failure-counter window
+  #
+  # Per-service overrides (a fast-recovery moderation endpoint shouldn't
+  # share a cooldown with an expensive batch endpoint):
+  # config.services = {
+  #   "api:openai:moderation" => { failure_threshold: 2, cooldown_seconds: 30 }
+  # }
+
+  # --- Fallback routing -------------------------------------------------
+  # One deliberate tier-hop per model is the recommended shape. Values may
+  # be a single model or an array of hops.
+  # config.fallback_models = {
+  #   "claude-haiku-4-5"  => "claude-sonnet-4-6",
+  #   "claude-sonnet-4-6" => "claude-opus-4-7",
+  #   "gemini-3.5-flash"  => "claude-sonnet-4-6"   # cross-provider safety net
+  # }
+
+  # --- Telemetry (alert on the error, graph the gauge, trend the counter) —
+  # config.on_error = ->(error, context) {
+  #   Rails.error.report(error, handled: true, context: context)
+  # }
+  # config.on_status = ->(service, state) {
+  #   Appsignal.set_gauge("circuit_breaker.state", state == :open ? 1 : 0, service: service)
+  # }
+  # config.on_fallback = ->(from:, to:, error:) {
+  #   Appsignal.increment_counter("llm.fallback", 1,
+  #     from: from[:service], to: to[:service], error: error.class.name)
+  # }
+
+  # --- Dashboard (mount RubyLLM::Resilience::Engine in routes.rb) --------
+  # DENY-BY-DEFAULT: every dashboard request 404s until you configure this.
+  # config.dashboard_auth = ->(controller) {
+  #   controller.head :not_found unless controller.respond_to?(:current_user) &&
+  #                                     controller.current_user&.admin?
+  # }
+  #
+  # Show a static fleet (instead of only breakers seen since boot), with
+  # descriptions for the About column:
+  # config.dashboard_services = %w[api:anthropic:sonnet api:openai:moderation]
+  # config.service_metadata = {
+  #   "api:anthropic:sonnet" => { description: "Coaching", consumers: "Chat" }
+  # }
+end

data/lib/ruby_llm/resilience/breaker.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "set"
+
+module RubyLLM
+  module Resilience
+    # Cache-backed circuit breaker for one service.
+    #
+    # State machine: CLOSED → OPEN → HALF_OPEN → CLOSED
+    #
+    # Three keys per service in the configured store:
+    #   {service}:failures    — consecutive-failure counter (windowed TTL)
+    #   {service}:open_until  — epoch float; presence means open/half-open
+    #   {service}:probe_lock  — SETNX lock so exactly one caller probes
+    #
+    # API purity contract (learned the hard way in production):
+    #   allow_request?  — the MUTATING gate. In half-open it CONSUMES the
+    #                     probe slot. Call it exactly once per real request.
+    #   open?/closed?/state/failure_count/seconds_until_probe — PURE reads,
+    #                     safe for dashboards, logging, and health checks.
+    #
+    # Fail-open everywhere: if the store is unreachable, the breaker reports
+    # closed and records nothing. The breaker must never take the app down
+    # when Redis blips — the API call itself is the thing being protected.
+    class Breaker
+      attr_reader :service
+
+      # Constants (not class-ivars) so subclasses share one registry —
+      # apps may subclass Breaker to add their own service lists/aliases.
+      REGISTRY = Set.new
+      REGISTRY_MUTEX = Mutex.new
+      private_constant :REGISTRY, :REGISTRY_MUTEX
+
+      class << self
+        def register(service)
+          REGISTRY_MUTEX.synchronize { REGISTRY.add(service) }
+        end
+
+        # Per-process registry of breakers seen since boot. A cache-store
+        # contract can't enumerate keys, so this (plus an explicit list) is
+        # how dashboards discover services.
+        def known_services
+          REGISTRY_MUTEX.synchronize { REGISTRY.to_a.sort }
+        end
+
+        def dashboard_status(services: nil)
+          config = Resilience.config
+          (services || known_services).map do |service|
+            breaker = new(service)
+            settings = config.settings_for(service)
+            {
+              service: service,
+              state: breaker.state,
+              failure_count: breaker.failure_count,
+              seconds_until_probe: breaker.seconds_until_probe,
+              metadata: config.metadata_for(service),
+              failure_threshold: settings.failure_threshold,
+              cooldown_seconds: settings.cooldown_seconds,
+              overridden: config.services.key?(service)
+            }
+          end
+        end
+
+        def reset_registry!
+          REGISTRY_MUTEX.synchronize { REGISTRY.clear }
+        end
+      end
+
+      def initialize(service)
+        @service = service.to_s
+        self.class.register(@service)
+      end
+
+      # The mutating gate: true if this request may proceed. In half-open,
+      # acquires the atomic probe lock — exactly one caller across all
+      # processes gets true; everyone else is treated as open.
+      def allow_request?
+        case current_state
+        when :closed    then true
+        when :open      then false
+        when :half_open then acquire_probe_lock
+        end
+      end
+
+      # Pure: true only when fully open. Half-open reports false (a request
+      # MAY be allowed). Never consumes the probe slot — dashboard-safe.
+      def open?
+        current_state == :open
+      end
+
+      def closed?
+        !open?
+      end
+
+      def state
+        current_state
+      end
+
+      def failure_count
+        safely(0) { cache.read(failures_key) }.to_i
+      end
+
+      # Seconds until the breaker will allow a probe (nil if closed).
+      def seconds_until_probe
+        open_until = safely { cache.read(open_until_key) }
+        return nil unless open_until
+
+        remaining = open_until.to_f - Time.now.to_f
+        remaining.positive? ? remaining.ceil : 0
+      end
+
+      # Reset failure count and close the breaker. Fires on_status with
+      # :closed on EVERY success — gauge semantics (idempotent), matching
+      # the production original. It is not a once-per-transition event.
+      def record_success
+        safely { cache.delete_multi([ failures_key, open_until_key, probe_lock_key ]) }
+        notify_status(:closed)
+      end
+
+      # Increment the failure counter; trip at threshold. In half-open, a
+      # single probe failure re-opens immediately (force: the open_until key
+      # still exists in half-open and must be overwritten, not skipped).
+      def record_failure
+        if current_state == :half_open
+          trip!(force: true)
+        else
+          count = safely do
+            cache.increment(failures_key, 1, expires_in: settings.failures_window_seconds)
+          end
+          trip! if count && count >= settings.failure_threshold
+        end
+      end
+
+      # Force-close (admin/console use).
+      def reset!
+        record_success
+      end
+
+      private
+
+      def current_state
+        open_until = safely(:store_error) { cache.read(open_until_key) }
+        return :closed if open_until == :store_error # fail open
+        return :closed if open_until.nil?
+
+        Time.now.to_f < open_until.to_f ? :open : :half_open
+      end
+
+      # Trips are ATOMIC: from closed, the open_until write uses SETNX so
+      # racing threads that all crossed the threshold produce exactly one
+      # transition (and one set of callbacks). From half-open (force:), the
+      # key already exists and must be overwritten — no race is possible
+      # there because the probe lock admitted exactly one caller.
+      def trip!(force: false)
+        cooldown = settings.cooldown_seconds
+        open_until = Time.now.to_f + cooldown
+
+        transitioned = safely(false) do
+          if force
+            cache.write(open_until_key, open_until, expires_in: cooldown * 2)
+            true
+          else
+            cache.write(open_until_key, open_until, expires_in: cooldown * 2, unless_exist: true)
+          end
+        end
+        return unless transitioned
+
+        safely { cache.delete(failures_key) }
+        notify_error(
+          BreakerTripped.new("Circuit breaker tripped for #{@service}"),
+          { service: @service, cooldown_seconds: cooldown }
+        )
+        notify_status(:open)
+      end
+
+      # Atomic SETNX: exactly one fiber/thread/process becomes the probe.
+      # Lock TTL = cooldown, so a probe that dies without reporting frees
+      # the slot after one cooldown period.
+      def acquire_probe_lock
+        safely(false) do
+          cache.write(probe_lock_key, true, expires_in: settings.cooldown_seconds, unless_exist: true)
+        end
+      end
+
+      # Store operations never raise into the caller: report via on_error
+      # and return the fallback. The call being protected matters more than
+      # the bookkeeping around it.
+      def safely(fallback = nil)
+        yield
+      rescue StandardError => e
+        notify_error(e, { service: @service, phase: "circuit_breaker_store" })
+        fallback
+      end
+
+      # User callbacks must not be able to break the call path either.
+      def notify_status(state)
+        config.on_status.call(@service, state)
+      rescue StandardError
+        nil
+      end
+
+      def notify_error(error, context)
+        config.on_error.call(error, context)
+      rescue StandardError
+        nil
+      end
+
+      def failures_key   = "#{@service}:failures"
+      def open_until_key = "#{@service}:open_until"
+      def probe_lock_key = "#{@service}:probe_lock"
+
+      def cache    = config.cache_store
+      def config   = Resilience.config
+      def settings = config.settings_for(@service)
+    end
+  end
+end

data/lib/ruby_llm/resilience/chain.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Resilience
+    # Orchestration: circuit-broken calls and fallback chains.
+    #
+    # The chain semantics are the part that isn't a textbook breaker:
+    #   - steps whose breaker is open are SKIPPED, not attempted
+    #   - the first success wins
+    #   - if the chain is exhausted and ANY step was skipped-open, raise
+    #     BreakerTripped (not the last real error) so callers can distinguish
+    #     "the providers are down" (fail closed) from "your request failed"
+    module Chain
+      module_function
+
+      # Circuit-break a single call. Raises the original error on failure,
+      # or BreakerTripped if the circuit is open.
+      def run(service)
+        breaker = Breaker.new(service)
+        raise BreakerTripped, "Circuit open for #{service}" unless breaker.allow_request?
+
+        result = yield
+        breaker.record_success
+        result
+      rescue BreakerTripped
+        raise
+      rescue StandardError => error
+        breaker.record_failure if Resilience.trippable?(error)
+        raise
+      end
+
+      # Model fallback via the configured map (or an explicit override). The
+      # block receives the model to use (primary first; hops on trippable or
+      # fallback-class errors). Breaker names come from the service_namer.
+      #
+      #   Resilience.run_with_model_fallback("claude-haiku-4-5") { |model|
+      #     RubyLLM.chat(model: model).ask(prompt)
+      #   }
+      #
+      # The fallback: keyword makes routing per-call configurable:
+      #   fallback: :map          — default: use config.fallback_models
+      #   fallback: false / nil   — no fallback; primary only
+      #   fallback: "model-name"  — explicit hop, ignoring the map
+      #   fallback: [ "a", "b" ]  — explicit multi-hop chain
+      def run_with_model_fallback(primary_model, fallback: :map, &block)
+        namer = Resilience.config.service_namer
+
+        hops =
+          case fallback
+          when :map       then Resilience.config.fallbacks_for(primary_model)
+          when nil, false then []
+          else Array(fallback)
+          end
+
+        steps = [ primary_model, *hops ].map do |model|
+          { service: namer.call(model), model: model.to_s, call: -> { block.call(model) } }
+        end
+
+        run_with_fallback(*steps)
+      end
+
+      # Try steps in order; each is { service:, call: -> { ... } } (an
+      # optional :model key is carried through to on_fallback for context).
+      # Every advance past a failed or skipped step fires config.on_fallback
+      # with from:, to:, error: — the audit trail.
+      def run_with_fallback(*steps)
+        raise ArgumentError, "run_with_fallback requires at least one step" if steps.empty?
+
+        last_error = nil
+        any_breaker_tripped = false
+
+        steps.each_with_index do |step, index|
+          begin
+            return run(step[:service]) { step[:call].call }
+          rescue BreakerTripped => error
+            any_breaker_tripped = true
+            last_error = error
+          rescue *Resilience.config.resolved_fallback_errors => error
+            last_error = error
+          end
+
+          next_step = steps[index + 1]
+          notify_fallback(step, next_step, last_error) if next_step
+        end
+
+        raise BreakerTripped, last_error&.message if any_breaker_tripped
+        raise last_error
+      end
+
+      # Callback failures must never break the call path.
+      def notify_fallback(from, to, error)
+        Resilience.config.on_fallback.call(
+          from: from.except(:call), to: to.except(:call), error: error
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/ruby_llm/resilience/configuration.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Resilience
+    # All five injection seams live here. Configure once at boot, then the
+    # configuration is frozen — concurrency safety comes from immutability,
+    # not locks. (Use RubyLLM::Resilience.reset_configuration! in tests.)
+    class Configuration
+      # Server-side unhealthiness — these TRIP the breaker. Client errors
+      # (4xx, auth, bad request) never do: those are your bug, not their
+      # outage. Resolved lazily so ruby_llm/faraday remain soft dependencies.
+      DEFAULT_TRIPPABLE_NAMES = %w[
+        RubyLLM::RateLimitError
+        RubyLLM::ServerError
+        RubyLLM::ServiceUnavailableError
+        RubyLLM::OverloadedError
+        Faraday::TimeoutError
+        Faraday::ConnectionFailed
+        Faraday::ServerError
+      ].freeze
+
+      # Errors that advance a fallback chain to its next step (without
+      # necessarily tripping). ModelNotFoundError is deliberate: it covers
+      # new-model rollout windows where a registry misses an ID. Programming
+      # bugs (NoMethodError, ArgumentError) are NOT here — those propagate so
+      # they surface in your error tracker instead of silently degrading.
+      DEFAULT_FALLBACK_NAMES = %w[
+        RubyLLM::Error
+        RubyLLM::ModelNotFoundError
+        Faraday::Error
+      ].freeze
+
+      attr_accessor :cache_store, :failure_threshold, :cooldown_seconds,
+                    :failures_window_seconds, :fallback_models,
+                    :on_error, :on_status, :on_fallback,
+                    :provider_resolver, :service_namer,
+                    :trippable_errors, :fallback_errors,
+                    :services, :service_metadata, :dashboard_auth, :dashboard_services
+
+      # Normalized fallback hops for a model: always an Array (possibly empty).
+      def fallbacks_for(model_name)
+        Array(@fallback_models[model_name.to_s])
+      end
+
+      # Effective per-service settings (global defaults + per-service override).
+      Settings = Struct.new(:failure_threshold, :cooldown_seconds, :failures_window_seconds)
+
+      def initialize
+        @cache_store = MemoryStore.new
+        @failure_threshold = 5
+        @cooldown_seconds = 120
+        @failures_window_seconds = 3600
+        # Model → fallback(s). Values may be a single model or an array of
+        # hops, tried in order:
+        #   c.fallback_models = {
+        #     "claude-haiku-4-5"  => "claude-sonnet-4-6",                       # one hop
+        #     "gemini-3.5-flash"  => ["claude-sonnet-4-6", "claude-opus-4-7"]   # multi-hop
+        #   }
+        # One deliberate hop is the recommended default (capacity incidents
+        # are tier-correlated; chains of desperation compound quality drift) —
+        # multi-hop is available, not encouraged.
+        @fallback_models = {}
+        @on_error = ->(_error, _context) {}
+        @on_status = ->(_service, _state) {}
+
+        # Fired every time a chain advances past a failed/skipped step —
+        # the audit trail that keeps fallback an emergency, not an ambient
+        # optimization:
+        #   c.on_fallback = ->(from:, to:, error:) {
+        #     Appsignal.increment_counter("llm.fallback", 1,
+        #       from: from[:service], to: to[:service], error: error.class.name)
+        #   }
+        @on_fallback = ->(from:, to:, error:) {}
+        @provider_resolver = default_provider_resolver
+        @service_namer = TierNamer
+        @trippable_errors = nil # nil => lazy defaults (see resolved_* below)
+        @fallback_errors = nil
+
+        # Per-service overrides for the three breaker knobs. A cheap
+        # fast-recovery moderation endpoint and an expensive batch endpoint
+        # shouldn't share one cooldown:
+        #   c.services = { "api:openai:moderation" => { cooldown_seconds: 30 } }
+        @services = {}
+
+        # App knowledge for dashboards — descriptions belong in config, not
+        # hardcoded in a controller:
+        #   c.service_metadata = { "api:anthropic:sonnet" =>
+        #     { description: "Coaching + generation", consumers: "Chat, Practice" } }
+        @service_metadata = {}
+
+        # Default service list for the dashboard engine. nil = the
+        # per-process registry (services seen since boot). Apps with a known
+        # static fleet should set this so the dashboard is complete from the
+        # first request:
+        #   c.dashboard_services = %w[api:anthropic:sonnet api:openai:gpt ...]
+        @dashboard_services = nil
+
+        # Auth hook for the mountable dashboard (see resilience/engine).
+        # DENY BY DEFAULT: mounting without configuring this renders 404 on
+        # every request. Override with e.g.
+        #   c.dashboard_auth = ->(controller) {
+        #     controller.head :not_found unless controller.current_user&.admin?
+        #   }
+        @dashboard_auth = ->(controller) { controller.head :not_found }
+      end
+
+      def settings_for(service)
+        override = @services[service.to_s] || {}
+        Settings.new(
+          override[:failure_threshold] || failure_threshold,
+          override[:cooldown_seconds] || cooldown_seconds,
+          override[:failures_window_seconds] || failures_window_seconds
+        )
+      end
+
+      def metadata_for(service)
+        @service_metadata[service.to_s] || {}
+      end
+
+      def resolved_trippable_errors
+        @trippable_errors || ErrorResolution.resolve(DEFAULT_TRIPPABLE_NAMES)
+      end
+
+      def resolved_fallback_errors
+        @fallback_errors || ErrorResolution.resolve(DEFAULT_FALLBACK_NAMES)
+      end
+
+      private
+
+      # RubyLLM's Models#find RAISES ModelNotFoundError for unknown ids — and
+      # unknown ids occur precisely during new-model rollout windows. Rescue
+      # to "unknown" rather than letting the resolver take down the call.
+      def default_provider_resolver
+        lambda do |model_name|
+          return "unknown" unless defined?(::RubyLLM) && ::RubyLLM.respond_to?(:models)
+
+          begin
+            ::RubyLLM.models.find(model_name.to_s)&.provider.to_s
+          rescue StandardError
+            "unknown"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/resilience/engine.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Optional mountable dashboard. Deliberately NOT loaded by the default
+# require — the core gem has zero dependencies and works outside Rails.
+#
+#   # config/application.rb (or an initializer)
+#   require "ruby_llm/resilience/engine"
+#
+#   # config/routes.rb
+#   mount RubyLLM::Resilience::Engine => "/resilience"
+#
+# SECURITY: the dashboard denies by default. Every request 404s until you
+# configure an auth hook:
+#
+#   RubyLLM::Resilience.configure do |c|
+#     c.dashboard_auth = ->(controller) {
+#       controller.head :not_found unless controller.respond_to?(:current_user) &&
+#                                         controller.current_user&.admin?
+#     }
+#   end
+raise LoadError, "ruby_llm/resilience/engine requires Rails" unless defined?(::Rails::Engine)
+
+require "ruby_llm/resilience"
+
+module RubyLLM
+  module Resilience
+    class Engine < ::Rails::Engine
+      isolate_namespace RubyLLM::Resilience
+
+      # Zeitwerk camelizes "ruby_llm" to "RubyLlm" without this. Matches the
+      # acronym RubyLLM's own Rails integration docs tell users to register,
+      # so host apps that already have it are unaffected.
+      initializer "ruby_llm_resilience.inflections" do
+        ActiveSupport::Inflector.inflections(:en) do |inflect|
+          inflect.acronym "LLM"
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/resilience/errors.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Resilience
+    # Raised when a call is blocked by an open circuit, or when a fallback
+    # chain is exhausted and at least one step was skipped because its
+    # breaker was open.
+    #
+    # Distinguishing this class from the underlying provider error lets
+    # callers fail CLOSED ("the provider is down, show the friendly banner")
+    # instead of treating it like a one-off request failure.
+    class BreakerTripped < StandardError; end
+
+    # Resolves error class names lazily so the gem has zero hard runtime
+    # dependencies: if ruby_llm or faraday isn't loaded, their error classes
+    # simply don't participate in the default lists.
+    module ErrorResolution
+      module_function
+
+      def resolve(names)
+        names.filter_map do |name|
+          Object.const_get(name)
+        rescue NameError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/resilience/memory_store.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Resilience
+    # Thread-safe in-process store implementing the five-method cache
+    # contract the breaker needs:
+    #
+    #   read(key)
+    #   write(key, value, expires_in: nil, unless_exist: false) -> true/false
+    #   increment(key, amount, expires_in: nil) -> Integer
+    #   delete(key)
+    #   delete_multi(keys)
+    #
+    # This is the DEFAULT store and it is per-process: two Puma workers each
+    # see their own breaker state. That's fine for development and small
+    # deployments, but multi-process production apps should configure a
+    # shared store (e.g. ActiveSupport::Cache::RedisCacheStore) so a breaker
+    # tripped in one process is open in all of them.
+    #
+    # TTL contract: `increment` applies `expires_in` only when it CREATES the
+    # counter; subsequent increments do not refresh the TTL. (This matches
+    # how the failure-counter window is meant to behave: N failures within
+    # the window of the first failure.)
+    class MemoryStore
+      Entry = Struct.new(:value, :expires_at)
+
+      def initialize
+        @data = {}
+        @mutex = Mutex.new
+      end
+
+      def read(key)
+        @mutex.synchronize { live_entry(key)&.value }
+      end
+
+      def write(key, value, expires_in: nil, unless_exist: false)
+        @mutex.synchronize do
+          return false if unless_exist && live_entry(key)
+
+          @data[key] = Entry.new(value, expires_in ? now + expires_in : nil)
+          true
+        end
+      end
+
+      def increment(key, amount = 1, expires_in: nil)
+        @mutex.synchronize do
+          entry = live_entry(key)
+          if entry
+            entry.value = entry.value.to_i + amount
+          else
+            entry = Entry.new(amount, expires_in ? now + expires_in : nil)
+            @data[key] = entry
+          end
+          entry.value
+        end
+      end
+
+      def delete(key)
+        @mutex.synchronize { !@data.delete(key).nil? }
+      end
+
+      def delete_multi(keys)
+        @mutex.synchronize { keys.count { |key| !@data.delete(key).nil? } }
+      end
+
+      def clear
+        @mutex.synchronize { @data.clear }
+      end
+
+      private
+
+      # Must be called while holding @mutex.
+      def live_entry(key)
+        entry = @data[key]
+        return nil unless entry
+
+        if entry.expires_at && now >= entry.expires_at
+          @data.delete(key)
+          nil
+        else
+          entry
+        end
+      end
+
+      def now
+        Time.now.to_f
+      end
+    end
+  end
+end