breaker_machines 0.3.0 → 0.4.0

data/lib/breaker_machines/dsl/hedged_builder.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module BreakerMachines
+  module DSL
+    # Builder for hedged request configuration
+    class HedgedBuilder
+      def initialize(config)
+        @config = config
+        @config[:hedged_requests] = true
+      end
+
+      def delay(milliseconds)
+        @config[:hedging_delay] = milliseconds
+      end
+
+      def max_requests(count)
+        @config[:max_hedged_requests] = count
+      end
+    end
+  end
+end

data/lib/breaker_machines/dsl/parallel_fallback_wrapper.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module BreakerMachines
+  module DSL
+    # Wrapper to indicate parallel execution for fallbacks
+    class ParallelFallbackWrapper
+      attr_reader :fallbacks
+
+      def initialize(fallbacks)
+        @fallbacks = fallbacks
+      end
+
+      def call(error)
+        # This will be handled by the circuit's fallback mechanism
+        # to execute fallbacks in parallel
+        raise NotImplementedError, 'ParallelFallbackWrapper should be handled by Circuit'
+      end
+    end
+  end
+end

data/lib/breaker_machines/dsl.rb

@@ -43,7 +43,7 @@ module BreakerMachines
         @circuits ||= {}
 
         if block_given?
-          builder = CircuitBuilder.new
+          builder = DSL::CircuitBuilder.new
           builder.instance_eval(&block)
           @circuits[name] = builder.config
         end
@@ -51,6 +51,19 @@ module BreakerMachines
         @circuits[name]
       end
 
+      # Define a cascading circuit breaker that can trip dependent circuits
+      def cascade_circuit(name, &block)
+        @circuits ||= {}
+
+        if block_given?
+          builder = DSL::CascadingCircuitBuilder.new
+          builder.instance_eval(&block)
+          @circuits[name] = builder.config.merge(circuit_type: :cascading)
+        end
+
+        @circuits[name]
+      end
+
       def circuits
         # Start with parent circuits if available
         base_circuits = if superclass.respond_to?(:circuits)
@@ -72,7 +85,7 @@ module BreakerMachines
         @circuit_templates ||= {}
 
         if block_given?
-          builder = CircuitBuilder.new
+          builder = DSL::CircuitBuilder.new
           builder.instance_eval(&block)
           @circuit_templates[name] = builder.config
         end
@@ -156,7 +169,16 @@ module BreakerMachines
     def circuit(name)
       self.class.circuits[name] ||= {}
       @circuit_instances ||= {}
-      @circuit_instances[name] ||= Circuit.new(name, self.class.circuits[name].merge(owner: self))
+
+      config = self.class.circuits[name].merge(owner: self)
+      circuit_type = config.delete(:circuit_type)
+
+      @circuit_instances[name] ||= case circuit_type
+                                   when :cascading
+                                     CascadingCircuit.new(name, config)
+                                   else
+                                     Circuit.new(name, config)
+                                   end
     end
 
     # Create a dynamic circuit breaker with inline configuration
@@ -173,7 +195,7 @@ module BreakerMachines
 
       # Apply additional configuration if block provided
       if config_block
-        builder = CircuitBuilder.new
+        builder = DSL::CircuitBuilder.new
         builder.instance_variable_set(:@config, base_config.deep_dup)
         builder.instance_eval(&config_block)
         base_config = builder.config
@@ -257,240 +279,5 @@ module BreakerMachines
     def cleanup_stale_dynamic_circuits(max_age_seconds = 3600)
       BreakerMachines.registry.cleanup_stale_dynamic_circuits(max_age_seconds)
     end
-
-    # DSL builder for configuring circuit breakers with a fluent interface
-    class CircuitBuilder
-      attr_reader :config
-
-      def initialize
-        @config = {
-          failure_threshold: 5,
-          failure_window: 60.seconds,
-          success_threshold: 1,
-          timeout: nil,
-          reset_timeout: 60.seconds,
-          half_open_calls: 1,
-          exceptions: [StandardError],
-          storage: nil,
-          metrics: nil,
-          fallback: nil,
-          on_open: nil,
-          on_close: nil,
-          on_half_open: nil,
-          on_reject: nil,
-          notifications: [],
-          fiber_safe: BreakerMachines.config.fiber_safe
-        }
-      end
-
-      def threshold(failures: nil, failure_rate: nil, minimum_calls: nil, within: 60.seconds, successes: nil)
-        if failure_rate
-          # Rate-based threshold
-          validate_failure_rate!(failure_rate)
-          validate_positive_integer!(:minimum_calls, minimum_calls) if minimum_calls
-
-          @config[:failure_rate] = failure_rate
-          @config[:minimum_calls] = minimum_calls || 5
-          @config[:use_rate_threshold] = true
-        elsif failures
-          # Absolute count threshold (existing behavior)
-          validate_positive_integer!(:failures, failures)
-          @config[:failure_threshold] = failures
-          @config[:use_rate_threshold] = false
-        end
-
-        validate_positive_integer!(:within, within.to_i)
-        @config[:failure_window] = within.to_i
-
-        return unless successes
-
-        validate_positive_integer!(:successes, successes)
-        @config[:success_threshold] = successes
-      end
-
-      def reset_after(duration, jitter: nil)
-        validate_positive_integer!(:duration, duration.to_i)
-        @config[:reset_timeout] = duration.to_i
-
-        return unless jitter
-
-        validate_jitter!(jitter)
-        @config[:reset_timeout_jitter] = jitter
-      end
-
-      def timeout(duration)
-        validate_non_negative_integer!(:timeout, duration.to_i)
-        @config[:timeout] = duration.to_i
-      end
-
-      def half_open_requests(count)
-        validate_positive_integer!(:half_open_requests, count)
-        @config[:half_open_calls] = count
-      end
-
-      def storage(backend, **options)
-        @config[:storage] = case backend
-                            when :memory
-                              Storage::Memory.new(**options)
-                            when :bucket_memory
-                              Storage::BucketMemory.new(**options)
-                            when :cache
-                              Storage::Cache.new(**options)
-                            when :null
-                              Storage::Null.new(**options)
-                            when :fallback_chain
-                              config = options.is_a?(Proc) ? options.call(timeout: 5) : options
-                              Storage::FallbackChain.new(config)
-                            when Class
-                              backend.new(**options)
-                            else
-                              backend
-                            end
-      end
-
-      def metrics(recorder = nil, &block)
-        @config[:metrics] = recorder || block
-      end
-
-      def fallback(value = nil, &block)
-        raise ArgumentError, 'Fallback requires either a value or a block' if value.nil? && !block_given?
-
-        fallback_value = block || value
-
-        if @config[:fallback].is_a?(Array)
-          @config[:fallback] << fallback_value
-        elsif @config[:fallback]
-          @config[:fallback] = [@config[:fallback], fallback_value]
-        else
-          @config[:fallback] = fallback_value
-        end
-      end
-
-      def on_open(&block)
-        @config[:on_open] = block
-      end
-
-      def on_close(&block)
-        @config[:on_close] = block
-      end
-
-      def on_half_open(&block)
-        @config[:on_half_open] = block
-      end
-
-      def on_reject(&block)
-        @config[:on_reject] = block
-      end
-
-      # Configure hedged requests
-      def hedged(&)
-        if block_given?
-          hedged_builder = HedgedBuilder.new(@config)
-          hedged_builder.instance_eval(&)
-        else
-          @config[:hedged_requests] = true
-        end
-      end
-
-      # Configure multiple backends
-      def backends(*backend_list)
-        @config[:backends] = backend_list.flatten
-      end
-
-      # Configure parallel fallback execution
-      def parallel_fallback(fallback_list)
-        @config[:fallback] = ParallelFallbackWrapper.new(fallback_list)
-      end
-
-      def notify(service, url = nil, events: %i[open close], **options)
-        notification = {
-          via: service,
-          url: url,
-          events: Array(events),
-          options: options
-        }
-        @config[:notifications] << notification
-      end
-
-      def handle(*exceptions)
-        @config[:exceptions] = exceptions
-      end
-
-      def fiber_safe(enabled = true) # rubocop:disable Style/OptionalBooleanParameter
-        @config[:fiber_safe] = enabled
-      end
-
-      def max_concurrent(limit)
-        validate_positive_integer!(:max_concurrent, limit)
-        @config[:max_concurrent] = limit
-      end
-
-      # Advanced features
-      def parallel_calls(count, timeout: nil)
-        @config[:parallel_calls] = count
-        @config[:parallel_timeout] = timeout
-      end
-
-      private
-
-      def validate_positive_integer!(name, value)
-        return if value.is_a?(Integer) && value.positive?
-
-        raise BreakerMachines::ConfigurationError,
-              "#{name} must be a positive integer, got: #{value.inspect}"
-      end
-
-      def validate_non_negative_integer!(name, value)
-        return if value.is_a?(Integer) && value >= 0
-
-        raise BreakerMachines::ConfigurationError,
-              "#{name} must be a non-negative integer, got: #{value.inspect}"
-      end
-
-      def validate_failure_rate!(rate)
-        return if rate.is_a?(Numeric) && rate >= 0.0 && rate <= 1.0
-
-        raise BreakerMachines::ConfigurationError,
-              "failure_rate must be between 0.0 and 1.0, got: #{rate.inspect}"
-      end
-
-      def validate_jitter!(jitter)
-        return if jitter.is_a?(Numeric) && jitter >= 0.0 && jitter <= 1.0
-
-        raise BreakerMachines::ConfigurationError,
-              "jitter must be between 0.0 and 1.0 (0% to 100%), got: #{jitter.inspect}"
-      end
-    end
-
-    # Builder for hedged request configuration
-    class HedgedBuilder
-      def initialize(config)
-        @config = config
-        @config[:hedged_requests] = true
-      end
-
-      def delay(milliseconds)
-        @config[:hedging_delay] = milliseconds
-      end
-
-      def max_requests(count)
-        @config[:max_hedged_requests] = count
-      end
-    end
-
-    # Wrapper to indicate parallel execution for fallbacks
-    class ParallelFallbackWrapper
-      attr_reader :fallbacks
-
-      def initialize(fallbacks)
-        @fallbacks = fallbacks
-      end
-
-      def call(error)
-        # This will be handled by the circuit's fallback mechanism
-        # to execute fallbacks in parallel
-        raise NotImplementedError, 'ParallelFallbackWrapper should be handled by Circuit'
-      end
-    end
   end
 end

data/lib/breaker_machines/registry.rb

@@ -102,13 +102,13 @@ module BreakerMachines
 
       {
         summary: stats_summary,
-        circuits: circuits.map(&:stats),
+        circuits: circuits.map { |c| c.stats.to_h },
         health: {
           open_count: circuits.count(&:open?),
           closed_count: circuits.count(&:closed?),
           half_open_count: circuits.count(&:half_open?),
-          total_failures: circuits.sum { |c| c.stats[:failure_count] },
-          total_successes: circuits.sum { |c| c.stats[:success_count] }
+          total_failures: circuits.sum { |c| c.stats.failure_count },
+          total_successes: circuits.sum { |c| c.stats.success_count }
         }
       }
     end

data/lib/breaker_machines/storage/backend_state.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module BreakerMachines
+  module Storage
+    # Manages the health state of a single storage backend using a state machine.
+    class BackendState
+      attr_reader :name, :failure_count, :last_failure_at
+      attr_accessor :health
+
+      def initialize(name, threshold:, timeout:)
+        @name = name
+        @threshold = threshold
+        @timeout = timeout
+        @failure_count = 0
+        @last_failure_at = nil
+        @health = :healthy
+      end
+
+      state_machine :health, initial: :healthy do
+        event :trip do
+          transition healthy: :unhealthy, if: :threshold_reached?
+        end
+
+        event :recover do
+          transition unhealthy: :healthy
+        end
+
+        event :reset do
+          transition all => :healthy
+        end
+
+        before_transition to: :unhealthy do |backend, _transition|
+          backend.instance_variable_set(:@unhealthy_until,
+                                        BreakerMachines.monotonic_time + backend.instance_variable_get(:@timeout))
+        end
+
+        after_transition to: :healthy do |backend, _transition|
+          backend.instance_variable_set(:@failure_count, 0)
+          backend.instance_variable_set(:@last_failure_at, nil)
+          backend.instance_variable_set(:@unhealthy_until, nil)
+        end
+      end
+
+      def record_failure
+        @failure_count += 1
+        @last_failure_at = BreakerMachines.monotonic_time
+        trip
+      end
+
+      def threshold_reached?
+        @failure_count >= @threshold
+      end
+
+      def unhealthy_due_to_timeout?
+        return false unless unhealthy?
+
+        unhealthy_until = instance_variable_get(:@unhealthy_until)
+        return false unless unhealthy_until
+
+        if BreakerMachines.monotonic_time > unhealthy_until
+          recover
+          false
+        else
+          true
+        end
+      end
+    end
+  end
+end

data/lib/breaker_machines/storage/bucket_memory.rb

@@ -27,10 +27,10 @@ module BreakerMachines
         circuit_data = @circuits[circuit_name]
         return nil unless circuit_data
 
-        {
+        BreakerMachines::Status.new(
           status: circuit_data[:status],
           opened_at: circuit_data[:opened_at]
-        }
+        )
       end
 
       def set_status(circuit_name, status, opened_at = nil)
@@ -160,7 +160,7 @@ module BreakerMachines
       end
 
       def monotonic_time
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        BreakerMachines.monotonic_time
       end
 
       def with_timeout(_timeout_ms)

data/lib/breaker_machines/storage/cache.rb

@@ -16,10 +16,10 @@ module BreakerMachines
         data = @cache.read(status_key(circuit_name))
         return nil unless data
 
-        {
+        BreakerMachines::Status.new(
           status: data[:status].to_sym,
           opened_at: data[:opened_at]
-        }
+        )
       end
 
       def set_status(circuit_name, status, opened_at = nil)
@@ -162,7 +162,7 @@ module BreakerMachines
       end
 
       def monotonic_time
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        BreakerMachines.monotonic_time
       end
     end
   end

data/lib/breaker_machines/storage/fallback_chain.rb

@@ -9,15 +9,18 @@ module BreakerMachines
     # cache stores (Redis, Memcached) will work properly. Memory-based backends (:memory,
     # :bucket_memory) are incompatible with DRb as they don't share state between processes.
     class FallbackChain < Base
-      attr_reader :storage_configs, :storage_instances, :unhealthy_until, :circuit_breaker_threshold, :circuit_breaker_timeout
+      attr_reader :storage_configs, :storage_instances, :backend_states
 
-      def initialize(storage_configs, **)
+      def initialize(storage_configs, circuit_breaker_threshold: 3, circuit_breaker_timeout: 30, **)
         super(**)
         @storage_configs = normalize_storage_configs(storage_configs)
         @storage_instances = {}
-        @unhealthy_until = {}
-        @circuit_breaker_threshold = 3 # After 3 failures, mark backend as unhealthy
-        @circuit_breaker_timeout = 30 # Keep marked as unhealthy for 30 seconds
+        @circuit_breaker_threshold = circuit_breaker_threshold
+        @circuit_breaker_timeout = circuit_breaker_timeout
+        @backend_states = @storage_configs.to_h do |config|
+          [config[:backend],
+           BackendState.new(config[:backend], threshold: @circuit_breaker_threshold, timeout: @circuit_breaker_timeout)]
+        end
         validate_configs!
       end
 
@@ -72,27 +75,28 @@ module BreakerMachines
           instance.clear_all if instance.respond_to?(:clear_all)
         end
         storage_instances.clear
-        @backend_failures&.clear
-        unhealthy_until.clear
+        backend_states.each_value(&:reset)
       end
 
       private
 
       def execute_with_fallback(method, *args, **kwargs)
-        chain_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        chain_started_at = BreakerMachines.monotonic_time
         attempted_backends = []
 
         storage_configs.each_with_index do |config, index|
-          attempted_backends << config[:backend]
+          backend_type = config[:backend]
+          attempted_backends << backend_type
+          backend_state = backend_states[backend_type]
 
-          if backend_unhealthy?(config[:backend])
-            emit_backend_skipped_notification(config[:backend], method, index)
+          if backend_state.unhealthy_due_to_timeout?
+            emit_backend_skipped_notification(backend_type, method, index)
             next
           end
 
           begin
-            backend = get_backend_instance(config[:backend])
-            started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            backend = get_backend_instance(backend_type)
+            started_at = BreakerMachines.monotonic_time
 
             result = backend.with_timeout(config[:timeout]) do
               if kwargs.any?
@@ -102,35 +106,30 @@ module BreakerMachines
               end
             end
 
-            # Success - emit success notification and reset failure count
-            duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round(2)
-            emit_operation_success_notification(config[:backend], method, duration_ms, index)
-            reset_backend_failures(config[:backend])
+            duration_ms = ((BreakerMachines.monotonic_time - started_at) * 1000).round(2)
+            emit_operation_success_notification(backend_type, method, duration_ms, index)
+            reset_backend_failures(backend_type)
 
-            # Emit chain success notification
-            chain_duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - chain_started_at) * 1000).round(2)
-            emit_chain_success_notification(method, attempted_backends, config[:backend], chain_duration_ms)
+            chain_duration_ms = ((BreakerMachines.monotonic_time - chain_started_at) * 1000).round(2)
+            emit_chain_success_notification(method, attempted_backends, backend_type, chain_duration_ms)
 
             return result
           rescue BreakerMachines::StorageTimeoutError, BreakerMachines::StorageError, StandardError => e
-            duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round(2)
-
-            # Record the failure
-            record_backend_failure(config[:backend], e, duration_ms)
-
-            # Emit notification about the fallback
-            emit_fallback_notification(config[:backend], e, duration_ms, index)
-
-            # If this is the last backend, re-raise the error
-            raise e if index == storage_configs.size - 1
+            duration_ms = ((BreakerMachines.monotonic_time - started_at) * 1000).round(2)
+            record_backend_failure(backend_type, e, duration_ms)
+            emit_fallback_notification(backend_type, e, duration_ms, index)
+
+            if index == storage_configs.size - 1
+              chain_duration_ms = ((BreakerMachines.monotonic_time - chain_started_at) * 1000).round(2)
+              emit_chain_failure_notification(method, attempted_backends, chain_duration_ms)
+              raise e
+            end
 
-            # Continue to next backend
             next
           end
         end
 
-        # If we get here, all backends were unhealthy
-        chain_duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - chain_started_at) * 1000).round(2)
+        chain_duration_ms = ((BreakerMachines.monotonic_time - chain_started_at) * 1000).round(2)
         emit_chain_failure_notification(method, attempted_backends, chain_duration_ms)
         raise BreakerMachines::StorageError, 'All storage backends are unhealthy'
       end
@@ -158,49 +157,34 @@ module BreakerMachines
         end
       end
 
-      def backend_unhealthy?(backend_type)
-        unhealthy_until_time = unhealthy_until[backend_type]
-        return false unless unhealthy_until_time
-
-        if Process.clock_gettime(Process::CLOCK_MONOTONIC) > unhealthy_until_time
-          unhealthy_until.delete(backend_type)
-          false
-        else
-          true
-        end
-      end
-
-      def record_backend_failure(backend_type, error, duration_ms)
-        @backend_failures ||= {}
-        @backend_failures[backend_type] ||= []
-        @backend_failures[backend_type] << {
-          error: error,
-          duration_ms: duration_ms,
-          timestamp: Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        }
-
-        # Keep only recent failures (last 60 seconds)
-        cutoff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - 60
-        @backend_failures[backend_type].reject! { |f| f[:timestamp] < cutoff }
+      def record_backend_failure(backend_type, _error, _duration_ms)
+        backend_state = backend_states[backend_type]
+        return unless backend_state
 
-        # Mark as unhealthy if too many failures
-        return unless @backend_failures[backend_type].size >= circuit_breaker_threshold
+        previous_health = backend_state.health_name
+        backend_state.record_failure
+        new_health = backend_state.health_name
 
-        unhealthy_until[backend_type] = Process.clock_gettime(Process::CLOCK_MONOTONIC) + circuit_breaker_timeout
-        emit_backend_health_change_notification(backend_type, :healthy, :unhealthy, @backend_failures[backend_type].size)
+        if new_health != previous_health
+          emit_backend_health_change_notification(backend_type, previous_health, new_health,
+                                                  backend_state.failure_count)
+        end
       rescue StandardError => e
         # Don't let failure recording cause the whole chain to hang
         Rails.logger&.error("FallbackChain: Failed to record backend failure: #{e.message}")
       end
 
       def reset_backend_failures(backend_type)
-        was_unhealthy = unhealthy_until.key?(backend_type)
-        @backend_failures&.delete(backend_type)
-        unhealthy_until.delete(backend_type)
+        backend_state = backend_states[backend_type]
+        return unless backend_state&.unhealthy?
 
-        if was_unhealthy
-          emit_backend_health_change_notification(backend_type, :unhealthy, :healthy, 0)
-        end
+        previous_health = backend_state.health_name
+        backend_state.reset
+        new_health = backend_state.health_name
+
+        return unless new_health != previous_health
+
+        emit_backend_health_change_notification(backend_type, previous_health, new_health, 0)
       end
 
       def emit_fallback_notification(backend_type, error, duration_ms, backend_index)
@@ -227,25 +211,27 @@ module BreakerMachines
       end
 
       def emit_backend_skipped_notification(backend_type, method, backend_index)
+        backend_state = backend_states[backend_type]
         ActiveSupport::Notifications.instrument(
           'storage_backend_skipped.breaker_machines',
           backend: backend_type,
           operation: method,
           backend_index: backend_index,
           reason: 'unhealthy',
-          unhealthy_until: unhealthy_until[backend_type]
+          unhealthy_until: backend_state&.instance_variable_get(:@unhealthy_until)
         )
       end
 
       def emit_backend_health_change_notification(backend_type, previous_state, new_state, failure_count)
+        backend_state = backend_states[backend_type]
         ActiveSupport::Notifications.instrument(
           'storage_backend_health.breaker_machines',
           backend: backend_type,
           previous_state: previous_state,
           new_state: new_state,
           failure_count: failure_count,
-          threshold: circuit_breaker_threshold,
-          recovery_time: new_state == :unhealthy ? unhealthy_until[backend_type] : nil
+          threshold: backend_state&.instance_variable_get(:@threshold),
+          recovery_time: new_state == :unhealthy ? backend_state&.instance_variable_get(:@unhealthy_until) : nil
         )
       end