breaker_machines 0.19.0 → 0.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6172f37c8307d66dbacea1526038a00deb457e16ea404ea877cd4d4c8f5c18bd
-  data.tar.gz: e246c2d0d5a4df91003105a4ccb2ee5d00593bda8a4d339f8eeb5fd28ac16d41
+  metadata.gz: a533c91306760488de32d6ef3b3a3bfc9a6c874b1b6e77c6fed42b429a2da6b6
+  data.tar.gz: 88991049dc9e83392a13880557ea199830291d21213749a84cd3716c56c9eaac
 SHA512:
-  metadata.gz: b4145bd486e2645b57eab3058671ba08143ac1422a11a68ba00b3079d3c35981d6c8b8c470e2f0de75af59c2799d780a360f74739e43b1022bfd447de6bc5dd0
-  data.tar.gz: 00f7382d4b5ea05c057c3ba77cf6289d3fe96e5a74351b5cc5280d8106240add688156b2b2a76ae1c609bb4c330bbec60745185a9abb1640bde2e5499c040218
+  metadata.gz: 7c25929aa77bacbca4857bc1961110680838e2ae83fee1356d10f3f67297e623f2b1bafe0a104485a372618281d45e542da063482e4ead55294fe9f2e7655478
+  data.tar.gz: 7ef7f97ca50b64a56589ff6a3a062fa4016e33b70e81e3becd3fb705254a3ffbab6af89e75a6d8f0fc17f2c8a3784b48dbdaba4d8fe78383ca244fe15e7a0a5f

data/lib/breaker_machines/circuit/admission.rb

@@ -18,14 +18,28 @@ module BreakerMachines
 
       private
 
+      # Runs on every call: the write lock is taken only to apply a changed
+      # shared status or to start recovery.
       def attempt_recovery_if_ready
-        return unless open? && reset_timeout_elapsed?
+        changed = stored_status_changed?
+        return unless changed || (open? && reset_timeout_elapsed?)
 
         @mutex.with_write_lock do
+          restore_status_from_storage if changed
           attempt_recovery if open? && reset_timeout_elapsed?
         end
       end
 
+      # Reads the shared status before taking the write lock, so storage I/O
+      # happens under it only when there is a change to apply.
+      def with_refreshed_status
+        changed = stored_status_changed?
+        @mutex.with_write_lock do
+          restore_status_from_storage if changed
+          yield
+        end
+      end
+
       def admit_call
         @mutex.with_read_lock do
           case status_name
@@ -55,7 +69,7 @@ module BreakerMachines
       def handle_success(admission)
         return unless admission.half_open?
 
-        @mutex.with_write_lock do
+        with_refreshed_status do
           return unless admission_current?(admission) && half_open?
 
           successful_attempts = @half_open_successes.increment
@@ -68,7 +82,7 @@ module BreakerMachines
       end
 
       def handle_failure(admission)
-        @mutex.with_write_lock do
+        with_refreshed_status do
           return unless admission_current?(admission)
 
           if admission.closed? && closed? && failure_threshold_exceeded?
@@ -84,7 +98,7 @@ module BreakerMachines
       def release_abandoned_admission(admission)
         return unless admission.half_open?
 
-        @mutex.with_write_lock do
+        with_refreshed_status do
           next unless admission_current?(admission) && half_open?
           next unless @half_open_attempts.value.positive?
 

data/lib/breaker_machines/circuit/base.rb

@@ -53,12 +53,50 @@ module BreakerMachines
         @semaphore = (Concurrent::Semaphore.new(@config[:max_concurrent]) if @config[:max_concurrent])
       end
 
+      # A round trip through epoch time is off by up to ~0.12µs (float64
+      # spacing near today's epoch); any larger difference is another opening.
+      OPENED_AT_TOLERANCE = 1e-6
+
+      # Reads the shared status without taking the lock. True when it differs
+      # from the last status this instance stored or observed.
+      def stored_status_changed?
+        stored_status = @storage&.get_status(@name)
+        !stored_status.nil? && !same_stored_status?(stored_status, @stored_status)
+      end
+
       def restore_status_from_storage
-        stored_status = @storage.get_status(@name)
-        return unless stored_status
+        stored_status = @storage&.get_status(@name)
+        return if stored_status.nil? || same_stored_status?(stored_status, @stored_status)
 
+        @stored_status = stored_status
         self.status = stored_status.status.to_s
-        @opened_at.value = stored_status.opened_at if stored_status.opened_at
+        @opened_at.value = stored_status.opened_at
+        @state_epoch.increment
+        @half_open_attempts.value = 0
+        @half_open_successes.value = 0
+        return unless closed?
+
+        @last_error.value = nil
+        @last_failure_at.value = nil
+      end
+
+      def same_stored_status?(stored_status, known_status)
+        return false unless known_status && stored_status.status == known_status.status
+
+        stored_at = stored_status.opened_at
+        known_at = known_status.opened_at
+        return stored_at == known_at if stored_at.nil? || known_at.nil?
+
+        (stored_at - known_at).abs <= OPENED_AT_TOLERANCE
+      end
+
+      # When the storage reports a failed write (false), keep the last status
+      # observed there: the stale entry then reads as unchanged and cannot
+      # undo this transition.
+      def persist_status(status, opened_at = nil)
+        return if @storage.nil? || @storage.set_status(@name, status, opened_at) == false
+
+        @stored_status = BreakerMachines::Status.new(status: status, opened_at: opened_at)
       end
     end
   end

data/lib/breaker_machines/circuit/state_callbacks.rb

@@ -11,7 +11,7 @@ module BreakerMachines
       def on_circuit_open
         @state_epoch.increment
         @opened_at.value = BreakerMachines.monotonic_time
-        @storage&.set_status(@name, :open, @opened_at.value)
+        persist_status(:open, @opened_at.value)
         if @storage.respond_to?(:record_event_with_details)
           @storage.record_event_with_details(@name, :state_change, 0,
                                              new_state: :open)
@@ -25,7 +25,7 @@ module BreakerMachines
         @opened_at.value = nil
         @last_error.value = nil
         @last_failure_at.value = nil
-        @storage&.set_status(@name, :closed)
+        persist_status(:closed)
         if @storage.respond_to?(:record_event_with_details)
           @storage.record_event_with_details(@name, :state_change, 0,
                                              new_state: :closed)
@@ -38,7 +38,7 @@ module BreakerMachines
         @state_epoch.increment
         @half_open_attempts.value = 0
         @half_open_successes.value = 0
-        @storage&.set_status(@name, :half_open)
+        persist_status(:half_open, @opened_at.value)
         if @storage.respond_to?(:record_event_with_details)
           @storage.record_event_with_details(@name, :state_change, 0,
                                              new_state: :half_open)

data/lib/breaker_machines/storage/base.rb

@@ -13,6 +13,8 @@ module BreakerMachines
         raise NotImplementedError
       end
 
+      # opened_at is a BreakerMachines.monotonic_time value. Return false when
+      # the write did not happen; any other value means it was stored.
       def set_status(circuit_name, status, opened_at = nil)
         raise NotImplementedError
       end

data/lib/breaker_machines/storage/cache.rb

@@ -10,6 +10,12 @@ module BreakerMachines
         @cache = cache_store
         @prefix = options[:prefix] || 'breaker_machines'
         @expires_in = options[:expires_in] || 300 # 5 minutes default
+        # Counter resolution in seconds. Each bucket is its own cache entry, so
+        # stores that keep expired entries around (Solid Cache) want coarser buckets.
+        @bucket_seconds = Integer(options[:bucket_seconds] || 1)
+        raise ArgumentError, 'bucket_seconds must be positive' unless @bucket_seconds.positive?
+
+        @warned_windows = Concurrent::Set.new
       end
 
       def get_status(circuit_name)
@@ -18,7 +24,7 @@ module BreakerMachines
 
         BreakerMachines::Status.new(
           status: data[:status].to_sym,
-          opened_at: data[:opened_at]
+          opened_at: local_opened_at(data)
         )
       end
 
@@ -28,6 +34,7 @@ module BreakerMachines
           {
             status: status,
             opened_at: opened_at,
+            opened_at_epoch: opened_at && BreakerMachines.monotonic_to_epoch(opened_at),
             updated_at: monotonic_time
           },
           expires_in: @expires_in
@@ -55,6 +62,12 @@ module BreakerMachines
         @cache.delete(success_key(circuit_name))
         @cache.delete(failure_key(circuit_name))
         @cache.delete(events_key(circuit_name))
+        return unless @cache.respond_to?(:increment)
+
+        keys = [success_key(circuit_name), failure_key(circuit_name)].flat_map do |key|
+          bucket_keys(key, bucket_count(@expires_in) + 1)
+        end
+        delete_counters(keys)
       end
 
       def clear_all
@@ -103,10 +116,20 @@ module BreakerMachines
 
       private
 
+      # Hosts do not share a monotonic clock origin, so other hosts read the
+      # epoch copy. Entries written before it existed carry only the writer's
+      # monotonic time.
+      def local_opened_at(data)
+        epoch = data[:opened_at_epoch]
+        epoch ? BreakerMachines.epoch_to_monotonic(epoch) : data[:opened_at]
+      end
+
       def increment_counter(key)
         # Use increment if available, otherwise fetch-and-update
         if @cache.respond_to?(:increment)
-          @cache.increment(key, 1, expires_in: @expires_in)
+          # Expiry counts from the first increment, so a bucket must outlive
+          # the retention period by its own length.
+          @cache.increment("#{key}:#{current_bucket}", 1, expires_in: @expires_in + @bucket_seconds)
         else
           # Fallback for caches without atomic increment
           current = @cache.fetch(key) { {} }
@@ -117,32 +140,58 @@ module BreakerMachines
       end
 
       def get_window_count(key, window_seconds)
+        warn_window_exceeds_retention(window_seconds)
+        count = bucket_count([window_seconds, @expires_in].min)
+
         if @cache.respond_to?(:increment)
-          # For simple counter-based caches, we can't get windowed counts
-          # Would need to implement bucketing similar to fallback
-          @cache.read(key) || 0
+          read_counters(bucket_keys(key, count)).sum(&:to_i)
         else
-          # Bucket-based counting for accurate windows
           buckets = @cache.read(key) || {}
-          current_time = current_bucket
+          now = current_bucket
+          count.times.sum { |offset| buckets[now - offset] || 0 }
+        end
+      end
 
-          total = 0
-          window_seconds.times do |i|
-            bucket_key = current_time - i
-            total += buckets[bucket_key] || 0
-          end
+      # Counters live for expires_in, so a longer window silently undercounts.
+      def warn_window_exceeds_retention(window_seconds)
+        return if window_seconds <= @expires_in
+        return unless @warned_windows.add?(window_seconds)
 
-          total
-        end
+        BreakerMachines.logger&.warn(
+          "[BreakerMachines] Cache storage keeps counts for #{@expires_in}s, so a #{window_seconds}s " \
+          "window only sees the last #{@expires_in}s. Set expires_in to at least the window."
+        )
+      end
+
+      # read_multi is a single round trip; stores without it read each bucket.
+      def read_counters(keys)
+        return @cache.read_multi(*keys, raw: true).values if @cache.respond_to?(:read_multi)
+
+        keys.filter_map { |key| @cache.read(key) }
+      end
+
+      def delete_counters(keys)
+        return @cache.delete_multi(keys) if @cache.respond_to?(:delete_multi)
+
+        keys.each { |key| @cache.delete(key) }
+      end
+
+      def bucket_count(seconds)
+        (seconds.to_f / @bucket_seconds).ceil
+      end
+
+      def bucket_keys(key, count)
+        now = current_bucket
+        Array.new(count) { |offset| "#{key}:#{now - offset}" }
       end
 
       def prune_old_buckets(buckets)
-        cutoff = current_bucket - 300 # Keep 5 minutes of data
-        buckets.delete_if { |time, _| time < cutoff }
+        cutoff = current_bucket - bucket_count(@expires_in)
+        buckets.delete_if { |bucket, _| bucket < cutoff }
       end
 
       def current_bucket
-        Time.now.to_i
+        Time.now.to_i / @bucket_seconds
       end
 
       def status_key(circuit_name)

data/lib/breaker_machines/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module BreakerMachines
-  VERSION = '0.19.0'
+  VERSION = '0.20.0'
 end

data/lib/breaker_machines.rb

@@ -184,6 +184,35 @@ module BreakerMachines
     def monotonic_time
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
+
+    # Converts a {monotonic_time} value to Unix epoch seconds. Each host's
+    # monotonic clock has its own origin, so shared storage must persist
+    # timestamps in epoch time for other hosts to read them.
+    #
+    # @param monotonic [Float] a value from {monotonic_time}
+    # @return [Float] Unix epoch seconds
+    def monotonic_to_epoch(monotonic)
+      monotonic + clock_offset
+    end
+
+    # Converts Unix epoch seconds to this host's {monotonic_time}.
+    #
+    # @param epoch [Float] Unix epoch seconds
+    # @return [Float] the matching {monotonic_time} value on this host
+    def epoch_to_monotonic(epoch)
+      epoch - clock_offset
+    end
+
+    private
+
+    # Wall clock minus monotonic clock. Memoized so that converting the same
+    # timestamp twice gives the same result; re-measured only when the wall
+    # clock has stepped by more than a second.
+    def clock_offset
+      offset = Process.clock_gettime(Process::CLOCK_REALTIME) - monotonic_time
+      @clock_offset = offset if @clock_offset.nil? || (offset - @clock_offset).abs > 1.0
+      @clock_offset
+    end
   end
 
   # Set up notifications on first use

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: breaker_machines
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.0
 platform: ruby
 authors:
 - Abdelkader Boudih
@@ -15,28 +15,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '8.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '8.0'
+        version: '8.1'
 - !ruby/object:Gem::Dependency
   name: chrono_machines
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -85,14 +85,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.16'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.16'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rb_sys
   requirement: !ruby/object:Gem::Requirement