network_resiliency 0.3.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19102b63c5f8090d6805ac5ce1d71a25e30ab7483ef11bf5d9cee771bb4ce508
-  data.tar.gz: a9f37e7fafbfc1d45e0f8b24b655cf72b33300b5b6335e396d9dbffc22479799
+  metadata.gz: 8949456e392f5198caa080238dff62a567114a9386d3a6f575fdb855fd5b5e98
+  data.tar.gz: 95d58f753ccb7f4aa5ffe1093d98d0f5e33016bb7e06caa111a181d29c8ac5fb
 SHA512:
-  metadata.gz: 27390e9651bef755a92a02218c286291971e7c77baabbddd062e3279d5a52f3a819fbbd9b0a9f91e289bd3124afdc028e42caf6b504c54bb198d959ba54dd1f6
-  data.tar.gz: 56763326a26679f03124a1a4f8b5dab2de10edeeebaff1a4dab8c821ee9b8a035b3b8cf285089bac944c722cc647bcb47264b7fe60295dfbf51d35c4977d2d1f
+  metadata.gz: d1544a231ebcdd48c6838a8b398f3d4463c9cae2e18182c8e28fda3c4cfeba51b75b6e355342f98a16a51979da035d65040dffdd83e1bcd0273c636f14332236
+  data.tar.gz: a90be5d5152dee3ce23ef7d682ce62dd495293e429d06494e5dcc91eb9cf3bf4a0bcf26b1cece0665815ab181394feff48016c9781e81b69408de3dca1572ec1

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+###  v0.4.1  (2023-11-14)
+- timeout units
+
+###  v0.4.0  (2023-11-13)
+- redis resiliency
+- timeouts_for
+- mode
+
 ###  v0.3.2  (2023-11-03)
 - stats observability
 - surface errors in specs

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    network_resiliency (0.3.2)
+    network_resiliency (0.4.1)
 
 GEM
   remote: https://rubygems.org/

data/lib/network_resiliency/adapter/redis.rb

@@ -36,35 +36,50 @@ module NetworkResiliency
       end
 
       module Instrumentation
-        # def initialize(...)
-        #   super
-
-        #   @network_resiliency_attempts = options[:reconnect_attempts]
-        #   options[:reconnect_attempts] = 0
-        # end
-
         def establish_connection
           return super unless NetworkResiliency.enabled?(:redis)
 
+          original_timeout = @options[:connect_timeout]
+
+          timeouts = NetworkResiliency.timeouts_for(
+            adapter: "redis",
+            action: "connect",
+            destination: host,
+            max: original_timeout,
+            units: :seconds,
+          )
+
+          attempts = 0
+          ts = -NetworkResiliency.timestamp
+
           begin
-            ts = -NetworkResiliency.timestamp
+            attempts += 1
+            error = nil
+
+            @options[:connect_timeout] = timeouts.shift
 
             super
           rescue ::Redis::CannotConnectError => e
             # capture error
+
+            # grab underlying exception within Redis wrapper
+            error = e.cause.class
+
+            retry if timeouts.size > 0
+
             raise
           ensure
             ts += NetworkResiliency.timestamp
-
-            # grab underlying exception within Redis wrapper
-            error = e ? e.cause.class : nil
+            @options[:connect_timeout] = original_timeout
 
             NetworkResiliency.record(
               adapter: "redis",
               action: "connect",
               destination: host,
-              error: error,
               duration: ts,
+              error: error,
+              timeout: @options[:connect_timeout] * 1_000,
+              attempts: attempts,
             )
           end
         end

data/lib/network_resiliency/refinements.rb

@@ -8,6 +8,14 @@ module NetworkResiliency
         log10 = Math.log10(self.round)
         10 ** (ceil ? log10.ceil : log10.floor)
       end
+
+      def power_ceil
+        return 0 if self <= 0
+        return 1 if self <= 1
+
+        digits = Math.log10(self).floor
+        10 ** digits * (self.to_f / 10 ** digits).ceil
+      end
     end
   end
 end

data/lib/network_resiliency/stats_engine.rb

@@ -31,7 +31,7 @@ module NetworkResiliency
 
       # select data to be synced
       data = synchronize do
-        # ensure sync is not run concurrently
+        # ensure sync does not run concurrently
         return [] if @syncing
         @syncing = Thread.current
 

data/lib/network_resiliency/version.rb

@@ -1,3 +1,3 @@
 module NetworkResiliency
-  VERSION = "0.3.2"
+  VERSION = "0.4.1"
 end

data/lib/network_resiliency.rb

@@ -14,6 +14,9 @@ module NetworkResiliency
     autoload :Postgres, "network_resiliency/adapter/postgres"
   end
 
+  MODE = [ :observe, :resilient ].freeze
+  RESILIENCY_SIZE_THRESHOLD = 1_000
+
   extend self
 
   attr_accessor :statsd, :redis
@@ -87,9 +90,21 @@ module NetworkResiliency
     Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1_000
   end
 
+  def mode
+    @mode || :observe
+  end
+
+  def mode=(mode)
+    unless MODE.include?(mode)
+      raise ArgumentError, "invalid NetworkResiliency mode: #{mode}"
+    end
+
+    @mode = mode
+  end
+
   # private
 
-  def record(adapter:, action:, destination:, duration:, error: nil)
+  def record(adapter:, action:, destination:, duration:, error:, timeout: nil, attempts: 1)
     return if ignore_destination?(adapter, action, destination)
 
     NetworkResiliency.statsd&.distribution(
@@ -99,6 +114,7 @@ module NetworkResiliency
         adapter: adapter,
         destination: destination,
         error: error,
+        attempts: (attempts if attempts > 1),
       }.compact,
     )
 
@@ -112,8 +128,37 @@ module NetworkResiliency
       }.compact,
     )
 
-    key = [ adapter, action, destination ].join(":")
-    StatsEngine.add(key, duration).tap do |stats|
+    NetworkResiliency.statsd&.gauge(
+      "network_resiliency.#{action}.timeout",
+      timeout,
+      tags: {
+        adapter: adapter,
+        destination: destination,
+      },
+    )
+
+    if error
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.time_saved",
+        timeout - duration,
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if timeout
+    else
+      # track successful retries
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.#{action}.resilient",
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if attempts > 1
+
+      # record stats
+      key = [ adapter, action, destination ].join(":")
+      stats = StatsEngine.add(key, duration)
       tags = {
         adapter: adapter,
         destination: destination,
@@ -138,6 +183,8 @@ module NetworkResiliency
         tags: tags,
       )
     end
+
+    nil
   rescue => e
     NetworkResiliency.statsd&.increment(
       "network_resiliency.error",
@@ -157,11 +204,98 @@ module NetworkResiliency
     IP_ADDRESS_REGEX.match?(destination)
   end
 
+  def timeouts_for(adapter:, action:, destination:, max: nil, units: :ms)
+    default = [ max ]
+
+    return default if NetworkResiliency.mode == :observe
+
+    key = [ adapter, action, destination ].join(":")
+    stats = StatsEngine.get(key)
+
+    return default unless stats.n >= RESILIENCY_SIZE_THRESHOLD
+
+    tags = {
+      adapter: adapter,
+      action: action,
+      destination: destination,
+    }
+
+    p99 = (stats.avg + stats.stdev * 3).power_ceil
+    timeouts = []
+
+    if max
+      max *= 1_000 if units == :s || units == :seconds
+
+      if p99 < max
+        timeouts << p99
+
+        # fallback attempt
+        if max - p99 > p99
+          # use remaining time for second attempt
+          timeouts << max - p99
+        else
+          timeouts << max
+
+          NetworkResiliency.statsd&.increment(
+            "network_resiliency.timeout.raised",
+            tags: tags,
+          )
+        end
+      else
+        # the specified timeout is less than our expected p99...awkward
+        timeouts << max
+
+        NetworkResiliency.statsd&.increment(
+          "network_resiliency.timeout.too_low",
+          tags: tags,
+        )
+      end
+    else
+      timeouts << p99
+
+      # timeouts << p99 * 10 if NetworkResiliency.mode == :resolute
+
+      # unbounded second attempt
+      timeouts << nil
+
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.timeout.missing",
+        tags: tags,
+      )
+    end
+
+    case units
+    when nil, :ms, :milliseconds
+      timeouts
+    when :s, :seconds
+      timeouts.map { |t| t.to_f / 1_000 if t }
+    else
+      raise ArgumentError, "invalid units: #{units}"
+    end
+  rescue => e
+    NetworkResiliency.statsd&.increment(
+      "network_resiliency.error",
+      tags: {
+        method: __method__,
+        type: e.class,
+      },
+    )
+
+    warn "[ERROR] NetworkResiliency: #{e.class}: #{e.message}"
+
+    default
+  end
+
   def reset
     @enabled = nil
+    @mode = nil
     Thread.current["network_resiliency"] = nil
     StatsEngine.reset
-    @sync_worker.kill if @sync_worker
+
+    if @sync_worker
+      @sync_worker.kill
+      @sync_worker = nil
+    end
   end
 
   private
@@ -176,13 +310,11 @@ module NetworkResiliency
     raise "Redis not configured" unless redis
 
     @sync_worker = Thread.new do
-      while true do
+      loop do
         StatsEngine.sync(redis)
 
         sleep(3)
       end
-    rescue Interrupt
-      # goodbye
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: network_resiliency
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.1
 platform: ruby
 authors:
 - Daniel Pepper
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-03 00:00:00.000000000 Z
+date: 2023-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug