network_resiliency 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19102b63c5f8090d6805ac5ce1d71a25e30ab7483ef11bf5d9cee771bb4ce508
-  data.tar.gz: a9f37e7fafbfc1d45e0f8b24b655cf72b33300b5b6335e396d9dbffc22479799
+  metadata.gz: c372873056610bf21197e55a265bdfd03e200b064c727a4713ea8725ae3d696d
+  data.tar.gz: 5da4933543bf9a57d46ed4a3b4d94a692524e641f6c1293d8bcb99aa74c0c9d9
 SHA512:
-  metadata.gz: 27390e9651bef755a92a02218c286291971e7c77baabbddd062e3279d5a52f3a819fbbd9b0a9f91e289bd3124afdc028e42caf6b504c54bb198d959ba54dd1f6
-  data.tar.gz: 56763326a26679f03124a1a4f8b5dab2de10edeeebaff1a4dab8c821ee9b8a035b3b8cf285089bac944c722cc647bcb47264b7fe60295dfbf51d35c4977d2d1f
+  metadata.gz: b8e9b66cb83ff5bfda080c43935e8834228cfbd16afd0c1229be2b7edbea8469702d917ae7b6bb7383b774ebf68802e9a5572ee89357723d479d1dd8b3a91f2c
+  data.tar.gz: 68f241f0bd9b7ef5b0b291e2e1be11aa3d2c377f6b4b70552ba32638122bff300bc1252b279a13c41e96af3ffa8843bc571b74bd9aac7f1300f27472e18001ff

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+###  v0.4.0  (2023-11-13)
+- redis resiliency
+- timeouts_for
+- mode
+
 ###  v0.3.2  (2023-11-03)
 - stats observability
 - surface errors in specs

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    network_resiliency (0.3.2)
+    network_resiliency (0.4.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/network_resiliency/adapter/redis.rb

@@ -36,35 +36,49 @@ module NetworkResiliency
       end
 
       module Instrumentation
-        # def initialize(...)
-        #   super
-
-        #   @network_resiliency_attempts = options[:reconnect_attempts]
-        #   options[:reconnect_attempts] = 0
-        # end
-
         def establish_connection
           return super unless NetworkResiliency.enabled?(:redis)
 
+          original_timeout = @options[:connect_timeout]
+
+          timeouts = NetworkResiliency.timeouts_for(
+            adapter: "redis",
+            action: "connect",
+            destination: host,
+            max: original_timeout,
+          )
+
+          attempts = 0
+          ts = -NetworkResiliency.timestamp
+
           begin
-            ts = -NetworkResiliency.timestamp
+            attempts += 1
+            error = nil
+
+            @options[:connect_timeout] = timeouts.shift
 
             super
           rescue ::Redis::CannotConnectError => e
             # capture error
+
+            # grab underlying exception within Redis wrapper
+            error = e.cause.class
+
+            retry if timeouts.size > 0
+
             raise
           ensure
             ts += NetworkResiliency.timestamp
-
-            # grab underlying exception within Redis wrapper
-            error = e ? e.cause.class : nil
+            @options[:connect_timeout] = original_timeout
 
             NetworkResiliency.record(
               adapter: "redis",
               action: "connect",
               destination: host,
-              error: error,
               duration: ts,
+              error: error,
+              timeout: @options[:connect_timeout],
+              attempts: attempts,
             )
           end
         end

data/lib/network_resiliency/refinements.rb

@@ -8,6 +8,14 @@ module NetworkResiliency
         log10 = Math.log10(self.round)
         10 ** (ceil ? log10.ceil : log10.floor)
       end
+
+      def power_ceil
+        return 0 if self <= 0
+        return 1 if self <= 1
+
+        digits = Math.log10(self).floor
+        10 ** digits * (self.to_f / 10 ** digits).ceil
+      end
     end
   end
 end

data/lib/network_resiliency/version.rb

@@ -1,3 +1,3 @@
 module NetworkResiliency
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

data/lib/network_resiliency.rb

@@ -14,6 +14,9 @@ module NetworkResiliency
     autoload :Postgres, "network_resiliency/adapter/postgres"
   end
 
+  MODE = [ :observe, :resilient ].freeze
+  RESILIENCY_SIZE_THRESHOLD = 1_000
+
   extend self
 
   attr_accessor :statsd, :redis
@@ -87,9 +90,21 @@ module NetworkResiliency
     Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1_000
   end
 
+  def mode
+    @mode || :observe
+  end
+
+  def mode=(mode)
+    unless MODE.include?(mode)
+      raise ArgumentError, "invalid NetworkResiliency mode: #{mode}"
+    end
+
+    @mode = mode
+  end
+
   # private
 
-  def record(adapter:, action:, destination:, duration:, error: nil)
+  def record(adapter:, action:, destination:, duration:, error:, timeout: nil, attempts: 1)
     return if ignore_destination?(adapter, action, destination)
 
     NetworkResiliency.statsd&.distribution(
@@ -99,6 +114,7 @@ module NetworkResiliency
         adapter: adapter,
         destination: destination,
         error: error,
+        attempts: (attempts if attempts > 1),
       }.compact,
     )
 
@@ -112,8 +128,37 @@ module NetworkResiliency
       }.compact,
     )
 
-    key = [ adapter, action, destination ].join(":")
-    StatsEngine.add(key, duration).tap do |stats|
+    NetworkResiliency.statsd&.gauge(
+      "network_resiliency.#{action}.timeout",
+      timeout,
+      tags: {
+        adapter: adapter,
+        destination: destination,
+      },
+    )
+
+    if error
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.time_saved",
+        timeout - duration,
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if timeout
+    else
+      # track successful retries
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.#{action}.resilient",
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if attempts > 1
+
+      # record stats
+      key = [ adapter, action, destination ].join(":")
+      stats = StatsEngine.add(key, duration)
       tags = {
         adapter: adapter,
         destination: destination,
@@ -138,6 +183,8 @@ module NetworkResiliency
         tags: tags,
       )
     end
+
+    nil
   rescue => e
     NetworkResiliency.statsd&.increment(
       "network_resiliency.error",
@@ -157,11 +204,89 @@ module NetworkResiliency
     IP_ADDRESS_REGEX.match?(destination)
   end
 
+  def timeouts_for(adapter:, action:, destination:, max: nil)
+    default = [ max ]
+
+    return default if NetworkResiliency.mode == :observe
+
+    key = [ adapter, action, destination ].join(":")
+    stats = StatsEngine.get(key)
+
+    return default unless stats.n >= RESILIENCY_SIZE_THRESHOLD
+
+    tags = {
+      adapter: adapter,
+      action: action,
+      destination: destination,
+    }
+
+    p99 = (stats.avg + stats.stdev * 3).power_ceil
+    timeouts = []
+
+    if max
+      if p99 < max
+        timeouts << p99
+
+        # fallback attempt
+        if max - p99 > p99
+          # use remaining time for second attempt
+          timeouts << max - p99
+        else
+          timeouts << max
+
+          NetworkResiliency.statsd&.increment(
+            "network_resiliency.timeout.raised",
+            tags: tags,
+          )
+        end
+      else
+        # the specified timeout is less than our expected p99...awkward
+        timeouts << max
+
+        NetworkResiliency.statsd&.increment(
+          "network_resiliency.timeout.too_low",
+          tags: tags,
+        )
+      end
+    else
+      timeouts << p99
+
+      # timeouts << p99 * 10 if NetworkResiliency.mode == :resolute
+
+      # unbounded second attempt
+      timeouts << nil
+
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.timeout.missing",
+        tags: tags,
+      )
+    end
+
+    timeouts
+  rescue => e
+    NetworkResiliency.statsd&.increment(
+      "network_resiliency.error",
+      tags: {
+        method: __method__,
+        type: e.class,
+      },
+    )
+
+    warn "[ERROR] NetworkResiliency: #{e.class}: #{e.message}"
+
+    default
+  end
+
   def reset
     @enabled = nil
+    @mode = nil
     Thread.current["network_resiliency"] = nil
     StatsEngine.reset
-    @sync_worker.kill if @sync_worker
+
+    if @sync_worker
+      @sync_worker.kill
+      @sync_worker = nil
+    end
   end
 
   private
@@ -176,13 +301,11 @@ module NetworkResiliency
     raise "Redis not configured" unless redis
 
     @sync_worker = Thread.new do
-      while true do
+      loop do
         StatsEngine.sync(redis)
 
         sleep(3)
       end
-    rescue Interrupt
-      # goodbye
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: network_resiliency
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Daniel Pepper
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-03 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug