network_resiliency 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9a1103993c635cbda0e6a3ba3ef523c8179680353fa67b93842f0bdfa44b197
-  data.tar.gz: aeb11a089950492bc6099933c137ccf79c27447a18ce6335e2a46c9982201591
+  metadata.gz: c372873056610bf21197e55a265bdfd03e200b064c727a4713ea8725ae3d696d
+  data.tar.gz: 5da4933543bf9a57d46ed4a3b4d94a692524e641f6c1293d8bcb99aa74c0c9d9
 SHA512:
-  metadata.gz: 7ea2cedcf4e6044299111e69c211a92a3d7df5305e66b4d6be83d32b949425daee581af33142f73f4d618ec178bd10e19ea0599175420838f57b0b61f157a8f5
-  data.tar.gz: 772d5749b23a32e6be7542c10d18794f1905fd64dde7da5b702bba4b4fb6ceb62fa3b971ce120c550db2fbc89ccec11670e3029ee0649ec7c21266597b0cb637
+  metadata.gz: b8e9b66cb83ff5bfda080c43935e8834228cfbd16afd0c1229be2b7edbea8469702d917ae7b6bb7383b774ebf68802e9a5572ee89357723d479d1dd8b3a91f2c
+  data.tar.gz: 68f241f0bd9b7ef5b0b291e2e1be11aa3d2c377f6b4b70552ba32638122bff300bc1252b279a13c41e96af3ffa8843bc571b74bd9aac7f1300f27472e18001ff

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+###  v0.4.0  (2023-11-13)
+- redis resiliency
+- timeouts_for
+- mode
+
+###  v0.3.2  (2023-11-03)
+- stats observability
+- surface errors in specs
+- destination filter
+- error observability
+- improve order of magnitude
+
 ###  v0.3.1  (2023-11-02)
 - sync thread safety
 - order of magnitude stats

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    network_resiliency (0.3.1)
+    network_resiliency (0.4.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/network_resiliency/adapter/redis.rb

@@ -36,35 +36,49 @@ module NetworkResiliency
       end
 
       module Instrumentation
-        # def initialize(...)
-        #   super
-
-        #   @network_resiliency_attempts = options[:reconnect_attempts]
-        #   options[:reconnect_attempts] = 0
-        # end
-
         def establish_connection
           return super unless NetworkResiliency.enabled?(:redis)
 
+          original_timeout = @options[:connect_timeout]
+
+          timeouts = NetworkResiliency.timeouts_for(
+            adapter: "redis",
+            action: "connect",
+            destination: host,
+            max: original_timeout,
+          )
+
+          attempts = 0
+          ts = -NetworkResiliency.timestamp
+
           begin
-            ts = -NetworkResiliency.timestamp
+            attempts += 1
+            error = nil
+
+            @options[:connect_timeout] = timeouts.shift
 
             super
           rescue ::Redis::CannotConnectError => e
             # capture error
+
+            # grab underlying exception within Redis wrapper
+            error = e.cause.class
+
+            retry if timeouts.size > 0
+
             raise
           ensure
             ts += NetworkResiliency.timestamp
-
-            # grab underlying exception within Redis wrapper
-            error = e ? e.cause.class : nil
+            @options[:connect_timeout] = original_timeout
 
             NetworkResiliency.record(
               adapter: "redis",
               action: "connect",
               destination: host,
-              error: error,
               duration: ts,
+              error: error,
+              timeout: @options[:connect_timeout],
+              attempts: attempts,
             )
           end
         end

data/lib/network_resiliency/refinements.rb

@@ -1,8 +1,20 @@
 module NetworkResiliency
   module Refinements
     refine Numeric do
-      def order_of_magnitude
-        self == 0 ? 0 : 10 ** Math.log10(self).ceil
+      def order_of_magnitude(ceil: false)
+        return 0 if self <= 0
+        return 1 if self <= 1
+
+        log10 = Math.log10(self.round)
+        10 ** (ceil ? log10.ceil : log10.floor)
+      end
+
+      def power_ceil
+        return 0 if self <= 0
+        return 1 if self <= 1
+
+        digits = Math.log10(self).floor
+        10 ** digits * (self.to_f / 10 ** digits).ceil
       end
     end
   end

data/lib/network_resiliency/version.rb

@@ -1,3 +1,3 @@
 module NetworkResiliency
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/network_resiliency.rb

@@ -14,6 +14,9 @@ module NetworkResiliency
     autoload :Postgres, "network_resiliency/adapter/postgres"
   end
 
+  MODE = [ :observe, :resilient ].freeze
+  RESILIENCY_SIZE_THRESHOLD = 1_000
+
   extend self
 
   attr_accessor :statsd, :redis
@@ -87,13 +90,22 @@ module NetworkResiliency
     Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1_000
   end
 
-  # private
+  def mode
+    @mode || :observe
+  end
 
-  IP_ADDRESS_REGEX = Regexp.new(/\d{1,3}(\.\d{1,3}){3}/)
+  def mode=(mode)
+    unless MODE.include?(mode)
+      raise ArgumentError, "invalid NetworkResiliency mode: #{mode}"
+    end
 
-  def record(adapter:, action:, destination:, duration:, error: nil)
-    # filter raw IP addresses
-    return if IP_ADDRESS_REGEX.match?(destination)
+    @mode = mode
+  end
+
+  # private
+
+  def record(adapter:, action:, destination:, duration:, error:, timeout: nil, attempts: 1)
+    return if ignore_destination?(adapter, action, destination)
 
     NetworkResiliency.statsd&.distribution(
       "network_resiliency.#{action}",
@@ -102,12 +114,13 @@ module NetworkResiliency
         adapter: adapter,
         destination: destination,
         error: error,
+        attempts: (attempts if attempts > 1),
       }.compact,
     )
 
     NetworkResiliency.statsd&.distribution(
       "network_resiliency.#{action}.magnitude",
-      duration.order_of_magnitude,
+      duration.order_of_magnitude(ceil: true),
       tags: {
         adapter: adapter,
         destination: destination,
@@ -115,24 +128,165 @@ module NetworkResiliency
       }.compact,
     )
 
+    NetworkResiliency.statsd&.gauge(
+      "network_resiliency.#{action}.timeout",
+      timeout,
+      tags: {
+        adapter: adapter,
+        destination: destination,
+      },
+    )
+
+    if error
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.time_saved",
+        timeout - duration,
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if timeout
+    else
+      # track successful retries
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.#{action}.resilient",
+        tags: {
+          adapter: adapter,
+          destination: destination,
+        },
+      ) if attempts > 1
+
+      # record stats
+      key = [ adapter, action, destination ].join(":")
+      stats = StatsEngine.add(key, duration)
+      tags = {
+        adapter: adapter,
+        destination: destination,
+        n: stats.n.order_of_magnitude,
+      }
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.n",
+        stats.n,
+        tags: tags,
+      )
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.avg",
+        stats.avg,
+        tags: tags,
+      )
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.stdev",
+        stats.stdev,
+        tags: tags,
+      )
+    end
+
+    nil
+  rescue => e
+    NetworkResiliency.statsd&.increment(
+      "network_resiliency.error",
+      tags: {
+        method: __method__,
+        type: e.class,
+      },
+    )
+
+    warn "[ERROR] NetworkResiliency: #{e.class}: #{e.message}"
+  end
+
+  IP_ADDRESS_REGEX = Regexp.new(/\d{1,3}(\.\d{1,3}){3}/)
+
+  def ignore_destination?(adapter, action, destination)
+    # filter raw IP addresses
+    IP_ADDRESS_REGEX.match?(destination)
+  end
+
+  def timeouts_for(adapter:, action:, destination:, max: nil)
+    default = [ max ]
+
+    return default if NetworkResiliency.mode == :observe
+
     key = [ adapter, action, destination ].join(":")
-    StatsEngine.add(key, duration)
+    stats = StatsEngine.get(key)
+
+    return default unless stats.n >= RESILIENCY_SIZE_THRESHOLD
+
+    tags = {
+      adapter: adapter,
+      action: action,
+      destination: destination,
+    }
+
+    p99 = (stats.avg + stats.stdev * 3).power_ceil
+    timeouts = []
+
+    if max
+      if p99 < max
+        timeouts << p99
+
+        # fallback attempt
+        if max - p99 > p99
+          # use remaining time for second attempt
+          timeouts << max - p99
+        else
+          timeouts << max
+
+          NetworkResiliency.statsd&.increment(
+            "network_resiliency.timeout.raised",
+            tags: tags,
+          )
+        end
+      else
+        # the specified timeout is less than our expected p99...awkward
+        timeouts << max
+
+        NetworkResiliency.statsd&.increment(
+          "network_resiliency.timeout.too_low",
+          tags: tags,
+        )
+      end
+    else
+      timeouts << p99
+
+      # timeouts << p99 * 10 if NetworkResiliency.mode == :resolute
+
+      # unbounded second attempt
+      timeouts << nil
+
+      NetworkResiliency.statsd&.increment(
+        "network_resiliency.timeout.missing",
+        tags: tags,
+      )
+    end
+
+    timeouts
   rescue => e
     NetworkResiliency.statsd&.increment(
       "network_resiliency.error",
       tags: {
+        method: __method__,
         type: e.class,
       },
     )
 
     warn "[ERROR] NetworkResiliency: #{e.class}: #{e.message}"
+
+    default
   end
 
   def reset
     @enabled = nil
+    @mode = nil
     Thread.current["network_resiliency"] = nil
     StatsEngine.reset
-    @sync_worker.kill if @sync_worker
+
+    if @sync_worker
+      @sync_worker.kill
+      @sync_worker = nil
+    end
   end
 
   private
@@ -147,13 +301,11 @@ module NetworkResiliency
     raise "Redis not configured" unless redis
 
     @sync_worker = Thread.new do
-      while true do
+      loop do
         StatsEngine.sync(redis)
 
         sleep(3)
       end
-    rescue Interrupt
-      # goodbye
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: network_resiliency
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Daniel Pepper
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-02 00:00:00.000000000 Z
+date: 2023-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug