network_resiliency 0.7.11 → 0.7.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1eec5786e7daaea4b6f3c285dffc1e4bf1c70d7096ce69d44f4cc4ea80111ea5
-  data.tar.gz: bc3ca5e592f1a809bfb28453e43d49a64405a424861aef2c6c6745ef2f27ebb4
+  metadata.gz: c006caf6d6fefeca3185a0a6a1d75be6bcf1cbc16ed9e1b00f2b147114b8442f
+  data.tar.gz: cad33936cc88b7d586b9b1ca92ab36308e37af11768509e67fb6f20e2af354ba
 SHA512:
-  metadata.gz: fa1af69b1f175fb708247ddd0e9501ae2b945909197b26c343347e113bb78e3e645c5217c5f74d63fda780a027b730746b8641b9f77cb5c31626bc2f985c60ff
-  data.tar.gz: e9733ddd7978b321f8c49218de503f42b300aa269757559cac442dcefb43e0b136e1507360629e9aad1976cedd31c7c0ccd4e9a256c41962a990e50f046d3979
+  metadata.gz: e4eaf6bf8b4a4b176a8bd2ae3b79a80fc1a9b01c0d0a6dc486e870d656eb3793c1626dd923c2afd1c2b06774012067d77815a1710e5de44f06d79f098dd8db2b
+  data.tar.gz: 36d9406a01c76ca09d0859194401bbd49a177f644c1ca38d5df3c1fb8ad27907304d8d17ad2c165ced7dcfc67b467ee4e2061ce63e80a884fc8a5a372e994f2b

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+###  v0.7.13  (2024-03-04)
+- downsampling
+- redis resiliency
+
+###  v0.7.12  (2024-02-23)
+- improve metric sampling
+- lower resiliency threshold
+- power buckets
+
 ###  v0.7.11  (2024-02-20)
 - lower dynamic timeout
 - fix metric specs

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    network_resiliency (0.7.11)
+    network_resiliency (0.7.13)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -39,3 +39,8 @@ Yes please  :)
 https://github.com/lostisland/faraday-retry/blob/main/lib/faraday/retry/middleware.rb
 
 https://github.com/ankane/the-ultimate-guide-to-ruby-timeouts
+
+
+https://reprep.io/writings/20220326_timeouts_deadline_propagation.html
+
+https://grpc.io/blog/deadlines

data/lib/network_resiliency/power_stats.rb

@@ -0,0 +1,132 @@
+require "network_resiliency/refinements"
+
+using NetworkResiliency::Refinements
+
+module NetworkResiliency
+  class PowerStats
+    MIN_VALUE = 1
+    LOCK = Thread::Mutex.new
+    STATS = {}
+
+    attr_reader :n
+
+    class << self
+      def [](key)
+        LOCK.synchronize { STATS[key] ||= new }
+      end
+
+      def reset
+        LOCK.synchronize { STATS.clear }
+      end
+
+      private
+
+      def synchronize(fn_name)
+        fn = instance_method(fn_name)
+
+        define_method(fn_name) do |*args|
+          @lock.synchronize { fn.bind(self).call(*args) }
+        end
+      end
+    end
+
+    def initialize(values = [])
+      @lock = Thread::Mutex.new
+      reset
+
+      values.each {|x| add(x) }
+    end
+
+    def <<(value)
+      case value
+      when Array
+        value.each {|x| add(x) }
+      when self.class
+        merge!(value)
+      else
+        add(value)
+      end
+
+      self
+    end
+
+    synchronize def add(value)
+      raise ArgumentError, "Numeric expected, found #{value.class}" unless value.is_a?(Numeric)
+
+      value = [ value, MIN_VALUE ].max
+      i = Math.log10(value).ceil
+
+      @buckets[i] ||= 0
+      @buckets[i] += 1
+      @n += 1
+    end
+
+    synchronize def percentile(p)
+      raise ArgumentError, "Percentile must be between 0 and 100" unless p.between?(0, 100)
+
+      return 0 if @n == 0
+
+      threshold = ((100 - p) / 100.0 * @n).floor
+      index = @buckets.size - 1
+
+      while index >= 0
+        if @buckets[index]
+          break if @buckets[index] >= threshold
+
+          threshold -= @buckets[index]
+        end
+
+        index -= 1
+      end
+
+      10 ** index
+    end
+    alias_method :p, :percentile
+
+    def p99
+      percentile(99)
+    end
+
+    def merge(other)
+      dup.merge!(other)
+    end
+    alias_method :+, :merge
+
+    synchronize def merge!(other)
+      raise ArgumentError unless other.is_a?(self.class)
+
+      other_buckets = other.instance_variable_get(:@buckets)
+
+      if @n == 0
+        @n = other.n
+        @buckets = other_buckets.dup
+      elsif other.n > 0
+        @n += other.n
+
+        other_buckets.each_with_index do |count, i|
+          next unless count
+
+          @buckets[i] ||= 0
+          @buckets[i] += count
+        end
+      end
+
+      self
+    end
+
+    synchronize def scale!(percentage)
+      raise ArgumentError, "Numeric expected, found #{percentage.class}" unless percentage.is_a?(Numeric)
+      raise ArgumentError, "argument must be between 0 and 100" unless percentage.between?(0, 100)
+
+      factor = percentage / 100.0
+
+      @buckets.map! {|x| (x * factor).round if x }
+      @n = @buckets.compact.sum
+    end
+
+    synchronize def reset
+      @n = 0
+      @buckets = []
+    end
+  end
+end

data/lib/network_resiliency/stats.rb

@@ -80,6 +80,16 @@ module NetworkResiliency
       self
     end
 
+    synchronize def scale!(percentage)
+      raise ArgumentError, "Numeric expected, found #{percentage.class}" unless percentage.is_a?(Numeric)
+      raise ArgumentError, "argument must be between 0 and 100" unless percentage.between?(0, 100)
+
+      factor = percentage / 100.0
+
+      @sq_dist *= factor
+      @n = (@n * factor).round
+    end
+
     def ==(other)
       return false unless other.is_a?(self.class)
 
@@ -94,7 +104,7 @@ module NetworkResiliency
       @sq_dist = 0.0 # sum of squared distance from mean
     end
 
-    MIN_SAMPLE_SIZE = 1000
+    MIN_SAMPLE_SIZE = 300
     MAX_WINDOW_LENGTH = 1000
     STATS_TTL = 24 * 60 * 60 # 1 day
     CACHE_TTL = 120 # seconds

data/lib/network_resiliency/syncer.rb

@@ -50,7 +50,9 @@ module NetworkResiliency
       NetworkResiliency.redis.disconnect! if NetworkResiliency.redis.connected?
 
       until @shutdown
-        StatsEngine.sync(NetworkResiliency.redis)
+        NetworkResiliency.redis.with_reconnect do
+          StatsEngine.sync(NetworkResiliency.redis)
+        end
 
         sleep(SLEEP_DURATION)
       end

data/lib/network_resiliency/version.rb

@@ -1,3 +1,3 @@
 module NetworkResiliency
-  VERSION = "0.7.11"
+  VERSION = "0.7.13"
 end

data/lib/network_resiliency.rb

@@ -1,3 +1,4 @@
+require "network_resiliency/power_stats"
 require "network_resiliency/refinements"
 require "network_resiliency/stats"
 require "network_resiliency/stats_engine"
@@ -19,7 +20,7 @@ module NetworkResiliency
   ACTIONS = [ :connect, :request ].freeze
   ADAPTERS = [ :http, :faraday, :redis, :mysql, :postgres, :rails ].freeze
   MODE = [ :observe, :resilient ].freeze
-  RESILIENCY_SIZE_THRESHOLD = 1_000
+  RESILIENCY_SIZE_THRESHOLD = 300
   SAMPLE_RATE = {
     timeout: 0.1,
     stats: 0.1,
@@ -256,6 +257,12 @@ module NetworkResiliency
       # record stats
       key = [ adapter, action, destination ].join(":")
       stats = StatsEngine.add(key, duration)
+
+      if stats.n > RESILIENCY_SIZE_THRESHOLD * 4
+        # downsample to age out old stats
+        stats.scale!(50)
+      end
+
       tags = {
         adapter: adapter,
         destination: destination,
@@ -266,26 +273,28 @@ module NetworkResiliency
       # ensure Syncer is running
       Syncer.start
 
-      NetworkResiliency.statsd&.distribution(
-        "network_resiliency.#{action}.stats.n",
-        stats.n,
-        tags: tags,
-        sample_rate: SAMPLE_RATE[:stats],
-      )
+      if rand < SAMPLE_RATE[:stats]
+        NetworkResiliency.statsd&.distribution(
+          "network_resiliency.#{action}.stats.n",
+          stats.n,
+          tags: tags,
+          sample_rate: SAMPLE_RATE[:stats],
+        )
 
-      NetworkResiliency.statsd&.distribution(
-        "network_resiliency.#{action}.stats.avg",
-        stats.avg,
-        tags: tags,
-        sample_rate: SAMPLE_RATE[:stats],
-      )
+        NetworkResiliency.statsd&.distribution(
+          "network_resiliency.#{action}.stats.avg",
+          stats.avg,
+          tags: tags,
+          sample_rate: SAMPLE_RATE[:stats],
+        )
 
-      NetworkResiliency.statsd&.distribution(
-        "network_resiliency.#{action}.stats.stdev",
-        stats.stdev,
-        tags: tags,
-        sample_rate: SAMPLE_RATE[:stats],
-      )
+        NetworkResiliency.statsd&.distribution(
+          "network_resiliency.#{action}.stats.stdev",
+          stats.stdev,
+          tags: tags,
+          sample_rate: SAMPLE_RATE[:stats],
+        )
+      end
     end
 
     nil
@@ -337,7 +346,7 @@ module NetworkResiliency
             "network_resiliency.timeout.raised",
             tags: tags,
             sample_rate: SAMPLE_RATE[:timeout],
-          )
+          ) if rand < SAMPLE_RATE[:timeout]
         end
       else
         # the specified timeout is less than our expected p99...awkward
@@ -347,7 +356,7 @@ module NetworkResiliency
           "network_resiliency.timeout.too_low",
           tags: tags,
           sample_rate: SAMPLE_RATE[:timeout],
-        )
+        ) if rand < SAMPLE_RATE[:timeout]
       end
     else
       timeouts << p99
@@ -361,7 +370,7 @@ module NetworkResiliency
         "network_resiliency.timeout.missing",
         tags: tags,
         sample_rate: SAMPLE_RATE[:timeout],
-      )
+      ) if rand < SAMPLE_RATE[:timeout]
     end
 
     NetworkResiliency.statsd&.distribution(
@@ -372,7 +381,7 @@ module NetworkResiliency
         destination: destination,
       },
       sample_rate: SAMPLE_RATE[:timeout],
-    )
+    ) if rand < SAMPLE_RATE[:timeout]
 
     case units
     when nil, :ms, :milliseconds

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: network_resiliency
 version: !ruby/object:Gem::Version
-  version: 0.7.11
+  version: 0.7.13
 platform: ruby
 authors:
 - Daniel Pepper
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-20 00:00:00.000000000 Z
+date: 2024-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -238,6 +238,7 @@ files:
 - lib/network_resiliency/adapter/postgres.rb
 - lib/network_resiliency/adapter/rails.rb
 - lib/network_resiliency/adapter/redis.rb
+- lib/network_resiliency/power_stats.rb
 - lib/network_resiliency/refinements.rb
 - lib/network_resiliency/stats.rb
 - lib/network_resiliency/stats_engine.rb