network_resiliency 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb3d39476429afedf4ec8532e509dd25d0c121e21fa59f36b69bad40a0985dc6
-  data.tar.gz: 2ef67d94b01e34a758ae52117e1e9ac5eb9fa8901841d5bcb9ef7c1cc3011f6f
+  metadata.gz: 19102b63c5f8090d6805ac5ce1d71a25e30ab7483ef11bf5d9cee771bb4ce508
+  data.tar.gz: a9f37e7fafbfc1d45e0f8b24b655cf72b33300b5b6335e396d9dbffc22479799
 SHA512:
-  metadata.gz: b3c19d756696ab7d4d0a11e13d6d99c70a27bd2414774e71eb31147e08685e74da0a4f5b56d15260cbbc0faf9e954b1976db307f116a47208697e83d2ff3f7ef
-  data.tar.gz: 28a018c2ad35c15e01309da7a4f0368a67f4c21aacaba8f81fbc1fb2b41435fea368b98ec84274fca03e406416f9fa6eb403e0426adb046ba9a8ea1ae826bd83
+  metadata.gz: 27390e9651bef755a92a02218c286291971e7c77baabbddd062e3279d5a52f3a819fbbd9b0a9f91e289bd3124afdc028e42caf6b504c54bb198d959ba54dd1f6
+  data.tar.gz: 56763326a26679f03124a1a4f8b5dab2de10edeeebaff1a4dab8c821ee9b8a035b3b8cf285089bac944c722cc647bcb47264b7fe60295dfbf51d35c4977d2d1f

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+###  v0.3.2  (2023-11-03)
+- stats observability
+- surface errors in specs
+- destination filter
+- error observability
+- improve order of magnitude
+
+###  v0.3.1  (2023-11-02)
+- sync thread safety
+- order of magnitude stats
+- stats engine
+- improve stats aggregation
+- lua v1
+- thread safety
+- postgres adapter
+
 ###  v0.3.0  (2023-10-04)
 - mysql adapter
 - faraday version constraint

data/Gemfile.lock

@@ -1,18 +1,21 @@
 PATH
   remote: .
   specs:
-    network_resiliency (0.3.0)
+    network_resiliency (0.3.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
     byebug (11.1.3)
-    ddtrace (1.14.0)
-      debase-ruby_core_source (= 3.2.1)
-      libdatadog (~> 3.0.0.1.0)
-      libddwaf (~> 1.9.0.0.0)
+    datadog-ci (0.2.0)
       msgpack
-    debase-ruby_core_source (3.2.1)
+    ddtrace (1.15.0)
+      datadog-ci (~> 0.2.0)
+      debase-ruby_core_source (= 3.2.2)
+      libdatadog (~> 5.0.0.1.0)
+      libddwaf (~> 1.14.0.0.0)
+      msgpack
+    debase-ruby_core_source (3.2.2)
     diff-lcs (1.5.0)
     docile (1.4.0)
     dogstatsd-ruby (4.8.3)
@@ -39,13 +42,14 @@ GEM
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
     faraday-retry (1.0.3)
-    ffi (1.15.5)
-    libdatadog (3.0.0.1.0)
-    libddwaf (1.9.0.0.1)
+    ffi (1.16.3)
+    libdatadog (5.0.0.1.0)
+    libddwaf (1.14.0.0.0)
       ffi (~> 1.0)
     msgpack (1.7.2)
     multipart-post (2.3.0)
     mysql2 (0.5.5)
+    pg (1.5.4)
     rack (3.0.8)
     rack-test (2.1.0)
       rack (>= 1.3)
@@ -82,6 +86,7 @@ DEPENDENCIES
   faraday-rack
   mysql2 (>= 0.5)
   network_resiliency!
+  pg (~> 1.1)
   rack
   rack-test
   redis (~> 4)
@@ -89,4 +94,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.4.19
+   2.4.20

data/README.md

@@ -3,13 +3,22 @@ NetworkResiliency
 ![Gem](https://img.shields.io/gem/dt/network_resiliency?style=plastic)
 [![codecov](https://codecov.io/gh/dpep/network_resiliency_rb/branch/main/graph/badge.svg)](https://codecov.io/gh/dpep/network_resiliency_rb)
 
-Making networks more resilient to errors.
-
-Resiliency: the ability to recover from adversity or adjust to change.
+Making network requests more resilient to error.
+- less errors, by retrying
+- less time, by setting granular timeouts
 
 
 ```ruby
 require "network_resiliency"
+
+NetworkResiliency.configure do |conf|
+  conf.statsd = Datadog::Statsd.new
+
+  # patch Redis instances
+  conf.patch :redis
+end
+
+Redis.new.connect
 ```
 
 

data/lib/network_resiliency/adapter/postgres.rb

@@ -0,0 +1,47 @@
+gem "pg", "~> 1.1"
+require "pg"
+
+module NetworkResiliency
+  module Adapter
+    module Postgres
+      extend self
+
+      def patch
+        return if patched?
+
+        PG::Connection.singleton_class.prepend(Instrumentation)
+      end
+
+      def patched?
+        PG::Connection.singleton_class.ancestors.include?(Instrumentation)
+      end
+
+      module Instrumentation
+        def connect_start(opts)
+          host = opts[:host].split(",")[0] if opts[:host]
+
+          return super unless NetworkResiliency.enabled?(:postgres)
+
+          begin
+            ts = -NetworkResiliency.timestamp
+
+            super
+          rescue PG::Error => e
+            # capture error
+            raise
+          ensure
+            ts += NetworkResiliency.timestamp
+
+            NetworkResiliency.record(
+              adapter: "postgres",
+              action: "connect",
+              destination: host,
+              error: e&.class,
+              duration: ts,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/network_resiliency/refinements.rb

@@ -0,0 +1,13 @@
+module NetworkResiliency
+  module Refinements
+    refine Numeric do
+      def order_of_magnitude(ceil: false)
+        return 0 if self <= 0
+        return 1 if self <= 1
+
+        log10 = Math.log10(self.round)
+        10 ** (ceil ? log10.ceil : log10.floor)
+      end
+    end
+  end
+end

data/lib/network_resiliency/stats.rb

@@ -2,20 +2,33 @@ module NetworkResiliency
   class Stats
     attr_reader :n, :avg
 
-    def self.from(n, avg, sq_dist)
-      new.tap do |instance|
-        instance.instance_eval do
-          @n = n
-          @avg = avg
-          @sq_dist = sq_dist
+    class << self
+      def from(n:, avg:, sq_dist:)
+        new.tap do |instance|
+          instance.instance_eval do
+            @n = n.to_i
+            @avg = avg.to_f
+            @sq_dist = sq_dist.to_f
+          end
         end
       end
+
+      private
+
+      def synchronize(fn_name)
+        make_private = private_method_defined?(fn_name)
+        fn = instance_method(fn_name)
+
+        define_method(fn_name) do |*args|
+          @lock.synchronize { fn.bind(self).call(*args) }
+        end
+        private fn_name if make_private
+      end
     end
 
     def initialize(values = [])
-      @n = 0
-      @avg = 0.0
-      @sq_dist = 0.0 # sum of squared distance from mean
+      @lock = Thread::Mutex.new
+      reset
 
       values.each {|x| update(x) }
     end
@@ -34,7 +47,7 @@ module NetworkResiliency
     end
 
     def variance(sample: false)
-      @sq_dist / (sample ? (@n - 1) : @n)
+      @n == 0 ? 0 : @sq_dist / (sample ? (@n - 1) : @n)
     end
 
     def stdev
@@ -46,24 +59,161 @@ module NetworkResiliency
     end
     alias_method :+, :merge
 
-    def merge!(other)
+    synchronize def merge!(other)
       raise ArgumentError unless other.is_a?(self.class)
 
-      prev_n = n
-      @n += other.n
+      if @n == 0
+        @n = other.n
+        @avg = other.avg
+        @sq_dist = other.sq_dist
+      elsif other.n > 0
+        prev_n = @n
+        @n += other.n
 
-      delta = other.avg - avg
-      @avg += delta * other.n / n
+        delta = other.avg - avg
+        @avg += delta * other.n / @n
 
-      @sq_dist += other.instance_variable_get(:@sq_dist)
-      @sq_dist += (delta ** 2) * prev_n * other.n / n
+        @sq_dist += other.sq_dist
+        @sq_dist += (delta ** 2) * prev_n * other.n / @n
+      end
 
       self
     end
 
+    def ==(other)
+      return false unless other.is_a?(self.class)
+
+      @n == other.n &&
+        @avg == other.avg &&
+        @sq_dist == other.sq_dist
+    end
+
+    synchronize def reset
+      @n = 0
+      @avg = 0.0
+      @sq_dist = 0.0 # sum of squared distance from mean
+    end
+
+    MIN_SAMPLE_SIZE = 1000
+    MAX_WINDOW_LENGTH = 1000
+    STATS_TTL = 24 * 60 * 60 # 1 day
+    CACHE_TTL = 60 # seconds
+
+    LUA_SCRIPT = <<~LUA
+      local results = {}
+
+      for i = 0, #KEYS / 2 - 1 do
+        local state_key = KEYS[i * 2 + 1]
+        local cache_key = KEYS[i * 2 + 2]
+
+        local n = tonumber(ARGV[i * 3 + 1])
+        local avg = ARGV[i * 3 + 2]
+        local sq_dist = math.floor(ARGV[i * 3 + 3])
+
+        if n > 0 then
+          -- save new data
+          local window_len = redis.call(
+            'LPUSH',
+            state_key,
+            string.format('%d|%f|%d', n, avg, sq_dist)
+          )
+          redis.call('EXPIRE', state_key, #{STATS_TTL})
+
+          if window_len > #{MAX_WINDOW_LENGTH} then
+            -- trim stats to window length
+            redis.call('LTRIM', state_key, 0, #{MAX_WINDOW_LENGTH - 1})
+          end
+        end
+
+        -- retrieve aggregated stats
+
+        local cached_stats = redis.call('GET', cache_key)
+        if cached_stats then
+          -- use cached stats
+          n, avg, sq_dist = string.match(cached_stats, "(%d+)|([%d.]+)|(%d+)")
+          n = tonumber(n)
+        else
+          -- calculate aggregated stats
+          n = 0
+          avg = 0.0
+          sq_dist = 0
+
+          local stats = redis.call('LRANGE', state_key, 0, -1)
+          for _, entry in ipairs(stats) do
+            local other_n, other_avg, other_sq_dist = string.match(entry, "(%d+)|([%d.]+)|(%d+)")
+            other_n = tonumber(other_n)
+            other_avg = tonumber(other_avg) + 0.0
+            other_sq_dist = tonumber(other_sq_dist)
+
+            local prev_n = n
+            n = n + other_n
+
+            local delta = other_avg - avg
+            avg = avg + delta * other_n / n
+
+            sq_dist = sq_dist + other_sq_dist
+            sq_dist = sq_dist + (delta ^ 2) * prev_n * other_n / n
+          end
+        end
+
+        -- update cache
+        if n >= #{MIN_SAMPLE_SIZE} then
+          cached_stats = string.format('%d|%f|%d', n, avg, sq_dist)
+          redis.call('SET', cache_key, cached_stats, 'EX', #{CACHE_TTL})
+        end
+
+        -- accumulate results
+        table.insert(results, n)
+        table.insert(results, tostring(avg))
+        table.insert(results, sq_dist)
+      end
+
+      return results
+    LUA
+
+    def sync(redis, key)
+      self.class.sync(redis, key => self)[key]
+    end
+
+    def self.sync(redis, **data)
+      keys = []
+      args = []
+
+      data.each do |key, stats|
+        keys += [
+          "network_resiliency:stats:#{key}",
+          "network_resiliency:stats:cache:#{key}",
+        ]
+
+        args += [ stats.n, stats.avg, stats.send(:sq_dist) ]
+      end
+
+      res = redis.eval(LUA_SCRIPT, keys, args)
+      data.keys.zip(res.each_slice(3)).map do |key, stats|
+        n, avg, sq_dist = *stats
+
+        [ key, Stats.from(n: n, avg: avg, sq_dist: sq_dist) ]
+      end.to_h
+    end
+
+    def self.fetch(redis, keys)
+      data = Array(keys).map { |k| [ k, new ] }.to_h
+      res = sync(redis, **data)
+
+      keys.is_a?(Array) ? res : res[keys]
+    end
+
+    def to_s
+      "#<#{self.class.name}:#{object_id} n=#{n} avg=#{avg} sq_dist=#{sq_dist}>"
+    end
+
+    protected
+
+    attr_reader :sq_dist
+
     private
 
-    def update(value)
+    synchronize def update(value)
       raise ArgumentError unless value.is_a?(Numeric)
 
       @n += 1
@@ -72,19 +222,6 @@ module NetworkResiliency
       @avg += (value - @avg) / @n
 
       @sq_dist += (value - prev_avg) * (value - @avg)
-      # @sq_dist += (sq_dist - @sq_dist) / @n
-
-# for x, w in data_weight_pairs:
-#   w_sum = w_sum + w
-#   mean_old = mean
-#   mean = mean_old + (w / w_sum) * (x - mean_old)
-#   S = S + w * (x - mean_old) * (x - mean)
-
-  # count += 1
-  # delta = newValue - mean
-  # mean += delta / count
-  # delta2 = newValue - mean
-  # M2 += delta * delta2
     end
   end
 end

data/lib/network_resiliency/stats_engine.rb

@@ -0,0 +1,112 @@
+module NetworkResiliency
+  module StatsEngine
+    extend self
+
+    LOCK = Thread::Mutex.new
+    STATS = {}
+    SYNC_LIMIT = 100
+
+    def add(key, value)
+      local, _ = synchronize do
+        STATS[key] ||= [ Stats.new, Stats.new ]
+      end
+
+      local << value
+    end
+
+    def get(key)
+      local, remote = synchronize do
+        STATS[key] ||= [ Stats.new, Stats.new ]
+      end
+
+      local + remote
+    end
+
+    def reset
+      synchronize { STATS.clear }
+    end
+
+    def sync(redis)
+      dirty_keys = {}
+
+      # select data to be synced
+      data = synchronize do
+        # ensure sync is not run concurrently
+        return [] if @syncing
+        @syncing = Thread.current
+
+        dirty_keys = STATS.map do |key, (local, remote)|
+          # skip if no new local stats and remote already synced
+          next if local.n == 0 && remote.n > 0
+
+          [ key, local.n ]
+        end.compact.to_h
+
+        # select keys to sync, prioritizing most used
+        keys = dirty_keys.sort_by do |key, weight|
+          -weight
+        end.take(SYNC_LIMIT).map(&:first)
+
+        # update stats for keys being synced
+        keys.map do |key|
+          local, remote = STATS[key]
+
+          remote << local # update remote stats until sync completes
+          STATS[key][0] = Stats.new # reset local stats
+
+          [ key, local ]
+        end.to_h
+      end
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.sync.keys",
+        data.size,
+        tags: {
+          empty: data.empty?,
+          truncated: data.size < dirty_keys.size,
+        }.select { |_, v| v },
+      )
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.sync.keys.dirty",
+        dirty_keys.select { |_, n| n > 0 }.count,
+      )
+
+      return [] if data.empty?
+
+      # sync data to redis
+      remote_stats = if NetworkResiliency.statsd
+        NetworkResiliency.statsd&.time("network_resiliency.sync") do
+          Stats.sync(redis, **data)
+        end
+      else
+        Stats.sync(redis, **data)
+      end
+
+      # integrate new remote stats
+      synchronize do
+        remote_stats.each do |key, stats|
+          local, remote = STATS[key]
+
+          remote.reset
+          remote << stats
+        end
+      end
+
+      remote_stats.keys
+    ensure
+      # release sync lock
+      @syncing = nil if @syncing == Thread.current
+    end
+
+    def syncing?
+      !!@syncing
+    end
+
+    private
+
+    def synchronize
+      LOCK.synchronize { yield }
+    end
+  end
+end

data/lib/network_resiliency/version.rb

@@ -1,3 +1,3 @@
 module NetworkResiliency
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

data/lib/network_resiliency.rb

@@ -1,19 +1,27 @@
+require "network_resiliency/refinements"
+require "network_resiliency/stats"
+require "network_resiliency/stats_engine"
 require "network_resiliency/version"
 
+using NetworkResiliency::Refinements
+
 module NetworkResiliency
   module Adapter
     autoload :HTTP, "network_resiliency/adapter/http"
     autoload :Faraday, "network_resiliency/adapter/faraday"
     autoload :Redis, "network_resiliency/adapter/redis"
     autoload :Mysql, "network_resiliency/adapter/mysql"
+    autoload :Postgres, "network_resiliency/adapter/postgres"
   end
 
   extend self
 
-  attr_accessor :statsd
+  attr_accessor :statsd, :redis
 
   def configure
-    yield self
+    yield self if block_given?
+
+    start_syncing if redis
   end
 
   def patch(*adapters)
@@ -25,6 +33,8 @@ module NetworkResiliency
         Adapter::Redis.patch
       when :mysql
         Adapter::Mysql.patch
+      when :postgres
+        Adapter::Postgres.patch
       else
         raise NotImplementedError
       end
@@ -32,6 +42,7 @@ module NetworkResiliency
   end
 
   def enabled?(adapter)
+    return thread_state["enabled"] if thread_state.key?("enabled")
     return true if @enabled.nil?
 
     if @enabled.is_a?(Proc)
@@ -55,20 +66,20 @@ module NetworkResiliency
 
   def enable!
     original = @enabled
-    @enabled = true
+    thread_state["enabled"] = true
 
     yield if block_given?
   ensure
-    @enabled = original if block_given?
+    thread_state.delete("enabled") if block_given?
   end
 
   def disable!
     original = @enabled
-    @enabled = false
+    thread_state["enabled"] = false
 
     yield if block_given?
   ensure
-    @enabled = original if block_given?
+    thread_state.delete("enabled") if block_given?
   end
 
   def timestamp
@@ -78,11 +89,8 @@ module NetworkResiliency
 
   # private
 
-  IP_ADDRESS_REGEX = Regexp.new(/\d{1,3}(\.\d{1,3}){3}/)
-
   def record(adapter:, action:, destination:, duration:, error: nil)
-    # filter raw IP addresses
-    return if IP_ADDRESS_REGEX.match?(destination)
+    return if ignore_destination?(adapter, action, destination)
 
     NetworkResiliency.statsd&.distribution(
       "network_resiliency.#{action}",
@@ -93,9 +101,88 @@ module NetworkResiliency
         error: error,
       }.compact,
     )
+
+    NetworkResiliency.statsd&.distribution(
+      "network_resiliency.#{action}.magnitude",
+      duration.order_of_magnitude(ceil: true),
+      tags: {
+        adapter: adapter,
+        destination: destination,
+        error: error,
+      }.compact,
+    )
+
+    key = [ adapter, action, destination ].join(":")
+    StatsEngine.add(key, duration).tap do |stats|
+      tags = {
+        adapter: adapter,
+        destination: destination,
+        n: stats.n.order_of_magnitude,
+      }
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.n",
+        stats.n,
+        tags: tags,
+      )
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.avg",
+        stats.avg,
+        tags: tags,
+      )
+
+      NetworkResiliency.statsd&.distribution(
+        "network_resiliency.#{action}.stats.stdev",
+        stats.stdev,
+        tags: tags,
+      )
+    end
+  rescue => e
+    NetworkResiliency.statsd&.increment(
+      "network_resiliency.error",
+      tags: {
+        method: __method__,
+        type: e.class,
+      },
+    )
+
+    warn "[ERROR] NetworkResiliency: #{e.class}: #{e.message}"
+  end
+
+  IP_ADDRESS_REGEX = Regexp.new(/\d{1,3}(\.\d{1,3}){3}/)
+
+  def ignore_destination?(adapter, action, destination)
+    # filter raw IP addresses
+    IP_ADDRESS_REGEX.match?(destination)
   end
 
   def reset
     @enabled = nil
+    Thread.current["network_resiliency"] = nil
+    StatsEngine.reset
+    @sync_worker.kill if @sync_worker
+  end
+
+  private
+
+  def thread_state
+    Thread.current["network_resiliency"] ||= {}
+  end
+
+  def start_syncing
+    @sync_worker.kill if @sync_worker
+
+    raise "Redis not configured" unless redis
+
+    @sync_worker = Thread.new do
+      while true do
+        StatsEngine.sync(redis)
+
+        sleep(3)
+      end
+    rescue Interrupt
+      # goodbye
+    end
   end
 end

data/network_resiliency.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "faraday", "~> 1"
   s.add_development_dependency "faraday-rack"
   s.add_development_dependency "mysql2", ">= 0.5"
+  s.add_development_dependency "pg", "~> 1.1"
   s.add_development_dependency "rack"
   s.add_development_dependency "rack-test"
   s.add_development_dependency "redis", "~> 4"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: network_resiliency
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Daniel Pepper
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-04 00:00:00.000000000 Z
+date: 2023-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -179,8 +193,11 @@ files:
 - lib/network_resiliency/adapter/faraday.rb
 - lib/network_resiliency/adapter/http.rb
 - lib/network_resiliency/adapter/mysql.rb
+- lib/network_resiliency/adapter/postgres.rb
 - lib/network_resiliency/adapter/redis.rb
+- lib/network_resiliency/refinements.rb
 - lib/network_resiliency/stats.rb
+- lib/network_resiliency/stats_engine.rb
 - lib/network_resiliency/version.rb
 - network_resiliency.gemspec
 homepage: https://github.com/dpep/network_resiliency_rb
@@ -202,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: NetworkResiliency