sidekiq-amigo 1.12.1 → 1.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab4f66f8d1d73bf0c9d9569ba86b4041d7642f86b8d10c246aa2eb155b3b6376
-  data.tar.gz: 6f5726a5708706dcbff900c5dcfde0656ab797a67a698608249abb96b2e1d93f
+  metadata.gz: 70bb0febcaf8c1a36e19fc4dd56fb766be4fa046c941b1f2aa66b7dcf5372872
+  data.tar.gz: 620b2c3a6bf664931eb8a6e84cc7d8762597f4aae91aa17765bc1d1fb0b432ad
 SHA512:
-  metadata.gz: 9ef0b460702cafafd2c25119434ef11fda3dc958de6ad575c2a3faae18ba5bd97de64302aa8facf27a97a87476be2ca3947faa9f5f65f68df0c078610e1d6239
-  data.tar.gz: cf55e3a4308cb10cd81c4f4831cc1ab967ebd2614adaf62889c192c1b439cbea694928a57b525beb4fafe992fb5a5242ab6cecad2e79ee7888341f9c9c104057
+  metadata.gz: 20bcdab3b140069cd0eeb6a559032e8e4d986ffdc6bdd7ed03030a45b5b83a50600bb7d465736a0a7dcdd2e369a2a054bb2f968a77111ec5cef807876d19d8f0
+  data.tar.gz: ddbb6ac910db1687c6e5f471f965d2b5f2dc78b650564f28c24153e8ce1d1d7b350f03da4b0571673084865ac8c9c5f9a27453e090ad94f6b12f9507649dde6d

data/lib/amigo/autoscaler/checkers/chain.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "amigo/autoscaler"
+
+module Amigo
+  class Autoscaler
+    module Checkers
+      class Chain < Amigo::Autoscaler::Checker
+        attr_accessor :chain
+
+        # Chain multiple checkers together.
+        # Latencies are merged, with the highest latency winning.
+        # Pool usage has the highest take precedence.
+        # @param chain [Array<Amigo::Autoscaler::Checker>]
+        def initialize(chain)
+          @chain = chain
+          super()
+        end
+
+        def get_latencies
+          h = {}
+          @chain.each do |c|
+            c.get_latencies.each do |k, v|
+              h[k] = [h[k], v].compact.max
+            end
+          end
+          return h
+        end
+
+        def get_pool_usage
+          return @chain.map(&:get_pool_usage).compact.max
+        end
+      end
+    end
+  end
+end

data/lib/amigo/autoscaler/checkers/fake.rb

@@ -6,8 +6,9 @@ module Amigo
   class Autoscaler
     module Checkers
       class Fake < Amigo::Autoscaler::Checker
-        def initialize(latencies)
+        def initialize(latencies: {}, pool_usage: nil)
           @latencies = latencies
+          @pool_usage = pool_usage
           super()
         end
 
@@ -16,6 +17,12 @@ module Amigo
           return @latencies.shift if @latencies.is_a?(Array)
           return @latencies
         end
+
+        def get_pool_usage
+          return @pool_usage.call if @pool_usage.respond_to?(:call)
+          return @pool_usage.shift if @pool_usage.is_a?(Array)
+          return @pool_usage
+        end
       end
     end
   end

data/lib/amigo/autoscaler/checkers/puma_pool_usage.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "puma/dsl"
+require "amigo/autoscaler"
+
+module Amigo
+  class Autoscaler
+    module Checkers
+      class PumaPoolUsage < Amigo::Autoscaler::Checker
+        NAMESPACE = "amigo/autoscaler/puma_pool_usage"
+
+        # The minimum number of usage readings before we report pool usage, to avoid spikes.
+        MIN_READINGS = 2
+
+        # How long to track the pool usage.
+        WINDOW = 60
+
+        def initialize(redis:, namespace: NAMESPACE, uid: SecureRandom.base64(4).delete_suffix("="))
+          @redis = redis
+          @key = "#{namespace}/v1"
+          @uid = uid
+          super()
+        end
+
+        # Set the pool usage, and trim old metrics.
+        def record(value, now:)
+          ts = now.to_f
+          member = "#{value}:#{@uid}:#{now.to_i}"
+          @redis.pipelined do |pipeline|
+            pipeline.call("ZADD", @key, ts, member)
+            pipeline.call("ZREMRANGEBYSCORE", @key, 0, ts - WINDOW)
+          end
+        end
+
+        def get_latencies = {}
+
+        def get_pool_usage
+          now = Time.now.to_f
+          members = @redis.call("ZRANGE", @key, now - WINDOW, now, "BYSCORE")
+          return nil if members.size < MIN_READINGS
+          values = members.map { |m| m.split(":", 2).first }
+          total_usage = values.sum(0, &:to_f)
+          return total_usage / values.size
+        end
+      end
+    end
+  end
+end
+
+module Puma
+  class DSL
+    def amigo_autoscaler_interval(interval)
+      @options[:amigo_autoscaler_interval] = interval
+    end
+
+    def amigo_puma_pool_usage_checker(ch)
+      @options[:amigo_puma_pool_usage_checker] = ch
+    end
+  end
+end

data/lib/amigo/autoscaler/checkers/sidekiq.rb

@@ -13,6 +13,18 @@ module Amigo
               map { |q| [q.name, q.latency] }.
               to_h
         end
+
+        def get_pool_usage
+          ps = ::Sidekiq::ProcessSet.new
+          total_concurrency = 0
+          total_busy = 0
+          ps.each do |process|
+            total_concurrency += process["concurrency"] || 0
+            total_busy += process["busy"] || 0
+          end
+          return 0.0 if total_concurrency.zero?
+          return total_busy.to_f / total_concurrency
+        end
       end
     end
   end

data/lib/amigo/autoscaler/checkers/web_latency.rb

@@ -34,6 +34,8 @@ module Amigo
           super()
         end
 
+        def get_pool_usage = nil
+
         def get_latencies
           now = Time.now.to_i
           window = (now - (WINDOW - 1))..now

data/lib/amigo/autoscaler/handlers/chain.rb

@@ -15,12 +15,12 @@ module Amigo
           super()
         end
 
-        def scale_up(*args, **kw)
-          @chain.each { |c| c.scale_up(*args, **kw) }
+        def scale_up(**kw)
+          @chain.each { |c| c.scale_up(**kw) }
         end
 
-        def scale_down(*args, **kw)
-          @chain.each { |c| c.scale_down(*args, **kw) }
+        def scale_down(**kw)
+          @chain.each { |c| c.scale_down(**kw) }
         end
       end
     end

data/lib/amigo/autoscaler/handlers/fake.rb

@@ -14,12 +14,12 @@ module Amigo
           super()
         end
 
-        def scale_up(checked_latencies, depth:, duration:, **kw)
-          @ups << [checked_latencies, depth, duration, kw]
+        def scale_up(**kw)
+          @ups << kw
         end
 
-        def scale_down(depth:, duration:, **kw)
-          @downs << [depth, duration, kw]
+        def scale_down(**kw)
+          @downs << kw
         end
       end
     end

data/lib/amigo/autoscaler/handlers/heroku.rb

@@ -103,7 +103,7 @@ module Amigo
         # Potentially add another worker to the formation.
         # @return [:noscale, :maxscale, :scaled] One of :noscale (no +active_event_initial_workers+),
         #   :maxscale (+max_additional_workers+ reached), or :scaled.
-        def scale_up(_queues_and_latencies, depth:, **)
+        def scale_up(depth:, **)
           # When the scaling event starts (or if this is the first time we've seen it
           # but the event is already in progress), store how many workers we have.
           # It needs to be stored in redis so it persists if

data/lib/amigo/autoscaler/handlers/log.rb

@@ -18,12 +18,12 @@ module Amigo
           super()
         end
 
-        def scale_up(checked_latencies, depth:, duration:, **_kw)
-          self._log(:warn, @message, queues: checked_latencies, depth: depth, duration: duration)
+        def scale_up(high_latencies:, depth:, duration:, pool_usage:, **)
+          self._log(:warn, @message, queues: high_latencies, depth:, duration:, pool_usage:)
         end
 
-        def scale_down(depth:, duration:, **_kw)
-          self._log(:info, "#{@message}_restored", depth: depth, duration: duration)
+        def scale_down(depth:, duration:, **)
+          self._log(:info, "#{@message}_restored", depth:, duration:)
         end
 
         protected def _log(level, msg, **kw)

data/lib/amigo/autoscaler/handlers/sentry.rb

@@ -20,12 +20,12 @@ module Amigo
           super()
         end
 
-        def scale_up(checked_latencies, depth:, duration:, **)
+        def scale_up(high_latencies:, depth:, duration:, pool_usage:, **)
           now = Time.now
           call_sentry = @last_alerted < (now - @interval)
           return unless call_sentry
           ::Sentry.with_scope do |scope|
-            scope&.set_extras(high_latency_queues: checked_latencies, depth:, duration:)
+            scope&.set_extras(high_latencies:, depth:, duration:, pool_usage:)
             ::Sentry.capture_message(@message, level: @level)
           end
           @last_alerted = now

data/lib/amigo/autoscaler.rb

@@ -3,6 +3,7 @@
 require "sidekiq/api"
 
 require "amigo"
+require "amigo/threading_event"
 
 # Generic autoscaling handler that will check for latency
 # and take an action.
@@ -42,12 +43,19 @@ module Amigo
     # @!attribute latency_event_started_at [Time] 0-time if not in a latency event.
     Persisted = Struct.new(:last_alerted_at, :depth, :latency_event_started_at)
 
-    # How often should Autoscaler check for latency?
+    # How often the Autoscaler checks for latency/usage statistics.
     # @return [Integer]
     attr_reader :poll_interval
-    # What latency should we alert on?
-    # @return [Integer]
+
+    # The latency, in seconds, that triggers an alert.
+    # @return [Numeric]
     attr_reader :latency_threshold
+
+    # The pool usage, as a float between 0 and 1 (or above), that triggers an alert.
+    # Note that usage-based autoscaling should generally not be used for background jobs.
+    # It is much more useful for web autoscaling, since it is more responsive than latency.
+    attr_reader :usage_threshold
+
     # What hosts/processes should this run on?
     # Looks at ENV['DYNO'] and Socket.gethostname for a match.
     # Default to only run on 'web.1', which is the first Heroku web dyno.
@@ -95,6 +103,7 @@ module Amigo
       checker:,
       poll_interval: 20,
       latency_threshold: 5,
+      usage_threshold: 1,
       hostname_regex: /^web\.1$/,
       alert_interval: 120,
       latency_restored_threshold: latency_threshold,
@@ -111,6 +120,7 @@ module Amigo
       @checker = checker
       @poll_interval = poll_interval
       @latency_threshold = latency_threshold
+      @usage_threshold = usage_threshold
       @hostname_regex = hostname_regex
       @alert_interval = alert_interval
       @latency_restored_threshold = latency_restored_threshold
@@ -124,9 +134,7 @@ module Amigo
     end
 
     def setup
-      # Store these as strings OR procs, rather than grabbing self.method here.
-      # It gets extremely hard ot test if we capture the method here.
-      @stop = false
+      @thr_event = ThreadingEvent.new
       persisted = self.fetch_persisted
       @last_alerted = persisted.last_alerted_at
       @depth = persisted.depth
@@ -171,16 +179,17 @@ module Amigo
       self._debug(:info, "async_autoscaler_starting")
       self.setup
       @polling_thread = Thread.new do
-        until @stop
-          Kernel.sleep(self.poll_interval)
-          self.check unless @stop
+        loop do
+          @thr_event.wait(self.poll_interval)
+          break if @thr_event.set?
+          self.check
         end
       end
       return true
     end
 
     def stop
-      @stop = true
+      @thr_event.set
     end
 
     def check
@@ -201,7 +210,8 @@ module Amigo
       self._debug(:info, "async_autoscaler_check")
       high_latency_queues = self.checker.get_latencies.
         select { |_, latency| latency > self.latency_threshold }
-      if high_latency_queues.empty?
+      high_pool_usage = !(pu = self.checker.get_pool_usage).nil? && pu > self.usage_threshold
+      if high_latency_queues.empty? && !high_pool_usage
         # Whenever we are in a latency event, we have a depth > 0. So a depth of 0 means
         # we're not in a latency event, and still have no latency, so can noop.
         return if @depth.zero?
@@ -224,9 +234,7 @@ module Amigo
         @latency_event_started = Time.now
         duration = 0.0
       end
-      # Alert each handler. For legacy reasons, we support handlers that accept
-      # ({queues and latencies}) and ({queues and latencies}, {}keywords}).
-      @handler.scale_up(high_latency_queues, depth: @depth, duration: duration)
+      @handler.scale_up(high_latencies: high_latency_queues, depth: @depth, duration: duration, pool_usage: pu)
       @last_alerted = now
       self.persist
     end
@@ -239,14 +247,22 @@ module Amigo
     class Checker
       # Return relevant latencies for this checker.
       # This could be the latencies of each Sidekiq queue, or web latencies, etc.
+      # If this is a pool usage checker only, return {}.
       # @return [Hash] Key is the queue name (or some other value); value is the latency in seconds.
       def get_latencies = raise NotImplementedError
+
+      # Return the pool usage for this checker.
+      # Values should be between 0 and 1, with values over 1 meaning a backlog.
+      # If this is a latency checker only, or there is not enough information to report on pool usage, return nil.
+      # @return [nil,Float]
+      def get_pool_usage = raise NotImplementedError
     end
 
     class Handler
       # Called when a latency event starts, and as it fails to resolve.
-      # @param checked_latencies [Hash] The +Hash+ returned from +Amigo::Autoscaler::Handler#check+.
+      # @param high_latencies [Hash] The +Hash+ returned from +Amigo::Autoscaler::Handler#check+.
       #   For Sidekiq, this will look like `{queue name => latency in seconds}`
+      # @param pool_usage [Float,nil] The pool usage value from the checker, or nil.
       # @param depth [Integer] Number of alerts as part of this latency event.
       #   For example, the first alert has a depth of 1, and if latency stays high,
       #   it'll be 2 on the next call, etc. +depth+ can be used to incrementally provision
@@ -256,7 +272,7 @@ module Amigo
       # @param kw [Hash] Additional undefined keywords. Handlers should accept additional options,
       #   like via `**kw` or `opts={}`, for compatibility.
       # @return [Array<String,Symbol,Proc,#call>]
-      def scale_up(checked_latencies, depth:, duration:, **kw) = raise NotImplementedError
+      def scale_up(high_latencies:, pool_usage:, depth:, duration:, **kw) = raise NotImplementedError
 
       # Called when a latency of +latency_restored_threshold+ is reached
       # (ie, when we get back to normal latency after a high latency event).

data/lib/amigo/threading_event.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Amigo
+  # Threading event on +Concurrent::Event+, ManualResetEvent, etc.
+  # Efficient way to sleep and wake up.
+  class ThreadingEvent
+    def initialize(initial=false)
+      @mutex = Mutex.new
+      @cv = ConditionVariable.new
+      @signaled = initial
+    end
+
+    # Sleep the current thread until +set+ is called by another thread.
+    # @param timeout [Numeric,nil] Passed to +Mutex#sleep+.
+    # @return See +Mutex#sleep+.
+    def wait(timeout=nil)
+      # _debug("wait")
+      @mutex.synchronize do
+        @cv.wait(@mutex, timeout)
+      end
+    end
+
+    # Signal the event. The waiting threads will wake up.
+    def set
+      #       _debug("set")
+      @mutex.synchronize do
+        @signaled = true
+        @cv.broadcast # wake up all waiters
+      end
+    end
+
+    # True if +set+ has been called.
+    def set? = @signaled
+
+    # Reset the event back to its original state.
+    def reset
+      #       _debug("reset")
+      @mutex.synchronize do
+        @signaled = false
+      end
+    end
+
+    #     # def _debug(msg)
+    #   puts "#{Thread.current.name}: #{msg}"
+    # end
+  end
+end

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.12.1"
+  VERSION = "1.13.0"
 end

data/lib/puma/plugin/amigo.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "amigo/autoscaler/checkers/puma_pool_usage"
+
+Puma::Plugin.create do
+  # @param [Puma::Launcher] launcher
+  def start(launcher)
+    interval = launcher.options[:amigo_autoscaler_interval] || 20
+    checker = launcher.options.fetch(:amigo_puma_pool_usage_checker)
+    event = Amigo::ThreadingEvent.new
+    in_background do
+      loop do
+        event.wait(interval)
+        break if event.set?
+        log_pool_usage(launcher, checker)
+      end
+    end
+
+    launcher.events.on_stopped do
+      event.set
+    end
+  end
+
+  # Find the Puma stats necessary depending on mode (single vs. cluster).
+  # Sends statistics for logging.
+  def log_pool_usage(launcher, checker)
+    now = Time.now
+    stats = launcher.stats
+    if stats[:worker_status]
+      stats[:worker_status].each { |worker| _log_pool_usage(checker, worker[:last_status], now:) }
+    else
+      _log_pool_usage(checker, stats, now:)
+    end
+  end
+
+  def _log_pool_usage(checker, stats, now:)
+    pool_usage = calculate_pool_usage(stats)
+    checker.record(pool_usage, now:)
+  end
+
+  # Pool usage is 0 at no busy threads, 1 at busy threads == max threads,
+  # or above 1 if there is a backlog (ie, 4 threads and 4 backlog items is a usage of 2).
+  # For our usage purposes, we don't want to deal with the case where we have a backlog,
+  # but fewer threads spawned than our max; in this case, we don't need to autoscale,
+  # since Puma can still launch threads.
+  def calculate_pool_usage(stats)
+    busy = stats[:busy_threads]
+    max = stats[:max_threads]
+    backlog = stats[:backlog]
+    return (busy + backlog) / max.to_f
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.12.1
+  version: 1.13.0
 platform: ruby
 authors:
 - Lithic Technology
@@ -37,6 +37,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: platform-api
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +65,20 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -203,7 +231,9 @@ files:
 - lib/amigo.rb
 - lib/amigo/audit_logger.rb
 - lib/amigo/autoscaler.rb
+- lib/amigo/autoscaler/checkers/chain.rb
 - lib/amigo/autoscaler/checkers/fake.rb
+- lib/amigo/autoscaler/checkers/puma_pool_usage.rb
 - lib/amigo/autoscaler/checkers/sidekiq.rb
 - lib/amigo/autoscaler/checkers/web_latency.rb
 - lib/amigo/autoscaler/handlers/chain.rb
@@ -221,7 +251,9 @@ files:
 - lib/amigo/scheduled_job.rb
 - lib/amigo/semaphore_backoff_job.rb
 - lib/amigo/spec_helpers.rb
+- lib/amigo/threading_event.rb
 - lib/amigo/version.rb
+- lib/puma/plugin/amigo.rb
 homepage: https://github.com/lithictech/sidekiq-amigo
 licenses:
 - MIT