sidekiq-amigo 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70d47770af144a84aedec89e8b74e4ebd13851119a5c329a0f4c65d56ec94a60
-  data.tar.gz: 69305abac1bb5eeb00eaf81613e591476bf1acb35cdac50dcc97a094d255aab8
+  metadata.gz: c51def3cb58812e0889c0768708747aefd3956495692ffcbb6521a1d24c6d0b0
+  data.tar.gz: 5081debc483ce040e8c6f3a26cfacdd85094c5e70895b390f23f58f795663da3
 SHA512:
-  metadata.gz: adf9aa76c352c9ddb9832f142d415b4efded423324dd78a87cfc3140a755eb266dd1a1b80e15a2f911ccafaaf8ea89f416d66dfe25a17955709fb1e4828554ef
-  data.tar.gz: f69c16bfd06a7ea5cc3ac15db4a962f3fb02345db60ac6f581123bf5425627775bb1958e9f99ff0be8b5956db57d4c6fec64d5422ff7903c0157bf0a0f8ba4d1
+  metadata.gz: b126004a168ea18f4842bd34268fc3edb263d21de40731d178e9bd4a5db64bf403da0953b6083ea0e5c5e8d32bc9aa4faf0e66e54f0f166a077780d6573565f6
+  data.tar.gz: 4a710ac1029e607ed60b1f448e4c443d287bcb599252801a70a17a9478e0f5c80b373f610a14034be1d2d3c98541512e64ab2f8b2bc85869b70fbc009308007d

data/lib/amigo/autoscaler/heroku.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "platform-api"
+
+require "amigo/autoscaler"
+
+module Amigo
+  class Autoscaler
+    # Autoscaler to use on Heroku, that starts additional worker processes when there is a high latency event
+    # and scales them down after the event is finished.
+    #
+    # When the first call of a high latency event happens (depth: 1), this class
+    # will ask Heroku how many dynos are in the formation. This is known as +active_event_initial_workers+.
+    #
+    # If +active_event_initial_workers+ is 0, no autoscaling will be done.
+    # This avoids a situation where a high latency event is triggered
+    # due to workers being deprovisioned intentionally, perhaps for maintenance.
+    #
+    # Each time the alert fires (see +Amigo::Autoscaler#alert_interval+),
+    # an additional worker will be added to the formation, up to +max_additional_workers+.
+    # So with +active_event_initial_workers+ of 1 and +max_additional_workers+ of 2,
+    # the first time the alert times, the formation will be set to 2 workers.
+    # The next time, it'll be set to 3 workers.
+    # After that, no additional workers will be provisioned.
+    #
+    # After the high latency event resolves,
+    # the dyno formation is restored to +active_event_initial_workers+.
+    #
+    # To use:
+    #
+    #   heroku = PlatformAPI.connect_oauth(heroku_oauth_token)
+    #   heroku_scaler = Amigo::Autoscaler::Heroku.new(heroku:, default_workers: 1)
+    #   Amigo::Autoscaler.new(
+    #     handlers: [heroku_scaler.alert_callback],
+    #     latency_restored_handlers: [heroku_scaler.restored_callback],
+    #   )
+    #
+    # See instance attributes for additional options.
+    #
+    # Note that this class is provided as an example, and potentially a base or implementation class.
+    # Your actual implementation may also want to alert when a max depth or duration is reached,
+    # since it can indicate a bigger problem. Autoscaling, especially of workers, is a tough problem
+    # without a one-size-fits-all approach.
+    class Heroku
+      # Heroku client, usually created via PlatformAPI.oauth_connect.
+      # @return [PlatformAPI::Client]
+      attr_reader :heroku
+
+      # Captured at the start of a high latency event.
+      # Nil otherwise.
+      # @return [Integer]
+      attr_reader :active_event_initial_workers
+
+      # Maximum number of workers to add.
+      #
+      # As the 'depth' of the alert is increased,
+      # workers are added to the recorded worker count until the max is reached.
+      # By default, this is 2 (so the max workers will be the recorded number, plus 2).
+      # Do not set this too high, since it can for example exhaust database connections or just end up
+      # increasing load.
+      #
+      # See class docs for more information.
+      # @return [Integer]
+      attr_reader :max_additional_workers
+
+      # Defaults to HEROKU_APP_NAME, which should already be set if you use Heroku dyna metadata,
+      # as per https://devcenter.heroku.com/articles/dyno-metadata.
+      # This must be provided if the env var is missing.
+      # @return [String]
+      attr_reader :app_id_or_app_name
+
+      # Defaults to 'worker', which is what you'll probably use if you have a simple system.
+      # If you use multiple worker processes for different queues, this class probably isn't sufficient.
+      # You will probably need to look at the slow queue names and determine the formation name to scale up.
+      # @return [String]
+      attr_reader :formation_id_or_formation_type
+
+      def initialize(
+        heroku:,
+        max_additional_workers: 2,
+        app_id_or_app_name: ENV.fetch("HEROKU_APP_NAME"),
+        formation_id_or_formation_type: "worker"
+      )
+
+        @heroku = heroku
+        @max_additional_workers = max_additional_workers
+        @app_id_or_app_name = app_id_or_app_name
+        @formation_id_or_formation_type = formation_id_or_formation_type
+        # Is nil outside of a latency event, set during a latency event. So if this is initialized to non-nil,
+        # we're already in a latency event.
+        @active_event_initial_workers = Sidekiq.redis do |r|
+          v = r.get("#{namespace}/active_event_initial_workers")
+          v&.to_i
+        end
+      end
+
+      def alert_callback
+        self.method(:scale_up)
+      end
+
+      def restored_callback
+        self.method(:scale_down)
+      end
+
+      protected def namespace
+        return "amigo/autoscaler/heroku"
+      end
+
+      # Potentially add another worker to the formation.
+      # @return [:noscale, :maxscale, :scaled] One of :noscale (no +active_event_initial_workers+),
+      #   :maxscale (+max_additional_workers+ reached), or :scaled.
+      def scale_up(_queues_and_latencies, depth:, **)
+        # When the scaling event starts (or if this is the first time we've seen it
+        # but the event is already in progress), store how many workers we have.
+        # It needs to be stored in redis so it persists if
+        # the latency event continues through restarts.
+        if @active_event_initial_workers.nil?
+          @active_event_initial_workers = @heroku.formation.info(@app_id_or_app_name, @formation_id_or_formation_type).
+            fetch("quantity")
+          Sidekiq.redis do |r|
+            r.set("#{namespace}/active_event_initial_workers", @active_event_initial_workers.to_s)
+          end
+        end
+        return :noscale if @active_event_initial_workers.zero?
+        new_quantity = @active_event_initial_workers + depth
+        max_quantity = @active_event_initial_workers + @max_additional_workers
+        return :maxscale if new_quantity > max_quantity
+        @heroku.formation.update(@app_id_or_app_name, @formation_id_or_formation_type, {quantity: new_quantity})
+        return :scaled
+      end
+
+      # Reset the formation to +active_event_initial_workers+.
+      # @return [:noscale, :scaled] :noscale if +active_event_initial_workers+ is 0, otherwise :scaled.
+      def scale_down(**)
+        initial_workers = @active_event_initial_workers
+        Sidekiq.redis do |r|
+          r.del("#{namespace}/active_event_initial_workers")
+        end
+        @active_event_initial_workers = nil
+        return :noscale if initial_workers.zero?
+        @heroku.formation.update(@app_id_or_app_name, @formation_id_or_formation_type, {quantity: initial_workers})
+        return :scaled
+      end
+    end
+  end
+end

data/lib/amigo/autoscaler.rb

@@ -6,20 +6,31 @@ require "amigo"
 
 # When queues achieve a latency that is too high,
 # take some action.
-# You should start this up at Sidekiq application startup:
+# You should start this up at Web application startup:
 #
-# # sidekiq.rb
-# Amigo::Autoscaler.new.start
+#   # puma.rb or similar
+#   Amigo::Autoscaler.new.start
 #
-# Right now, this is pretty simple- we alert any time
-# there is a latency over a threshold.
+# When latency grows beyond +latency_threshold+,
+# a "high latency event" is started.
+# Some action is taken, which is defined by the +handlers+ argument.
+# This includes logging, alerting, and/or autoscaling.
 #
-# In the future, we can:
+# When latency returns to normal (defined by +latency_restored_threshold+),
+# the high latency event finishes.
+# Some additional action is taken, which is defined by the +latency_restored_handlers+ argument.
+# Usually this is logging, and/or returning autoscaling to its original status.
 #
-# 1) actually autoscale rather than just alert
-#    (this may take the form of a POST to a configurable endpoint),
-# 2) become more sophisticated with how we detect latency growth.
+# There are several parameters to control behavior, such as how often polling is done,
+# how often alerting/scaling is done, and more.
 #
+# As an example autoscaler that includes actual resource scaling,
+# check out +Amigo::Autoscaler::Heroku+.
+# Its ideas can easily be expanded to other platforms.
+#
+# Note that +Autoscaler+ maintains its state over multiple processes;
+# it needs to keep track of high latency events even if the process running the autoscaler
+# (usually a web process) restarts.
 module Amigo
   class Autoscaler
     class InvalidHandler < StandardError; end
@@ -31,18 +42,26 @@ module Amigo
     # @return [Integer]
     attr_reader :latency_threshold
     # What hosts/processes should this run on?
-    # Look at ENV['DYNO'] and Socket.gethostname.
+    # Looks at ENV['DYNO'] and Socket.gethostname for a match.
     # Default to only run on 'web.1', which is the first Heroku web dyno.
     # We run on the web, not worker, dyno, so we report backed up queues
     # in case we, say, turn off all workers (broken web processes
     # are generally easier to find).
     # @return [Regexp]
     attr_reader :hostname_regex
-    # Methods to call when alerting.
-    # Valid values are 'log' and 'sentry' (requires Sentry to be required already).
-    # Anything that responds to +call+ will be invoked with a hash of
-    # `{queue name => latency in seconds}`.
-    # @return [Array<String,Proc>]
+    # Methods to call when alerting, as strings/symbols or procs.
+    # Valid string values are 'log' and 'sentry' (requires Sentry to be required already).
+    # Anything that responds to +call+ will be invoked with:
+    # - Positional argument which is a +Hash+ of `{queue name => latency in seconds}`
+    # - Keyword argument +:depth+: Number of alerts as part of this latency event.
+    #   For example, the first alert has a depth of 1, and if latency stays high,
+    #   it'll be 2 on the next call, etc. +depth+ can be used to incrementally provision
+    #   additional processing capacity, and stop adding capacity at a certain depth
+    #   to avoid problems with too many workers (like excessive DB load).
+    # - Keyword argument +:duration+: Number of seconds since this latency spike started.
+    # - Additional undefined keywords. Handlers should accept additional options,
+    #   like via `**kw` or `opts={}`, for compatibility.
+    # @return [Array<String,Symbol,Proc,#call>]
     attr_reader :handlers
     # Only alert this often.
     # For example, with poll_interval of 10 seconds
@@ -50,20 +69,55 @@ module Amigo
     # we'd alert once and then 210 seconds later.
     # @return [Integer]
     attr_reader :alert_interval
+    # After an alert happens, what latency should be considered "back to normal" and
+    # +latency_restored_handlers+ will be called?
+    # In most cases this should be the same as (and defaults to) +latency_threshold+
+    # so that we're 'back to normal' once we're below the threshold.
+    # It may also commonly be 0, so that the callback is fired when the queue is entirely clear.
+    # Note that, if +latency_restored_threshold+ is less than +latency_threshold+,
+    # while the latency is between the two, no alerts will fire.
+    attr_reader :latency_restored_threshold
+    # Methods to call when a latency of +latency_restored_threshold+ is reached
+    # (ie, when we get back to normal latency after a high latency event).
+    # Valid string values are 'log'.
+    # Usually this handler will deprovision capacity procured as part of the alert +handlers+.
+    # Anything that responds to +call+ will be invoked with:
+    # - Keyword +:depth+, the number of times an alert happened before
+    #   the latency spike was resolved.
+    # - Keyword +:duration+, the number of seconds for the latency spike has been going on.
+    # - Additional undefined keywords. Handlers should accept additional options,
+    #   like via `**kw`, for compatibility.
+    # @return [Array<String,Symbol,Proc,#call>]
+    attr_reader :latency_restored_handlers
+    # Proc/callable called with (level, message, params={}).
+    # By default, use +Amigo.log+ (which logs to the Sidekiq logger).
+    attr_reader :log
 
     def initialize(
       poll_interval: 20,
       latency_threshold: 5,
       hostname_regex: /^web\.1$/,
-      handlers: ["log"],
-      alert_interval: 120
+      handlers: [:log],
+      alert_interval: 120,
+      latency_restored_threshold: latency_threshold,
+      latency_restored_handlers: [:log],
+      log: ->(level, message, params={}) { Amigo.log(nil, level, message, params) }
     )
 
+      raise ArgumentError, "latency_threshold must be > 0" if
+        latency_threshold <= 0
+      raise ArgumentError, "latency_restored_threshold must be >= 0" if
+        latency_restored_threshold.negative?
+      raise ArgumentError, "latency_restored_threshold must be <= latency_threshold" if
+        latency_restored_threshold > latency_threshold
       @poll_interval = poll_interval
       @latency_threshold = latency_threshold
       @hostname_regex = hostname_regex
-      @handlers = handlers
+      @handlers = handlers.freeze
       @alert_interval = alert_interval
+      @latency_restored_threshold = latency_restored_threshold
+      @latency_restored_handlers = latency_restored_handlers.freeze
+      @log = log
     end
 
     def polling_thread
@@ -73,17 +127,44 @@ module Amigo
     def setup
       # Store these as strings OR procs, rather than grabbing self.method here.
       # It gets extremely hard ot test if we capture the method here.
-      @alert_methods = self.handlers.map do |a|
-        if a.respond_to?(:call)
-          a
-        else
-          method_name = meth = "alert_#{a.strip}".to_sym
-          raise InvalidHandler, a.inspect unless self.method(method_name)
-          meth
-        end
-      end
-      @last_alerted = Time.at(0)
+      @alert_methods = self.handlers.map { |a| _handler_to_method("alert_", a) }
+      @restored_methods = self.latency_restored_handlers.map { |a| _handler_to_method("alert_restored_", a) }
       @stop = false
+      Sidekiq.redis do |r|
+        @last_alerted = Time.at((r.get("#{namespace}/last_alerted") || 0).to_f)
+        @depth = (r.get("#{namespace}/depth") || 0).to_i
+        @latency_event_started = Time.at((r.get("#{namespace}/latency_event_started") || 0).to_f)
+      end
+    end
+
+    private def persist
+      Sidekiq.redis do |r|
+        r.set("#{namespace}/last_alerted", @last_alerted.to_f.to_s)
+        r.set("#{namespace}/depth", @depth.to_s)
+        r.set("#{namespace}/latency_event_started", @latency_event_started.to_f.to_s)
+      end
+    end
+
+    # Delete all the keys that Autoscaler stores.
+    # Can be used in extreme cases where things need to be cleaned up,
+    # but should not be normally used.
+    def unpersist
+      Sidekiq.redis do |r|
+        r.del("#{namespace}/last_alerted")
+        r.del("#{namespace}/depth")
+        r.del("#{namespace}/latency_event_started")
+      end
+    end
+
+    protected def namespace
+      return "amigo/autoscaler"
+    end
+
+    private def _handler_to_method(prefix, a)
+      return a if a.respond_to?(:call)
+      method_name = "#{prefix}#{a.to_s.strip}".to_sym
+      raise InvalidHandler, a.inspect unless (meth = self.method(method_name))
+      return meth
     end
 
     def start
@@ -92,7 +173,7 @@ module Amigo
       hostname = ENV.fetch("DYNO") { Socket.gethostname }
       return false unless self.hostname_regex.match?(hostname)
 
-      self.log(:info, "async_autoscaler_starting")
+      self._log(:info, "async_autoscaler_starting")
       self.setup
       @polling_thread = Thread.new do
         until @stop
@@ -109,21 +190,53 @@ module Amigo
 
     def check
       now = Time.now
-      skip_check = now < (@last_alerted + self.poll_interval)
+      skip_check = now < (@last_alerted + self.alert_interval)
       if skip_check
-        self.log(:debug, "async_autoscaler_skip_check")
+        self._log(:debug, "async_autoscaler_skip_check")
         return
       end
-      self.log(:info, "async_autoscaler_check")
+      self._log(:info, "async_autoscaler_check")
       high_latency_queues = Sidekiq::Queue.all.
         map { |q| [q.name, q.latency] }.
         select { |(_, latency)| latency > self.latency_threshold }.
         to_h
-      return if high_latency_queues.empty?
+      if high_latency_queues.empty?
+        # Whenever we are in a latency event, we have a depth > 0. So a depth of 0 means
+        # we're not in a latency event, and still have no latency, so can noop.
+        return if @depth.zero?
+        # We WERE in a latency event, and now we're not, so report on it.
+        @restored_methods.each do |m|
+          m.call(depth: @depth, duration: (Time.now - @latency_event_started).to_f)
+        end
+        # Reset back to 0 depth so we know we're not in a latency event.
+        @depth = 0
+        @latency_event_started = Time.at(0)
+        @last_alerted = now
+        self.persist
+        return
+      end
+      if @depth.positive?
+        # We have already alerted, so increment the depth and when the latency started.
+        @depth += 1
+        duration = (Time.now - @latency_event_started).to_f
+      else
+        # Indicate we are starting a high latency event.
+        @depth = 1
+        @latency_event_started = Time.now
+        duration = 0.0
+      end
+      # Alert each handler. For legacy reasons, we support handlers that accept
+      # ({queues and latencies}) and ({queues and latencies}, {}keywords}).
+      kw = {depth: @depth, duration: duration}
       @alert_methods.each do |m|
-        m.respond_to?(:call) ? m.call(high_latency_queues) : self.send(m, high_latency_queues)
+        if m.respond_to?(:arity) && m.arity == 1
+          m.call(high_latency_queues)
+        else
+          m.call(high_latency_queues, **kw)
+        end
       end
       @last_alerted = now
+      self.persist
     end
 
     def alert_sentry(names_and_latencies)
@@ -134,14 +247,18 @@ module Amigo
       end
     end
 
-    def alert_log(names_and_latencies)
-      self.log(:warn, "high_latency_queues", queues: names_and_latencies)
+    def alert_log(names_and_latencies, depth:, duration:)
+      self._log(:warn, "high_latency_queues", queues: names_and_latencies, depth: depth, duration: duration)
     end
 
-    def alert_test(_names_and_latencies); end
+    def alert_test(_names_and_latencies, _opts={}); end
+
+    def alert_restored_log(depth:, duration:)
+      self._log(:info, "high_latency_queues_restored", depth: depth, duration: duration)
+    end
 
-    protected def log(level, msg, **kw)
-      Amigo.log(nil, level, msg, kw)
+    protected def _log(level, msg, **kw)
+      self.log[level, msg, kw]
     end
   end
 end

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Lithic Technology
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-04 00:00:00.000000000 Z
+date: 2023-04-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: platform-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.48'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '1.48'
 - !ruby/object:Gem::Dependency
   name: rubocop-performance
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.16'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: sentry-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -136,10 +150,24 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 'sidekiq-amigo provides a pubsub system and other enhancements around
   Sidekiq.
 
-'
+  '
 email: hello@lithic.tech
 executables: []
 extensions: []
@@ -148,6 +176,7 @@ files:
 - lib/amigo.rb
 - lib/amigo/audit_logger.rb
 - lib/amigo/autoscaler.rb
+- lib/amigo/autoscaler/heroku.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
 - lib/amigo/queue_backoff_job.rb
@@ -163,7 +192,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -178,8 +207,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: Pubsub system and other enhancements around Sidekiq.
 test_files: []