sidekiq-amigo 1.3.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56470f3e16978212a6e8a23d970c7709c0f24854b80c15d522989389e0cc829e
-  data.tar.gz: 2bed22f2a190e056aa715814f460ba009fd2c2d3884e6d34eb866084659af962
+  metadata.gz: defd2060d1c4a56780815bbaff9cfc54169ed12857fc5c1e20a075a5ea60ee8d
+  data.tar.gz: 1cefc43ca64867f333d5706c5f2df3998a3fac294d957ccdf95f05763589d1aa
 SHA512:
-  metadata.gz: 18ac44e7cffa487844980f937193ac4601585e380aa08c85599591181271e3ded429271be3407d120e0bf280c541c0f9c28ca23c2c5e9a528deda26c16ca8735
-  data.tar.gz: 243229bf9a1f661eb6ef4b9fe847a979634fa729df34e756016f6f2c403ae1e8f73d444e437e640b86d300e38169a391b72f1676a49e985aeac11d41684c4974
+  metadata.gz: fde4becc89be41126f447ce282a9c6ccbe5adb05b979cc6dbea455d7bd7735043001115722450b412434253a23dc0b80bab6766b43804ad58698d0eddfd6713b
+  data.tar.gz: 43209e4289474df8d62bc1d8052869df1cd7c375df921ead70f5aeb69f03be79dbaba53f33b1f04d66153681fd0f6e92a598176a3e7261718cda972306b571af

data/lib/amigo/autoscaler.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "sidekiq/api"
+
+require "amigo"
+
+# When queues achieve a latency that is too high,
+# take some action.
+# You should start this up at Sidekiq application startup:
+#
+# # sidekiq.rb
+# Amigo::Autoscaler.new.start
+#
+# Right now, this is pretty simple- we alert any time
+# there is a latency over a threshold.
+#
+# In the future, we can:
+#
+# 1) actually autoscale rather than just alert
+#    (this may take the form of a POST to a configurable endpoint),
+# 2) become more sophisticated with how we detect latency growth.
+#
+module Amigo
+  class Autoscaler
+    class InvalidHandler < StandardError; end
+
+    # How often should Autoscaler check for latency?
+    # @return [Integer]
+    attr_reader :poll_interval
+    # What latency should we alert on?
+    # @return [Integer]
+    attr_reader :latency_threshold
+    # What hosts/processes should this run on?
+    # Look at ENV['DYNO'] and Socket.gethostname.
+    # Default to only run on 'web.1', which is the first Heroku web dyno.
+    # We run on the web, not worker, dyno, so we report backed up queues
+    # in case we, say, turn off all workers (broken web processes
+    # are generally easier to find).
+    # @return [Regexp]
+    attr_reader :hostname_regex
+    # Methods to call when alerting.
+    # Valid values are 'log' and 'sentry' (requires Sentry to be required already).
+    # Anything that responds to +call+ will be invoked with a hash of
+    # `{queue name => latency in seconds}`.
+    # @return [Array<String,Proc>]
+    attr_reader :handlers
+    # Only alert this often.
+    # For example, with poll_interval of 10 seconds
+    # and alert_interval of 200 seconds,
+    # we'd alert once and then 210 seconds later.
+    # @return [Integer]
+    attr_reader :alert_interval
+
+    def initialize(
+      poll_interval: 20,
+      latency_threshold: 5,
+      hostname_regex: /^web\.1$/,
+      handlers: ["log"],
+      alert_interval: 120
+    )
+
+      @poll_interval = poll_interval
+      @latency_threshold = latency_threshold
+      @hostname_regex = hostname_regex
+      @handlers = handlers
+      @alert_interval = alert_interval
+    end
+
+    def polling_thread
+      return @polling_thread
+    end
+
+    def setup
+      # Store these as strings OR procs, rather than grabbing self.method here.
+      # It gets extremely hard ot test if we capture the method here.
+      @alert_methods = self.handlers.map do |a|
+        if a.respond_to?(:call)
+          a
+        else
+          method_name = meth = "alert_#{a.strip}".to_sym
+          raise InvalidHandler, a.inspect unless self.method(method_name)
+          meth
+        end
+      end
+      @last_alerted = Time.at(0)
+      @stop = false
+    end
+
+    def start
+      raise "already started" unless @polling_thread.nil?
+
+      hostname = ENV.fetch("DYNO") { Socket.gethostname }
+      return false unless self.hostname_regex.match?(hostname)
+
+      self.log(:info, "async_autoscaler_starting")
+      self.setup
+      @polling_thread = Thread.new do
+        until @stop
+          Kernel.sleep(self.poll_interval)
+          self.check unless @stop
+        end
+      end
+      return true
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def check
+      now = Time.now
+      skip_check = now < (@last_alerted + self.poll_interval)
+      if skip_check
+        self.log(:debug, "async_autoscaler_skip_check")
+        return
+      end
+      self.log(:info, "async_autoscaler_check")
+      high_latency_queues = Sidekiq::Queue.all.
+        map { |q| [q.name, q.latency] }.
+        select { |(_, latency)| latency > self.latency_threshold }.
+        to_h
+      return if high_latency_queues.empty?
+      @alert_methods.each do |m|
+        m.respond_to?(:call) ? m.call(high_latency_queues) : self.send(m, high_latency_queues)
+      end
+      @last_alerted = now
+    end
+
+    def alert_sentry(names_and_latencies)
+      Sentry.with_scope do |scope|
+        scope.set_extras(high_latency_queues: names_and_latencies)
+        names = names_and_latencies.map(&:first).sort.join(", ")
+        Sentry.capture_message("Some queues have a high latency: #{names}")
+      end
+    end
+
+    def alert_log(names_and_latencies)
+      self.log(:warn, "high_latency_queues", queues: names_and_latencies)
+    end
+
+    def alert_test(_names_and_latencies); end
+
+    protected def log(level, msg, **kw)
+      Amigo.log(nil, level, msg, kw)
+    end
+  end
+end

data/lib/amigo/retry.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "sidekiq/api"
 
 # Middleware so Sidekiq workers can use a custom retry logic.
 # See +Amigo::Retry::Retry+, +Amigo::Retry::Die+,
@@ -64,11 +65,15 @@ module Amigo
 
       def handle_retry(worker, job, e)
         Sidekiq.logger.info("scheduling_retry")
+        job["error_class"] = e.class.to_s
+        job["error_message"] = e.to_s
         self.amigo_retry_in(worker.class, job, e.interval_or_timestamp)
       end
 
-      def handle_die(_worker, job, _e)
+      def handle_die(_worker, job, e)
         Sidekiq.logger.warn("sending_to_deadset")
+        job["error_class"] = e.class.to_s
+        job["error_message"] = e.to_s
         payload = Sidekiq.dump_json(job)
         Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
       end

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.3.0"
+  VERSION = "1.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Lithic Technology
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-29 00:00:00.000000000 Z
+date: 2023-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -133,6 +147,7 @@ extra_rdoc_files: []
 files:
 - lib/amigo.rb
 - lib/amigo/audit_logger.rb
+- lib/amigo/autoscaler.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
 - lib/amigo/queue_backoff_job.rb