sidekiq-amigo 1.2.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 53361c37c9ec31b3886fb7b33ff61bd022802128b458f93e624c4f0d0cbcc963
-  data.tar.gz: 54a9e75e5b011d6441b8906404a0d856394e19c4d27e66e7580c37d4928e3d41
+  metadata.gz: ef7c94e7a10ea6d0023d775d364242fb3f361bd782a142ce6e969756d4067d14
+  data.tar.gz: a4fda58bbee1d07b9e3e009925d6ad7fd85d5cd378f837667b4ee2bb07adecb7
 SHA512:
-  metadata.gz: b469583cc5c2e339d359834763d3a67c154e9977212b35f039623f937b4b55997b7a523ba8f5c414898295c18c65d401a020b9679c4d695c50d636ef5c7334db
-  data.tar.gz: ab70f5da69a2bb8868c9ec864c760eb1d6fc124a99d71e2ad332a8dccd7ba8503971dfb0ee3a60d19d122e5ce079b5f9a4dc6a5a3df2a8c05057b19f2963c172
+  metadata.gz: 8d86da5a86d2bbdb5251552ab925ddb7e2b14231c9ddce0ea26b9d2c35ce96227addb95345f4e9b309bca6c4951b13601237d62040cfde69776a4d84c08d90bc
+  data.tar.gz: 38002ddf39af631ec5bee7ec6cdee6fdfb127094f41279719b53d4a85313444c80001a84b67bf10f5bc6c7e239adfa2aae6d4fda5ec4e8d6de6d13b6dc065ffa

data/lib/amigo/autoscaler.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require "sidekiq/api"
+
+require "amigo"
+
+# When queues achieve a latency that is too high,
+# take some action.
+# You should start this up at Sidekiq application startup:
+#
+# # sidekiq.rb
+# Amigo::Autoscaler.new.start
+#
+# Right now, this is pretty simple- we alert any time
+# there is a latency over a threshold.
+#
+# In the future, we can:
+#
+# 1) actually autoscale rather than just alert
+#    (this may take the form of a POST to a configurable endpoint),
+# 2) become more sophisticated with how we detect latency growth.
+#
+module Amigo
+  class Autoscaler
+    class InvalidHandler < StandardError; end
+
+    # How often should Autoscaler check for latency?
+    # @return [Integer]
+    attr_reader :poll_interval
+    # What latency should we alert on?
+    # @return [Integer]
+    attr_reader :latency_threshold
+    # What hosts/processes should this run on?
+    # Look at ENV['DYNO'] and Socket.gethostname.
+    # Default to only run on 'web.1', which is the first Heroku web dyno.
+    # We run on the web, not worker, dyno, so we report backed up queues
+    # in case we, say, turn off all workers (broken web processes
+    # are generally easier to find).
+    # @return [Regexp]
+    attr_reader :hostname_regex
+    # Methods to call when alerting.
+    # Valid values are 'log' and 'sentry' (requires Sentry to be required already).
+    # Anything that responds to +call+ will be invoked with a hash of
+    # `{queue name => latency in seconds}`.
+    # @return [Array<String,Proc>]
+    attr_reader :handlers
+    # Only alert this often.
+    # For example, with poll_interval of 10 seconds
+    # and alert_interval of 200 seconds,
+    # we'd alert once and then 210 seconds later.
+    # @return [Integer]
+    attr_reader :alert_interval
+
+    def initialize(
+      poll_interval: 20,
+      latency_threshold: 5,
+      hostname_regex: /^web\.1$/,
+      handlers: ["log"],
+      alert_interval: 120
+    )
+
+      @poll_interval = poll_interval
+      @latency_threshold = latency_threshold
+      @hostname_regex = hostname_regex
+      @handlers = handlers
+      @alert_interval = alert_interval
+    end
+
+    def polling_thread
+      return @polling_thread
+    end
+
+    def setup
+      # Store these as strings OR procs, rather than grabbing self.method here.
+      # It gets extremely hard ot test if we capture the method here.
+      @alert_methods = self.handlers.map do |a|
+        if a.respond_to?(:call)
+          a
+        else
+          method_name = meth = "alert_#{a.strip}".to_sym
+          raise InvalidHandler, a.inspect unless self.method(method_name)
+          meth
+        end
+      end
+      @last_alerted = Time.at(0)
+      @stop = false
+    end
+
+    def start
+      raise "already started" unless @polling_thread.nil?
+
+      hostname = ENV.fetch("DYNO") { Socket.gethostname }
+      return false unless self.hostname_regex.match?(hostname)
+
+      self.log(:info, "async_autoscaler_starting")
+      self.setup
+      @polling_thread = Thread.new do
+        until @stop
+          Kernel.sleep(self.poll_interval)
+          self.check unless @stop
+        end
+      end
+      return true
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def check
+      now = Time.now
+      skip_check = now < (@last_alerted + self.poll_interval)
+      if skip_check
+        self.log(:debug, "async_autoscaler_skip_check")
+        return
+      end
+      self.log(:info, "async_autoscaler_check")
+      high_latency_queues = Sidekiq::Queue.all.
+        map { |q| [q.name, q.latency] }.
+        select { |(_, latency)| latency > self.latency_threshold }.
+        to_h
+      return if high_latency_queues.empty?
+      @alert_methods.each do |m|
+        m.respond_to?(:call) ? m.call(high_latency_queues) : self.send(m, high_latency_queues)
+      end
+      @last_alerted = now
+    end
+
+    def alert_sentry(names_and_latencies)
+      Sentry.with_scope do |scope|
+        scope.set_extras(high_latency_queues: names_and_latencies)
+        names = names_and_latencies.map(&:first).sort.join(", ")
+        Sentry.capture_message("Some queues have a high latency: #{names}")
+      end
+    end
+
+    def alert_log(names_and_latencies)
+      self.log(:warn, "high_latency_queues", queues: names_and_latencies)
+    end
+
+    def alert_test(_names_and_latencies); end
+
+    protected def log(level, msg, **kw)
+      Amigo.log(nil, level, msg, kw)
+    end
+  end
+end

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Amigo
-  VERSION = "1.2.2"
+  VERSION = "1.4.0"
 end

data/lib/amigo.rb

@@ -140,7 +140,18 @@ module Amigo
     # An Array of callbacks to be run when an event is published.
     attr_accessor :subscribers
 
-    # A single callback to be run when an event publication errors.
+    # A single callback to be run when an event publication errors,
+    # almost always due to an error in a subscriber.
+    #
+    # The callback receives the exception, the event being published, and the erroring subscriber.
+    #
+    # If this is not set, errors from subscribers will be re-raised immediately,
+    # since broken subscribers usually indicate a broken application.
+    #
+    # Note also that when an error occurs, Amigo.log is always called first.
+    # You do NOT need a callback that just logs and swallows the error.
+    # If all you want to do is log, and not propogate the error,
+    # you can use `Amigo.on_publish_error = proc {}`.
     attr_accessor :on_publish_error
 
     # Publish an event with the specified +eventname+ and +payload+
@@ -151,12 +162,18 @@ module Amigo
       self.subscribers.to_a.each do |hook|
         hook.call(ev)
       rescue StandardError => e
-        self.log(nil, :error, "amigo_subscriber_hook_error", error: e, hook: hook, event: ev)
-        self.on_publish_error.call(e)
+        self.log(nil, :error, "amigo_subscriber_hook_error", error: e, hook: hook, event: ev&.as_json)
+        raise e if self.on_publish_error.nil?
+        if self.on_publish_error.respond_to?(:arity) && self.on_publish_error.arity == 1
+          self.on_publish_error.call(e)
+        else
+          self.on_publish_error.call(e, ev, hook)
+        end
       end
     end
 
     # Register a hook to be called when an event is sent.
+    # If a subscriber errors, on_publish_error is called with the exception, event, and subscriber.
     def register_subscriber(&block)
       raise LocalJumpError, "no block given" unless block
       self.log nil, :info, "amigo_installed_subscriber", block: block
@@ -191,7 +208,15 @@ module Amigo
 
     def _subscriber(event)
       event_json = event.as_json
-      self.audit_logger_class.perform_async(event_json)
+      begin
+        self.audit_logger_class.perform_async(event_json)
+      rescue StandardError => e
+        # If the audit logger cannot perform, let's say because Redis is down,
+        # we can run the job manually. This is pretty important for anything used for auditing;
+        # it should be as resilient as possible.
+        self.log(nil, :error, "amigo_audit_log_subscriber_error", error: e, event: event_json)
+        self.audit_logger_class.new.perform(event_json)
+      end
       self.router_class.perform_async(event_json)
     end
 
@@ -272,7 +297,6 @@ Amigo.reset_logging
 Amigo.synchronous_mode = false
 Amigo.registered_jobs = []
 Amigo.subscribers = Set.new
-Amigo.on_publish_error = proc {}
 
 require "amigo/audit_logger"
 require "amigo/router"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Lithic Technology
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-05 00:00:00.000000000 Z
+date: 2022-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -133,6 +147,7 @@ extra_rdoc_files: []
 files:
 - lib/amigo.rb
 - lib/amigo/audit_logger.rb
+- lib/amigo/autoscaler.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
 - lib/amigo/queue_backoff_job.rb