sidekiq-amigo 1.1.2 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1ad3126cf6ef41459f944083e16b5c64a83a59e2df2617e238112b608c9d545c
-  data.tar.gz: c53b63fa0b89f7d5f8f7c84f9199ae23941aa400991d8835d0f11e272060fc13
+  metadata.gz: 1dc6d4ac6f9e7a89afecc711780ceeeb3ab1204435a19be2683623256b2abc49
+  data.tar.gz: 725dc4719b949d5f93bd84125ed6d2bea713be872f7e8413e6509d90c9090544
 SHA512:
-  metadata.gz: e4e05313b891b464942f591bf63ab199bc586dff134779581621364a93e928014bc2f96f5184d1df367d1f2ef1be7e4a36b5af12351daca6fc9e7da723577221
-  data.tar.gz: 53b8a94510e1396c5fc779d36c9eeccb379ca02b11f5b9cc71822565b0b88874c381fda32c854e9addc664f1c08ec46ebafc6c5c9833f2f566140ae6c56b0fed
+  metadata.gz: e1a5e2c7beab44fecabb7d7b7f9f610ce1138029eb247b46c04d90399550d976990f6db36717c73f225c195318ea303b1607984878167dfac4087024e809c735
+  data.tar.gz: 807e4c88665fd4ac7a6a4004e654de2a697b4ebd7c90546be7f965955ecd84690f2e0f5eebe913d31e2673f32f7bbd560e05695f79154284f3d7bd8a6ee77eb9

data/lib/amigo/audit_logger.rb

@@ -2,7 +2,7 @@
 
 require "amigo"
 
-class Amigo
+module Amigo
   class AuditLogger
     include Sidekiq::Worker
 

data/lib/amigo/deprecated_jobs.rb

@@ -10,7 +10,7 @@ require "amigo/job"
 # So, make the class exist, but noop so it won't be scheduled and won't be retried.
 # Then it can be deleted later.
 #
-class Amigo
+module Amigo
   module DeprecatedJobs
     def self.install(const_base, *names)
       cls = self.noop_class

data/lib/amigo/job.rb

@@ -4,13 +4,14 @@ require "sidekiq"
 
 require "amigo"
 
-class Amigo
+module Amigo
   module Job
     def self.extended(cls)
       cls.include(Sidekiq::Worker)
       cls.extend(ClassMethods)
       cls.pattern = ""
       cls.include(InstanceMethods)
+      Amigo.register_job(cls)
     end
 
     module InstanceMethods

data/lib/amigo/queue_backoff_job.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+require "sidekiq/api"
+
+# Queue backoff jobs are used for jobs that should not saturate workers,
+# such that jobs on dependent queues end up not running for a while.
+#
+# For example, imagine a queue dedicated to long-running jobs ('slow'),
+# a queue of critical, short-running tasks ('critical'), and 10 worker threads.
+# Imagine 20 'slow' jobs enter that queue, then 2 'critical' jobs.
+#
+# The 10 worker threads start processing the 'slow' queue,
+# and one completes. When that worker thread goes to find its next job,
+# it pulls a job off the 'slow' queue (even with Sidekiq queue priorities,
+# lopsided queue sizes mean it's likely we'll ge ta 'slow' job).
+#
+# When this job starts, it checks the 'critical' queue,
+# which is specified as a *dependent* queue of this job.
+# If it sees the 'critical' queue has latency,
+# the job reschedules itself in the future and then processes the next job.
+#
+# This keeps happening until the worker thread finds a job from
+# the 'critical' queue and processes it successfully.
+#
+# Implementers can override two methods:
+#
+# - `dependent_queues` should return an array of the names of queues that should be checked,
+#   in order of higher-priority-first. See below for Redis performance notes.
+# - `calculate_backoff` is passed a queue name and its latency,
+#   and should return either:
+#   - the backoff duration in seconds (ie the argument to `perform_in`),
+#   - 0 to perform the job immediately, or
+#   - nil to check the next queue with latency.
+#   - Note that if all calls to `calculate_backoff` return nil, the job is performed immediately.
+#
+# BackoffJob supports multiple dependent queues but it checks them one-at-a-time
+# to avoid any unnecessary calls to Redis.
+#
+# == Redis Impacts
+#
+# Using BackoffJob adds an overhead to each perform of a job-
+# specifically, a call to `Redis.lrange` through the Sidekiq API's Sidekiq:::Queue#latency
+# potentially for each queue in `dependent_queues`.
+# This is a fast call (it just gets the last item), but it's not free,
+# so users should be aware of it.
+#
+module Amigo
+  module QueueBackoffJob
+    def self.included(cls)
+      cls.include InstanceMethods
+      cls.prepend PrependedMethods
+    end
+
+    class << self
+      # Reset class state. Mostly used just for testing.
+      def reset
+        @max_backoff = 10
+        is_testing = defined?(::Sidekiq::Testing) && ::Sidekiq::Testing.enabled?
+        @enabled = !is_testing
+        @cache_queue_names = true
+        @cache_latencies = true
+        @all_queue_names = nil
+        @latency_cache_duration = 5
+        @latency_cache = {}
+      end
+
+      # Maximum time into the future a job will reschedule itself for.
+      # Ie, if latency is 30s, and max_backoff is 10, the job will be scheduled
+      # for 10s into the future if it finds backoff pressure.
+      attr_accessor :max_backoff
+
+      # Return true if backoff checks are enabled.
+      attr_accessor :enabled
+
+      def enabled?
+        return @enabled
+      end
+
+      # Cached value of all Sidekiq queues, since they rarely change.
+      # If your queue names change at runtime, set +cache_queue_names+ to false.
+      def all_queue_names
+        return @all_queue_names if @cache_queue_names && @all_queue_names
+        @all_queue_names = ::Sidekiq::Queue.all.map(&:name)
+        return @all_queue_names
+      end
+
+      # Whether all_queue_names should be cached.
+      attr_reader :cache_queue_names
+
+      def cache_queue_names=(v)
+        @cache_queue_names = v
+        @all_queue_names = nil if v == false
+      end
+
+      # Return how long queue latencies should be cached before they are re-fetched from Redis.
+      # Avoids hitting Redis to check latency too often.
+      # Default to 5 seconds. Set to 0 to avoid caching.
+      attr_accessor :latency_cache_duration
+
+      # Check the latency of the queue with the given now.
+      # If the queue has been checked more recently than latency_cache_duration specified,
+      # return the cached value.
+      def check_latency(qname, now: Time.now)
+        return ::Sidekiq::Queue.new(qname).latency if self.latency_cache_duration.zero?
+        cached = @latency_cache[qname]
+        if cached.nil? || (cached[:at] + self.latency_cache_duration) < now
+          @latency_cache[qname] = {at: now, value: ::Sidekiq::Queue.new(qname).latency}
+        end
+        return @latency_cache[qname][:value]
+      end
+    end
+    self.reset
+
+    module InstanceMethods
+      def dependent_queues
+        qname = self.class.get_sidekiq_options["queue"]
+        return ::Amigo::QueueBackoffJob.all_queue_names.reject { |x| x == qname }
+      end
+
+      def calculate_backoff(_queue_name, latency, _args)
+        return [latency, ::Amigo::QueueBackoffJob.max_backoff].min
+      end
+    end
+
+    module PrependedMethods
+      def perform(*args)
+        return super unless ::Amigo::QueueBackoffJob.enabled?
+        # rubocop:disable Style/GuardClause, Lint/NonLocalExitFromIterator
+        dependent_queues.each do |qname|
+          latency = Amigo::QueueBackoffJob.check_latency(qname)
+          # If latency is <= 0, we can skip this queue.
+          next unless latency.positive?
+          # If backoff is nil, ignore this queue and check the next
+          # If it's > 0, defer until the future
+          # If it's <= 0, run the job and check no more queues
+          backoff = calculate_backoff(qname, latency, args)
+          next if backoff.nil?
+          if backoff.positive?
+            self.class.perform_in(backoff, *args)
+            return
+          else
+            return super
+          end
+        end
+        # rubocop:enable Style/GuardClause, Lint/NonLocalExitFromIterator
+        super
+      end
+    end
+  end
+end

data/lib/amigo/rate_limited_error_handler.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "digest"
+
+# Wrap another Sidekiq error handler so invoking it is rate limited.
+#
+# Useful when wrapping a usage-based error reporter like Sentry,
+# which can be hammered in the case of an issue like connectivity
+# that causes all jobs and retries to fail.
+# It is suggested that all errors are still reported to something
+# like application logs, since entirely silencing errors
+# can make debugging problems tricky.
+#
+# Usage:
+#
+#   Sidekiq.configure_server do |config|
+#     config.error_handlers << Amigo::RateLimitedErrorHandler.new(
+#       Sentry::Sidekiq::ErrorHandler.new,
+#       sample_rate: ENV.fetch('ASYNC_ERROR_RATE_LIMITER_SAMPLE_RATE', '0.5').to_f,
+#       ttl: ENV.fetch('ASYNC_ERROR_RATE_LIMITER_TTL', '120').to_f,
+#     )
+#   end
+#
+# See notes about +sample_rate+ and +ttl+,
+# and +fingerprint+ for how exceptions are fingerprinted for uniqueness.
+#
+# Rate limiting is done in-memory so is unique across the entire process-
+# threads/workers share rate limiting, but multiple processes do not.
+# So if 2 processes have 10 threads each,
+# the error handler would be invoked twice if they all error
+# for the same reason.
+#
+# Thread-based limiting (20 errors in the case above)
+# or cross-process limiting (1 error in the case above)
+# can be added in the future.
+module Amigo
+  class RateLimitedErrorHandler
+    # The error handler that will be called to report the error.
+    attr_reader :wrapped
+
+    # After the first error with a fingerprint is seen,
+    # how many future errors with the same fingerprint should we sample,
+    # until the fingerprint expires +ttl+ after the first error?
+    # Use 1 to called the wrapped handler on all errors with the same fingerprint,
+    # and 0 to never call the wrapped handler on those errors until ttl has elapsed.
+    attr_reader :sample_rate
+
+    # How long does the fingerprint live for an error?
+    # For example, with a sample rate of 0 and a ttl of 2 minutes,
+    # the rate will be at most one of the same error every 2 minutes;
+    # the error is always sent when the key is set; then no events are sent until the key expires.
+    #
+    # Note that, unlike Redis TTL, the ttl is set only when the error is first seen
+    # (and then after it's seen once the fingerprint expires);
+    # this means that, if an error is seen once a minute, with a TTL of 2 minutes,
+    # even with a sample rate of 0, an error is recorded every 2 minutes,
+    # rather than just once and never again.
+    attr_reader :ttl
+
+    def initialize(wrapped, sample_rate: 0.1, ttl: 120)
+      @mutex = Mutex.new
+      @wrapped = wrapped
+      @sample_rate = sample_rate
+      @inverse_sample_rate = 1 - @sample_rate
+      @ttl = ttl
+      # Key is fingerprint, value is when to expire
+      @store = {}
+      # Add some fast-paths to handle 0 and 1 sample rates.
+      @call = if sample_rate == 1
+                ->(*a) { @wrapped.call(*a) }
+      elsif sample_rate.zero?
+        self.method(:call_zero)
+      else
+        self.method(:call_sampled)
+      end
+    end
+
+    def call(ex, context)
+      @call[ex, context]
+    end
+
+    private def call_zero(ex, context)
+      call_impl(ex, context) { false }
+    end
+
+    private def call_sampled(ex, context)
+      call_impl(ex, context) { rand <= @sample_rate }
+    end
+
+    private def call_impl(ex, context)
+      now = Time.now
+      invoke = @mutex.synchronize do
+        @store.delete_if { |_sig, t| t < now }
+        fingerprint = self.fingerprint(ex)
+        if @store.key?(fingerprint)
+          yield
+        else
+          @store[fingerprint] = now + @ttl
+          true
+        end
+      end
+      @wrapped.call(ex, context) if invoke
+    end
+
+    # Fingerprint an exception.
+    # - No two exceptions with the same class can be the same.
+    # - If an exception has no backtrace (it was manually constructed),
+    #   the identity of the exception instance (object_id) is the fingerprint.
+    # - If an exception has a backtrace,
+    #   the md5 of the backtrace is the fingerprint.
+    def fingerprint(ex)
+      md5 = Digest::MD5.new
+      md5.update ex.class.to_s
+      if ex.backtrace.nil?
+        md5.update ex.object_id.to_s
+      else
+        ex.backtrace.each { |line| md5.update(line) }
+      end
+      md5.update(self.fingerprint(ex.cause)) if ex.cause
+      return md5.hexdigest
+    end
+  end
+end

data/lib/amigo/retry.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+# Middleware so Sidekiq workers can use a custom retry logic.
+# See +Amigo::Retry::Retry+, +Amigo::Retry::Die+,
+# and +Amigo::Retry::OrDie+ for more details
+# on how these should be used.
+#
+# NOTE: You MUST register +Amigo::Retry::ServerMiddleware+,
+# and you SHOULD increase the size of the dead set if you are relying on 'die' behavior:
+#
+#   Sidekiq.configure_server do |config|
+#     config.options[:dead_max_jobs] = 999_999_999
+#     config.server_middleware.add(Amigo::Retry::ServerMiddleware)
+#   end
+module Amigo
+  module Retry
+    class Error < StandardError; end
+
+    # Raise this class, or a subclass of it, to schedule a later retry,
+    # rather than using an error to trigger Sidekiq's default retry behavior.
+    # The benefit here is that it allows a consistent, customizable behavior,
+    # so is better for 'expected' errors like rate limiting.
+    class Retry < Error
+      attr_accessor :interval_or_timestamp
+
+      def initialize(interval_or_timestamp, msg=nil)
+        @interval_or_timestamp = interval_or_timestamp
+        super(msg || "retry job in #{interval_or_timestamp}")
+      end
+    end
+
+    # Raise this class, or a subclass of it, to send the job to the DeadSet,
+    # rather than going through Sidekiq's retry mechanisms.
+    # This allows jobs to hard-fail when there is something like a total outage,
+    # rather than retrying.
+    class Die < Error
+    end
+
+    # Raise this class, or a subclass of it, to:
+    # - Use +Retry+ exception semantics while the current attempt is <= +attempts+, or
+    # - Use +Die+ exception semantics if the current attempt is > +attempts+.
+    class OrDie < Error
+      attr_reader :attempts, :interval_or_timestamp
+
+      def initialize(attempts, interval_or_timestamp, msg=nil)
+        @attempts = attempts
+        @interval_or_timestamp = interval_or_timestamp
+        super(msg || "retry every #{interval_or_timestamp} up to #{attempts} times")
+      end
+    end
+
+    class ServerMiddleware
+      def call(worker, job, _queue)
+        yield
+      rescue Amigo::Retry::Retry => e
+        handle_retry(worker, job, e)
+      rescue Amigo::Retry::Die => e
+        handle_die(worker, job, e)
+      rescue Amigo::Retry::OrDie => e
+        handle_retry_or_die(worker, job, e)
+      end
+
+      def handle_retry(worker, job, e)
+        Sidekiq.logger.info("scheduling_retry")
+        self.amigo_retry_in(worker.class, job, e.interval_or_timestamp)
+      end
+
+      def handle_die(_worker, job, _e)
+        Sidekiq.logger.warn("sending_to_deadset")
+        payload = Sidekiq.dump_json(job)
+        Sidekiq::DeadSet.new.kill(payload, notify_failure: false)
+      end
+
+      def handle_retry_or_die(worker, job, e)
+        retry_count = job.fetch("retry_count", 0)
+        if retry_count <= e.attempts
+          handle_retry(worker, job, e)
+        else
+          handle_die(worker, job, e)
+        end
+      end
+
+      def amigo_retry_in(worker_class, item, interval)
+        # pulled from perform_in
+        int = interval.to_f
+        now = Time.now.to_f
+        ts = (int < 1_000_000_000 ? now + int : int)
+        item["at"] = ts if ts > now
+        item["retry_count"] = item.fetch("retry_count", 0) + 1
+        worker_class.client_push(item)
+      end
+    end
+  end
+end

data/lib/amigo/router.rb

@@ -4,7 +4,7 @@ require "sidekiq"
 
 require "amigo"
 
-class Amigo
+module Amigo
   class Router
     include Sidekiq::Worker
 

data/lib/amigo/scheduled_job.rb

@@ -5,7 +5,7 @@ require "sidekiq-cron"
 
 require "amigo"
 
-class Amigo
+module Amigo
   module ScheduledJob
     def self.extended(cls)
       cls.include(Sidekiq::Worker)
@@ -13,6 +13,7 @@ class Amigo
       cls.extend(ClassMethods)
       cls.splay_duration = 30
       cls.include(InstanceMethods)
+      Amigo.register_job(cls)
     end
 
     module InstanceMethods

data/lib/amigo/semaphore_backoff_job.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "sidekiq"
+
+module Amigo
+  # This is a placeholder until it's migrated to Amigo proper
+end
+
+# Semaphore backoff jobs can reschedule themselves to happen at a later time
+# if there is too high a contention on a semaphore.
+# Ie, if there are too many jobs with the same key,
+# they start to reschedule themselves for the future.
+#
+# This is useful when a certain job (or job for a certain target)
+# can be slow so should not consume all available resources.
+#
+# In general, you should not use semaphore backoff jobs for singletons,
+# as the guarantees are not strong enough.
+# It is useful for many rapid jobs.
+#
+# Implementers must override the following methods:
+#
+# - `semaphore_key` must return the Redis key to use as the semaphore.
+#   This may be something like "sbj-user-1" to limit jobs for user 1.
+# - `semaphore_size` is the number of concurrent jobs.
+#   Returning 5 would mean at most 5 jobs could be running at a time.
+#
+# And may override the following methods:
+#
+# - `semaphore_backoff` is called to know when to schedule the backoff retry.
+#   By default, it is 10 seconds, plus between 0 and 10 seconds more,
+#   so the job will be retried in between 10 and 20 seconds.
+#   Return whatever you want for the backoff.
+# - `semaphore_expiry` should return the TTL of the semaphore key.
+#   Defaults to 30 seconds. See below for key expiry and negative semaphore value details.
+# - `before_perform` is called before calling the `perform` method.
+#   This is required so that implementers can set worker state, based on job arguments,
+#   that can be used for calculating the semaphore key.
+#
+# Note that we give the semaphore key an expiry. This is to avoid situation where
+# jobs are killed, the decrement is not done, and the counter increases to the point we
+# have fewer than the expected number of jobs running.
+#
+# This does mean that, when a job runs longer than the semaphore expiry,
+# another worker can be started, which would increment the counter back to 1.
+# When the original job ends, the counter would be 0; then when the new job ends,
+# the counter would be -1. To avoid negative counters (which create the same issue
+# around missing decrements), if we ever detect a negative 'jobs running',
+# we warn and remove the key entirely.
+#
+module Amigo
+  module SemaphoreBackoffJob
+    def self.included(cls)
+      cls.include InstanceMethods
+      cls.prepend PrependedMethods
+    end
+
+    class << self
+      # Reset class state. Mostly used just for testing.
+      def reset
+        is_testing = defined?(::Sidekiq::Testing) && ::Sidekiq::Testing.enabled?
+        @enabled = !is_testing
+      end
+
+      # Return true if backoff checks are enabled.
+      attr_accessor :enabled
+
+      def enabled?
+        return @enabled
+      end
+    end
+
+    self.reset
+
+    module InstanceMethods
+      def semaphore_key
+        raise NotImplementedError, "must be implemented on worker"
+      end
+
+      def semaphore_size
+        raise NotImplementedError, "must be implemented on worker"
+      end
+
+      def semaphore_backoff
+        return 10 + (rand * 10)
+      end
+
+      def semaphore_expiry
+        return 30
+      end
+    end
+
+    module PrependedMethods
+      def perform(*args)
+        self.before_perform(*args) if self.respond_to?(:before_perform)
+        return super unless ::Amigo::SemaphoreBackoffJob.enabled?
+        key = self.semaphore_key
+        size = self.semaphore_size
+        # Create a simple counter for the semaphore key.
+        # Always increment; also set an expiration if this is the first job.
+        # If we need to retry later, make sure we decrement, then schedule for the future.
+        # If we run it now, decrement the counter afterwards.
+        # If some corruption results in a negative number of jobs in the semaphore,
+        # we can delete the key and get back to a default state
+        # (this can cause problems but the idea is that
+        # we should run at least the configured number of jobs,
+        # and eventually the semaphore key will expire/get rebalanced).
+        jobs_in_semaphore = Sidekiq.redis do |conn|
+          cnt = conn.incr(key)
+          conn.expire(key, self.semaphore_expiry) if cnt == 1
+          cnt
+        end
+        if jobs_in_semaphore > size
+          Sidekiq.redis { |conn| conn.decr(key) }
+          backoff = self.semaphore_backoff
+          self.class.perform_in(backoff, *args)
+          return
+        end
+        begin
+          super
+        ensure
+          Sidekiq.redis do |conn|
+            new_job_count = conn.decr(key)
+            if new_job_count.negative?
+              conn.del(key)
+              Sidekiq.logger.warn("negative_semaphore_backoff_job_count")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/amigo/spec_helpers.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
 
 require "amigo"
+require "sidekiq/worker"
 
-class Amigo
+module Amigo
   module SpecHelpers
     def self.included(context)
       context.before(:each) do |example|
@@ -121,7 +122,7 @@ class Amigo
 
         @missing.each do |event, payload|
           message = "expected a '%s' event to be fired" % [event]
-          message << " with a payload of %p" % [payload] unless payload.nil?
+          message << (" with a payload of %p" % [payload]) unless payload.nil?
           message << " but none was."
 
           messages << message
@@ -131,7 +132,7 @@ class Amigo
           messages << "No events were sent."
         else
           parts = @recorded_events.map(&:inspect)
-          messages << "The following events were recorded: %s" % [parts.join(", ")]
+          messages << ("The following events were recorded: %s" % [parts.join(", ")])
         end
 
         return messages.join("\n")
@@ -141,7 +142,7 @@ class Amigo
         messages = []
         @matched.each do |event, _payload|
           message = "expected a '%s' event not to be fired" % [event]
-          message << " with a payload of %p" % [@expected_payload] if @expected_payload
+          message << (" with a payload of %p" % [@expected_payload]) if @expected_payload
           message << " but one was."
           messages << message
         end
@@ -229,5 +230,73 @@ class Amigo
     def perform_async_job(job)
       return PerformAsyncJobMatcher.new(job)
     end
+
+    # Like a Sidekiq worker's perform_inline,
+    # but allows an arbitrary item to be used, rather than just the
+    # given class and args. For example, when testing,
+    # you may need to assume something like 'retry_count' is in the job payload,
+    # but that can't be included with perform_inline.
+    # This allows those arbitrary job payload fields
+    # to be included when the job is run.
+    module_function def sidekiq_perform_inline(klass, args, item=nil)
+      Sidekiq::Worker::Setter.override_item = item
+      begin
+        klass.perform_inline(*args)
+      ensure
+        Sidekiq::Worker::Setter.override_item = nil
+      end
+    end
+
+    module_function def drain_sidekiq_jobs(q)
+      all_sidekiq_jobs(q).each do |job|
+        klass = job.item.fetch("class")
+        klass = Sidekiq::Testing.constantize(klass) if klass.is_a?(String)
+        sidekiq_perform_inline(klass, job.item["args"], job.item)
+        job.delete
+      end
+    end
+
+    module_function def all_sidekiq_jobs(q)
+      arr = []
+      q.each { |j| arr << j }
+      return arr
+    end
+
+    # Use this middleware to pass an arbitrary callback evaluated before a job runs.
+    # Make sure to call +reset+ after the test.
+    class ServerCallbackMiddleware
+      class << self
+        attr_accessor :callback
+      end
+
+      def self.reset
+        self.callback = nil
+        return self
+      end
+
+      def self.new
+        return self
+      end
+
+      def self.call(worker, job, queue)
+        self.callback[worker, job, queue] if self.callback
+        yield
+      end
+    end
+  end
+end
+
+module ::Sidekiq
+  module Worker
+    class Setter
+      class << self
+        attr_accessor :override_item
+      end
+      def normalize_item(item)
+        result = super
+        result.merge!(self.class.override_item || {})
+        return result
+      end
+    end
   end
 end

data/lib/amigo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-class Amigo
-  VERSION = "1.1.2"
+module Amigo
+  VERSION = "1.2.1"
 end

data/lib/amigo.rb

@@ -97,18 +97,21 @@ require "sidekiq-cron"
 # Splay exists to avoid a "thundering herd" issue.
 # Splay defaults to 30s; you may wish to always provide splay, whatever you think for your job.
 #
-class Amigo
+module Amigo
   class Error < StandardError; end
 
   class StartSchedulerFailed < Error; end
 
   class << self
-    attr_accessor :structured_logging
+    attr_accessor :structured_logging, :audit_logger_class, :router_class
 
     # Proc called with [job, level, message, params].
     # By default, logs to the job's logger (or Sidekiq's if job is nil).
-    # If structured_logging is true, the message will be an 'event' without any dynamic info,
-    # if false, the params will be rendered into the message so are suitable for unstructured logging.
+    # If structured_logging is true, the message will be an 'event' string (like 'registered_subscriber')
+    # without any dynamic info.
+    # If structured_logging is false, the params will be rendered into the message
+    # so are suitable for unstructured logging. Also, the params will also have an :log_message key
+    # which will contain the original log message.
     attr_accessor :log_callback
 
     def reset_logging
@@ -118,8 +121,9 @@ class Amigo
 
     def log(job, level, message, params)
       params ||= {}
-      if self.structured_logging && !params.empty?
+      if !self.structured_logging && !params.empty?
         paramstr = params.map { |k, v| "#{k}=#{v}" }.join(" ")
+        params[:log_message] = message
         message = "#{message} #{paramstr}"
       end
       self.log_callback[job, level, message, params]
@@ -180,8 +184,6 @@ class Amigo
     # Install Amigo so that every publish will be sent to the AuditLogger job
     # and will invoke the relevant jobs in registered_jobs via the Router job.
     def install_amigo_jobs
-      require "amigo/audit_logger"
-      require "amigo/router"
       return self.register_subscriber do |ev|
         self._subscriber(ev)
       end
@@ -189,12 +191,13 @@ class Amigo
 
     def _subscriber(event)
       event_json = event.as_json
-      Amigo::AuditLogger.perform_async(event_json)
-      Amigo::Router.perform_async(event_json)
+      self.audit_logger_class.perform_async(event_json)
+      self.router_class.perform_async(event_json)
     end
 
     def register_job(job)
       self.registered_jobs << job
+      self.registered_jobs.uniq!
     end
 
     # Start the scheduler.
@@ -276,3 +279,8 @@ Amigo.synchronous_mode = false
 Amigo.registered_jobs = []
 Amigo.subscribers = Set.new
 Amigo.on_publish_error = proc {}
+
+require "amigo/audit_logger"
+require "amigo/router"
+Amigo.audit_logger_class = Amigo::AuditLogger
+Amigo.router_class = Amigo::Router

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-amigo
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.1
 platform: ruby
 authors:
 - Lithic Technology
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-23 00:00:00.000000000 Z
+date: 2022-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -135,14 +135,19 @@ files:
 - lib/amigo/audit_logger.rb
 - lib/amigo/deprecated_jobs.rb
 - lib/amigo/job.rb
+- lib/amigo/queue_backoff_job.rb
+- lib/amigo/rate_limited_error_handler.rb
+- lib/amigo/retry.rb
 - lib/amigo/router.rb
 - lib/amigo/scheduled_job.rb
+- lib/amigo/semaphore_backoff_job.rb
 - lib/amigo/spec_helpers.rb
 - lib/amigo/version.rb
 homepage: https://github.com/lithictech/sidekiq-amigo
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths: