sidekiq 6.5.10 → 8.1.7

data/lib/sidekiq/client.rb

@@ -21,7 +21,6 @@ module Sidekiq
     # Sidekiq.client_middleware but you can change as necessary.
     #
     def middleware(&block)
-      @chain ||= Sidekiq.client_middleware
       if block
         @chain = @chain.dup
         yield @chain
@@ -31,18 +30,47 @@ module Sidekiq
 
     attr_accessor :redis_pool
 
-    # Sidekiq::Client normally uses the default Redis pool but you may
-    # pass a custom ConnectionPool if you want to shard your
-    # Sidekiq jobs across several Redis instances (for scalability
-    # reasons, e.g.)
+    # Sidekiq::Client is responsible for pushing job payloads to Redis.
+    # Requires the :pool or :config keyword argument.
     #
-    #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
+    #   Sidekiq::Client.new(pool: Sidekiq::RedisConnection.create)
     #
-    # Generally this is only needed for very large Sidekiq installs processing
-    # thousands of jobs per second.  I don't recommend sharding unless you
-    # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool = nil)
-      @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
+    # Inside the Sidekiq process, you can reuse the configured resources:
+    #
+    #   Sidekiq::Client.new(config: config)
+    #
+    # @param pool [ConnectionPool] explicit Redis pool to use
+    # @param config [Sidekiq::Config] use the pool and middleware from the given Sidekiq container
+    # @param chain [Sidekiq::Middleware::Chain] use the given middleware chain
+    def initialize(*args, **kwargs)
+      if args.size == 1 && kwargs.size == 0
+        warn "Sidekiq::Client.new(pool) is deprecated, please use Sidekiq::Client.new(pool: pool), #{caller(0..3)}"
+        # old calling method, accept 1 pool argument
+        @redis_pool = args[0]
+        @chain = Sidekiq.default_configuration.client_middleware
+        @config = Sidekiq.default_configuration
+      else
+        # new calling method: keyword arguments
+        @config = kwargs[:config] || Sidekiq.default_configuration
+        @redis_pool = kwargs[:pool] || Thread.current[:sidekiq_redis_pool] || @config&.redis_pool
+        @chain = kwargs[:chain] || @config&.client_middleware
+        raise ArgumentError, "No Redis pool available for Sidekiq::Client" unless @redis_pool
+      end
+    end
+
+    # Cancel the IterableJob with the given JID.
+    # **NB: Cancellation is asynchronous.** Iteration checks every
+    # five seconds so this will not immediately stop the given job.
+    def cancel!(jid)
+      key = "it-#{jid}"
+      _, result, _ = Sidekiq.redis do |c|
+        c.pipelined do |p|
+          p.hsetnx(key, "cancelled", Time.now.to_i)
+          p.hget(key, "cancelled")
+          p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
+        end
+      end
+      result.to_i
     end
 
     ##
@@ -53,6 +81,7 @@ module Sidekiq
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
+    #   retry_for - relative amount of time to retry this job if it fails, default nil
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
@@ -60,7 +89,7 @@ module Sidekiq
     #
     # Any options valid for a job class's sidekiq_options are also available here.
     #
-    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # All keys must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
     #
@@ -83,42 +112,76 @@ module Sidekiq
 
     ##
     # Push a large number of jobs to Redis. This method cuts out the redis
-    # network round trip latency.  I wouldn't recommend pushing more than
-    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # network round trip latency. It pushes jobs in batches if more than
+    # `:batch_size` (1000 by default) of jobs are passed. I wouldn't recommend making `:batch_size`
+    # larger than 1000 but YMMV based on network quality, size of job args, etc.
     # A large number of jobs can cause a bit of Redis command processing latency.
     #
+    # Accepts an `:at` option to schedule the jobs for future execution. It
+    # accepts either a single Numeric timestamp (or seconds-from-now) applied
+    # to every job, or an Array of Numeric values with the same size as `args`
+    # to schedule each job at its corresponding time.
+    #
+    # Accepts an additional `:spread_interval` option (in seconds) to randomly spread
+    # the jobs schedule times over the specified interval.
+    #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
-    # than the number given if the middleware stopped processing for one or more jobs.
+    # Returns an array of the of pushed jobs' jids, may contain nils if any client middleware
+    # prevented a job push.
+    #
+    # Example (pushing jobs in batches):
+    #   push_bulk('class' => MyJob, 'args' => (1..100_000).to_a, batch_size: 1_000)
+    #
     def push_bulk(items)
       args = items["args"]
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
-      return [] if args.empty? # no jobs to push
-
-      at = items.delete("at")
+      at = items.delete("at") || items.delete(:at)
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       jid = items.delete("jid")
       raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
+      spread_interval = items.delete(:spread_interval) || items.delete("spread_interval")
+      raise ArgumentError, "Jobs 'spread_interval' must be a positive Numeric" if spread_interval && (!spread_interval.is_a?(Numeric) || spread_interval <= 0)
+      raise ArgumentError, "Only one of 'at' or 'spread_interval' can be provided" if at && spread_interval
+
+      if !at && spread_interval
+        # Do not use spread interval smaller than pooling interval.
+        spread_interval = [spread_interval, 5].max
+        now = Time.now.to_f
+        at = args.map { now + rand * spread_interval }
+      end
+
+      # Use a smaller batch size by default for scheduled jobs since adding to sorted sets is more costly.
+      batch_size = items.delete(:batch_size) || items.delete("batch_size") || (at ? 100 : 1_000)
+
       normed = normalize_item(items)
-      payloads = args.map.with_index { |job_args, index|
-        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
-        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
-          verify_json(copy)
-          copy
-        end
-        result || nil
-      }.compact
+      slice_index = 0
+      result = args.each_slice(batch_size).flat_map do |slice|
+        raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless slice.is_a?(Array) && slice.all?(Array)
+        break [] if slice.empty? # no jobs to push
+
+        payloads = slice.map.with_index { |job_args, index|
+          copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+          copy["at"] = (at.is_a?(Array) ? at[slice_index + index] : at) if at
+          result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+            verify_json(copy)
+            copy
+          end
+          result || nil
+        }
+        slice_index += batch_size
+
+        to_push = payloads.compact
+        raw_push(to_push) unless to_push.empty?
+        payloads.map { |payload| payload&.[]("jid") }
+      end
 
-      raw_push(payloads) unless payloads.empty?
-      payloads.collect { |payload| payload["jid"] }
+      result.is_a?(Enumerator::Lazy) ? result.force : result
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -135,11 +198,11 @@ module Sidekiq
     # you cannot scale any other way (e.g. splitting your app into smaller apps).
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
-      current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      Thread.current[:sidekiq_via_pool] = pool
+      current_sidekiq_pool = Thread.current[:sidekiq_redis_pool]
+      Thread.current[:sidekiq_redis_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
+      Thread.current[:sidekiq_redis_pool] = current_sidekiq_pool
     end
 
     class << self
@@ -147,8 +210,8 @@ module Sidekiq
         new.push(item)
       end
 
-      def push_bulk(items)
-        new.push_bulk(items)
+      def push_bulk(...)
+        new.push_bulk(...)
       end
 
       # Resque compatibility helpers.  Note all helpers
@@ -201,14 +264,14 @@ module Sidekiq
           conn.pipelined do |pipeline|
             atomic_push(pipeline, payloads)
           end
-        rescue RedisConnection.adapter::BaseError => ex
+        rescue RedisClient::Error => ex
           # 2550 Failover can cause the server to become a replica, need
           # to disconnect and reopen the socket to get back to the primary.
           # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
           # 4985 Use the same logic when a blocking command is force-unblocked
           # The retry logic is copied from sidekiq.rb
           if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
-            conn.disconnect!
+            conn.close
             retryable = false
             retry
           end
@@ -221,18 +284,22 @@ module Sidekiq
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
         conn.zadd("schedule", payloads.flat_map { |hash|
-          at = hash.delete("at").to_s
+          at = hash["at"].to_s
+          # ActiveJob sets enqueued_at but the job has not been enqueued yet
+          hash = hash.except("enqueued_at", "at")
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        queue = payloads.first["queue"]
-        now = Time.now.to_f
-        to_push = payloads.map { |entry|
-          entry["enqueued_at"] = now
-          Sidekiq.dump_json(entry)
-        }
-        conn.sadd("queues", [queue])
-        conn.lpush("queue:#{queue}", to_push)
+        now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
+        grouped_queues = payloads.group_by { |job| job["queue"] }
+        conn.sadd("queues", grouped_queues.keys)
+        grouped_queues.each do |queue, grouped_payloads|
+          to_push = grouped_payloads.map { |entry|
+            entry["enqueued_at"] = now
+            Sidekiq.dump_json(entry)
+          }
+          conn.lpush("queue:#{queue}", to_push)
+        end
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -1,9 +1,39 @@
+# frozen_string_literal: true
+
 module Sidekiq
+  # Ruby's default thread priority is 0, which uses 100ms time slices.
+  # This can lead to some surprising thread starvation; if using a lot of
+  # CPU-heavy concurrency, it may take several seconds before a Thread gets
+  # on the CPU.
+  #
+  # Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
+  # With more frequent timeslices, we reduce the risk of unintentional timeouts
+  # and starvation.
+  #
+  # Customize like so:
+  #
+  #   Sidekiq.configure_server do |cfg|
+  #     cfg.thread_priority = 0
+  #   end
+  #
+  DEFAULT_THREAD_PRIORITY = -1
+
   ##
-  # Sidekiq::Component assumes a config instance is available at @config
+  # Sidekiq::Component provides a set of utility methods depending only
+  # on Sidekiq::Config. It assumes a config instance is available at @config.
   module Component # :nodoc:
     attr_reader :config
 
+    # This is epoch milliseconds, appropriate for persistence
+    def real_ms
+      ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
+    end
+
+    # used for time difference and relative comparisons, not persistence.
+    def mono_ms
+      ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
+    end
+
     def watchdog(last_words)
       yield
     rescue Exception => ex
@@ -11,11 +41,11 @@ module Sidekiq
       raise ex
     end
 
-    def safe_thread(name, &block)
+    def safe_thread(name, priority: nil, &block)
       Thread.new do
-        Thread.current.name = name
+        Thread.current.name = "sidekiq.#{name}"
         watchdog(name, &block)
-      end
+      end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
     end
 
     def logger
@@ -27,6 +57,9 @@ module Sidekiq
     end
 
     def tid
+      # We XOR with PID to ensure Thread IDs changes after fork.
+      # I'm unclear why we don't multiply the two values to better guarantee
+      # a unique value but it's been this way for quite a while now. #3685
       Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
     end
 
@@ -50,6 +83,7 @@ module Sidekiq
       oneshot = options.fetch(:oneshot, true)
       reverse = options[:reverse]
       reraise = options[:reraise]
+      logger.debug("Firing #{event} event") if oneshot
 
       arr = config[:lifecycle_events][event]
       arr.reverse! if reverse
@@ -61,5 +95,38 @@ module Sidekiq
       end
       arr.clear if oneshot # once we've fired an event, we never fire it again
     end
+
+    # When you have a large tree of components, the `inspect` output
+    # can get out of hand, especially with lots of Sidekiq::Config
+    # references everywhere. We avoid calling `inspect` on more complex
+    # state and use `to_s` instead to keep output manageable, #6553
+    def inspect
+      "#<#{self.class.name} #{
+        instance_variables.map do |name|
+          value = instance_variable_get(name)
+          case value
+          when Proc
+            "#{name}=#{value}"
+          when Sidekiq::Config
+            "#{name}=#{value}"
+          when Sidekiq::Component
+            "#{name}=#{value}"
+          else
+            "#{name}=#{value.inspect}"
+          end
+        end.join(", ")
+      }>"
+    end
+
+    def default_tag(dir = Dir.pwd)
+      name = File.basename(dir)
+      prevdir = File.dirname(dir) # Capistrano release directory?
+      if name.to_i != 0 && prevdir
+        if File.basename(prevdir) == "releases"
+          return File.basename(File.dirname(prevdir))
+        end
+      end
+      name
+    end
   end
 end

data/lib/sidekiq/config.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "sidekiq/redis_connection"
+
+module Sidekiq
+  # Sidekiq::Config represents the global configuration for an instance of Sidekiq.
+  class Config
+    extend Forwardable
+
+    DEFAULTS = {
+      labels: Set.new,
+      require: ".",
+      environment: nil,
+      concurrency: 5,
+      timeout: 25,
+      poll_interval_average: nil,
+      average_scheduled_poll_interval: 5,
+      on_complex_arguments: :raise,
+      # if the Iterable job runs longer than this value (in seconds), then the job
+      # will be interrupted after the current iteration and re-enqueued at the back of the queue
+      max_iteration_runtime: nil,
+      error_handlers: [],
+      death_handlers: [],
+      lifecycle_events: {
+        startup: [],
+        quiet: [],
+        shutdown: [],
+        exit: [],
+        # triggers when we fire the first heartbeat on startup OR repairing a network partition
+        heartbeat: [],
+        # triggers on EVERY heartbeat call, every 10 seconds
+        beat: []
+      },
+      dead_max_jobs: 10_000,
+      dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+      reloader: proc { |&block| block.call },
+      backtrace_cleaner: ->(backtrace) { backtrace },
+      logged_job_attributes: ["bid", "tags"],
+      redis_idle_timeout: nil
+    }
+
+    ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
+      Sidekiq::Context.with(ctx) do
+        dev = cfg[:environment] == "development"
+        fancy = dev && $stdout.tty? # 🎩
+        # Weird logic here but we want to show the backtrace in local
+        # development or if verbose logging is enabled.
+        #
+        # `full_message` contains the error class, message and backtrace
+        # `detailed_message` contains the error class and message
+        #
+        # Absolutely terrible API names. Not useful at all to have two
+        # methods with similar but obscure names.
+        if dev || cfg.logger.debug?
+          cfg.logger.info { ex.full_message(highlight: fancy) }
+        else
+          cfg.logger.info { ex.detailed_message(highlight: fancy) }
+        end
+      end
+    }
+
+    def initialize(options = {})
+      @options = DEFAULTS.merge(options)
+      @options[:error_handlers] << ERROR_HANDLER if @options[:error_handlers].empty?
+      @directory = {}
+      @redis_config = {}
+      @capsules = {}
+    end
+
+    def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
+    attr_reader :capsules
+    attr_accessor :thread_priority
+
+    def inspect
+      "#<#{self.class.name} @options=#{
+        @options.except(:lifecycle_events, :reloader, :death_handlers, :error_handlers).inspect
+      }>"
+    end
+
+    def to_json(*)
+      Sidekiq.dump_json(@options)
+    end
+
+    # LEGACY: edits the default capsule
+    # config.concurrency = 5
+    def concurrency=(val)
+      default_capsule.concurrency = Integer(val)
+    end
+
+    def concurrency
+      default_capsule.concurrency
+    end
+
+    def total_concurrency
+      capsules.each_value.sum(&:concurrency)
+    end
+
+    # Edit the default capsule.
+    # config.queues = %w( high default low )                 # strict
+    # config.queues = %w( high,3 default,2 low,1 )           # weighted
+    # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+    #
+    # With weighted priority, queue will be checked first (weight / total) of the time.
+    # high will be checked first (3/6) or 50% of the time.
+    # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+    # are ridiculous and unnecessarily expensive. You can get random queue ordering
+    # by explicitly setting all weights to 1.
+    def queues=(val)
+      default_capsule.queues = val
+    end
+
+    def queues
+      default_capsule.queues
+    end
+
+    def client_middleware
+      @client_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @client_chain if block_given?
+      @client_chain
+    end
+
+    def server_middleware
+      @server_chain ||= Sidekiq::Middleware::Chain.new(self)
+      yield @server_chain if block_given?
+      @server_chain
+    end
+
+    def default_capsule(&block)
+      capsule("default", &block)
+    end
+
+    # register a new queue processing subsystem
+    def capsule(name)
+      nm = name.to_s
+      cap = @capsules.fetch(nm) do
+        cap = Sidekiq::Capsule.new(nm, self)
+        @capsules[nm] = cap
+      end
+      yield cap if block_given?
+      cap
+    end
+
+    # All capsules must use the same Redis configuration
+    def redis=(hash)
+      @redis_config = @redis_config.merge(hash)
+    end
+
+    def reap_idle_redis_connections(timeout = 60)
+      self[:redis_idle_timeout] = timeout
+    end
+
+    def redis_pool
+      Thread.current[:sidekiq_redis_pool] || Thread.current[:sidekiq_capsule]&.redis_pool || local_redis_pool
+    end
+
+    def local_redis_pool
+      # this is our internal client/housekeeping pool. each capsule has its
+      # own pool for executing threads.
+      @redis ||= new_redis_pool(10, "internal")
+    end
+
+    def new_redis_pool(size, name = "unset")
+      # connection pool is lazy, it will not create connections unless you actually need them
+      # so don't be skimpy!
+      RedisConnection.create({size: size, logger: logger, pool_name: name}.merge(@redis_config))
+    end
+
+    def redis_info
+      redis do |conn|
+        conn.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
+      rescue RedisClientAdapter::CommandError => ex
+        # 2850 return fake version when INFO command has (probably) been renamed
+        raise unless /unknown command/.match?(ex.message)
+        {
+          "redis_version" => "9.9.9",
+          "uptime_in_days" => "9999",
+          "connected_clients" => "9999",
+          "used_memory_human" => "9P",
+          "used_memory_peak_human" => "9P"
+        }.freeze
+      end
+    end
+
+    def redis
+      raise ArgumentError, "requires a block" unless block_given?
+      redis_pool.with do |conn|
+        retryable = true
+        begin
+          yield conn
+        rescue RedisClientAdapter::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The same retry logic is also used in client.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.close
+            retryable = false
+            retry
+          end
+          raise
+        end
+      end
+    end
+
+    # register global singletons which can be accessed elsewhere
+    def register(name, instance)
+      # logger.debug("register[#{name}] = #{instance}")
+      # Sidekiq Enterprise lazy registers a few services so we
+      # can't lock down this hash completely.
+      hash = @directory.dup
+      hash[name] = instance
+      @directory = hash.freeze
+      instance
+    end
+
+    # find a singleton
+    def lookup(name, default_class = nil)
+      # JNDI is just a fancy name for a hash lookup
+      @directory.fetch(name) do |key|
+        return nil unless default_class
+        register(key, default_class.new(self))
+      end
+    end
+
+    def freeze!
+      @directory.freeze
+      @options.freeze
+      true
+    end
+
+    ##
+    # Death handlers are called when all retries for a job have been exhausted and
+    # the job dies.  It's the notification to your application
+    # that this job will not succeed without manual intervention.
+    #
+    # Sidekiq.configure_server do |config|
+    #   config.death_handlers << ->(job, ex) do
+    #   end
+    # end
+    def death_handlers
+      @options[:death_handlers]
+    end
+
+    # How frequently Redis should be checked by a random Sidekiq process for
+    # scheduled and retriable jobs. Each individual process will take turns by
+    # waiting some multiple of this value.
+    #
+    # See sidekiq/scheduled.rb for an in-depth explanation of this value
+    def average_scheduled_poll_interval=(interval)
+      @options[:average_scheduled_poll_interval] = interval
+    end
+
+    # Register a proc to handle any error which occurs within the Sidekiq process.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.error_handlers << proc {|ex,ctx_hash,config| MyErrorService.notify(ex, ctx_hash) }
+    #   end
+    #
+    # The default error handler logs errors to @logger.
+    def error_handlers
+      @options[:error_handlers]
+    end
+
+    # Register a block to run at a point in the Sidekiq lifecycle.
+    # :startup, :quiet, :shutdown, or :exit are valid events.
+    #
+    #   Sidekiq.configure_server do |config|
+    #     config.on(:shutdown) do
+    #       puts "Goodbye cruel world!"
+    #     end
+    #   end
+    def on(event, &block)
+      raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+      raise ArgumentError, "Invalid event name: #{event}" unless @options[:lifecycle_events].key?(event)
+      @options[:lifecycle_events][event] << block
+    end
+
+    def logger
+      @logger ||= Sidekiq::Logger.new($stdout, level: :info).tap do |log|
+        log.level = Logger::INFO
+        log.formatter = if ENV["DYNO"]
+          Sidekiq::Logger::Formatters::WithoutTimestamp.new
+        else
+          Sidekiq::Logger::Formatters::Pretty.new
+        end
+      end
+    end
+
+    def logger=(logger)
+      if logger.nil?
+        self.logger.level = Logger::FATAL
+        return
+      end
+
+      @logger = logger
+    end
+
+    # INTERNAL USE ONLY
+    def handle_exception(ex, ctx = {})
+      if @options[:error_handlers].empty?
+        logger.error { "No error handlers configured, logging exception directly" }
+        logger.error { ex }
+      end
+      @options[:error_handlers].each do |handler|
+        handler.call(ex, ctx, self)
+      rescue Exception => e
+        l = logger
+        l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+        l.error e
+        l.error e.backtrace.join("\n") unless e.backtrace.nil?
+      end
+    end
+  end
+end