sidekiq 3.5.4 → 5.2.7

data/lib/sidekiq/logging.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
 require 'time'
 require 'logger'
+require 'fcntl'
 
 module Sidekiq
   module Logging
@@ -9,7 +11,7 @@ module Sidekiq
 
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
-        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+        "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
       end
 
       def context
@@ -20,10 +22,26 @@ module Sidekiq
 
     class WithoutTimestamp < Pretty
       def call(severity, time, program_name, message)
-        "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+        "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
       end
     end
 
+    def self.tid
+      Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def self.job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      klass = job_hash['wrapped'] || job_hash["class"]
+      bid = job_hash['bid']
+      "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
+    end
+
+    def self.with_job_hash_context(job_hash, &block)
+      with_context(job_hash_context(job_hash), &block)
+    end
+
     def self.with_context(msg)
       Thread.current[:sidekiq_context] ||= []
       Thread.current[:sidekiq_context] << msg
@@ -46,7 +64,7 @@ module Sidekiq
     end
 
     def self.logger=(log)
-      @logger = (log ? log : Logger.new('/dev/null'))
+      @logger = (log ? log : Logger.new(File::NULL))
     end
 
     # This reopens ALL logfiles in the process that have been rotated

data/lib/sidekiq/manager.rb

@@ -1,156 +1,100 @@
-# encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq/util'
-require 'sidekiq/actor'
 require 'sidekiq/processor'
 require 'sidekiq/fetch'
+require 'thread'
+require 'set'
 
 module Sidekiq
 
   ##
-  # The main router in the system.  This
-  # manages the processor state and accepts messages
-  # from Redis to be dispatched to an idle processor.
+  # The Manager is the central coordination point in Sidekiq, controlling
+  # the lifecycle of the Processors.
+  #
+  # Tasks:
+  #
+  # 1. start: Spin up Processors.
+  # 3. processor_died: Handle job failure, throw away Processor, create new one.
+  # 4. quiet: shutdown idle Processors.
+  # 5. stop: hard stop the Processors by deadline.
+  #
+  # Note that only the last task requires its own Thread since it has to monitor
+  # the shutdown process.  The other tasks are performed by other threads.
   #
   class Manager
     include Util
-    include Actor
-    trap_exit :processor_died
 
-    attr_reader :ready
-    attr_reader :busy
-    attr_accessor :fetcher
+    attr_reader :workers
+    attr_reader :options
 
-    SPIN_TIME_FOR_GRACEFUL_SHUTDOWN = 1
-    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
-
-    def initialize(condvar, options={})
+    def initialize(options={})
       logger.debug { options.inspect }
       @options = options
-      @count = options[:concurrency] || 25
+      @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
-      @done_callback = nil
-      @finished = condvar
 
-      @in_progress = {}
-      @threads = {}
       @done = false
-      @busy = []
-      @ready = @count.times.map do
-        p = Processor.new_link(current_actor)
-        p.proxy_id = p.object_id
-        p
+      @workers = Set.new
+      @count.times do
+        @workers << Processor.new(self)
       end
+      @plock = Mutex.new
     end
 
-    def stop(options={})
-      watchdog('Manager#stop died') do
-        should_shutdown = options[:shutdown]
-        timeout = options[:timeout]
-
-        @done = true
-
-        logger.info { "Terminating #{@ready.size} quiet workers" }
-        @ready.each { |x| x.terminate if x.alive? }
-        @ready.clear
-
-        return if clean_up_for_graceful_shutdown
-
-        hard_shutdown_in timeout if should_shutdown
+    def start
+      @workers.each do |x|
+        x.start
       end
     end
 
-    def clean_up_for_graceful_shutdown
-      if @busy.empty?
-        shutdown
-        return true
-      end
+    def quiet
+      return if @done
+      @done = true
 
-      after(SPIN_TIME_FOR_GRACEFUL_SHUTDOWN) { clean_up_for_graceful_shutdown }
-      false
+      logger.info { "Terminating quiet workers" }
+      @workers.each { |x| x.terminate }
+      fire_event(:quiet, reverse: true)
     end
 
-    def start
-      @ready.each { dispatch }
-    end
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
 
-    def when_done(&blk)
-      @done_callback = blk
-    end
+    def stop(deadline)
+      quiet
+      fire_event(:shutdown, reverse: true)
 
-    def processor_done(processor)
-      watchdog('Manager#processor_done died') do
-        @done_callback.call(processor) if @done_callback
-        @in_progress.delete(processor.object_id)
-        @threads.delete(processor.object_id)
-        @busy.delete(processor)
-        if stopped?
-          processor.terminate if processor.alive?
-          shutdown if @busy.empty?
-        else
-          @ready << processor if processor.alive?
-        end
-        dispatch
-      end
-    end
+      # some of the shutdown events can be async,
+      # we don't have any way to know when they're done but
+      # give them a little time to take effect
+      sleep PAUSE_TIME
+      return if @workers.empty?
 
-    def processor_died(processor, reason)
-      watchdog("Manager#processor_died died") do
-        @in_progress.delete(processor.object_id)
-        @threads.delete(processor.object_id)
-        @busy.delete(processor)
-
-        unless stopped?
-          p = Processor.new_link(current_actor)
-          p.proxy_id = p.object_id
-          @ready << p
-          dispatch
-        else
-          shutdown if @busy.empty?
-        end
+      logger.info { "Pausing to allow workers to finish..." }
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if @workers.empty?
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
-    end
+      return if @workers.empty?
 
-    def assign(work)
-      watchdog("Manager#assign died") do
-        if stopped?
-          # Race condition between Manager#stop if Fetcher
-          # is blocked on redis and gets a message after
-          # all the ready Processors have been stopped.
-          # Push the message back to redis.
-          work.requeue
-        else
-          processor = @ready.pop
-          @in_progress[processor.object_id] = work
-          @busy << processor
-          processor.async.process(work)
-        end
-      end
+      hard_shutdown
     end
 
-    # A hack worthy of Rube Goldberg.  We need to be able
-    # to hard stop a working thread.  But there's no way for us to
-    # get handle to the underlying thread performing work for a processor
-    # so we have it call us and tell us.
-    def real_thread(proxy_id, thr)
-      @threads[proxy_id] = thr if thr.alive?
+    def processor_stopped(processor)
+      @plock.synchronize do
+        @workers.delete(processor)
+      end
     end
 
-    PROCTITLES = [
-      proc { 'sidekiq'.freeze },
-      proc { Sidekiq::VERSION },
-      proc { |mgr, data| data['tag'] },
-      proc { |mgr, data| "[#{mgr.busy.size} of #{data['concurrency']} busy]" },
-      proc { |mgr, data| "stopping" if mgr.stopped? },
-    ]
-
-    def heartbeat(key, data, json)
-      results = PROCTITLES.map {|x| x.(self, data) }
-      results.compact!
-      $0 = results.join(' ')
-
-      ❤(key, json)
-      after(5) do
-        heartbeat(key, data, json)
+    def processor_died(processor, reason)
+      @plock.synchronize do
+        @workers.delete(processor)
+        unless @done
+          p = Processor.new(self)
+          @workers << p
+          p.start
+        end
       end
     end
 
@@ -160,77 +104,34 @@ module Sidekiq
 
     private
 
-    def ❤(key, json)
-      begin
-        _, _, _, msg = Sidekiq.redis do |conn|
-          conn.multi do
-            conn.sadd('processes', key)
-            conn.hmset(key, 'info', json, 'busy', @busy.size, 'beat', Time.now.to_f)
-            conn.expire(key, 60)
-            conn.rpop("#{key}-signals")
-          end
-        end
-
-        return unless msg
-
-        if JVM_RESERVED_SIGNALS.include?(msg)
-          Sidekiq::CLI.instance.handle_signal(msg)
-        else
-          ::Process.kill(msg, $$)
-        end
-      rescue => e
-        # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+    def hard_shutdown
+      # We've reached the timeout and we still have busy workers.
+      # They must die but their jobs shall live on.
+      cleanup = nil
+      @plock.synchronize do
+        cleanup = @workers.dup
       end
-    end
 
-    def hard_shutdown_in(delay)
-      logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
+      if cleanup.size > 0
+        jobs = cleanup.map {|p| p.job }.compact
 
-      after(delay) do
-        watchdog("Manager#hard_shutdown_in died") do
-          # We've reached the timeout and we still have busy workers.
-          # They must die but their messages shall live on.
-          logger.warn { "Terminating #{@busy.size} busy worker threads" }
-          logger.warn { "Work still in progress #{@in_progress.values.inspect}" }
+        logger.warn { "Terminating #{cleanup.size} busy worker threads" }
+        logger.warn { "Work still in progress #{jobs.inspect}" }
 
-          requeue
-
-          @busy.each do |processor|
-            if processor.alive? && t = @threads.delete(processor.object_id)
-              t.raise Shutdown
-            end
-          end
-
-          @finished.signal
-        end
+        # Re-enqueue unfinished jobs
+        # NOTE: You may notice that we may push a job back to redis before
+        # the worker thread is terminated. This is ok because Sidekiq's
+        # contract says that jobs are run AT LEAST once. Process termination
+        # is delayed until we're certain the jobs are back in Redis because
+        # it is worse to lose a job than to run it twice.
+        strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+        strategy.bulk_requeue(jobs, @options)
       end
-    end
-
-    def dispatch
-      return if stopped?
-      # This is a safety check to ensure we haven't leaked
-      # processors somehow.
-      raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
-      raise "No ready processor!?" if @ready.empty?
 
-      @fetcher.async.fetch
-    end
-
-    def shutdown
-      requeue
-      @finished.signal
+      cleanup.each do |processor|
+        processor.kill
+      end
     end
 
-    def requeue
-      # Re-enqueue terminated jobs
-      # NOTE: You may notice that we may push a job back to redis before
-      # the worker thread is terminated. This is ok because Sidekiq's
-      # contract says that jobs are run AT LEAST once. Process termination
-      # is delayed until we're certain the jobs are back in Redis because
-      # it is worse to lose a job than to run it twice.
-      Sidekiq::Fetcher.strategy.bulk_requeue(@in_progress.values, @options)
-      @in_progress.clear
-    end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack

data/lib/sidekiq/middleware/i18n.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 #
 # Simple middleware to save the current locale and restore it when the job executes.
 # Use it by requiring it in your initializer:

data/lib/sidekiq/middleware/server/active_record.rb

@@ -1,7 +1,17 @@
+# frozen_string_literal: true
 module Sidekiq
   module Middleware
     module Server
       class ActiveRecord
+
+        def initialize
+          # With Rails 5+ we must use the Reloader **always**.
+          # The reloader handles code loading and db connection management.
+          if defined?(::Rails) && ::Rails::VERSION::MAJOR >= 5
+            raise ArgumentError, "Rails 5 no longer needs or uses the ActiveRecord middleware."
+          end
+        end
+
         def call(*args)
           yield
         ensure

data/lib/sidekiq/paginator.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Paginator
 

data/lib/sidekiq/processor.rb

@@ -1,161 +1,279 @@
+# frozen_string_literal: true
 require 'sidekiq/util'
-require 'sidekiq/actor'
-
-require 'sidekiq/middleware/server/retry_jobs'
-require 'sidekiq/middleware/server/logging'
+require 'sidekiq/fetch'
+require 'sidekiq/job_logger'
+require 'sidekiq/job_retry'
+require 'thread'
 
 module Sidekiq
   ##
-  # The Processor receives a message from the Manager and actually
-  # processes it.  It instantiates the worker, runs the middleware
-  # chain and then calls Sidekiq::Worker#perform.
+  # The Processor is a standalone thread which:
+  #
+  # 1. fetches a job from Redis
+  # 2. executes the job
+  #   a. instantiate the Worker
+  #   b. run the middleware chain
+  #   c. call #perform
+  #
+  # A Processor can exit due to shutdown (processor_stopped)
+  # or due to an error during job execution (processor_died)
+  #
+  # If an error occurs in the job execution, the
+  # Processor calls the Manager to create a new one
+  # to replace itself and exits.
+  #
   class Processor
-    # To prevent a memory leak, ensure that stats expire. However, they should take up a minimal amount of storage
-    # so keep them around for a long time
-    STATS_TIMEOUT = 24 * 60 * 60 * 365 * 5
 
     include Util
-    include Actor
-
-    def self.default_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Logging
-        m.add Middleware::Server::RetryJobs
-        if defined?(::ActiveRecord::Base)
-          require 'sidekiq/middleware/server/active_record'
-          m.add Sidekiq::Middleware::Server::ActiveRecord
+
+    attr_reader :thread
+    attr_reader :job
+
+    def initialize(mgr)
+      @mgr = mgr
+      @down = false
+      @done = false
+      @job = nil
+      @thread = nil
+      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+      @reloader = Sidekiq.options[:reloader]
+      @logging = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @retrier = Sidekiq::JobRetry.new
+    end
+
+    def terminate(wait=false)
+      @done = true
+      return if !@thread
+      @thread.value if wait
+    end
+
+    def kill(wait=false)
+      @done = true
+      return if !@thread
+      # unlike the other actors, terminate does not wait
+      # for the thread to finish because we don't know how
+      # long the job will take to finish.  Instead we
+      # provide a `kill` method to call after the shutdown
+      # timeout passes.
+      @thread.raise ::Sidekiq::Shutdown
+      @thread.value if wait
+    end
+
+    def start
+      @thread ||= safe_thread("processor", &method(:run))
+    end
+
+    private unless $TESTING
+
+    def run
+      begin
+        while !@done
+          process_one
         end
+        @mgr.processor_stopped(self)
+      rescue Sidekiq::Shutdown
+        @mgr.processor_stopped(self)
+      rescue Exception => ex
+        @mgr.processor_died(self, ex)
       end
     end
 
-    attr_accessor :proxy_id
+    def process_one
+      @job = fetch
+      process(@job) if @job
+      @job = nil
+    end
 
-    def initialize(boss)
-      @boss = boss
+    def get_one
+      begin
+        work = @strategy.retrieve_work
+        (logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }; @down = nil) if @down
+        work
+      rescue Sidekiq::Shutdown
+      rescue => ex
+        handle_fetch_exception(ex)
+      end
+    end
+
+    def fetch
+      j = get_one
+      if j && @done
+        j.requeue
+        nil
+      else
+        j
+      end
+    end
+
+    def handle_fetch_exception(ex)
+      if !@down
+        @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        logger.error("Error fetching job: #{ex}")
+        handle_exception(ex)
+      end
+      sleep(1)
+      nil
+    end
+
+    def dispatch(job_hash, queue)
+      # since middleware can mutate the job hash
+      # we clone here so we report the original
+      # job structure to the Web UI
+      pristine = cloned(job_hash)
+
+      Sidekiq::Logging.with_job_hash_context(job_hash) do
+        @retrier.global(pristine, queue) do
+          @logging.call(job_hash, queue) do
+            stats(pristine, queue) do
+              # Rails 5 requires a Reloader to wrap code execution.  In order to
+              # constantize the worker and instantiate an instance, we have to call
+              # the Reloader.  It handles code loading, db connection management, etc.
+              # Effectively this block denotes a "unit of work" to Rails.
+              @reloader.call do
+                klass  = constantize(job_hash['class'])
+                worker = klass.new
+                worker.jid = job_hash['jid']
+                @retrier.local(worker, pristine, queue) do
+                  yield worker
+                end
+              end
+            end
+          end
+        end
+      end
     end
 
     def process(work)
-      msgstr = work.message
+      jobstr = work.job
       queue = work.queue_name
 
-      @boss.async.real_thread(proxy_id, Thread.current)
+      # Treat malformed JSON as a special case: job goes straight to the morgue.
+      job_hash = nil
+      begin
+        job_hash = Sidekiq.load_json(jobstr)
+      rescue => ex
+        handle_exception(ex, { :context => "Invalid JSON for job", :jobstr => jobstr })
+        # we can't notify because the job isn't a valid hash payload.
+        DeadSet.new.kill(jobstr, notify_failure: false)
+        return work.acknowledge
+      end
 
-      ack = false
+      ack = true
       begin
-        msg = Sidekiq.load_json(msgstr)
-        klass  = msg['class'.freeze].constantize
-        worker = klass.new
-        worker.jid = msg['jid'.freeze]
-
-        stats(worker, msg, queue) do
-          Sidekiq.server_middleware.invoke(worker, msg, queue) do
-            # Only ack if we either attempted to start this job or
-            # successfully completed it. This prevents us from
-            # losing jobs if a middleware raises an exception before yielding
-            ack = true
-            execute_job(worker, cloned(msg['args'.freeze]))
+        dispatch(job_hash, queue) do |worker|
+          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
+            execute_job(worker, cloned(job_hash['args']))
           end
         end
-        ack = true
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
         # we didn't properly finish it.
         ack = false
+      rescue Sidekiq::JobRetry::Handled => h
+        # this is the common case: job raised error and Sidekiq::JobRetry::Handled
+        # signals that we created a retry successfully.  We can acknowlege the job.
+        e = h.cause ? h.cause : h
+        handle_exception(e, { :context => "Job raised exception", :job => job_hash, :jobstr => jobstr })
+        raise e
       rescue Exception => ex
-        handle_exception(ex, msg || { :message => msgstr })
-        raise
+        # Unexpected error!  This is very bad and indicates an exception that got past
+        # the retry subsystem (e.g. network partition).  We won't acknowledge the job
+        # so it can be rescued when using Sidekiq Pro.
+        ack = false
+        handle_exception(ex, { :context => "Internal exception!", :job => job_hash, :jobstr => jobstr })
+        raise e
       ensure
         work.acknowledge if ack
       end
-
-      @boss.async.processor_done(current_actor)
-    end
-
-    def inspect
-      "<Processor##{object_id.to_s(16)}>"
     end
 
     def execute_job(worker, cloned_args)
       worker.perform(*cloned_args)
     end
 
-    private
+    # Ruby doesn't provide atomic counters out of the box so we'll
+    # implement something simple ourselves.
+    # https://bugs.ruby-lang.org/issues/14706
+    class Counter
+      def initialize
+        @value = 0
+        @lock = Mutex.new
+      end
 
-    def thread_identity
-      @str ||= Thread.current.object_id.to_s(36)
+      def incr(amount=1)
+        @lock.synchronize { @value = @value + amount }
+      end
+
+      def reset
+        @lock.synchronize { val = @value; @value = 0; val }
+      end
     end
 
-    def stats(worker, msg, queue)
-      # Do not conflate errors from the job with errors caused by updating
-      # stats so calling code can react appropriately
-      retry_and_suppress_exceptions do
-        hash = Sidekiq.dump_json({:queue => queue, :payload => msg, :run_at => Time.now.to_i })
-        Sidekiq.redis do |conn|
-          conn.multi do
-            conn.hmset("#{identity}:workers", thread_identity, hash)
-            conn.expire("#{identity}:workers", 60*60*4)
-          end
-        end
+    # jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
+    class SharedWorkerState
+      def initialize
+        @worker_state = {}
+        @lock = Mutex.new
+      end
+
+      def set(tid, hash)
+        @lock.synchronize { @worker_state[tid] = hash }
+      end
+
+      def delete(tid)
+        @lock.synchronize { @worker_state.delete(tid) }
+      end
+
+      def dup
+        @lock.synchronize { @worker_state.dup }
+      end
+
+      def size
+        @lock.synchronize { @worker_state.size }
+      end
+
+      def clear
+        @lock.synchronize { @worker_state.clear }
       end
+    end
+
+    PROCESSED = Counter.new
+    FAILURE = Counter.new
+    WORKER_STATE = SharedWorkerState.new
+
+    def stats(job_hash, queue)
+      tid = Sidekiq::Logging.tid
+      WORKER_STATE.set(tid, {:queue => queue, :payload => job_hash, :run_at => Time.now.to_i })
 
-      nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
       begin
         yield
       rescue Exception
-        retry_and_suppress_exceptions do
-          failed = "stat:failed:#{nowdate}"
-          Sidekiq.redis do |conn|
-            conn.multi do
-              conn.incrby("stat:failed".freeze, 1)
-              conn.incrby(failed, 1)
-              conn.expire(failed, STATS_TIMEOUT)
-            end
-          end
-        end
+        FAILURE.incr
         raise
       ensure
-        retry_and_suppress_exceptions do
-          processed = "stat:processed:#{nowdate}"
-          Sidekiq.redis do |conn|
-            conn.multi do
-              conn.hdel("#{identity}:workers", thread_identity)
-              conn.incrby("stat:processed".freeze, 1)
-              conn.incrby(processed, 1)
-              conn.expire(processed, STATS_TIMEOUT)
-            end
-          end
-        end
+        WORKER_STATE.delete(tid)
+        PROCESSED.incr
       end
     end
 
     # Deep clone the arguments passed to the worker so that if
-    # the message fails, what is pushed back onto Redis hasn't
+    # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
-    def cloned(ary)
-      Marshal.load(Marshal.dump(ary))
+    def cloned(thing)
+      Marshal.load(Marshal.dump(thing))
     end
 
-    # If an exception occurs in the block passed to this method, that block will be retried up to max_retries times.
-    # All exceptions will be swallowed and logged.
-    def retry_and_suppress_exceptions(max_retries = 5)
-      retry_count = 0
-      begin
-        yield
-      rescue => e
-        retry_count += 1
-        if retry_count <= max_retries
-          Sidekiq.logger.debug {"Suppressing and retrying error: #{e.inspect}"}
-          pause_for_recovery(retry_count)
-          retry
-        else
-          handle_exception(e, { :message => "Exhausted #{max_retries} retries"})
-        end
+    def constantize(str)
+      names = str.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      names.inject(Object) do |constant, name|
+        # the false flag limits search for name to under the constant namespace
+        #   which mimics Rails' behaviour
+        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
       end
     end
 
-    def pause_for_recovery(retry_count)
-      sleep(retry_count)
-    end
   end
 end