sidekiq 6.2.1 → 6.5.1

data/lib/sidekiq/logger.rb

@@ -16,6 +16,10 @@ module Sidekiq
     def self.current
       Thread.current[:sidekiq_context] ||= {}
     end
+
+    def self.add(k, v)
+      current[k] = v
+    end
   end
 
   module LoggingUtils
@@ -31,24 +35,10 @@ module Sidekiq
       nil
     end
 
-    def debug?
-      level <= 0
-    end
-
-    def info?
-      level <= 1
-    end
-
-    def warn?
-      level <= 2
-    end
-
-    def error?
-      level <= 3
-    end
-
-    def fatal?
-      level <= 4
+    LEVELS.each do |level, numeric_level|
+      define_method("#{level}?") do
+        local_level.nil? ? super() : local_level <= numeric_level
+      end
     end
 
     def local_level

data/lib/sidekiq/manager.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/processor"
 require "sidekiq/fetch"
 require "set"
@@ -21,43 +20,37 @@ module Sidekiq
   # the shutdown process.  The other tasks are performed by other threads.
   #
   class Manager
-    include Util
+    include Sidekiq::Component
 
     attr_reader :workers
-    attr_reader :options
 
     def initialize(options = {})
+      @config = options
       logger.debug { options.inspect }
-      @options = options
       @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self, options)
+        @workers << Processor.new(@config, &method(:processor_result))
       end
       @plock = Mutex.new
     end
 
     def start
-      @workers.each do |x|
-        x.start
-      end
+      @workers.each(&:start)
     end
 
     def quiet
       return if @done
       @done = true
 
-      logger.info { "Terminating quiet workers" }
-      @workers.each { |x| x.terminate }
+      logger.info { "Terminating quiet threads" }
+      @workers.each(&:terminate)
       fire_event(:quiet, reverse: true)
     end
 
-    # hack for quicker development / testing environment #2774
-    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
-
     def stop(deadline)
       quiet
       fire_event(:shutdown, reverse: true)
@@ -68,29 +61,18 @@ module Sidekiq
       sleep PAUSE_TIME
       return if @workers.empty?
 
-      logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      while remaining > PAUSE_TIME
-        return if @workers.empty?
-        sleep PAUSE_TIME
-        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      end
+      logger.info { "Pausing to allow jobs to finish..." }
+      wait_for(deadline) { @workers.empty? }
       return if @workers.empty?
 
       hard_shutdown
     end
 
-    def processor_stopped(processor)
-      @plock.synchronize do
-        @workers.delete(processor)
-      end
-    end
-
-    def processor_died(processor, reason)
+    def processor_result(processor, reason = nil)
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self, options)
+          p = Processor.new(@config, &method(:processor_result))
           @workers << p
           p.start
         end
@@ -104,7 +86,7 @@ module Sidekiq
     private
 
     def hard_shutdown
-      # We've reached the timeout and we still have busy workers.
+      # We've reached the timeout and we still have busy threads.
       # They must die but their jobs shall live on.
       cleanup = nil
       @plock.synchronize do
@@ -114,22 +96,41 @@ module Sidekiq
       if cleanup.size > 0
         jobs = cleanup.map { |p| p.job }.compact
 
-        logger.warn { "Terminating #{cleanup.size} busy worker threads" }
-        logger.warn { "Work still in progress #{jobs.inspect}" }
+        logger.warn { "Terminating #{cleanup.size} busy threads" }
+        logger.debug { "Jobs still in progress #{jobs.inspect}" }
 
         # Re-enqueue unfinished jobs
         # NOTE: You may notice that we may push a job back to redis before
-        # the worker thread is terminated. This is ok because Sidekiq's
+        # the thread is terminated. This is ok because Sidekiq's
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = @options[:fetch]
-        strategy.bulk_requeue(jobs, @options)
+        strategy = @config[:fetch]
+        strategy.bulk_requeue(jobs, @config)
       end
 
       cleanup.each do |processor|
         processor.kill
       end
+
+      # when this method returns, we immediately call `exit` which may not give
+      # the remaining threads time to run `ensure` blocks, etc. We pause here up
+      # to 3 seconds to give threads a minimal amount of time to run `ensure` blocks.
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + 3
+      wait_for(deadline) { @workers.empty? }
+    end
+
+    # hack for quicker development / testing environment #2774
+    PAUSE_TIME = $stdout.tty? ? 0.1 : 0.5
+
+    # Wait for the orblock to be true or the deadline passed.
+    def wait_for(deadline, &condblock)
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      while remaining > PAUSE_TIME
+        return if condblock.call
+        sleep PAUSE_TIME
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
     end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "sidekiq/middleware/modules"
+
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack
@@ -44,10 +46,12 @@ module Sidekiq
   # This is an example of a minimal server middleware:
   #
   # class MyServerHook
-  #   def call(worker_instance, msg, queue)
-  #     puts "Before work"
+  #   include Sidekiq::ServerMiddleware
+  #   def call(job_instance, msg, queue)
+  #     logger.info "Before job"
+  #     redis {|conn| conn.get("foo") } # do something in Redis
   #     yield
-  #     puts "After work"
+  #     logger.info "After job"
   #   end
   # end
   #
@@ -56,10 +60,11 @@ module Sidekiq
   # to Redis:
   #
   # class MyClientHook
-  #   def call(worker_class, msg, queue, redis_pool)
-  #     puts "Before push"
+  #   include Sidekiq::ClientMiddleware
+  #   def call(job_class, msg, queue, redis_pool)
+  #     logger.info "Before push"
   #     result = yield
-  #     puts "After push"
+  #     logger.info "After push"
   #     result
   #   end
   # end
@@ -76,7 +81,8 @@ module Sidekiq
         entries.each(&block)
       end
 
-      def initialize
+      def initialize(config = nil)
+        @config = config
         @entries = nil
         yield self if block_given?
       end
@@ -90,25 +96,25 @@ module Sidekiq
       end
 
       def add(klass, *args)
-        remove(klass) if exists?(klass)
-        entries << Entry.new(klass, *args)
+        remove(klass)
+        entries << Entry.new(@config, klass, *args)
       end
 
       def prepend(klass, *args)
-        remove(klass) if exists?(klass)
-        entries.insert(0, Entry.new(klass, *args))
+        remove(klass)
+        entries.insert(0, Entry.new(@config, klass, *args))
       end
 
       def insert_before(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || 0
         entries.insert(i, new_entry)
       end
 
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
-        new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
+        new_entry = i.nil? ? Entry.new(@config, newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
         entries.insert(i + 1, new_entry)
       end
@@ -132,7 +138,7 @@ module Sidekiq
       def invoke(*args)
         return yield if empty?
 
-        chain = retrieve.dup
+        chain = retrieve
         traverse_chain = proc do
           if chain.empty?
             yield
@@ -144,16 +150,21 @@ module Sidekiq
       end
     end
 
+    private
+
     class Entry
       attr_reader :klass
 
-      def initialize(klass, *args)
+      def initialize(config, klass, *args)
+        @config = config
         @klass = klass
         @args = args
       end
 
       def make_new
-        @klass.new(*@args)
+        x = @klass.new(*@args)
+        x.config = @config if @config && x.respond_to?(:config=)
+        x
       end
     end
   end

data/lib/sidekiq/middleware/current_attributes.rb

@@ -0,0 +1,61 @@
+require "active_support/current_attributes"
+
+module Sidekiq
+  ##
+  # Automatically save and load any current attributes in the execution context
+  # so context attributes "flow" from Rails actions into any associated jobs.
+  # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
+  # per-request attribute. See +ActiveSupport::CurrentAttributes+.
+  #
+  # @example
+  #
+  #   # in your initializer
+  #   require "sidekiq/middleware/current_attributes"
+  #   Sidekiq::CurrentAttributes.persist(Myapp::Current)
+  #
+  module CurrentAttributes
+    class Save
+      include Sidekiq::ClientMiddleware
+
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, _)
+        attrs = @klass.attributes
+        if job.has_key?("cattr")
+          job["cattr"].merge!(attrs)
+        else
+          job["cattr"] = attrs
+        end
+        yield
+      end
+    end
+
+    class Load
+      include Sidekiq::ServerMiddleware
+
+      def initialize(cattr)
+        @klass = cattr
+      end
+
+      def call(_, job, _, &block)
+        if job.has_key?("cattr")
+          @klass.set(job["cattr"], &block)
+        else
+          yield
+        end
+      end
+    end
+
+    def self.persist(klass)
+      Sidekiq.configure_client do |config|
+        config.client_middleware.add Save, klass
+      end
+      Sidekiq.configure_server do |config|
+        config.client_middleware.add Save, klass
+        config.server_middleware.add Load, klass
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/i18n.rb

@@ -10,16 +10,18 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
-    def call(_worker, msg, _queue, _redis)
-      msg["locale"] ||= I18n.locale
+    include Sidekiq::ClientMiddleware
+    def call(_jobclass, job, _queue, _redis)
+      job["locale"] ||= I18n.locale
       yield
     end
   end
 
   # Pull the msg locale out and set the current thread to use it.
   class Server
-    def call(_worker, msg, _queue, &block)
-      I18n.with_locale(msg.fetch("locale", I18n.default_locale), &block)
+    include Sidekiq::ServerMiddleware
+    def call(_jobclass, job, _queue, &block)
+      I18n.with_locale(job.fetch("locale", I18n.default_locale), &block)
     end
   end
 end

data/lib/sidekiq/middleware/modules.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  # Server-side middleware must import this Module in order
+  # to get access to server resources during `call`.
+  module ServerMiddleware
+    attr_accessor :config
+    def redis_pool
+      config.redis_pool
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+  end
+
+  # no difference for now
+  ClientMiddleware = ServerMiddleware
+end

data/lib/sidekiq/monitor.rb

@@ -17,7 +17,7 @@ class Sidekiq::Monitor
       end
       send(section)
     rescue => e
-      puts "Couldn't get status: #{e}"
+      abort "Couldn't get status: #{e}"
     end
 
     def all

data/lib/sidekiq/paginator.rb

@@ -16,22 +16,22 @@ module Sidekiq
 
         case type
         when "zset"
-          total_size, items = conn.multi {
-            conn.zcard(key)
+          total_size, items = conn.multi { |transaction|
+            transaction.zcard(key)
             if rev
-              conn.zrevrange(key, starting, ending, with_scores: true)
+              transaction.zrevrange(key, starting, ending, withscores: true)
             else
-              conn.zrange(key, starting, ending, with_scores: true)
+              transaction.zrange(key, starting, ending, withscores: true)
             end
           }
           [current_page, total_size, items]
         when "list"
-          total_size, items = conn.multi {
-            conn.llen(key)
+          total_size, items = conn.multi { |transaction|
+            transaction.llen(key)
             if rev
-              conn.lrange(key, -ending - 1, -starting - 1)
+              transaction.lrange(key, -ending - 1, -starting - 1)
             else
-              conn.lrange(key, starting, ending)
+              transaction.lrange(key, starting, ending)
             end
           }
           items.reverse! if rev

data/lib/sidekiq/processor.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "sidekiq/util"
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
@@ -11,33 +10,34 @@ module Sidekiq
   #
   # 1. fetches a job from Redis
   # 2. executes the job
-  #   a. instantiate the Worker
+  #   a. instantiate the job class
   #   b. run the middleware chain
   #   c. call #perform
   #
-  # A Processor can exit due to shutdown (processor_stopped)
-  # or due to an error during job execution (processor_died)
+  # A Processor can exit due to shutdown or due to
+  # an error during job execution.
   #
   # If an error occurs in the job execution, the
   # Processor calls the Manager to create a new one
   # to replace itself and exits.
   #
   class Processor
-    include Util
+    include Sidekiq::Component
 
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr, options)
-      @mgr = mgr
+    def initialize(options, &block)
+      @callback = block
       @down = false
       @done = false
       @job = nil
       @thread = nil
+      @config = options
       @strategy = options[:fetch]
       @reloader = options[:reloader] || proc { |&block| block.call }
       @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
-      @retrier = Sidekiq::JobRetry.new
+      @retrier = Sidekiq::JobRetry.new(options)
     end
 
     def terminate(wait = false)
@@ -66,26 +66,26 @@ module Sidekiq
 
     def run
       process_one until @done
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Sidekiq::Shutdown
-      @mgr.processor_stopped(self)
+      @callback.call(self)
     rescue Exception => ex
-      @mgr.processor_died(self, ex)
+      @callback.call(self, ex)
     end
 
-    def process_one
+    def process_one(&block)
       @job = fetch
       process(@job) if @job
       @job = nil
     end
 
     def get_one
-      work = @strategy.retrieve_work
+      uow = @strategy.retrieve_work
       if @down
         logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
         @down = nil
       end
-      work
+      uow
     rescue Sidekiq::Shutdown
     rescue => ex
       handle_fetch_exception(ex)
@@ -130,10 +130,10 @@ module Sidekiq
               # Effectively this block denotes a "unit of work" to Rails.
               @reloader.call do
                 klass = constantize(job_hash["class"])
-                worker = klass.new
-                worker.jid = job_hash["jid"]
-                @retrier.local(worker, jobstr, queue) do
-                  yield worker
+                inst = klass.new
+                inst.jid = job_hash["jid"]
+                @retrier.local(inst, jobstr, queue) do
+                  yield inst
                 end
               end
             end
@@ -142,9 +142,9 @@ module Sidekiq
       end
     end
 
-    def process(work)
-      jobstr = work.job
-      queue = work.queue_name
+    def process(uow)
+      jobstr = uow.job
+      queue = uow.queue_name
 
       # Treat malformed JSON as a special case: job goes straight to the morgue.
       job_hash = nil
@@ -154,14 +154,14 @@ module Sidekiq
         handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
         # we can't notify because the job isn't a valid hash payload.
         DeadSet.new.kill(jobstr, notify_failure: false)
-        return work.acknowledge
+        return uow.acknowledge
       end
 
       ack = false
       begin
-        dispatch(job_hash, queue, jobstr) do |worker|
-          Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-            execute_job(worker, job_hash["args"])
+        dispatch(job_hash, queue, jobstr) do |inst|
+          @config.server_middleware.invoke(inst, job_hash, queue) do
+            execute_job(inst, job_hash["args"])
           end
         end
         ack = true
@@ -186,14 +186,14 @@ module Sidekiq
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.
           Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
-            work.acknowledge
+            uow.acknowledge
           end
         end
       end
     end
 
-    def execute_job(worker, cloned_args)
-      worker.perform(*cloned_args)
+    def execute_job(inst, cloned_args)
+      inst.perform(*cloned_args)
     end
 
     # Ruby doesn't provide atomic counters out of the box so we'll
@@ -219,39 +219,39 @@ module Sidekiq
     end
 
     # jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
-    class SharedWorkerState
+    class SharedWorkState
       def initialize
-        @worker_state = {}
+        @work_state = {}
         @lock = Mutex.new
       end
 
       def set(tid, hash)
-        @lock.synchronize { @worker_state[tid] = hash }
+        @lock.synchronize { @work_state[tid] = hash }
       end
 
       def delete(tid)
-        @lock.synchronize { @worker_state.delete(tid) }
+        @lock.synchronize { @work_state.delete(tid) }
       end
 
       def dup
-        @lock.synchronize { @worker_state.dup }
+        @lock.synchronize { @work_state.dup }
       end
 
       def size
-        @lock.synchronize { @worker_state.size }
+        @lock.synchronize { @work_state.size }
       end
 
       def clear
-        @lock.synchronize { @worker_state.clear }
+        @lock.synchronize { @work_state.clear }
       end
     end
 
     PROCESSED = Counter.new
     FAILURE = Counter.new
-    WORKER_STATE = SharedWorkerState.new
+    WORK_STATE = SharedWorkState.new
 
     def stats(jobstr, queue)
-      WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
+      WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
 
       begin
         yield
@@ -259,7 +259,7 @@ module Sidekiq
         FAILURE.incr
         raise
       ensure
-        WORKER_STATE.delete(tid)
+        WORK_STATE.delete(tid)
         PROCESSED.incr
       end
     end