sidekiq 5.2.8 → 6.1.3

data/lib/sidekiq/client.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
-require 'securerandom'
-require 'sidekiq/middleware/chain'
+
+require "securerandom"
+require "sidekiq/middleware/chain"
 
 module Sidekiq
   class Client
-
     ##
     # Define client-side middleware:
     #
@@ -19,7 +19,7 @@ module Sidekiq
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -38,7 +38,7 @@ module Sidekiq
     # Generally this is only needed for very large Sidekiq installs processing
     # thousands of jobs per second.  I don't recommend sharding unless you
     # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool=nil)
+    def initialize(redis_pool = nil)
       @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
     end
 
@@ -68,11 +68,11 @@ module Sidekiq
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item['class'], normed)
+      payload = process_single(item["class"], normed)
 
       if payload
         raw_push([payload])
-        payload['jid']
+        payload["jid"]
       end
     end
 
@@ -90,19 +90,25 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items['args'].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
+
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items['args'].map do |args|
-        copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
-        result = process_single(items['class'], copy)
-        result ? result : nil
-      end.compact
-
-      raw_push(payloads) if !payloads.empty?
-      payloads.collect { |payload| payload['jid'] }
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+
+        result = process_single(items["class"], copy)
+        result || nil
+      }.compact
+
+      raw_push(payloads) unless payloads.empty?
+      payloads.collect { |payload| payload["jid"] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -127,7 +133,6 @@ module Sidekiq
     end
 
     class << self
-
       def push(item)
         new.push(item)
       end
@@ -145,14 +150,14 @@ module Sidekiq
       # Messages are enqueued to the 'default' queue.
       #
       def enqueue(klass, *args)
-        klass.client_push('class' => klass, 'args' => args)
+        klass.client_push("class" => klass, "args" => args)
       end
 
       # Example usage:
       #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
-        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
+        klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
@@ -163,8 +168,8 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
-        item.delete('at') if ts <= now
+        item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
+        item.delete("at") if ts <= now
 
         klass.client_push(item)
       end
@@ -189,51 +194,66 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first['at']
-        conn.zadd('schedule', payloads.map do |hash|
-          at = hash.delete('at').to_s
+      if payloads.first.key?("at")
+        conn.zadd("schedule", payloads.map { |hash|
+          at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
-        end)
+        })
       else
-        q = payloads.first['queue']
+        queue = payloads.first["queue"]
         now = Time.now.to_f
-        to_push = payloads.map do |entry|
-          entry['enqueued_at'] = now
+        to_push = payloads.map { |entry|
+          entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
-        end
-        conn.sadd('queues', q)
-        conn.lpush("queue:#{q}", to_push)
+        }
+        conn.sadd("queues", queue)
+        conn.lpush("queue:#{queue}", to_push)
       end
     end
 
     def process_single(worker_class, item)
-      queue = item['queue']
+      queue = item["queue"]
 
       middleware.invoke(worker_class, item, queue, @redis_pool) do
         item
       end
     end
 
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
     def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
-      raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
-      #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      normalized_hash(item['class'])
-        .each{ |key, value| item[key] = value if item[key].nil? }
-
-      item['class'] = item['class'].to_s
-      item['queue'] = item['queue'].to_s
-      item['jid'] ||= SecureRandom.hex(12)
-      item['created_at'] ||= Time.now.to_f
+      # 6.0.0 push_bulk bug, #4321
+      # TODO Remove after a while...
+      item.delete("at") if item.key?("at") && item["at"].nil?
+
+      validate(item)
+      # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
+
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
+      item["class"] = item["class"].to_s
+      item["queue"] = item["queue"].to_s
+      item["jid"] ||= SecureRandom.hex(12)
+      item["created_at"] ||= Time.now.to_f
+
       item
     end
 
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?("get_sidekiq_options")
         item_class.get_sidekiq_options
       else
         Sidekiq.default_worker_options

data/lib/sidekiq/delay.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
+
 module Sidekiq
   module Extensions
-
     def self.enable_delay!
       if defined?(::ActiveSupport)
-        require 'sidekiq/extensions/active_record'
-        require 'sidekiq/extensions/action_mailer'
+        require "sidekiq/extensions/active_record"
+        require "sidekiq/extensions/action_mailer"
 
         # Need to patch Psych so it can autoload classes whose names are serialized
         # in the delayed YAML.
@@ -19,7 +19,7 @@ module Sidekiq
         end
       end
 
-      require 'sidekiq/extensions/class_methods'
+      require "sidekiq/extensions/class_methods"
       Module.__send__(:include, Sidekiq::Extensions::Klass)
     end
 
@@ -27,7 +27,7 @@ module Sidekiq
       def resolve_class(klass_name)
         return nil if !klass_name || klass_name.empty?
         # constantize
-        names = klass_name.split('::')
+        names = klass_name.split("::")
         names.shift if names.empty? || names.first.empty?
 
         names.inject(Object) do |constant, name|
@@ -39,4 +39,3 @@ module Sidekiq
     end
   end
 end
-

data/lib/sidekiq/exception_handler.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
-require 'sidekiq'
+
+require "sidekiq"
 
 module Sidekiq
   module ExceptionHandler
-
     class Logger
-      def call(ex, ctxHash)
-        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
+      def call(ex, ctx)
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
         Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
         Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
       end
@@ -14,15 +14,13 @@ module Sidekiq
       Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
     end
 
-    def handle_exception(ex, ctxHash={})
+    def handle_exception(ex, ctx = {})
       Sidekiq.error_handlers.each do |handler|
-        begin
-          handler.call(ex, ctxHash)
-        rescue => ex
-          Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
-          Sidekiq.logger.error ex
-          Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
-        end
+        handler.call(ex, ctx)
+      rescue => ex
+        Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+        Sidekiq.logger.error ex
+        Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
       end
     end
   end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
-    # delivery to Sidekiq.  Example:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.
     #
+    # @example
     #    UserMailer.delay.send_welcome_email(new_user)
     #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
     #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -19,39 +21,28 @@ module Sidekiq
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
         if msg
-          deliver(msg)
-        else
-          raise "#{target.name}##{method_name} returned an undeliverable mail object"
-        end
-      end
-
-      private
-
-      def deliver(msg)
-        if msg.respond_to?(:deliver_now)
-          # Rails 4.2/5.0
           msg.deliver_now
         else
-          # Rails 3.2/4.0/4.1
-          msg.deliver
+          raise "#{target.name}##{method_name} returned an undeliverable mail object"
         end
       end
     end
 
     module ActionMailer
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedMailer, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end

data/lib/sidekiq/extensions/active_record.rb

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.
@@ -22,19 +24,20 @@ module Sidekiq
     end
 
     module ActiveRecord
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedModel, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker
@@ -20,20 +22,21 @@ module Sidekiq
     end
 
     module Klass
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedClass, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end
 

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
-require 'yaml'
+
+require "yaml"
 
 module Sidekiq
   module Extensions
     SIZE_LIMIT = 8_192
 
     class Proxy < BasicObject
-      def initialize(performable, target, options={})
+      def initialize(performable, target, options = {})
         @performable = performable
         @target = target
         @opts = options
@@ -23,9 +24,8 @@ module Sidekiq
         if marshalled.size > SIZE_LIMIT
           ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
         end
-        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
+        @performable.client_push({"class" => @performable, "args" => [marshalled]}.merge(@opts))
       end
     end
-
   end
 end

data/lib/sidekiq/fetch.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-require 'sidekiq'
+
+require "sidekiq"
 
 module Sidekiq
   class BasicFetch
@@ -7,27 +8,29 @@ module Sidekiq
     # can check if the process is shutting down.
     TIMEOUT = 2
 
-    UnitOfWork = Struct.new(:queue, :job) do
+    UnitOfWork = Struct.new(:queue, :job) {
       def acknowledge
         # nothing to do
       end
 
       def queue_name
-        queue.sub(/.*queue:/, '')
+        queue.delete_prefix("queue:")
       end
 
       def requeue
         Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+          conn.rpush(queue, job)
         end
       end
-    end
+    }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
+        @queues.uniq!
         @queues << TIMEOUT
       end
     end
@@ -37,38 +40,20 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle.uniq
-        queues << TIMEOUT
-        queues
-      end
-    end
-
-
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
         conn.pipelined do
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            conn.rpush(queue, jobs)
           end
         end
       end
@@ -77,5 +62,19 @@ module Sidekiq
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle!.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
   end
 end

data/lib/sidekiq/job_logger.rb

@@ -1,25 +1,63 @@
 # frozen_string_literal: true
+
 module Sidekiq
   class JobLogger
+    def initialize(logger = Sidekiq.logger)
+      @logger = logger
+    end
 
     def call(item, queue)
       start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
-      logger.info("start")
+      @logger.info("start")
+
       yield
-      logger.info("done: #{elapsed(start)} sec")
+
+      with_elapsed_time_context(start) do
+        @logger.info("done")
+      end
     rescue Exception
-      logger.info("fail: #{elapsed(start)} sec")
+      with_elapsed_time_context(start) do
+        @logger.info("fail")
+      end
+
       raise
     end
 
+    def prepare(job_hash, &block)
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level) do
+          Sidekiq::Context.with(job_hash_context(job_hash), &block)
+        end
+      else
+        Sidekiq::Context.with(job_hash_context(job_hash), &block)
+      end
+    end
+
+    def job_hash_context(job_hash)
+      # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+      # attribute to expose the underlying thing.
+      h = {
+        class: job_hash["wrapped"] || job_hash["class"],
+        jid: job_hash["jid"]
+      }
+      h[:bid] = job_hash["bid"] if job_hash["bid"]
+      h[:tags] = job_hash["tags"] if job_hash["tags"]
+      h
+    end
+
+    def with_elapsed_time_context(start, &block)
+      Sidekiq::Context.with(elapsed_time_context(start), &block)
+    end
+
+    def elapsed_time_context(start)
+      {elapsed: elapsed(start).to_s}
+    end
+
     private
 
     def elapsed(start)
       (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
     end
-
-    def logger
-      Sidekiq.logger
-    end
   end
 end