sidekiq 4.2.10 → 6.5.7

data/lib/sidekiq/client.rb

@@ -1,9 +1,12 @@
 # frozen_string_literal: true
-require 'securerandom'
-require 'sidekiq/middleware/chain'
+
+require "securerandom"
+require "sidekiq/middleware/chain"
+require "sidekiq/job_util"
 
 module Sidekiq
   class Client
+    include Sidekiq::JobUtil
 
     ##
     # Define client-side middleware:
@@ -12,14 +15,14 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
     #
     def middleware(&block)
       @chain ||= Sidekiq.client_middleware
-      if block_given?
+      if block
         @chain = @chain.dup
         yield @chain
       end
@@ -38,7 +41,7 @@ module Sidekiq
     # Generally this is only needed for very large Sidekiq installs processing
     # thousands of jobs per second.  I don't recommend sharding unless you
     # cannot scale any other way (e.g. splitting your app into smaller apps).
-    def initialize(redis_pool=nil)
+    def initialize(redis_pool = nil)
       @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
     end
 
@@ -46,11 +49,17 @@ module Sidekiq
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
+    # If class is set to the class name, the jobs' options will be based on Sidekiq's default
+    # job options. Otherwise, they will be based on the job class's options.
+    #
+    # Any options valid for a job class's sidekiq_options are also available here.
+    #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
     # space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
@@ -58,22 +67,25 @@ module Sidekiq
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item['class'], normed)
-
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
-        payload['jid']
+        payload["jid"]
       end
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing thousands of jobs or more.  This method
-    # cuts out the redis network round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
@@ -83,19 +95,30 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items['args'].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
+
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
+
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
 
       normed = normalize_item(items)
-      payloads = items['args'].map do |args|
-        copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
-        result = process_single(items['class'], copy)
-        result ? result : nil
-      end.compact
-
-      raw_push(payloads) if !payloads.empty?
-      payloads.collect { |payload| payload['jid'] }
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+          verify_json(copy)
+          copy
+        end
+        result || nil
+      }.compact
+
+      raw_push(payloads) unless payloads.empty?
+      payloads.collect { |payload| payload["jid"] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -103,8 +126,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -113,15 +136,13 @@ module Sidekiq
     def self.via(pool)
       raise ArgumentError, "No pool given" if pool.nil?
       current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
-      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
       Thread.current[:sidekiq_via_pool] = pool
       yield
     ensure
-      Thread.current[:sidekiq_via_pool] = nil
+      Thread.current[:sidekiq_via_pool] = current_sidekiq_pool
     end
 
     class << self
-
       def push(item)
         new.push(item)
       end
@@ -131,40 +152,40 @@ module Sidekiq
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
       def enqueue(klass, *args)
-        klass.client_push('class' => klass, 'args' => args)
+        klass.client_push("class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
-        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
+        klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
-        item.delete('at'.freeze) if ts <= now
+        item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
+        item.delete("at") if ts <= now
 
         klass.client_push(item)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -175,61 +196,43 @@ module Sidekiq
 
     def raw_push(payloads)
       @redis_pool.with do |conn|
-        conn.multi do
-          atomic_push(conn, payloads)
+        retryable = true
+        begin
+          conn.pipelined do |pipeline|
+            atomic_push(pipeline, payloads)
+          end
+        rescue RedisConnection.adapter::BaseError => ex
+          # 2550 Failover can cause the server to become a replica, need
+          # to disconnect and reopen the socket to get back to the primary.
+          # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+          # 4985 Use the same logic when a blocking command is force-unblocked
+          # The retry logic is copied from sidekiq.rb
+          if retryable && ex.message =~ /READONLY|NOREPLICAS|UNBLOCKED/
+            conn.disconnect!
+            retryable = false
+            retry
+          end
+          raise
         end
       end
       true
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first['at']
-        conn.zadd('schedule'.freeze, payloads.map do |hash|
-          at = hash.delete('at'.freeze).to_s
+      if payloads.first.key?("at")
+        conn.zadd("schedule", payloads.flat_map { |hash|
+          at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
-        end)
+        })
       else
-        q = payloads.first['queue']
+        queue = payloads.first["queue"]
         now = Time.now.to_f
-        to_push = payloads.map do |entry|
-          entry['enqueued_at'.freeze] = now
+        to_push = payloads.map { |entry|
+          entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
-        end
-        conn.sadd('queues'.freeze, q)
-        conn.lpush("queue:#{q}", to_push)
-      end
-    end
-
-    def process_single(worker_class, item)
-      queue = item['queue']
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
-
-    def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
-      raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
-      #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
-
-      normalized_hash(item['class'.freeze])
-        .each{ |key, value| item[key] = value if item[key].nil? }
-
-      item['class'.freeze] = item['class'.freeze].to_s
-      item['queue'.freeze] = item['queue'.freeze].to_s
-      item['jid'.freeze] ||= SecureRandom.hex(12)
-      item['created_at'.freeze] ||= Time.now.to_f
-      item
-    end
-
-    def normalized_hash(item_class)
-      if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
-        item_class.get_sidekiq_options
-      else
-        Sidekiq.default_worker_options
+        }
+        conn.sadd("queues", [queue])
+        conn.lpush("queue:#{queue}", to_push)
       end
     end
   end

data/lib/sidekiq/component.rb

@@ -0,0 +1,65 @@
+module Sidekiq
+  ##
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component # :nodoc:
+    attr_reader :config
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, {context: last_words})
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current.name = name
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+
+    def tid
+      Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def hostname
+      ENV["DYNO"] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
+    end
+
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
+    def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
+      arr = config[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        block.call
+      rescue => ex
+        handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
+        raise ex if reraise
+      end
+      arr.clear if oneshot # once we've fired an event, we never fire it again
+    end
+  end
+end

data/lib/sidekiq/delay.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Sidekiq # :nodoc:
+  module Extensions
+    def self.enable_delay!
+      warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1
+
+      if defined?(::ActiveSupport)
+        require "sidekiq/extensions/active_record"
+        require "sidekiq/extensions/action_mailer"
+
+        # Need to patch Psych so it can autoload classes whose names are serialized
+        # in the delayed YAML.
+        Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
+
+        ActiveSupport.on_load(:active_record) do
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require "sidekiq/extensions/class_methods"
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        # constantize
+        names = klass_name.split("::")
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      rescue NameError
+        super
+      end
+    end
+  end
+end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
-    # delivery to Sidekiq.  Example:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.
     #
+    # @example
     #    UserMailer.delay.send_welcome_email(new_user)
     #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
     #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
@@ -19,39 +21,28 @@ module Sidekiq
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
         if msg
-          deliver(msg)
-        else
-          raise "#{target.name}##{method_name} returned an undeliverable mail object"
-        end
-      end
-
-      private
-
-      def deliver(msg)
-        if msg.respond_to?(:deliver_now)
-          # Rails 4.2/5.0
           msg.deliver_now
         else
-          # Rails 3.2/4.0/4.1
-          msg.deliver
+          raise "#{target.name}##{method_name} returned an undeliverable mail object"
         end
       end
     end
 
     module ActionMailer
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedMailer, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedMailer, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedMailer, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end

data/lib/sidekiq/extensions/active_record.rb

@@ -1,13 +1,15 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.
@@ -22,19 +24,20 @@ module Sidekiq
     end
 
     module ActiveRecord
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedModel, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedModel, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedModel, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
-require 'sidekiq/extensions/generic_proxy'
+
+require "sidekiq/extensions/generic_proxy"
 
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker
@@ -20,20 +22,21 @@ module Sidekiq
     end
 
     module Klass
-      def sidekiq_delay(options={})
+      def sidekiq_delay(options = {})
         Proxy.new(DelayedClass, self, options)
       end
-      def sidekiq_delay_for(interval, options={})
-        Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
+
+      def sidekiq_delay_for(interval, options = {})
+        Proxy.new(DelayedClass, self, options.merge("at" => Time.now.to_f + interval.to_f))
       end
-      def sidekiq_delay_until(timestamp, options={})
-        Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
+
+      def sidekiq_delay_until(timestamp, options = {})
+        Proxy.new(DelayedClass, self, options.merge("at" => timestamp.to_f))
       end
       alias_method :delay, :sidekiq_delay
       alias_method :delay_for, :sidekiq_delay_for
       alias_method :delay_until, :sidekiq_delay_until
     end
-
   end
 end
 

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
-require 'yaml'
+
+require "yaml"
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
-      def initialize(performable, target, options={})
+      def initialize(performable, target, options = {})
         @performable = performable
         @target = target
-        @opts = options
+        @opts = options.transform_keys(&:to_s)
       end
 
       def method_missing(name, *args)
@@ -17,9 +20,14 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({"class" => @performable,
+                                  "args" => [marshalled],
+                                  "display_class" => "#{@target}.#{name}"}.merge(@opts))
       end
     end
-
   end
 end