sidekiq 6.0.0 → 6.1.2

data/lib/sidekiq/cli.rb

@@ -33,17 +33,21 @@ module Sidekiq
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run
-      boot_system
+    def run(boot_app: true)
+      boot_application if boot_app
+
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << "USR2" if Sidekiq.pro? && !jruby?
       sigs.each do |sig|
         trap sig do
-          self_write.write("#{sig}\n")
+          self_write.puts(sig)
         end
       rescue ArgumentError
         puts "Signal #{sig} not supported"
@@ -51,12 +55,12 @@ module Sidekiq
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
-      logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info["redis_version"]
-      raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
+      raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -160,17 +164,14 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
-      handy = SIGNAL_HANDLERS[sig]
-      if handy
-        handy.call(self)
-      else
-        Sidekiq.logger.info { "No signal handler for #{sig}" }
-      end
+      SIGNAL_HANDLERS[sig].call(self)
     end
 
     private
@@ -182,7 +183,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -204,7 +209,7 @@ module Sidekiq
 
       # check config file presence
       if opts[:config_file]
-        if opts[:config_file] && !File.exist?(opts[:config_file])
+        unless File.exist?(opts[:config_file])
           raise ArgumentError, "No such file #{opts[:config_file]}"
         end
       else
@@ -224,8 +229,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -236,7 +240,7 @@ module Sidekiq
       Sidekiq.options
     end
 
-    def boot_system
+    def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
       if File.directory?(options[:require])
@@ -283,8 +287,13 @@ module Sidekiq
 
     def parse_options(argv)
       opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
 
-      @parser = OptionParser.new { |o|
+    def option_parser(opts)
+      parser = OptionParser.new { |o|
         o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
           opts[:concurrency] = Integer(arg)
         end
@@ -336,15 +345,13 @@ module Sidekiq
         end
       }
 
-      @parser.banner = "sidekiq [options]"
-      @parser.on_tail "-h", "--help", "Show help" do
-        logger.info @parser
+      parser.banner = "sidekiq [options]"
+      parser.on_tail "-h", "--help", "Show help" do
+        logger.info parser
         die 1
       end
 
-      @parser.parse!(argv)
-
-      opts
+      parser
     end
 
     def initialize_logger
@@ -361,6 +368,8 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
@@ -372,9 +381,16 @@ module Sidekiq
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
       [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -90,13 +90,19 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
+
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items["args"].map { |args|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+
         result = process_single(items["class"], copy)
         result || nil
       }.compact
@@ -188,7 +194,7 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first["at"]
+      if payloads.first.key?("at")
         conn.zadd("schedule", payloads.map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
@@ -213,20 +219,35 @@ module Sidekiq
       end
     end
 
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
     def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
+      # 6.0.0 push_bulk bug, #4321
+      # TODO Remove after a while...
+      item.delete("at") if item.key?("at") && item["at"].nil?
+
+      validate(item)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item["class"])
-        .each { |key, value| item[key] = value if item[key].nil? }
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["jid"] ||= SecureRandom.hex(12)
       item["created_at"] ||= Time.now.to_f
+
       item
     end
 

data/lib/sidekiq/extensions/action_mailer.rb

@@ -5,9 +5,10 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
-    # delivery to Sidekiq.  Example:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.
     #
+    # @example
     #    UserMailer.delay.send_welcome_email(new_user)
     #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
     #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)

data/lib/sidekiq/extensions/active_record.rb

@@ -5,10 +5,11 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # Adds +delay+, +delay_for+ and +delay_until+ methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.

data/lib/sidekiq/extensions/class_methods.rb

@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker

data/lib/sidekiq/fetch.rb

@@ -14,21 +14,23 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, "")
+        queue.delete_prefix("queue:")
       end
 
       def requeue
         Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+          conn.rpush(queue, job)
         end
       end
     }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
+        @queues.uniq!
         @queues << TIMEOUT
       end
     end
@@ -38,37 +40,20 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle.uniq
-        queues << TIMEOUT
-        queues
-      end
-    end
-
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
         conn.pipelined do
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            conn.rpush(queue, jobs)
           end
         end
       end
@@ -76,5 +61,20 @@ module Sidekiq
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle!.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
   end
 end

data/lib/sidekiq/job_logger.rb

@@ -23,8 +23,15 @@ module Sidekiq
       raise
     end
 
-    def with_job_hash_context(job_hash, &block)
-      @logger.with_context(job_hash_context(job_hash), &block)
+    def prepare(job_hash, &block)
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level) do
+          Sidekiq::Context.with(job_hash_context(job_hash), &block)
+        end
+      else
+        Sidekiq::Context.with(job_hash_context(job_hash), &block)
+      end
     end
 
     def job_hash_context(job_hash)
@@ -32,14 +39,15 @@ module Sidekiq
       # attribute to expose the underlying thing.
       h = {
         class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
+      h[:tags] = job_hash["tags"] if job_hash["tags"]
       h
     end
 
     def with_elapsed_time_context(start, &block)
-      @logger.with_context(elapsed_time_context(start), &block)
+      Sidekiq::Context.with(elapsed_time_context(start), &block)
     end
 
     def elapsed_time_context(start)

data/lib/sidekiq/job_retry.rb

@@ -3,6 +3,9 @@
 require "sidekiq/scheduled"
 require "sidekiq/api"
 
+require "zlib"
+require "base64"
+
 module Sidekiq
   ##
   # Automatically retry jobs that fail in Sidekiq.
@@ -71,7 +74,7 @@ module Sidekiq
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
     # require the worker to be instantiated.
-    def global(msg, queue)
+    def global(jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -82,6 +85,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
         attempt_retry(nil, msg, queue, e)
       else
@@ -103,7 +107,7 @@ module Sidekiq
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, msg, queue)
+    def local(worker, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -114,6 +118,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
         msg["retry"] = worker.class.get_sidekiq_options["retry"]
       end
@@ -151,12 +156,14 @@ module Sidekiq
         msg["retry_count"] = 0
       end
 
-      if msg["backtrace"] == true
-        msg["error_backtrace"] = exception.backtrace
-      elsif !msg["backtrace"]
-        # do nothing
-      elsif msg["backtrace"].to_i != 0
-        msg["error_backtrace"] = exception.backtrace[0...msg["backtrace"].to_i]
+      if msg["backtrace"]
+        lines = if msg["backtrace"] == true
+          exception.backtrace
+        else
+          exception.backtrace[0...msg["backtrace"].to_i]
+        end
+
+        msg["error_backtrace"] = compress_backtrace(lines)
       end
 
       if count < max_retry_attempts
@@ -182,13 +189,13 @@ module Sidekiq
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      send_to_morgue(msg) unless msg["dead"] == false
+
       Sidekiq.death_handlers.each do |handler|
         handler.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling death handler", job: msg})
       end
-
-      send_to_morgue(msg) unless msg["dead"] == false
     end
 
     def send_to_morgue(msg)
@@ -245,5 +252,11 @@ module Sidekiq
     rescue
       +"!!! ERROR MESSAGE THREW AN ERROR !!!"
     end
+
+    def compress_backtrace(backtrace)
+      serialized = Sidekiq.dump_json(backtrace)
+      compressed = Zlib::Deflate.deflate(serialized)
+      Base64.encode64(compressed)
+    end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -16,12 +16,13 @@ module Sidekiq
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
-      proc { |me, data| "stopping" if me.stopping? },
+      proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
       @done = false
@@ -56,7 +57,7 @@ module Sidekiq
 
       # Requeue everything in case there was a worker who grabbed work while stopped
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+      strategy = @options[:fetch]
       strategy.bulk_requeue([], @options)
 
       clear_heartbeat
@@ -83,7 +84,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.pipelined do
           conn.srem("processes", identity)
-          conn.del("#{identity}:workers")
+          conn.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -96,6 +97,32 @@ module Sidekiq
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
@@ -118,7 +145,7 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
@@ -129,15 +156,13 @@ module Sidekiq
         fails = procd = 0
 
         _, exists, _, _, msg = Sidekiq.redis { |conn|
-          res = conn.multi {
+          conn.multi {
             conn.sadd("processes", key)
-            conn.exists(key)
+            conn.exists?(key)
             conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           }
-
-          res
         }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
@@ -148,7 +173,7 @@ module Sidekiq
         ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
@@ -165,7 +190,7 @@ module Sidekiq
           "concurrency" => @options[:concurrency],
           "queues" => @options[:queues].uniq,
           "labels" => @options[:labels],
-          "identity" => identity,
+          "identity" => identity
         }
       end
     end