sidekiq 6.0.0 → 6.0.2

data/lib/sidekiq/cli.rb

@@ -41,9 +41,11 @@ module Sidekiq
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << "USR2" unless jruby?
       sigs.each do |sig|
         trap sig do
-          self_write.write("#{sig}\n")
+          self_write.puts(sig)
         end
       rescue ArgumentError
         puts "Signal #{sig} not supported"
@@ -162,15 +164,12 @@ module Sidekiq
         end
       },
     }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
-      handy = SIGNAL_HANDLERS[sig]
-      if handy
-        handy.call(self)
-      else
-        Sidekiq.logger.info { "No signal handler for #{sig}" }
-      end
+      SIGNAL_HANDLERS[sig].call(self)
     end
 
     private
@@ -204,7 +203,7 @@ module Sidekiq
 
       # check config file presence
       if opts[:config_file]
-        if opts[:config_file] && !File.exist?(opts[:config_file])
+        unless File.exist?(opts[:config_file])
           raise ArgumentError, "No such file #{opts[:config_file]}"
         end
       else
@@ -224,7 +223,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
+      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
@@ -283,8 +282,13 @@ module Sidekiq
 
     def parse_options(argv)
       opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
 
-      @parser = OptionParser.new { |o|
+    def option_parser(opts)
+      parser = OptionParser.new { |o|
         o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
           opts[:concurrency] = Integer(arg)
         end
@@ -336,15 +340,13 @@ module Sidekiq
         end
       }
 
-      @parser.banner = "sidekiq [options]"
-      @parser.on_tail "-h", "--help", "Show help" do
-        logger.info @parser
+      parser.banner = "sidekiq [options]"
+      parser.on_tail "-h", "--help", "Show help" do
+        logger.info parser
         die 1
       end
 
-      @parser.parse!(argv)
-
-      opts
+      parser
     end
 
     def initialize_logger

data/lib/sidekiq/client.rb

@@ -94,9 +94,14 @@ module Sidekiq
       return [] unless arg # no jobs to push
       raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
 
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+
       normed = normalize_item(items)
-      payloads = items["args"].map { |args|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = items["args"].map.with_index { |args, index|
+        single_at = at.is_a?(Array) ? at[index] : at
+        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "at" => single_at, "enqueued_at" => Time.now.to_f)
+
         result = process_single(items["class"], copy)
         result || nil
       }.compact
@@ -218,6 +223,7 @@ module Sidekiq
       raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
       raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
       normalized_hash(item["class"])

data/lib/sidekiq/fetch.rb

@@ -14,12 +14,12 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, "")
+        queue.delete_prefix("queue:")
       end
 
       def requeue
         Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+          conn.rpush(queue, job)
         end
       end
     }
@@ -28,7 +28,7 @@ module Sidekiq
       @strictly_ordered_queues = !!options[:strict]
       @queues = options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
+        @queues.uniq!
         @queues << TIMEOUT
       end
     end
@@ -47,7 +47,7 @@ module Sidekiq
       if @strictly_ordered_queues
         @queues
       else
-        queues = @queues.shuffle.uniq
+        queues = @queues.shuffle!.uniq
         queues << TIMEOUT
         queues
       end
@@ -61,14 +61,14 @@ module Sidekiq
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
         conn.pipelined do
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            conn.rpush(queue, jobs)
           end
         end
       end

data/lib/sidekiq/job_logger.rb

@@ -23,8 +23,15 @@ module Sidekiq
       raise
     end
 
-    def with_job_hash_context(job_hash, &block)
-      @logger.with_context(job_hash_context(job_hash), &block)
+    def prepare(job_hash, &block)
+      level = job_hash["log_level"]
+      if level
+        @logger.log_at(level) do
+          Sidekiq::Context.with(job_hash_context(job_hash), &block)
+        end
+      else
+        Sidekiq::Context.with(job_hash_context(job_hash), &block)
+      end
     end
 
     def job_hash_context(job_hash)
@@ -35,11 +42,12 @@ module Sidekiq
         jid: job_hash["jid"],
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
+      h[:tags] = job_hash["tags"] if job_hash["tags"]
       h
     end
 
     def with_elapsed_time_context(start, &block)
-      @logger.with_context(elapsed_time_context(start), &block)
+      Sidekiq::Context.with(elapsed_time_context(start), &block)
     end
 
     def elapsed_time_context(start)

data/lib/sidekiq/job_retry.rb

@@ -3,6 +3,9 @@
 require "sidekiq/scheduled"
 require "sidekiq/api"
 
+require "zlib"
+require "base64"
+
 module Sidekiq
   ##
   # Automatically retry jobs that fail in Sidekiq.
@@ -71,7 +74,7 @@ module Sidekiq
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
     # require the worker to be instantiated.
-    def global(msg, queue)
+    def global(jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -82,6 +85,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
         attempt_retry(nil, msg, queue, e)
       else
@@ -103,7 +107,7 @@ module Sidekiq
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, msg, queue)
+    def local(worker, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -114,6 +118,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
         msg["retry"] = worker.class.get_sidekiq_options["retry"]
       end
@@ -151,12 +156,14 @@ module Sidekiq
         msg["retry_count"] = 0
       end
 
-      if msg["backtrace"] == true
-        msg["error_backtrace"] = exception.backtrace
-      elsif !msg["backtrace"]
-        # do nothing
-      elsif msg["backtrace"].to_i != 0
-        msg["error_backtrace"] = exception.backtrace[0...msg["backtrace"].to_i]
+      if msg["backtrace"]
+        lines = if msg["backtrace"] == true
+          exception.backtrace
+        else
+          exception.backtrace[0...msg["backtrace"].to_i]
+        end
+
+        msg["error_backtrace"] = compress_backtrace(lines)
       end
 
       if count < max_retry_attempts
@@ -245,5 +252,11 @@ module Sidekiq
     rescue
       +"!!! ERROR MESSAGE THREW AN ERROR !!!"
     end
+
+    def compress_backtrace(backtrace)
+      serialized = Sidekiq.dump_json(backtrace)
+      compressed = Zlib::Deflate.deflate(serialized)
+      Base64.encode64(compressed)
+    end
   end
 end

data/lib/sidekiq/launcher.rb

@@ -129,15 +129,13 @@ module Sidekiq
         fails = procd = 0
 
         _, exists, _, _, msg = Sidekiq.redis { |conn|
-          res = conn.multi {
+          conn.multi {
             conn.sadd("processes", key)
             conn.exists(key)
             conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
           }
-
-          res
         }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat

data/lib/sidekiq/logger.rb

@@ -4,22 +4,109 @@ require "logger"
 require "time"
 
 module Sidekiq
-  class Logger < ::Logger
-    def initialize(*args)
-      super
+  module Context
+    def self.with(hash)
+      current.merge!(hash)
+      yield
+    ensure
+      hash.each_key { |key| current.delete(key) }
+    end
 
-      self.formatter = Sidekiq.log_formatter
+    def self.current
+      Thread.current[:sidekiq_context] ||= {}
+    end
+  end
+
+  module LoggingUtils
+    LEVELS = {
+      "debug" => 0,
+      "info" => 1,
+      "warn" => 2,
+      "error" => 3,
+      "fatal" => 4,
+    }
+    LEVELS.default_proc = proc do |_, level|
+      Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
+      nil
+    end
+
+    def debug?
+      level >= 0
+    end
+
+    def info?
+      level >= 1
+    end
+
+    def warn?
+      level >= 2
+    end
+
+    def error?
+      level >= 3
+    end
+
+    def fatal?
+      level >= 4
     end
 
-    def with_context(hash)
-      ctx.merge!(hash)
+    def local_level
+      Thread.current[:sidekiq_log_level]
+    end
+
+    def local_level=(level)
+      case level
+      when Integer
+        Thread.current[:sidekiq_log_level] = level
+      when Symbol, String
+        Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
+      when nil
+        Thread.current[:sidekiq_log_level] = nil
+      else
+        raise ArgumentError, "Invalid log level: #{level.inspect}"
+      end
+    end
+
+    def level
+      local_level || super
+    end
+
+    # Change the thread-local level for the duration of the given block.
+    def log_at(level)
+      old_local_level = local_level
+      self.local_level = level
       yield
     ensure
-      hash.keys.each { |key| ctx.delete(key) }
+      self.local_level = old_local_level
     end
 
-    def ctx
-      Thread.current[:sidekiq_context] ||= {}
+    # Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
+    # FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= UNKNOWN
+      progname ||= @progname
+
+      return true if @logdev.nil? || severity < level
+
+      if message.nil?
+        if block_given?
+          message = yield
+        else
+          message = progname
+          progname = @progname
+        end
+      end
+
+      @logdev.write format_message(format_severity(severity), Time.now, progname, message)
+    end
+  end
+
+  class Logger < ::Logger
+    include LoggingUtils
+
+    def initialize(*args)
+      super
+      self.formatter = Sidekiq.log_formatter
     end
 
     module Formatters
@@ -29,11 +116,20 @@ module Sidekiq
         end
 
         def ctx
-          Thread.current[:sidekiq_context] ||= {}
+          Sidekiq::Context.current
         end
 
         def format_context
-          " " + ctx.compact.map { |k, v| "#{k}=#{v}" }.join(" ") if ctx.any?
+          if ctx.any?
+            " " + ctx.compact.map { |k, v|
+              case v
+              when Array
+                "#{k}=#{v.join(",")}"
+              else
+                "#{k}=#{v}"
+              end
+            }.join(" ")
+          end
         end
       end
 

data/lib/sidekiq/middleware/chain.rb

@@ -67,7 +67,6 @@ module Sidekiq
   module Middleware
     class Chain
       include Enumerable
-      attr_reader :entries
 
       def initialize_copy(copy)
         copy.instance_variable_set(:@entries, entries.dup)
@@ -78,10 +77,14 @@ module Sidekiq
       end
 
       def initialize
-        @entries = []
+        @entries = nil
         yield self if block_given?
       end
 
+      def entries
+        @entries ||= []
+      end
+
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
@@ -114,6 +117,10 @@ module Sidekiq
         any? { |entry| entry.klass == klass }
       end
 
+      def empty?
+        @entries.nil? || @entries.empty?
+      end
+
       def retrieve
         map(&:make_new)
       end
@@ -123,6 +130,8 @@ module Sidekiq
       end
 
       def invoke(*args)
+        return yield if empty?
+
         chain = retrieve.dup
         traverse_chain = lambda do
           if chain.empty?

data/lib/sidekiq/monitor.rb

@@ -4,21 +4,6 @@ require "fileutils"
 require "sidekiq/api"
 
 class Sidekiq::Monitor
-  CMD = File.basename($PROGRAM_NAME)
-
-  attr_reader :stage
-
-  def self.print_usage
-    puts "#{CMD} - monitor Sidekiq from the command line."
-    puts
-    puts "Usage: #{CMD} status <section>"
-    puts
-    puts "       <section> (optional) view a specific section of the status output"
-    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
-    puts
-    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
-  end
-
   class Status
     VALID_SECTIONS = %w[all version overview processes queues]
     COL_PAD = 2
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
 
     def version
       puts "Sidekiq #{Sidekiq::VERSION}"
-      puts Time.now
+      puts Time.now.utc
     end
 
     def overview

data/lib/sidekiq/paginator.rb

@@ -12,10 +12,10 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         type = conn.type(key)
+        rev = opts && opts[:reverse]
 
         case type
         when "zset"
-          rev = opts && opts[:reverse]
           total_size, items = conn.multi {
             conn.zcard(key)
             if rev
@@ -28,8 +28,13 @@ module Sidekiq
         when "list"
           total_size, items = conn.multi {
             conn.llen(key)
-            conn.lrange(key, starting, ending)
+            if rev
+              conn.lrange(key, -ending - 1, -starting - 1)
+            else
+              conn.lrange(key, starting, ending)
+            end
           }
+          items.reverse! if rev
           [current_page, total_size, items]
         when "none"
           [1, 0, []]

data/lib/sidekiq/processor.rb

@@ -111,16 +111,19 @@ module Sidekiq
       nil
     end
 
-    def dispatch(job_hash, queue)
+    def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
-      # we clone here so we report the original
+      # we need to clone it to report the original
       # job structure to the Web UI
-      pristine = cloned(job_hash)
+      # or to push back to redis when retrying.
+      # To avoid costly and, most of the time, useless cloning here,
+      # we pass original String of JSON to respected methods
+      # to re-parse it there if we need access to the original, untouched job
 
-      @job_logger.with_job_hash_context(job_hash) do
-        @retrier.global(pristine, queue) do
+      @job_logger.prepare(job_hash) do
+        @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
-            stats(pristine, queue) do
+            stats(jobstr, queue) do
               # Rails 5 requires a Reloader to wrap code execution.  In order to
               # constantize the worker and instantiate an instance, we have to call
               # the Reloader.  It handles code loading, db connection management, etc.
@@ -129,7 +132,7 @@ module Sidekiq
                 klass = constantize(job_hash["class"])
                 worker = klass.new
                 worker.jid = job_hash["jid"]
-                @retrier.local(worker, pristine, queue) do
+                @retrier.local(worker, jobstr, queue) do
                   yield worker
                 end
               end
@@ -156,9 +159,9 @@ module Sidekiq
 
       ack = false
       begin
-        dispatch(job_hash, queue) do |worker|
+        dispatch(job_hash, queue, jobstr) do |worker|
           Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-            execute_job(worker, cloned(job_hash["args"]))
+            execute_job(worker, job_hash["args"])
           end
         end
         ack = true
@@ -247,8 +250,8 @@ module Sidekiq
     FAILURE = Counter.new
     WORKER_STATE = SharedWorkerState.new
 
-    def stats(job_hash, queue)
-      WORKER_STATE.set(tid, {queue: queue, payload: job_hash, run_at: Time.now.to_i})
+    def stats(jobstr, queue)
+      WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
 
       begin
         yield
@@ -261,21 +264,16 @@ module Sidekiq
       end
     end
 
-    # Deep clone the arguments passed to the worker so that if
-    # the job fails, what is pushed back onto Redis hasn't
-    # been mutated by the worker.
-    def cloned(thing)
-      Marshal.load(Marshal.dump(thing))
-    end
-
     def constantize(str)
+      return Object.const_get(str) unless str.include?("::")
+
       names = str.split("::")
       names.shift if names.empty? || names.first.empty?
 
       names.inject(Object) do |constant, name|
         # the false flag limits search for name to under the constant namespace
         #   which mimics Rails' behaviour
-        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+        constant.const_get(name, false)
       end
     end
   end

data/lib/sidekiq/scheduled.rb

@@ -14,18 +14,19 @@ module Sidekiq
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get the next item in the queue if it's score (time to execute) is <= now.
-            # We need to go through the list one at a time to reduce the risk of something
-            # going wrong between the time jobs are popped from the scheduled queue and when
-            # they are pushed onto a work queue and losing the jobs.
-            while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
-
-              # Pop item off the queue and add it to the work queue. If the job can't be popped from
-              # the queue, it's because another process already popped it so we can move on to the
-              # next one.
-              if conn.zrem(sorted_set, job)
-                Sidekiq::Client.push(Sidekiq.load_json(job))
-                Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+            # Get next items in the queue with scores (time to execute) <= now.
+            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
+              # We need to go through the list one at a time to reduce the risk of something
+              # going wrong between the time jobs are popped from the scheduled queue and when
+              # they are pushed onto a work queue and losing the jobs.
+              jobs.each do |job|
+                # Pop item off the queue and add it to the work queue. If the job can't be popped from
+                # the queue, it's because another process already popped it so we can move on to the
+                # next one.
+                if conn.zrem(sorted_set, job)
+                  Sidekiq::Client.push(Sidekiq.load_json(job))
+                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+                end
               end
             end
           end

data/lib/sidekiq/testing.rb

@@ -323,6 +323,18 @@ module Sidekiq
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+      end
+    end
+  end
+
+  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
+  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?

data/lib/sidekiq/util.rb

@@ -11,8 +11,6 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60 * 24
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.0"
+  VERSION = "6.0.2"
 end