sidekiq 6.0.0 → 6.0.6

data/lib/sidekiq/api.rb

@@ -2,23 +2,11 @@
 
 require "sidekiq"
 
-module Sidekiq
-  module RedisScanner
-    def sscan(conn, key)
-      cursor = "0"
-      result = []
-      loop do
-        cursor, values = conn.sscan(key, cursor)
-        result.push(*values)
-        break if cursor == "0"
-      end
-      result
-    end
-  end
+require "zlib"
+require "base64"
 
+module Sidekiq
   class Stats
-    include RedisScanner
-
     def initialize
       fetch_stats!
     end
@@ -77,11 +65,11 @@ module Sidekiq
       }
 
       processes = Sidekiq.redis { |conn|
-        sscan(conn, "processes")
+        conn.sscan_each("processes").to_a
       }
 
       queues = Sidekiq.redis { |conn|
-        sscan(conn, "queues")
+        conn.sscan_each("queues").to_a
       }
 
       pipe2_res = Sidekiq.redis { |conn|
@@ -92,8 +80,8 @@ module Sidekiq
       }
 
       s = processes.size
-      workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
-      enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
+      workers_size = pipe2_res[0...s].sum(&:to_i)
+      enqueued = pipe2_res[s..-1].sum(&:to_i)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
@@ -117,7 +105,7 @@ module Sidekiq
 
         default_queue_latency: default_queue_latency,
         workers_size: workers_size,
-        enqueued: enqueued,
+        enqueued: enqueued
       }
     end
 
@@ -142,11 +130,9 @@ module Sidekiq
     end
 
     class Queues
-      include RedisScanner
-
       def lengths
         Sidekiq.redis do |conn|
-          queues = sscan(conn, "queues")
+          queues = conn.sscan_each("queues").to_a
 
           lengths = conn.pipelined {
             queues.each do |queue|
@@ -154,13 +140,8 @@ module Sidekiq
             end
           }
 
-          i = 0
-          array_of_arrays = queues.each_with_object({}) { |queue, memo|
-            memo[queue] = lengths[i]
-            i += 1
-          }.sort_by { |_, size| size }
-
-          Hash[array_of_arrays.reverse]
+          array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
+          Hash[array_of_arrays]
         end
       end
     end
@@ -182,18 +163,12 @@ module Sidekiq
       private
 
       def date_stat_hash(stat)
-        i = 0
         stat_hash = {}
-        keys = []
-        dates = []
-
-        while i < @days_previous
-          date = @start_date - i
-          datestr = date.strftime("%Y-%m-%d")
-          keys << "stat:#{stat}:#{datestr}"
-          dates << datestr
-          i += 1
-        end
+        dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
+          date.strftime("%Y-%m-%d")
+        }
+
+        keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
 
         begin
           Sidekiq.redis do |conn|
@@ -225,13 +200,12 @@ module Sidekiq
   #
   class Queue
     include Enumerable
-    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| sscan(c, "queues") }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| c.sscan_each("queues").to_a }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
@@ -299,7 +273,7 @@ module Sidekiq
     def clear
       Sidekiq.redis do |conn|
         conn.multi do
-          conn.del(@rname)
+          conn.unlink(@rname)
           conn.srem("queues", name)
         end
       end
@@ -349,7 +323,7 @@ module Sidekiq
                    end
                  when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
                    job_class = @item["wrapped"] || args[0]
-                   if job_class == "ActionMailer::DeliveryJob"
+                   if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
                      # MailerClass#mailer_method
                      args[0]["arguments"][0..1].join("#")
                    else
@@ -372,6 +346,9 @@ module Sidekiq
                   if (self["wrapped"] || args[0]) == "ActionMailer::DeliveryJob"
                     # remove MailerClass, mailer_method and 'deliver_now'
                     job_args.drop(3)
+                  elsif (self["wrapped"] || args[0]) == "ActionMailer::MailDeliveryJob"
+                    # remove MailerClass, mailer_method and 'deliver_now'
+                    job_args.drop(3).first["args"]
                   else
                     job_args
                   end
@@ -400,6 +377,20 @@ module Sidekiq
       Time.at(self["created_at"] || self["enqueued_at"] || 0).utc
     end
 
+    def tags
+      self["tags"] || []
+    end
+
+    def error_backtrace
+      # Cache nil values
+      if defined?(@error_backtrace)
+        @error_backtrace
+      else
+        value = self["error_backtrace"]
+        @error_backtrace = value && uncompress_backtrace(value)
+      end
+    end
+
     attr_reader :queue
 
     def latency
@@ -433,6 +424,23 @@ module Sidekiq
       Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == "development"
       default
     end
+
+    def uncompress_backtrace(backtrace)
+      if backtrace.is_a?(Array)
+        # Handle old jobs with raw Array backtrace format
+        backtrace
+      else
+        decoded = Base64.decode64(backtrace)
+        uncompressed = Zlib::Inflate.inflate(decoded)
+        begin
+          Sidekiq.load_json(uncompressed)
+        rescue
+          # Handle old jobs with marshalled backtrace format
+          # TODO Remove in 7.x
+          Marshal.load(uncompressed)
+        end
+      end
+    end
   end
 
   class SortedEntry < Job
@@ -458,8 +466,9 @@ module Sidekiq
     end
 
     def reschedule(at)
-      delete
-      @parent.schedule(at, item)
+      Sidekiq.redis do |conn|
+        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
+      end
     end
 
     def add_to_queue
@@ -503,7 +512,7 @@ module Sidekiq
         else
           # multiple jobs with the same score
           # find the one with the right JID and push it
-          hash = results.group_by { |message|
+          matched, nonmatched = results.partition { |message|
             if message.index(jid)
               msg = Sidekiq.load_json(message)
               msg["jid"] == jid
@@ -512,12 +521,12 @@ module Sidekiq
             end
           }
 
-          msg = hash.fetch(true, []).first
+          msg = matched.first
           yield msg if msg
 
           # push the rest back onto the sorted set
           conn.multi do
-            hash.fetch(false, []).each do |message|
+            nonmatched.each do |message|
               conn.zadd(parent.name, score.to_f.to_s, message)
             end
           end
@@ -540,9 +549,20 @@ module Sidekiq
       Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    def scan(match, count = 100)
+      return to_enum(:scan, match, count) unless block_given?
+
+      match = "*#{match}*" unless match.include?("*")
+      Sidekiq.redis do |conn|
+        conn.zscan_each(name, match: match, count: count) do |entry, score|
+          yield SortedEntry.new(self, score, entry)
+        end
+      end
+    end
+
     def clear
       Sidekiq.redis do |conn|
-        conn.del(name)
+        conn.unlink(name)
       end
     end
     alias_method :💣, :clear
@@ -576,28 +596,40 @@ module Sidekiq
       end
     end
 
+    ##
+    # Fetch jobs that match a given time or Range. Job ID is an
+    # optional second argument.
     def fetch(score, jid = nil)
+      begin_score, end_score =
+        if score.is_a?(Range)
+          [score.first, score.last]
+        else
+          [score, score]
+        end
+
       elements = Sidekiq.redis { |conn|
-        conn.zrangebyscore(name, score, score)
+        conn.zrangebyscore(name, begin_score, end_score, with_scores: true)
       }
 
       elements.each_with_object([]) do |element, result|
-        entry = SortedEntry.new(self, score, element)
-        if jid
-          result << entry if entry.jid == jid
-        else
-          result << entry
-        end
+        data, job_score = element
+        entry = SortedEntry.new(self, job_score, data)
+        result << entry if jid.nil? || entry.jid == jid
       end
     end
 
     ##
     # Find the job with the given JID within this sorted set.
-    #
-    # This is a slow, inefficient operation.  Do not use under
-    # normal conditions.  Sidekiq Pro contains a faster version.
+    # This is a slower O(n) operation.  Do not use for app logic.
     def find_job(jid)
-      detect { |j| j.jid == jid }
+      Sidekiq.redis do |conn|
+        conn.zscan_each(name, match: "*#{jid}*", count: 100) do |entry, score|
+          job = JSON.parse(entry)
+          matched = job["jid"] == jid
+          return SortedEntry.new(self, score, entry) if matched
+        end
+      end
+      nil
     end
 
     def delete_by_value(name, value)
@@ -612,11 +644,13 @@ module Sidekiq
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
         elements.each do |element|
-          message = Sidekiq.load_json(element)
-          if message["jid"] == jid
-            ret = conn.zrem(name, element)
-            @_size -= 1 if ret
-            break ret
+          if element.index(jid)
+            message = Sidekiq.load_json(element)
+            if message["jid"] == jid
+              ret = conn.zrem(name, element)
+              @_size -= 1 if ret
+              break ret
+            end
           end
         end
       end
@@ -720,7 +754,6 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
-    include RedisScanner
 
     def initialize(clean_plz = true)
       cleanup if clean_plz
@@ -731,7 +764,7 @@ module Sidekiq
     def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes").sort
+        procs = conn.sscan_each("processes").to_a.sort
         heartbeats = conn.pipelined {
           procs.each do |key|
             conn.hget(key, "info")
@@ -741,40 +774,37 @@ module Sidekiq
         # the hash named key has an expiry of 60 seconds.
         # if it's not found, that means the process has not reported
         # in to Redis and probably died.
-        to_prune = []
-        heartbeats.each_with_index do |beat, i|
-          to_prune << procs[i] if beat.nil?
-        end
+        to_prune = procs.select.with_index { |proc, i|
+          heartbeats[i].nil?
+        }
         count = conn.srem("processes", to_prune) unless to_prune.empty?
       end
       count
     end
 
     def each
-      procs = Sidekiq.redis { |conn| sscan(conn, "processes") }.sort
+      result = Sidekiq.redis { |conn|
+        procs = conn.sscan_each("processes").to_a.sort
 
-      Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
         # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
         # you'll be happier this way
-        result = conn.pipelined {
+        conn.pipelined do
           procs.each do |key|
             conn.hmget(key, "info", "busy", "beat", "quiet")
           end
-        }
+        end
+      }
 
-        result.each do |info, busy, at_s, quiet|
-          # If a process is stopped between when we query Redis for `procs` and
-          # when we query for `result`, we will have an item in `result` that is
-          # composed of `nil` values.
-          next if info.nil?
+      result.each do |info, busy, at_s, quiet|
+        # If a process is stopped between when we query Redis for `procs` and
+        # when we query for `result`, we will have an item in `result` that is
+        # composed of `nil` values.
+        next if info.nil?
 
-          hash = Sidekiq.load_json(info)
-          yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
-        end
+        hash = Sidekiq.load_json(info)
+        yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
       end
-
-      nil
     end
 
     # This method is not guaranteed accurate since it does not prune the set
@@ -885,11 +915,10 @@ module Sidekiq
   #
   class Workers
     include Enumerable
-    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes")
+        procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
           valid, workers = conn.pipelined {
             conn.exists(key)
@@ -897,7 +926,11 @@ module Sidekiq
           }
           next unless valid
           workers.each_pair do |tid, json|
-            yield key, tid, Sidekiq.load_json(json)
+            hsh = Sidekiq.load_json(json)
+            p = hsh["payload"]
+            # avoid breaking API, this is a side effect of the JSON optimization in #4316
+            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
+            yield key, tid, hsh
           end
         end
       end
@@ -911,7 +944,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = sscan(conn, "processes")
+        procs = conn.sscan_each("processes").to_a
         if procs.empty?
           0
         else
@@ -919,7 +952,7 @@ module Sidekiq
             procs.each do |key|
               conn.hget(key, "busy")
             end
-          }.map(&:to_i).inject(:+)
+          }.sum(&:to_i)
         end
       end
     end

data/lib/sidekiq/cli.rb

@@ -38,12 +38,15 @@ module Sidekiq
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << "USR2" unless jruby?
       sigs.each do |sig|
         trap sig do
-          self_write.write("#{sig}\n")
+          self_write.puts(sig)
         end
       rescue ArgumentError
         puts "Signal #{sig} not supported"
@@ -56,7 +59,7 @@ module Sidekiq
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info["redis_version"]
-      raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
+      raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -160,17 +163,14 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
-      handy = SIGNAL_HANDLERS[sig]
-      if handy
-        handy.call(self)
-      else
-        Sidekiq.logger.info { "No signal handler for #{sig}" }
-      end
+      SIGNAL_HANDLERS[sig].call(self)
     end
 
     private
@@ -182,7 +182,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -204,7 +208,7 @@ module Sidekiq
 
       # check config file presence
       if opts[:config_file]
-        if opts[:config_file] && !File.exist?(opts[:config_file])
+        unless File.exist?(opts[:config_file])
           raise ArgumentError, "No such file #{opts[:config_file]}"
         end
       else
@@ -224,7 +228,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = Array(opts[:queues]) << "default" if opts[:queues].nil? || opts[:queues].empty?
+      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
       opts[:strict] = true if opts[:strict].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
@@ -283,8 +287,13 @@ module Sidekiq
 
     def parse_options(argv)
       opts = {}
+      @parser = option_parser(opts)
+      @parser.parse!(argv)
+      opts
+    end
 
-      @parser = OptionParser.new { |o|
+    def option_parser(opts)
+      parser = OptionParser.new { |o|
         o.on "-c", "--concurrency INT", "processor threads to use" do |arg|
           opts[:concurrency] = Integer(arg)
         end
@@ -336,15 +345,13 @@ module Sidekiq
         end
       }
 
-      @parser.banner = "sidekiq [options]"
-      @parser.on_tail "-h", "--help", "Show help" do
-        logger.info @parser
+      parser.banner = "sidekiq [options]"
+      parser.on_tail "-h", "--help", "Show help" do
+        logger.info parser
         die 1
       end
 
-      @parser.parse!(argv)
-
-      opts
+      parser
     end
 
     def initialize_logger
@@ -376,5 +383,11 @@ module Sidekiq
       [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -94,9 +94,14 @@ module Sidekiq
       return [] unless arg # no jobs to push
       raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
 
+      at = items.delete("at")
+      raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+
       normed = normalize_item(items)
-      payloads = items["args"].map { |args|
+      payloads = items["args"].map.with_index { |args, index|
         copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
+
         result = process_single(items["class"], copy)
         result || nil
       }.compact
@@ -188,7 +193,7 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first["at"]
+      if payloads.first.key?("at")
         conn.zadd("schedule", payloads.map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
@@ -214,19 +219,30 @@ module Sidekiq
     end
 
     def normalize_item(item)
+      # 6.0.0 push_bulk bug, #4321
+      # TODO Remove after a while...
+      item.delete("at") if item.key?("at") && item["at"].nil?
+
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
       raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
       raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item["class"])
-        .each { |key, value| item[key] = value if item[key].nil? }
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["jid"] ||= SecureRandom.hex(12)
       item["created_at"] ||= Time.now.to_f
+
       item
     end
 

data/lib/sidekiq/fetch.rb

@@ -14,12 +14,12 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, "")
+        queue.delete_prefix("queue:")
       end
 
       def requeue
         Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", job)
+          conn.rpush(queue, job)
         end
       end
     }
@@ -28,7 +28,7 @@ module Sidekiq
       @strictly_ordered_queues = !!options[:strict]
       @queues = options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
-        @queues = @queues.uniq
+        @queues.uniq!
         @queues << TIMEOUT
       end
     end
@@ -47,7 +47,7 @@ module Sidekiq
       if @strictly_ordered_queues
         @queues
       else
-        queues = @queues.shuffle.uniq
+        queues = @queues.shuffle!.uniq
         queues << TIMEOUT
         queues
       end
@@ -61,14 +61,14 @@ module Sidekiq
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
-        jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+        jobs_to_requeue[unit_of_work.queue] ||= []
+        jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
         conn.pipelined do
           jobs_to_requeue.each do |queue, jobs|
-            conn.rpush("queue:#{queue}", jobs)
+            conn.rpush(queue, jobs)
           end
         end
       end