sidekiq 4.2.10 → 5.2.7

data/lib/sidekiq.rb

@@ -1,12 +1,13 @@
-# encoding: utf-8
 # frozen_string_literal: true
+
 require 'sidekiq/version'
-fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.0.0." if RUBY_PLATFORM != 'java' && RUBY_VERSION < '2.0.0'
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.2.2." if RUBY_PLATFORM != 'java' && Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.2.2')
 
 require 'sidekiq/logging'
 require 'sidekiq/client'
 require 'sidekiq/worker'
 require 'sidekiq/redis_connection'
+require 'sidekiq/delay'
 
 require 'json'
 
@@ -17,13 +18,14 @@ module Sidekiq
   DEFAULTS = {
     queues: [],
     labels: [],
-    concurrency: 25,
+    concurrency: 10,
     require: '.',
     environment: nil,
     timeout: 8,
     poll_interval_average: nil,
-    average_scheduled_poll_interval: 15,
+    average_scheduled_poll_interval: 5,
     error_handlers: [],
+    death_handlers: [],
     lifecycle_events: {
       startup: [],
       quiet: [],
@@ -33,7 +35,6 @@ module Sidekiq
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
     reloader: proc { |&block| block.call },
-    executor: proc { |&block| block.call },
   }
 
   DEFAULT_WORKER_OPTIONS = {
@@ -47,7 +48,7 @@ module Sidekiq
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
     "used_memory_peak_human" => "9P"
-  }.freeze
+  }
 
   def self.❨╯°□°❩╯︵┻━┻
     puts "Calm down, yo."
@@ -56,6 +57,7 @@ module Sidekiq
   def self.options
     @options ||= DEFAULTS.dup
   end
+
   def self.options=(opts)
     @options = opts
   end
@@ -94,8 +96,8 @@ module Sidekiq
       begin
         yield conn
       rescue Redis::CommandError => ex
-        #2550 Failover can cause the server to become a slave, need
-        # to disconnect and reopen the socket to get back to the master.
+        #2550 Failover can cause the server to become a replica, need
+        # to disconnect and reopen the socket to get back to the primary.
         (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
         raise
       end
@@ -145,32 +147,34 @@ module Sidekiq
   end
 
   def self.default_server_middleware
-    require 'sidekiq/middleware/server/retry_jobs'
-    require 'sidekiq/middleware/server/logging'
-
-    Middleware::Chain.new do |m|
-      m.add Middleware::Server::RetryJobs
-      m.add Middleware::Server::Logging
-    end
+    Middleware::Chain.new
   end
 
   def self.default_worker_options=(hash)
-    @default_worker_options = default_worker_options.merge(hash.stringify_keys)
+    # stringify
+    @default_worker_options = default_worker_options.merge(Hash[hash.map{|k, v| [k.to_s, v]}])
   end
   def self.default_worker_options
     defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
   end
 
+  def self.default_retries_exhausted=(prok)
+    logger.info { "default_retries_exhausted is deprecated, please use `config.death_handlers << -> {|job, ex| }`" }
+    return nil unless prok
+    death_handlers << prok
+  end
+
+  ##
+  # Death handlers are called when all retries for a job have been exhausted and
+  # the job dies.  It's the notification to your application
+  # that this job will not succeed without manual intervention.
+  #
   # Sidekiq.configure_server do |config|
-  #   config.default_retries_exhausted = -> (job, ex) do
+  #   config.death_handlers << ->(job, ex) do
   #   end
   # end
-  def self.default_retries_exhausted=(prok)
-    @default_retries_exhausted = prok
-  end
-  @default_retries_exhausted = ->(job, ex) { }
-  def self.default_retries_exhausted
-    @default_retries_exhausted
+  def self.death_handlers
+    options[:death_handlers]
   end
 
   def self.load_json(string)
@@ -228,10 +232,6 @@ module Sidekiq
   # otherwise Ruby's Thread#kill will commit.  See #377.
   # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
   class Shutdown < Interrupt; end
-
 end
 
-require 'sidekiq/extensions/class_methods'
-require 'sidekiq/extensions/action_mailer'
-require 'sidekiq/extensions/active_record'
 require 'sidekiq/rails' if defined?(::Rails::Engine)

data/lib/sidekiq/api.rb

@@ -1,9 +1,24 @@
-# encoding: utf-8
 # frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
+
+  module RedisScanner
+    def sscan(conn, key)
+      cursor = '0'
+      result = []
+      loop do
+        cursor, values = conn.sscan(key, cursor)
+        result.push(*values)
+        break if cursor == '0'
+      end
+      result
+    end
+  end
+
   class Stats
+    include RedisScanner
+
     def initialize
       fetch_stats!
     end
@@ -51,33 +66,39 @@ module Sidekiq
     def fetch_stats!
       pipe1_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          conn.get('stat:processed'.freeze)
-          conn.get('stat:failed'.freeze)
-          conn.zcard('schedule'.freeze)
-          conn.zcard('retry'.freeze)
-          conn.zcard('dead'.freeze)
-          conn.scard('processes'.freeze)
-          conn.lrange('queue:default'.freeze, -1, -1)
-          conn.smembers('processes'.freeze)
-          conn.smembers('queues'.freeze)
+          conn.get('stat:processed')
+          conn.get('stat:failed')
+          conn.zcard('schedule')
+          conn.zcard('retry')
+          conn.zcard('dead')
+          conn.scard('processes')
+          conn.lrange('queue:default', -1, -1)
         end
       end
 
+      processes = Sidekiq.redis do |conn|
+        sscan(conn, 'processes')
+      end
+
+      queues = Sidekiq.redis do |conn|
+        sscan(conn, 'queues')
+      end
+
       pipe2_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          pipe1_res[7].each {|key| conn.hget(key, 'busy'.freeze) }
-          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+          processes.each {|key| conn.hget(key, 'busy') }
+          queues.each {|queue| conn.llen("queue:#{queue}") }
         end
       end
 
-      s = pipe1_res[7].size
+      s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
-                                job = Sidekiq.load_json(entry)
+                                job = Sidekiq.load_json(entry) rescue {}
                                 now = Time.now.to_f
-                                thence = job['enqueued_at'.freeze] || now
+                                thence = job['enqueued_at'] || now
                                 now - thence
                               else
                                 0
@@ -117,9 +138,11 @@ module Sidekiq
     end
 
     class Queues
+      include RedisScanner
+
       def lengths
         Sidekiq.redis do |conn|
-          queues = conn.smembers('queues'.freeze)
+          queues = sscan(conn, 'queues')
 
           lengths = conn.pipelined do
             queues.each do |queue|
@@ -146,11 +169,11 @@ module Sidekiq
       end
 
       def processed
-        date_stat_hash("processed")
+        @processed ||= date_stat_hash("processed")
       end
 
       def failed
-        date_stat_hash("failed")
+        @failed ||= date_stat_hash("failed")
       end
 
       private
@@ -163,16 +186,21 @@ module Sidekiq
 
         while i < @days_previous
           date = @start_date - i
-          datestr = date.strftime("%Y-%m-%d".freeze)
+          datestr = date.strftime("%Y-%m-%d")
           keys << "stat:#{stat}:#{datestr}"
           dates << datestr
           i += 1
         end
 
-        Sidekiq.redis do |conn|
-          conn.mget(keys).each_with_index do |value, idx|
-            stat_hash[dates[idx]] = value ? value.to_i : 0
+        begin
+          Sidekiq.redis do |conn|
+            conn.mget(keys).each_with_index do |value, idx|
+              stat_hash[dates[idx]] = value ? value.to_i : 0
+            end
           end
+        rescue Redis::CommandError
+          # mget will trigger a CROSSSLOT error when run against a Cluster
+          # TODO Someone want to add Cluster support?
         end
 
         stat_hash
@@ -194,18 +222,19 @@ module Sidekiq
   #
   class Queue
     include Enumerable
+    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| c.smembers('queues'.freeze) }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
     def initialize(name="default")
-      @name = name
+      @name = name.to_s
       @rname = "queue:#{name}"
     end
 
@@ -268,7 +297,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.multi do
           conn.del(@rname)
-          conn.srem("queues".freeze, name)
+          conn.srem("queues", name)
         end
       end
     end
@@ -287,13 +316,25 @@ module Sidekiq
     attr_reader :value
 
     def initialize(item, queue_name=nil)
+      @args = nil
       @value = item
-      @item = item.is_a?(Hash) ? item : Sidekiq.load_json(item)
+      @item = item.is_a?(Hash) ? item : parse(item)
       @queue = queue_name || @item['queue']
     end
 
+    def parse(item)
+      Sidekiq.load_json(item)
+    rescue JSON::ParserError
+      # If the job payload in Redis is invalid JSON, we'll load
+      # the item as an empty hash and store the invalid JSON as
+      # the job 'args' for display in the Web UI.
+      @invalid = true
+      @args = [item]
+      {}
+    end
+
     def klass
-      @item['class']
+      self['class']
     end
 
     def display_class
@@ -318,38 +359,42 @@ module Sidekiq
 
     def display_args
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
-      @args ||= case klass
+      @display_args ||= case klass
                 when /\ASidekiq::Extensions::Delayed/
                   safe_load(args[0], args) do |_, _, arg|
                     arg
                   end
                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
-                  job_args = @item['wrapped'] ? args[0]["arguments"] : []
-                  if 'ActionMailer::DeliveryJob' == (@item['wrapped'] || args[0])
-                   # remove MailerClass, mailer_method and 'deliver_now'
-                   job_args.drop(3)
+                  job_args = self['wrapped'] ? args[0]["arguments"] : []
+                  if 'ActionMailer::DeliveryJob' == (self['wrapped'] || args[0])
+                    # remove MailerClass, mailer_method and 'deliver_now'
+                    job_args.drop(3)
                   else
-                   job_args
+                    job_args
                   end
                 else
+                  if self['encrypt']
+                    # no point in showing 150+ bytes of random garbage
+                    args[-1] = '[encrypted data]'
+                  end
                   args
                 end
     end
 
     def args
-      @item['args']
+      @args || @item['args']
     end
 
     def jid
-      @item['jid']
+      self['jid']
     end
 
     def enqueued_at
-      @item['enqueued_at'] ? Time.at(@item['enqueued_at']).utc : nil
+      self['enqueued_at'] ? Time.at(self['enqueued_at']).utc : nil
     end
 
     def created_at
-      Time.at(@item['created_at'] || @item['enqueued_at'] || 0).utc
+      Time.at(self['created_at'] || self['enqueued_at'] || 0).utc
     end
 
     def queue
@@ -371,7 +416,10 @@ module Sidekiq
     end
 
     def [](name)
-      @item[name]
+      # nil will happen if the JSON fails to parse.
+      # We don't guarantee Sidekiq will work with bad job JSON but we should
+      # make a best effort to minimize the damage.
+      @item ? @item[name] : nil
     end
 
     private
@@ -434,14 +482,7 @@ module Sidekiq
     # Place job in the dead set
     def kill
       remove_job do |message|
-        now = Time.now.to_f
-        Sidekiq.redis do |conn|
-          conn.multi do
-            conn.zadd('dead', now, message)
-            conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
-            conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
-          end
-        end
+        DeadSet.new.kill(message)
       end
     end
 
@@ -531,7 +572,7 @@ module Sidekiq
         end
         break if elements.empty?
         page -= 1
-        elements.each do |element, score|
+        elements.reverse.each do |element, score|
           yield SortedEntry.new(self, score, element)
         end
         offset_size = initial_size - @_size
@@ -629,6 +670,12 @@ module Sidekiq
         each(&:retry)
       end
     end
+
+    def kill_all
+      while size > 0
+        each(&:kill)
+      end
+    end
   end
 
   ##
@@ -639,6 +686,27 @@ module Sidekiq
       super 'dead'
     end
 
+    def kill(message, opts={})
+      now = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.zadd(name, now.to_s, message)
+          conn.zremrangebyscore(name, '-inf', now - self.class.timeout)
+          conn.zremrangebyrank(name, 0, - self.class.max_jobs)
+        end
+      end
+
+      if opts[:notify_failure] != false
+        job = Sidekiq.load_json(message)
+        r = RuntimeError.new("Job killed by API")
+        r.set_backtrace(caller)
+        Sidekiq.death_handlers.each do |handle|
+          handle.call(job, r)
+        end
+      end
+      true
+    end
+
     def retry_all
       while size > 0
         each(&:retry)
@@ -663,17 +731,18 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
+    include RedisScanner
 
     def initialize(clean_plz=true)
-      self.class.cleanup if clean_plz
+      cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
-    def self.cleanup
+    def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes').sort
+        procs = sscan(conn, 'processes').sort
         heartbeats = conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'info')
@@ -693,7 +762,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+      procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -706,6 +775,11 @@ module Sidekiq
         end
 
         result.each do |info, busy, at_s, quiet|
+          # If a process is stopped between when we query Redis for `procs` and
+          # when we query for `result`, we will have an item in `result` that is
+          # composed of `nil` values.
+          next if info.nil?
+
           hash = Sidekiq.load_json(info)
           yield Process.new(hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f, 'quiet' => quiet))
         end
@@ -721,6 +795,18 @@ module Sidekiq
     def size
       Sidekiq.redis { |conn| conn.scard('processes') }
     end
+
+    # Returns the identity of the current cluster leader or "" if no leader.
+    # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
+    # or Sidekiq Pro.
+    def leader
+      @leader ||= begin
+        x = Sidekiq.redis {|c| c.get("dear-leader") }
+        # need a non-falsy value so we can memoize
+        x = "" unless x
+        x
+      end
+    end
   end
 
   #
@@ -755,8 +841,12 @@ module Sidekiq
       @attribs[key]
     end
 
+    def identity
+      self['identity']
+    end
+
     def quiet!
-      signal('USR1')
+      signal('TSTP')
     end
 
     def stop!
@@ -783,9 +873,6 @@ module Sidekiq
       end
     end
 
-    def identity
-      self['identity']
-    end
   end
 
   ##
@@ -810,10 +897,11 @@ module Sidekiq
   #
   class Workers
     include Enumerable
+    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         procs.sort.each do |key|
           valid, workers = conn.pipelined do
             conn.exists(key)
@@ -835,7 +923,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         if procs.empty?
           0
         else