sidekiq 2.15.1 → 4.2.10

data/lib/generators/sidekiq/templates/worker.rb.erb

@@ -0,0 +1,9 @@
+<% module_namespacing do -%>
+class <%= class_name %>Worker
+  include Sidekiq::Worker
+
+  def perform(*args)
+    # Do something
+  end
+end
+<% end -%>

data/lib/generators/sidekiq/templates/worker_spec.rb.erb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Worker, type: :worker do
+    pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/sidekiq/templates/worker_test.rb.erb

@@ -0,0 +1,8 @@
+require 'test_helper'
+<% module_namespacing do -%>
+class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
+  def test_example
+    skip "add some examples to (or delete) #{__FILE__}"
+  end
+end
+<% end -%>

data/lib/generators/sidekiq/worker_generator.rb

@@ -0,0 +1,49 @@
+require 'rails/generators/named_base'
+
+module Sidekiq
+  module Generators # :nodoc:
+    class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
+      desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
+
+      check_class_collision suffix: 'Worker'
+
+      def self.default_generator_root
+        File.dirname(__FILE__)
+      end
+
+      def create_worker_file
+        template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
+      end
+
+      def create_test_file
+        if defined?(RSpec)
+          create_worker_spec
+        else
+          create_worker_test
+        end
+      end
+
+      private
+
+      def create_worker_spec
+        template_file = File.join(
+            'spec/workers',
+            class_path,
+            "#{file_name}_worker_spec.rb"
+        )
+        template 'worker_spec.rb.erb', template_file
+      end
+
+      def create_worker_test
+        template_file = File.join(
+            'test/workers',
+            class_path,
+            "#{file_name}_worker_test.rb"
+        )
+        template 'worker_test.rb.erb', template_file
+      end
+
+
+    end
+  end
+end

data/lib/sidekiq.rb

@@ -1,35 +1,61 @@
 # encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq/version'
+fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.0.0." if RUBY_PLATFORM != 'java' && RUBY_VERSION < '2.0.0'
+
 require 'sidekiq/logging'
 require 'sidekiq/client'
 require 'sidekiq/worker'
 require 'sidekiq/redis_connection'
-require 'sidekiq/util'
-require 'sidekiq/api'
 
 require 'json'
 
 module Sidekiq
-  NAME = "Sidekiq"
+  NAME = 'Sidekiq'
   LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
 
   DEFAULTS = {
-    :queues => [],
-    :concurrency => 25,
-    :require => '.',
-    :environment => nil,
-    :timeout => 8,
-    :profile => false,
+    queues: [],
+    labels: [],
+    concurrency: 25,
+    require: '.',
+    environment: nil,
+    timeout: 8,
+    poll_interval_average: nil,
+    average_scheduled_poll_interval: 15,
+    error_handlers: [],
+    lifecycle_events: {
+      startup: [],
+      quiet: [],
+      shutdown: [],
+      heartbeat: [],
+    },
+    dead_max_jobs: 10_000,
+    dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
+    reloader: proc { |&block| block.call },
+    executor: proc { |&block| block.call },
+  }
+
+  DEFAULT_WORKER_OPTIONS = {
+    'retry' => true,
+    'queue' => 'default'
   }
 
+  FAKE_INFO = {
+    "redis_version" => "9.9.9",
+    "uptime_in_days" => "9999",
+    "connected_clients" => "9999",
+    "used_memory_human" => "9P",
+    "used_memory_peak_human" => "9P"
+  }.freeze
+
   def self.❨╯°□°❩╯︵┻━┻
-    puts "Calm down, bro"
+    puts "Calm down, yo."
   end
 
   def self.options
     @options ||= DEFAULTS.dup
   end
-
   def self.options=(opts)
     @options = opts
   end
@@ -61,46 +87,95 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" if !block
-    @redis ||= Sidekiq::RedisConnection.create(@hash || {})
-    @redis.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
-  def self.redis=(hash)
-    return @redis = hash if hash.is_a?(ConnectionPool)
+  def self.redis_info
+    redis do |conn|
+      begin
+        # admin commands can't go through redis-namespace starting
+        # in redis-namespace 2.0
+        if conn.respond_to?(:namespace)
+          conn.redis.info
+        else
+          conn.info
+        end
+      rescue Redis::CommandError => ex
+        #2850 return fake version when INFO command has (probably) been renamed
+        raise unless ex.message =~ /unknown command/
+        FAKE_INFO
+      end
+    end
+  end
 
-    if hash.is_a?(Hash)
-      @hash = hash
+  def self.redis_pool
+    @redis ||= Sidekiq::RedisConnection.create
+  end
+
+  def self.redis=(hash)
+    @redis = if hash.is_a?(ConnectionPool)
+      hash
     else
-      raise ArgumentError, "redis= requires a Hash or ConnectionPool"
+      Sidekiq::RedisConnection.create(hash)
     end
   end
 
   def self.client_middleware
-    @client_chain ||= Client.default_middleware
+    @client_chain ||= Middleware::Chain.new
     yield @client_chain if block_given?
     @client_chain
   end
 
   def self.server_middleware
-    @server_chain ||= Processor.default_middleware
+    @server_chain ||= default_server_middleware
     yield @server_chain if block_given?
     @server_chain
   end
 
-  def self.default_worker_options=(hash)
-    @default_worker_options = default_worker_options.merge(hash)
+  def self.default_server_middleware
+    require 'sidekiq/middleware/server/retry_jobs'
+    require 'sidekiq/middleware/server/logging'
+
+    Middleware::Chain.new do |m|
+      m.add Middleware::Server::RetryJobs
+      m.add Middleware::Server::Logging
+    end
   end
 
+  def self.default_worker_options=(hash)
+    @default_worker_options = default_worker_options.merge(hash.stringify_keys)
+  end
   def self.default_worker_options
-    @default_worker_options || { 'retry' => true, 'queue' => 'default' }
+    defined?(@default_worker_options) ? @default_worker_options : DEFAULT_WORKER_OPTIONS
+  end
+
+  # Sidekiq.configure_server do |config|
+  #   config.default_retries_exhausted = -> (job, ex) do
+  #   end
+  # end
+  def self.default_retries_exhausted=(prok)
+    @default_retries_exhausted = prok
+  end
+  @default_retries_exhausted = ->(job, ex) { }
+  def self.default_retries_exhausted
+    @default_retries_exhausted
   end
 
   def self.load_json(string)
     JSON.parse(string)
   end
-
   def self.dump_json(object)
     JSON.generate(object)
   end
@@ -108,15 +183,52 @@ module Sidekiq
   def self.logger
     Sidekiq::Logging.logger
   end
-
   def self.logger=(log)
     Sidekiq::Logging.logger = log
   end
 
-  def self.poll_interval=(interval)
-    self.options[:poll_interval] = interval
+  # How frequently Redis should be checked by a random Sidekiq process for
+  # scheduled and retriable jobs. Each individual process will take turns by
+  # waiting some multiple of this value.
+  #
+  # See sidekiq/scheduled.rb for an in-depth explanation of this value
+  def self.average_scheduled_poll_interval=(interval)
+    self.options[:average_scheduled_poll_interval] = interval
   end
 
+  # Register a proc to handle any error which occurs within the Sidekiq process.
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+  #   end
+  #
+  # The default error handler logs errors to Sidekiq.logger.
+  def self.error_handlers
+    self.options[:error_handlers]
+  end
+
+  # Register a block to run at a point in the Sidekiq lifecycle.
+  # :startup, :quiet or :shutdown are valid events.
+  #
+  #   Sidekiq.configure_server do |config|
+  #     config.on(:shutdown) do
+  #       puts "Goodbye cruel world!"
+  #     end
+  #   end
+  def self.on(event, &block)
+    raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
+    raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
+    options[:lifecycle_events][event] << block
+  end
+
+  # We are shutting down Sidekiq but what about workers that
+  # are working on some long job?  This error is
+  # raised in workers that have not finished within the hard
+  # timeout limit.  This is needed to rollback db transactions,
+  # otherwise Ruby's Thread#kill will commit.  See #377.
+  # DO NOT RESCUE THIS ERROR IN YOUR WORKERS
+  class Shutdown < Interrupt; end
+
 end
 
 require 'sidekiq/extensions/class_methods'

data/lib/sidekiq/api.rb

@@ -1,45 +1,142 @@
+# encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
   class Stats
+    def initialize
+      fetch_stats!
+    end
+
     def processed
-      Sidekiq.redis { |conn| conn.get("stat:processed") }.to_i
+      stat :processed
     end
 
     def failed
-      Sidekiq.redis { |conn| conn.get("stat:failed") }.to_i
+      stat :failed
     end
 
-    def reset
-      Sidekiq.redis do |conn|
-        conn.set("stat:failed", 0)
-        conn.set("stat:processed", 0)
-      end
+    def scheduled_size
+      stat :scheduled_size
+    end
+
+    def retry_size
+      stat :retry_size
+    end
+
+    def dead_size
+      stat :dead_size
+    end
+
+    def enqueued
+      stat :enqueued
+    end
+
+    def processes_size
+      stat :processes_size
+    end
+
+    def workers_size
+      stat :workers_size
+    end
+
+    def default_queue_latency
+      stat :default_queue_latency
     end
 
     def queues
-      Sidekiq.redis do |conn|
-        queues = conn.smembers('queues')
+      Sidekiq::Stats::Queues.new.lengths
+    end
+
+    def fetch_stats!
+      pipe1_res = Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.get('stat:processed'.freeze)
+          conn.get('stat:failed'.freeze)
+          conn.zcard('schedule'.freeze)
+          conn.zcard('retry'.freeze)
+          conn.zcard('dead'.freeze)
+          conn.scard('processes'.freeze)
+          conn.lrange('queue:default'.freeze, -1, -1)
+          conn.smembers('processes'.freeze)
+          conn.smembers('queues'.freeze)
+        end
+      end
 
-        array_of_arrays = queues.inject({}) do |memo, queue|
-          memo[queue] = conn.llen("queue:#{queue}")
-          memo
-        end.sort_by { |_, size| size }
+      pipe2_res = Sidekiq.redis do |conn|
+        conn.pipelined do
+          pipe1_res[7].each {|key| conn.hget(key, 'busy'.freeze) }
+          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+        end
+      end
 
-        Hash[array_of_arrays.reverse]
+      s = pipe1_res[7].size
+      workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
+      enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
+
+      default_queue_latency = if (entry = pipe1_res[6].first)
+                                job = Sidekiq.load_json(entry)
+                                now = Time.now.to_f
+                                thence = job['enqueued_at'.freeze] || now
+                                now - thence
+                              else
+                                0
+                              end
+      @stats = {
+        processed:             pipe1_res[0].to_i,
+        failed:                pipe1_res[1].to_i,
+        scheduled_size:        pipe1_res[2],
+        retry_size:            pipe1_res[3],
+        dead_size:             pipe1_res[4],
+        processes_size:        pipe1_res[5],
+
+        default_queue_latency: default_queue_latency,
+        workers_size:          workers_size,
+        enqueued:              enqueued
+      }
+    end
+
+    def reset(*stats)
+      all   = %w(failed processed)
+      stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
+
+      mset_args = []
+      stats.each do |stat|
+        mset_args << "stat:#{stat}"
+        mset_args << 0
+      end
+      Sidekiq.redis do |conn|
+        conn.mset(*mset_args)
       end
     end
 
-    def enqueued
-      queues.values.inject(&:+) || 0
-    end
+    private
 
-    def scheduled_size
-      Sidekiq.redis {|c| c.zcard('schedule') }
+    def stat(s)
+      @stats[s]
     end
 
-    def retry_size
-      Sidekiq.redis {|c| c.zcard('retry') }
+    class Queues
+      def lengths
+        Sidekiq.redis do |conn|
+          queues = conn.smembers('queues'.freeze)
+
+          lengths = conn.pipelined do
+            queues.each do |queue|
+              conn.llen("queue:#{queue}")
+            end
+          end
+
+          i = 0
+          array_of_arrays = queues.inject({}) do |memo, queue|
+            memo[queue] = lengths[i]
+            i += 1
+            memo
+          end.sort_by { |_, size| size }
+
+          Hash[array_of_arrays.reverse]
+        end
+      end
     end
 
     class History
@@ -61,15 +158,20 @@ module Sidekiq
       def date_stat_hash(stat)
         i = 0
         stat_hash = {}
+        keys = []
+        dates = []
+
+        while i < @days_previous
+          date = @start_date - i
+          datestr = date.strftime("%Y-%m-%d".freeze)
+          keys << "stat:#{stat}:#{datestr}"
+          dates << datestr
+          i += 1
+        end
 
         Sidekiq.redis do |conn|
-          while i < @days_previous
-            date = @start_date - i
-            value = conn.get("stat:#{stat}:#{date}")
-
-            stat_hash[date.to_s] = value ? value.to_i : 0
-
-            i += 1
+          conn.mget(keys).each_with_index do |value, idx|
+            stat_hash[dates[idx]] = value ? value.to_i : 0
           end
         end
 
@@ -93,8 +195,11 @@ module Sidekiq
   class Queue
     include Enumerable
 
+    ##
+    # Return all known queues within Redis.
+    #
     def self.all
-      Sidekiq.redis {|c| c.smembers('queues') }.map {|q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| c.smembers('queues'.freeze) }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
@@ -108,42 +213,66 @@ module Sidekiq
       Sidekiq.redis { |con| con.llen(@rname) }
     end
 
+    # Sidekiq Pro overrides this
+    def paused?
+      false
+    end
+
+    ##
+    # Calculates this queue's latency, the difference in seconds since the oldest
+    # job in the queue was enqueued.
+    #
+    # @return Float
     def latency
       entry = Sidekiq.redis do |conn|
         conn.lrange(@rname, -1, -1)
       end.first
       return 0 unless entry
-      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+      job = Sidekiq.load_json(entry)
+      now = Time.now.to_f
+      thence = job['enqueued_at'] || now
+      now - thence
     end
 
-    def each(&block)
+    def each
+      initial_size = size
+      deleted_size = 0
       page = 0
       page_size = 50
 
-      loop do
+      while true do
+        range_start = page * page_size - deleted_size
+        range_end   = range_start + page_size - 1
         entries = Sidekiq.redis do |conn|
-          conn.lrange @rname, page * page_size, (page * page_size) + page_size - 1
+          conn.lrange @rname, range_start, range_end
         end
         break if entries.empty?
         page += 1
         entries.each do |entry|
-          block.call Job.new(entry, @name)
+          yield Job.new(entry, @name)
         end
+        deleted_size = initial_size - size
       end
     end
 
+    ##
+    # Find the job with the given JID within this queue.
+    #
+    # This is a slow, inefficient operation.  Do not use under
+    # normal conditions.  Sidekiq Pro contains a faster version.
     def find_job(jid)
-      self.detect { |j| j.jid == jid }
+      detect { |j| j.jid == jid }
     end
 
     def clear
       Sidekiq.redis do |conn|
         conn.multi do
           conn.del(@rname)
-          conn.srem("queues", name)
+          conn.srem("queues".freeze, name)
         end
       end
     end
+    alias_method :💣, :clear
   end
 
   ##
@@ -155,10 +284,11 @@ module Sidekiq
   #
   class Job
     attr_reader :item
+    attr_reader :value
 
     def initialize(item, queue_name=nil)
       @value = item
-      @item = Sidekiq.load_json(item)
+      @item = item.is_a?(Hash) ? item : Sidekiq.load_json(item)
       @queue = queue_name || @item['queue']
     end
 
@@ -166,6 +296,46 @@ module Sidekiq
       @item['class']
     end
 
+    def display_class
+      # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
+      @klass ||= case klass
+                 when /\ASidekiq::Extensions::Delayed/
+                   safe_load(args[0], klass) do |target, method, _|
+                     "#{target}.#{method}"
+                   end
+                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+                   job_class = @item['wrapped'] || args[0]
+                   if 'ActionMailer::DeliveryJob' == job_class
+                     # MailerClass#mailer_method
+                     args[0]['arguments'][0..1].join('#')
+                   else
+                    job_class
+                   end
+                 else
+                   klass
+                 end
+    end
+
+    def display_args
+      # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
+      @args ||= case klass
+                when /\ASidekiq::Extensions::Delayed/
+                  safe_load(args[0], args) do |_, _, arg|
+                    arg
+                  end
+                when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+                  job_args = @item['wrapped'] ? args[0]["arguments"] : []
+                  if 'ActionMailer::DeliveryJob' == (@item['wrapped'] || args[0])
+                   # remove MailerClass, mailer_method and 'deliver_now'
+                   job_args.drop(3)
+                  else
+                   job_args
+                  end
+                else
+                  args
+                end
+    end
+
     def args
       @item['args']
     end
@@ -175,7 +345,11 @@ module Sidekiq
     end
 
     def enqueued_at
-      Time.at(@item['enqueued_at'] || 0).utc
+      @item['enqueued_at'] ? Time.at(@item['enqueued_at']).utc : nil
+    end
+
+    def created_at
+      Time.at(@item['created_at'] || @item['enqueued_at'] || 0).utc
     end
 
     def queue
@@ -183,25 +357,40 @@ module Sidekiq
     end
 
     def latency
-      Time.now.to_f - @item['enqueued_at']
+      now = Time.now.to_f
+      now - (@item['enqueued_at'] || @item['created_at'] || now)
     end
 
     ##
     # Remove this job from the queue.
     def delete
       count = Sidekiq.redis do |conn|
-        conn.lrem("queue:#{@queue}", 0, @value)
+        conn.lrem("queue:#{@queue}", 1, @value)
       end
       count != 0
     end
 
     def [](name)
-      @item.send(:[], name)
+      @item[name]
+    end
+
+    private
+
+    def safe_load(content, default)
+      begin
+        yield(*YAML.load(content))
+      rescue => ex
+        # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
+        # memory yet so the YAML can't be loaded.
+        Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == 'development'
+        default
+      end
     end
   end
 
   class SortedEntry < Job
     attr_reader :score
+    attr_reader :parent
 
     def initialize(parent, score, item)
       super(item)
@@ -214,76 +403,144 @@ module Sidekiq
     end
 
     def delete
-      @parent.delete(score, jid)
+      if @value
+        @parent.delete_by_value(@parent.name, @value)
+      else
+        @parent.delete_by_jid(score, jid)
+      end
     end
 
     def reschedule(at)
-      @parent.delete(score, jid)
+      delete
       @parent.schedule(at, item)
     end
 
     def add_to_queue
-      Sidekiq.redis do |conn|
-        results = conn.zrangebyscore('schedule', score, score)
-        conn.zremrangebyscore('schedule', score, score)
-        results.map do |message|
-          msg = Sidekiq.load_json(message)
-          Sidekiq::Client.push(msg)
-        end
+      remove_job do |message|
+        msg = Sidekiq.load_json(message)
+        Sidekiq::Client.push(msg)
       end
     end
 
     def retry
-      raise "Retry not available on jobs not in the Retry queue." unless item["failed_at"]
+      remove_job do |message|
+        msg = Sidekiq.load_json(message)
+        msg['retry_count'] -= 1 if msg['retry_count']
+        Sidekiq::Client.push(msg)
+      end
+    end
+
+    ##
+    # Place job in the dead set
+    def kill
+      remove_job do |message|
+        now = Time.now.to_f
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.zadd('dead', now, message)
+            conn.zremrangebyscore('dead', '-inf', now - DeadSet.timeout)
+            conn.zremrangebyrank('dead', 0, - DeadSet.max_jobs)
+          end
+        end
+      end
+    end
+
+    def error?
+      !!item['error_class']
+    end
+
+    private
+
+    def remove_job
       Sidekiq.redis do |conn|
-        results = conn.zrangebyscore('retry', score, score)
-        conn.zremrangebyscore('retry', score, score)
-        results.map do |message|
-          msg = Sidekiq.load_json(message)
-          msg['retry_count'] = msg['retry_count'] - 1
-          Sidekiq::Client.push(msg)
+        results = conn.multi do
+          conn.zrangebyscore(parent.name, score, score)
+          conn.zremrangebyscore(parent.name, score, score)
+        end.first
+
+        if results.size == 1
+          yield results.first
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          hash = results.group_by do |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg['jid'] == jid
+            else
+              false
+            end
+          end
+
+          msg = hash.fetch(true, []).first
+          yield msg if msg
+
+          # push the rest back onto the sorted set
+          conn.multi do
+            hash.fetch(false, []).each do |message|
+              conn.zadd(parent.name, score.to_f.to_s, message)
+            end
+          end
         end
       end
     end
+
   end
 
   class SortedSet
     include Enumerable
 
+    attr_reader :name
+
     def initialize(name)
-      @zset = name
+      @name = name
+      @_size = size
     end
 
     def size
-      Sidekiq.redis {|c| c.zcard(@zset) }
+      Sidekiq.redis { |c| c.zcard(name) }
     end
 
+    def clear
+      Sidekiq.redis do |conn|
+        conn.del(name)
+      end
+    end
+    alias_method :💣, :clear
+  end
+
+  class JobSet < SortedSet
+
     def schedule(timestamp, message)
       Sidekiq.redis do |conn|
-        conn.zadd(@zset, timestamp.to_f.to_s, Sidekiq.dump_json(message))
+        conn.zadd(name, timestamp.to_f.to_s, Sidekiq.dump_json(message))
       end
     end
 
-    def each(&block)
-      # page thru the sorted set backwards so deleting entries doesn't screw up indexing
+    def each
+      initial_size = @_size
+      offset_size = 0
       page = -1
       page_size = 50
 
-      loop do
+      while true do
+        range_start = page * page_size + offset_size
+        range_end   = range_start + page_size - 1
         elements = Sidekiq.redis do |conn|
-          conn.zrange @zset, page * page_size, (page * page_size) + (page_size - 1), :with_scores => true
+          conn.zrange name, range_start, range_end, with_scores: true
         end
         break if elements.empty?
         page -= 1
         elements.each do |element, score|
-          block.call SortedEntry.new(self, score, element)
+          yield SortedEntry.new(self, score, element)
         end
+        offset_size = initial_size - @_size
       end
     end
 
     def fetch(score, jid = nil)
       elements = Sidekiq.redis do |conn|
-        conn.zrangebyscore(@zset, score, score)
+        conn.zrangebyscore(name, score, score)
       end
 
       elements.inject([]) do |result, element|
@@ -297,52 +554,54 @@ module Sidekiq
       end
     end
 
+    ##
+    # Find the job with the given JID within this sorted set.
+    #
+    # This is a slow, inefficient operation.  Do not use under
+    # normal conditions.  Sidekiq Pro contains a faster version.
     def find_job(jid)
       self.detect { |j| j.jid == jid }
     end
 
-    def delete(score, jid = nil)
-      if jid
-        elements = Sidekiq.redis do |conn|
-          conn.zrangebyscore(@zset, score, score)
-        end
+    def delete_by_value(name, value)
+      Sidekiq.redis do |conn|
+        ret = conn.zrem(name, value)
+        @_size -= 1 if ret
+        ret
+      end
+    end
 
-        elements_with_jid = elements.map do |element|
+    def delete_by_jid(score, jid)
+      Sidekiq.redis do |conn|
+        elements = conn.zrangebyscore(name, score, score)
+        elements.each do |element|
           message = Sidekiq.load_json(element)
-
           if message["jid"] == jid
-            Sidekiq.redis { |conn| conn.zrem(@zset, element) }
+            ret = conn.zrem(name, element)
+            @_size -= 1 if ret
+            break ret
           end
+          false
         end
-        elements_with_jid.count != 0
-      else
-        count = Sidekiq.redis do |conn|
-          conn.zremrangebyscore(@zset, score, score)
-        end
-        count != 0
       end
     end
 
-    def clear
-      Sidekiq.redis do |conn|
-        conn.del(@zset)
-      end
-    end
+    alias_method :delete, :delete_by_jid
   end
 
   ##
   # Allows enumeration of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
+  # and deleting them from the schedule queue.
   #
   #   r = Sidekiq::ScheduledSet.new
-  #   r.select do |retri|
-  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
-  #     retri.args[0] == 'User' &&
-  #     retri.args[1] == 'setup_new_subscriber'
+  #   r.select do |scheduled|
+  #     scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     scheduled.args[0] == 'User' &&
+  #     scheduled.args[1] == 'setup_new_subscriber'
   #   end.map(&:delete)
-  class ScheduledSet < SortedSet
+  class ScheduledSet < JobSet
     def initialize
       super 'schedule'
     end
@@ -360,7 +619,7 @@ module Sidekiq
   #     retri.args[0] == 'User' &&
   #     retri.args[1] == 'setup_new_subscriber'
   #   end.map(&:delete)
-  class RetrySet < SortedSet
+  class RetrySet < JobSet
     def initialize
       super 'retry'
     end
@@ -372,8 +631,165 @@ module Sidekiq
     end
   end
 
+  ##
+  # Allows enumeration of dead jobs within Sidekiq.
+  #
+  class DeadSet < JobSet
+    def initialize
+      super 'dead'
+    end
+
+    def retry_all
+      while size > 0
+        each(&:retry)
+      end
+    end
+
+    def self.max_jobs
+      Sidekiq.options[:dead_max_jobs]
+    end
+
+    def self.timeout
+      Sidekiq.options[:dead_timeout_in_seconds]
+    end
+  end
 
   ##
+  # Enumerates the set of Sidekiq processes which are actively working
+  # right now.  Each process send a heartbeat to Redis every 5 seconds
+  # so this set should be relatively accurate, barring network partitions.
+  #
+  # Yields a Sidekiq::Process.
+  #
+  class ProcessSet
+    include Enumerable
+
+    def initialize(clean_plz=true)
+      self.class.cleanup if clean_plz
+    end
+
+    # Cleans up dead processes recorded in Redis.
+    # Returns the number of processes cleaned.
+    def self.cleanup
+      count = 0
+      Sidekiq.redis do |conn|
+        procs = conn.smembers('processes').sort
+        heartbeats = conn.pipelined do
+          procs.each do |key|
+            conn.hget(key, 'info')
+          end
+        end
+
+        # the hash named key has an expiry of 60 seconds.
+        # if it's not found, that means the process has not reported
+        # in to Redis and probably died.
+        to_prune = []
+        heartbeats.each_with_index do |beat, i|
+          to_prune << procs[i] if beat.nil?
+        end
+        count = conn.srem('processes', to_prune) unless to_prune.empty?
+      end
+      count
+    end
+
+    def each
+      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+
+      Sidekiq.redis do |conn|
+        # We're making a tradeoff here between consuming more memory instead of
+        # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
+        # you'll be happier this way
+        result = conn.pipelined do
+          procs.each do |key|
+            conn.hmget(key, 'info', 'busy', 'beat', 'quiet')
+          end
+        end
+
+        result.each do |info, busy, at_s, quiet|
+          hash = Sidekiq.load_json(info)
+          yield Process.new(hash.merge('busy' => busy.to_i, 'beat' => at_s.to_f, 'quiet' => quiet))
+        end
+      end
+
+      nil
+    end
+
+    # This method is not guaranteed accurate since it does not prune the set
+    # based on current heartbeat.  #each does that and ensures the set only
+    # contains Sidekiq processes which have sent a heartbeat within the last
+    # 60 seconds.
+    def size
+      Sidekiq.redis { |conn| conn.scard('processes') }
+    end
+  end
+
+  #
+  # Sidekiq::Process represents an active Sidekiq process talking with Redis.
+  # Each process has a set of attributes which look like this:
+  #
+  # {
+  #   'hostname' => 'app-1.example.com',
+  #   'started_at' => <process start time>,
+  #   'pid' => 12345,
+  #   'tag' => 'myapp'
+  #   'concurrency' => 25,
+  #   'queues' => ['default', 'low'],
+  #   'busy' => 10,
+  #   'beat' => <last heartbeat>,
+  #   'identity' => <unique string identifying the process>,
+  # }
+  class Process
+    def initialize(hash)
+      @attribs = hash
+    end
+
+    def tag
+      self['tag']
+    end
+
+    def labels
+      Array(self['labels'])
+    end
+
+    def [](key)
+      @attribs[key]
+    end
+
+    def quiet!
+      signal('USR1')
+    end
+
+    def stop!
+      signal('TERM')
+    end
+
+    def dump_threads
+      signal('TTIN')
+    end
+
+    def stopping?
+      self['quiet'] == 'true'
+    end
+
+    private
+
+    def signal(sig)
+      key = "#{identity}-signals"
+      Sidekiq.redis do |c|
+        c.multi do
+          c.lpush(key, sig)
+          c.expire(key, 60)
+        end
+      end
+    end
+
+    def identity
+      self['identity']
+    end
+  end
+
+  ##
+  # A worker is a thread that is currently processing a job.
   # Programmatic access to the current active worker set.
   #
   # WARNING WARNING WARNING
@@ -384,34 +800,52 @@ module Sidekiq
   #
   #    workers = Sidekiq::Workers.new
   #    workers.size => 2
-  #    workers.each do |name, work, started_at|
-  #      # name is a unique identifier per worker
+  #    workers.each do |process_id, thread_id, work|
+  #      # process_id is a unique identifier per Sidekiq process
+  #      # thread_id is a unique identifier per thread
   #      # work is a Hash which looks like:
   #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
-  #      # started_at is a String rep of the time when the worker started working on the job
+  #      # run_at is an epoch Integer.
   #    end
-
+  #
   class Workers
     include Enumerable
 
-    def each(&block)
+    def each
       Sidekiq.redis do |conn|
-        workers = conn.smembers("workers")
-        workers.each do |w|
-          msg, time = conn.multi do
-            conn.get("worker:#{w}")
-            conn.get("worker:#{w}:started")
+        procs = conn.smembers('processes')
+        procs.sort.each do |key|
+          valid, workers = conn.pipelined do
+            conn.exists(key)
+            conn.hgetall("#{key}:workers")
+          end
+          next unless valid
+          workers.each_pair do |tid, json|
+            yield key, tid, Sidekiq.load_json(json)
           end
-          next unless msg
-          block.call(w, Sidekiq.load_json(msg), time)
         end
       end
     end
 
+    # Note that #size is only as accurate as Sidekiq's heartbeat,
+    # which happens every 5 seconds.  It is NOT real-time.
+    #
+    # Not very efficient if you have lots of Sidekiq
+    # processes but the alternative is a global counter
+    # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        conn.scard("workers")
-      end.to_i
+        procs = conn.smembers('processes')
+        if procs.empty?
+          0
+        else
+          conn.pipelined do
+            procs.each do |key|
+              conn.hget(key, 'busy')
+            end
+          end.map(&:to_i).inject(:+)
+        end
+      end
     end
   end