sidekiq 2.17.8 → 3.0.0

data/lib/sidekiq/processor.rb

@@ -11,7 +11,9 @@ module Sidekiq
   # processes it.  It instantiates the worker, runs the middleware
   # chain and then calls Sidekiq::Worker#perform.
   class Processor
-    STATS_TIMEOUT = 180 * 24 * 60 * 60
+    # To prevent a memory leak, ensure that stats expire. However, they should take up a minimal amount of storage
+    # so keep them around for a long time
+    STATS_TIMEOUT = 24 * 60 * 60 * 365 * 5
 
     include Util
     include Actor
@@ -34,32 +36,30 @@ module Sidekiq
       msgstr = work.message
       queue = work.queue_name
 
-      do_defer do
-        @boss.async.real_thread(proxy_id, Thread.current)
+      @boss.async.real_thread(proxy_id, Thread.current)
 
-        ack = true
-        begin
-          msg = Sidekiq.load_json(msgstr)
-          klass  = msg['class'].constantize
-          worker = klass.new
-          worker.jid = msg['jid']
-
-          stats(worker, msg, queue) do
-            Sidekiq.server_middleware.invoke(worker, msg, queue) do
-              worker.perform(*cloned(msg['args']))
-            end
+      ack = true
+      begin
+        msg = Sidekiq.load_json(msgstr)
+        klass  = msg['class'].constantize
+        worker = klass.new
+        worker.jid = msg['jid']
+
+        stats(worker, msg, queue) do
+          Sidekiq.server_middleware.invoke(worker, msg, queue) do
+            worker.perform(*cloned(msg['args']))
           end
-        rescue Sidekiq::Shutdown
-          # Had to force kill this job because it didn't finish
-          # within the timeout.  Don't acknowledge the work since
-          # we didn't properly finish it.
-          ack = false
-        rescue Exception => ex
-          handle_exception(ex, msg || { :message => msgstr })
-          raise
-        ensure
-          work.acknowledge if ack
         end
+      rescue Sidekiq::Shutdown
+        # Had to force kill this job because it didn't finish
+        # within the timeout.  Don't acknowledge the work since
+        # we didn't properly finish it.
+        ack = false
+      rescue Exception => ex
+        handle_exception(ex, msg || { :message => msgstr })
+        raise
+      ensure
+        work.acknowledge if ack
       end
 
       @boss.async.processor_done(current_actor)
@@ -71,35 +71,19 @@ module Sidekiq
 
     private
 
-    # We use Celluloid's defer to workaround tiny little
-    # Fiber stacks (4kb!) in MRI 1.9.
-    #
-    # For some reason, Celluloid's thread dispatch, TaskThread,
-    # is unstable under heavy concurrency but TaskFiber has proven
-    # itself stable.
-    NEED_DEFER = (RUBY_ENGINE == 'ruby' && RUBY_VERSION < '2.0.0')
-
-    def do_defer(&block)
-      if NEED_DEFER
-        defer(&block)
-      else
-        yield
-      end
-    end
-
-    def identity
-      @str ||= "#{hostname}:#{process_id}-#{Thread.current.object_id}:default"
+    def thread_identity
+      @str ||= Thread.current.object_id.to_s(36)
     end
 
     def stats(worker, msg, queue)
-      # Do not conflate errors from the job with errors caused by updating stats so calling code can react appropriately
+      # Do not conflate errors from the job with errors caused by updating
+      # stats so calling code can react appropriately
       retry_and_suppress_exceptions do
-        redis do |conn|
+        hash = Sidekiq.dump_json({:queue => queue, :payload => msg, :run_at => Time.now.to_i })
+        Sidekiq.redis do |conn|
           conn.multi do
-            conn.sadd('workers', identity)
-            conn.setex("worker:#{identity}:started", EXPIRY, Time.now.to_s)
-            hash = {:queue => queue, :payload => msg, :run_at => Time.now.to_i }
-            conn.setex("worker:#{identity}", EXPIRY, Sidekiq.dump_json(hash))
+            conn.hmset("#{identity}:workers", thread_identity, hash)
+            conn.expire("#{identity}:workers", 60*60)
           end
         end
       end
@@ -108,7 +92,7 @@ module Sidekiq
         yield
       rescue Exception
         retry_and_suppress_exceptions do
-          redis do |conn|
+          Sidekiq.redis do |conn|
             failed = "stat:failed:#{Time.now.utc.to_date}"
             result = conn.multi do
               conn.incrby("stat:failed", 1)
@@ -120,12 +104,10 @@ module Sidekiq
         raise
       ensure
         retry_and_suppress_exceptions do
-          redis do |conn|
+          Sidekiq.redis do |conn|
             processed = "stat:processed:#{Time.now.utc.to_date}"
             result = conn.multi do
-              conn.srem("workers", identity)
-              conn.del("worker:#{identity}")
-              conn.del("worker:#{identity}:started")
+              conn.hdel("#{identity}:workers", thread_identity)
               conn.incrby("stat:processed", 1)
               conn.incrby(processed, 1)
             end
@@ -158,7 +140,7 @@ module Sidekiq
           sleep(1)
           retry
         else
-          Sidekiq.logger.info {"Exhausted #{max_retries} retries due to Redis timeouts: #{e.inspect}"}
+          handle_exception(e, { :message => "Exhausted #{max_retries} retries"})
         end
       end
     end

data/lib/sidekiq/redis_connection.rb

@@ -68,9 +68,7 @@ module Sidekiq
       end
 
       def determine_redis_provider
-        # REDISTOGO_URL is only support for legacy reasons
-        provider = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
-        ENV[provider] || ENV['REDISTOGO_URL']
+        ENV[ENV['REDIS_PROVIDER'] || 'REDIS_URL']
       end
 
     end

data/lib/sidekiq/util.rb

@@ -26,12 +26,12 @@ module Sidekiq
       Sidekiq.redis(&block)
     end
 
-    def process_id
-      @@process_id ||= SecureRandom.hex
-    end
-
     def hostname
       Socket.gethostname
     end
+
+    def identity
+      @@identity ||= "#{hostname}:#{$$}"
+    end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.17.8"
+  VERSION = "3.0.0"
 end

data/lib/sidekiq/web.rb

@@ -20,10 +20,11 @@ module Sidekiq
 
     DEFAULT_TABS = {
       "Dashboard" => '',
-      "Workers"   => 'workers',
+      "Busy"      => 'busy',
       "Queues"    => 'queues',
       "Retries"   => 'retries',
       "Scheduled" => 'scheduled',
+      "Dead"      => 'morgue',
     }
 
     class << self
@@ -37,8 +38,8 @@ module Sidekiq
       alias_method :tabs, :custom_tabs
     end
 
-    get "/workers" do
-      erb :workers
+    get "/busy" do
+      erb :busy
     end
 
     get "/queues" do
@@ -55,11 +56,6 @@ module Sidekiq
       erb :queue
     end
 
-    post "/reset" do
-      reset_worker_list
-      redirect root_path
-    end
-
     post "/queues/:name" do
       Sidekiq::Queue.new(params[:name]).clear
       redirect "#{root_path}queues"
@@ -70,6 +66,59 @@ module Sidekiq
       redirect_with_query("#{root_path}queues/#{params[:name]}")
     end
 
+    get '/morgue' do
+      @count = (params[:count] || 25).to_i
+      (@current_page, @total_size, @dead) = page("dead", params[:page], @count)
+      @dead = @dead.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      erb :morgue
+    end
+
+    get "/morgue/:key" do
+      halt 404 unless params['key']
+      @dead = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
+      redirect "#{root_path}morgue" if @dead.nil?
+      erb :dead
+    end
+
+    post '/morgue' do
+      halt 404 unless params['key']
+
+      params['key'].each do |key|
+        job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
+        next unless job
+        if params['retry']
+          job.retry
+        elsif params['delete']
+          job.delete
+        end
+      end
+      redirect_with_query("#{root_path}morgue")
+    end
+
+    post "/morgue/all/delete" do
+      Sidekiq::DeadSet.new.clear
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/all/retry" do
+      Sidekiq::DeadSet.new.retry_all
+      redirect "#{root_path}morgue"
+    end
+
+    post "/morgue/:key" do
+      halt 404 unless params['key']
+      job = Sidekiq::DeadSet.new.fetch(*parse_params(params['key'])).first
+      if job
+        if params['retry']
+          job.retry
+        elsif params['delete']
+          job.delete
+        end
+      end
+      redirect_with_query("#{root_path}morgue")
+    end
+
+
     get '/retries' do
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @retries) = page("retry", params[:page], @count)

data/lib/sidekiq/web_helpers.rb

@@ -31,28 +31,19 @@ module Sidekiq
 
     def t(msg, options={})
       string = get_locale[msg] || msg
-      string % options
-    end
-
-    def reset_worker_list
-      Sidekiq.redis do |conn|
-        workers = conn.smembers('workers')
-        conn.srem('workers', workers) if !workers.empty?
+      if options.empty?
+        string
+      else
+        string % options
       end
     end
 
     def workers_size
-      @workers_size ||= Sidekiq.redis do |conn|
-        conn.scard('workers')
-      end
+      @workers_size ||= workers.size
     end
 
     def workers
-      @workers ||= begin
-        Sidekiq::Workers.new.tap do |w|
-          w.prune
-        end
-      end
+      @workers ||= Sidekiq::Workers.new
     end
 
     def stats

data/lib/sidekiq/worker.rb

@@ -62,6 +62,7 @@ module Sidekiq
       #   :retry - enable the RetryJobs middleware for this Worker, default *true*
       #   :backtrace - whether to save any error backtrace in the retry payload to display in web UI,
       #      can be true, false or an integer number of lines to save, default *false*
+      #   :pool - use the given Redis connection pool to push this type of job to a given shard.
       def sidekiq_options(opts={})
         self.sidekiq_options_hash = get_sidekiq_options.merge((opts || {}).stringify_keys)
         ::Sidekiq.logger.warn("#{self.name} - :timeout is unsafe and support has been removed from Sidekiq, see http://bit.ly/OtYpK for details") if opts.include? :timeout
@@ -80,7 +81,8 @@ module Sidekiq
       end
 
       def client_push(item) # :nodoc:
-        Sidekiq::Client.push(item.stringify_keys)
+        pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
+        Sidekiq::Client.new(pool).push(item.stringify_keys)
       end
 
     end

data/sidekiq.gemspec

@@ -14,14 +14,14 @@ Gem::Specification.new do |gem|
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
-  gem.add_dependency                  'redis', '~> 3.1'
-  gem.add_dependency                  'redis-namespace', '~> 1.3'
-  gem.add_dependency                  'connection_pool', '~> 2.0'
-  gem.add_dependency                  'celluloid', '0.15.2'
+  gem.add_dependency                  'redis', '>= 3.0.6'
+  gem.add_dependency                  'redis-namespace', '>= 1.3.1'
+  gem.add_dependency                  'connection_pool', '>= 2.0.0'
+  gem.add_dependency                  'celluloid', '>= 0.15.2'
   gem.add_dependency                  'json'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'minitest', '~> 4.2'
   gem.add_development_dependency      'rake'
-  gem.add_development_dependency      'rails', '4.0.5'
+  gem.add_development_dependency      'rails', '>= 4.0.0'
   gem.add_development_dependency      'coveralls'
 end

data/test/test_api.rb

@@ -1,6 +1,7 @@
 require 'helper'
 
 class TestApi < Sidekiq::Test
+
   describe "stats" do
     before do
       Sidekiq.redis {|c| c.flushdb }
@@ -160,6 +161,8 @@ class TestApi < Sidekiq::Test
   end
 
   describe 'with an empty database' do
+    include Sidekiq::Util
+
     before do
       Sidekiq.redis {|c| c.flushdb }
     end
@@ -339,6 +342,23 @@ class TestApi < Sidekiq::Test
       assert_equal 0, r.size
     end
 
+    it 'can enumerate processes' do
+      odata = { 'pid' => 123, 'hostname' => hostname, 'key' => "#{hostname}:123", 'started_at' => Time.now.to_f - 15 }
+      time = Time.now.to_f
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.sadd('processes', odata['key'])
+          conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
+          conn.sadd('processes', 'fake:pid')
+        end
+      end
+
+      ps = Sidekiq::ProcessSet.new.to_a
+      assert_equal 1, ps.size
+      data = ps.first
+      assert_equal odata.merge('busy' => 10, 'beat' => time), data
+    end
+
     it 'can enumerate workers' do
       w = Sidekiq::Workers.new
       assert_equal 0, w.size
@@ -346,38 +366,36 @@ class TestApi < Sidekiq::Test
         assert false
       end
 
-      s = '12345'
+      key = "#{hostname}:#{$$}"
+      pdata = { 'pid' => $$, 'hostname' => hostname, 'started_at' => Time.now.to_i }
+      Sidekiq.redis do |conn|
+        conn.sadd('processes', key)
+        conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
+      end
+
+      s = "#{key}:workers"
       data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
       Sidekiq.redis do |c|
-        c.multi do
-          c.sadd('workers', s)
-          c.set("worker:#{s}", data)
-          c.set("worker:#{s}:started", Time.now.to_s)
-        end
+        c.hmset(s, '1234', data)
       end
 
-      assert_equal 1, w.size
-      w.each do |x, y, z|
-        assert_equal s, x
+      w.each do |p, x, y|
+        assert_equal key, p
+        assert_equal "1234", x
         assert_equal 'default', y['queue']
-        assert_equal Time.now.year, DateTime.parse(z).year
+        assert_equal Time.now.year, Time.at(y['run_at']).year
       end
 
-      s = '12346'
+      s = "#{key}:workers"
       data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
       Sidekiq.redis do |c|
         c.multi do
-          c.sadd('workers', s)
-          c.set("worker:#{s}", data)
-          c.set("worker:#{s}:started", Time.now.to_s)
-          c.sadd('workers', '123457')
+          c.hmset(s, '5678', data)
+          c.hmset("b#{s}", '5678', data)
         end
       end
 
-      assert_equal 3, w.size
-      count = w.prune
-      assert_equal 1, w.size
-      assert_equal 2, count
+      assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
     end
 
     it 'can reschedule jobs' do
@@ -396,6 +414,28 @@ class TestApi < Sidekiq::Test
       assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
     end
 
+    it 'prunes processes which have died' do
+      data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
+      key = "#{data['hostname']}:#{data['pid']}"
+      Sidekiq.redis do |conn|
+        conn.sadd('processes', key)
+        conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
+      end
+
+      ps = Sidekiq::ProcessSet.new
+      assert_equal 1, ps.size
+      assert_equal 1, ps.to_a.size
+
+      Sidekiq.redis do |conn|
+        conn.sadd('processes', "bar:987")
+        conn.sadd('processes', "bar:986")
+      end
+
+      ps = Sidekiq::ProcessSet.new
+      assert_equal 3, ps.size
+      assert_equal 1, ps.to_a.size
+    end
+
     def add_retry(jid = 'bob', at = Time.now.to_f)
       payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
       Sidekiq.redis do |conn|