sidekiq 2.5.4 → 2.6.0

data/test/config.yml

@@ -3,6 +3,7 @@
 :environment:  xzibit
 :require:      ./test/fake_env.rb
 :pidfile:      /tmp/sidekiq-config-test.pid
+:logfile:      /tmp/sidekiq.log
 :concurrency:  50
 :queues:
   - [<%="very_"%>often, 2]

data/test/test_api.rb

@@ -1,6 +1,149 @@
 require 'helper'
 
 class TestApi < MiniTest::Unit::TestCase
+  describe "stats" do
+    before do
+      Sidekiq.redis {|c| c.flushdb }
+    end
+
+    describe "processed" do
+      it "is initially zero" do
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.processed
+      end
+
+      it "returns number of processed jobs" do
+        Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
+        s = Sidekiq::Stats.new
+        assert_equal 5, s.processed
+      end
+    end
+
+    describe "failed" do
+      it "is initially zero" do
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.processed
+      end
+
+      it "returns number of failed jobs" do
+        Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
+        s = Sidekiq::Stats.new
+        assert_equal 5, s.failed
+      end
+    end
+
+    describe "queues" do
+      it "is initially empty" do
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.queues.size
+      end
+
+      it "returns a hash of queue and size in order" do
+        Sidekiq.redis do |conn|
+          conn.rpush 'queue:foo', '{}'
+          conn.sadd 'queues', 'foo'
+
+          3.times { conn.rpush 'queue:bar', '{}' }
+          conn.sadd 'queues', 'bar'
+        end
+
+        s = Sidekiq::Stats.new
+        assert_equal ({ "foo" => 1, "bar" => 3 }), s.queues
+        assert_equal "bar", s.queues.first.first
+      end
+    end
+
+    describe "enqueued" do
+      it "is initially empty" do
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.enqueued
+      end
+
+      it "returns total enqueued jobs" do
+        Sidekiq.redis do |conn|
+          conn.rpush 'queue:foo', '{}'
+          conn.sadd 'queues', 'foo'
+
+          3.times { conn.rpush 'queue:bar', '{}' }
+          conn.sadd 'queues', 'bar'
+        end
+
+        s = Sidekiq::Stats.new
+        assert_equal 4, s.enqueued
+      end
+    end
+
+    describe "over time" do
+      describe "processed" do
+        it 'retrieves hash of dates' do
+          Sidekiq.redis do |c|
+            c.incrby("stat:processed:2012-12-24", 4)
+            c.incrby("stat:processed:2012-12-25", 1)
+            c.incrby("stat:processed:2012-12-26", 6)
+            c.incrby("stat:processed:2012-12-27", 2)
+          end
+          Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
+            s = Sidekiq::Stats::History.new(2)
+            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.processed
+
+            s = Sidekiq::Stats::History.new(3)
+            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
+
+            s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
+            assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
+          end
+        end
+      end
+
+      describe "failed" do
+        it 'retrieves hash of dates' do
+          Sidekiq.redis do |c|
+            c.incrby("stat:failed:2012-12-24", 4)
+            c.incrby("stat:failed:2012-12-25", 1)
+            c.incrby("stat:failed:2012-12-26", 6)
+            c.incrby("stat:failed:2012-12-27", 2)
+          end
+          Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
+            s = Sidekiq::Stats::History.new(2)
+            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
+
+            s = Sidekiq::Stats::History.new(3)
+            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
+
+            s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
+            assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
+          end
+        end
+      end
+
+      describe "cleanup" do
+        it 'removes processed stats outside of keep window' do
+          Sidekiq.redis do |c|
+            c.incrby("stat:processed:2012-05-03", 4)
+            c.incrby("stat:processed:2012-06-03", 4)
+            c.incrby("stat:processed:2012-07-03", 1)
+          end
+          Time.stub(:now, Time.parse("2012-12-01 1:00:00 -0500")) do
+            Sidekiq::Stats::History.cleanup
+            assert_equal false, Sidekiq.redis { |c| c.exists("stat:processed:2012-05-03") }
+          end
+        end
+
+        it 'removes failed stats outside of keep window' do
+          Sidekiq.redis do |c|
+            c.incrby("stat:failed:2012-05-03", 4)
+            c.incrby("stat:failed:2012-06-03", 4)
+            c.incrby("stat:failed:2012-07-03", 1)
+          end
+          Time.stub(:now, Time.parse("2012-12-01 1:00:00 -0500")) do
+            Sidekiq::Stats::History.cleanup
+            assert_equal false, Sidekiq.redis { |c| c.exists("stat:failed:2012-05-03") }
+          end
+        end
+      end
+    end
+  end
+
   describe 'with an empty database' do
     before do
       Sidekiq.redis {|c| c.flushdb }
@@ -47,6 +190,24 @@ class TestApi < MiniTest::Unit::TestCase
       end
     end
 
+    it 'can fetch by score' do
+      same_time = Time.now.to_f
+      add_retry('bob1', same_time)
+      add_retry('bob2', same_time)
+      r = Sidekiq::RetrySet.new
+      assert_equal 2, r.fetch(same_time).size
+    end
+
+    it 'can fetch by score and jid' do
+      same_time = Time.now.to_f
+      add_retry('bob1', same_time)
+      add_retry('bob2', same_time)
+      r = Sidekiq::RetrySet.new
+      # jobs = r.fetch(same_time)
+      # puts jobs[1].jid
+      assert_equal 1, r.fetch(same_time, 'bob1').size
+    end
+
     it 'shows empty retries' do
       r = Sidekiq::RetrySet.new
       assert_equal 0, r.size
@@ -67,12 +228,36 @@ class TestApi < MiniTest::Unit::TestCase
       assert_in_delta Time.now.to_f, retri.at.to_f, 0.01
     end
 
-    it 'can delete retries' do
+    it 'can delete multiple retries from score' do
+      same_time = Time.now.to_f
+      add_retry('bob1', same_time)
+      add_retry('bob2', same_time)
+      r = Sidekiq::RetrySet.new
+      assert_equal 2, r.size
+      Sidekiq::RetrySet.new.delete(same_time)
+      assert_equal 0, r.size
+    end
+
+    it 'can delete a single retry from score and jid' do
+      same_time = Time.now.to_f
+      add_retry('bob1', same_time)
+      add_retry('bob2', same_time)
+      r = Sidekiq::RetrySet.new
+      assert_equal 2, r.size
+      Sidekiq::RetrySet.new.delete(same_time, 'bob1')
+      assert_equal 1, r.size
+    end
+
+    it 'can retry a retry' do
       add_retry
       r = Sidekiq::RetrySet.new
       assert_equal 1, r.size
-      r.map(&:delete)
+      r.first.retry
       assert_equal 0, r.size
+      assert_equal 1, Sidekiq::Queue.new('default').size
+      job = Sidekiq::Queue.new('default').first
+      assert_equal 'bob', job.jid
+      assert_equal 1, job['retry_count']
     end
 
     it 'can clear retries' do
@@ -84,9 +269,8 @@ class TestApi < MiniTest::Unit::TestCase
       assert_equal 0, r.size
     end
 
-    def add_retry(jid = 'bob')
-      at = Time.now.to_f
-      payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid)
+    def add_retry(jid = 'bob', at = Time.now.to_f)
+      payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.utc)
       Sidekiq.redis do |conn|
         conn.zadd('retry', at.to_s, payload)
       end

data/test/test_cli.rb

@@ -80,9 +80,51 @@ class TestCli < MiniTest::Unit::TestCase
       @cli.parse(['sidekiq', '-v', '-r', './test/fake_env.rb'])
       assert_equal Logger::DEBUG, Sidekiq.logger.level
       # If we leave the logger at DEBUG it'll add a lot of noise to the test output
+      Sidekiq.options.delete(:verbose)
       Sidekiq.logger.level = old
     end
 
+    describe 'with logfile' do
+      before do
+        @old_logger = Sidekiq.logger
+        @tmp_log_path = '/tmp/sidekiq.log'
+      end
+
+      after do
+        Sidekiq.logger = @old_logger
+        Sidekiq.options.delete(:logfile)
+        File.unlink @tmp_log_path if File.exists?(@tmp_log_path)
+      end
+
+      it 'sets the logfile path' do
+        @cli.parse(['sidekiq', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        assert_equal @tmp_log_path, Sidekiq.options[:logfile]
+      end
+
+      it 'creates and writes to a logfile' do
+        @cli.parse(['sidekiq', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        Sidekiq.logger.info('test message')
+
+        assert_match /test message/, File.read(@tmp_log_path), "didn't include the log message"
+      end
+
+      it 'appends messages to a logfile' do
+        File.open(@tmp_log_path, 'w') do |f|
+          f.puts 'already existant log message'
+        end
+
+        @cli.parse(['sidekiq', '-L', @tmp_log_path, '-r', './test/fake_env.rb'])
+
+        Sidekiq.logger.info('test message')
+
+        log_file_content = File.read(@tmp_log_path)
+        assert_match /already existant/, log_file_content, "didn't include the old message"
+        assert_match /test message/, log_file_content, "didn't include the new message"
+      end
+    end
+
     describe 'with pidfile' do
       before do
         @tmp_file = Tempfile.new('sidekiq-test')
@@ -134,6 +176,10 @@ class TestCli < MiniTest::Unit::TestCase
         assert_equal '/tmp/sidekiq-config-test.pid', Sidekiq.options[:pidfile]
       end
 
+      it 'sets logfile' do
+        assert_equal '/tmp/sidekiq.log', Sidekiq.options[:logfile]
+      end
+
       it 'sets queues' do
         assert_equal 2, Sidekiq.options[:queues].count { |q| q == 'very_often' }
         assert_equal 1, Sidekiq.options[:queues].count { |q| q == 'seldom' }

data/test/test_processor.rb

@@ -67,5 +67,58 @@ class TestProcessor < MiniTest::Unit::TestCase
       processor.process(msgstr, 'default')
       assert_equal [['myarg']], msg['args']
     end
+
+    describe 'stats' do
+      before do
+        Sidekiq.redis {|c| c.flushdb }
+      end
+
+      describe 'when successful' do
+        def successful_job
+          msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+          actor = MiniTest::Mock.new
+          actor.expect(:processor_done, nil, [@processor])
+          @boss.expect(:async, actor, [])
+          @processor.process(msg, 'default')
+        end
+
+        it 'increments processed stat' do
+          successful_job
+          assert_equal 1, Sidekiq::Stats.new.processed
+        end
+
+        it 'increments date processed stat' do
+          Time.stub(:now, Time.parse("2012-12-25 1:00:00 -0500")) do
+            successful_job
+            date_processed = Sidekiq.redis { |conn| conn.get("stat:processed:2012-12-25") }.to_i
+            assert_equal 1, date_processed
+          end
+        end
+      end
+
+      describe 'when failed' do
+        def failed_job
+          msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+          begin
+            @processor.process(msg, 'default')
+          rescue TestException
+          end
+        end
+
+        it 'increments failed stat' do
+          failed_job
+          assert_equal 1, Sidekiq::Stats.new.failed
+        end
+
+        it 'increments date failed stat' do
+          Time.stub(:now, Time.parse("2012-12-25 1:00:00 -0500")) do
+            failed_job
+            date_failed = Sidekiq.redis { |conn| conn.get("stat:failed:2012-12-25") }.to_i
+            assert_equal 1, date_failed
+          end
+        end
+      end
+
+    end
   end
 end

data/test/test_sidekiq.rb

@@ -0,0 +1,14 @@
+require 'helper'
+
+class TestSidekiq < MiniTest::Unit::TestCase
+  describe 'json processing' do
+    it 'loads json' do
+      assert_equal ({"foo" => "bar"}), Sidekiq.load_json("{\"foo\":\"bar\"}")
+    end
+
+    it 'dumps json' do
+      assert_equal "{\"foo\":\"bar\"}", Sidekiq.dump_json({ "foo" => "bar" })
+    end
+  end
+
+end

data/test/test_stats.rb

@@ -18,54 +18,6 @@ class TestStats < MiniTest::Unit::TestCase
       end
     end
 
-    it 'updates global stats in the success case' do
-      msg = Sidekiq.dump_json({ 'class' => DumbWorker.to_s, 'args' => [""] })
-      boss = MiniTest::Mock.new
-      actor = MiniTest::Mock.new
-
-      @redis.with do |conn|
-
-        set = conn.smembers('workers')
-        assert_equal 0, set.size
-
-        processor = Sidekiq::Processor.new(boss)
-        3.times do
-          actor.expect(:processor_done, nil, [processor])
-          boss.expect(:async, actor, [])
-        end
-
-        assert_equal 0, Sidekiq.info[:failed]
-        assert_equal 0, Sidekiq.info[:processed]
-
-        processor.process(msg, 'xyzzy')
-        processor.process(msg, 'xyzzy')
-        processor.process(msg, 'xyzzy')
-
-        assert_equal 0, Sidekiq.info[:failed]
-        assert_equal 3, Sidekiq.info[:processed]
-      end
-    end
-
-    it 'updates global stats in the error case' do
-      msg = Sidekiq.dump_json({ 'class' => DumbWorker.to_s, 'args' => [nil] })
-      boss = MiniTest::Mock.new
-
-      @redis.with do |conn|
-        assert_equal [], conn.smembers('workers')
-        assert_equal 0, Sidekiq.info[:failed]
-        assert_equal 0, Sidekiq.info[:processed]
-
-        processor = Sidekiq::Processor.new(boss)
-
-        assert_raises RuntimeError do
-          processor.process(msg, 'xyzzy')
-        end
-
-        assert_equal 1, Sidekiq.info[:failed]
-        assert_equal 1, Sidekiq.info[:processed]
-      end
-    end
-
     describe "info counts" do
       before do
         @redis.with do |conn|
@@ -80,18 +32,6 @@ class TestStats < MiniTest::Unit::TestCase
         end
       end
 
-      describe "queues_with_sizes" do
-        it "returns queue names and corresponding job counts" do
-          assert_equal [["foo", 1], ["baz", 2], ["bar", 3]], Sidekiq.info[:queues_with_sizes]
-        end
-      end
-
-      describe "backlog" do
-        it "returns count of all jobs yet to be processed" do
-          assert_equal 6, Sidekiq.info[:backlog]
-        end
-      end
-
       describe "size" do
         it "returns size of queues" do
           assert_equal 0, Sidekiq.size("foox")

data/test/test_web.rb

@@ -11,6 +11,10 @@ class TestWeb < MiniTest::Unit::TestCase
       Sidekiq::Web
     end
 
+    def job_params(job, score)
+      "#{score}-#{job['jid']}"
+    end
+
     before do
       Sidekiq.redis = REDIS
       Sidekiq.redis {|c| c.flushdb }
@@ -31,14 +35,6 @@ class TestWeb < MiniTest::Unit::TestCase
       refute_match /default/, last_response.body
     end
 
-    it 'can display poll' do
-      get '/poll'
-      assert_equal 200, last_response.status
-      assert_match /summary/, last_response.body
-      assert_match /workers/, last_response.body
-      refute_match /navbar/, last_response.body
-    end
-
     it 'can display queues' do
       assert Sidekiq::Client.push('queue' => :foo, 'class' => WebWorker, 'args' => [1, 3])
 
@@ -48,11 +44,6 @@ class TestWeb < MiniTest::Unit::TestCase
       refute_match /HardWorker/, last_response.body
     end
 
-    it 'handles missing retry' do
-      get '/retries/12391982.123'
-      assert_equal 302, last_response.status
-    end
-
     it 'handles queue view' do
       get '/queues/default'
       assert_equal 200, last_response.status
@@ -94,31 +85,6 @@ class TestWeb < MiniTest::Unit::TestCase
       end
     end
 
-    it 'can display scheduled' do
-      get '/scheduled'
-      assert_equal 200, last_response.status
-      assert_match /found/, last_response.body
-      refute_match /HardWorker/, last_response.body
-
-      add_scheduled
-
-      get '/scheduled'
-      assert_equal 200, last_response.status
-      refute_match /found/, last_response.body
-      assert_match /HardWorker/, last_response.body
-    end
-
-    it 'can delete scheduled' do
-      msg,score = add_scheduled
-      Sidekiq.redis do |conn|
-        assert_equal 1, conn.zcard('schedule')
-        post '/scheduled', 'score' => [score], 'delete' => 'Delete'
-        assert_equal 302, last_response.status
-        assert_equal 'http://example.org/scheduled', last_response.header['Location']
-        assert_equal 0, conn.zcard('schedule')
-      end
-    end
-
     it 'can display retries' do
       get '/retries'
       assert_equal 200, last_response.status
@@ -134,25 +100,28 @@ class TestWeb < MiniTest::Unit::TestCase
     end
 
     it 'can display a single retry' do
-      get '/retries/12938712.123333'
+      params = add_retry
+      get '/retries/2c4c17969825a384a92f023b'
       assert_equal 302, last_response.status
-      _, score = add_retry
-
-      get "/retries/#{score}"
+      get "/retries/#{job_params(*params)}"
       assert_equal 200, last_response.status
       assert_match /HardWorker/, last_response.body
     end
 
-    it 'can delete a single retry' do
-      _, score = add_retry
+    it 'handles missing retry' do
+      get "/retries/2c4c17969825a384a92f023b"
+      assert_equal 302, last_response.status
+    end
 
-      post "/retries/#{score}", 'delete' => 'Delete'
+    it 'can delete a single retry' do
+      params = add_retry
+      post "/retries/#{job_params(*params)}", 'delete' => 'Delete'
       assert_equal 302, last_response.status
       assert_equal 'http://example.org/retries', last_response.header['Location']
 
       get "/retries"
       assert_equal 200, last_response.status
-      refute_match /#{score}/, last_response.body
+      refute_match /#{params.first['args'][2]}/, last_response.body
     end
 
     it 'can delete all retries' do
@@ -165,15 +134,39 @@ class TestWeb < MiniTest::Unit::TestCase
     end
 
     it 'can retry a single retry now' do
-      msg, score = add_retry
-
-      post "/retries/#{score}", 'retry' => 'Retry'
+      params = add_retry
+      post "/retries/#{job_params(*params)}", 'retry' => 'Retry'
       assert_equal 302, last_response.status
       assert_equal 'http://example.org/retries', last_response.header['Location']
 
       get '/queues/default'
       assert_equal 200, last_response.status
-      assert_match /#{msg['args'][2]}/, last_response.body
+      assert_match /#{params.first['args'][2]}/, last_response.body
+    end
+
+    it 'can display scheduled' do
+      get '/scheduled'
+      assert_equal 200, last_response.status
+      assert_match /found/, last_response.body
+      refute_match /HardWorker/, last_response.body
+
+      add_scheduled
+
+      get '/scheduled'
+      assert_equal 200, last_response.status
+      refute_match /found/, last_response.body
+      assert_match /HardWorker/, last_response.body
+    end
+
+    it 'can delete scheduled' do
+      params = add_scheduled
+      Sidekiq.redis do |conn|
+        assert_equal 1, conn.zcard('schedule')
+        post '/scheduled', 'key' => [job_params(*params)], 'delete' => 'Delete'
+        assert_equal 302, last_response.status
+        assert_equal 'http://example.org/scheduled', last_response.header['Location']
+        assert_equal 0, conn.zcard('schedule')
+      end
     end
 
     it 'can retry all retries' do
@@ -202,11 +195,27 @@ class TestWeb < MiniTest::Unit::TestCase
       end
     end
 
+    it 'can display the dashboard' do
+      get '/dashboard'
+      assert_equal 200, last_response.status
+    end
+
+    it 'can refresh dashboard stats' do
+      Sidekiq.redis do |conn|
+        conn.set("stat:processed", 5)
+        conn.set("stat:failed", 2)
+      end
+      get '/dashboard/stats'
+      assert_equal 200, last_response.status
+      assert_equal "{\"processed\":5,\"failed\":2}", last_response.body
+    end
+
     def add_scheduled
+      score = Time.now.to_f
       msg = { 'class' => 'HardWorker',
               'args' => ['bob', 1, Time.now.to_f],
-              'at' => Time.now.to_f }
-      score = Time.now.to_f
+              'at' => score,
+              'jid' => 'f39af2a05e8f4b24dbc0f1e4' }
       Sidekiq.redis do |conn|
         conn.zadd('schedule', score, Sidekiq.dump_json(msg))
       end
@@ -220,7 +229,8 @@ class TestWeb < MiniTest::Unit::TestCase
               'error_message' => 'Some fake message',
               'error_class' => 'RuntimeError',
               'retry_count' => 0,
-              'failed_at' => Time.now.utc, }
+              'failed_at' => Time.now.utc,
+              'jid' => 'f39af2a05e8f4b24dbc0f1e4'}
       score = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.zadd('retry', score, Sidekiq.dump_json(msg))