sidekiq 3.4.1 → 4.0.2

data/test/test_manager.rb

@@ -5,160 +5,45 @@ class TestManager < Sidekiq::Test
 
   describe 'manager' do
     before do
-      Sidekiq.redis = REDIS
       Sidekiq.redis {|c| c.flushdb }
     end
 
     def new_manager(opts)
-      condvar = Minitest::Mock.new
-      condvar.expect(:signal, nil, [])
-      Sidekiq::Manager.new(condvar, opts)
+      Sidekiq::Manager.new(opts)
     end
 
     it 'creates N processor instances' do
       mgr = new_manager(options)
-      assert_equal options[:concurrency], mgr.ready.size
-      assert_equal [], mgr.busy
-    end
-
-    it 'assigns work to a processor' do
-      uow = Object.new
-      processor = Minitest::Mock.new
-      processor.expect(:async, processor, [])
-      processor.expect(:process, nil, [uow])
-
-      mgr = new_manager(options)
-      mgr.ready << processor
-      mgr.assign(uow)
-      assert_equal 1, mgr.busy.size
-
-      processor.verify
-    end
-
-    it 'requeues work if stopping' do
-      uow = Minitest::Mock.new
-      uow.expect(:requeue, nil, [])
-
-      mgr = new_manager(options)
-      mgr.fetcher = Sidekiq::BasicFetch.new({:queues => []})
-      mgr.stop
-      mgr.assign(uow)
-      uow.verify
+      assert_equal options[:concurrency], mgr.workers.size
     end
 
     it 'shuts down the system' do
       mgr = new_manager(options)
-      mgr.fetcher = Sidekiq::BasicFetch.new({:queues => []})
-      mgr.stop
-
-      assert mgr.busy.empty?
-      assert mgr.ready.empty?
-    end
-
-    it 'returns finished processors to the ready pool' do
-      fetcher = MiniTest::Mock.new
-      fetcher.expect :async, fetcher, []
-      fetcher.expect :fetch, nil, []
-      mgr = new_manager(options)
-      mgr.fetcher = fetcher
-      init_size = mgr.ready.size
-      processor = mgr.ready.pop
-      mgr.busy << processor
-      mgr.processor_done(processor)
-
-      assert_equal 0, mgr.busy.size
-      assert_equal init_size, mgr.ready.size
-      fetcher.verify
+      mgr.stop(Time.now)
     end
 
     it 'throws away dead processors' do
-      fetcher = MiniTest::Mock.new
-      fetcher.expect :async, fetcher, []
-      fetcher.expect :fetch, nil, []
       mgr = new_manager(options)
-      mgr.fetcher = fetcher
-      init_size = mgr.ready.size
-      processor = mgr.ready.pop
-      mgr.busy << processor
-      mgr.processor_died(processor, 'ignored')
-
-      assert_equal 0, mgr.busy.size
-      assert_equal init_size, mgr.ready.size
-      refute mgr.ready.include?(processor)
-      fetcher.verify
-    end
-
-    describe 'heartbeat' do
-      before do
-        uow = Object.new
-
-        @processor = Minitest::Mock.new
-        @processor.expect(:async, @processor, [])
-        @processor.expect(:process, nil, [uow])
-
-        @mgr = new_manager(options)
-        @mgr.ready << @processor
-        @mgr.assign(uow)
-
-        @processor.verify
-        @proctitle = $0
-      end
-
-      after do
-        $0 = @proctitle
-      end
-
-      describe 'when manager is active' do
-        before do
-          Sidekiq::Manager::PROCTITLES << proc { "xyz" }
-          @mgr.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
-          Sidekiq::Manager::PROCTITLES.pop
-        end
-
-        it 'sets useful info to proctitle' do
-          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy] xyz", $0
-        end
-
-        it 'stores process info in redis' do
-          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
-          assert_equal ["1"], info
-          expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 500
-        end
+      init_size = mgr.workers.size
+      processor = mgr.workers.first
+      begin
+        mgr.processor_died(processor, 'ignored')
+
+        assert_equal init_size, mgr.workers.size
+        refute mgr.workers.include?(processor)
+      ensure
+        mgr.workers.each {|p| p.terminate(true) }
       end
+    end
 
-      describe 'when manager is stopped' do
-        before do
-          @processor.expect(:alive?, [])
-          @processor.expect(:terminate, [])
-
-          @mgr.stop
-          @mgr.processor_done(@processor)
-          @mgr.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
-
-          @processor.verify
-        end
-
-        it 'indicates stopping status in proctitle' do
-          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [0 of 3 busy] stopping", $0
-        end
-
-        it 'stores process info in redis' do
-          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
-          assert_equal ["0"], info
-          expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 50
-        end
-      end
+    it 'does not support invalid concurrency' do
+      assert_raises(ArgumentError) { new_manager(concurrency: 0) }
+      assert_raises(ArgumentError) { new_manager(concurrency: -1) }
     end
 
     def options
       { :concurrency => 3, :queues => ['default'] }
     end
 
-    def heartbeat_data
-      { 'concurrency' => 3, 'tag' => 'myapp' }
-    end
   end
-
 end

data/test/test_middleware.rb

@@ -79,12 +79,10 @@ class TestMiddleware < Sidekiq::Test
       end
 
       boss = Minitest::Mock.new
+      boss.expect(:options, {:queues => ['default'] }, [])
+      boss.expect(:options, {:queues => ['default'] }, [])
       processor = Sidekiq::Processor.new(boss)
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      boss.expect(:async, actor, [])
-      boss.expect(:async, actor, [])
+      boss.expect(:processor_done, nil, [processor])
       processor.process(Sidekiq::BasicFetch::UnitOfWork.new('queue:default', msg))
       assert_equal %w(2 before 3 before 1 before work_performed 1 after 3 after 2 after), $recorder.flatten
     end

data/test/test_processor.rb

@@ -1,17 +1,19 @@
 require_relative 'helper'
+require 'sidekiq/fetch'
+require 'sidekiq/cli'
 require 'sidekiq/processor'
 
 class TestProcessor < Sidekiq::Test
   TestException = Class.new(StandardError)
   TEST_EXCEPTION = TestException.new("kerboom!")
 
-  describe 'with mock setup' do
+  describe 'processor' do
     before do
       $invokes = 0
-      @boss = Minitest::Mock.new
-      @processor = ::Sidekiq::Processor.new(@boss)
-      Celluloid.logger = nil
-      Sidekiq.redis = REDIS
+      @mgr = Minitest::Mock.new
+      @mgr.expect(:options, {:queues => ['default']})
+      @mgr.expect(:options, {:queues => ['default']})
+      @processor = ::Sidekiq::Processor.new(@mgr)
     end
 
     class MockWorker
@@ -29,13 +31,7 @@ class TestProcessor < Sidekiq::Test
 
     it 'processes as expected' do
       msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [@processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      @boss.expect(:async, actor, [])
       @processor.process(work(msg))
-      @boss.verify
       assert_equal 1, $invokes
     end
 
@@ -45,100 +41,147 @@ class TestProcessor < Sidekiq::Test
       @processor.execute_job(worker, [1, 2, 3])
     end
 
-    it 'passes exceptions to ExceptionHandler' do
-      actor = Minitest::Mock.new
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
-      begin
-        @processor.process(work(msg))
-        flunk "Expected #process to raise exception"
-      rescue TestException
-      end
-
-      assert_equal 0, $invokes
-    end
-
     it 're-raises exceptions after handling' do
       msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
       re_raise = false
-      actor = Minitest::Mock.new
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
 
       begin
         @processor.process(work(msg))
+        flunk "Expected exception"
       rescue TestException
         re_raise = true
       end
 
+      assert_equal 0, $invokes
       assert re_raise, "does not re-raise exceptions after handling"
     end
 
     it 'does not modify original arguments' do
       msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
       msgstr = Sidekiq.dump_json(msg)
-      processor = ::Sidekiq::Processor.new(@boss)
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      @boss.expect(:async, actor, [])
-      processor.process(work(msgstr))
+      @mgr.expect(:processor_done, nil, [@processor])
+      @processor.process(work(msgstr))
       assert_equal [['myarg']], msg['args']
     end
 
-    describe 'stats' do
+    describe 'acknowledgement' do
+      class ExceptionRaisingMiddleware
+        def initialize(raise_before_yield, raise_after_yield, skip)
+          @raise_before_yield = raise_before_yield
+          @raise_after_yield = raise_after_yield
+          @skip = skip
+        end
+
+        def call(worker, item, queue)
+          raise TEST_EXCEPTION if @raise_before_yield
+          yield unless @skip
+          raise TEST_EXCEPTION if @raise_after_yield
+        end
+      end
+
+      let(:raise_before_yield) { false }
+      let(:raise_after_yield) { false }
+      let(:skip_job) { false }
+      let(:worker_args) { ['myarg'] }
+      let(:work) { MiniTest::Mock.new }
+
       before do
-        Sidekiq.redis {|c| c.flushdb }
+        work.expect(:queue_name, 'queue:default')
+        work.expect(:job, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
+        Sidekiq.server_middleware do |chain|
+          chain.prepend ExceptionRaisingMiddleware, raise_before_yield, raise_after_yield, skip_job
+        end
+      end
+
+      after do
+        Sidekiq.server_middleware do |chain|
+          chain.remove ExceptionRaisingMiddleware
+        end
+        work.verify
+      end
+
+      describe 'middleware throws an exception before processing the work' do
+        let(:raise_before_yield) { true }
+
+        it 'does not ack' do
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
+        end
       end
 
-      def with_expire(time)
-        begin
-          old = Sidekiq::Processor::STATS_TIMEOUT
-          silence_warnings { Sidekiq::Processor.const_set(:STATS_TIMEOUT, time) }
-          yield
-        ensure
-          silence_warnings { Sidekiq::Processor.const_set(:STATS_TIMEOUT, old) }
+      describe 'middleware throws an exception after processing the work' do
+        let(:raise_after_yield) { true }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
         end
       end
 
+      describe 'middleware decides to skip work' do
+        let(:skip_job) { true }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          @mgr.expect(:processor_done, nil, [@processor])
+          @processor.process(work)
+        end
+      end
+
+      describe 'worker raises an exception' do
+        let(:worker_args) { ['boom'] }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
+        end
+      end
+
+      describe 'everything goes well' do
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          @mgr.expect(:processor_done, nil, [@processor])
+          @processor.process(work)
+        end
+      end
+    end
+
+    describe 'stats' do
+      before do
+        Sidekiq.redis {|c| c.flushdb }
+      end
+
       describe 'when successful' do
-        let(:processed_today_key) { "stat:processed:#{Time.now.utc.to_date}" }
+        let(:processed_today_key) { "stat:processed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
 
         def successful_job
           msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
-          actor = Minitest::Mock.new
-          actor.expect(:real_thread, nil, [nil, Thread])
-          actor.expect(:processor_done, nil, [@processor])
-          @boss.expect(:async, actor, [])
-          @boss.expect(:async, actor, [])
+          @mgr.expect(:processor_done, nil, [@processor])
           @processor.process(work(msg))
         end
 
         it 'increments processed stat' do
+          Sidekiq::Processor::PROCESSED.value = 0
           successful_job
-          assert_equal 1, Sidekiq::Stats.new.processed
-        end
-
-        it 'expires processed stat' do
-          successful_job
-          assert_equal Sidekiq::Processor::STATS_TIMEOUT, Sidekiq.redis { |conn| conn.ttl(processed_today_key) }
-        end
-
-        it 'increments date processed stat' do
-          successful_job
-          assert_equal 1, Sidekiq.redis { |conn| conn.get(processed_today_key) }.to_i
+          assert_equal 1, Sidekiq::Processor::PROCESSED.value
         end
       end
 
       describe 'when failed' do
-        let(:failed_today_key) { "stat:failed:#{Time.now.utc.to_date}" }
+        let(:failed_today_key) { "stat:failed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
 
         def failed_job
-          actor = Minitest::Mock.new
-          actor.expect(:real_thread, nil, [nil, Thread])
-          @boss.expect(:async, actor, [])
           msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
           begin
             @processor.process(work(msg))
@@ -147,18 +190,9 @@ class TestProcessor < Sidekiq::Test
         end
 
         it 'increments failed stat' do
+          Sidekiq::Processor::FAILURE.value = 0
           failed_job
-          assert_equal 1, Sidekiq::Stats.new.failed
-        end
-
-        it 'increments date failed stat' do
-          failed_job
-          assert_equal 1, Sidekiq.redis { |conn| conn.get(failed_today_key) }.to_i
-        end
-
-        it 'expires failed stat' do
-          failed_job
-          assert_equal Sidekiq::Processor::STATS_TIMEOUT, Sidekiq.redis { |conn| conn.ttl(failed_today_key) }
+          assert_equal 1, Sidekiq::Processor::FAILURE.value
         end
       end
     end

data/test/test_rails.rb

@@ -0,0 +1,21 @@
+require_relative 'helper'
+
+$HAS_AJ = true
+begin
+  require 'active_job'
+rescue LoadError
+  $HAS_AJ = false
+end
+
+class TestRails < Sidekiq::Test
+
+  describe 'ActiveJob' do
+    it 'does not allow Sidekiq::Worker in AJ::Base classes' do
+      ex = assert_raises ArgumentError do
+        c = Class.new(ActiveJob::Base)
+        c.send(:include, Sidekiq::Worker)
+      end
+      assert_includes ex.message, "cannot include"
+    end if $HAS_AJ
+  end
+end

data/test/test_redis_connection.rb

@@ -1,5 +1,4 @@
 require_relative 'helper'
-require 'sidekiq/redis_connection'
 
 class TestRedisConnection < Sidekiq::Test