sidekiq 3.5.4 → 4.0.0.pre1

data/test/test_fetch.rb

@@ -20,7 +20,7 @@ class TestFetcher < Sidekiq::Test
       uow = fetch.retrieve_work
       refute_nil uow
       assert_equal 'basic', uow.queue_name
-      assert_equal 'msg', uow.message
+      assert_equal 'msg', uow.job
       q = Sidekiq::Queue.new('basic')
       assert_equal 0, q.size
       uow.requeue
@@ -31,7 +31,7 @@ class TestFetcher < Sidekiq::Test
     it 'retrieves with strict setting' do
       fetch = Sidekiq::BasicFetch.new(:queues => ['basic', 'bar', 'bar'], :strict => true)
       cmd = fetch.queues_cmd
-      assert_equal cmd, ['queue:basic', 'queue:bar', 1]
+      assert_equal cmd, ['queue:basic', 'queue:bar', Sidekiq::BasicFetch::TIMEOUT]
     end
 
     it 'bulk requeues' do
@@ -45,60 +45,5 @@ class TestFetcher < Sidekiq::Test
       assert_equal 1, q2.size
     end
 
-    describe 'fetching' do
-      before do
-        Sidekiq::Fetcher.reset
-      end
-
-      it 'instantiates' do
-        begin
-          Sidekiq.options[:fetch] = NullFetch
-          mgr = Minitest::Mock.new
-          fetch = Sidekiq::Fetcher.new(mgr, {})
-          fetch.fetch
-          Sidekiq::Fetcher.done!
-        ensure
-          Sidekiq.options[:fetch] = Sidekiq::BasicFetch
-        end
-      end
-
-      class NullFetch
-        def initialize(opts)
-        end
-        def retrieve_work
-        end
-        def self.bulk_requeue(*args)
-        end
-      end
-
-      it 'handles redis network errors' do
-        begin
-          Sidekiq.logger.level = Logger::FATAL
-          Sidekiq.options[:fetch] = ErrorFetch
-          mgr = Minitest::Mock.new
-          fetch = Sidekiq::Fetcher.new(mgr, {})
-          def fetch.pause
-          end
-          refute fetch.down
-          fetch.fetch
-          Sidekiq::Fetcher.done!
-          assert fetch.down
-        ensure
-          Sidekiq.options[:fetch] = Sidekiq::BasicFetch
-          Sidekiq.logger.level = Logger::ERROR
-        end
-      end
-
-      class ErrorFetch
-        def initialize(opts)
-        end
-        def retrieve_work
-          raise IOError, "ker-BOOM"
-        end
-        def self.bulk_requeue(*args)
-        end
-      end
-    end
-
   end
 end

data/test/test_launcher.rb

@@ -0,0 +1,80 @@
+require_relative 'helper'
+require 'sidekiq/launcher'
+
+class TestLauncher < Sidekiq::Test
+
+  describe 'launcher' do
+    before do
+      Sidekiq.redis {|c| c.flushdb }
+    end
+
+    def new_manager(opts)
+      Sidekiq::Manager.new(opts)
+    end
+
+    describe 'heartbeat' do
+      before do
+        uow = Object.new
+
+        @mgr = new_manager(options)
+        @launcher = Sidekiq::Launcher.new(options)
+        @launcher.manager = @mgr
+
+        Sidekiq::Processor::WORKER_STATE['a'] = {'b' => 1}
+
+        @proctitle = $0
+      end
+
+      after do
+        Sidekiq::Processor::WORKER_STATE.clear
+        $0 = @proctitle
+      end
+
+      describe 'when manager is active' do
+        before do
+          Sidekiq::CLI::PROCTITLES << proc { "xyz" }
+          @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
+          Sidekiq::CLI::PROCTITLES.pop
+        end
+
+        it 'sets useful info to proctitle' do
+          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy] xyz", $0
+        end
+
+        it 'stores process info in redis' do
+          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
+          assert_equal ["1"], info
+          expires = Sidekiq.redis { |c| c.pttl('identity') }
+          assert_in_delta 60000, expires, 500
+        end
+      end
+
+      describe 'when manager is stopped' do
+        before do
+          @launcher.quiet
+          @launcher.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
+        end
+
+        it 'indicates stopping status in proctitle' do
+          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy] stopping", $0
+        end
+
+        it 'stores process info in redis' do
+          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
+          assert_equal ["1"], info
+          expires = Sidekiq.redis { |c| c.pttl('identity') }
+          assert_in_delta 60000, expires, 50
+        end
+      end
+    end
+
+    def options
+      { :concurrency => 3, :queues => ['default'] }
+    end
+
+    def heartbeat_data
+      { 'concurrency' => 3, 'tag' => 'myapp' }
+    end
+  end
+
+end

data/test/test_manager.rb

@@ -9,82 +9,31 @@ class TestManager < Sidekiq::Test
     end
 
     def new_manager(opts)
-      condvar = Minitest::Mock.new
-      condvar.expect(:signal, nil, [])
-      Sidekiq::Manager.new(condvar, opts)
+      Sidekiq::Manager.new(opts)
     end
 
     it 'creates N processor instances' do
       mgr = new_manager(options)
-      assert_equal options[:concurrency], mgr.ready.size
-      assert_equal [], mgr.busy
-    end
-
-    it 'assigns work to a processor' do
-      uow = Object.new
-      processor = Minitest::Mock.new
-      processor.expect(:async, processor, [])
-      processor.expect(:process, nil, [uow])
-
-      mgr = new_manager(options)
-      mgr.ready << processor
-      mgr.assign(uow)
-      assert_equal 1, mgr.busy.size
-
-      processor.verify
-    end
-
-    it 'requeues work if stopping' do
-      uow = Minitest::Mock.new
-      uow.expect(:requeue, nil, [])
-
-      mgr = new_manager(options)
-      mgr.fetcher = Sidekiq::BasicFetch.new({:queues => []})
-      mgr.stop
-      mgr.assign(uow)
-      uow.verify
+      assert_equal options[:concurrency], mgr.workers.size
     end
 
     it 'shuts down the system' do
       mgr = new_manager(options)
-      mgr.fetcher = Sidekiq::BasicFetch.new({:queues => []})
-      mgr.stop
-
-      assert mgr.busy.empty?
-      assert mgr.ready.empty?
-    end
-
-    it 'returns finished processors to the ready pool' do
-      fetcher = MiniTest::Mock.new
-      fetcher.expect :async, fetcher, []
-      fetcher.expect :fetch, nil, []
-      mgr = new_manager(options)
-      mgr.fetcher = fetcher
-      init_size = mgr.ready.size
-      processor = mgr.ready.pop
-      mgr.busy << processor
-      mgr.processor_done(processor)
-
-      assert_equal 0, mgr.busy.size
-      assert_equal init_size, mgr.ready.size
-      fetcher.verify
+      mgr.stop(Time.now)
     end
 
     it 'throws away dead processors' do
-      fetcher = MiniTest::Mock.new
-      fetcher.expect :async, fetcher, []
-      fetcher.expect :fetch, nil, []
       mgr = new_manager(options)
-      mgr.fetcher = fetcher
-      init_size = mgr.ready.size
-      processor = mgr.ready.pop
-      mgr.busy << processor
-      mgr.processor_died(processor, 'ignored')
-
-      assert_equal 0, mgr.busy.size
-      assert_equal init_size, mgr.ready.size
-      refute mgr.ready.include?(processor)
-      fetcher.verify
+      init_size = mgr.workers.size
+      processor = mgr.workers.first
+      begin
+        mgr.processor_died(processor, 'ignored')
+
+        assert_equal init_size, mgr.workers.size
+        refute mgr.workers.include?(processor)
+      ensure
+        mgr.workers.each {|p| p.terminate(true) }
+      end
     end
 
     it 'does not support invalid concurrency' do
@@ -92,77 +41,9 @@ class TestManager < Sidekiq::Test
       assert_raises(ArgumentError) { new_manager(concurrency: -1) }
     end
 
-    describe 'heartbeat' do
-      before do
-        uow = Object.new
-
-        @processor = Minitest::Mock.new
-        @processor.expect(:async, @processor, [])
-        @processor.expect(:process, nil, [uow])
-
-        @mgr = new_manager(options)
-        @mgr.ready << @processor
-        @mgr.assign(uow)
-
-        @processor.verify
-        @proctitle = $0
-      end
-
-      after do
-        $0 = @proctitle
-      end
-
-      describe 'when manager is active' do
-        before do
-          Sidekiq::Manager::PROCTITLES << proc { "xyz" }
-          @mgr.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
-          Sidekiq::Manager::PROCTITLES.pop
-        end
-
-        it 'sets useful info to proctitle' do
-          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy] xyz", $0
-        end
-
-        it 'stores process info in redis' do
-          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
-          assert_equal ["1"], info
-          expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 500
-        end
-      end
-
-      describe 'when manager is stopped' do
-        before do
-          @processor.expect(:alive?, [])
-          @processor.expect(:terminate, [])
-
-          @mgr.stop
-          @mgr.processor_done(@processor)
-          @mgr.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
-
-          @processor.verify
-        end
-
-        it 'indicates stopping status in proctitle' do
-          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [0 of 3 busy] stopping", $0
-        end
-
-        it 'stores process info in redis' do
-          info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
-          assert_equal ["0"], info
-          expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 50
-        end
-      end
-    end
-
     def options
       { :concurrency => 3, :queues => ['default'] }
     end
 
-    def heartbeat_data
-      { 'concurrency' => 3, 'tag' => 'myapp' }
-    end
   end
-
 end

data/test/test_middleware.rb

@@ -79,12 +79,10 @@ class TestMiddleware < Sidekiq::Test
       end
 
       boss = Minitest::Mock.new
+      boss.expect(:options, {:queues => ['default'] }, [])
+      boss.expect(:options, {:queues => ['default'] }, [])
       processor = Sidekiq::Processor.new(boss)
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      boss.expect(:async, actor, [])
-      boss.expect(:async, actor, [])
+      boss.expect(:processor_done, nil, [processor])
       processor.process(Sidekiq::BasicFetch::UnitOfWork.new('queue:default', msg))
       assert_equal %w(2 before 3 before 1 before work_performed 1 after 3 after 2 after), $recorder.flatten
     end

data/test/test_processor.rb

@@ -1,15 +1,19 @@
 require_relative 'helper'
+require 'sidekiq/fetch'
+require 'sidekiq/cli'
 require 'sidekiq/processor'
 
 class TestProcessor < Sidekiq::Test
   TestException = Class.new(StandardError)
   TEST_EXCEPTION = TestException.new("kerboom!")
 
-  describe 'with mock setup' do
+  describe 'processor' do
     before do
       $invokes = 0
-      @boss = Minitest::Mock.new
-      @processor = ::Sidekiq::Processor.new(@boss)
+      @mgr = Minitest::Mock.new
+      @mgr.expect(:options, {:queues => ['default']})
+      @mgr.expect(:options, {:queues => ['default']})
+      @processor = ::Sidekiq::Processor.new(@mgr)
     end
 
     class MockWorker
@@ -27,13 +31,7 @@ class TestProcessor < Sidekiq::Test
 
     it 'processes as expected' do
       msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [@processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      @boss.expect(:async, actor, [])
       @processor.process(work(msg))
-      @boss.verify
       assert_equal 1, $invokes
     end
 
@@ -43,46 +41,26 @@ class TestProcessor < Sidekiq::Test
       @processor.execute_job(worker, [1, 2, 3])
     end
 
-    it 'passes exceptions to ExceptionHandler' do
-      actor = Minitest::Mock.new
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
-      begin
-        @processor.process(work(msg))
-        flunk "Expected #process to raise exception"
-      rescue TestException
-      end
-
-      assert_equal 0, $invokes
-    end
-
     it 're-raises exceptions after handling' do
       msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
       re_raise = false
-      actor = Minitest::Mock.new
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
 
       begin
         @processor.process(work(msg))
+        flunk "Expected exception"
       rescue TestException
         re_raise = true
       end
 
+      assert_equal 0, $invokes
       assert re_raise, "does not re-raise exceptions after handling"
     end
 
     it 'does not modify original arguments' do
       msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
       msgstr = Sidekiq.dump_json(msg)
-      processor = ::Sidekiq::Processor.new(@boss)
-      actor = Minitest::Mock.new
-      actor.expect(:processor_done, nil, [processor])
-      actor.expect(:real_thread, nil, [nil, Thread])
-      @boss.expect(:async, actor, [])
-      @boss.expect(:async, actor, [])
-      processor.process(work(msgstr))
+      @mgr.expect(:processor_done, nil, [@processor])
+      @processor.process(work(msgstr))
       assert_equal [['myarg']], msg['args']
     end
 
@@ -106,17 +84,13 @@ class TestProcessor < Sidekiq::Test
       let(:skip_job) { false }
       let(:worker_args) { ['myarg'] }
       let(:work) { MiniTest::Mock.new }
-      let(:actor) { Minitest::Mock.new }
 
       before do
-        work.expect(:queue_name, 'queues:default')
-        work.expect(:message, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
+        work.expect(:queue_name, 'queue:default')
+        work.expect(:job, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
         Sidekiq.server_middleware do |chain|
           chain.prepend ExceptionRaisingMiddleware, raise_before_yield, raise_after_yield, skip_job
         end
-
-        actor.expect(:real_thread, nil, [nil, Thread])
-        @boss.expect(:async, actor, [])
       end
 
       after do
@@ -156,8 +130,7 @@ class TestProcessor < Sidekiq::Test
 
         it 'acks the job' do
           work.expect(:acknowledge, nil)
-          @boss.expect(:async, actor, [])
-          actor.expect(:processor_done, nil, [@processor])
+          @mgr.expect(:processor_done, nil, [@processor])
           @processor.process(work)
         end
       end
@@ -178,8 +151,7 @@ class TestProcessor < Sidekiq::Test
       describe 'everything goes well' do
         it 'acks the job' do
           work.expect(:acknowledge, nil)
-          @boss.expect(:async, actor, [])
-          actor.expect(:processor_done, nil, [@processor])
+          @mgr.expect(:processor_done, nil, [@processor])
           @processor.process(work)
         end
       end
@@ -195,19 +167,14 @@ class TestProcessor < Sidekiq::Test
 
         def successful_job
           msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
-          actor = Minitest::Mock.new
-          actor.expect(:real_thread, nil, [nil, Thread])
-          actor.expect(:processor_done, nil, [@processor])
-          @boss.expect(:async, actor, [])
-          @boss.expect(:async, actor, [])
+          @mgr.expect(:processor_done, nil, [@processor])
           @processor.process(work(msg))
         end
 
         it 'increments processed stat' do
-          assert_equal 0, Sidekiq::Stats.new.processed
+          Sidekiq::Processor::PROCESSED.value = 0
           successful_job
-          assert_equal 1, Sidekiq::Stats.new.processed
-          assert_equal Sidekiq::Processor::STATS_TIMEOUT, Sidekiq.redis { |conn| conn.ttl(processed_today_key) }
+          assert_equal 1, Sidekiq::Processor::PROCESSED.value
         end
       end
 
@@ -215,9 +182,6 @@ class TestProcessor < Sidekiq::Test
         let(:failed_today_key) { "stat:failed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
 
         def failed_job
-          actor = Minitest::Mock.new
-          actor.expect(:real_thread, nil, [nil, Thread])
-          @boss.expect(:async, actor, [])
           msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
           begin
             @processor.process(work(msg))
@@ -226,10 +190,9 @@ class TestProcessor < Sidekiq::Test
         end
 
         it 'increments failed stat' do
-          assert_equal 0, Sidekiq::Stats.new.failed
+          Sidekiq::Processor::FAILURE.value = 0
           failed_job
-          assert_equal 1, Sidekiq::Stats.new.failed
-          assert_equal Sidekiq::Processor::STATS_TIMEOUT, Sidekiq.redis { |conn| conn.ttl(failed_today_key) }
+          assert_equal 1, Sidekiq::Processor::FAILURE.value
         end
       end
     end