sr-sidekiq 4.1.6

data/test/test_processor.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+require_relative 'helper'
+require 'sidekiq/fetch'
+require 'sidekiq/cli'
+require 'sidekiq/processor'
+
+class TestProcessor < Sidekiq::Test
+  TestException = Class.new(StandardError)
+  TEST_EXCEPTION = TestException.new("kerboom!")
+
+  describe 'processor' do
+    before do
+      $invokes = 0
+      @mgr = Minitest::Mock.new
+      @mgr.expect(:options, {:queues => ['default']})
+      @mgr.expect(:options, {:queues => ['default']})
+      @processor = ::Sidekiq::Processor.new(@mgr)
+    end
+
+    class MockWorker
+      include Sidekiq::Worker
+      def perform(args)
+        raise TEST_EXCEPTION if args == 'boom'
+        args.pop if args.is_a? Array
+        $invokes += 1
+      end
+    end
+
+    def work(msg, queue='queue:default')
+      Sidekiq::BasicFetch::UnitOfWork.new(queue, msg)
+    end
+
+    it 'processes as expected' do
+      msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+      @processor.process(work(msg))
+      assert_equal 1, $invokes
+    end
+
+    it 'executes a worker as expected' do
+      worker = Minitest::Mock.new
+      worker.expect(:perform, nil, [1, 2, 3])
+      @processor.execute_job(worker, [1, 2, 3])
+    end
+
+    it 're-raises exceptions after handling' do
+      msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+      re_raise = false
+
+      begin
+        @processor.process(work(msg))
+        flunk "Expected exception"
+      rescue TestException
+        re_raise = true
+      end
+
+      assert_equal 0, $invokes
+      assert re_raise, "does not re-raise exceptions after handling"
+    end
+
+    it 'does not modify original arguments' do
+      msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
+      msgstr = Sidekiq.dump_json(msg)
+      @mgr.expect(:processor_done, nil, [@processor])
+      @processor.process(work(msgstr))
+      assert_equal [['myarg']], msg['args']
+    end
+
+    describe 'acknowledgement' do
+      class ExceptionRaisingMiddleware
+        def initialize(raise_before_yield, raise_after_yield, skip)
+          @raise_before_yield = raise_before_yield
+          @raise_after_yield = raise_after_yield
+          @skip = skip
+        end
+
+        def call(worker, item, queue)
+          raise TEST_EXCEPTION if @raise_before_yield
+          yield unless @skip
+          raise TEST_EXCEPTION if @raise_after_yield
+        end
+      end
+
+      let(:raise_before_yield) { false }
+      let(:raise_after_yield) { false }
+      let(:skip_job) { false }
+      let(:worker_args) { ['myarg'] }
+      let(:work) { MiniTest::Mock.new }
+
+      before do
+        work.expect(:queue_name, 'queue:default')
+        work.expect(:job, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
+        Sidekiq.server_middleware do |chain|
+          chain.prepend ExceptionRaisingMiddleware, raise_before_yield, raise_after_yield, skip_job
+        end
+      end
+
+      after do
+        Sidekiq.server_middleware do |chain|
+          chain.remove ExceptionRaisingMiddleware
+        end
+        work.verify
+      end
+
+      describe 'middleware throws an exception before processing the work' do
+        let(:raise_before_yield) { true }
+
+        it 'does not ack' do
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
+        end
+      end
+
+      describe 'middleware throws an exception after processing the work' do
+        let(:raise_after_yield) { true }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
+        end
+      end
+
+      describe 'middleware decides to skip work' do
+        let(:skip_job) { true }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          @mgr.expect(:processor_done, nil, [@processor])
+          @processor.process(work)
+        end
+      end
+
+      describe 'worker raises an exception' do
+        let(:worker_args) { ['boom'] }
+
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          begin
+            @processor.process(work)
+            flunk "Expected #process to raise exception"
+          rescue TestException
+          end
+        end
+      end
+
+      describe 'everything goes well' do
+        it 'acks the job' do
+          work.expect(:acknowledge, nil)
+          @mgr.expect(:processor_done, nil, [@processor])
+          @processor.process(work)
+        end
+      end
+    end
+
+    describe 'stats' do
+      before do
+        Sidekiq.redis {|c| c.flushdb }
+      end
+
+      describe 'when successful' do
+        let(:processed_today_key) { "stat:processed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
+
+        def successful_job
+          msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
+          @mgr.expect(:processor_done, nil, [@processor])
+          @processor.process(work(msg))
+        end
+
+        it 'increments processed stat' do
+          Sidekiq::Processor::PROCESSED.value = 0
+          successful_job
+          assert_equal 1, Sidekiq::Processor::PROCESSED.value
+        end
+      end
+
+      describe 'when failed' do
+        let(:failed_today_key) { "stat:failed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
+
+        def failed_job
+          msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
+          begin
+            @processor.process(work(msg))
+          rescue TestException
+          end
+        end
+
+        it 'increments failed stat' do
+          Sidekiq::Processor::FAILURE.value = 0
+          failed_job
+          assert_equal 1, Sidekiq::Processor::FAILURE.value
+        end
+      end
+    end
+  end
+end

data/test/test_rails.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+require_relative 'helper'
+
+$HAS_AJ = true
+begin
+  require 'active_job'
+rescue LoadError
+  $HAS_AJ = false
+end
+
+class TestRails < Sidekiq::Test
+
+  describe 'ActiveJob' do
+    it 'does not allow Sidekiq::Worker in AJ::Base classes' do
+      ex = assert_raises ArgumentError do
+        c = Class.new(ActiveJob::Base)
+        c.send(:include, Sidekiq::Worker)
+      end
+      assert_includes ex.message, "cannot include"
+    end if $HAS_AJ
+  end
+end

data/test/test_redis_connection.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+require_relative 'helper'
+
+class TestRedisConnection < Sidekiq::Test
+
+  describe ".create" do
+
+    it "creates a pooled redis connection" do
+      pool = Sidekiq::RedisConnection.create
+      assert_equal Redis, pool.checkout.class
+    end
+
+    describe "network_timeout" do
+      it "sets a custom network_timeout if specified" do
+        pool = Sidekiq::RedisConnection.create(:network_timeout => 8)
+        redis = pool.checkout
+
+        assert_equal 8, redis.client.timeout
+      end
+
+      it "uses the default network_timeout if none specified" do
+        pool = Sidekiq::RedisConnection.create
+        redis = pool.checkout
+
+        assert_equal 5, redis.client.timeout
+      end
+    end
+
+    describe "namespace" do
+      it "uses a given :namespace" do
+        pool = Sidekiq::RedisConnection.create(:namespace => "xxx")
+        assert_equal "xxx", pool.checkout.namespace
+      end
+
+      it "uses given :namespace over :namespace from Sidekiq.options" do
+        Sidekiq.options[:namespace] = "xxx"
+        pool = Sidekiq::RedisConnection.create(:namespace => "yyy")
+        assert_equal "yyy", pool.checkout.namespace
+      end
+    end
+
+    describe "socket path" do
+      it "uses a given :path" do
+        pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock")
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/0", pool.checkout.client.id
+      end
+
+      it "uses a given :path and :db" do
+        pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
+        assert_equal "unix", pool.checkout.client.scheme
+        assert_equal "redis:///var/run/redis.sock/8", pool.checkout.client.id
+      end
+    end
+
+    describe "pool_timeout" do
+      it "uses a given :timeout over the default of 1" do
+        pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)
+
+        assert_equal 5, pool.instance_eval{ @timeout }
+      end
+
+      it "uses the default timeout of 1 if no override" do
+        pool = Sidekiq::RedisConnection.create
+
+        assert_equal 1, pool.instance_eval{ @timeout }
+      end
+    end
+  end
+
+  describe ".determine_redis_provider" do
+
+    before do
+      @old_env = ENV.to_hash
+    end
+
+    after do
+      ENV.update(@old_env)
+    end
+
+    def with_env_var(var, uri, skip_provider=false)
+      vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
+      vars.each do |v|
+        next if skip_provider
+        ENV[v] = nil
+      end
+      ENV[var] = uri
+      assert_equal uri, Sidekiq::RedisConnection.__send__(:determine_redis_provider)
+      ENV[var] = nil
+    end
+
+    describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
+      it "sets connection URI to the provider" do
+        uri = 'redis://sidekiq-redis-provider:6379/0'
+        provider = 'SIDEKIQ_REDIS_PROVIDER'
+
+        ENV['REDIS_PROVIDER'] = provider
+        ENV[provider] = uri
+        ENV['REDISTOGO_URL'] = 'redis://redis-to-go:6379/0'
+        with_env_var provider, uri, true
+
+        ENV[provider] = nil
+      end
+    end
+
+    describe "with REDIS_PROVIDER set" do
+      it "sets connection URI to the provider" do
+        uri = 'redis://sidekiq-redis-provider:6379/0'
+        provider = 'SIDEKIQ_REDIS_PROVIDER'
+
+        ENV['REDIS_PROVIDER'] = provider
+        ENV[provider] = uri
+
+        with_env_var provider, uri, true
+
+        ENV[provider] = nil
+      end
+    end
+
+    describe "with REDIS_URL set" do
+      it "sets connection URI to custom uri" do
+        with_env_var 'REDIS_URL', 'redis://redis-uri:6379/0'
+      end
+    end
+
+  end
+end

data/test/test_retry.rb

@@ -0,0 +1,326 @@
+# frozen_string_literal: true
+# encoding: utf-8
+require_relative 'helper'
+require 'sidekiq/scheduled'
+require 'sidekiq/middleware/server/retry_jobs'
+
+class TestRetry < Sidekiq::Test
+  describe 'middleware' do
+    class SomeWorker
+      include Sidekiq::Worker
+    end
+
+    before do
+      Sidekiq.redis {|c| c.flushdb }
+    end
+
+    def worker
+      @worker ||= SomeWorker.new
+    end
+
+    def handler(options={})
+      @handler ||= Sidekiq::Middleware::Server::RetryJobs.new(options)
+    end
+
+    def job(options={})
+      @job ||= { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }.merge(options)
+    end
+
+    it 'allows disabling retry' do
+      assert_raises RuntimeError do
+        handler.call(worker, job('retry' => false), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 0, Sidekiq::RetrySet.new.size
+    end
+
+    it 'allows a numeric retry' do
+      assert_raises RuntimeError do
+        handler.call(worker, job('retry' => 2), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 1, Sidekiq::RetrySet.new.size
+      assert_equal 0, Sidekiq::DeadSet.new.size
+    end
+
+    it 'allows 0 retry => no retry and dead queue' do
+      assert_raises RuntimeError do
+        handler.call(worker, job('retry' => 0), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 0, Sidekiq::RetrySet.new.size
+      assert_equal 1, Sidekiq::DeadSet.new.size
+    end
+
+    it 'handles zany characters in error message, #1705' do
+      skip 'skipped! test requires ruby 2.1+' if RUBY_VERSION <= '2.1.0'
+
+      assert_raises RuntimeError do
+        handler.call(worker, job, 'default') do
+          raise "kerblammo! #{195.chr}"
+        end
+      end
+      assert_equal "kerblammo! �", job["error_message"]
+    end
+
+
+    it 'allows a max_retries option in initializer' do
+      max_retries = 7
+      1.upto(max_retries + 1) do
+        assert_raises RuntimeError do
+          handler(:max_retries => max_retries).call(worker, job, 'default') do
+            raise "kerblammo!"
+          end
+        end
+      end
+
+      assert_equal max_retries, Sidekiq::RetrySet.new.size
+      assert_equal 1, Sidekiq::DeadSet.new.size
+    end
+
+    it 'saves backtraces' do
+      c = nil
+      assert_raises RuntimeError do
+        handler.call(worker, job('backtrace' => true), 'default') do
+          c = caller(0); raise "kerblammo!"
+        end
+      end
+      assert job["error_backtrace"]
+      assert_equal c[0], job["error_backtrace"][0]
+    end
+
+    it 'saves partial backtraces' do
+      c = nil
+      assert_raises RuntimeError do
+        handler.call(worker, job('backtrace' => 3), 'default') do
+          c = caller(0)[0...3]; raise "kerblammo!"
+        end
+      end
+      assert job["error_backtrace"]
+      assert_equal c, job["error_backtrace"]
+      assert_equal 3, c.size
+    end
+
+    it 'handles a new failed message' do
+      assert_raises RuntimeError do
+        handler.call(worker, job, 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', job["queue"]
+      assert_equal 'kerblammo!', job["error_message"]
+      assert_equal 'RuntimeError', job["error_class"]
+      assert_equal 0, job["retry_count"]
+      refute job["error_backtrace"]
+      assert job["failed_at"]
+    end
+
+    it 'shuts down without retrying work-in-progress, which will resume' do
+      rs = Sidekiq::RetrySet.new
+      assert_equal 0, rs.size
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      assert_raises Sidekiq::Shutdown do
+        handler.call(worker, msg, 'default') do
+          raise Sidekiq::Shutdown
+        end
+      end
+      assert_equal 0, rs.size
+    end
+
+    it 'shuts down cleanly when shutdown causes exception' do
+      skip('Not supported in Ruby < 2.1.0') if RUBY_VERSION < '2.1.0'
+
+      rs = Sidekiq::RetrySet.new
+      assert_equal 0, rs.size
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      assert_raises Sidekiq::Shutdown do
+        handler.call(worker, msg, 'default') do
+          begin
+            raise Sidekiq::Shutdown
+          rescue Interrupt
+            raise "kerblammo!"
+          end
+        end
+      end
+      assert_equal 0, rs.size
+    end
+
+    it 'shuts down cleanly when shutdown causes chained exceptions' do
+      skip('Not supported in Ruby < 2.1.0') if RUBY_VERSION < '2.1.0'
+
+      rs = Sidekiq::RetrySet.new
+      assert_equal 0, rs.size
+      assert_raises Sidekiq::Shutdown do
+        handler.call(worker, job, 'default') do
+          begin
+            raise Sidekiq::Shutdown
+          rescue Interrupt
+            begin
+              raise "kerblammo!"
+            rescue
+              raise "kablooie!"
+            end
+          end
+        end
+      end
+      assert_equal 0, rs.size
+    end
+
+    it 'allows a retry queue' do
+      assert_raises RuntimeError do
+        handler.call(worker, job("retry_queue" => 'retryx'), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'retryx', job["queue"]
+      assert_equal 'kerblammo!', job["error_message"]
+      assert_equal 'RuntimeError', job["error_class"]
+      assert_equal 0, job["retry_count"]
+      refute job["error_backtrace"]
+      assert job["failed_at"]
+    end
+
+    it 'handles a recurring failed message' do
+      now = Time.now.to_f
+      msg = {"queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>10}
+      assert_raises RuntimeError do
+        handler.call(worker, job(msg), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 'default', job["queue"]
+      assert_equal 'kerblammo!', job["error_message"]
+      assert_equal 'RuntimeError', job["error_class"]
+      assert_equal 11, job["retry_count"]
+      assert job["failed_at"]
+    end
+
+    it 'throws away old messages after too many retries (using the default)' do
+      q = Sidekiq::Queue.new
+      rs = Sidekiq::RetrySet.new
+      ds = Sidekiq::DeadSet.new
+      assert_equal 0, q.size
+      assert_equal 0, rs.size
+      assert_equal 0, ds.size
+      now = Time.now.to_f
+      msg = {"queue"=>"default", "error_message"=>"kerblammo!", "error_class"=>"RuntimeError", "failed_at"=>now, "retry_count"=>25}
+      assert_raises RuntimeError do
+        handler.call(worker, job(msg), 'default') do
+          raise "kerblammo!"
+        end
+      end
+      assert_equal 0, q.size
+      assert_equal 0, rs.size
+      assert_equal 1, ds.size
+    end
+
+    describe "custom retry delay" do
+      before do
+        @old_logger    = Sidekiq.logger
+        @tmp_log_path  = '/tmp/sidekiq-retries.log'
+        Sidekiq.logger = Logger.new(@tmp_log_path)
+      end
+
+      after do
+        Sidekiq.logger = @old_logger
+        Sidekiq.options.delete(:logfile)
+        File.unlink @tmp_log_path if File.exist?(@tmp_log_path)
+      end
+
+      class CustomWorkerWithoutException
+        include Sidekiq::Worker
+
+        sidekiq_retry_in do |count|
+          count * 2
+        end
+      end
+
+      class CustomWorkerWithException
+        include Sidekiq::Worker
+
+        sidekiq_retry_in do |count, exception|
+          case exception
+          when ArgumentError
+            count * 4
+          else
+            count * 2
+          end
+        end
+      end
+
+      class ErrorWorker
+        include Sidekiq::Worker
+
+        sidekiq_retry_in do |count|
+          count / 0
+        end
+      end
+
+      it "retries with a default delay" do
+        refute_equal 4, handler.__send__(:delay_for, worker, 2, StandardError.new)
+      end
+
+      it "retries with a custom delay and exception 1" do
+        assert_equal 8, handler.__send__(:delay_for, CustomWorkerWithException, 2, ArgumentError.new)
+      end
+
+      it "retries with a custom delay and exception 2" do
+        assert_equal 4, handler.__send__(:delay_for, CustomWorkerWithException, 2, StandardError.new)
+      end
+
+      it "retries with a custom delay without exception" do
+        assert_equal 4, handler.__send__(:delay_for, CustomWorkerWithoutException, 2, StandardError.new)
+      end
+
+      it "falls back to the default retry on exception" do
+        refute_equal 4, handler.__send__(:delay_for, ErrorWorker, 2, StandardError.new)
+        assert_match(/Failure scheduling retry using the defined `sidekiq_retry_in`/,
+                     File.read(@tmp_log_path), 'Log entry missing for sidekiq_retry_in')
+      end
+    end
+
+    describe 'handles errors withouth cause' do
+      before do
+        @error = nil
+        begin
+          raise ::StandardError, 'Error'
+        rescue ::StandardError => e
+          @error = e
+        end
+      end
+
+      it "does not recurse infinitely checking if it's a shutdown" do
+        assert(!Sidekiq::Middleware::Server::RetryJobs.new.send(
+          :exception_caused_by_shutdown?, @error))
+      end
+    end
+
+    describe 'handles errors with circular causes' do
+      before do
+        @error = nil
+        begin
+          begin
+            raise ::StandardError, 'Error 1'
+          rescue ::StandardError => e1
+            begin
+              raise ::StandardError, 'Error 2'
+            rescue ::StandardError
+              raise e1
+            end
+          end
+        rescue ::StandardError => e
+          @error = e
+        end
+      end
+
+      it "does not recurse infinitely checking if it's a shutdown" do
+        assert(!Sidekiq::Middleware::Server::RetryJobs.new.send(
+          :exception_caused_by_shutdown?, @error))
+      end
+    end
+  end
+
+end