sidekiq 2.13.0 → 2.13.1

data/.gitignore

@@ -6,4 +6,6 @@ Gemfile.lock
 dump.rdb
 .rbx
 coverage/
+vendor/
+.bundle/
 .sass-cache/

data/Changes.md

@@ -1,5 +1,26 @@
+2.13.1
+-----------
+
+- Make Sidekiq::Middleware::Chain Enumerable
+- Make summary bar and graphs responsive [manishval, #1025]
+- Adds a job status page for scheduled jobs [jonhyman]
+- Handle race condition in retrying and deleting jobs in the Web UI
+- The Web UI relative times are now i18n. [MadRabbit, #1088]
+- Allow for default number of retry attempts to be set for
+  `Sidekiq::Middleware::Server::RetryJobs` middleware. [czarneckid] [#1091]
+
+```ruby
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Middleare::Server::RetryJobs, :max_retries => 10
+  end
+end
+```
+
+
 2.13.0
 -----------
+
 - Adding button to move scheduled job to main queue [guiceolin, #1020]
 - fix i18n support resetting saved locale when job is retried [#1011]
 - log rotation via USR2 now closes the old logger [#1008]

data/Contributing.md

@@ -0,0 +1,26 @@
+# Contributing
+
+First of all, thank you for even opening this file up!  I hope you find
+it worthwhile to help out with Sidekiq.
+
+
+## Code
+
+If you're interested in helping or contributing to Sidekiq, here's
+some known areas which need improvement:
+
+* The Web UI and sidekiq.org website could always use more polish.  If you have an eye for design and an idea for improvement, please open an issue and let us know.
+* The Sidekiq API has a serious issue: deleting elements and iterating
+  at the same time is broken, see #866, #1060.
+* Make normal testing and inline testing dynamic: #1053
+* Your brilliant idea which I haven't listed!
+
+It's always best to open an issue before investing a lot of time into a
+fix or new functionality.  Functionality must meet my design goals and
+vision for the project to be accepted; I would be happy to discuss how
+your idea can best fit into Sidekiq.
+
+
+## Sponsorship
+
+If you've got more money than time and want to sponsor Sidekiq's continued support, your company can buy [Sidekiq Pro](http://sidekiq.org/pro).  You get great functionality, I continue to fix bugs and enhance Sidekiq for years to come.

data/Pro-Changes.md

@@ -3,6 +3,28 @@ Sidekiq Pro Changelog
 
 Please see http://sidekiq.org/pro for more details and how to buy.
 
+1.2.0
+-----------
+
+- **LEAK** Fix batch key which didn't expire in Redis.  Keys match
+  /b-[a-f0-9]{16}-pending/, e.g. "b-4f55163ddba10aa0-pending" [#1057]
+- **Reliable fetch now supports multiple queues**, using the algorithm spec'd
+  by @jackrg [#1102]
+- Fix issue with reliable\_push where it didn't return the JID for a pushed
+  job when sending previously cached jobs to Redis.
+- Add fast Sidekiq::Queue#delete\_job(jid) API which leverages Lua so job lookup is
+  100% server-side.  Benchmark vs Sidekiq's Job#delete API:
+
+```
+Sidekiq Pro API
+  0.030000   0.020000   0.050000 (  1.640659)
+Sidekiq API
+ 17.250000   2.220000  19.470000 ( 22.193300)
+```
+
+- Add fast Sidekiq::Queue#delete\_by\_class(klass) API to remove all
+  jobs of a given type.  Uses server-side Lua for performance.
+
 1.1.0
 -----------
 

data/lib/sidekiq/exception_handler.rb

@@ -33,7 +33,7 @@ module Sidekiq
     end
 
     def send_to_exception_notifier(msg, ex)
-      ::ExceptionNotifier::Notifier.background_exception_notification(ex, :data => { :message => msg }).deliver
+      ::ExceptionNotifier.notify_exception(ex, :data => {:message => msg})
     end
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -61,8 +61,13 @@ module Sidekiq
   #
   module Middleware
     class Chain
+      include Enumerable
       attr_reader :entries
 
+      def each(&block)
+        entries.each(&block)
+      end
+
       def initialize
         @entries = []
         yield self if block_given?
@@ -73,7 +78,8 @@ module Sidekiq
       end
 
       def add(klass, *args)
-        entries << Entry.new(klass, *args) unless exists?(klass)
+        remove(klass) if exists?(klass)
+        entries << Entry.new(klass, *args)
       end
 
       def insert_before(oldklass, newklass, *args)
@@ -117,6 +123,7 @@ module Sidekiq
 
     class Entry
       attr_reader :klass
+
       def initialize(klass, *args)
         @klass = klass
         @args  = args

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -40,11 +40,24 @@ module Sidekiq
       #
       # We don't store the backtrace as that can add a lot of overhead
       # to the message and everyone is using Airbrake, right?
+      #
+      # The default number of retry attempts is 25. You can pass a value for the
+      # number of retry attempts when adding the middleware using the options hash:
+      #
+      #   Sidekiq.configure_server do |config|
+      #     config.server_middleware do |chain|
+      #       chain.add Middleware::Server::RetryJobs, {:max_retries => 7}
+      #     end
+      #   end
       class RetryJobs
         include Sidekiq::Util
 
         DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
+        def initialize(options = {})
+          @max_retries = options.fetch(:max_retries, DEFAULT_MAX_RETRY_ATTEMPTS)
+        end
+
         def call(worker, msg, queue)
           yield
         rescue Sidekiq::Shutdown
@@ -52,7 +65,7 @@ module Sidekiq
           raise
         rescue Exception => e
           raise e unless msg['retry']
-          max_retry_attempts = retry_attempts_from(msg['retry'], DEFAULT_MAX_RETRY_ATTEMPTS)
+          max_retry_attempts = retry_attempts_from(msg['retry'], @max_retries)
 
           msg['queue'] = if msg['retry_queue']
             msg['retry_queue']

data/lib/sidekiq/redis_connection.rb

@@ -9,18 +9,19 @@ module Sidekiq
         url = options[:url] || determine_redis_provider || 'redis://localhost:6379/0'
         # need a connection for Fetcher and Retry
         size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 2) : 5)
+        pool_timeout = options[:pool_timeout] || 1
 
         log_info(url, options)
 
-        ConnectionPool.new(:timeout => 1, :size => size) do
-          build_client(url, options[:namespace], options[:driver] || 'ruby')
+        ConnectionPool.new(:timeout => pool_timeout, :size => size) do
+          build_client(url, options[:namespace], options[:driver] || 'ruby', options[:network_timeout])
         end
       end
 
       private
 
-      def build_client(url, namespace, driver)
-        client = Redis.connect(:url => url, :driver => driver)
+      def build_client(url, namespace, driver, network_timeout)
+        client = Redis.new client_opts(url, driver, network_timeout)
         if namespace
           require 'redis/namespace'
           Redis::Namespace.new(namespace, :redis => client)
@@ -29,11 +30,19 @@ module Sidekiq
         end
       end
 
+      def client_opts(url, driver, timeout)
+        if timeout
+          { :url => url, :driver => driver, :timeout => timeout }
+        else
+          { :url => url, :driver => driver }
+        end
+      end
+
       def log_info(url, options)
         opts = options.dup
         opts.delete(:url)
         if Sidekiq.server?
-          Sidekiq.logger.info("Booting #{Sidekiq::NAME} #{Sidekiq::VERSION} using #{url} with options #{opts}")
+          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} using #{url} with options #{opts}")
         else
           Sidekiq.logger.info("#{Sidekiq::NAME} client using #{url} with options #{opts}")
         end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.13.0"
+  VERSION = "2.13.1"
 end

data/lib/sidekiq/web.rb

@@ -12,25 +12,28 @@ module Sidekiq
   class Web < Sinatra::Base
     include Sidekiq::Paginator
 
-    dir = File.expand_path(File.dirname(__FILE__) + "/../../web")
-
-    set :public_folder, "#{dir}/assets"
-    set :views,  "#{dir}/views"
-    set :root, "#{dir}/public"
-    set :locales, "#{dir}/locales"
+    set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
+    set :public_folder, Proc.new { "#{root}/assets" }
+    set :views, Proc.new { "#{root}/views" }
+    set :locales, Proc.new { "#{root}/locales" }
     set :slim, :pretty => true
 
     helpers do
       def strings
         @strings ||= begin
           Dir["#{settings.locales}/*.yml"].inject({}) do |memo, file|
-            memo.merge(YAML.load(File.read(file)))
+            memo.merge(YAML.load(File.open(file)))
           end
         end
       end
 
+      def locale
+        lang = (request.env["HTTP_ACCEPT_LANGUAGE"] || 'en')[0,2]
+        strings[lang] ? lang : 'en'
+      end
+
       def get_locale
-        strings[(request.env["HTTP_ACCEPT_LANGUAGE"] || 'en')[0,2]] || strings['en']
+        strings[locale]
       end
 
       def t(msg, options={})
@@ -46,7 +49,7 @@ module Sidekiq
       end
 
       def workers_size
-        Sidekiq.redis do |conn|
+        @workers_size ||= Sidekiq.redis do |conn|
           conn.scard('workers')
         end
       end
@@ -107,10 +110,10 @@ module Sidekiq
         [score.to_f, jid]
       end
 
-      def display_args(args, count=100)
+      def display_args(args, truncate_after_chars = 2000)
         args.map do |arg|
           a = arg.inspect
-          a.size > count ? "#{a[0..count]}..." : a
+          truncate_after_chars && a.size > truncate_after_chars ? "#{a[0..truncate_after_chars]}..." : a
         end.join(", ")
       end
 
@@ -212,6 +215,7 @@ module Sidekiq
 
       params['key'].each do |key|
         job = Sidekiq::RetrySet.new.fetch(*parse_params(key)).first
+        next unless job
         if params['retry']
           job.retry
         elsif params['delete']
@@ -249,6 +253,13 @@ module Sidekiq
       slim :scheduled
     end
 
+    get "/scheduled/:key" do
+      halt 404 unless params['key']
+      @job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
+      redirect "#{root_path}scheduled" if @job.nil?
+      slim :scheduled_job_info
+    end
+
     post '/scheduled' do
       halt 404 unless params['key']
 
@@ -263,6 +274,17 @@ module Sidekiq
       redirect "#{root_path}scheduled"
     end
 
+    post "/scheduled/:key" do
+      halt 404 unless params['key']
+      job = Sidekiq::ScheduledSet.new.fetch(*parse_params(params['key'])).first
+      if params['add_to_queue']
+        job.add_to_queue
+      elsif params['delete']
+        job.delete
+      end
+      redirect "#{root_path}scheduled"
+    end
+
     get '/' do
       @redis_info = Sidekiq.redis { |conn| conn.info }.select{ |k, v| redis_keys.include? k }
       stats_history = Sidekiq::Stats::History.new((params[:days] || 30).to_i)

data/test/test_exception_handler.rb

@@ -73,8 +73,7 @@ class TestExceptionHandler < Minitest::Test
 
   describe "with fake ExceptionNotifier" do
     before do
-      ::ExceptionNotifier = Module.new
-      ::ExceptionNotifier::Notifier = MiniTest::Mock.new
+      ::ExceptionNotifier = MiniTest::Mock.new
     end
 
     after do
@@ -82,12 +81,9 @@ class TestExceptionHandler < Minitest::Test
     end
 
     it "notifies ExceptionNotifier" do
-      mail = MiniTest::Mock.new
-      mail.expect(:deliver,nil)
-      ::ExceptionNotifier::Notifier.expect(:background_exception_notification,mail,[TEST_EXCEPTION, :data => { :message => { :b => 2 } }])
+      ::ExceptionNotifier.expect(:notify_exception,true,[TEST_EXCEPTION, :data => { :message => { :b => 2 } }])
       Component.new.invoke_exception(:b => 2)
-      ::ExceptionNotifier::Notifier.verify
-      mail.verify
+      ::ExceptionNotifier.verify
     end
   end
 

data/test/test_middleware.rb

@@ -72,7 +72,7 @@ class TestMiddleware < Minitest::Test
       msg = Sidekiq.dump_json({ 'class' => CustomWorker.to_s, 'args' => [$recorder] })
 
       Sidekiq.server_middleware do |chain|
-        # should only add once, second should be ignored
+        # should only add once, second should replace the first
         2.times { |i| chain.add CustomMiddleware, i.to_s, $recorder }
         chain.insert_before CustomMiddleware, AnotherCustomMiddleware, '2', $recorder
         chain.insert_after AnotherCustomMiddleware, YetAnotherCustomMiddleware, '3', $recorder
@@ -86,7 +86,14 @@ class TestMiddleware < Minitest::Test
       boss.expect(:async, actor, [])
       boss.expect(:async, actor, [])
       processor.process(Sidekiq::BasicFetch::UnitOfWork.new('queue:default', msg))
-      assert_equal %w(2 before 3 before 0 before work_performed 0 after 3 after 2 after), $recorder.flatten
+      assert_equal %w(2 before 3 before 1 before work_performed 1 after 3 after 2 after), $recorder.flatten
+    end
+
+    it 'correctly replaces middleware when using middleware with options in the initializer' do
+      chain = Sidekiq::Middleware::Chain.new
+      chain.add Sidekiq::Middleware::Server::RetryJobs
+      chain.add Sidekiq::Middleware::Server::RetryJobs, {:max_retries => 5}
+      assert_equal 1, chain.count
     end
 
     it 'allows middleware to abruptly stop processing rest of chain' do

data/test/test_redis_connection.rb

@@ -10,6 +10,22 @@ class TestRedisConnection < Minitest::Test
       assert_equal Redis, pool.checkout.class
     end
 
+    describe "network_timeout" do
+      it "sets a custom network_timeout if specified" do
+        pool = Sidekiq::RedisConnection.create(:network_timeout => 8)
+        redis = pool.checkout
+        
+        assert_equal 8, redis.client.timeout
+      end
+
+      it "uses the default network_timeout if none specified" do
+        pool = Sidekiq::RedisConnection.create
+        redis = pool.checkout
+
+        assert_equal 5, redis.client.timeout
+      end
+    end
+
     describe "namespace" do
       it "uses a given :namespace" do
         pool = Sidekiq::RedisConnection.create(:namespace => "xxx")
@@ -22,6 +38,20 @@ class TestRedisConnection < Minitest::Test
         assert_equal "yyy", pool.checkout.namespace
       end
     end
+
+    describe "pool_timeout" do
+      it "uses a given :timeout over the default of 1" do
+        pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)
+        
+        assert_equal 5, pool.instance_eval{ @timeout }
+      end
+
+      it "uses the default timeout of 1 if no override" do
+        pool = Sidekiq::RedisConnection.create
+
+        assert_equal 1, pool.instance_eval{ @timeout }
+      end
+    end
   end
 
   describe ".determine_redis_provider" do

data/test/test_retry.rb

@@ -45,6 +45,23 @@ class TestRetry < Minitest::Test
       @redis.verify
     end
 
+    it 'allows a max_retries option in initializer' do
+      max_retries = 7
+      1.upto(max_retries) do
+        @redis.expect :zadd, 1, ['retry', String, String]
+      end
+      msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true }
+      handler = Sidekiq::Middleware::Server::RetryJobs.new({:max_retries => max_retries})
+      1.upto(max_retries + 1) do
+        assert_raises RuntimeError do
+          handler.call(worker, msg, 'default') do
+            raise "kerblammo!"
+          end
+        end
+      end
+      @redis.verify
+    end
+
     it 'saves backtraces' do
       @redis.expect :zadd, 1, ['retry', String, String]
       msg = { 'class' => 'Bob', 'args' => [1,2,'foo'], 'retry' => true, 'backtrace' => true }

data/test/test_web.rb

@@ -130,7 +130,7 @@ class TestWeb < Minitest::Test
 
     it 'can display a single retry' do
       params = add_retry
-      get '/retries/2c4c17969825a384a92f023b'
+      get '/retries/0-shouldntexist'
       assert_equal 302, last_response.status
       get "/retries/#{job_params(*params)}"
       assert_equal 200, last_response.status
@@ -138,7 +138,7 @@ class TestWeb < Minitest::Test
     end
 
     it 'handles missing retry' do
-      get "/retries/2c4c17969825a384a92f023b"
+      get "/retries/0-shouldntexist"
       assert_equal 302, last_response.status
     end
 
@@ -187,6 +187,42 @@ class TestWeb < Minitest::Test
       assert_match /HardWorker/, last_response.body
     end
 
+    it 'can display a single scheduled job' do
+      params = add_scheduled
+      get '/scheduled/0-shouldntexist'
+      assert_equal 302, last_response.status
+      get "/scheduled/#{job_params(*params)}"
+      assert_equal 200, last_response.status
+      assert_match /HardWorker/, last_response.body
+    end
+
+    it 'handles missing scheduled job' do
+      get "/scheduled/0-shouldntexist"
+      assert_equal 302, last_response.status
+    end
+
+    it 'can add to queue a single scheduled job' do
+      params = add_scheduled
+      post "/scheduled/#{job_params(*params)}", 'add_to_queue' => true
+      assert_equal 302, last_response.status
+      assert_equal 'http://example.org/scheduled', last_response.header['Location']
+
+      get '/queues/default'
+      assert_equal 200, last_response.status
+      assert_match /#{params.first['args'][2]}/, last_response.body
+    end
+
+    it 'can delete a single scheduled job' do
+      params = add_scheduled
+      post "/scheduled/#{job_params(*params)}", 'delete' => 'Delete'
+      assert_equal 302, last_response.status
+      assert_equal 'http://example.org/scheduled', last_response.header['Location']
+
+      get "/scheduled"
+      assert_equal 200, last_response.status
+      refute_match /#{params.first['args'][2]}/, last_response.body
+    end
+
     it 'can delete scheduled' do
       params = add_scheduled
       Sidekiq.redis do |conn|
@@ -199,14 +235,16 @@ class TestWeb < Minitest::Test
     end
 
     it "can move scheduled to default queue" do
+      q = Sidekiq::Queue.new
       params = add_scheduled
       Sidekiq.redis do |conn|
         assert_equal 1, conn.zcard('schedule')
-        assert_equal 0, conn.zcard('default')
+        assert_equal 0, q.size
         post '/scheduled', 'key' => [job_params(*params)], 'add_to_queue' => 'AddToQueue'
         assert_equal 302, last_response.status
         assert_equal 'http://example.org/scheduled', last_response.header['Location']
         assert_equal 0, conn.zcard('schedule')
+        assert_equal 1, q.size
         get '/queues/default'
         assert_equal 200, last_response.status
         assert_match /#{params[0]['args'][2]}/, last_response.body