sidekiq 5.1.1 → 5.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2fff631710094e52bd8a57c3ddd0367fa2466425
-  data.tar.gz: 1310f9de542f0c123d39d45b1a8c7e1c09f36d58
+  metadata.gz: 985b2f1c0778b1e4d2587eb2f182fc58723277c7
+  data.tar.gz: 2d20a6e15e248bea802b1ca9438c9507a1f94ab4
 SHA512:
-  metadata.gz: c66ac1497ae96aa4c83b89966bfd79babd6ef7629e2916c3d0f441ca7e8f0a34c3d19cd86f37b1660965beded437f1059853a28547b0b5371456c5fe159648a2
-  data.tar.gz: 26bbe0b74f3d76e821f05e7811c1ae7dd9021cffad3fe9f9644e85b8ecc8f9b2ece98632cd4de17f27a3ffa2f3913ce8e60e6149c8dd5758fd409b4e93725347
+  metadata.gz: 4e4be1e23c16eb53d43424a0a07e6b69ab2001ffbbaf28c7c09db1dac89d9f1e9ac3513c70ac5abd398efb12c8a712da20f563df5fd055c6855eb5d2d7abea05
+  data.tar.gz: 8947fc9287e9a1336ba57b6ef380e5f76d7ea7382fff3930acd914c6c08fed9c08940780c1537bbedf349ed29257b9c249f8356eed095d8537a4195e7426b676

data/.travis.yml

@@ -7,8 +7,8 @@ before_install:
   - gem install bundler
   - gem update bundler
 rvm:
-  - 2.2.2
-  - 2.3.5
-  - 2.4.2
-  - 2.5.0
-  - jruby-9.1.14.0
+  - 2.2.10
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
+  - jruby-9.1.17.0

data/Changes.md

@@ -2,6 +2,45 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+5.2.2
+---------
+
+- Raise error for duplicate queue names in config to avoid unexpected fetch algorithm change [#3911]
+- Fix concurrency bug on JRuby [#3958, mattbooks]
+- Add "Kill All" button to the retries page [#3938]
+
+5.2.1
+-----------
+
+- Fix concurrent modification error during heartbeat [#3921]
+
+5.2.0
+-----------
+
+- **Decrease default concurrency from 25 to 10** [#3892]
+- Verify connection pool sizing upon startup [#3917]
+- Smoother scheduling for large Sidekiq clusters [#3889]
+- Switch Sidekiq::Testing impl from alias\_method to Module#prepend, for resiliency [#3852]
+- Update Sidekiq APIs to use SCAN for scalability [#3848, ffiller]
+- Remove concurrent-ruby gem dependency [#3830]
+- Optimize Web UI's bootstrap.css [#3914]
+
+5.1.3
+-----------
+
+- Fix version comparison so Ruby 2.2.10 works. [#3808, nateberkopec]
+
+5.1.2
+-----------
+
+- Add link to docs in Web UI footer
+- Fix crash on Ctrl-C in Windows [#3775, Bernica]
+- Remove `freeze` calls on String constants. This is superfluous with Ruby
+  2.3+ and `frozen_string_literal: true`. [#3759]
+- Fix use of AR middleware outside of Rails [#3787]
+- Sidekiq::Worker `sidekiq_retry_in` block can now return nil or 0 to use
+  the default backoff delay [#3796, dsalahutdinov]
+
 5.1.1
 -----------
 

data/Ent-Changes.md

@@ -4,6 +4,17 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+-------------
+
+- Add support for sidekiqswarm memory monitoring on FreeBSD [#3884]
+
+1.7.1
+-------------
+
+- Fix Lua error in concurrent rate limiter under heavy contention
+- Remove superfluous `freeze` calls on Strings [#3759]
+
 1.7.0
 -------------
 

data/Gemfile

@@ -1,8 +1,14 @@
 source 'https://rubygems.org'
 gemspec
 
-gem 'rails', '>= 5.0.1'
+# load testing
 #gem "hiredis"
-gem 'simplecov'
-gem 'minitest'
 #gem 'toxiproxy'
+
+group :test do
+  gem 'rails', '>= 5.0.1'
+  gem 'minitest'
+  gem 'rake'
+  gem 'redis-namespace'
+  gem 'simplecov'
+end

data/LICENSE

@@ -5,5 +5,5 @@ the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.
 
 Sidekiq Pro has a commercial-friendly license allowing private forks
-and modifications of Sidekiq.  Please see http://sidekiq.org/pro/ for
+and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
 more detail.  You can find the commercial license terms in COMM-LICENSE.

data/Pro-Changes.md

@@ -4,6 +4,36 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+4.0.4
+---------
+
+- Update Sidekiq::Client patches to work with new Module#prepend
+  mechanism in Sidekiq 5.2.0. [#3930]
+
+4.0.3
+---------
+
+- Add at\_exit handler to push any saved jobs in `reliable_push` when exiting. [#3823]
+- Implement batch death callback.  This is fired the first time a job within a batch dies. [#3841]
+```ruby
+batch = Sidekiq::Batch.new
+batch.on(:death, ...)
+```
+
+4.0.2
+---------
+
+- Remove super\_fetch edge case leading to an unnecessary `sleep(1)`
+  call and resulting latency [#3790]
+- Fix possible bad statsd metric call on super\_fetch startup
+- Remove superfluous `freeze` calls on Strings [#3759]
+
+4.0.1
+---------
+
+- Fix incompatibility with the statsd-ruby gem [#3740]
+- Add tags to Statsd metrics when using Datadog [#3744]
+
 4.0.0
 ---------
 

data/lib/sidekiq/api.rb

@@ -1,9 +1,24 @@
-# encoding: utf-8
 # frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
+
+  module RedisScanner
+    def sscan(conn, key)
+      cursor = '0'
+      result = []
+      loop do
+        cursor, values = conn.sscan(key, cursor)
+        result.push(*values)
+        break if cursor == '0'
+      end
+      result
+    end
+  end
+
   class Stats
+    include RedisScanner
+
     def initialize
       fetch_stats!
     end
@@ -51,33 +66,39 @@ module Sidekiq
     def fetch_stats!
       pipe1_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          conn.get('stat:processed'.freeze)
-          conn.get('stat:failed'.freeze)
-          conn.zcard('schedule'.freeze)
-          conn.zcard('retry'.freeze)
-          conn.zcard('dead'.freeze)
-          conn.scard('processes'.freeze)
-          conn.lrange('queue:default'.freeze, -1, -1)
-          conn.smembers('processes'.freeze)
-          conn.smembers('queues'.freeze)
+          conn.get('stat:processed')
+          conn.get('stat:failed')
+          conn.zcard('schedule')
+          conn.zcard('retry')
+          conn.zcard('dead')
+          conn.scard('processes')
+          conn.lrange('queue:default', -1, -1)
         end
       end
 
+      processes = Sidekiq.redis do |conn|
+        sscan(conn, 'processes')
+      end
+
+      queues = Sidekiq.redis do |conn|
+        sscan(conn, 'queues')
+      end
+
       pipe2_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          pipe1_res[7].each {|key| conn.hget(key, 'busy'.freeze) }
-          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+          processes.each {|key| conn.hget(key, 'busy') }
+          queues.each {|queue| conn.llen("queue:#{queue}") }
         end
       end
 
-      s = pipe1_res[7].size
+      s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
                                 job = Sidekiq.load_json(entry) rescue {}
                                 now = Time.now.to_f
-                                thence = job['enqueued_at'.freeze] || now
+                                thence = job['enqueued_at'] || now
                                 now - thence
                               else
                                 0
@@ -117,9 +138,11 @@ module Sidekiq
     end
 
     class Queues
+      include RedisScanner
+
       def lengths
         Sidekiq.redis do |conn|
-          queues = conn.smembers('queues'.freeze)
+          queues = sscan(conn, 'queues')
 
           lengths = conn.pipelined do
             queues.each do |queue|
@@ -163,7 +186,7 @@ module Sidekiq
 
         while i < @days_previous
           date = @start_date - i
-          datestr = date.strftime("%Y-%m-%d".freeze)
+          datestr = date.strftime("%Y-%m-%d")
           keys << "stat:#{stat}:#{datestr}"
           dates << datestr
           i += 1
@@ -199,18 +222,19 @@ module Sidekiq
   #
   class Queue
     include Enumerable
+    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| c.smembers('queues'.freeze) }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
     def initialize(name="default")
-      @name = name
+      @name = name.to_s
       @rname = "queue:#{name}"
     end
 
@@ -273,7 +297,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.multi do
           conn.del(@rname)
-          conn.srem("queues".freeze, name)
+          conn.srem("queues", name)
         end
       end
     end
@@ -349,9 +373,9 @@ module Sidekiq
                     job_args
                   end
                 else
-                  if self['encrypt'.freeze]
+                  if self['encrypt']
                     # no point in showing 150+ bytes of random garbage
-                    args[-1] = '[encrypted data]'.freeze
+                    args[-1] = '[encrypted data]'
                   end
                   args
                 end
@@ -646,6 +670,12 @@ module Sidekiq
         each(&:retry)
       end
     end
+
+    def kill_all
+      while size > 0
+        each(&:kill)
+      end
+    end
   end
 
   ##
@@ -701,17 +731,18 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
+    include RedisScanner
 
     def initialize(clean_plz=true)
-      self.class.cleanup if clean_plz
+      cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
-    def self.cleanup
+    def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes').sort
+        procs = sscan(conn, 'processes').sort
         heartbeats = conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'info')
@@ -731,7 +762,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+      procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -866,10 +897,11 @@ module Sidekiq
   #
   class Workers
     include Enumerable
+    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         procs.sort.each do |key|
           valid, workers = conn.pipelined do
             conn.exists(key)
@@ -891,7 +923,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         if procs.empty?
           0
         else

data/lib/sidekiq/cli.rb

@@ -1,4 +1,3 @@
-# encoding: utf-8
 # frozen_string_literal: true
 $stdout.sync = true
 
@@ -17,7 +16,7 @@ module Sidekiq
     include Singleton unless $TESTING
 
     PROCTITLES = [
-      proc { 'sidekiq'.freeze },
+      proc { 'sidekiq' },
       proc { Sidekiq::VERSION },
       proc { |me, data| data['tag'] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
@@ -65,7 +64,7 @@ module Sidekiq
       sigs.each do |sig|
         begin
           trap sig do
-            self_write.puts(sig)
+            self_write.write("#{sig}\n")
           end
         rescue ArgumentError
           puts "Signal #{sig} not supported"
@@ -81,6 +80,12 @@ module Sidekiq
       ver = Sidekiq.redis_info['redis_version']
       raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
 
+      # Since the user can pass us a connection pool explicitly in the initializer, we
+      # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
+      cursize = Sidekiq.redis_pool.size
+      needed = Sidekiq.options[:concurrency] + 2
+      raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
+
       # cache process identity
       Sidekiq.options[:identity] = identity
 
@@ -330,6 +335,8 @@ module Sidekiq
           opts[:tag] = arg
         end
 
+        # this index remains here for backwards compatibility but none of the Sidekiq
+        # family use this value anymore.  it was used by Pro's original reliable_fetch.
         o.on '-i', '--index INT', "unique process index on this machine" do |arg|
           opts[:index] = Integer(arg.match(/\d+/)[0])
         end
@@ -429,9 +436,9 @@ module Sidekiq
     end
 
     def parse_queue(opts, q, weight=nil)
-      [weight.to_i, 1].max.times do
-       (opts[:queues] ||= []) << q
-      end
+      opts[:queues] ||= []
+      raise ArgumentError, "queues: #{q} cannot be defined twice" if opts[:queues].include?(q)
+      [weight.to_i, 1].max.times { opts[:queues] << q }
       opts[:strict] = false if weight.to_i > 0
     end
   end

data/lib/sidekiq/client.rb

@@ -68,18 +68,19 @@ module Sidekiq
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item['class'.freeze], normed)
+      payload = process_single(item['class'], normed)
 
       if payload
         raw_push([payload])
-        payload['jid'.freeze]
+        payload['jid']
       end
     end
 
     ##
-    # Push a large number of jobs to Redis.  In practice this method is only
-    # useful if you are pushing thousands of jobs or more.  This method
-    # cuts out the redis network round trip latency.
+    # Push a large number of jobs to Redis. This method cuts out the redis
+    # network round trip latency.  I wouldn't recommend pushing more than
+    # 1000 per call but YMMV based on network quality, size of job args, etc.
+    # A large number of jobs can cause a bit of Redis command processing latency.
     #
     # Takes the same arguments as #push except that args is expected to be
     # an Array of Arrays.  All other keys are duplicated for each job.  Each job
@@ -89,19 +90,19 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items['args'.freeze].first
+      arg = items['args'].first
       return [] unless arg # no jobs to push
       raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
 
       normed = normalize_item(items)
-      payloads = items['args'.freeze].map do |args|
-        copy = normed.merge('args'.freeze => args, 'jid'.freeze => SecureRandom.hex(12), 'enqueued_at'.freeze => Time.now.to_f)
-        result = process_single(items['class'.freeze], copy)
+      payloads = items['args'].map do |args|
+        copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
+        result = process_single(items['class'], copy)
         result ? result : nil
       end.compact
 
       raw_push(payloads) if !payloads.empty?
-      payloads.collect { |payload| payload['jid'.freeze] }
+      payloads.collect { |payload| payload['jid'] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -144,14 +145,14 @@ module Sidekiq
       # Messages are enqueued to the 'default' queue.
       #
       def enqueue(klass, *args)
-        klass.client_push('class'.freeze => klass, 'args'.freeze => args)
+        klass.client_push('class' => klass, 'args' => args)
       end
 
       # Example usage:
       #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
-        klass.client_push('queue'.freeze => queue, 'class'.freeze => klass, 'args'.freeze => args)
+        klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
       end
 
       # Example usage:
@@ -162,8 +163,8 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = { 'class'.freeze => klass, 'args'.freeze => args, 'at'.freeze => ts, 'queue'.freeze => queue }
-        item.delete('at'.freeze) if ts <= now
+        item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
+        item.delete('at') if ts <= now
 
         klass.client_push(item)
       end
@@ -188,25 +189,25 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first['at'.freeze]
-        conn.zadd('schedule'.freeze, payloads.map do |hash|
-          at = hash.delete('at'.freeze).to_s
+      if payloads.first['at']
+        conn.zadd('schedule', payloads.map do |hash|
+          at = hash.delete('at').to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
-        q = payloads.first['queue'.freeze]
+        q = payloads.first['queue']
         now = Time.now.to_f
         to_push = payloads.map do |entry|
-          entry['enqueued_at'.freeze] = now
+          entry['enqueued_at'] = now
           Sidekiq.dump_json(entry)
         end
-        conn.sadd('queues'.freeze, q)
+        conn.sadd('queues', q)
         conn.lpush("queue:#{q}", to_push)
       end
     end
 
     def process_single(worker_class, item)
-      queue = item['queue'.freeze]
+      queue = item['queue']
 
       middleware.invoke(worker_class, item, queue, @redis_pool) do
         item
@@ -214,25 +215,25 @@ module Sidekiq
     end
 
     def normalize_item(item)
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class') && item.has_key?('args')
       raise(ArgumentError, "Job args must be an Array") unless item['args'].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at'.freeze) && !item['at'].is_a?(Numeric)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'].is_a?(Class) || item['class'].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.has_key?('at') && !item['at'].is_a?(Numeric)
       #raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item['class'.freeze])
+      normalized_hash(item['class'])
         .each{ |key, value| item[key] = value if item[key].nil? }
 
-      item['class'.freeze] = item['class'.freeze].to_s
-      item['queue'.freeze] = item['queue'.freeze].to_s
-      item['jid'.freeze] ||= SecureRandom.hex(12)
-      item['created_at'.freeze] ||= Time.now.to_f
+      item['class'] = item['class'].to_s
+      item['queue'] = item['queue'].to_s
+      item['jid'] ||= SecureRandom.hex(12)
+      item['created_at'] ||= Time.now.to_f
       item
     end
 
     def normalized_hash(item_class)
       if item_class.is_a?(Class)
-        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
+        raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options')
         item_class.get_sidekiq_options
       else
         Sidekiq.default_worker_options

data/lib/sidekiq/delay.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Sidekiq
   module Extensions
 

data/lib/sidekiq/fetch.rb

@@ -13,7 +13,7 @@ module Sidekiq
       end
 
       def queue_name
-        queue.sub(/.*queue:/, ''.freeze)
+        queue.sub(/.*queue:/, '')
       end
 
       def requeue

data/lib/sidekiq/job_logger.rb

@@ -1,9 +1,10 @@
+# frozen_string_literal: true
 module Sidekiq
   class JobLogger
 
     def call(item, queue)
       start = Time.now
-      logger.info("start".freeze)
+      logger.info("start")
       yield
       logger.info("done: #{elapsed(start)} sec")
     rescue Exception

data/lib/sidekiq/job_retry.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/scheduled'
 require 'sidekiq/api'
 
@@ -204,7 +205,11 @@ module Sidekiq
     end
 
     def delay_for(worker, count, exception)
-      worker && worker.sidekiq_retry_in_block && retry_in(worker, count, exception) || seconds_to_delay(count)
+      if worker && worker.sidekiq_retry_in_block
+        custom_retry_in = retry_in(worker, count, exception).to_i
+        return custom_retry_in if custom_retry_in > 0
+      end
+      seconds_to_delay(count)
     end
 
     # delayed_job uses the same basic formula
@@ -214,7 +219,7 @@ module Sidekiq
 
     def retry_in(worker, count, exception)
       begin
-        worker.sidekiq_retry_in_block.call(count, exception).to_i
+        worker.sidekiq_retry_in_block.call(count, exception)
       rescue Exception => e
         handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
         nil