sidekiq 5.0.5 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96202101069aaee71e462e2f3b91f5d4c58e952f
-  data.tar.gz: ad75e0b5dc019bb8d1058b124535190f00b06f60
+  metadata.gz: 2b5307056552014ab284241c16aa83a8d334d31b
+  data.tar.gz: acc9588810366d8c6dbb03d3c5cbe140a169f501
 SHA512:
-  metadata.gz: 4de121e1a00611b97f5c8838c5beb1d5a4fa5e6fb37e7cd7b2ac2074239d8a48d1d9526d4a8c0a94fcfcebe7ee3507209735569c78309f18d29363fbab24b992
-  data.tar.gz: 239a247dd69bab58148229f64d7b8ff12a658bb696c08c9f00b9ee9170c5d40ea052891eef5d89742633282d99095b4eb6004df298aa2f06b5f114fbbcfb10de
+  metadata.gz: 8b051912f610e763585e847a991cee6e684d4b27111f91cfe395a1e02539424e931f5ba16141e30a06c3dcf9de3c54db578be47a9fcd629222b81d8295f7c12a
+  data.tar.gz: 836cc95385406201ad9e26363698d838381ef993e75034bfc030c4ef0d2821c80e3a4612c5a7ec8bfca1d7acdadd02bcf29f5b979d56ece6752686f6293216ef

data/.github/issue_template.md

@@ -4,6 +4,8 @@ Sidekiq / Pro / Enterprise version(s):
 Please include your initializer and any error message with the full backtrace.
 
 Are you using an old version?
-Have you checked the changelog to see if your issue has been fixed in a later version?
+Have you checked the changelogs to see if your issue has been fixed in a later version?
 
 https://github.com/mperham/sidekiq/blob/master/Changes.md
+https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md
+https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md

data/.travis.yml

@@ -7,7 +7,8 @@ before_install:
   - gem install bundler
   - gem update bundler
 rvm:
-  - 2.2.4
-  - 2.3.0
-  - 2.4.0
-  - jruby-9.1.6.0
+  - 2.2.10
+  - 2.3.7
+  - 2.4.4
+  - 2.5.1
+  - jruby-9.1.17.0

data/Changes.md

@@ -2,12 +2,64 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-HEAD
+5.2.1
+-----------
+
+- Fix concurrent modification error during heartbeat [#3921]
+
+5.2.0
+-----------
+
+- **Decrease default concurrency from 25 to 10** [#3892]
+- Verify connection pool sizing upon startup [#3917]
+- Smoother scheduling for large Sidekiq clusters [#3889]
+- Switch Sidekiq::Testing impl from alias\_method to Module#prepend, for resiliency [#3852]
+- Update Sidekiq APIs to use SCAN for scalability [#3848, ffiller]
+- Remove concurrent-ruby gem dependency [#3830]
+- Optimize Web UI's bootstrap.css [#3914]
+
+5.1.3
+-----------
+
+- Fix version comparison so Ruby 2.2.10 works. [#3808, nateberkopec]
+
+5.1.2
+-----------
+
+- Add link to docs in Web UI footer
+- Fix crash on Ctrl-C in Windows [#3775, Bernica]
+- Remove `freeze` calls on String constants. This is superfluous with Ruby
+  2.3+ and `frozen_string_literal: true`. [#3759]
+- Fix use of AR middleware outside of Rails [#3787]
+- Sidekiq::Worker `sidekiq_retry_in` block can now return nil or 0 to use
+  the default backoff delay [#3796, dsalahutdinov]
+
+5.1.1
+-----------
+
+- Fix Web UI incompatibility with Redis 3.x gem [#3749]
+
+5.1.0
+-----------
+
+- **NEW** Global death handlers - called when your job exhausts all
+  retries and dies.  Now you can take action when a job fails permanently. [#3721]
+- **NEW** Enable ActiveRecord query cache within jobs by default [#3718, sobrinho]
+  This will prevent duplicate SELECTS; cache is cleared upon any UPDATE/INSERT/DELETE.
+  See the issue for how to bypass the cache or disable it completely.
+- Scheduler timing is now more accurate, 15 -> 5 seconds [#3734]
+- Exceptions during the :startup event will now kill the process [#3717]
+- Make `Sidekiq::Client.via` reentrant [#3715]
+- Fix use of Sidekiq logger outside of the server process [#3714]
+- Tweak `constantize` to better match Rails class lookup. [#3701, caffeinated-tech]
+
+5.0.5
 -----------
 
 - Update gemspec to allow newer versions of the Redis gem [#3617]
 - Refactor Worker.set so it can be memoized [#3602]
 - Fix display of Redis URL in web footer, broken in 5.0.3 [#3560]
+- Update `Sidekiq::Job#display_args` to avoid mutation [#3621]
 
 5.0.4
 -----------

data/Ent-Changes.md

@@ -4,6 +4,27 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+-------------
+
+- Add support for sidekiqswarm memory monitoring on FreeBSD [#3884]
+
+1.7.1
+-------------
+
+- Fix Lua error in concurrent rate limiter under heavy contention
+- Remove superfluous `freeze` calls on Strings [#3759]
+
+1.7.0
+-------------
+
+- **NEW FEATURE** [Rolling restarts](https://github.com/mperham/sidekiq/wiki/Ent-Rolling-Restarts) - great for long running jobs!
+- Adjust middleware so unique jobs that don't push aren't registered in a Batch [#3662]
+- Add new unlimited rate limiter, useful for testing [#3743]
+```ruby
+limiter = Sidekiq::Limiter.unlimited(...any args...)
+```
+
 1.6.1
 -------------
 

data/Gemfile

@@ -1,32 +1,14 @@
 source 'https://rubygems.org'
 gemspec
 
-gem 'rails', '>= 5.0.1'
-gem "hiredis"
-gem 'simplecov'
-gem 'minitest'
-#gem 'minitest-utils'
-gem 'toxiproxy'
-
-# For Redis 4.0 support. Unreleased 9cb81bf.
-gem 'redis-namespace', git: 'https://github.com/resque/redis-namespace'
-
-platforms :rbx do
-  gem 'rubysl', '~> 2.0'         # if using anything in the ruby standard library
-  gem 'psych'                    # if using yaml
-  gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
-end
-
-platforms :ruby do
-  gem 'sqlite3'
+# load testing
+#gem "hiredis"
+#gem 'toxiproxy'
+
+group :test do
+  gem 'rails', '>= 5.0.1'
+  gem 'minitest'
+  gem 'rake'
+  gem 'redis-namespace'
+  gem 'simplecov'
 end
-
-platforms :mri do
-  gem 'pry-byebug'
-  gem 'ruby-prof'
-end
-
-#platforms :jruby do
-  #gem 'jruby-openssl'
-  #gem 'activerecord-jdbcsqlite3-adapter'
-#end

data/LICENSE

@@ -5,5 +5,5 @@ the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
 for license text.
 
 Sidekiq Pro has a commercial-friendly license allowing private forks
-and modifications of Sidekiq.  Please see http://sidekiq.org/pro/ for
+and modifications of Sidekiq.  Please see https://sidekiq.org/products/pro.html for
 more detail.  You can find the commercial license terms in COMM-LICENSE.

data/Pro-4.0-Upgrade.md

@@ -0,0 +1,35 @@
+# Welcome to Sidekiq Pro 4.0!
+
+Sidekiq Pro 4.0 is designed to work with Sidekiq 5.0.
+
+## What's New
+
+* Batches now "die" if any of their jobs die.  You can enumerate the set
+  of dead batches and their associated dead jobs.  The success callback
+  for a dead batch will never fire unless these jobs are fixed.
+```ruby
+Sidekiq::Batch::DeadSet.new.each do |status|
+  status.dead? # => true
+  status.dead_jobs # => [...]
+end
+```
+This API allows you to enumerate the batches which need help.
+If you fix the issue and the dead jobs succeed, the batch will succeed.
+* The older `reliable_fetch` and `timed_fetch` algorithms have been
+  removed.  Only super\_fetch is available in 4.0.
+* The statsd middleware has been tweaked to remove support for legacy,
+  pre-3.6.0 configuration and add relevant tags.
+* Requires Sidekiq 5.0.5+.
+
+## Upgrade
+
+* Upgrade to the latest Sidekiq Pro 3.x.
+```ruby
+gem 'sidekiq-pro', '< 4'
+```
+* Fix any deprecation warnings you see.
+* Upgrade to 4.x.
+```ruby
+gem 'sidekiq-pro', '< 5'
+```
+

data/Pro-Changes.md

@@ -4,6 +4,92 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+4.0.3
+---------
+
+- Add at\_exit handler to push any saved jobs in `reliable_push` when exiting. [#3823]
+- Implement batch death callback.  This is fired the first time a job within a batch dies. [#3841]
+```ruby
+batch = Sidekiq::Batch.new
+batch.on(:death, ...)
+```
+
+4.0.2
+---------
+
+- Remove super\_fetch edge case leading to an unnecessary `sleep(1)`
+  call and resulting latency [#3790]
+- Fix possible bad statsd metric call on super\_fetch startup
+- Remove superfluous `freeze` calls on Strings [#3759]
+
+4.0.1
+---------
+
+- Fix incompatibility with the statsd-ruby gem [#3740]
+
+4.0.0
+---------
+
+- See the [Sidekiq Pro 4.0](Pro-4.0-Upgrade.md) release notes.
+
+
+3.7.1
+---------
+
+- Deprecate timed\_fetch.  Switch to super\_fetch:
+```ruby
+config.super_fetch!
+```
+
+
+3.7.0
+---------
+
+- Refactor batch job success/failure to gracefully handle several edge
+  cases with regard to Sidekiq::Shutdown.  This should greatly reduce
+  the chances of seeing the long-standing "negative pending count" problem. [#3710]
+
+
+3.6.1
+---------
+
+- Add support for Datadog::Statsd, it is the recommended Statsd client.  [#3699]
+```ruby
+Sidekiq::Pro.dogstatsd = ->{ Datadog::Statsd.new("metrics.example.com", 8125) }
+```
+- Size the statsd connection pool based on Sidekiq's concurrency [#3700]
+
+
+3.6.0
+---------
+
+This release overhauls the Statsd metrics support and adds more
+metrics for tracking Pro feature usage.  In your initializer:
+```ruby
+Sidekiq::Pro.statsd = ->{ ::Statsd.new("127.0.0.1", 8125) }
+```
+Sidekiq Pro will emit more metrics to Statsd:
+```
+jobs.expired - when a job is expired
+jobs.recovered.push - when a job is recovered by reliable_push after network outage
+jobs.recovered.fetch - when a job is recovered by super_fetch after process crash
+batch.created - when a batch is created
+batch.complete - when a batch is completed
+batch.success - when a batch is successful
+```
+Sidekiq Pro's existing Statsd middleware has been rewritten to leverage the new API.
+Everything should be backwards compatible with one deprecation notice.
+
+
+3.5.4
+---------
+
+- Fix case in SuperFetch where Redis downtime can lead to processor thread death [#3684]
+- Fix case where TimedFetch might not recover some pending jobs
+- Fix edge case in Batch::Status#poll leading to premature completion [#3640]
+- Adjust scan API to check 100 elements at a time, to minimize network round trips
+  when scanning large sets.
+
 3.5.3
 ---------
 

data/README.md

@@ -77,16 +77,18 @@ Problems?
 If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
 Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
 
+Sidekiq Pro and Sidekiq Enterprise customers get private email support.  You can purchase at http://sidekiq.org; email support@contribsys.com for help.
+
 Useful resources:
 
 * Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
 * Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
 
-**No support via Twitter, 140 characters is not enough.**
+**No support via Twitter**
 
 Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
-See the [Sidekiq support page](http://sidekiq.org/support) for details.
+See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
 
 Thanks
 -----------------

data/Rakefile

@@ -6,7 +6,3 @@ Rake::TestTask.new(:test) do |test|
 end
 
 task :default => :test
-
-task :appraise do
-  exec("cd myapp && rake appraise")
-end

data/bin/sidekiqload

@@ -72,7 +72,7 @@ def handle_signal(launcher, sig)
     launcher.quiet
   when 'TTIN'
     Thread.list.each do |thread|
-      Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
       if thread.backtrace
         Sidekiq.logger.warn thread.backtrace.join("\n")
       else

data/lib/sidekiq/api.rb

@@ -1,9 +1,24 @@
-# encoding: utf-8
 # frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
+
+  module RedisScanner
+    def sscan(conn, key)
+      cursor = '0'
+      result = []
+      loop do
+        cursor, values = conn.sscan(key, cursor)
+        result.push(*values)
+        break if cursor == '0'
+      end
+      result
+    end
+  end
+
   class Stats
+    include RedisScanner
+
     def initialize
       fetch_stats!
     end
@@ -51,33 +66,39 @@ module Sidekiq
     def fetch_stats!
       pipe1_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          conn.get('stat:processed'.freeze)
-          conn.get('stat:failed'.freeze)
-          conn.zcard('schedule'.freeze)
-          conn.zcard('retry'.freeze)
-          conn.zcard('dead'.freeze)
-          conn.scard('processes'.freeze)
-          conn.lrange('queue:default'.freeze, -1, -1)
-          conn.smembers('processes'.freeze)
-          conn.smembers('queues'.freeze)
+          conn.get('stat:processed')
+          conn.get('stat:failed')
+          conn.zcard('schedule')
+          conn.zcard('retry')
+          conn.zcard('dead')
+          conn.scard('processes')
+          conn.lrange('queue:default', -1, -1)
         end
       end
 
+      processes = Sidekiq.redis do |conn|
+        sscan(conn, 'processes')
+      end
+
+      queues = Sidekiq.redis do |conn|
+        sscan(conn, 'queues')
+      end
+
       pipe2_res = Sidekiq.redis do |conn|
         conn.pipelined do
-          pipe1_res[7].each {|key| conn.hget(key, 'busy'.freeze) }
-          pipe1_res[8].each {|queue| conn.llen("queue:#{queue}") }
+          processes.each {|key| conn.hget(key, 'busy') }
+          queues.each {|queue| conn.llen("queue:#{queue}") }
         end
       end
 
-      s = pipe1_res[7].size
+      s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
                                 job = Sidekiq.load_json(entry) rescue {}
                                 now = Time.now.to_f
-                                thence = job['enqueued_at'.freeze] || now
+                                thence = job['enqueued_at'] || now
                                 now - thence
                               else
                                 0
@@ -117,9 +138,11 @@ module Sidekiq
     end
 
     class Queues
+      include RedisScanner
+
       def lengths
         Sidekiq.redis do |conn|
-          queues = conn.smembers('queues'.freeze)
+          queues = sscan(conn, 'queues')
 
           lengths = conn.pipelined do
             queues.each do |queue|
@@ -163,7 +186,7 @@ module Sidekiq
 
         while i < @days_previous
           date = @start_date - i
-          datestr = date.strftime("%Y-%m-%d".freeze)
+          datestr = date.strftime("%Y-%m-%d")
           keys << "stat:#{stat}:#{datestr}"
           dates << datestr
           i += 1
@@ -199,18 +222,19 @@ module Sidekiq
   #
   class Queue
     include Enumerable
+    extend RedisScanner
 
     ##
     # Return all known queues within Redis.
     #
     def self.all
-      Sidekiq.redis { |c| c.smembers('queues'.freeze) }.sort.map { |q| Sidekiq::Queue.new(q) }
+      Sidekiq.redis { |c| sscan(c, 'queues') }.sort.map { |q| Sidekiq::Queue.new(q) }
     end
 
     attr_reader :name
 
     def initialize(name="default")
-      @name = name
+      @name = name.to_s
       @rname = "queue:#{name}"
     end
 
@@ -273,7 +297,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.multi do
           conn.del(@rname)
-          conn.srem("queues".freeze, name)
+          conn.srem("queues", name)
         end
       end
     end
@@ -349,9 +373,9 @@ module Sidekiq
                     job_args
                   end
                 else
-                  if self['encrypt'.freeze]
+                  if self['encrypt']
                     # no point in showing 150+ bytes of random garbage
-                    args[-1] = '[encrypted data]'.freeze
+                    args[-1] = '[encrypted data]'
                   end
                   args
                 end
@@ -548,7 +572,7 @@ module Sidekiq
         end
         break if elements.empty?
         page -= 1
-        elements.each do |element, score|
+        elements.reverse.each do |element, score|
           yield SortedEntry.new(self, score, element)
         end
         offset_size = initial_size - @_size
@@ -656,7 +680,7 @@ module Sidekiq
       super 'dead'
     end
 
-    def kill(message)
+    def kill(message, opts={})
       now = Time.now.to_f
       Sidekiq.redis do |conn|
         conn.multi do
@@ -665,6 +689,16 @@ module Sidekiq
           conn.zremrangebyrank(name, 0, - self.class.max_jobs)
         end
       end
+
+      if opts[:notify_failure] != false
+        job = Sidekiq.load_json(message)
+        r = RuntimeError.new("Job killed by API")
+        r.set_backtrace(caller)
+        Sidekiq.death_handlers.each do |handle|
+          handle.call(job, r)
+        end
+      end
+      true
     end
 
     def retry_all
@@ -691,17 +725,18 @@ module Sidekiq
   #
   class ProcessSet
     include Enumerable
+    include RedisScanner
 
     def initialize(clean_plz=true)
-      self.class.cleanup if clean_plz
+      cleanup if clean_plz
     end
 
     # Cleans up dead processes recorded in Redis.
     # Returns the number of processes cleaned.
-    def self.cleanup
+    def cleanup
       count = 0
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes').sort
+        procs = sscan(conn, 'processes').sort
         heartbeats = conn.pipelined do
           procs.each do |key|
             conn.hget(key, 'info')
@@ -721,7 +756,7 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.smembers('processes') }.sort
+      procs = Sidekiq.redis { |conn| sscan(conn, 'processes') }.sort
 
       Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
@@ -856,10 +891,11 @@ module Sidekiq
   #
   class Workers
     include Enumerable
+    include RedisScanner
 
     def each
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         procs.sort.each do |key|
           valid, workers = conn.pipelined do
             conn.exists(key)
@@ -881,7 +917,7 @@ module Sidekiq
     # which can easily get out of sync with crashy processes.
     def size
       Sidekiq.redis do |conn|
-        procs = conn.smembers('processes')
+        procs = sscan(conn, 'processes')
         if procs.empty?
           0
         else