sidekiq 4.1.4 → 4.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4e031cb26d8dcc6ab4f404563cd0a711577e586
-  data.tar.gz: 0a05484ed39032f71dcd2db901afec38f4c9d023
+  metadata.gz: 595500a3810c25e8e04e86c644f966ff01cb035f
+  data.tar.gz: 3f92a9f61e61848bc2c0639469ce3588f3848fc0
 SHA512:
-  metadata.gz: c807c5e398088ee6b344cdce5fa57a77d51c7206a3cc11c4aeb8bc9af186eb82c292c215facb6ae54589ed5b1d95573b74a07622bc452b7ba970d0fb49c207df
-  data.tar.gz: a56f9815bfc2954b12a9598b6678aee7771ee949d7e886dfc6ddaacbe05f5294e008deccf063c9c39ac430f18322cbdff4468a9ebd3530a9ef8ce780b064bf7e
+  metadata.gz: c4e97c3665d1e902f47cfb579fda437dc834ac4b584606f1ca3b8fd9225e8502cacf5fb687673b4735b359de6af179589759a34369f1c74e904b37f8fe70a4ee
+  data.tar.gz: a4e6d41067bf6b88c77577def0c1f9ecc3f5d46f0bdfb0821952c26867e91a71c64231aab09a25096d22c3272dea2dcc37547bc6096dd7622a192fa00e0a4e50

data/.github/issue_template.md

@@ -1,4 +1,9 @@
 Ruby version:
 Sidekiq / Pro / Enterprise version(s):
 
-If relevant, please include your initializer and any error message with the full backtrace.
+Please include your initializer and any error message with the full backtrace.
+
+Are you using an old version?
+Have you checked the changelog to see if your issue has been fixed in a later version?
+
+https://github.com/mperham/sidekiq/blob/master/Changes.md

data/.travis.yml

@@ -6,13 +6,13 @@ services:
 before_install:
   - gem install bundler
   - gem update bundler
-rvm:
-  - 2.0.0
-  - 2.1.8
-  - 2.2.4
-  - 2.3.0
-  - jruby-head
-  - rbx-2
 matrix:
-  allow_failures:
-    - rvm: rbx-2
+  include:
+    - rvm: 2.2.4
+      env: "PERCY_ENABLE=0"
+    - rvm: 2.3.0
+      env: "PERCY_ENABLE=0"
+    - rvm: 2.4.0
+      env: "PERCY_ENABLE=1"
+    - rvm: jruby-9.1.6.0
+      env: "PERCY_ENABLE=0"

data/Changes.md

@@ -1,5 +1,105 @@
 # Sidekiq Changes
 
+HEAD
+-----------
+
+- Scheduled jobs can now be moved directly to the Dead queue via API [#3390]
+- Fix edge case leading to job duplication when using Sidekiq Pro's
+  reliability feature [#3388]
+- Fix error class name display on retry page [#3348]
+- More robust latency calculation [#3340]
+
+4.2.9
+-----------
+
+- Rollback [#3303] which broke Heroku Redis users [#3311]
+- Add support for TSTP signal, for Sidekiq 5.0 forward compatibility. [#3302]
+
+4.2.8
+-----------
+
+- Fix rare edge case with Redis driver that can create duplicate jobs [#3303]
+- Fix Rails 5 loading issue [#3275]
+- Restore missing tooltips to timestamps in Web UI [#3310]
+- Work on **Sidekiq 5.0** is now active! [#3301]
+
+4.2.7
+-----------
+
+- Add new integration testing to verify code loading and job execution
+  in development and production modes with Rails 4 and 5 [#3241]
+- Fix delayed extensions in development mode [#3227, DarthSim]
+- Use Worker's `retry` default if job payload does not have a retry
+  attribute [#3234, mlarraz]
+
+4.2.6
+-----------
+
+- Run Rails Executor when in production [#3221, eugeneius]
+
+4.2.5
+-----------
+
+- Re-enable eager loading of all code when running non-development Rails 5. [#3203]
+- Better root URL handling for zany web servers [#3207]
+
+4.2.4
+-----------
+
+- Log errors coming from the Rails 5 reloader. [#3212, eugeneius]
+- Clone job data so middleware changes don't appear in Busy tab
+
+4.2.3
+-----------
+
+- Disable use of Rails 5's Reloader API in non-development modes, it has proven
+  to be unstable under load [#3154]
+- Allow disabling of Sidekiq::Web's cookie session to handle the
+  case where the app provides a session already [#3180, inkstak]
+```ruby
+Sidekiq::Web.set :sessions, false
+```
+- Fix Web UI sharding support broken in 4.2.2. [#3169]
+- Fix timestamps not updating during UI polling [#3193, shaneog]
+- Relax rack-protection version to >= 1.5.0
+- Provide consistent interface to exception handlers, changing the structure of the context hash. [#3161]
+
+4.2.2
+-----------
+
+- Fix ever-increasing cookie size with nginx [#3146, cconstantine]
+- Fix so Web UI works without trailing slash [#3158, timdorr]
+
+4.2.1
+-----------
+
+- Ensure browser does not cache JSON/AJAX responses. [#3136]
+- Support old Sinatra syntax for setting config [#3139]
+
+4.2.0
+-----------
+
+- Enable development-mode code reloading.  **With Rails 5.0+, you don't need
+  to restart Sidekiq to pick up your Sidekiq::Worker changes anymore!** [#2457]
+- **Remove Sinatra dependency**.  Sidekiq's Web UI now uses Rack directly.
+  Thank you to Sidekiq's newest committer, **badosu**, for writing the code
+  and doing a lot of testing to ensure compatibility with many different
+  3rd party plugins.  If your Web UI works with 4.1.4 but fails with
+  4.2.0, please open an issue. [#3075]
+- Allow tuning of concurrency with the `RAILS_MAX_THREADS` env var. [#2985]
+  This is the same var used by Puma so you can tune all of your systems
+  the same way:
+```sh
+web: RAILS_MAX_THREADS=5 bundle exec puma ...
+worker: RAILS_MAX_THREADS=10 bundle exec sidekiq ...
+```
+Using `-c` or `config/sidekiq.yml` overrides this setting.  I recommend
+adjusting your `config/database.yml` to use it too so connections are
+auto-scaled:
+```yaml
+  pool: <%= ENV['RAILS_MAX_THREADS'] || 5 %>
+```
+
 4.1.4
 -----------
 

data/Ent-Changes.md

@@ -3,7 +3,57 @@ Sidekiq Enterprise Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-HEAD
+1.5.2
+-------------
+
+- Fix encrypted arguments double-encrypted by retry or rate limiting [#3368]
+- Fix leak in concurrent rate limiter, run this in Rails console to clean up existing data [#3323]
+```ruby
+expiry = 1.month.to_i; Sidekiq::Limiter.redis { |c| c.scan_each(match: "lmtr-cfree-*") { |key| c.expire(key, expiry) } }
+```
+
+1.5.1
+-------------
+
+- Fix issue with census startup when not using Bundler configuration for
+  source credentials.
+
+1.5.0
+-------------
+
+- Add new web authorization API [#3251]
+- Update all sidekiqswarm env vars to use SIDEKIQ\_ prefix [#3218]
+- Add census reporting, the leader will ping contribsys nightly with aggregate usage metrics
+
+1.4.0
+-------------
+
+- No functional changes, require latest Sidekiq and Sidekiq Pro versions
+
+1.3.2
+-------------
+
+- Upgrade encryption to use OpenSSL's more secure GCM mode. [#3060]
+
+1.3.1
+-------------
+
+- Fix multi-process memory monitoring on CentOS 6.x [#3063]
+- Polish the new encryption feature a bit.
+
+1.3.0
+-------------
+
+- **BETA** [New encryption feature](https://github.com/mperham/sidekiq/wiki/Ent-Encryption)
+  which automatically encrypts the last argument of a Worker, aka the secret bag.
+
+1.2.4
+-------------
+
+- Fix issue causing some minutely jobs to execute every other minute.
+- Log a warning if slow periodic processing causes us to miss a clock tick.
+
+1.2.3
 -------------
 
 - Periodic jobs could stop executing until process restart if Redis goes down [#3047]

data/Gemfile

@@ -1,11 +1,11 @@
 source 'https://rubygems.org'
 gemspec
 
+gem 'rails', '>= 5.0.1'
 gem "hiredis"
-gem 'rails', '~> 4.2'
 gem 'simplecov'
 gem 'minitest'
-gem 'minitest-utils'
+#gem 'minitest-utils'
 gem 'toxiproxy'
 
 platforms :rbx do
@@ -23,7 +23,7 @@ platforms :mri do
   gem 'ruby-prof'
 end
 
-platforms :jruby do
-  gem 'jruby-openssl'
-  gem 'activerecord-jdbcsqlite3-adapter'
-end
+#platforms :jruby do
+  #gem 'jruby-openssl'
+  #gem 'activerecord-jdbcsqlite3-adapter'
+#end

data/Pro-Changes.md

@@ -3,6 +3,75 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+---------
+
+- Make Batch UI progress bar more friendly to the colorblind [#3387]
+
+3.4.5
+---------
+
+- Fix potential job loss with reliable scheduler when lots of jobs are scheduled
+  at precisely the same time. Thanks to raivil for his hard work in
+  reproducing the bug. [#3371]
+
+3.4.4
+---------
+
+- Optimize super\_fetch shutdown to restart jobs quicker [#3249]
+
+3.4.3
+---------
+
+- Limit reliable scheduler to enqueue up to 100 jobs per call, minimizing Redis latency [#3332]
+- Fix bug in super\_fetch logic for queues with `_` in the name [#3339]
+
+3.4.2
+---------
+
+- Add `Batch::Status#invalidated?` API which returns true if any/all
+  JIDs were invalidated within the batch. [#3326]
+
+3.4.1
+---------
+
+- Allow super\_fetch's orphan job check to happen as often as every hour [#3273]
+- Officially deprecate reliable\_fetch algorithm, I now recommend you use `super_fetch` instead:
+```ruby
+Sidekiq.configure_server do |config|
+  config.super_fetch!
+end
+```
+Also note that Sidekiq's `-i/--index` option is no longer used/relevant with super\_fetch.
+- Don't display "Delete/Retry All" buttons when filtering in Web UI [#3243]
+- Reimplement Sidekiq::JobSet#find\_job with ZSCAN [#3197]
+
+3.4.0
+---------
+
+- Introducing the newest reliable fetching algorithm: `super_fetch`!  This
+  algorithm will replace reliable\_fetch in Pro 4.0.  super\_fetch is
+  bullet-proof across all environments, no longer requiring stable
+  hostnames or an index to be set per-process. [#3077]
+```ruby
+Sidekiq.configure_server do |config|
+  config.super_fetch!
+end
+```
+  Thank you to @jonhyman for code review and the Sidekiq Pro customers that
+  beta tested super\_fetch.
+
+3.3.3
+---------
+
+- Update Web UI extension to work with Sidekiq 4.2.0's new Web UI. [#3075]
+
+3.3.2
+---------
+
+- Minimize batch memory usage after success [#3083]
+- Extract batch's 24 hr linger expiry to a LINGER constant so it can be tuned. [#3011]
+
 3.3.1
 ---------
 

data/README.md

@@ -50,7 +50,8 @@ Getting Started
 -----------------
 
 See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
-You can watch [Railscast #366](http://railscasts.com/episodes/366-sidekiq) to see Sidekiq in action.  If you do everything right, you should see this:
+You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
+Sidekiq and see its features in action.  Here's the Web UI:
 
 ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
 
@@ -84,8 +85,8 @@ Useful resources:
 
 **No support via Twitter, 140 characters is not enough.**
 
-Every Friday morning 9am Pacific is Sidekiq happy hour: I video chat and answer questions.
-See the [Sidekiq support page](http://sidekiq.org/support).
+Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
+See the [Sidekiq support page](http://sidekiq.org/support) for details.
 
 Thanks
 -----------------

data/Rakefile

@@ -1,9 +1,12 @@
 require 'bundler/gem_tasks'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
-  #SO MUCH NOISE
-  #test.warning = true
+  test.warning = true
   test.pattern = 'test/**/test_*.rb'
 end
 
 task :default => :test
+
+task :appraise do
+  exec("cd myapp && rake appraise")
+end

data/bin/sidekiqload

@@ -12,22 +12,9 @@ require_relative '../lib/sidekiq/launcher'
 
 include Sidekiq::Util
 
-# brew tap shopify/shopify
-# brew install toxiproxy
-# gem install toxiproxy
-require 'toxiproxy'
-# simulate a non-localhost network for realer-world conditions.
-# adding 1ms of network latency has an ENORMOUS impact on benchmarks
-Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
-}])
-
-
 Sidekiq.configure_server do |config|
   #config.options[:concurrency] = 1
-  config.redis = { driver: :hiredis, db: 13, port: 6380 }
+  config.redis = { db: 13 }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -49,14 +36,14 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
-require 'toxiproxy'
+#require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
-Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
-}])
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
 
 self_read, self_write = IO.pipe
 %w(INT TERM USR1 USR2 TTIN).each do |sig|
@@ -121,7 +108,7 @@ Sidekiq.logger.error "Created #{count*iter} jobs"
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 2
+      sleep 1
       qsize, retries = Sidekiq.redis do |conn|
         conn.pipelined do
           conn.llen "queue:default"
@@ -143,8 +130,8 @@ begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
   fire_event(:startup)
-  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -152,7 +139,7 @@ begin
       signal = readable_io.first[0].gets.strip
       handle_signal(launcher, signal)
     end
-  end
+  #end
 rescue SystemExit => e
   #Sidekiq.logger.error("Profiling...")
   #result = RubyProf.stop

data/lib/generators/sidekiq/templates/worker_test.rb.erb

@@ -1,4 +1,4 @@
-require_relative 'test_helper'
+require 'test_helper'
 <% module_namespacing do -%>
 class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
   def test_example

data/lib/sidekiq/api.rb

@@ -1,5 +1,5 @@
-# frozen_string_literal: true
 # encoding: utf-8
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
@@ -75,7 +75,10 @@ module Sidekiq
       enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
-                                Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at'.freeze]
+                                job = Sidekiq.load_json(entry)
+                                now = Time.now.to_f
+                                thence = job['enqueued_at'.freeze] || now
+                                now - thence
                               else
                                 0
                               end
@@ -225,7 +228,10 @@ module Sidekiq
         conn.lrange(@rname, -1, -1)
       end.first
       return 0 unless entry
-      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+      job = Sidekiq.load_json(entry)
+      now = Time.now.to_f
+      thence = job['enqueued_at'] || now
+      now - thence
     end
 
     def each
@@ -351,7 +357,8 @@ module Sidekiq
     end
 
     def latency
-      Time.now.to_f - (@item['enqueued_at'] || @item['created_at'])
+      now = Time.now.to_f
+      now - (@item['enqueued_at'] || @item['created_at'] || now)
     end
 
     ##
@@ -416,10 +423,9 @@ module Sidekiq
     end
 
     def retry
-      raise "Retry not available on jobs which have not failed" unless item["failed_at"]
       remove_job do |message|
         msg = Sidekiq.load_json(message)
-        msg['retry_count'] -= 1
+        msg['retry_count'] -= 1 if msg['retry_count']
         Sidekiq::Client.push(msg)
       end
     end
@@ -427,9 +433,7 @@ module Sidekiq
     ##
     # Place job in the dead set
     def kill
-      raise 'Kill not available on jobs which have not failed' unless item['failed_at']
       remove_job do |message|
-        Sidekiq.logger.info { "Killing job #{message['jid']}" }
         now = Time.now.to_f
         Sidekiq.redis do |conn|
           conn.multi do
@@ -441,6 +445,10 @@ module Sidekiq
       end
     end
 
+    def error?
+      !!item['error_class']
+    end
+
     private
 
     def remove_job
@@ -585,13 +593,13 @@ module Sidekiq
   # Allows enumeration of scheduled jobs within Sidekiq.
   # Based on this, you can search/filter for jobs.  Here's an
   # example where I'm selecting all jobs of a certain type
-  # and deleting them from the retry queue.
+  # and deleting them from the schedule queue.
   #
   #   r = Sidekiq::ScheduledSet.new
-  #   r.select do |retri|
-  #     retri.klass == 'Sidekiq::Extensions::DelayedClass' &&
-  #     retri.args[0] == 'User' &&
-  #     retri.args[1] == 'setup_new_subscriber'
+  #   r.select do |scheduled|
+  #     scheduled.klass == 'Sidekiq::Extensions::DelayedClass' &&
+  #     scheduled.args[0] == 'User' &&
+  #     scheduled.args[1] == 'setup_new_subscriber'
   #   end.map(&:delete)
   class ScheduledSet < JobSet
     def initialize

data/lib/sidekiq/cli.rb

@@ -1,5 +1,5 @@
-# frozen_string_literal: true
 # encoding: utf-8
+# frozen_string_literal: true
 $stdout.sync = true
 
 require 'yaml'
@@ -52,7 +52,7 @@ module Sidekiq
 
       self_read, self_write = IO.pipe
 
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+      %w(INT TERM USR1 USR2 TTIN TSTP).each do |sig|
         begin
           trap sig do
             self_write.puts(sig)
@@ -135,6 +135,10 @@ module Sidekiq
       when 'USR1'
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
         launcher.quiet
+      when 'TSTP'
+        # USR1 is not available on JVM, allow TSTP as an alternate signal
+        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        launcher.quiet
       when 'USR2'
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
@@ -142,7 +146,7 @@ module Sidekiq
         end
       when 'TTIN'
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['sidekiq_label']}"
           if thread.backtrace
             Sidekiq.logger.warn thread.backtrace.join("\n")
           else
@@ -208,6 +212,8 @@ module Sidekiq
       opts = parse_config(cfile).merge(opts) if cfile
 
       opts[:strict] = true if opts[:strict].nil?
+      opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if !opts[:concurrency] && ENV["RAILS_MAX_THREADS"]
+      opts[:identity] = identity
 
       options.merge!(opts)
     end
@@ -227,18 +233,26 @@ module Sidekiq
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
           ::Rails.application.eager_load!
-        else
+        elsif ::Rails::VERSION::MAJOR == 4
           # Painful contortions, see 1791 for discussion
+          # No autoloading, we want to force eager load for everything.
           require File.expand_path("#{options[:require]}/config/application.rb")
           ::Rails::Application.initializer "sidekiq.eager_load" do
             ::Rails.application.config.eager_load = true
           end
           require 'sidekiq/rails'
           require File.expand_path("#{options[:require]}/config/environment.rb")
+        else
+          # Rails 5+ && development mode, use Reloader
+          require 'sidekiq/rails'
+          require File.expand_path("#{options[:require]}/config/environment.rb")
         end
         options[:tag] ||= default_tag
       else
-        require options[:require]
+        not_required_message = "#{options[:require]} was not required, you should use an explicit path: " +
+            "./#{options[:require]} or /path/to/#{options[:require]}"
+
+        require(options[:require]) || raise(ArgumentError, not_required_message)
       end
     end
 

data/lib/sidekiq/core_ext.rb

@@ -104,3 +104,16 @@ rescue LoadError
 end
 
 
+begin
+  require 'active_support/core_ext/kernel/reporting'
+rescue LoadError
+  module Kernel
+    module_function
+    def silence_warnings
+      old_verbose, $VERBOSE = $VERBOSE, nil
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+  end
+end