sidekiq 5.2.4 → 6.0.1

data/Gemfile

@@ -1,29 +1,24 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec
 
-gem 'rake'
-gem 'redis-namespace'
-gem 'rails', '~> 5.2'
-gem 'sqlite3', platforms: :ruby
-gem 'activerecord-jdbcsqlite3-adapter', platforms: :jruby
-
-group :development do
-  gem 'appraisal'
-end
+gem "rake"
+gem "redis-namespace"
+gem "rails"
+gem "sqlite3", platforms: :ruby
+gem "activerecord-jdbcsqlite3-adapter", platforms: :jruby
 
 group :test do
-  gem 'minitest'
-  gem 'minitest-focus'
-  gem 'minitest-reporters'
-  gem 'simplecov'
+  gem "minitest"
+  gem "simplecov"
 end
 
 group :development, :test do
-  gem 'pry-byebug', platforms: :mri
+  gem "pry-byebug", platforms: :mri
+  gem "standard"
 end
 
 group :load_test do
-  gem 'hiredis'
-  gem 'toxiproxy'
+  gem "hiredis"
+  gem "toxiproxy"
 end

data/Gemfile.lock

@@ -0,0 +1,196 @@
+PATH
+  remote: .
+  specs:
+    sidekiq (6.0.1)
+      connection_pool (>= 2.2.2)
+      rack (>= 2.0.0)
+      rack-protection (>= 2.0.0)
+      redis (>= 4.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (6.0.0)
+      actionpack (= 6.0.0)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+    actionmailbox (6.0.0)
+      actionpack (= 6.0.0)
+      activejob (= 6.0.0)
+      activerecord (= 6.0.0)
+      activestorage (= 6.0.0)
+      activesupport (= 6.0.0)
+      mail (>= 2.7.1)
+    actionmailer (6.0.0)
+      actionpack (= 6.0.0)
+      actionview (= 6.0.0)
+      activejob (= 6.0.0)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 2.0)
+    actionpack (6.0.0)
+      actionview (= 6.0.0)
+      activesupport (= 6.0.0)
+      rack (~> 2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.0)
+      actionpack (= 6.0.0)
+      activerecord (= 6.0.0)
+      activestorage (= 6.0.0)
+      activesupport (= 6.0.0)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.0)
+      activesupport (= 6.0.0)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.0)
+      activesupport (= 6.0.0)
+      globalid (>= 0.3.6)
+    activemodel (6.0.0)
+      activesupport (= 6.0.0)
+    activerecord (6.0.0)
+      activemodel (= 6.0.0)
+      activesupport (= 6.0.0)
+    activestorage (6.0.0)
+      actionpack (= 6.0.0)
+      activejob (= 6.0.0)
+      activerecord (= 6.0.0)
+      marcel (~> 0.3.1)
+    activesupport (6.0.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 0.7, < 2)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+      zeitwerk (~> 2.1, >= 2.1.8)
+    ast (2.4.0)
+    builder (3.2.3)
+    byebug (11.0.1)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.5)
+    connection_pool (2.2.2)
+    crass (1.0.4)
+    docile (1.3.2)
+    erubi (1.8.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
+    hiredis (0.6.3)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.3)
+    json (2.2.0)
+    loofah (2.2.3)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    marcel (0.3.3)
+      mimemagic (~> 0.3.2)
+    method_source (0.9.2)
+    mimemagic (0.3.3)
+    mini_mime (1.0.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    nio4r (2.5.1)
+    nokogiri (1.10.4)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.17.0)
+    parser (2.6.4.1)
+      ast (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    pry-byebug (3.7.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
+    rack (2.0.7)
+    rack-protection (2.0.7)
+      rack
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails (6.0.0)
+      actioncable (= 6.0.0)
+      actionmailbox (= 6.0.0)
+      actionmailer (= 6.0.0)
+      actionpack (= 6.0.0)
+      actiontext (= 6.0.0)
+      actionview (= 6.0.0)
+      activejob (= 6.0.0)
+      activemodel (= 6.0.0)
+      activerecord (= 6.0.0)
+      activestorage (= 6.0.0)
+      activesupport (= 6.0.0)
+      bundler (>= 1.3.0)
+      railties (= 6.0.0)
+      sprockets-rails (>= 2.0.0)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.2.0)
+      loofah (~> 2.2, >= 2.2.2)
+    railties (6.0.0)
+      actionpack (= 6.0.0)
+      activesupport (= 6.0.0)
+      method_source
+      rake (>= 0.8.7)
+      thor (>= 0.20.3, < 2.0)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    redis (4.1.2)
+    redis-namespace (1.6.0)
+      redis (>= 3.0.4)
+    rubocop (0.72.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-performance (1.4.1)
+      rubocop (>= 0.71.0)
+    ruby-progressbar (1.10.1)
+    simplecov (0.17.0)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    sprockets (3.7.2)
+      concurrent-ruby (~> 1.0)
+      rack (> 1, < 3)
+    sprockets-rails (3.2.1)
+      actionpack (>= 4.0)
+      activesupport (>= 4.0)
+      sprockets (>= 3.0.0)
+    sqlite3 (1.4.1)
+    standard (0.1.4)
+      rubocop (~> 0.72.0)
+      rubocop-performance (~> 1.4.0)
+    thor (0.20.3)
+    thread_safe (0.3.6)
+    toxiproxy (1.0.3)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    unicode-display_width (1.6.0)
+    websocket-driver (0.7.1)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.4)
+    zeitwerk (2.1.9)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord-jdbcsqlite3-adapter
+  hiredis
+  minitest
+  pry-byebug
+  rails
+  rake
+  redis-namespace
+  sidekiq!
+  simplecov
+  sqlite3
+  standard
+  toxiproxy

data/Pro-5.0-Upgrade.md

@@ -0,0 +1,25 @@
+# Welcome to Sidekiq Pro 5.0!
+
+Sidekiq Pro 5.0 is mainly a cleanup release for Sidekiq 6.0. The
+migration should be as close to trivial as a major version bump can be.
+Note that Sidekiq 6.0 does have major breaking changes.
+
+## What's New
+
+* New localizations for the Sidekiq Pro Web UI: ES, ZH, PT, JA, RU
+* Removed deprecated APIs and warnings.
+* Various changes for Sidekiq 6.0
+* Requires Ruby 2.5+ and Redis 4.0+
+* Requires Sidekiq 6.0+.
+
+## Upgrade
+
+* Upgrade to the latest Sidekiq Pro 4.x.
+```ruby
+gem 'sidekiq-pro', '< 5'
+```
+* Fix any deprecation warnings you see.
+* Upgrade to 5.x.
+```ruby
+gem 'sidekiq-pro', '< 6'
+```

data/Pro-Changes.md

@@ -4,11 +4,35 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
-HEAD
+5.0.1
 ---------
 
+- Rejigger batch failures UI to add direct links to retries and scheduled jobs [#4209]
+- Delete batch data with `UNLINK` [#4155]
+- Fix bug where a scheduled job can lose its scheduled time when using reliable push [#4267]
+- Sidekiq::JobSet#scan and #find_job APIs have been promoted to Sidekiq OSS. [#4259]
+
+5.0.0
+---------
+
+- There is no significant migration from Sidekiq Pro 4.0 to 5.0
+  but make sure you read the [update notes for Sidekiq
+6.0](https://github.com/mperham/sidekiq/blob/master/6.0-Upgrade.md).
+- Removed various deprecated APIs and associated warnings.
+- **BREAKING CHANGE** Remove the `Sidekiq::Batch::Status#dead_jobs` API in favor of
+  `Sidekiq::Batch::Status#dead_jids`. [#4217]
+- Update Sidekiq Pro codebase to use StandardRB formatting
+- Fix lingering "b-XXX-died" elements in Redis which could cause
+  excessive memory usage. [#4217]
+- Add ES translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
+
+4.0.5
+---------
+
+- Increase super\_fetch retriever thread count from 1 to 2 to make it
+  less sensitive to Redis latency.
 - Better handling of invalid job JSON by reliable scheduler [#4053]
-- Added ZH, PT, JA and RU translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
+- Added ZH, PT, JA and RU translations.
 
 4.0.4
 ---------

data/README.md

@@ -3,7 +3,8 @@ Sidekiq
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
 [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
-[![Build Status](https://travis-ci.org/mperham/sidekiq.svg)](https://travis-ci.org/mperham/sidekiq)
+[![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
+[![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
 [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
 
 
@@ -13,32 +14,26 @@ Sidekiq uses threads to handle many jobs at the same time in the
 same process.  It does not require Rails but will integrate tightly with
 Rails to make background processing dead simple.
 
-Sidekiq is compatible with Resque.  It uses the exact same
-message format as Resque so it can integrate into an existing Resque processing farm.
-You can have Sidekiq and Resque run side-by-side at the same time and
-use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
-
 Performance
 ---------------
 
-Version |	Latency | Garbage created for 10,000 jobs	| Time to process 100,000 jobs |	Throughput
------------------|------|---------|---------|------------------------
-Sidekiq 4.0.0    | 10ms	| 151 MB  | 22 sec  | **4500 jobs/sec**
-Sidekiq 3.5.1    | 22ms	| 1257 MB | 125 sec | 800 jobs/sec
-Resque 1.25.2    |  -	  | -       | 420 sec | 240 jobs/sec
-DelayedJob 4.1.1 |  -   | -       | 465 sec | 215 jobs/sec
+Version |	Latency | Garbage created for 10k jobs	| Time to process 100k jobs |	Throughput | Ruby
+-----------------|------|---------|---------|------------------------|-----
+Sidekiq 6.0.0    | 3 ms	| 156 MB  | 19 sec  | **5200 jobs/sec** | MRI 2.6.3
+Sidekiq 4.0.0    | 10 ms	| 151 MB  | 22 sec  | 4500 jobs/sec |
+Sidekiq 3.5.1    | 22 ms	| 1257 MB | 125 sec | 800 jobs/sec |
+Resque 1.25.2    |  -	  | -       | 420 sec | 240 jobs/sec |
+DelayedJob 4.1.1 |  -   | -       | 465 sec | 215 jobs/sec |
 
-<small>This benchmark can be found in `bin/sidekiqload`.</small>
+This benchmark can be found in `bin/sidekiqload` and assumes a Redis network latency of 1ms.
 
 Requirements
 -----------------
 
-Sidekiq supports CRuby 2.2.2+ and JRuby 9k.
-
-All Rails releases >= 4.0 are officially supported.
+- Redis: 4.0+
+- Ruby: MRI 2.5+ or JRuby 9.2+.
 
-Redis 2.8 or greater is required.  3.0.3+ is recommended for large
-installations with thousands of worker threads.
+Sidekiq 6.0 supports Rails 5.0+ but does not require it.
 
 
 Installation
@@ -63,7 +58,7 @@ Want to Upgrade?
 I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
 features, a commercial-friendly license and allow you to support high
 quality open source development all at the same time.  Please see the
-[Sidekiq](http://sidekiq.org/) homepage for more detail.
+[Sidekiq](https://sidekiq.org/) homepage for more detail.
 
 Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
 features and changes to Sidekiq and its bigger siblings.
@@ -77,24 +72,16 @@ Problems?
 If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
 Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
 
-Sidekiq Pro and Sidekiq Enterprise customers get private email support.  You can purchase at http://sidekiq.org; email support@contribsys.com for help.
+Sidekiq Pro and Sidekiq Enterprise customers get private email support.  You can purchase at https://sidekiq.org; email support@contribsys.com for help.
 
 Useful resources:
 
 * Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
-* Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
+* Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
 * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
 
-**No support via Twitter**
-
 Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
-See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
-
-Thanks
------------------
-
-Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
-performance problems on JRuby.  Unfortunately MRI does not have good multithreaded profiling tools.
+See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
 
 
 License
@@ -106,4 +93,4 @@ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for
 Author
 -----------------
 
-Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
+Mike Perham, [@mperham@mastodon.xyz](https://mastodon.xyz/@mperham) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)

data/Rakefile

@@ -1,9 +1,10 @@
-require 'bundler/gem_tasks'
-require 'rake/testtask'
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "standard/rake"
 
 Rake::TestTask.new(:test) do |test|
   test.warning = true
-  test.pattern = 'test/**/test_*.rb'
+  test.pattern = "test/**/test_*.rb"
 end
 
-task default: :test
+task default: [:standard, :test]

data/bin/sidekiqload

@@ -5,7 +5,8 @@
 $TESTING = false
 
 #require 'ruby-prof'
-Bundler.require(:default)
+require 'bundler/setup'
+Bundler.require(:default, :load_test)
 
 require_relative '../lib/sidekiq/cli'
 require_relative '../lib/sidekiq/launcher'
@@ -13,8 +14,8 @@ require_relative '../lib/sidekiq/launcher'
 include Sidekiq::Util
 
 Sidekiq.configure_server do |config|
-  #config.options[:concurrency] = 1
-  config.redis = { db: 13 }
+  config.options[:concurrency] = 10
+  config.redis = { db: 13, port: 6380, driver: :hiredis }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -28,7 +29,8 @@ class LoadWorker
     1
   end
 
-  def perform(idx)
+  def perform(idx, ts=nil)
+    puts(Time.now.to_f - ts) if ts != nil
     #raise idx.to_s if idx % 100 == 1
   end
 end
@@ -36,14 +38,15 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
-#require 'toxiproxy'
+# run `toxiproxy-server` in a separate terminal window.
+require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
-#Toxiproxy.populate([{
-    #"name": "redis",
-    #"listen": "127.0.0.1:6380",
-    #"upstream": "127.0.0.1:6379"
-#}])
+Toxiproxy.populate([{
+    "name": "redis",
+    "listen": "127.0.0.1:6380",
+    "upstream": "127.0.0.1:6379"
+}])
 
 self_read, self_write = IO.pipe
 %w(INT TERM TSTP TTIN).each do |sig|
@@ -65,7 +68,7 @@ def handle_signal(launcher, sig)
     # http://jira.codehaus.org/browse/JRUBY-4637
     raise Interrupt
   when 'TERM'
-    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    # Heroku sends TERM and then waits 30 seconds for process to exit.
     raise Interrupt
   when 'TSTP'
     Sidekiq.logger.info "Received TSTP, no longer accepting new work"
@@ -100,21 +103,26 @@ iter.times do
 end
 Sidekiq.logger.error "Created #{count*iter} jobs"
 
+start = Time.now
+
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 1
-      qsize, retries = Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.llen "queue:default"
-          conn.zcard "retry"
-        end
-      end.map(&:to_i)
-      total = qsize + retries
-      #GC.start
-      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      sleep 0.2
+      qsize = Sidekiq.redis do |conn|
+        conn.llen "queue:default"
+      end
+      total = qsize
+      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
-        Sidekiq.logger.error("Done")
+        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+        Sidekiq.logger.error("Now here's the latency for three jobs")
+
+        LoadWorker.perform_async(1, Time.now.to_f)
+        LoadWorker.perform_async(2, Time.now.to_f)
+        LoadWorker.perform_async(3, Time.now.to_f)
+
+        sleep 0.2
         exit(0)
       end
     end
@@ -125,8 +133,8 @@ begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
   fire_event(:startup)
-  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -134,7 +142,7 @@ begin
       signal = readable_io.first[0].gets.strip
       handle_signal(launcher, signal)
     end
-  #end
+  end
 rescue SystemExit => e
   #Sidekiq.logger.error("Profiling...")
   #result = RubyProf.stop