sidekiq 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b52bf9136f69ca7203686496bcdf680f62013981
-  data.tar.gz: 4ffaee1757a6f975e95789832d7f809f6224cd5e
+  metadata.gz: 92ef60dcba913bb8fdbb05e1c0a8dec0071146f3
+  data.tar.gz: 7c607121e01d8df3e83c09ab77876fde573cb1af
 SHA512:
-  metadata.gz: 08a8a7b31a1f59c197c18b98c670d9991ee660397bdc09b6a5f06f73b885d5fc09e57de4d1ca60e7a2956e06e541a9245627367525d4b52f36be57040b362d79
-  data.tar.gz: 75ad33dfa747d4d2a89bd98d2cabe601b99cc3a6085190361d134c3128d7faddfcbf182d5a278cf8cedb087ed5f0564bd7e839a1f9665a21777f31430c8264c9
+  metadata.gz: 980ecb9625038a8b0211d73ab6ac8176f6c7bf55c03da3c7e810016169ba76449ad43e602be337e1cc4d094aa2a9229ff565d37dd8bb6f094de62a34f61bc8e5
+  data.tar.gz: 420995af74088296a8ebf722f438be5b57f1a459acb0678956223d85341b8a4352a268da65b9ef4bcab3c275b5b66f7db3113fe75d6f52f6c1d2188a264d7867

data/COMM-LICENSE

@@ -74,7 +74,7 @@ In no event will Contributed Systems' liability exceed the Software license pric
 
 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.
 
-13.2 Amendment. Contributed Systems reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to thie Agreement can be located at: https://github.com/mperham/sidekiq/blob/master/COMM-LICENSE.
+13.2 Amendment. Contributed Systems reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/mperham/sidekiq/blob/master/COMM-LICENSE.
 
 13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Contributed Systems and any attempted assignment without such consent shall be void.
 

data/Changes.md

@@ -1,3 +1,11 @@
+3.5.1
+-----------
+
+- **FIX MEMORY LEAK** Under rare conditions, threads may leak [#2598]
+- Add Ukranian locale [#2561, elrakita]
+- Disconnect and retry Redis operations if we see a READONLY error [#2550]
+- Add server middleware testing harness; see [wiki](https://github.com/mperham/sidekiq/wiki/Testing#testing-server-middleware) [#2534, ryansch]
+
 3.5.0
 -----------
 
@@ -401,7 +409,7 @@ middleware, see 3.0-Upgrade.md.**
 [Testing](https://github.com/mperham/sidekiq/wiki/Testing) for detail. [#1193]
 - The Retries table has a new column for the error message.
 - The Web UI topbar now contains the status and live poll button.
-- Orphaned worker records are now auto-vacuumed when you vist the
+- Orphaned worker records are now auto-vacuumed when you visit the
   Workers page in the Web UI.
 - Sidekiq.default\_worker\_options allows you to configure default
   options for all Sidekiq worker types.

data/Ent-Changes.md

@@ -3,6 +3,20 @@ Sidekiq Enterprise Changelog
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+0.7.3
+----------
+
+- Rework `Sidekiq::Limiter` redis handling to match global redis handling.
+- Allow user to customize rate limit backoff logic and handle custom
+  rate limit errors.
+- Fix scalability issue with Limiter index page.
+
+
+0.7.2
+----------
+
+- Fix typo which prevented limiters with '0' in their names.
+
 0.7.1
 ----------
 

data/Gemfile

@@ -4,6 +4,7 @@ gemspec
 gem 'rails', '~> 4.2'
 gem 'simplecov'
 gem 'minitest'
+gem 'toxiproxy'
 
 platforms :rbx do
   gem 'rubysl', '~> 2.0'         # if using anything in the ruby standard library

data/Pro-Changes.md

@@ -1,7 +1,18 @@
 Sidekiq Pro Changelog
 =======================
 
-Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and how to buy.
+Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
+
+2.1.0
+-----------
+
+- Explicit support for sharding batches.  You list your Redis shards and
+  Sidekiq Pro will randomly spread batches across the shards.  The BID
+  will indicate which shard contains the batch data.  Jobs within a
+  batch may be spread across all shards too. [#2548, jonhyman]
+- Officially deprecate Sidekiq::Notifications code.  Notifications have
+  been undocumented for months now. [#2575]
+
 
 2.0.8
 -----------

data/bin/sidekiqload

@@ -0,0 +1,153 @@
+#!/usr/bin/env ruby
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+$CELLULOID_DEBUG = false
+
+require 'celluloid/current'
+puts Celluloid::VERSION
+require_relative '../lib/sidekiq/cli'
+require_relative '../lib/sidekiq/launcher'
+Celluloid.logger = nil
+
+include Sidekiq::Util
+
+Sidekiq.configure_server do |config|
+  config.redis = { db: 13, port: 6380 }
+  config.options[:queues] << 'default'
+  config.logger.level = Logger::ERROR
+  config.average_scheduled_poll_interval = 2
+end
+
+class LoadWorker
+  include Sidekiq::Worker
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx)
+    #raise idx.to_s if idx % 100 == 1
+  end
+end
+
+# brew tap shopify/shopify
+# brew install toxiproxy
+# gem install toxiproxy
+require 'toxiproxy'
+# simulate a non-localhost network for realer-world conditions.
+# adding 1ms of network latency has an ENORMOUS impact on benchmarks
+Toxiproxy.populate([{
+    "name": "redis",
+    "listen": "127.0.0.1:6380",
+    "upstream": "127.0.0.1:6379"
+}])
+
+self_read, self_write = IO.pipe
+%w(INT TERM USR1 USR2 TTIN).each do |sig|
+  begin
+    trap sig do
+      self_write.puts(sig)
+    end
+  rescue ArgumentError
+    puts "Signal #{sig} not supported"
+  end
+end
+
+Sidekiq.redis {|c| c.flushdb}
+def handle_signal(launcher, sig)
+  Sidekiq.logger.debug "Got #{sig} signal"
+  case sig
+  when 'INT'
+    # Handle Ctrl-C in JRuby like MRI
+    # http://jira.codehaus.org/browse/JRUBY-4637
+    raise Interrupt
+  when 'TERM'
+    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    raise Interrupt
+  when 'USR1'
+    Sidekiq.logger.info "Received USR1, no longer accepting new work"
+    launcher.manager.async.stop
+    #fire_event(:quiet, true)
+  when 'USR2'
+    if Sidekiq.options[:logfile]
+      Sidekiq.logger.info "Received USR2, reopening log file"
+      Sidekiq::Logging.reopen_logs
+    end
+  when 'TTIN'
+    Thread.list.each do |thread|
+      Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+      if thread.backtrace
+        Sidekiq.logger.warn thread.backtrace.join("\n")
+      else
+        Sidekiq.logger.warn "<no backtrace available>"
+      end
+    end
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+iter = 10
+count = 10_000
+
+iter.times do
+  arr = Array.new(count) do
+    []
+  end
+  count.times do |idx|
+    arr[idx][0] = idx
+  end
+  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
+end
+Sidekiq.logger.error "Created #{count*iter} jobs"
+
+Monitoring = Thread.new do
+  watchdog("monitor thread") do
+    while true
+      sleep 2
+      qsize, retries = Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.llen "queue:default"
+          conn.zcard "retry"
+        end
+      end.map(&:to_i)
+      total = qsize + retries
+      #GC.start
+      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      if total == 0
+        Sidekiq.logger.error("Done")
+        exit(0)
+      end
+    end
+  end
+end
+
+begin
+  #RubyProf::exclude_threads = [ Monitoring ]
+  #RubyProf.start
+  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher.run
+
+    while readable_io = IO.select([self_read])
+      signal = readable_io.first[0].gets.strip
+      handle_signal(launcher, signal)
+    end
+  end
+rescue SystemExit => e
+  #Sidekiq.logger.error("Profiling...")
+  #result = RubyProf.stop
+  #printer = RubyProf::GraphHtmlPrinter.new(result)
+  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # normal
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/sidekiq.rb

@@ -76,9 +76,19 @@ module Sidekiq
     defined?(Sidekiq::CLI)
   end
 
-  def self.redis(&block)
-    raise ArgumentError, "requires a block" unless block
-    redis_pool.with(&block)
+  def self.redis
+    raise ArgumentError, "requires a block" unless block_given?
+    redis_pool.with do |conn|
+      retryable = true
+      begin
+        yield conn
+      rescue Redis::CommandError => ex
+        #2550 Failover can cause the server to become a slave, need
+        # to disconnect and reopen the socket to get back to the master.
+        (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/
+        raise
+      end
+    end
   end
 
   def self.redis_pool

data/lib/sidekiq/cli.rb

@@ -105,16 +105,16 @@ module Sidekiq
 %q{
          m,
          `$b
-    .ss,  $$b         .,d$
-    `$$P,d$P'    .,md$$P'  ____  _     _      _    _
-     ,$$$$$$bmmd$$$^'     / ___|(_) __| | ___| | _(_) __ _
-   ,d$$$$$$$$$$$P         \___ \| |/ _` |/ _ \ |/ / |/ _` |
-   $s^' `"^$$$'            ___) | | (_| |  __/   <| | (_| |
-   $:     ,$$P            |____/|_|\__,_|\___|_|\_\_|\__, |
-   `b     :$$                                           |_|
-          $$:
-          $$
-        .d$$
+    .ss,  $$:         .,d$
+    `$$P,d$P'    .,md$P"'
+     ,$$$$$bmmd$$$P^'
+   .d$$$$$$$$$$P'
+   $$^' `"^$$$'       ____  _     _      _    _
+   $:     ,$$:       / ___|(_) __| | ___| | _(_) __ _
+   `b     :$$        \___ \| |/ _` |/ _ \ |/ / |/ _` |
+          $$:         ___) | | (_| |  __/   <| | (_| |
+          $$         |____/|_|\__,_|\___|_|\_\_|\__, |
+        .d$$                                       |_|
 }
     end
 

data/lib/sidekiq/manager.rb

@@ -132,7 +132,7 @@ module Sidekiq
     # get handle to the underlying thread performing work for a processor
     # so we have it call us and tell us.
     def real_thread(proxy_id, thr)
-      @threads[proxy_id] = thr
+      @threads[proxy_id] = thr if thr.alive?
     end
 
     PROCTITLES = [

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -181,7 +181,7 @@ module Sidekiq
 
         def retry_in(worker, count)
           begin
-            worker.sidekiq_retry_in_block.call(count)
+            worker.sidekiq_retry_in_block.call(count).to_i
           rescue Exception => e
             handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
             nil

data/lib/sidekiq/processor.rb

@@ -40,18 +40,23 @@ module Sidekiq
 
       @boss.async.real_thread(proxy_id, Thread.current)
 
-      ack = true
+      ack = false
       begin
         msg = Sidekiq.load_json(msgstr)
-        klass  = msg['class'].constantize
+        klass  = msg['class'.freeze].constantize
         worker = klass.new
-        worker.jid = msg['jid']
+        worker.jid = msg['jid'.freeze]
 
         stats(worker, msg, queue) do
           Sidekiq.server_middleware.invoke(worker, msg, queue) do
-            execute_job(worker, cloned(msg['args']))
+            # Only ack if we either attempted to start this job or
+            # successfully completed it. This prevents us from
+            # losing jobs if a middleware raises an exception before yielding
+            ack = true
+            execute_job(worker, cloned(msg['args'.freeze]))
           end
         end
+        ack = true
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
@@ -94,14 +99,15 @@ module Sidekiq
         end
       end
 
+      nowdate = Time.now.utc.strftime("%Y-%m-%d".freeze)
       begin
         yield
       rescue Exception
         retry_and_suppress_exceptions do
-          failed = "stat:failed:#{Time.now.utc.to_date}"
+          failed = "stat:failed:#{nowdate}"
           Sidekiq.redis do |conn|
             conn.multi do
-              conn.incrby("stat:failed", 1)
+              conn.incrby("stat:failed".freeze, 1)
               conn.incrby(failed, 1)
               conn.expire(failed, STATS_TIMEOUT)
             end
@@ -110,11 +116,11 @@ module Sidekiq
         raise
       ensure
         retry_and_suppress_exceptions do
-          processed = "stat:processed:#{Time.now.utc.to_date}"
+          processed = "stat:processed:#{nowdate}"
           Sidekiq.redis do |conn|
             conn.multi do
               conn.hdel("#{identity}:workers", thread_identity)
-              conn.incrby("stat:processed", 1)
+              conn.incrby("stat:processed".freeze, 1)
               conn.incrby(processed, 1)
               conn.expire(processed, STATS_TIMEOUT)
             end

data/lib/sidekiq/testing.rb

@@ -48,6 +48,12 @@ module Sidekiq
       def inline?
         self.__test_mode == :inline
       end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new
+        yield @server_chain if block_given?
+        @server_chain
+      end
     end
   end
 
@@ -150,10 +156,7 @@ module Sidekiq
       # Drain and run all jobs for this worker
       def drain
         while job = jobs.shift do
-          worker = new
-          worker.jid = job['jid']
-          worker.bid = job['bid'] if worker.respond_to?(:bid=)
-          execute_job(worker, job['args'])
+          process_job(job)
         end
       end
 
@@ -161,10 +164,16 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         job = jobs.shift
+        process_job(job)
+      end
+
+      def process_job(job)
         worker = new
         worker.jid = job['jid']
         worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        execute_job(worker, job['args'])
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
+          execute_job(worker, job['args'])
+        end
       end
 
       def execute_job(worker, args)

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.5.0"
+  VERSION = "3.5.1"
 end

data/sidekiq.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'redis', '~> 3.2', '>= 3.2.1'
   gem.add_dependency                  'redis-namespace', '~> 1.5', '>= 1.5.2'
   gem.add_dependency                  'connection_pool', '~> 2.2', '>= 2.2.0'
-  gem.add_dependency                  'celluloid', '~> 0.17.0'
+  gem.add_dependency                  'celluloid', '~> 0.17.2'
   gem.add_dependency                  'json', '~> 1.0'
   gem.add_development_dependency      'sinatra', '~> 1.4', '>= 1.4.6'
   gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'

data/test/test_api.rb

@@ -3,23 +3,16 @@ require_relative 'helper'
 class TestApi < Sidekiq::Test
 
   describe "stats" do
-    before do
-      @before = DateTime::DATE_FORMATS[:default]
-      DateTime::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
-      Sidekiq.redis = REDIS
-      Sidekiq.redis {|c| c.flushdb }
-    end
 
-    after do
-      DateTime::DATE_FORMATS[:default] = @before
+    it "is initially zero" do
+      Sidekiq.redis {|c| c.flushdb }
+      s = Sidekiq::Stats.new
+      assert_equal 0, s.processed
+      assert_equal 0, s.failed
+      assert_equal 0, s.enqueued
     end
 
     describe "processed" do
-      it "is initially zero" do
-        s = Sidekiq::Stats.new
-        assert_equal 0, s.processed
-      end
-
       it "returns number of processed jobs" do
         Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
         s = Sidekiq::Stats.new
@@ -28,11 +21,6 @@ class TestApi < Sidekiq::Test
     end
 
     describe "failed" do
-      it "is initially zero" do
-        s = Sidekiq::Stats.new
-        assert_equal 0, s.failed
-      end
-
       it "returns number of failed jobs" do
         Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
         s = Sidekiq::Stats.new
@@ -50,40 +38,36 @@ class TestApi < Sidekiq::Test
 
       it 'will reset all stats by default' do
         Sidekiq::Stats.new.reset
-        Sidekiq.redis do |conn|
-          assert_equal '0', conn.get('stat:processed')
-          assert_equal '0', conn.get('stat:failed')
-        end
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.failed
+        assert_equal 0, s.processed
       end
 
       it 'can reset individual stats' do
         Sidekiq::Stats.new.reset('failed')
-        Sidekiq.redis do |conn|
-          assert_equal '5', conn.get('stat:processed')
-          assert_equal '0', conn.get('stat:failed')
-        end
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.failed
+        assert_equal 5, s.processed
       end
 
       it 'can accept anything that responds to #to_s' do
         Sidekiq::Stats.new.reset(:failed)
-        Sidekiq.redis do |conn|
-          assert_equal '5', conn.get('stat:processed')
-          assert_equal '0', conn.get('stat:failed')
-        end
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.failed
+        assert_equal 5, s.processed
       end
 
       it 'ignores anything other than "failed" or "processed"' do
         Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
-        Sidekiq.redis do |conn|
-          assert_equal '5', conn.get('stat:processed')
-          assert_equal '0', conn.get('stat:failed')
-        end
+        s = Sidekiq::Stats.new
+        assert_equal 0, s.failed
+        assert_equal 5, s.processed
       end
     end
 
     describe "queues" do
-      it "returns all queues" do
-        assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
+      before do
+        Sidekiq.redis {|c| c.flushdb }
       end
 
       it "is initially empty" do
@@ -103,17 +87,15 @@ class TestApi < Sidekiq::Test
         s = Sidekiq::Stats::Queues.new
         assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
         assert_equal "bar", s.lengths.first.first
+
+        assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
       end
     end
 
     describe "enqueued" do
-      it "is initially empty" do
-        s = Sidekiq::Stats.new
-        assert_equal 0, s.enqueued
-      end
-
       it "returns total enqueued jobs" do
         Sidekiq.redis do |conn|
+          conn.flushdb
           conn.rpush 'queue:foo', '{}'
           conn.sadd 'queues', 'foo'
 
@@ -127,6 +109,15 @@ class TestApi < Sidekiq::Test
     end
 
     describe "over time" do
+      before do
+        @before = DateTime::DATE_FORMATS[:default]
+        DateTime::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
+      end
+
+      after do
+        DateTime::DATE_FORMATS[:default] = @before
+      end
+
       describe "processed" do
         it 'retrieves hash of dates' do
           Sidekiq.redis do |c|
@@ -137,13 +128,13 @@ class TestApi < Sidekiq::Test
           end
           Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
             s = Sidekiq::Stats::History.new(2)
-            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.processed
+            assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed)
 
             s = Sidekiq::Stats::History.new(3)
-            assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
+            assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
 
             s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
-            assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.processed
+            assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
           end
         end
       end
@@ -172,10 +163,7 @@ class TestApi < Sidekiq::Test
   end
 
   describe 'with an empty database' do
-    include Sidekiq::Util
-
     before do
-      Sidekiq.redis = REDIS
       Sidekiq.redis {|c| c.flushdb }
     end
 
@@ -389,7 +377,7 @@ class TestApi < Sidekiq::Test
       identity_string = "identity_string"
       odata = {
         'pid' => 123,
-        'hostname' => hostname,
+        'hostname' => Socket.gethostname,
         'key' => identity_string,
         'identity' => identity_string,
         'started_at' => Time.now.to_f - 15,
@@ -424,8 +412,9 @@ class TestApi < Sidekiq::Test
         assert false
       end
 
-      key = "#{hostname}:#{$$}"
-      pdata = { 'pid' => $$, 'hostname' => hostname, 'started_at' => Time.now.to_i }
+      hn = Socket.gethostname
+      key = "#{hn}:#{$$}"
+      pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i }
       Sidekiq.redis do |conn|
         conn.sadd('processes', key)
         conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)