sidekiq 6.0.1 → 6.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 792692164ce00b070d352bda8869518e3d20d92cfdb70cda88456b53d5a4a627
-  data.tar.gz: b489c2ecc7b3708dd578a0184a9da58ac32d36e737a3b704d094e1ef375e93e1
+  metadata.gz: 9dd138f78183ff31972192acbcee3ea7aad67403dfdd478500f3bbbfebf14698
+  data.tar.gz: ea4ab3b7c40bf358a80df9043013a63ee7c0562322ce5f212f938cecdc93c62f
 SHA512:
-  metadata.gz: 80d7bd6a6fc9ef0a6026c9c5bb8aaf727db7323e1f1d9e057550681329991ddf82de030df70dc80e8f3a3608fc1f999ecb5d251765a5bcf09f23f764d86365a7
-  data.tar.gz: 3148011168f88fbc4fa67f244a56be0b0e7da36d5009f4d465e0df2a7df93ddf8a99fc1c57c43fa0365a0cdb1de9fb770306daf070bcea5f330c1f3522210bf9
+  metadata.gz: c34d01bffdf5af462b98afa03e4b71356b8afffd873533eb953305314e1c0fd0ea4a4c0eea53c7169b61aca51420a50d5eda9127ad815c46c4053954677be24d
+  data.tar.gz: fdfe2b1704bc7d3a071756ca983c9b81bf4534ceb5f532bec59ec87fe4c3094c95120a7b1471163327f55e132c9773a3fd05d9a6211c6c1a1342c58872e8d980

data/Changes.md

@@ -2,7 +2,13 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-HEAD
+6.0.2
+---------
+
+- Fix Sidekiq Enterprise's rolling restart functionality, broken by refactoring in 6.0.0. [#4334]
+- More internal refactoring and performance tuning [fatkodima]
+
+6.0.1
 ---------
 
 - **Performance tuning**, Sidekiq should be 10-15% faster now [#4303, 4299,

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sidekiq (6.0.1)
+    sidekiq (6.0.2)
       connection_pool (>= 2.2.2)
       rack (>= 2.0.0)
       rack-protection (>= 2.0.0)

data/README.md

@@ -19,7 +19,8 @@ Performance
 
 Version |	Latency | Garbage created for 10k jobs	| Time to process 100k jobs |	Throughput | Ruby
 -----------------|------|---------|---------|------------------------|-----
-Sidekiq 6.0.0    | 3 ms	| 156 MB  | 19 sec  | **5200 jobs/sec** | MRI 2.6.3
+Sidekiq 6.0.2    | 3 ms	| 156 MB  | 14.0 sec| **7100 jobs/sec** | MRI 2.6.3
+Sidekiq 6.0.0    | 3 ms	| 156 MB  | 19 sec  | 5200 jobs/sec | MRI 2.6.3
 Sidekiq 4.0.0    | 10 ms	| 151 MB  | 22 sec  | 4500 jobs/sec |
 Sidekiq 3.5.1    | 22 ms	| 1257 MB | 125 sec | 800 jobs/sec |
 Resque 1.25.2    |  -	  | -       | 420 sec | 240 jobs/sec |

data/lib/sidekiq/api.rb

@@ -80,8 +80,8 @@ module Sidekiq
       }
 
       s = processes.size
-      workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
-      enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
+      workers_size = pipe2_res[0...s].sum(&:to_i)
+      enqueued = pipe2_res[s..-1].sum(&:to_i)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
@@ -438,12 +438,18 @@ module Sidekiq
 
     def uncompress_backtrace(backtrace)
       if backtrace.is_a?(Array)
-        # Handle old jobs with previous backtrace format
+        # Handle old jobs with raw Array backtrace format
         backtrace
       else
         decoded = Base64.decode64(backtrace)
         uncompressed = Zlib::Inflate.inflate(decoded)
-        Marshal.load(uncompressed)
+        begin
+          Sidekiq.load_json(uncompressed)
+        rescue
+          # Handle old jobs with marshalled backtrace format
+          # TODO Remove in 7.x
+          Marshal.load(uncompressed)
+        end
       end
     end
   end
@@ -471,8 +477,9 @@ module Sidekiq
     end
 
     def reschedule(at)
-      delete
-      @parent.schedule(at, item)
+      Sidekiq.redis do |conn|
+        conn.zincrby(@parent.name, at - @score, Sidekiq.dump_json(@item))
+      end
     end
 
     def add_to_queue
@@ -554,7 +561,7 @@ module Sidekiq
     end
 
     def scan(match, count = 100)
-      return to_enum(:scan, match) unless block_given?
+      return to_enum(:scan, match, count) unless block_given?
 
       match = "*#{match}*" unless match.include?("*")
       Sidekiq.redis do |conn|
@@ -648,11 +655,13 @@ module Sidekiq
       Sidekiq.redis do |conn|
         elements = conn.zrangebyscore(name, score, score)
         elements.each do |element|
-          message = Sidekiq.load_json(element)
-          if message["jid"] == jid
-            ret = conn.zrem(name, element)
-            @_size -= 1 if ret
-            break ret
+          if element.index(jid)
+            message = Sidekiq.load_json(element)
+            if message["jid"] == jid
+              ret = conn.zrem(name, element)
+              @_size -= 1 if ret
+              break ret
+            end
           end
         end
       end
@@ -786,30 +795,28 @@ module Sidekiq
     end
 
     def each
-      procs = Sidekiq.redis { |conn| conn.sscan_each("processes").to_a }.sort
+      result = Sidekiq.redis { |conn|
+        procs = conn.sscan_each("processes").to_a.sort
 
-      Sidekiq.redis do |conn|
         # We're making a tradeoff here between consuming more memory instead of
         # making more roundtrips to Redis, but if you have hundreds or thousands of workers,
         # you'll be happier this way
-        result = conn.pipelined {
+        conn.pipelined do
           procs.each do |key|
             conn.hmget(key, "info", "busy", "beat", "quiet")
           end
-        }
+        end
+      }
 
-        result.each do |info, busy, at_s, quiet|
-          # If a process is stopped between when we query Redis for `procs` and
-          # when we query for `result`, we will have an item in `result` that is
-          # composed of `nil` values.
-          next if info.nil?
+      result.each do |info, busy, at_s, quiet|
+        # If a process is stopped between when we query Redis for `procs` and
+        # when we query for `result`, we will have an item in `result` that is
+        # composed of `nil` values.
+        next if info.nil?
 
-          hash = Sidekiq.load_json(info)
-          yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
-        end
+        hash = Sidekiq.load_json(info)
+        yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
       end
-
-      nil
     end
 
     # This method is not guaranteed accurate since it does not prune the set
@@ -953,7 +960,7 @@ module Sidekiq
             procs.each do |key|
               conn.hget(key, "busy")
             end
-          }.map(&:to_i).inject(:+)
+          }.sum(&:to_i)
         end
       end
     end

data/lib/sidekiq/cli.rb

@@ -41,6 +41,8 @@ module Sidekiq
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << "USR2" unless jruby?
       sigs.each do |sig|
         trap sig do
           self_write.puts(sig)

data/lib/sidekiq/job_retry.rb

@@ -74,7 +74,7 @@ module Sidekiq
     # The global retry handler requires only the barest of data.
     # We want to be able to retry as much as possible so we don't
     # require the worker to be instantiated.
-    def global(msg, queue)
+    def global(jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -85,6 +85,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"]
         attempt_retry(nil, msg, queue, e)
       else
@@ -106,7 +107,7 @@ module Sidekiq
     # exception so the global block does not reprocess the error.  The
     # Skip exception is unwrapped within Sidekiq::Processor#process before
     # calling the handle_exception handlers.
-    def local(worker, msg, queue)
+    def local(worker, jobstr, queue)
       yield
     rescue Handled => ex
       raise ex
@@ -117,6 +118,7 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
+      msg = Sidekiq.load_json(jobstr)
       if msg["retry"].nil?
         msg["retry"] = worker.class.get_sidekiq_options["retry"]
       end
@@ -252,7 +254,7 @@ module Sidekiq
     end
 
     def compress_backtrace(backtrace)
-      serialized = Marshal.dump(backtrace)
+      serialized = Sidekiq.dump_json(backtrace)
       compressed = Zlib::Deflate.deflate(serialized)
       Base64.encode64(compressed)
     end

data/lib/sidekiq/monitor.rb

@@ -4,21 +4,6 @@ require "fileutils"
 require "sidekiq/api"
 
 class Sidekiq::Monitor
-  CMD = File.basename($PROGRAM_NAME)
-
-  attr_reader :stage
-
-  def self.print_usage
-    puts "#{CMD} - monitor Sidekiq from the command line."
-    puts
-    puts "Usage: #{CMD} <section>"
-    puts
-    puts "       <section> (optional) view a specific section of the status output"
-    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
-    puts
-    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
-  end
-
   class Status
     VALID_SECTIONS = %w[all version overview processes queues]
     COL_PAD = 2

data/lib/sidekiq/processor.rb

@@ -111,16 +111,19 @@ module Sidekiq
       nil
     end
 
-    def dispatch(job_hash, queue)
+    def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
-      # we clone here so we report the original
+      # we need to clone it to report the original
       # job structure to the Web UI
-      pristine = json_clone(job_hash)
+      # or to push back to redis when retrying.
+      # To avoid costly and, most of the time, useless cloning here,
+      # we pass original String of JSON to respected methods
+      # to re-parse it there if we need access to the original, untouched job
 
       @job_logger.prepare(job_hash) do
-        @retrier.global(pristine, queue) do
+        @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
-            stats(pristine, queue) do
+            stats(jobstr, queue) do
               # Rails 5 requires a Reloader to wrap code execution.  In order to
               # constantize the worker and instantiate an instance, we have to call
               # the Reloader.  It handles code loading, db connection management, etc.
@@ -129,7 +132,7 @@ module Sidekiq
                 klass = constantize(job_hash["class"])
                 worker = klass.new
                 worker.jid = job_hash["jid"]
-                @retrier.local(worker, pristine, queue) do
+                @retrier.local(worker, jobstr, queue) do
                   yield worker
                 end
               end
@@ -156,7 +159,7 @@ module Sidekiq
 
       ack = false
       begin
-        dispatch(job_hash, queue) do |worker|
+        dispatch(job_hash, queue, jobstr) do |worker|
           Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
             execute_job(worker, job_hash["args"])
           end
@@ -247,8 +250,8 @@ module Sidekiq
     FAILURE = Counter.new
     WORKER_STATE = SharedWorkerState.new
 
-    def stats(job_hash, queue)
-      WORKER_STATE.set(tid, {queue: queue, payload: job_hash, run_at: Time.now.to_i})
+    def stats(jobstr, queue)
+      WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
 
       begin
         yield
@@ -273,30 +276,5 @@ module Sidekiq
         constant.const_get(name, false)
       end
     end
-
-    # Deep clone the arguments passed to the worker so that if
-    # the job fails, what is pushed back onto Redis hasn't
-    # been mutated by the worker.
-    def json_clone(obj)
-      if Integer === obj || Float === obj || TrueClass === obj || FalseClass === obj || NilClass === obj
-        return obj
-      elsif String === obj
-        return obj.dup
-      elsif Array === obj
-        duped = Array.new(obj.size)
-        obj.each_with_index do |value, index|
-          duped[index] = json_clone(value)
-        end
-      elsif Hash === obj
-        duped = obj.dup
-        duped.each_pair do |key, value|
-          duped[key] = json_clone(value)
-        end
-      else
-        duped = obj.dup
-      end
-
-      duped
-    end
   end
 end

data/lib/sidekiq/util.rb

@@ -11,8 +11,6 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60 * 24
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.1"
+  VERSION = "6.0.2"
 end

data/lib/sidekiq/web/application.rb

@@ -5,7 +5,6 @@ module Sidekiq
     extend WebRouter
 
     CONTENT_LENGTH = "Content-Length"
-    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -307,7 +306,7 @@ module Sidekiq
 
       resp[1] = resp[1].dup
 
-      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
+      resp[1][CONTENT_LENGTH] = resp[2].sum(&:bytesize).to_s
 
       resp
     end

data/lib/sidekiq/web/helpers.rb

@@ -156,12 +156,6 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def retries_with_score(score)
-      Sidekiq.redis { |conn|
-        conn.zrangebyscore("retry", score, score)
-      }.map { |msg| Sidekiq.load_json(msg) }
-    end
-
     def redis_connection
       Sidekiq.redis do |conn|
         c = conn.connection

data/web/locales/de.yml

@@ -13,7 +13,7 @@ de:
   Retries: Versuche
   Enqueued: In der Warteschlange
   Worker: Arbeiter
-  LivePoll: Live Poll
+  LivePoll: Echtzeitabfrage
   StopPolling: Abfrage stoppen
   Queue: Warteschlange
   Class: Klasse
@@ -33,12 +33,13 @@ de:
   NextRetry: Nächster Versuch
   RetryCount: Anzahl der Versuche
   RetryNow: Jetzt erneut versuchen
-  Kill: Töten
+  Kill: Vernichten
   LastRetry: Letzter Versuch
   OriginallyFailed: Ursprünglich fehlgeschlagen
   AreYouSure: Bist du sicher?
   DeleteAll: Alle löschen
   RetryAll: Alle erneut versuchen
+  KillAll: Alle vernichten
   NoRetriesFound: Keine erneuten Versuche gefunden
   Error: Fehler
   ErrorClass: Fehlerklasse
@@ -67,3 +68,14 @@ de:
   Thread: Thread
   Threads: Threads
   Jobs: Jobs
+  Paused: Pausiert
+  Stop: Stopp
+  Quiet: Leise
+  StopAll: Alle stoppen
+  QuietAll: Alle leise
+  PollingInterval: Abfrageintervall
+  Plugins: Erweiterungen
+  NotYetEnqueued: Noch nicht in der Warteschlange
+  CreatedAt: Erstellt
+  BackToApp: Zurück zur Anwendung
+  Latency: Latenz

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 6.0.1
+  version: 6.0.2
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-02 00:00:00.000000000 Z
+date: 2019-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -214,8 +214,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Simple, efficient background processing for Ruby