sidekiq 6.0.0 → 6.0.5

data/lib/sidekiq/middleware/chain.rb

@@ -67,7 +67,6 @@ module Sidekiq
   module Middleware
     class Chain
       include Enumerable
-      attr_reader :entries
 
       def initialize_copy(copy)
         copy.instance_variable_set(:@entries, entries.dup)
@@ -78,10 +77,14 @@ module Sidekiq
       end
 
       def initialize
-        @entries = []
+        @entries = nil
         yield self if block_given?
       end
 
+      def entries
+        @entries ||= []
+      end
+
       def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
@@ -114,6 +117,10 @@ module Sidekiq
         any? { |entry| entry.klass == klass }
       end
 
+      def empty?
+        @entries.nil? || @entries.empty?
+      end
+
       def retrieve
         map(&:make_new)
       end
@@ -123,6 +130,8 @@ module Sidekiq
       end
 
       def invoke(*args)
+        return yield if empty?
+
         chain = retrieve.dup
         traverse_chain = lambda do
           if chain.empty?

data/lib/sidekiq/monitor.rb

@@ -4,21 +4,6 @@ require "fileutils"
 require "sidekiq/api"
 
 class Sidekiq::Monitor
-  CMD = File.basename($PROGRAM_NAME)
-
-  attr_reader :stage
-
-  def self.print_usage
-    puts "#{CMD} - monitor Sidekiq from the command line."
-    puts
-    puts "Usage: #{CMD} status <section>"
-    puts
-    puts "       <section> (optional) view a specific section of the status output"
-    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
-    puts
-    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
-  end
-
   class Status
     VALID_SECTIONS = %w[all version overview processes queues]
     COL_PAD = 2
@@ -47,7 +32,7 @@ class Sidekiq::Monitor
 
     def version
       puts "Sidekiq #{Sidekiq::VERSION}"
-      puts Time.now
+      puts Time.now.utc
     end
 
     def overview

data/lib/sidekiq/paginator.rb

@@ -12,10 +12,10 @@ module Sidekiq
 
       Sidekiq.redis do |conn|
         type = conn.type(key)
+        rev = opts && opts[:reverse]
 
         case type
         when "zset"
-          rev = opts && opts[:reverse]
           total_size, items = conn.multi {
             conn.zcard(key)
             if rev
@@ -28,8 +28,13 @@ module Sidekiq
         when "list"
           total_size, items = conn.multi {
             conn.llen(key)
-            conn.lrange(key, starting, ending)
+            if rev
+              conn.lrange(key, -ending - 1, -starting - 1)
+            else
+              conn.lrange(key, starting, ending)
+            end
           }
+          items.reverse! if rev
           [current_page, total_size, items]
         when "none"
           [1, 0, []]

data/lib/sidekiq/processor.rb

@@ -111,16 +111,19 @@ module Sidekiq
       nil
     end
 
-    def dispatch(job_hash, queue)
+    def dispatch(job_hash, queue, jobstr)
       # since middleware can mutate the job hash
-      # we clone here so we report the original
+      # we need to clone it to report the original
       # job structure to the Web UI
-      pristine = cloned(job_hash)
+      # or to push back to redis when retrying.
+      # To avoid costly and, most of the time, useless cloning here,
+      # we pass original String of JSON to respected methods
+      # to re-parse it there if we need access to the original, untouched job
 
-      @job_logger.with_job_hash_context(job_hash) do
-        @retrier.global(pristine, queue) do
+      @job_logger.prepare(job_hash) do
+        @retrier.global(jobstr, queue) do
           @job_logger.call(job_hash, queue) do
-            stats(pristine, queue) do
+            stats(jobstr, queue) do
               # Rails 5 requires a Reloader to wrap code execution.  In order to
               # constantize the worker and instantiate an instance, we have to call
               # the Reloader.  It handles code loading, db connection management, etc.
@@ -129,7 +132,7 @@ module Sidekiq
                 klass = constantize(job_hash["class"])
                 worker = klass.new
                 worker.jid = job_hash["jid"]
-                @retrier.local(worker, pristine, queue) do
+                @retrier.local(worker, jobstr, queue) do
                   yield worker
                 end
               end
@@ -156,9 +159,9 @@ module Sidekiq
 
       ack = false
       begin
-        dispatch(job_hash, queue) do |worker|
+        dispatch(job_hash, queue, jobstr) do |worker|
           Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
-            execute_job(worker, cloned(job_hash["args"]))
+            execute_job(worker, job_hash["args"])
           end
         end
         ack = true
@@ -178,7 +181,7 @@ module Sidekiq
         # the retry subsystem (e.g. network partition).  We won't acknowledge the job
         # so it can be rescued when using Sidekiq Pro.
         handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
-        raise e
+        raise ex
       ensure
         if ack
           # We don't want a shutdown signal to interrupt job acknowledgment.
@@ -247,8 +250,8 @@ module Sidekiq
     FAILURE = Counter.new
     WORKER_STATE = SharedWorkerState.new
 
-    def stats(job_hash, queue)
-      WORKER_STATE.set(tid, {queue: queue, payload: job_hash, run_at: Time.now.to_i})
+    def stats(jobstr, queue)
+      WORKER_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
 
       begin
         yield
@@ -261,21 +264,16 @@ module Sidekiq
       end
     end
 
-    # Deep clone the arguments passed to the worker so that if
-    # the job fails, what is pushed back onto Redis hasn't
-    # been mutated by the worker.
-    def cloned(thing)
-      Marshal.load(Marshal.dump(thing))
-    end
-
     def constantize(str)
+      return Object.const_get(str) unless str.include?("::")
+
       names = str.split("::")
       names.shift if names.empty? || names.first.empty?
 
       names.inject(Object) do |constant, name|
         # the false flag limits search for name to under the constant namespace
         #   which mimics Rails' behaviour
-        constant.const_defined?(name, false) ? constant.const_get(name, false) : constant.const_missing(name)
+        constant.const_get(name, false)
       end
     end
   end

data/lib/sidekiq/redis_connection.rb

@@ -103,6 +103,9 @@ module Sidekiq
         if scrubbed_options[:password]
           scrubbed_options[:password] = redacted
         end
+        scrubbed_options[:sentinels]&.each do |sentinel|
+          sentinel[:password] = redacted if sentinel[:password]
+        end
         if Sidekiq.server?
           Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
         else

data/lib/sidekiq/scheduled.rb

@@ -14,18 +14,19 @@ module Sidekiq
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get the next item in the queue if it's score (time to execute) is <= now.
-            # We need to go through the list one at a time to reduce the risk of something
-            # going wrong between the time jobs are popped from the scheduled queue and when
-            # they are pushed onto a work queue and losing the jobs.
-            while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
-
-              # Pop item off the queue and add it to the work queue. If the job can't be popped from
-              # the queue, it's because another process already popped it so we can move on to the
-              # next one.
-              if conn.zrem(sorted_set, job)
-                Sidekiq::Client.push(Sidekiq.load_json(job))
-                Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+            # Get next items in the queue with scores (time to execute) <= now.
+            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
+              # We need to go through the list one at a time to reduce the risk of something
+              # going wrong between the time jobs are popped from the scheduled queue and when
+              # they are pushed onto a work queue and losing the jobs.
+              jobs.each do |job|
+                # Pop item off the queue and add it to the work queue. If the job can't be popped from
+                # the queue, it's because another process already popped it so we can move on to the
+                # next one.
+                if conn.zrem(sorted_set, job)
+                  Sidekiq::Client.push(Sidekiq.load_json(job))
+                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+                end
               end
             end
           end

data/lib/sidekiq/testing.rb

@@ -323,6 +323,18 @@ module Sidekiq
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+      end
+    end
+  end
+
+  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
+  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?

data/lib/sidekiq/util.rb

@@ -11,8 +11,6 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60 * 24
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.0"
+  VERSION = "6.0.5"
 end

data/lib/sidekiq/web/application.rb

@@ -5,7 +5,6 @@ module Sidekiq
     extend WebRouter
 
     CONTENT_LENGTH = "Content-Length"
-    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -84,14 +83,22 @@ module Sidekiq
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count)
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
       @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
 
       erb(:queue)
     end
 
     post "/queues/:name" do
-      Sidekiq::Queue.new(route_params[:name]).clear
+      queue = Sidekiq::Queue.new(route_params[:name])
+
+      if Sidekiq.pro? && params["pause"]
+        queue.pause!
+      elsif Sidekiq.pro? && params["unpause"]
+        queue.unpause!
+      else
+        queue.clear
+      end
 
       redirect "#{root_path}queues"
     end
@@ -283,36 +290,30 @@ module Sidekiq
       action = self.class.match(env)
       return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
 
-      resp = catch(:halt) {
-        app = @klass
+      app = @klass
+      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
         self.class.run_befores(app, action)
-        begin
-          resp = action.instance_exec env, &action.block
-        ensure
-          self.class.run_afters(app, action)
-        end
-
-        resp
-      }
+        action.instance_exec env, &action.block
+      ensure
+        self.class.run_afters(app, action)
+      end
 
       resp = case resp
       when Array
+        # redirects go here
         resp
       else
+        # rendered content goes here
         headers = {
           "Content-Type" => "text/html",
           "Cache-Control" => "no-cache",
           "Content-Language" => action.locale,
           "Content-Security-Policy" => CSP_HEADER,
         }
-
+        # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
 
-      resp[1] = resp[1].dup
-
-      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
-
       resp
     end
 

data/lib/sidekiq/web/helpers.rb

@@ -65,7 +65,10 @@ module Sidekiq
 
     def poll_path
       if current_path != "" && params["poll"]
-        root_path + current_path
+        path = root_path + current_path
+        query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
+        path += "?#{query_string}" unless query_string.empty?
+        path
       else
         ""
       end
@@ -112,6 +115,13 @@ module Sidekiq
       end
     end
 
+    # within is used by Sidekiq Pro
+    def display_tags(job, within = nil)
+      job.tags.map { |tag|
+        "<span class='jobtag label label-info'>#{::Rack::Utils.escape_html(tag)}</span>"
+      }.join(" ")
+    end
+
     # mperham/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
@@ -130,6 +140,10 @@ module Sidekiq
       end
     end
 
+    def sort_direction_label
+      params[:direction] == "asc" ? "&uarr;" : "&darr;"
+    end
+
     def workers
       @workers ||= Sidekiq::Workers.new
     end
@@ -142,12 +156,6 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def retries_with_score(score)
-      Sidekiq.redis { |conn|
-        conn.zrangebyscore("retry", score, score)
-      }.map { |msg| Sidekiq.load_json(msg) }
-    end
-
     def redis_connection
       Sidekiq.redis do |conn|
         c = conn.connection
@@ -189,7 +197,7 @@ module Sidekiq
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page poll]
+    SAFE_QPARAMS = %w[page poll direction]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
@@ -198,7 +206,11 @@ module Sidekiq
         options[key.to_s] = options.delete(key)
       end
 
-      params.merge(options).map { |key, value|
+      to_query_string(params.merge(options))
+    end
+
+    def to_query_string(params)
+      params.map { |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       }.compact.join("&")
     end
@@ -238,7 +250,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
-      created_at
+      created_at tags
     ])
 
     def retry_extra_items(retry_job)
@@ -283,7 +295,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/worker.rb

@@ -171,9 +171,9 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        payload = @opts.merge("class" => @klass, "args" => args, "at" => ts)
+        payload = @opts.merge("class" => @klass, "args" => args)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload.delete("at") if ts <= now
+        payload["at"] = ts if ts > now
         @klass.client_push(payload)
       end
       alias_method :perform_at, :perform_in
@@ -207,10 +207,10 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = {"class" => self, "args" => args, "at" => ts}
+        item = {"class" => self, "args" => args}
 
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        item.delete("at") if ts <= now
+        item["at"] = ts if ts > now
 
         client_push(item)
       end