sidekiq 7.0.9 → 7.1.6

data/lib/sidekiq/middleware/current_attributes.rb

@@ -7,26 +7,32 @@ module Sidekiq
   # This can be useful for multi-tenancy, i18n locale, timezone, any implicit
   # per-request attribute. See +ActiveSupport::CurrentAttributes+.
   #
+  # For multiple current attributes, pass an array of current attributes.
+  #
   # @example
   #
   #   # in your initializer
   #   require "sidekiq/middleware/current_attributes"
   #   Sidekiq::CurrentAttributes.persist("Myapp::Current")
+  #   # or multiple current attributes
+  #   Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
   #
   module CurrentAttributes
     class Save
       include Sidekiq::ClientMiddleware
 
-      def initialize(cattr)
-        @strklass = cattr
+      def initialize(cattrs)
+        @cattrs = cattrs
       end
 
       def call(_, job, _, _)
-        if !job.has_key?("cattr")
-          attrs = @strklass.constantize.attributes
-          # Retries can push the job N times, we don't
-          # want retries to reset cattr. #5692, #5090
-          job["cattr"] = attrs if attrs.any?
+        @cattrs.each do |(key, strklass)|
+          if !job.has_key?(key)
+            attrs = strklass.constantize.attributes
+            # Retries can push the job N times, we don't
+            # want retries to reset cattr. #5692, #5090
+            job[key] = attrs if attrs.any?
+          end
         end
         yield
       end
@@ -35,22 +41,55 @@ module Sidekiq
     class Load
       include Sidekiq::ServerMiddleware
 
-      def initialize(cattr)
-        @strklass = cattr
+      def initialize(cattrs)
+        @cattrs = cattrs
       end
 
       def call(_, job, _, &block)
-        if job.has_key?("cattr")
-          @strklass.constantize.set(job["cattr"], &block)
-        else
-          yield
+        cattrs_to_reset = []
+
+        @cattrs.each do |(key, strklass)|
+          if job.has_key?(key)
+            constklass = strklass.constantize
+            cattrs_to_reset << constklass
+
+            job[key].each do |(attribute, value)|
+              constklass.public_send("#{attribute}=", value)
+            end
+          end
         end
+
+        yield
+      ensure
+        cattrs_to_reset.each(&:reset)
       end
     end
 
-    def self.persist(klass, config = Sidekiq.default_configuration)
-      config.client_middleware.add Save, klass.to_s
-      config.server_middleware.add Load, klass.to_s
+    class << self
+      def persist(klass_or_array, config = Sidekiq.default_configuration)
+        cattrs = build_cattrs_hash(klass_or_array)
+
+        config.client_middleware.add Save, cattrs
+        config.server_middleware.add Load, cattrs
+      end
+
+      private
+
+      def build_cattrs_hash(klass_or_array)
+        if klass_or_array.is_a?(Array)
+          {}.tap do |hash|
+            klass_or_array.each_with_index do |klass, index|
+              hash[key_at(index)] = klass.to_s
+            end
+          end
+        else
+          {key_at(0) => klass_or_array.to_s}
+        end
+      end
+
+      def key_at(index)
+        (index == 0) ? "cattr" : "cattr_#{index}"
+      end
     end
   end
 end

data/lib/sidekiq/paginator.rb

@@ -19,7 +19,7 @@ module Sidekiq
           total_size, items = conn.multi { |transaction|
             transaction.zcard(key)
             if rev
-              transaction.zrevrange(key, starting, ending, withscores: true)
+              transaction.zrange(key, starting, ending, "REV", withscores: true)
             else
               transaction.zrange(key, starting, ending, withscores: true)
             end

data/lib/sidekiq/processor.rb

@@ -148,6 +148,8 @@ module Sidekiq
 
     IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
     private_constant :IGNORE_SHUTDOWN_INTERRUPTS
+    ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
+    private_constant :ALLOW_SHUTDOWN_INTERRUPTS
 
     def process(uow)
       jobstr = uow.job
@@ -171,36 +173,35 @@ module Sidekiq
       end
 
       ack = false
-      begin
-        dispatch(job_hash, queue, jobstr) do |inst|
-          config.server_middleware.invoke(inst, job_hash, queue) do
-            execute_job(inst, job_hash["args"])
+      Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
+        Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
+          dispatch(job_hash, queue, jobstr) do |inst|
+            config.server_middleware.invoke(inst, job_hash, queue) do
+              execute_job(inst, job_hash["args"])
+            end
           end
+          ack = true
+        rescue Sidekiq::Shutdown
+          # Had to force kill this job because it didn't finish
+          # within the timeout.  Don't acknowledge the work since
+          # we didn't properly finish it.
+        rescue Sidekiq::JobRetry::Handled => h
+          # this is the common case: job raised error and Sidekiq::JobRetry::Handled
+          # signals that we created a retry successfully.  We can acknowlege the job.
+          ack = true
+          e = h.cause || h
+          handle_exception(e, {context: "Job raised exception", job: job_hash})
+          raise e
+        rescue Exception => ex
+          # Unexpected error!  This is very bad and indicates an exception that got past
+          # the retry subsystem (e.g. network partition).  We won't acknowledge the job
+          # so it can be rescued when using Sidekiq Pro.
+          handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
+          raise ex
         end
-        ack = true
-      rescue Sidekiq::Shutdown
-        # Had to force kill this job because it didn't finish
-        # within the timeout.  Don't acknowledge the work since
-        # we didn't properly finish it.
-      rescue Sidekiq::JobRetry::Handled => h
-        # this is the common case: job raised error and Sidekiq::JobRetry::Handled
-        # signals that we created a retry successfully.  We can acknowlege the job.
-        ack = true
-        e = h.cause || h
-        handle_exception(e, {context: "Job raised exception", job: job_hash})
-        raise e
-      rescue Exception => ex
-        # Unexpected error!  This is very bad and indicates an exception that got past
-        # the retry subsystem (e.g. network partition).  We won't acknowledge the job
-        # so it can be rescued when using Sidekiq Pro.
-        handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
-        raise ex
       ensure
         if ack
-          # We don't want a shutdown signal to interrupt job acknowledgment.
-          Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
-            uow.acknowledge
-          end
+          uow.acknowledge
         end
       end
     end

data/lib/sidekiq/rails.rb

@@ -39,14 +39,9 @@ module Sidekiq
       end
     end
 
-    initializer "sidekiq.rails_logger" do
+    initializer "sidekiq.backtrace_cleaner" do
       Sidekiq.configure_server do |config|
-        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
-        # it will appear in the Sidekiq console with all of the job context. See #5021 and
-        # https://github.com/rails/rails/blob/b5f2b550f69a99336482739000c58e4e04e033aa/railties/lib/rails/commands/server/server_command.rb#L82-L84
-        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
-        end
+        config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
       end
     end
 
@@ -57,6 +52,16 @@ module Sidekiq
     config.after_initialize do
       Sidekiq.configure_server do |config|
         config[:reloader] = Sidekiq::Rails::Reloader.new
+
+        # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
+        # it will appear in the Sidekiq console with all of the job context.
+        unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
+          if ::Rails::VERSION::STRING < "7.1"
+            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
+          else
+            ::Rails.logger.broadcast_to(config.logger)
+          end
+        end
       end
     end
   end

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "set"
 require "redis_client"
 require "redis_client/decorator"
 
@@ -8,13 +9,14 @@ module Sidekiq
     BaseError = RedisClient::Error
     CommandError = RedisClient::CommandError
 
+    # You can add/remove items or clear the whole thing if you don't want deprecation warnings.
+    DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
+
     module CompatMethods
-      # TODO Deprecate and remove this
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
       end
 
-      # TODO Deprecate and remove this
       def evalsha(sha, keys, argv)
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
@@ -24,6 +26,7 @@ module Sidekiq
       # this allows us to use methods like `conn.hmset(...)` instead of having to use
       # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
+        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
         @client.call(*args, *block)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
@@ -39,28 +42,6 @@ module Sidekiq
       def config
         @client.config
       end
-
-      def message
-        yield nil, @queue.pop
-      end
-
-      # NB: this method does not return
-      def subscribe(chan)
-        @queue = ::Queue.new
-
-        pubsub = @client.pubsub
-        pubsub.call("subscribe", chan)
-
-        loop do
-          evt = pubsub.next_event
-          next if evt.nil?
-          next unless evt[0] == "message" && evt[1] == chan
-
-          (_, _, msg) = evt
-          @queue << msg
-          yield self
-        end
-      end
     end
 
     def initialize(options)
@@ -82,8 +63,7 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace."
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
       end
 
       opts.delete(:size)

data/lib/sidekiq/redis_connection.rb

@@ -14,6 +14,7 @@ module Sidekiq
         logger = symbolized_options.delete(:logger)
         logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
+        raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
         pool_name = symbolized_options.delete(:pool_name)

data/lib/sidekiq/scheduled.rb

@@ -12,7 +12,7 @@ module Sidekiq
 
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
-        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
         if jobs[1] then
           redis.call("zrem", key, jobs[1])
           return jobs[1]

data/lib/sidekiq/testing.rb

@@ -6,22 +6,35 @@ require "sidekiq"
 module Sidekiq
   class Testing
     class << self
-      attr_accessor :__test_mode
+      attr_accessor :__global_test_mode
 
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
-          current_mode = __test_mode
           begin
-            self.__test_mode = mode
+            self.__local_test_mode = mode
             yield
           ensure
-            self.__test_mode = current_mode
+            self.__local_test_mode = nil
           end
         else
-          self.__test_mode = mode
+          self.__global_test_mode = mode
         end
       end
 
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
       def disable!(&block)
         __set_test_mode(:disable, &block)
       end
@@ -64,7 +77,7 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def raw_push(payloads)
+    def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))

data/lib/sidekiq/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.0.9"
+  VERSION = "7.1.6"
   MAJOR = 7
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -328,9 +328,59 @@ module Sidekiq
       json Sidekiq::Stats.new.queues
     end
 
+    ########
+    # Filtering
+    get "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    post "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    get "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    post "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    get "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
+    end
+
+    post "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
+    end
+
     def call(env)
       action = self.class.match(env)
-      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -347,10 +397,10 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "content-type" => "text/html",
-          "cache-control" => "private, no-store",
-          "content-language" => action.locale,
-          "content-security-policy" => CSP_HEADER
+          Rack::CONTENT_TYPE => "text/html",
+          Rack::CACHE_CONTROL => "private, no-store",
+          Web::CONTENT_LANGUAGE => action.locale,
+          Web::CONTENT_SECURITY_POLICY => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]

data/lib/sidekiq/web/csrf_protection.rb

@@ -62,7 +62,7 @@ module Sidekiq
 
       def deny(env)
         logger(env).warn "attack prevented by #{self.class}"
-        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+        [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
       end
 
       def session(env)

data/lib/sidekiq/web/helpers.rb

@@ -49,8 +49,29 @@ module Sidekiq
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which)
+      erb(:filtering, locals: {which: which})
+    end
+
+    def filter_link(jid, within = "retries")
+      if within.nil?
+        ::Rack::Utils.escape_html(jid)
+      else
+        "<a href='#{root_path}filter/#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -111,13 +132,6 @@ module Sidekiq
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
     # sidekiq/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")

data/lib/sidekiq/web.rb

@@ -34,6 +34,18 @@ module Sidekiq
       "Metrics" => "metrics"
     }
 
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
+    end
+
     class << self
       def settings
         self
@@ -137,7 +149,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],

data/sidekiq.gemspec

@@ -23,17 +23,8 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.11.0"
+  gem.add_dependency "redis-client", ">= 0.14.0"
   gem.add_dependency "connection_pool", ">= 2.3.0"
   gem.add_dependency "rack", ">= 2.2.4"
   gem.add_dependency "concurrent-ruby", "< 2"
-  gem.post_install_message = <<~EOM
-    
-    Welcome to Sidekiq 7.0!
-    
-    1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
-    2. Read the release notes at https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md
-    3. If you have problems, search for open/closed issues at https://github.com/sidekiq/sidekiq/issues/
-
-  EOM
 end

data/web/assets/javascripts/application.js

@@ -140,6 +140,7 @@ function checkResponse(resp) {
 
 function scheduleLivePoll() {
   let ti = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+  if (ti < 2000) { ti = 2000 }
   livePollTimer = setTimeout(livePollCallback, ti);
 }
 

data/web/assets/javascripts/dashboard-charts.js

@@ -57,7 +57,9 @@ class DashboardChart extends BaseChart {
 class RealtimeChart extends DashboardChart {
   constructor(el, options) {
     super(el, options);
-    this.delay = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    let d = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    if (d < 2000) { d = 2000; }
+    this.delay = d
     this.startPolling();
     document.addEventListener("interval:update", this.handleUpdate.bind(this));
   }

data/web/locales/en.yml

@@ -97,3 +97,5 @@ en:
   Context: Context
   Bucket: Bucket
   NoJobMetricsFound: No recent job metrics were found
+  Filter: Filter
+  AnyJobContent: Any job content

data/web/locales/fr.yml

@@ -17,14 +17,17 @@ fr:
   DeadJobs: Tâches mortes
   Delete: Supprimer
   DeleteAll: Tout supprimer
+  Deploy: Déploiement
   Enqueued: En attente
   Error: Erreur
   ErrorBacktrace: Backtrace d’erreur
   ErrorClass: Classe d’erreur
   ErrorMessage: Message d’erreur
+  ExecutionTime: Temps d'exécution
   Extras: Extras
   Failed: Échouées
   Failures: Echecs
+  Failure: Echec
   GoBack: ← Retour
   History: Historique
   Job: Tâche
@@ -35,6 +38,7 @@ fr:
   Latency: Latence
   LivePoll: Temps réel
   MemoryUsage: Mémoire utilisée
+  Name: Nom
   Namespace: Namespace
   NextRetry: Prochain essai
   NoDeadJobsFound: Aucune tâche morte n'a été trouvée
@@ -63,6 +67,7 @@ fr:
   RetryNow: Réessayer maintenant
   Scheduled: Planifiées
   ScheduledJobs: Tâches planifiées
+  Seconds: Secondes
   ShowAll: Tout montrer
   SixMonths: 6 mois
   Size: Taille
@@ -71,6 +76,8 @@ fr:
   Stop: Arrêter
   StopAll: Tout arrêter
   StopPolling: Arrêt du temps réel
+  Success: Succès
+  Summary: Résumé
   Thread: Thread
   Threads: Threads
   ThreeMonths: 3 mois
@@ -83,3 +90,10 @@ fr:
   Worker: Travailleur
   active: actif
   idle: inactif
+  Metrics: Métriques
+  NoDataFound: Aucune donnée disponible
+  TotalExecutionTime: Temps d'exécution total
+  AvgExecutionTime: Temps d'exécution moyen
+  Context: Contexte
+  Bucket: Bucket
+  NoJobMetricsFound: Aucune statistique de tâche récente n'a été trouvée