sidekiq 7.0.0 → 7.3.0

data/lib/sidekiq/redis_client_adapter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "set"
 require "redis_client"
 require "redis_client/decorator"
 
@@ -8,6 +9,9 @@ module Sidekiq
     BaseError = RedisClient::Error
     CommandError = RedisClient::CommandError
 
+    # You can add/remove items or clear the whole thing if you don't want deprecation warnings.
+    DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
+
     module CompatMethods
       def info
         @client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
@@ -17,11 +21,28 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
+      # this is the set of Redis commands used by Sidekiq. Not guaranteed
+      # to be comprehensive, we use this as a performance enhancement to
+      # avoid calling method_missing on most commands
+      USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
+        get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
+        lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
+        publish rpop rpush sadd scard script set sismember smembers
+        srem ttl type unlink zadd zcard zincrby zrange zrem
+        zremrangebyrank zremrangebyscore]
+
+      USED_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs|
+          @client.call(name, *args, **kwargs)
+        end
+      end
+
       private
 
       # this allows us to use methods like `conn.hmset(...)` instead of having to use
       # redis-client's native `conn.call("hmset", ...)`
       def method_missing(*args, &block)
+        warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
         @client.call(*args, *block)
       end
       ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
@@ -34,42 +55,22 @@ module Sidekiq
     CompatClient = RedisClient::Decorator.create(CompatMethods)
 
     class CompatClient
-      # underscore methods are not official API
-      def _client
-        @client
-      end
-
-      def _config
+      def config
         @client.config
       end
-
-      def message
-        yield nil, @queue.pop
-      end
-
-      # NB: this method does not return
-      def subscribe(chan)
-        @queue = ::Queue.new
-
-        pubsub = @client.pubsub
-        pubsub.call("subscribe", chan)
-
-        loop do
-          evt = pubsub.next_event
-          next if evt.nil?
-          next unless evt[0] == "message" && evt[1] == chan
-
-          (_, _, msg) = evt
-          @queue << msg
-          yield self
-        end
-      end
     end
 
     def initialize(options)
       opts = client_opts(options)
       @config = if opts.key?(:sentinels)
         RedisClient.sentinel(**opts)
+      elsif opts.key?(:nodes)
+        # Sidekiq does not support Redis clustering but Sidekiq Enterprise's
+        # rate limiters are cluster-safe so we can scale to millions
+        # of rate limiters using a Redis cluster. This requires the
+        # `redis-cluster-client` gem.
+        # Sidekiq::Limiter.redis = { nodes: [...] }
+        RedisClient.cluster(**opts)
       else
         RedisClient.config(**opts)
       end
@@ -85,8 +86,7 @@ module Sidekiq
       opts = options.dup
 
       if opts[:namespace]
-        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature isn't supported by redis-client. " \
-          "Either use the redis adapter or remove the namespace."
+        raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
       end
 
       opts.delete(:size)
@@ -97,13 +97,9 @@ module Sidekiq
         opts.delete(:network_timeout)
       end
 
-      if opts[:driver]
-        opts[:driver] = opts[:driver].to_sym
-      end
-
       opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
       opts[:role] = opts[:role].to_sym if opts.key?(:role)
-      opts.delete(:url) if opts.key?(:sentinels)
+      opts[:driver] = opts[:driver].to_sym if opts.key?(:driver)
 
       # Issue #3303, redis-rb will silently retry an operation.
       # This can lead to duplicate jobs if Sidekiq::Client's LPUSH

data/lib/sidekiq/redis_connection.rb

@@ -8,16 +8,28 @@ module Sidekiq
   module RedisConnection
     class << self
       def create(options = {})
-        symbolized_options = options.transform_keys(&:to_sym)
+        symbolized_options = deep_symbolize_keys(options)
         symbolized_options[:url] ||= determine_redis_provider
 
         logger = symbolized_options.delete(:logger)
         logger&.info { "Sidekiq #{Sidekiq::VERSION} connecting to Redis with options #{scrub(symbolized_options)}" }
 
+        raise "Sidekiq 7+ does not support Redis protocol 2" if symbolized_options[:protocol] == 2
+
+        safe = !!symbolized_options.delete(:cluster_safe)
+        raise ":nodes not allowed, Sidekiq is not safe to run on Redis Cluster" if !safe && symbolized_options.key?(:nodes)
+
         size = symbolized_options.delete(:size) || 5
         pool_timeout = symbolized_options.delete(:pool_timeout) || 1
         pool_name = symbolized_options.delete(:pool_name)
 
+        # Default timeout in redis-client is 1 second, which can be too aggressive
+        # if the Sidekiq process is CPU-bound. With 10-15 threads and a thread quantum of 100ms,
+        # it can be easy to get the occasional ReadTimeoutError. You can still provide
+        # a smaller timeout explicitly:
+        #     config.redis = { url: "...", timeout: 1 }
+        symbolized_options[:timeout] ||= 3
+
         redis_config = Sidekiq::RedisClientAdapter.new(symbolized_options)
         ConnectionPool.new(timeout: pool_timeout, size: size, name: pool_name) do
           redis_config.new_client
@@ -26,6 +38,19 @@ module Sidekiq
 
       private
 
+      def deep_symbolize_keys(object)
+        case object
+        when Hash
+          object.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = deep_symbolize_keys(value)
+          end
+        when Array
+          object.map { |e| deep_symbolize_keys(e) }
+        else
+          object
+        end
+      end
+
       def scrub(options)
         redacted = "REDACTED"
 
@@ -38,9 +63,8 @@ module Sidekiq
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        if scrubbed_options[:password]
-          scrubbed_options[:password] = redacted
-        end
+        scrubbed_options[:password] = redacted if scrubbed_options[:password]
+        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end
@@ -66,9 +90,7 @@ module Sidekiq
           EOM
         end
 
-        ENV[
-          p || "REDIS_URL"
-        ]
+        ENV[p.to_s] || ENV["REDIS_URL"]
       end
     end
   end

data/lib/sidekiq/scheduled.rb

@@ -12,7 +12,7 @@ module Sidekiq
 
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
-        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
         if jobs[1] then
           redis.call("zrem", key, jobs[1])
           return jobs[1]
@@ -54,7 +54,7 @@ module Sidekiq
           @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
+        conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
       rescue RedisClient::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
@@ -144,7 +144,7 @@ module Sidekiq
         # In the example above, each process should schedule every 10 seconds on average. We special
         # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
         # As we run more processes, the scheduling interval average will approach an even spread
-        # between 0 and poll interval so we don't need this artifical boost.
+        # between 0 and poll interval so we don't need this artificial boost.
         #
         count = process_count
         interval = poll_interval_average(count)
@@ -193,7 +193,7 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
         count = 0
         redis do |conn|

data/lib/sidekiq/testing.rb

@@ -5,23 +5,42 @@ require "sidekiq"
 
 module Sidekiq
   class Testing
+    class TestModeAlreadySetError < RuntimeError; end
     class << self
-      attr_accessor :__test_mode
+      attr_accessor :__global_test_mode
 
+      # Calling without a block sets the global test mode, affecting
+      # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
-          current_mode = __test_mode
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
           begin
-            self.__test_mode = mode
             yield
           ensure
-            self.__test_mode = current_mode
+            self.__local_test_mode = nil
           end
         else
-          self.__test_mode = mode
+          self.__global_test_mode = mode
         end
       end
 
+      def __test_mode
+        __local_test_mode || __global_test_mode
+      end
+
+      def __local_test_mode
+        Thread.current[:__sidekiq_test_mode]
+      end
+
+      def __local_test_mode=(value)
+        Thread.current[:__sidekiq_test_mode] = value
+      end
+
       def disable!(&block)
         __set_test_mode(:disable, &block)
       end
@@ -64,7 +83,7 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def raw_push(payloads)
+    def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
@@ -93,7 +112,7 @@ module Sidekiq
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
     # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
-    # to the array. Because the array was dervied from a filter of the total
+    # to the array. Because the array was derived from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
@@ -259,7 +278,7 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, to_s)
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
         process_job(next_job)
       end
 

data/lib/sidekiq/transaction_aware_client.rb

@@ -9,7 +9,14 @@ module Sidekiq
       @redis_client = Client.new(pool: pool, config: config)
     end
 
+    def batching?
+      Thread.current[:sidekiq_batch]
+    end
+
     def push(item)
+      # 6160 we can't support both Sidekiq::Batch and transactions.
+      return @redis_client.push(item) if batching?
+
       # pre-allocate the JID so we can return it immediately and
       # save it to the database as part of the transaction.
       item["jid"] ||= SecureRandom.hex(12)

data/lib/sidekiq/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.0.0"
+  VERSION = "7.3.0"
   MAJOR = 7
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,16 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
+    end
+
+    def reload_page
+      current_location = request.referer.gsub(request.base_url, "")
+      redirect current_location
     end
 
     def params
@@ -42,7 +47,8 @@ module Sidekiq
     def erb(content, options = {})
       if content.is_a? Symbol
         unless respond_to?(:"_erb_#{content}")
-          src = ERB.new(File.read("#{Web.settings.views}/#{content}.erb")).src
+          views = options[:views] || Web.settings.views
+          src = ERB.new(File.read("#{views}/#{content}.erb")).src
           WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
             def _erb_#{content}
               #{src}
@@ -68,7 +74,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -5,7 +5,7 @@ module Sidekiq
     extend WebRouter
 
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
-    CSP_HEADER = [
+    CSP_HEADER_TEMPLATE = [
       "default-src 'self' https: http:",
       "child-src 'self'",
       "connect-src 'self' https: http: wss: ws:",
@@ -15,11 +15,17 @@ module Sidekiq
       "manifest-src 'self'",
       "media-src 'self'",
       "object-src 'none'",
-      "script-src 'self' https: http: 'unsafe-inline'",
-      "style-src 'self' https: http: 'unsafe-inline'",
+      "script-src 'self' 'nonce-!placeholder!'",
+      "style-src 'self' https: http: 'unsafe-inline'", # TODO Nonce in 8.0
       "worker-src 'self'",
       "base-uri 'self'"
     ].join("; ").freeze
+    METRICS_PERIODS = {
+      "1h" => 60,
+      "2h" => 120,
+      "4h" => 240,
+      "8h" => 480
+    }
 
     def initialize(klass)
       @klass = klass
@@ -43,9 +49,9 @@ module Sidekiq
 
     head "/" do
       # HEAD / is the cheapest heartbeat possible,
-      # it hits Redis to ensure connectivity
-      Sidekiq.redis { |c| c.llen("queue:default") }
-      ""
+      # it hits Redis to ensure connectivity and returns
+      # the size of the default queue
+      Sidekiq.redis { |c| c.llen("queue:default") }.to_s
     end
 
     get "/" do
@@ -62,14 +68,20 @@ module Sidekiq
 
     get "/metrics" do
       q = Sidekiq::Metrics::Query.new
-      @query_result = q.top_jobs
+      @period = h((params[:period] || "")[0..1])
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.top_jobs(minutes: minutes)
       erb(:metrics)
     end
 
     get "/metrics/:name" do
       @name = route_params[:name]
+      @period = h((params[:period] || "")[0..1])
       q = Sidekiq::Metrics::Query.new
-      @query_result = q.for_job(@name)
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.for_job(@name, minutes: minutes)
       erb(:metrics_for_job)
     end
 
@@ -82,11 +94,14 @@ module Sidekiq
 
     post "/busy" do
       if params["identity"]
-        p = Sidekiq::Process.new("identity" => params["identity"])
-        p.quiet! if params["quiet"]
-        p.stop! if params["stop"]
+        pro = Sidekiq::ProcessSet[params["identity"]]
+
+        pro.quiet! if params["quiet"]
+        pro.stop! if params["stop"]
       else
         processes.each do |pro|
+          next if pro.embedded?
+
           pro.quiet! if params["quiet"]
           pro.stop! if params["stop"]
         end
@@ -313,9 +328,87 @@ module Sidekiq
       json Sidekiq::Stats.new.queues
     end
 
+    ########
+    # Filtering
+
+    get "/filter/metrics" do
+      redirect "#{root_path}metrics"
+    end
+
+    post "/filter/metrics" do
+      x = params[:substr]
+      q = Sidekiq::Metrics::Query.new
+      @period = h((params[:period] || "")[0..1])
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.top_jobs(minutes: minutes, class_filter: Regexp.new(Regexp.escape(x), Regexp::IGNORECASE))
+
+      erb :metrics
+    end
+
+    get "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    post "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    get "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    post "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    get "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
+    end
+
+    post "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
+    end
+
+    post "/change_locale" do
+      locale = params["locale"]
+
+      match = available_locales.find { |available|
+        locale == available
+      }
+
+      session[:locale] = match if match
+
+      reload_page
+    end
+
     def call(env)
       action = self.class.match(env)
-      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -332,16 +425,20 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "content-type" => "text/html",
-          "cache-control" => "private, no-store",
-          "content-language" => action.locale,
-          "content-security-policy" => CSP_HEADER
+          Rack::CONTENT_TYPE => "text/html",
+          Rack::CACHE_CONTROL => "private, no-store",
+          Web::CONTENT_LANGUAGE => action.locale,
+          Web::CONTENT_SECURITY_POLICY => process_csp(env, CSP_HEADER_TEMPLATE)
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
     end
 
+    def process_csp(env, input)
+      input.gsub("!placeholder!", env[:csp_nonce])
+    end
+
     def self.helpers(mod = nil, &block)
       if block
         WebAction.class_eval(&block)

data/lib/sidekiq/web/csrf_protection.rb

@@ -27,7 +27,6 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require "securerandom"
-require "base64"
 require "rack/request"
 
 module Sidekiq
@@ -57,12 +56,12 @@ module Sidekiq
       end
 
       def logger(env)
-        @logger ||= (env["rack.logger"] || ::Logger.new(env["rack.errors"]))
+        @logger ||= env["rack.logger"] || ::Logger.new(env["rack.errors"])
       end
 
       def deny(env)
         logger(env).warn "attack prevented by #{self.class}"
-        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+        [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
       end
 
       def session(env)
@@ -116,7 +115,7 @@ module Sidekiq
         sess = session(env)
         localtoken = sess[:csrf]
 
-        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        # Checks that Rack::Session::Cookie actually contains the csrf token
         return false if localtoken.nil?
 
         # Rotate the session token after every use
@@ -143,7 +142,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.urlsafe_encode64(masked_token)
+        encode_token(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -168,8 +167,12 @@ module Sidekiq
         ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
       end
 
+      def encode_token(token)
+        [token].pack("m0").tr("+/", "-_")
+      end
+
       def decode_token(token)
-        Base64.urlsafe_decode64(token)
+        token.tr("-_", "+/").unpack1("m0")
       end
 
       def xor_byte_strings(s1, s2)