sidekiq 6.5.12 → 7.1.5

data/lib/sidekiq/scheduled.rb

@@ -8,16 +8,20 @@ module Sidekiq
     SETS = %w[retry schedule]
 
     class Enq
+      include Sidekiq::Component
+
       LUA_ZPOPBYSCORE = <<~LUA
         local key, now = KEYS[1], ARGV[1]
-        local jobs = redis.call("zrangebyscore", key, "-inf", now, "limit", 0, 1)
+        local jobs = redis.call("zrange", key, "-inf", now, "byscore", "limit", 0, 1)
         if jobs[1] then
           redis.call("zrem", key, jobs[1])
           return jobs[1]
         end
       LUA
 
-      def initialize
+      def initialize(container)
+        @config = container
+        @client = Sidekiq::Client.new(config: container)
         @done = false
         @lua_zpopbyscore_sha = nil
       end
@@ -25,15 +29,15 @@ module Sidekiq
       def enqueue_jobs(sorted_sets = SETS)
         # A job's "score" in Redis is the time at which it should be processed.
         # Just check Redis for the set of jobs with a timestamp before now.
-        Sidekiq.redis do |conn|
+        redis do |conn|
           sorted_sets.each do |sorted_set|
             # Get next item in the queue with score (time to execute) <= now.
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
             while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s]))
-              Sidekiq::Client.push(Sidekiq.load_json(job))
-              Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+              @client.push(Sidekiq.load_json(job))
+              logger.debug { "enqueued #{sorted_set}: #{job}" }
             end
           end
         end
@@ -47,12 +51,11 @@ module Sidekiq
 
       def zpopbyscore(conn, keys: nil, argv: nil)
         if @lua_zpopbyscore_sha.nil?
-          raw_conn = conn.respond_to?(:redis) ? conn.redis : conn
-          @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
+          @lua_zpopbyscore_sha = conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
-      rescue RedisConnection.adapter::CommandError => e
+        conn.call("EVALSHA", @lua_zpopbyscore_sha, keys.size, *keys, *argv)
+      rescue RedisClient::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -70,9 +73,9 @@ module Sidekiq
 
       INITIAL_WAIT = 10
 
-      def initialize(options)
-        @config = options
-        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(config)
+        @config = config
+        @enq = (config[:scheduled_enq] || Sidekiq::Scheduled::Enq).new(config)
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -82,14 +85,10 @@ module Sidekiq
       # Shut down this instance, will pause until the thread is dead.
       def terminate
         @done = true
-        @enq.terminate if @enq.respond_to?(:terminate)
+        @enq.terminate
 
-        if @thread
-          t = @thread
-          @thread = nil
-          @sleeper << 0
-          t.value
-        end
+        @sleeper << 0
+        @thread&.value
       end
 
       def start
@@ -194,11 +193,11 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless Sidekiq.redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
 
         count = 0
-        Sidekiq.redis do |conn|
-          procs = conn.sscan_each("processes").to_a
+        redis do |conn|
+          procs = conn.sscan("processes").to_a
           heartbeats = conn.pipelined { |pipeline|
             procs.each do |key|
               pipeline.hget(key, "info")

data/lib/sidekiq/testing.rb

@@ -51,19 +51,10 @@ module Sidekiq
       end
 
       def server_middleware
-        @server_chain ||= Middleware::Chain.new
+        @server_chain ||= Middleware::Chain.new(Sidekiq.default_configuration)
         yield @server_chain if block_given?
         @server_chain
       end
-
-      def constantize(str)
-        names = str.split("::")
-        names.shift if names.empty? || names.first.empty?
-
-        names.inject(Object) do |constant, name|
-          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-        end
-      end
     end
   end
 
@@ -73,7 +64,7 @@ module Sidekiq
   class EmptyQueueError < RuntimeError; end
 
   module TestingClient
-    def raw_push(payloads)
+    def atomic_push(conn, payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
           job = Sidekiq.load_json(Sidekiq.dump_json(job))
@@ -83,7 +74,7 @@ module Sidekiq
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          klass = Sidekiq::Testing.constantize(job["class"])
+          klass = Object.const_get(job["class"])
           job["id"] ||= SecureRandom.hex(12)
           job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
           klass.process_job(job_hash)
@@ -218,25 +209,9 @@ module Sidekiq
     #   assert_equal 1, HardJob.jobs.size
     #   assert_equal :something, HardJob.jobs[0]['args'][0]
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #
     # You can also clear and drain all job types:
     #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   MyMailer.delay.send_welcome_email('foo@example.com')
-    #   MyModel.delay.do_something_hard
-    #
-    #   assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 1, Sidekiq::Extensions::DelayedModel.jobs.size
-    #
-    #   Sidekiq::Worker.clear_all # or .drain_all
-    #
-    #   assert_equal 0, Sidekiq::Extensions::DelayedMailer.jobs.size
-    #   assert_equal 0, Sidekiq::Extensions::DelayedModel.jobs.size
+    #   Sidekiq::Job.clear_all # or .drain_all
     #
     # This can be useful to make sure jobs don't linger between tests:
     #
@@ -318,7 +293,7 @@ module Sidekiq
           job_classes = jobs.map { |job| job["class"] }.uniq
 
           job_classes.each do |job_class|
-            Sidekiq::Testing.constantize(job_class).drain
+            Object.const_get(job_class).drain
           end
         end
       end
@@ -329,13 +304,10 @@ module Sidekiq
     def jobs_for(klass)
       jobs.select do |job|
         marshalled = job["args"][0]
-        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+        marshalled.index(klass.to_s) && YAML.safe_load(marshalled)[0] == klass
       end
     end
   end
-
-  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
-  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING

data/lib/sidekiq/transaction_aware_client.rb

@@ -5,8 +5,8 @@ require "sidekiq/client"
 
 module Sidekiq
   class TransactionAwareClient
-    def initialize(redis_pool)
-      @redis_client = Client.new(redis_pool)
+    def initialize(pool: nil, config: nil)
+      @redis_client = Client.new(pool: pool, config: config)
     end
 
     def push(item)
@@ -34,11 +34,10 @@ module Sidekiq
     begin
       require "after_commit_everywhere"
     rescue LoadError
-      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
-      raise
+      raise %q(You need to add `gem "after_commit_everywhere"` to your Gemfile to use Sidekiq's transactional client)
     end
 
-    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq.default_job_options["client_class"] = Sidekiq::TransactionAwareClient
     Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
     true
   end

data/lib/sidekiq/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.5.12"
+  VERSION = "7.1.5"
+  MAJOR = 7
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
+      throw :halt, [res, {Rack::CONTENT_TYPE => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
+      [200, {Rack::CONTENT_TYPE => "application/json", Rack::CACHE_CONTROL => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -20,6 +20,12 @@ module Sidekiq
       "worker-src 'self'",
       "base-uri 'self'"
     ].join("; ").freeze
+    METRICS_PERIODS = {
+      "1h" => 60,
+      "2h" => 120,
+      "4h" => 240,
+      "8h" => 480
+    }
 
     def initialize(klass)
       @klass = klass
@@ -62,14 +68,20 @@ module Sidekiq
 
     get "/metrics" do
       q = Sidekiq::Metrics::Query.new
-      @query_result = q.top_jobs
+      @period = h((params[:period] || "")[0..1])
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.top_jobs(minutes: minutes)
       erb(:metrics)
     end
 
     get "/metrics/:name" do
       @name = route_params[:name]
+      @period = h((params[:period] || "")[0..1])
       q = Sidekiq::Metrics::Query.new
-      @query_result = q.for_job(@name)
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.for_job(@name, minutes: minutes)
       erb(:metrics_for_job)
     end
 
@@ -82,11 +94,14 @@ module Sidekiq
 
     post "/busy" do
       if params["identity"]
-        p = Sidekiq::Process.new("identity" => params["identity"])
-        p.quiet! if params["quiet"]
-        p.stop! if params["stop"]
+        pro = Sidekiq::ProcessSet[params["identity"]]
+
+        pro.quiet! if params["quiet"]
+        pro.stop! if params["stop"]
       else
         processes.each do |pro|
+          next if pro.embedded?
+
           pro.quiet! if params["quiet"]
           pro.stop! if params["stop"]
         end
@@ -310,12 +325,62 @@ module Sidekiq
     end
 
     get "/stats/queues" do
-      json Sidekiq::Stats::Queues.new.lengths
+      json Sidekiq::Stats.new.queues
+    end
+
+    ########
+    # Filtering
+    get "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    post "/filter/retries" do
+      x = params[:substr]
+      return redirect "#{root_path}retries" unless x && x != ""
+
+      @retries = search(Sidekiq::RetrySet.new, params[:substr])
+      erb :retries
+    end
+
+    get "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    post "/filter/scheduled" do
+      x = params[:substr]
+      return redirect "#{root_path}scheduled" unless x && x != ""
+
+      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
+      erb :scheduled
+    end
+
+    get "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
+    end
+
+    post "/filter/dead" do
+      x = params[:substr]
+      return redirect "#{root_path}morgue" unless x && x != ""
+
+      @dead = search(Sidekiq::DeadSet.new, params[:substr])
+      erb :morgue
     end
 
     def call(env)
       action = self.class.match(env)
-      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action
 
       app = @klass
       resp = catch(:halt) do
@@ -332,10 +397,10 @@ module Sidekiq
       else
         # rendered content goes here
         headers = {
-          "content-type" => "text/html",
-          "cache-control" => "private, no-store",
-          "content-language" => action.locale,
-          "content-security-policy" => CSP_HEADER
+          Rack::CONTENT_TYPE => "text/html",
+          Rack::CACHE_CONTROL => "private, no-store",
+          Web::CONTENT_LANGUAGE => action.locale,
+          Web::CONTENT_SECURITY_POLICY => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]

data/lib/sidekiq/web/csrf_protection.rb

@@ -62,7 +62,7 @@ module Sidekiq
 
       def deny(env)
         logger(env).warn "attack prevented by #{self.class}"
-        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+        [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
       end
 
       def session(env)
@@ -152,7 +152,7 @@ module Sidekiq
         # value and decrypt it
         token_length = masked_token.length / 2
         one_time_pad = masked_token[0...token_length]
-        encrypted_token = masked_token[token_length..-1]
+        encrypted_token = masked_token[token_length..]
         xor_byte_strings(one_time_pad, encrypted_token)
       end
 

data/lib/sidekiq/web/helpers.rb

@@ -15,7 +15,7 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.safe_load(File.open(file))
+          strs = YAML.safe_load(File.read(file))
           global.merge!(strs[lang])
         end
       end
@@ -49,8 +49,29 @@ module Sidekiq
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which)
+      erb(:filtering, locals: {which: which})
+    end
+
+    def filter_link(jid, within = "retries")
+      if within.nil?
+        ::Rack::Utils.escape_html(jid)
+      else
+        "<a href='#{root_path}filter/#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -111,14 +132,7 @@ module Sidekiq
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
-    # mperham/sidekiq#3243
+    # sidekiq/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
     end
@@ -161,22 +175,26 @@ module Sidekiq
       end
     end
 
+    def busy_weights(capsule_weights)
+      # backwards compat with 7.0.0, remove in 7.1
+      cw = [capsule_weights].flatten
+      cw.map { |hash|
+        hash.map { |name, weight| (weight > 0) ? +name << ": " << weight.to_s : name }.join(", ")
+      }.join("; ")
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
 
-    def redis_connection
+    def redis_url
       Sidekiq.redis do |conn|
-        conn.connection[:id]
+        conn.config.server_url
       end
     end
 
-    def namespace
-      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-    end
-
     def redis_info
-      Sidekiq.redis_info
+      Sidekiq.default_configuration.redis_info
     end
 
     def root_path
@@ -314,7 +332,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.default_configuration[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end
@@ -327,11 +345,8 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
-    def redis_connection_and_namespace
-      @redis_connection_and_namespace ||= begin
-        namespace_suffix = namespace.nil? ? "" : "##{namespace}"
-        "#{redis_connection}#{namespace_suffix}"
-      end
+    def pollable?
+      !(current_path == "" || current_path.start_with?("metrics"))
     end
 
     def retry_or_delete_or_kill(job, params)

data/lib/sidekiq/web.rb

@@ -30,11 +30,20 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue"
+      "Dead" => "morgue",
+      "Metrics" => "metrics"
     }
 
-    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
-      DEFAULT_TABS["Metrics"] = "metrics"
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
     end
 
     class << self
@@ -51,6 +60,10 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
+      def custom_job_info_rows
+        @custom_job_info_rows ||= []
+      end
+
       def locales
         @locales ||= LOCALES
       end
@@ -79,14 +92,6 @@ module Sidekiq
         send(:"#{attribute}=", value)
       end
 
-      def sessions=(val)
-        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
-      def session_secret=(val)
-        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
       attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
@@ -133,10 +138,6 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def sessions=(val)
-      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
-    end
-
     def self.register(extension)
       extension.registered(WebApplication)
     end
@@ -148,7 +149,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Worker = Job
+end