sidekiq 6.4.1 → 6.5.10

data/lib/sidekiq/cli.rb

@@ -9,18 +9,34 @@ require "erb"
 require "fileutils"
 
 require "sidekiq"
+require "sidekiq/component"
 require "sidekiq/launcher"
-require "sidekiq/util"
 
-module Sidekiq
+# module ScoutApm
+# VERSION = "5.3.1"
+# end
+fail <<~EOM if defined?(ScoutApm::VERSION) && ScoutApm::VERSION < "5.2.0"
+  
+  
+  scout_apm v#{ScoutApm::VERSION} is unsafe with Sidekiq 6.5. Please run `bundle up scout_apm` to upgrade to 5.2.0 or greater.
+  
+  
+EOM
+
+module Sidekiq # :nodoc:
   class CLI
-    include Util
+    include Sidekiq::Component
     include Singleton unless $TESTING
 
     attr_accessor :launcher
     attr_accessor :environment
+    attr_accessor :config
+
+    def parse(args = ARGV.dup)
+      @config = Sidekiq
+      @config[:error_handlers].clear
+      @config[:error_handlers] << @config.method(:default_error_handler)
 
-    def parse(args = ARGV)
       setup_options(args)
       initialize_logger
       validate!
@@ -36,7 +52,7 @@ module Sidekiq
     def run(boot_app: true)
       boot_application if boot_app
 
-      if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
+      if environment == "development" && $stdout.tty? && @config.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
       logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
@@ -67,7 +83,7 @@ module Sidekiq
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
-      info = Sidekiq.redis_info
+      info = @config.redis_info
       ver = info["redis_version"]
       raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
@@ -85,22 +101,22 @@ module Sidekiq
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
-      cursize = Sidekiq.redis_pool.size
-      needed = Sidekiq.options[:concurrency] + 2
+      cursize = @config.redis_pool.size
+      needed = @config[:concurrency] + 2
       raise "Your pool of #{cursize} Redis connections is too small, please increase the size to at least #{needed}" if cursize < needed
 
       # cache process identity
-      Sidekiq.options[:identity] = identity
+      @config[:identity] = identity
 
       # Touch middleware so it isn't lazy loaded by multiple threads, #3043
-      Sidekiq.server_middleware
+      @config.server_middleware
 
       # Before this point, the process is initializing with just the main thread.
       # Starting here the process will now have multiple threads running.
       fire_event(:startup, reverse: false, reraise: true)
 
-      logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(", ")}" }
-      logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Client Middleware: #{@config.client_middleware.map(&:klass).join(", ")}" }
+      logger.debug { "Server Middleware: #{@config.server_middleware.map(&:klass).join(", ")}" }
 
       launch(self_read)
     end
@@ -110,13 +126,13 @@ module Sidekiq
         logger.info "Starting processing, hit Ctrl-C to stop"
       end
 
-      @launcher = Sidekiq::Launcher.new(options)
+      @launcher = Sidekiq::Launcher.new(@config)
 
       begin
         launcher.run
 
-        while (readable_io = self_read.wait_readable)
-          signal = readable_io.gets.strip
+        while self_read.wait_readable
+          signal = self_read.gets.strip
           handle_signal(signal)
         end
       rescue Interrupt
@@ -173,25 +189,25 @@ module Sidekiq
       # Heroku sends TERM and then waits 30 seconds for process to exit.
       "TERM" => ->(cli) { raise Interrupt },
       "TSTP" => ->(cli) {
-        Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+        cli.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
       "TTIN" => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
+          cli.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
-            Sidekiq.logger.warn thread.backtrace.join("\n")
+            cli.logger.warn thread.backtrace.join("\n")
           else
-            Sidekiq.logger.warn "<no backtrace available>"
+            cli.logger.warn "<no backtrace available>"
           end
         end
       }
     }
-    UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
+    UNHANDLED_SIGNAL_HANDLER = ->(cli) { cli.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
 
     def handle_signal(sig)
-      Sidekiq.logger.debug "Got #{sig} signal"
+      logger.debug "Got #{sig} signal"
       SIGNAL_HANDLERS[sig].call(self)
     end
 
@@ -209,6 +225,7 @@ module Sidekiq
       # Both Sinatra 2.0+ and Sidekiq support this term.
       # RAILS_ENV and RACK_ENV are there for legacy support.
       @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      config[:environment] = @environment
     end
 
     def symbolize_keys_deep!(hash)
@@ -237,7 +254,7 @@ module Sidekiq
         config_dir = if File.directory?(opts[:require].to_s)
           File.join(opts[:require], "config")
         else
-          File.join(options[:require], "config")
+          File.join(@config[:require], "config")
         end
 
         %w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
@@ -254,27 +271,23 @@ module Sidekiq
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
-      options.merge!(opts)
-    end
-
-    def options
-      Sidekiq.options
+      @config.merge!(opts)
     end
 
     def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
-      if File.directory?(options[:require])
+      if File.directory?(@config[:require])
         require "rails"
         if ::Rails::VERSION::MAJOR < 5
           raise "Sidekiq no longer supports this version of Rails"
         else
           require "sidekiq/rails"
-          require File.expand_path("#{options[:require]}/config/environment.rb")
+          require File.expand_path("#{@config[:require]}/config/environment.rb")
         end
-        options[:tag] ||= default_tag
+        @config[:tag] ||= default_tag
       else
-        require options[:require]
+        require @config[:require]
       end
     end
 
@@ -291,18 +304,18 @@ module Sidekiq
     end
 
     def validate!
-      if !File.exist?(options[:require]) ||
-          (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+      if !File.exist?(@config[:require]) ||
+          (File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
-        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "  to load your job classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
         die(1)
       end
 
       [:concurrency, :timeout].each do |opt|
-        raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.key?(opt) && options[opt].to_i <= 0
+        raise ArgumentError, "#{opt}: #{@config[opt]} is not a valid value" if @config[opt].to_i <= 0
       end
     end
 
@@ -336,7 +349,7 @@ module Sidekiq
           parse_queue opts, queue, weight
         end
 
-        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with workers or file to require" do |arg|
+        o.on "-r", "--require [PATH|DIR]", "Location of Rails application with jobs or file to require" do |arg|
           opts[:require] = arg
         end
 
@@ -376,7 +389,7 @@ module Sidekiq
     end
 
     def initialize_logger
-      Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
+      @config.logger.level = ::Logger::DEBUG if @config[:verbose]
     end
 
     def parse_config(path)
@@ -425,3 +438,4 @@ module Sidekiq
 end
 
 require "sidekiq/systemd"
+require "sidekiq/metrics/tracking" if ENV["SIDEKIQ_METRICS_BETA"]

data/lib/sidekiq/client.rb

@@ -15,7 +15,7 @@ module Sidekiq
     #   client.middleware do |chain|
     #     chain.use MyClientMiddleware
     #   end
-    #   client.push('class' => 'SomeWorker', 'args' => [1,2,3])
+    #   client.push('class' => 'SomeJob', 'args' => [1,2,3])
     #
     # All client instances default to the globally-defined
     # Sidekiq.client_middleware but you can change as necessary.
@@ -49,16 +49,16 @@ module Sidekiq
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
     #   queue - the named queue to use, default 'default'
-    #   class - the worker class to call, required
+    #   class - the job class to call, required
     #   args - an array of simple arguments to the perform method, must be JSON-serializable
     #   at - timestamp to schedule the job (optional), must be Numeric (e.g. Time.now.to_f)
     #   retry - whether to retry this job if it fails, default true or an integer number of retries
     #   backtrace - whether to save any error backtrace, default false
     #
     # If class is set to the class name, the jobs' options will be based on Sidekiq's default
-    # worker options. Otherwise, they will be based on the job class's options.
+    # job options. Otherwise, they will be based on the job class's options.
     #
-    # Any options valid for a worker class's sidekiq_options are also available here.
+    # Any options valid for a job class's sidekiq_options are also available here.
     #
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.  Note that +backtrace: true+ can take quite a bit of
@@ -67,13 +67,15 @@ module Sidekiq
     # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
-    #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #   push('queue' => 'my_queue', 'class' => MyJob, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def push(item)
       normed = normalize_item(item)
-      payload = process_single(item["class"], normed)
-
+      payload = middleware.invoke(item["class"], normed, normed["queue"], @redis_pool) do
+        normed
+      end
       if payload
+        verify_json(payload)
         raw_push([payload])
         payload["jid"]
       end
@@ -101,12 +103,17 @@ module Sidekiq
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all? { |entry| entry.is_a?(Numeric) })
       raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
+      jid = items.delete("jid")
+      raise ArgumentError, "Explicitly passing 'jid' when pushing more than one job is not supported" if jid && args.size > 1
+
       normed = normalize_item(items)
       payloads = args.map.with_index { |job_args, index|
         copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12))
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
-
-        result = process_single(items["class"], copy)
+        result = middleware.invoke(items["class"], copy, copy["queue"], @redis_pool) do
+          verify_json(copy)
+          copy
+        end
         result || nil
       }.compact
 
@@ -119,8 +126,8 @@ module Sidekiq
     #
     #   pool = ConnectionPool.new { Redis.new }
     #   Sidekiq::Client.via(pool) do
-    #     SomeWorker.perform_async(1,2,3)
-    #     SomeOtherWorker.perform_async(1,2,3)
+    #     SomeJob.perform_async(1,2,3)
+    #     SomeOtherJob.perform_async(1,2,3)
     #   end
     #
     # Generally this is only needed for very large Sidekiq installs processing
@@ -145,10 +152,10 @@ module Sidekiq
       end
 
       # Resque compatibility helpers.  Note all helpers
-      # should go through Worker#client_push.
+      # should go through Sidekiq::Job#client_push.
       #
       # Example usage:
-      #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue(MyJob, 'foo', 1, :bat => 'bar')
       #
       # Messages are enqueued to the 'default' queue.
       #
@@ -157,19 +164,19 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to(:queue_name, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to(queue, klass, *args)
         klass.client_push("queue" => queue, "class" => klass, "args" => args)
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_to_in(queue, interval, klass, *args)
         int = interval.to_f
         now = Time.now.to_f
-        ts = (int < 1_000_000_000 ? now + int : int)
+        ts = ((int < 1_000_000_000) ? now + int : int)
 
         item = {"class" => klass, "args" => args, "at" => ts, "queue" => queue}
         item.delete("at") if ts <= now
@@ -178,7 +185,7 @@ module Sidekiq
       end
 
       # Example usage:
-      #   Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
+      #   Sidekiq::Client.enqueue_in(3.minutes, MyJob, 'foo', 1, :bat => 'bar')
       #
       def enqueue_in(interval, klass, *args)
         klass.perform_in(interval, *args)
@@ -194,7 +201,7 @@ module Sidekiq
           conn.pipelined do |pipeline|
             atomic_push(pipeline, payloads)
           end
-        rescue Redis::BaseError => ex
+        rescue RedisConnection.adapter::BaseError => ex
           # 2550 Failover can cause the server to become a replica, need
           # to disconnect and reopen the socket to get back to the primary.
           # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
@@ -213,7 +220,7 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first.key?("at")
-        conn.zadd("schedule", payloads.map { |hash|
+        conn.zadd("schedule", payloads.flat_map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
         })
@@ -224,17 +231,9 @@ module Sidekiq
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", queue)
+        conn.sadd("queues", [queue])
         conn.lpush("queue:#{queue}", to_push)
       end
     end
-
-    def process_single(worker_class, item)
-      queue = item["queue"]
-
-      middleware.invoke(worker_class, item, queue, @redis_pool) do
-        item
-      end
-    end
   end
 end

data/lib/sidekiq/component.rb

@@ -0,0 +1,65 @@
+module Sidekiq
+  ##
+  # Sidekiq::Component assumes a config instance is available at @config
+  module Component # :nodoc:
+    attr_reader :config
+
+    def watchdog(last_words)
+      yield
+    rescue Exception => ex
+      handle_exception(ex, {context: last_words})
+      raise ex
+    end
+
+    def safe_thread(name, &block)
+      Thread.new do
+        Thread.current.name = name
+        watchdog(name, &block)
+      end
+    end
+
+    def logger
+      config.logger
+    end
+
+    def redis(&block)
+      config.redis(&block)
+    end
+
+    def tid
+      Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+    end
+
+    def hostname
+      ENV["DYNO"] || Socket.gethostname
+    end
+
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
+    def identity
+      @@identity ||= "#{hostname}:#{::Process.pid}:#{process_nonce}"
+    end
+
+    def handle_exception(ex, ctx = {})
+      config.handle_exception(ex, ctx)
+    end
+
+    def fire_event(event, options = {})
+      oneshot = options.fetch(:oneshot, true)
+      reverse = options[:reverse]
+      reraise = options[:reraise]
+
+      arr = config[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
+        block.call
+      rescue => ex
+        handle_exception(ex, {context: "Exception during Sidekiq lifecycle event.", event: event})
+        raise ex if reraise
+      end
+      arr.clear if oneshot # once we've fired an event, we never fire it again
+    end
+  end
+end

data/lib/sidekiq/delay.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module Sidekiq
+module Sidekiq # :nodoc:
   module Extensions
     def self.enable_delay!
       warn "Sidekiq's Delayed Extensions will be removed in Sidekiq 7.0", uplevel: 1

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -10,7 +10,7 @@ module Sidekiq
       def initialize(performable, target, options = {})
         @performable = performable
         @target = target
-        @opts = options
+        @opts = options.transform_keys(&:to_s)
       end
 
       def method_missing(name, *args)

data/lib/sidekiq/fetch.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 require "sidekiq"
+require "sidekiq/component"
 
-module Sidekiq
+module Sidekiq # :nodoc:
   class BasicFetch
+    include Sidekiq::Component
     # We want the fetch operation to timeout every few seconds so the thread
     # can check if the process is shutting down.
     TIMEOUT = 2
 
-    UnitOfWork = Struct.new(:queue, :job) {
+    UnitOfWork = Struct.new(:queue, :job, :config) {
       def acknowledge
         # nothing to do
       end
@@ -18,20 +20,20 @@ module Sidekiq
       end
 
       def requeue
-        Sidekiq.redis do |conn|
+        config.redis do |conn|
           conn.rpush(queue, job)
         end
       end
     }
 
-    def initialize(options)
-      raise ArgumentError, "missing queue list" unless options[:queues]
-      @options = options
-      @strictly_ordered_queues = !!@options[:strict]
-      @queues = @options[:queues].map { |q| "queue:#{q}" }
+    def initialize(config)
+      raise ArgumentError, "missing queue list" unless config[:queues]
+      @config = config
+      @strictly_ordered_queues = !!@config[:strict]
+      @queues = @config[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
-        @queues << TIMEOUT
+        @queues << {timeout: TIMEOUT}
       end
     end
 
@@ -44,30 +46,30 @@ module Sidekiq
         return nil
       end
 
-      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
-      UnitOfWork.new(*work) if work
+      queue, job = redis { |conn| conn.brpop(*qs) }
+      UnitOfWork.new(queue, job, config) if queue
     end
 
     def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
-      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
         jobs_to_requeue[unit_of_work.queue] ||= []
         jobs_to_requeue[unit_of_work.queue] << unit_of_work.job
       end
 
-      Sidekiq.redis do |conn|
+      redis do |conn|
         conn.pipelined do |pipeline|
           jobs_to_requeue.each do |queue, jobs|
             pipeline.rpush(queue, jobs)
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+      logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
-      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+      logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
     # Creating the Redis#brpop command takes into account any
@@ -81,7 +83,7 @@ module Sidekiq
       else
         permute = @queues.shuffle
         permute.uniq!
-        permute << TIMEOUT
+        permute << {timeout: TIMEOUT}
         permute
       end
     end