sidekiq 6.0.4 → 6.1.1

data/Pro-Changes.md

@@ -2,7 +2,26 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
+Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy.
+
+5.1.1
+---------
+
+- Fix broken basic fetcher [#4616]
+
+5.1.0
+---------
+
+- Remove old Statsd metrics with `WorkerName` in the name [#4377]
+```
+job.WorkerName.count -> job.count with tag worker:WorkerName
+job.WorkerName.perform -> job.perform with tag worker:WorkerName
+job.WorkerName.failure -> job.failure with tag worker:WorkerName
+```
+- Remove `concurrent-ruby` gem dependency [#4586]
+- Update `constantize` for batch callbacks. [#4469]
+- Add queue tag to `jobs.recovered.fetch` metric [#4594]
+- Refactor Pro's fetch infrastructure [#4602]
 
 5.0.1
 ---------

data/README.md

@@ -2,11 +2,8 @@ Sidekiq
 ==============
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
-[![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
-[![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
+[![Codecov](https://codecov.io/gh/mperham/sidekiq/branch/master/graph/badge.svg)](https://codecov.io/gh/mperham/sidekiq)
 [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
-[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
-
 
 Simple, efficient background processing for Ruby.
 
@@ -94,4 +91,4 @@ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for
 Author
 -----------------
 
-Mike Perham, [@mperham@mastodon.xyz](https://mastodon.xyz/@mperham) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
+Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)

data/bin/sidekiq

@@ -6,13 +6,37 @@ $TESTING = false
 
 require_relative '../lib/sidekiq/cli'
 
+def integrate_with_systemd
+  return unless ENV["NOTIFY_SOCKET"]
+
+  Sidekiq.configure_server do |config|
+    Sidekiq.logger.info "Enabling systemd notification integration"
+    require "sidekiq/sd_notify"
+    config.on(:startup) do
+      Sidekiq::SdNotify.ready
+    end
+    config.on(:shutdown) do
+      Sidekiq::SdNotify.stopping
+    end
+    Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
+  end
+end
+
 begin
   cli = Sidekiq::CLI.instance
   cli.parse
+
+  integrate_with_systemd
+
   cli.run
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  if Sidekiq.error_handlers.length == 0
+    STDERR.puts e.message
+    STDERR.puts e.backtrace.join("\n")
+  else
+    cli.handle_exception e
+  end
+
   exit 1
 end

data/lib/sidekiq.rb

@@ -20,6 +20,7 @@ module Sidekiq
     labels: [],
     concurrency: 10,
     require: ".",
+    strict: true,
     environment: nil,
     timeout: 25,
     poll_interval_average: nil,
@@ -30,16 +31,16 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: [],
+      heartbeat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-    reloader: proc { |&block| block.call },
+    reloader: proc { |&block| block.call }
   }
 
   DEFAULT_WORKER_OPTIONS = {
     "retry" => true,
-    "queue" => "default",
+    "queue" => "default"
   }
 
   FAKE_INFO = {
@@ -47,7 +48,7 @@ module Sidekiq
     "uptime_in_days" => "9999",
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
-    "used_memory_peak_human" => "9P",
+    "used_memory_peak_human" => "9P"
   }
 
   def self.❨╯°□°❩╯︵┻━┻
@@ -95,10 +96,11 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::CommandError => ex
+      rescue Redis::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
-        if retryable && ex.message =~ /READONLY/
+        # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+        if retryable && ex.message =~ /READONLY|NOREPLICAS/
           conn.disconnect!
           retryable = false
           retry
@@ -154,7 +156,7 @@ module Sidekiq
 
   def self.default_worker_options=(hash)
     # stringify
-    @default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
+    @default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
   end
 
   def self.default_worker_options

data/lib/sidekiq/api.rb

@@ -105,7 +105,7 @@ module Sidekiq
 
         default_queue_latency: default_queue_latency,
         workers_size: workers_size,
-        enqueued: enqueued,
+        enqueued: enqueued
       }
     end
 
@@ -273,7 +273,7 @@ module Sidekiq
     def clear
       Sidekiq.redis do |conn|
         conn.multi do
-          conn.del(@rname)
+          conn.unlink(@rname)
           conn.srem("queues", name)
         end
       end
@@ -562,7 +562,7 @@ module Sidekiq
 
     def clear
       Sidekiq.redis do |conn|
-        conn.del(name)
+        conn.unlink(name)
       end
     end
     alias_method :💣, :clear
@@ -916,12 +916,13 @@ module Sidekiq
   class Workers
     include Enumerable
 
-    def each
+    def each(&block)
+      results = []
       Sidekiq.redis do |conn|
         procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
           valid, workers = conn.pipelined {
-            conn.exists(key)
+            conn.exists?(key)
             conn.hgetall("#{key}:workers")
           }
           next unless valid
@@ -930,10 +931,12 @@ module Sidekiq
             p = hsh["payload"]
             # avoid breaking API, this is a side effect of the JSON optimization in #4316
             hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
-            yield key, tid, hsh
+            results << [key, tid, hsh]
           end
         end
       end
+
+      results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
     end
 
     # Note that #size is only as accurate as Sidekiq's heartbeat,

data/lib/sidekiq/cli.rb

@@ -33,8 +33,9 @@ module Sidekiq
     # Code within this method is not tested because it alters
     # global process state irreversibly.  PRs which improve the
     # test coverage of Sidekiq::CLI are welcomed.
-    def run
-      boot_system
+    def run(boot_app: true)
+      boot_application if boot_app
+
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
@@ -54,7 +55,7 @@ module Sidekiq
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
-      logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
@@ -163,7 +164,7 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
     UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
@@ -182,7 +183,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -224,8 +229,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -236,7 +240,7 @@ module Sidekiq
       Sidekiq.options
     end
 
-    def boot_system
+    def boot_application
       ENV["RACK_ENV"] = ENV["RAILS_ENV"] = environment
 
       if File.directory?(options[:require])
@@ -364,6 +368,8 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(:strict)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
@@ -375,13 +381,16 @@ module Sidekiq
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
       [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
 
     def rails_app?
-      defined?(::Rails)
+      defined?(::Rails) && ::Rails.respond_to?(:application)
     end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -90,16 +90,17 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
 
       at = items.delete("at")
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items["args"].map.with_index { |args, index|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
 
         result = process_single(items["class"], copy)
@@ -218,16 +219,20 @@ module Sidekiq
       end
     end
 
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
     def normalize_item(item)
       # 6.0.0 push_bulk bug, #4321
       # TODO Remove after a while...
       item.delete("at") if item.key?("at") && item["at"].nil?
 
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
+      validate(item)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
       # merge in the default sidekiq_options for the item's class and/or wrapped element
@@ -236,6 +241,8 @@ module Sidekiq
       defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
       item = defaults.merge(item)
 
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
+
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["jid"] ||= SecureRandom.hex(12)

data/lib/sidekiq/extensions/active_record.rb

@@ -6,9 +6,10 @@ module Sidekiq
   module Extensions
     ##
     # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.

data/lib/sidekiq/extensions/class_methods.rb

@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker

data/lib/sidekiq/fetch.rb

@@ -25,8 +25,10 @@ module Sidekiq
     }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
         @queues << TIMEOUT
@@ -38,24 +40,7 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
-      end
-    end
-
-    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
-    # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
@@ -76,5 +61,20 @@ module Sidekiq
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle!.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
   end
 end

data/lib/sidekiq/job_logger.rb

@@ -39,7 +39,7 @@ module Sidekiq
       # attribute to expose the underlying thing.
       h = {
         class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
       h[:tags] = job_hash["tags"] if job_hash["tags"]

data/lib/sidekiq/launcher.rb

@@ -16,12 +16,13 @@ module Sidekiq
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
-      proc { |me, data| "stopping" if me.stopping? },
+      proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
+      options[:fetch] ||= BasicFetch.new(options)
       @manager = Sidekiq::Manager.new(options)
       @poller = Sidekiq::Scheduled::Poller.new
       @done = false
@@ -56,7 +57,7 @@ module Sidekiq
 
       # Requeue everything in case there was a worker who grabbed work while stopped
       # This call is a no-op in Sidekiq but necessary for Sidekiq Pro.
-      strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+      strategy = @options[:fetch]
       strategy.bulk_requeue([], @options)
 
       clear_heartbeat
@@ -83,7 +84,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.pipelined do
           conn.srem("processes", identity)
-          conn.del("#{identity}:workers")
+          conn.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -96,6 +97,32 @@ module Sidekiq
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
@@ -118,7 +145,7 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
@@ -131,7 +158,7 @@ module Sidekiq
         _, exists, _, _, msg = Sidekiq.redis { |conn|
           conn.multi {
             conn.sadd("processes", key)
-            conn.exists(key)
+            conn.exists?(key)
             conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
@@ -146,7 +173,7 @@ module Sidekiq
         ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
@@ -163,7 +190,7 @@ module Sidekiq
           "concurrency" => @options[:concurrency],
           "queues" => @options[:queues].uniq,
           "labels" => @options[:labels],
-          "identity" => identity,
+          "identity" => identity
         }
       end
     end