sidekiq 6.0.3 → 6.1.0

data/Pro-Changes.md

@@ -2,7 +2,21 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
+Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy.
+
+5.1.0
+---------
+
+- Remove old Statsd metrics with `WorkerName` in the name [#4377]
+```
+job.WorkerName.count -> job.count with tag worker:WorkerName
+job.WorkerName.perform -> job.perform with tag worker:WorkerName
+job.WorkerName.failure -> job.failure with tag worker:WorkerName
+```
+- Remove `concurrent-ruby` gem dependency [#4586]
+- Update `constantize` for batch callbacks. [#4469]
+- Add queue tag to `jobs.recovered.fetch` metric [#4594]
+- Refactor Pro's fetch infrastructure [#4602]
 
 5.0.1
 ---------

data/README.md

@@ -2,11 +2,8 @@ Sidekiq
 ==============
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
-[![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
-[![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
+[![Codecov](https://codecov.io/gh/mperham/sidekiq/branch/master/graph/badge.svg)](https://codecov.io/gh/mperham/sidekiq)
 [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
-[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
-
 
 Simple, efficient background processing for Ruby.
 
@@ -94,4 +91,4 @@ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for
 Author
 -----------------
 
-Mike Perham, [@mperham@mastodon.xyz](https://mastodon.xyz/@mperham) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
+Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)

data/bin/sidekiq

@@ -6,13 +6,37 @@ $TESTING = false
 
 require_relative '../lib/sidekiq/cli'
 
+def integrate_with_systemd
+  return unless ENV["NOTIFY_SOCKET"]
+
+  Sidekiq.configure_server do |config|
+    Sidekiq.logger.info "Enabling systemd notification integration"
+    require "sidekiq/sd_notify"
+    config.on(:startup) do
+      Sidekiq::SdNotify.ready
+    end
+    config.on(:shutdown) do
+      Sidekiq::SdNotify.stopping
+    end
+    Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
+  end
+end
+
 begin
   cli = Sidekiq::CLI.instance
   cli.parse
+
+  integrate_with_systemd
+
   cli.run
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  if Sidekiq.error_handlers.length == 0
+    STDERR.puts e.message
+    STDERR.puts e.backtrace.join("\n")
+  else
+    cli.handle_exception e
+  end
+
   exit 1
 end

data/lib/generators/sidekiq/worker_generator.rb

@@ -18,7 +18,7 @@ module Sidekiq
       def create_test_file
         return unless test_framework
 
-        if defined?(RSpec)
+        if test_framework == :rspec
           create_worker_spec
         else
           create_worker_test

data/lib/sidekiq.rb

@@ -20,6 +20,7 @@ module Sidekiq
     labels: [],
     concurrency: 10,
     require: ".",
+    strict: true,
     environment: nil,
     timeout: 25,
     poll_interval_average: nil,
@@ -30,16 +31,16 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: [],
+      heartbeat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-    reloader: proc { |&block| block.call },
+    reloader: proc { |&block| block.call }
   }
 
   DEFAULT_WORKER_OPTIONS = {
     "retry" => true,
-    "queue" => "default",
+    "queue" => "default"
   }
 
   FAKE_INFO = {
@@ -47,7 +48,7 @@ module Sidekiq
     "uptime_in_days" => "9999",
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
-    "used_memory_peak_human" => "9P",
+    "used_memory_peak_human" => "9P"
   }
 
   def self.❨╯°□°❩╯︵┻━┻
@@ -95,10 +96,11 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::CommandError => ex
+      rescue Redis::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
-        if retryable && ex.message =~ /READONLY/
+        # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+        if retryable && ex.message =~ /READONLY|NOREPLICAS/
           conn.disconnect!
           retryable = false
           retry
@@ -154,7 +156,7 @@ module Sidekiq
 
   def self.default_worker_options=(hash)
     # stringify
-    @default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
+    @default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
   end
 
   def self.default_worker_options
@@ -210,6 +212,10 @@ module Sidekiq
     @logger = logger
   end
 
+  def self.pro?
+    defined?(Sidekiq::Pro)
+  end
+
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.

data/lib/sidekiq/api.rb

@@ -105,7 +105,7 @@ module Sidekiq
 
         default_queue_latency: default_queue_latency,
         workers_size: workers_size,
-        enqueued: enqueued,
+        enqueued: enqueued
       }
     end
 
@@ -273,7 +273,7 @@ module Sidekiq
     def clear
       Sidekiq.redis do |conn|
         conn.multi do
-          conn.del(@rname)
+          conn.unlink(@rname)
           conn.srem("queues", name)
         end
       end
@@ -562,7 +562,7 @@ module Sidekiq
 
     def clear
       Sidekiq.redis do |conn|
-        conn.del(name)
+        conn.unlink(name)
       end
     end
     alias_method :💣, :clear
@@ -921,12 +921,16 @@ module Sidekiq
         procs = conn.sscan_each("processes").to_a
         procs.sort.each do |key|
           valid, workers = conn.pipelined {
-            conn.exists(key)
+            conn.exists?(key)
             conn.hgetall("#{key}:workers")
           }
           next unless valid
           workers.each_pair do |tid, json|
-            yield key, tid, Sidekiq.load_json(json)
+            hsh = Sidekiq.load_json(json)
+            p = hsh["payload"]
+            # avoid breaking API, this is a side effect of the JSON optimization in #4316
+            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
+            yield key, tid, hsh
           end
         end
       end

data/lib/sidekiq/cli.rb

@@ -38,6 +38,7 @@ module Sidekiq
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
@@ -53,7 +54,7 @@ module Sidekiq
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
-      logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
@@ -162,7 +163,7 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
     UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
@@ -181,7 +182,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -223,8 +228,7 @@ module Sidekiq
       opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file]
 
       # set defaults
-      opts[:queues] = ["default"] if opts[:queues].nil? || opts[:queues].empty?
-      opts[:strict] = true if opts[:strict].nil?
+      opts[:queues] = ["default"] if opts[:queues].nil?
       opts[:concurrency] = Integer(ENV["RAILS_MAX_THREADS"]) if opts[:concurrency].nil? && ENV["RAILS_MAX_THREADS"]
 
       # merge with defaults
@@ -353,6 +357,12 @@ module Sidekiq
       Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
     end
 
+    INTERNAL_OPTIONS = [
+      # These are options that are set internally and cannot be
+      # set via the config file or command line arguments.
+      :strict
+    ]
+
     def parse_config(path)
       opts = YAML.load(ERB.new(File.read(path)).result) || {}
 
@@ -363,6 +373,8 @@ module Sidekiq
       end
 
       opts = opts.merge(opts.delete(environment.to_sym) || {})
+      opts.delete(*INTERNAL_OPTIONS)
+
       parse_queues(opts, opts.delete(:queues) || [])
 
       opts
@@ -374,9 +386,16 @@ module Sidekiq
 
     def parse_queue(opts, queue, weight = nil)
       opts[:queues] ||= []
+      opts[:strict] = true if opts[:strict].nil?
       raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
       [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -90,16 +90,17 @@ module Sidekiq
     # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
     # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
-      arg = items["args"].first
-      return [] unless arg # no jobs to push
-      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless arg.is_a?(Array)
+      args = items["args"]
+      raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" unless args.is_a?(Array) && args.all?(Array)
+      return [] if args.empty? # no jobs to push
 
       at = items.delete("at")
       raise ArgumentError, "Job 'at' must be a Numeric or an Array of Numeric timestamps" if at && (Array(at).empty? || !Array(at).all?(Numeric))
+      raise ArgumentError, "Job 'at' Array must have same size as 'args' Array" if at.is_a?(Array) && at.size != args.size
 
       normed = normalize_item(items)
-      payloads = items["args"].map.with_index { |args, index|
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+      payloads = args.map.with_index { |job_args, index|
+        copy = normed.merge("args" => job_args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
         copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
 
         result = process_single(items["class"], copy)
@@ -218,25 +219,35 @@ module Sidekiq
       end
     end
 
+    def validate(item)
+      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: `#{item}`") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
+      raise(ArgumentError, "Job args must be an Array: `#{item}`") unless item["args"].is_a?(Array)
+      raise(ArgumentError, "Job class must be either a Class or String representation of the class name: `#{item}`") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
+      raise(ArgumentError, "Job 'at' must be a Numeric timestamp: `#{item}`") if item.key?("at") && !item["at"].is_a?(Numeric)
+      raise(ArgumentError, "Job tags must be an Array: `#{item}`") if item["tags"] && !item["tags"].is_a?(Array)
+    end
+
     def normalize_item(item)
       # 6.0.0 push_bulk bug, #4321
       # TODO Remove after a while...
       item.delete("at") if item.key?("at") && item["at"].nil?
 
-      raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
-      raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
-      raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
-      raise(ArgumentError, "Job 'at' must be a Numeric timestamp") if item.key?("at") && !item["at"].is_a?(Numeric)
-      raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
+      validate(item)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item["class"])
-        .each { |key, value| item[key] = value if item[key].nil? }
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["jid"] ||= SecureRandom.hex(12)
       item["created_at"] ||= Time.now.to_f
+
       item
     end
 

data/lib/sidekiq/extensions/active_record.rb

@@ -6,9 +6,10 @@ module Sidekiq
   module Extensions
     ##
     # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
-    # execution to Sidekiq.  Examples:
+    # execution to Sidekiq.
     #
-    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    # @example
+    #   User.recent_signups.each { |user| user.delay.mark_as_awesome }
     #
     # Please note, this is not recommended as this will serialize the entire
     # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.

data/lib/sidekiq/extensions/class_methods.rb

@@ -5,11 +5,12 @@ require "sidekiq/extensions/generic_proxy"
 module Sidekiq
   module Extensions
     ##
-    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
-    # execution to Sidekiq.  Examples:
+    # Adds `delay`, `delay_for` and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.
     #
-    # User.delay.delete_inactive
-    # Wikipedia.delay.download_changes_for(Date.today)
+    # @example
+    #   User.delay.delete_inactive
+    #   Wikipedia.delay.download_changes_for(Date.today)
     #
     class DelayedClass
       include Sidekiq::Worker

data/lib/sidekiq/fetch.rb

@@ -25,8 +25,10 @@ module Sidekiq
     }
 
     def initialize(options)
-      @strictly_ordered_queues = !!options[:strict]
-      @queues = options[:queues].map { |q| "queue:#{q}" }
+      raise ArgumentError, "missing queue list" unless options[:queues]
+      @options = options
+      @strictly_ordered_queues = !!@options[:strict]
+      @queues = @options[:queues].map { |q| "queue:#{q}" }
       if @strictly_ordered_queues
         @queues.uniq!
         @queues << TIMEOUT
@@ -38,24 +40,9 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      if @strictly_ordered_queues
-        @queues
-      else
-        queues = @queues.shuffle!.uniq
-        queues << TIMEOUT
-        queues
-      end
-    end
-
     # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
     # an instance method will make it async to the Fetcher actor
-    def self.bulk_requeue(inprogress, options)
+    def bulk_requeue(inprogress, options)
       return if inprogress.empty?
 
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
@@ -76,5 +63,20 @@ module Sidekiq
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle!.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
   end
 end