sidekiq 6.0.2 → 6.0.7

data/Pro-Changes.md

@@ -2,7 +2,13 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
-Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
+Please see [http://sidekiq.org](http://sidekiq.org) for more details and how to buy.
+
+HEAD
+---------
+
+- Update `constantize` for batch callbacks. [#4469]
+
 
 5.0.1
 ---------

data/bin/sidekiq

@@ -6,9 +6,28 @@ $TESTING = false
 
 require_relative '../lib/sidekiq/cli'
 
+def integrate_with_systemd
+  return unless ENV["NOTIFY_SOCKET"]
+
+  Sidekiq.configure_server do |config|
+    Sidekiq.logger.info "Enabling systemd notification integration"
+    require "sidekiq/sd_notify"
+    config.on(:startup) do
+      Sidekiq::SdNotify.ready
+    end
+    config.on(:shutdown) do
+      Sidekiq::SdNotify.stopping
+    end
+    Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
+  end
+end
+
 begin
   cli = Sidekiq::CLI.instance
   cli.parse
+
+  integrate_with_systemd
+
   cli.run
 rescue => e
   raise e if $DEBUG

data/lib/generators/sidekiq/worker_generator.rb

@@ -18,7 +18,7 @@ module Sidekiq
       def create_test_file
         return unless test_framework
 
-        if defined?(RSpec)
+        if test_framework == :rspec
           create_worker_spec
         else
           create_worker_test

data/lib/sidekiq.rb

@@ -30,16 +30,16 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: [],
+      heartbeat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-    reloader: proc { |&block| block.call },
+    reloader: proc { |&block| block.call }
   }
 
   DEFAULT_WORKER_OPTIONS = {
     "retry" => true,
-    "queue" => "default",
+    "queue" => "default"
   }
 
   FAKE_INFO = {
@@ -47,7 +47,7 @@ module Sidekiq
     "uptime_in_days" => "9999",
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
-    "used_memory_peak_human" => "9P",
+    "used_memory_peak_human" => "9P"
   }
 
   def self.❨╯°□°❩╯︵┻━┻
@@ -154,7 +154,7 @@ module Sidekiq
 
   def self.default_worker_options=(hash)
     # stringify
-    @default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
+    @default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
   end
 
   def self.default_worker_options
@@ -210,6 +210,10 @@ module Sidekiq
     @logger = logger
   end
 
+  def self.pro?
+    defined?(Sidekiq::Pro)
+  end
+
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.

data/lib/sidekiq/api.rb

@@ -105,7 +105,7 @@ module Sidekiq
 
         default_queue_latency: default_queue_latency,
         workers_size: workers_size,
-        enqueued: enqueued,
+        enqueued: enqueued
       }
     end
 
@@ -140,13 +140,8 @@ module Sidekiq
             end
           }
 
-          i = 0
-          array_of_arrays = queues.each_with_object({}) { |queue, memo|
-            memo[queue] = lengths[i]
-            i += 1
-          }.sort_by { |_, size| size }
-
-          Hash[array_of_arrays.reverse]
+          array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
+          Hash[array_of_arrays]
         end
       end
     end
@@ -168,18 +163,12 @@ module Sidekiq
       private
 
       def date_stat_hash(stat)
-        i = 0
         stat_hash = {}
-        keys = []
-        dates = []
-
-        while i < @days_previous
-          date = @start_date - i
-          datestr = date.strftime("%Y-%m-%d")
-          keys << "stat:#{stat}:#{datestr}"
-          dates << datestr
-          i += 1
-        end
+        dates = @start_date.downto(@start_date - @days_previous + 1).map { |date|
+          date.strftime("%Y-%m-%d")
+        }
+
+        keys = dates.map { |datestr| "stat:#{stat}:#{datestr}" }
 
         begin
           Sidekiq.redis do |conn|
@@ -284,7 +273,7 @@ module Sidekiq
     def clear
       Sidekiq.redis do |conn|
         conn.multi do
-          conn.del(@rname)
+          conn.unlink(@rname)
           conn.srem("queues", name)
         end
       end
@@ -478,7 +467,7 @@ module Sidekiq
 
     def reschedule(at)
       Sidekiq.redis do |conn|
-        conn.zincrby(@parent.name, at - @score, Sidekiq.dump_json(@item))
+        conn.zincrby(@parent.name, at.to_f - @score, Sidekiq.dump_json(@item))
       end
     end
 
@@ -523,7 +512,7 @@ module Sidekiq
         else
           # multiple jobs with the same score
           # find the one with the right JID and push it
-          hash = results.group_by { |message|
+          matched, nonmatched = results.partition { |message|
             if message.index(jid)
               msg = Sidekiq.load_json(message)
               msg["jid"] == jid
@@ -532,12 +521,12 @@ module Sidekiq
             end
           }
 
-          msg = hash.fetch(true, []).first
+          msg = matched.first
           yield msg if msg
 
           # push the rest back onto the sorted set
           conn.multi do
-            hash.fetch(false, []).each do |message|
+            nonmatched.each do |message|
               conn.zadd(parent.name, score.to_f.to_s, message)
             end
           end
@@ -573,7 +562,7 @@ module Sidekiq
 
     def clear
       Sidekiq.redis do |conn|
-        conn.del(name)
+        conn.unlink(name)
       end
     end
     alias_method :💣, :clear
@@ -785,10 +774,9 @@ module Sidekiq
         # the hash named key has an expiry of 60 seconds.
         # if it's not found, that means the process has not reported
         # in to Redis and probably died.
-        to_prune = []
-        heartbeats.each_with_index do |beat, i|
-          to_prune << procs[i] if beat.nil?
-        end
+        to_prune = procs.select.with_index { |proc, i|
+          heartbeats[i].nil?
+        }
         count = conn.srem("processes", to_prune) unless to_prune.empty?
       end
       count
@@ -938,7 +926,11 @@ module Sidekiq
           }
           next unless valid
           workers.each_pair do |tid, json|
-            yield key, tid, Sidekiq.load_json(json)
+            hsh = Sidekiq.load_json(json)
+            p = hsh["payload"]
+            # avoid breaking API, this is a side effect of the JSON optimization in #4316
+            hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
+            yield key, tid, hsh
           end
         end
       end

data/lib/sidekiq/cli.rb

@@ -38,6 +38,7 @@ module Sidekiq
       if environment == "development" && $stdout.tty? && Sidekiq.log_formatter.is_a?(Sidekiq::Logger::Formatters::Pretty)
         print_banner
       end
+      logger.info "Booted Rails #{::Rails.version} application in #{environment} environment" if rails_app?
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
@@ -53,12 +54,12 @@ module Sidekiq
 
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
-      logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
+      logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
 
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info["redis_version"]
-      raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
+      raise "You are connecting to Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -162,7 +163,7 @@ module Sidekiq
             Sidekiq.logger.warn "<no backtrace available>"
           end
         end
-      },
+      }
     }
     UNHANDLED_SIGNAL_HANDLER = ->(cli) { Sidekiq.logger.info "No signal handler registered, ignoring" }
     SIGNAL_HANDLERS.default = UNHANDLED_SIGNAL_HANDLER
@@ -181,7 +182,11 @@ module Sidekiq
     end
 
     def set_environment(cli_env)
-      @environment = cli_env || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      # See #984 for discussion.
+      # APP_ENV is now the preferred ENV term since it is not tech-specific.
+      # Both Sinatra 2.0+ and Sidekiq support this term.
+      # RAILS_ENV and RACK_ENV are there for legacy support.
+      @environment = cli_env || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
     end
 
     def symbolize_keys_deep!(hash)
@@ -378,5 +383,11 @@ module Sidekiq
       [weight.to_i, 1].max.times { opts[:queues] << queue }
       opts[:strict] = false if weight.to_i > 0
     end
+
+    def rails_app?
+      defined?(::Rails) && ::Rails.respond_to?(:application)
+    end
   end
 end
+
+require "sidekiq/systemd"

data/lib/sidekiq/client.rb

@@ -99,8 +99,8 @@ module Sidekiq
 
       normed = normalize_item(items)
       payloads = items["args"].map.with_index { |args, index|
-        single_at = at.is_a?(Array) ? at[index] : at
-        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "at" => single_at, "enqueued_at" => Time.now.to_f)
+        copy = normed.merge("args" => args, "jid" => SecureRandom.hex(12), "enqueued_at" => Time.now.to_f)
+        copy["at"] = (at.is_a?(Array) ? at[index] : at) if at
 
         result = process_single(items["class"], copy)
         result || nil
@@ -193,7 +193,7 @@ module Sidekiq
     end
 
     def atomic_push(conn, payloads)
-      if payloads.first["at"]
+      if payloads.first.key?("at")
         conn.zadd("schedule", payloads.map { |hash|
           at = hash.delete("at").to_s
           [at, Sidekiq.dump_json(hash)]
@@ -219,6 +219,10 @@ module Sidekiq
     end
 
     def normalize_item(item)
+      # 6.0.0 push_bulk bug, #4321
+      # TODO Remove after a while...
+      item.delete("at") if item.key?("at") && item["at"].nil?
+
       raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.key?("class") && item.key?("args")
       raise(ArgumentError, "Job args must be an Array") unless item["args"].is_a?(Array)
       raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item["class"].is_a?(Class) || item["class"].is_a?(String)
@@ -226,13 +230,19 @@ module Sidekiq
       raise(ArgumentError, "Job tags must be an Array") if item["tags"] && !item["tags"].is_a?(Array)
       # raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
 
-      normalized_hash(item["class"])
-        .each { |key, value| item[key] = value if item[key].nil? }
+      # merge in the default sidekiq_options for the item's class and/or wrapped element
+      # this allows ActiveJobs to control sidekiq_options too.
+      defaults = normalized_hash(item["class"])
+      defaults = defaults.merge(item["wrapped"].get_sidekiq_options) if item["wrapped"].respond_to?("get_sidekiq_options")
+      item = defaults.merge(item)
+
+      raise(ArgumentError, "Job must include a valid queue name") if item["queue"].nil? || item["queue"] == ""
 
       item["class"] = item["class"].to_s
       item["queue"] = item["queue"].to_s
       item["jid"] ||= SecureRandom.hex(12)
       item["created_at"] ||= Time.now.to_f
+
       item
     end
 

data/lib/sidekiq/job_logger.rb

@@ -39,7 +39,7 @@ module Sidekiq
       # attribute to expose the underlying thing.
       h = {
         class: job_hash["wrapped"] || job_hash["class"],
-        jid: job_hash["jid"],
+        jid: job_hash["jid"]
       }
       h[:bid] = job_hash["bid"] if job_hash["bid"]
       h[:tags] = job_hash["tags"] if job_hash["tags"]

data/lib/sidekiq/job_retry.rb

@@ -189,13 +189,13 @@ module Sidekiq
         handle_exception(e, {context: "Error calling retries_exhausted", job: msg})
       end
 
+      send_to_morgue(msg) unless msg["dead"] == false
+
       Sidekiq.death_handlers.each do |handler|
         handler.call(msg, exception)
       rescue => e
         handle_exception(e, {context: "Error calling death handler", job: msg})
       end
-
-      send_to_morgue(msg) unless msg["dead"] == false
     end
 
     def send_to_morgue(msg)

data/lib/sidekiq/launcher.rb

@@ -16,7 +16,7 @@ module Sidekiq
       proc { Sidekiq::VERSION },
       proc { |me, data| data["tag"] },
       proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
-      proc { |me, data| "stopping" if me.stopping? },
+      proc { |me, data| "stopping" if me.stopping? }
     ]
 
     attr_accessor :manager, :poller, :fetcher
@@ -83,7 +83,7 @@ module Sidekiq
       Sidekiq.redis do |conn|
         conn.pipelined do
           conn.srem("processes", identity)
-          conn.del("#{identity}:workers")
+          conn.unlink("#{identity}:workers")
         end
       end
     rescue
@@ -96,6 +96,32 @@ module Sidekiq
       ❤
     end
 
+    def self.flush_stats
+      fails = Processor::FAILURE.reset
+      procd = Processor::PROCESSED.reset
+      return if fails + procd == 0
+
+      nowdate = Time.now.utc.strftime("%Y-%m-%d")
+      begin
+        Sidekiq.redis do |conn|
+          conn.pipelined do
+            conn.incrby("stat:processed", procd)
+            conn.incrby("stat:processed:#{nowdate}", procd)
+            conn.expire("stat:processed:#{nowdate}", STATS_TTL)
+
+            conn.incrby("stat:failed", fails)
+            conn.incrby("stat:failed:#{nowdate}", fails)
+            conn.expire("stat:failed:#{nowdate}", STATS_TTL)
+          end
+        end
+      rescue => ex
+        # we're exiting the process, things might be shut down so don't
+        # try to handle the exception
+        Sidekiq.logger.warn("Unable to flush stats: #{ex}")
+      end
+    end
+    at_exit(&method(:flush_stats))
+
     def ❤
       key = identity
       fails = procd = 0
@@ -118,7 +144,7 @@ module Sidekiq
             conn.incrby("stat:failed:#{nowdate}", fails)
             conn.expire("stat:failed:#{nowdate}", STATS_TTL)
 
-            conn.del(workers_key)
+            conn.unlink(workers_key)
             curstate.each_pair do |tid, hash|
               conn.hset(workers_key, tid, Sidekiq.dump_json(hash))
             end
@@ -146,7 +172,7 @@ module Sidekiq
         ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
-        logger.error("heartbeat: #{e.message}")
+        logger.error("heartbeat: #{e}")
         # don't lose the counts if there was a network issue
         Processor::PROCESSED.incr(procd)
         Processor::FAILURE.incr(fails)
@@ -163,7 +189,7 @@ module Sidekiq
           "concurrency" => @options[:concurrency],
           "queues" => @options[:queues].uniq,
           "labels" => @options[:labels],
-          "identity" => identity,
+          "identity" => identity
         }
       end
     end

data/lib/sidekiq/logger.rb

@@ -23,7 +23,7 @@ module Sidekiq
       "info" => 1,
       "warn" => 2,
       "error" => 3,
-      "fatal" => 4,
+      "fatal" => 4
     }
     LEVELS.default_proc = proc do |_, level|
       Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
@@ -31,23 +31,23 @@ module Sidekiq
     end
 
     def debug?
-      level >= 0
+      level <= 0
     end
 
     def info?
-      level >= 1
+      level <= 1
     end
 
     def warn?
-      level >= 2
+      level <= 2
     end
 
     def error?
-      level >= 3
+      level <= 3
     end
 
     def fatal?
-      level >= 4
+      level <= 4
     end
 
     def local_level
@@ -83,7 +83,7 @@ module Sidekiq
     # Redefined to check severity against #level, and thus the thread-local level, rather than +@level+.
     # FIXME: Remove when the minimum Ruby version supports overriding Logger#level.
     def add(severity, message = nil, progname = nil, &block)
-      severity ||= UNKNOWN
+      severity ||= ::Logger::UNKNOWN
       progname ||= @progname
 
       return true if @logdev.nil? || severity < level
@@ -104,7 +104,7 @@ module Sidekiq
   class Logger < ::Logger
     include LoggingUtils
 
-    def initialize(*args)
+    def initialize(*args, **kwargs)
       super
       self.formatter = Sidekiq.log_formatter
     end
@@ -152,7 +152,7 @@ module Sidekiq
             pid: ::Process.pid,
             tid: tid,
             lvl: severity,
-            msg: message,
+            msg: message
           }
           c = ctx
           hash["ctx"] = c unless c.empty?