sidekiq 6.0.0.pre1 → 6.0.0

data/bin/sidekiqload

@@ -13,8 +13,8 @@ require_relative '../lib/sidekiq/launcher'
 include Sidekiq::Util
 
 Sidekiq.configure_server do |config|
-  #config.options[:concurrency] = 1
-  config.redis = { db: 13 }
+  config.options[:concurrency] = 10
+  config.redis = { db: 13, port: 6380, driver: :hiredis }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -28,7 +28,8 @@ class LoadWorker
     1
   end
 
-  def perform(idx)
+  def perform(idx, ts=nil)
+    puts(Time.now.to_f - ts) if ts != nil
     #raise idx.to_s if idx % 100 == 1
   end
 end
@@ -36,14 +37,15 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
-#require 'toxiproxy'
+# run `toxiproxy-server` in a separate terminal window.
+require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
-#Toxiproxy.populate([{
-    #"name": "redis",
-    #"listen": "127.0.0.1:6380",
-    #"upstream": "127.0.0.1:6379"
-#}])
+Toxiproxy.populate([{
+    "name": "redis",
+    "listen": "127.0.0.1:6380",
+    "upstream": "127.0.0.1:6379"
+}])
 
 self_read, self_write = IO.pipe
 %w(INT TERM TSTP TTIN).each do |sig|
@@ -103,18 +105,20 @@ Sidekiq.logger.error "Created #{count*iter} jobs"
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 1
-      qsize, retries = Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.llen "queue:default"
-          conn.zcard "retry"
-        end
-      end.map(&:to_i)
-      total = qsize + retries
-      #GC.start
+      sleep 0.5
+      qsize = Sidekiq.redis do |conn|
+        conn.llen "queue:default"
+      end
+      total = qsize
       Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
-        Sidekiq.logger.error("Done")
+        Sidekiq.logger.error("Done, now here's the latency for three jobs")
+
+        LoadWorker.perform_async(1, Time.now.to_f)
+        LoadWorker.perform_async(2, Time.now.to_f)
+        LoadWorker.perform_async(3, Time.now.to_f)
+
+        sleep 0.2
         exit(0)
       end
     end
@@ -125,8 +129,8 @@ begin
   #RubyProf::exclude_threads = [ Monitoring ]
   #RubyProf.start
   fire_event(:startup)
-  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+  Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
     launcher = Sidekiq::Launcher.new(Sidekiq.options)
     launcher.run
 
@@ -134,7 +138,7 @@ begin
       signal = readable_io.first[0].gets.strip
       handle_signal(launcher, signal)
     end
-  #end
+  end
 rescue SystemExit => e
   #Sidekiq.logger.error("Profiling...")
   #result = RubyProf.stop

data/bin/sidekiqmon

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'sidekiq/monitor'
+
+if ARGV[0] == 'status'
+  Sidekiq::Monitor::Status.new.display(ARGV[1])
+else
+  Sidekiq::Monitor.print_usage
+end

data/lib/generators/sidekiq/templates/worker_test.rb.erb

@@ -1,6 +1,6 @@
 require 'test_helper'
 <% module_namespacing do -%>
-class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
+class <%= class_name %>WorkerTest < Minitest::Test
   def test_example
     skip "add some examples to (or delete) #{__FILE__}"
   end

data/lib/sidekiq.rb

@@ -119,7 +119,7 @@ module Sidekiq
       end
     rescue Redis::CommandError => ex
       # 2850 return fake version when INFO command has (probably) been renamed
-      raise unless ex.message =~ /unknown command/
+      raise unless /unknown command/.match?(ex.message)
       FAKE_INFO
     end
   end
@@ -154,7 +154,7 @@ module Sidekiq
 
   def self.default_worker_options=(hash)
     # stringify
-    @default_worker_options = default_worker_options.merge(Hash[hash.map {|k, v| [k.to_s, v]}])
+    @default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
   end
 
   def self.default_worker_options

data/lib/sidekiq/api.rb

@@ -86,14 +86,14 @@ module Sidekiq
 
       pipe2_res = Sidekiq.redis { |conn|
         conn.pipelined do
-          processes.each {|key| conn.hget(key, "busy") }
-          queues.each {|queue| conn.llen("queue:#{queue}") }
+          processes.each { |key| conn.hget(key, "busy") }
+          queues.each { |queue| conn.llen("queue:#{queue}") }
         end
       }
 
       s = processes.size
       workers_size = pipe2_res[0...s].map(&:to_i).inject(0, &:+)
-      enqueued     = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
+      enqueued = pipe2_res[s..-1].map(&:to_i).inject(0, &:+)
 
       default_queue_latency = if (entry = pipe1_res[6].first)
         job = begin
@@ -122,7 +122,7 @@ module Sidekiq
     end
 
     def reset(*stats)
-      all   = %w[failed processed]
+      all = %w[failed processed]
       stats = stats.empty? ? all : all & stats.flatten.compact.map(&:to_s)
 
       mset_args = []
@@ -274,7 +274,7 @@ module Sidekiq
 
       loop do
         range_start = page * page_size - deleted_size
-        range_end   = range_start + page_size - 1
+        range_end = range_start + page_size - 1
         entries = Sidekiq.redis { |conn|
           conn.lrange @rname, range_start, range_end
         }
@@ -563,7 +563,7 @@ module Sidekiq
 
       loop do
         range_start = page * page_size + offset_size
-        range_end   = range_start + page_size - 1
+        range_end = range_start + page_size - 1
         elements = Sidekiq.redis { |conn|
           conn.zrange name, range_start, range_end, with_scores: true
         }
@@ -713,7 +713,7 @@ module Sidekiq
 
   ##
   # Enumerates the set of Sidekiq processes which are actively working
-  # right now.  Each process send a heartbeat to Redis every 5 seconds
+  # right now.  Each process sends a heartbeat to Redis every 5 seconds
   # so this set should be relatively accurate, barring network partitions.
   #
   # Yields a Sidekiq::Process.
@@ -790,7 +790,7 @@ module Sidekiq
     # or Sidekiq Pro.
     def leader
       @leader ||= begin
-        x = Sidekiq.redis {|c| c.get("dear-leader") }
+        x = Sidekiq.redis { |c| c.get("dear-leader") }
         # need a non-falsy value so we can memoize
         x ||= ""
         x

data/lib/sidekiq/cli.rb

@@ -41,12 +41,6 @@ module Sidekiq
 
       self_read, self_write = IO.pipe
       sigs = %w[INT TERM TTIN TSTP]
-      # USR1 and USR2 don't work on the JVM
-      unless jruby?
-        sigs << "USR1"
-        sigs << "USR2"
-      end
-
       sigs.each do |sig|
         trap sig do
           self_write.write("#{sig}\n")
@@ -62,8 +56,7 @@ module Sidekiq
       # touch the connection pool so it is created before we
       # fire startup and start multithreading.
       ver = Sidekiq.redis_info["redis_version"]
-      raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < "2.8"
-      logger.warn "Sidekiq 6.0 will require Redis 4.0+, you are using Redis v#{ver}" if ver < "4"
+      raise "You are using Redis v#{ver}, Sidekiq requires Redis v4.0.0 or greater" if ver < "4"
 
       # Since the user can pass us a connection pool explicitly in the initializer, we
       # need to verify the size is large enough or else Sidekiq's performance is dramatically slowed.
@@ -104,9 +97,13 @@ module Sidekiq
       rescue Interrupt
         logger.info "Shutting down"
         launcher.stop
-        # Explicitly exit so busy Processor threads can't block
-        # process shutdown.
         logger.info "Bye!"
+
+        # Explicitly exit so busy Processor threads won't block process shutdown.
+        #
+        # NB: slow at_exit handlers will prevent a timely exit if they take
+        # a while to run. If Sidekiq is getting here but the process isn't exiting,
+        # use the TTIN signal to determine where things are stuck.
         exit(0)
       end
     end
@@ -150,17 +147,13 @@ module Sidekiq
       # TERM is the signal that Sidekiq must exit.
       # Heroku sends TERM and then waits 30 seconds for process to exit.
       "TERM" => ->(cli) { raise Interrupt },
-      "USR1" => ->(cli) {
-        Sidekiq.logger.info "Received USR1, no longer accepting new work"
-        cli.launcher.quiet
-      },
       "TSTP" => ->(cli) {
         Sidekiq.logger.info "Received TSTP, no longer accepting new work"
         cli.launcher.quiet
       },
       "TTIN" => ->(cli) {
         Thread.list.each do |thread|
-          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["sidekiq_label"]}"
+          Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread.name}"
           if thread.backtrace
             Sidekiq.logger.warn thread.backtrace.join("\n")
           else
@@ -276,7 +269,7 @@ module Sidekiq
       if !File.exist?(options[:require]) ||
           (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
-        logger.info "  Please point sidekiq to a Rails 4/5 application or a Ruby file  "
+        logger.info "  Please point Sidekiq to a Rails application or a Ruby file  "
         logger.info "  to load your worker classes with -r [DIR|FILE]."
         logger.info "=================================================================="
         logger.info @parser
@@ -297,7 +290,7 @@ module Sidekiq
         end
 
         o.on "-d", "--daemon", "Daemonize process" do |arg|
-          puts "WARNING: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
+          puts "ERROR: Daemonization mode was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
         end
 
         o.on "-e", "--environment ENV", "Application environment" do |arg|
@@ -330,11 +323,11 @@ module Sidekiq
         end
 
         o.on "-L", "--logfile PATH", "path to writable logfile" do |arg|
-          puts "WARNING: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
+          puts "ERROR: Logfile redirection was removed in Sidekiq 6.0, Sidekiq will only log to STDOUT"
         end
 
         o.on "-P", "--pidfile PATH", "path to pidfile" do |arg|
-          puts "WARNING: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
+          puts "ERROR: PID file creation was removed in Sidekiq 6.0, please use a proper process supervisor to start and manage your services"
         end
 
         o.on "-V", "--version", "Print version and exit" do |arg|
@@ -370,12 +363,6 @@ module Sidekiq
       opts = opts.merge(opts.delete(environment.to_sym) || {})
       parse_queues(opts, opts.delete(:queues) || [])
 
-      ns = opts.delete(:namespace)
-      if ns
-        # logger hasn't been initialized yet, puts is all we have.
-        puts("namespace should be set in your ruby initializer, is ignored in config file")
-        puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
-      end
       opts
     end
 

data/lib/sidekiq/client.rb

@@ -194,14 +194,14 @@ module Sidekiq
           [at, Sidekiq.dump_json(hash)]
         })
       else
-        q = payloads.first["queue"]
+        queue = payloads.first["queue"]
         now = Time.now.to_f
         to_push = payloads.map { |entry|
           entry["enqueued_at"] = now
           Sidekiq.dump_json(entry)
         }
-        conn.sadd("queues", q)
-        conn.lpush("queue:#{q}", to_push)
+        conn.sadd("queues", queue)
+        conn.lpush("queue:#{queue}", to_push)
       end
     end
 

data/lib/sidekiq/launcher.rb

@@ -5,10 +5,7 @@ require "sidekiq/fetch"
 require "sidekiq/scheduled"
 
 module Sidekiq
-  # The Launcher is a very simple Actor whose job is to
-  # start, monitor and stop the core Actors in Sidekiq.
-  # If any of these actors die, the Sidekiq process exits
-  # immediately.
+  # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
   class Launcher
     include Util
 
@@ -148,7 +145,7 @@ module Sidekiq
 
         return unless msg
 
-        ::Process.kill(msg, $PID)
+        ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")
@@ -163,7 +160,7 @@ module Sidekiq
         {
           "hostname" => hostname,
           "started_at" => Time.now.to_f,
-          "pid" => $PID,
+          "pid" => ::Process.pid,
           "tag" => @options[:tag] || "",
           "concurrency" => @options[:concurrency],
           "queues" => @options[:queues].uniq,

data/lib/sidekiq/manager.rb

@@ -112,7 +112,7 @@ module Sidekiq
       end
 
       if cleanup.size > 0
-        jobs = cleanup.map {|p| p.job }.compact
+        jobs = cleanup.map { |p| p.job }.compact
 
         logger.warn { "Terminating #{cleanup.size} busy worker threads" }
         logger.warn { "Work still in progress #{jobs.inspect}" }

data/lib/sidekiq/middleware/chain.rb

@@ -140,7 +140,7 @@ module Sidekiq
 
       def initialize(klass, *args)
         @klass = klass
-        @args  = args
+        @args = args
       end
 
       def make_new

data/lib/sidekiq/{ctl.rb → monitor.rb}

@@ -3,19 +3,20 @@
 require "fileutils"
 require "sidekiq/api"
 
-class Sidekiq::Ctl
+class Sidekiq::Monitor
   CMD = File.basename($PROGRAM_NAME)
 
   attr_reader :stage
 
   def self.print_usage
-    puts "#{CMD} - control Sidekiq from the command line."
+    puts "#{CMD} - monitor Sidekiq from the command line."
     puts
     puts "Usage: #{CMD} status <section>"
     puts
     puts "       <section> (optional) view a specific section of the status output"
-    puts "       Valid sections are: #{Sidekiq::Ctl::Status::VALID_SECTIONS.join(", ")}"
+    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
     puts
+    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
   end
 
   class Status

data/lib/sidekiq/processor.rb

@@ -4,6 +4,7 @@ require "sidekiq/util"
 require "sidekiq/fetch"
 require "sidekiq/job_logger"
 require "sidekiq/job_retry"
+
 module Sidekiq
   ##
   # The Processor is a standalone thread which:
@@ -125,7 +126,7 @@ module Sidekiq
               # the Reloader.  It handles code loading, db connection management, etc.
               # Effectively this block denotes a "unit of work" to Rails.
               @reloader.call do
-                klass  = constantize(job_hash["class"])
+                klass = constantize(job_hash["class"])
                 worker = klass.new
                 worker.jid = job_hash["jid"]
                 @retrier.local(worker, pristine, queue) do
@@ -153,21 +154,22 @@ module Sidekiq
         return work.acknowledge
       end
 
-      ack = true
+      ack = false
       begin
         dispatch(job_hash, queue) do |worker|
           Sidekiq.server_middleware.invoke(worker, job_hash, queue) do
             execute_job(worker, cloned(job_hash["args"]))
           end
         end
+        ack = true
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
         # we didn't properly finish it.
-        ack = false
       rescue Sidekiq::JobRetry::Handled => h
         # this is the common case: job raised error and Sidekiq::JobRetry::Handled
         # signals that we created a retry successfully.  We can acknowlege the job.
+        ack = true
         e = h.cause || h
         handle_exception(e, {context: "Job raised exception", job: job_hash, jobstr: jobstr})
         raise e
@@ -175,11 +177,15 @@ module Sidekiq
         # Unexpected error!  This is very bad and indicates an exception that got past
         # the retry subsystem (e.g. network partition).  We won't acknowledge the job
         # so it can be rescued when using Sidekiq Pro.
-        ack = false
         handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
         raise e
       ensure
-        work.acknowledge if ack
+        if ack
+          # We don't want a shutdown signal to interrupt job acknowledgment.
+          Thread.handle_interrupt(Sidekiq::Shutdown => :never) do
+            work.acknowledge
+          end
+        end
       end
     end
 

data/lib/sidekiq/rails.rb

@@ -1,15 +1,33 @@
 # frozen_string_literal: true
 
+require "sidekiq/worker"
+
 module Sidekiq
   class Rails < ::Rails::Engine
+    # By including the Options module, we allow AJs to directly control sidekiq features
+    # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
+    # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
+    # manually retried, don't automatically die, etc.
+    #
+    #   class SomeJob < ActiveJob::Base
+    #     queue_as :default
+    #     sidekiq_options retry: 3, backtrace: 10
+    #     def perform
+    #     end
+    #   end
+    initializer "sidekiq.active_job_integration" do
+      ActiveSupport.on_load(:active_job) do
+        include ::Sidekiq::Worker::Options unless respond_to?(:sidekiq_options)
+      end
+    end
+
+    # This hook happens after all initializers are run, just before returning
+    # from config/environment.rb back to sidekiq/cli.rb.
+    # We have to add the reloader after initialize to see if cache_classes has
+    # been turned on.
+    #
+    # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
-      # This hook happens after all initializers are run, just before returning
-      # from config/environment.rb back to sidekiq/cli.rb.
-      # We have to add the reloader after initialize to see if cache_classes has
-      # been turned on.
-      #
-      # None of this matters on the client-side, only within the Sidekiq process itself.
-      #
       Sidekiq.configure_server do |_|
         Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
       end