sidekiq 4.2.10 → 7.3.10

data/bin/sidekiqload

@@ -1,154 +1,256 @@
 #!/usr/bin/env ruby
 
+#
+# bin/sidekiqload is a helpful script to load test and
+# performance tune Sidekiq's core. It creates 500,000 no-op
+# jobs and executes them as fast as possible.
+# Example Usage:
+#
+# > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
+# Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
+#
+# Use LATENCY=1 to get a more real world network setup
+# but you'll need to setup and start toxiproxy as noted below.
+#
+# Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
+# you can see the runtime performance difference between the two APIs.
+#
+# None of this script is considered a public API and may change over time.
+#
+
 # Quiet some warnings we see when running in warning mode:
 # RUBYOPT=-w bundle exec sidekiq
 $TESTING = false
+puts RUBY_DESCRIPTION
+puts(%w[THREADS LATENCY AJ PROFILE].map { |x| "#{x}: #{ENV[x] || "nil"}" }.join(", "))
 
-#require 'ruby-prof'
-Bundler.require(:default)
+require "ruby-prof" if ENV["PROFILE"]
+require "bundler/setup"
+Bundler.require(:default, :load_test)
 
-require_relative '../lib/sidekiq/cli'
-require_relative '../lib/sidekiq/launcher'
+latency = Integer(ENV["LATENCY"] || 1)
+if latency > 0
+  # brew tap shopify/shopify
+  # brew install toxiproxy
+  # run `toxiproxy-server` in a separate terminal window.
+  require "toxiproxy"
+  # simulate a non-localhost network for realer-world conditions.
+  # adding 1ms of network latency has an ENORMOUS impact on benchmarks
+  Toxiproxy.populate([{
+    name: "redis",
+    listen: "127.0.0.1:6380",
+    upstream: "127.0.0.1:6379"
+  }])
+end
 
-include Sidekiq::Util
+if ENV["AJ"]
+  require "active_job"
+  puts "Using ActiveJob #{ActiveJob::VERSION::STRING}"
+  ActiveJob::Base.queue_adapter = :sidekiq
+  ActiveJob::Base.logger.level = Logger::WARN
 
-Sidekiq.configure_server do |config|
-  #config.options[:concurrency] = 1
-  config.redis = { db: 13 }
-  config.options[:queues] << 'default'
-  config.logger.level = Logger::ERROR
-  config.average_scheduled_poll_interval = 2
-  config.reliable! if defined?(Sidekiq::Pro)
+  class LoadJob < ActiveJob::Base
+    def perform(string, idx, hash, ts = nil)
+      puts(Time.now.to_f - ts) if !ts.nil?
+    end
+  end
 end
 
 class LoadWorker
-  include Sidekiq::Worker
+  include Sidekiq::Job
+  $count = 0
+  $lock = Mutex.new
+ 
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1
   end
 
-  def perform(idx)
-    #raise idx.to_s if idx % 100 == 1
+  def perform(string, idx, hash, ts = nil)
+    $lock.synchronize do
+      $count += 1
+      if $count % 100_000 == 0
+        logger.warn("#{Time.now} Done #{$count}")
+      end
+    end
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
   end
 end
 
-# brew tap shopify/shopify
-# brew install toxiproxy
-# gem install toxiproxy
-#require 'toxiproxy'
-# simulate a non-localhost network for realer-world conditions.
-# adding 1ms of network latency has an ENORMOUS impact on benchmarks
-#Toxiproxy.populate([{
-    #"name": "redis",
-    #"listen": "127.0.0.1:6380",
-    #"upstream": "127.0.0.1:6379"
-#}])
-
-self_read, self_write = IO.pipe
-%w(INT TERM USR1 USR2 TTIN).each do |sig|
-  begin
-    trap sig do
-      self_write.puts(sig)
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+class Loader
+  def initialize
+    @iter = ENV["GC"] ? 10 : 500
+    @count = Integer(ENV["COUNT"] || 1_000)
+    @latency = Integer(ENV["LATENCY"] || 1)
+  end
+
+  def configure
+    @x = Sidekiq.configure_embed do |config|
+      config.redis = {db: 13, port: ((@latency > 0) ? 6380 : 6379)}
+      config.concurrency = Integer(ENV.fetch("THREADS", "10"))
+      # config.redis = { db: 13, port: 6380, driver: :hiredis}
+      config.queues = %w[default]
+      config.logger.level = Logger::WARN
+      config.average_scheduled_poll_interval = 2
+      config.reliable! if defined?(Sidekiq::Pro)
+    end
+
+    @self_read, @self_write = IO.pipe
+    %w[INT TERM TSTP TTIN].each do |sig|
+      trap sig do
+        @self_write.puts(sig)
+      end
+    rescue ArgumentError
+      puts "Signal #{sig} not supported"
     end
-  rescue ArgumentError
-    puts "Signal #{sig} not supported"
   end
-end
 
-Sidekiq.redis {|c| c.flushdb}
-def handle_signal(launcher, sig)
-  Sidekiq.logger.debug "Got #{sig} signal"
-  case sig
-  when 'INT'
-    # Handle Ctrl-C in JRuby like MRI
-    # http://jira.codehaus.org/browse/JRUBY-4637
-    raise Interrupt
-  when 'TERM'
-    # Heroku sends TERM and then waits 10 seconds for process to exit.
-    raise Interrupt
-  when 'USR1'
-    Sidekiq.logger.info "Received USR1, no longer accepting new work"
-    launcher.quiet
-  when 'USR2'
-    if Sidekiq.options[:logfile]
-      Sidekiq.logger.info "Received USR2, reopening log file"
-      Sidekiq::Logging.reopen_logs
+  def handle_signal(sig)
+    launcher = @x
+    Sidekiq.logger.debug "Got #{sig} signal"
+    case sig
+    when "INT"
+      # Handle Ctrl-C in JRuby like MRI
+      # http://jira.codehaus.org/browse/JRUBY-4637
+      raise Interrupt
+    when "TERM"
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      raise Interrupt
+    when "TSTP"
+      Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+      launcher.quiet
+    when "TTIN"
+      Thread.list.each do |thread|
+        Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
+        if thread.backtrace
+          Sidekiq.logger.warn thread.backtrace.join("\n")
+        else
+          Sidekiq.logger.warn "<no backtrace available>"
+        end
+      end
     end
-  when 'TTIN'
-    Thread.list.each do |thread|
-      Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
-      if thread.backtrace
-        Sidekiq.logger.warn thread.backtrace.join("\n")
-      else
-        Sidekiq.logger.warn "<no backtrace available>"
+  end
+
+  def setup
+    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.redis { |c| c.flushdb }
+    start = Time.now
+    if ENV["AJ"]
+      @iter.times do
+        ActiveJob.perform_all_later(@count.times.map do |idx|
+          LoadJob.new("mike", idx, {mike: "bob"})
+        end)
+      end
+    else
+      @iter.times do
+        arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
+        Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
       end
     end
+    Sidekiq.logger.warn "Created #{@count * @iter} jobs in #{Time.now - start} sec"
   end
-end
 
-def Process.rss
-  `ps -o rss= -p #{Process.pid}`.chomp.to_i
-end
+  def monitor
+    @monitor = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        qsize = Sidekiq.redis do |conn|
+          conn.llen "queue:default"
+        end
+        total = qsize
+        if total == 0
+          ending = Time.now - @start
+          size = @iter * @count
+          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
+          Sidekiq.logger.error("Now here's the latency for three jobs")
 
-iter = 10
-count = 10_000
+          if ENV["AJ"]
+            LoadJob.perform_later("", 1, {}, Time.now.to_f)
+            LoadJob.perform_later("", 2, {}, Time.now.to_f)
+            LoadJob.perform_later("", 3, {}, Time.now.to_f)
+          else
+            LoadWorker.perform_async("", 1, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 2, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 3, {}, Time.now.to_f)
+          end
 
-iter.times do
-  arr = Array.new(count) do
-    []
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+      end
+    end
   end
-  count.times do |idx|
-    arr[idx][0] = idx
+
+  def with_latency(latency, &block)
+    Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+    if latency > 0
+      Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
+    else
+      yield
+    end
   end
-  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
-end
-Sidekiq.logger.error "Created #{count*iter} jobs"
-
-Monitoring = Thread.new do
-  watchdog("monitor thread") do
-    while true
-      sleep 1
-      qsize, retries = Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.llen "queue:default"
-          conn.zcard "retry"
-        end
-      end.map(&:to_i)
-      total = qsize + retries
-      #GC.start
-      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
-      if total == 0
-        Sidekiq.logger.error("Done")
-        exit(0)
+
+  def run(name)
+    Sidekiq.logger.warn("Starting #{name}")
+    monitor
+
+    if ENV["PROFILE"]
+      RubyProf.exclude_threads = [@monitor]
+      RubyProf.start
+    elsif ENV["GC"]
+      GC.start
+      GC.compact
+      GC.disable
+      Sidekiq.logger.error("GC Start RSS: #{Process.rss}")
+    end
+    @start = Time.now
+    with_latency(@latency) do
+      @x.run
+
+      while (readable_io = IO.select([@self_read]))
+        signal = readable_io.first[0].gets.strip
+        handle_signal(signal)
       end
     end
+    # normal
+  rescue Interrupt
+  rescue => e
+    raise e if $DEBUG
+    warn e.message
+    warn e.backtrace.join("\n")
+    exit 1
+  ensure
+    @x.stop
   end
-end
 
-begin
-  #RubyProf::exclude_threads = [ Monitoring ]
-  #RubyProf.start
-  fire_event(:startup)
-  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
-  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
-    launcher = Sidekiq::Launcher.new(Sidekiq.options)
-    launcher.run
-
-    while readable_io = IO.select([self_read])
-      signal = readable_io.first[0].gets.strip
-      handle_signal(launcher, signal)
+  def done
+    Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
+    if ENV["PROFILE"]
+      Sidekiq.logger.error("Profiling...")
+      result = RubyProf.stop
+      printer = RubyProf::GraphHtmlPrinter.new(result)
+      printer.print(File.new("output.html", "w"), min_percent: 1)
     end
-  #end
-rescue SystemExit => e
-  #Sidekiq.logger.error("Profiling...")
-  #result = RubyProf.stop
-  #printer = RubyProf::GraphHtmlPrinter.new(result)
-  #printer.print(File.new("output.html", "w"), :min_percent => 1)
-  # normal
-rescue => e
-  raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
-  exit 1
+  end
 end
+
+ll = Loader.new
+ll.configure
+
+if ENV["WARM"]
+  ll.setup
+  ll.run("warmup")
+end
+
+ll.setup
+ll.run("load")
+ll.done

data/bin/sidekiqmon

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "sidekiq/monitor"
+
+# disable the Redis connection pool logging
+Sidekiq.default_configuration.logger.level = :warn
+
+section = "all"
+section = ARGV[0] if ARGV.size == 1
+
+Sidekiq::Monitor::Status.new.display(section)

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    # Explicitly remove the implementation existing in older rails'.
+    remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
+
+    # Sidekiq adapter for Active Job
+    #
+    # To use Sidekiq set the queue_adapter config to +:sidekiq+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :sidekiq
+    class SidekiqAdapter
+      # Defines whether enqueuing should happen implicitly to after commit when called
+      # from inside a transaction.
+      # @api private
+      def enqueue_after_transaction_commit?
+        true
+      end
+
+      # @api private
+      def enqueue(job)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_async(job.serialize)
+      end
+
+      # @api private
+      def enqueue_at(job, timestamp)
+        job.provider_job_id = JobWrapper.set(
+          wrapped: job.class,
+          queue: job.queue_name
+        ).perform_at(timestamp, job.serialize)
+      end
+
+      # @api private
+      def enqueue_all(jobs)
+        enqueued_count = 0
+        jobs.group_by(&:class).each do |job_class, same_class_jobs|
+          same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
+            immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
+
+            if immediate_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => immediate_jobs.map { |job| [job.serialize] }
+              )
+              enqueued_count += jids.compact.size
+            end
+
+            if scheduled_jobs.any?
+              jids = Sidekiq::Client.push_bulk(
+                "class" => JobWrapper,
+                "wrapped" => job_class,
+                "queue" => queue,
+                "args" => scheduled_jobs.map { |job| [job.serialize] },
+                "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
+              )
+              enqueued_count += jids.compact.size
+            end
+          end
+        end
+        enqueued_count
+      end
+
+      # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
+      # @api private
+      class JobWrapper < Sidekiq::ActiveJob::Wrapper
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/job_generator.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "rails/generators/named_base"
+
+module Sidekiq
+  module Generators # :nodoc:
+    class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
+      desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
+
+      check_class_collision suffix: "Job"
+
+      def self.default_generator_root
+        File.dirname(__FILE__)
+      end
+
+      def create_job_file
+        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+      end
+
+      def create_test_file
+        return unless test_framework
+
+        if test_framework == :rspec
+          create_job_spec
+        else
+          create_job_test
+        end
+      end
+
+      private
+
+      def create_job_spec
+        template_file = File.join(
+          "spec/sidekiq",
+          class_path,
+          "#{file_name}_job_spec.rb"
+        )
+        template "job_spec.rb.erb", template_file
+      end
+
+      def create_job_test
+        template_file = File.join(
+          "test/sidekiq",
+          class_path,
+          "#{file_name}_job_test.rb"
+        )
+        template "job_test.rb.erb", template_file
+      end
+
+      def file_name
+        @_file_name ||= super.sub(/_?job\z/i, "")
+      end
+
+      def test_framework
+        ::Rails.application.config.generators.options[:rails][:test_framework]
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb}

@@ -1,6 +1,6 @@
 <% module_namespacing do -%>
-class <%= class_name %>Worker
-  include Sidekiq::Worker
+class <%= class_name %>Job
+  include Sidekiq::Job
 
   def perform(*args)
     # Do something

data/lib/generators/sidekiq/templates/job_spec.rb.erb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Job, type: :job do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb}

@@ -1,6 +1,6 @@
 require 'test_helper'
 <% module_namespacing do -%>
-class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
+class <%= class_name %>JobTest < Minitest::Test
   def test_example
     skip "add some examples to (or delete) #{__FILE__}"
   end