sidekiq 7.1.4 → 8.0.9

data/bin/multi_queue_bench

@@ -0,0 +1,271 @@
+#!/usr/bin/env ruby
+
+#
+# bin/bench is a helpful script to load test and
+# performance tune Sidekiq's core. It's a configurable script,
+# which accepts the following parameters as ENV variables.
+#
+# QUEUES
+# Number of queues to consume from. Default is 8
+#
+# PROCESSES
+# The number of processes this benchmark will create. Each process, consumes
+# from one of the available queues. When processes are more than the number of
+# queues, they are distributed to processes in round robin. Default is 8
+#
+# ELEMENTS
+# Number of jobs to push to each queue. Default is 1000
+#
+# ITERATIONS
+# Each queue pushes ITERATIONS times ELEMENTS jobs. Default is 1000
+#
+# PORT
+# The port of the Dragonfly instance. Default is 6379
+#
+# IP
+# The ip of the Dragonfly instance. Default is 127.0.0.1
+#
+# Example Usage:
+#
+# > RUBY_YJIT_ENABLE=1 THREADS=10 PROCESSES=8 QUEUES=8 bin/multi_queue_bench
+#
+# None of this script is considered a public API and may change over time.
+#
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+puts RUBY_DESCRIPTION
+
+require "bundler/setup"
+Bundler.require(:default, :load_test)
+
+class LoadWorker
+  include Sidekiq::Job
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx, ts = nil)
+    puts(Time.now.to_f - ts) if !ts.nil?
+    # raise idx.to_s if idx % 100 == 1
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+$iterations = ENV["ITERATIONS"] ? Integer(ENV["ITERATIONS"]) : 1_000
+$elements = ENV["ELEMENTS"] ? Integer(ENV["ELEMENTS"]) : 1_000
+$port = ENV["PORT"] ? Integer(ENV["PORT"]) : 6379
+$ip = ENV["IP"] ? String(ENV["IP"]) : "127.0.0.1"
+
+class Loader
+  def initialize
+    @iter =  $iterations
+    @count = $elements
+  end
+
+  def configure(queue)
+    @x = Sidekiq.configure_embed do |config|
+      config.redis = {db: 0, host: $ip, port: $port}
+      config.concurrency = Integer(ENV.fetch("THREADS", "30"))
+      config.queues = queue
+      config.logger.level = Logger::WARN
+      config.average_scheduled_poll_interval = 2
+      config.reliable! if defined?(Sidekiq::Pro)
+    end
+
+    @self_read, @self_write = IO.pipe
+    %w[INT TERM TSTP TTIN].each do |sig|
+      trap sig do
+        @self_write.puts(sig)
+      end
+    rescue ArgumentError
+      puts "Signal #{sig} not supported"
+    end
+  end
+
+  def handle_signal(sig)
+    launcher = @x
+    Sidekiq.logger.debug "Got #{sig} signal"
+    case sig
+    when "INT"
+      # Handle Ctrl-C in JRuby like MRI
+      # http://jira.codehaus.org/browse/JRUBY-4637
+      raise Interrupt
+    when "TERM"
+      # Heroku sends TERM and then waits 30 seconds for process to exit.
+      raise Interrupt
+    when "TSTP"
+      Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+      launcher.quiet
+    when "TTIN"
+      Thread.list.each do |thread|
+        Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
+        if thread.backtrace
+          Sidekiq.logger.warn thread.backtrace.join("\n")
+        else
+          Sidekiq.logger.warn "<no backtrace available>"
+        end
+      end
+    end
+  end
+
+  def setup(queue)
+    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.logger.error("Pushing work to queue: #{queue}")
+    start = Time.now
+    @iter.times do
+      arr = Array.new(@count) { |idx| [idx] }
+      # Sidekiq always prepends "queue:" to the queue name,
+      # that's why we pass 'q1', 'q2', etc instead of 'queue:q1'
+      Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr, "queue" => queue)
+      $stdout.write "."
+    end
+    puts "Done"
+  end
+
+  def monitor_single(queue)
+    q = "queue:#{queue}"
+    @monitor_single = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        total = Sidekiq.redis do |conn|
+          conn.llen q
+        end
+
+        if total == 0
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+
+      end
+    end
+  end
+
+  def monitor_all(queues)
+    @monitor_all = Thread.new do
+      GC.start
+      loop do
+        sleep 0.2
+        qsize = 0
+        queues.each do |q|
+          tmp = Sidekiq.redis do |conn|
+            conn.llen q
+          end
+          qsize = qsize + tmp
+        end
+        total = qsize
+
+        if total == 0
+          ending = Time.now - @start
+          size = @iter * @count * queues.length()
+          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
+
+          sleep 0.1
+          @x.stop
+          Process.kill("INT", $$)
+          break
+        end
+      end
+    end
+  end
+
+  def run(queues, queue, monitor_all_queues)
+    Sidekiq.logger.warn("Consuming from #{queue}")
+    if monitor_all_queues
+      monitor_all(queues)
+    else
+      monitor_single(queue)
+    end
+
+    @start = Time.now
+    @x.run
+
+    while (readable_io = IO.select([@self_read]))
+      signal = readable_io.first[0].gets.strip
+      handle_signal(signal)
+    end
+    # normal
+  rescue Interrupt
+  rescue => e
+    raise e if $DEBUG
+    warn e.message
+    warn e.backtrace.join("\n")
+    exit 1
+  ensure
+    @x.stop
+  end
+end
+
+def setup(queue)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.setup(queue)
+end
+
+def consume(queues, queue, monitor_all_queues)
+  ll = Loader.new
+  ll.configure(queue)
+  ll.run(queues, queue, monitor_all_queues)
+end
+
+# We assign one queue to each sidekiq process
+def run(number_of_processes, total_queues)
+  read_stream, write_stream = IO.pipe
+
+  queues = []
+  (0..total_queues-1).each do |idx|
+    queues.push("queue:q#{idx}")
+  end
+
+  Sidekiq.logger.info("Queues are: #{queues}")
+
+  # Produce
+  start = Time.now
+  (0..total_queues-1).each do |idx|
+    Process.fork do
+      queue_num = "q#{idx}"
+      setup(queue_num)
+    end
+  end
+
+  queue_sz = $iterations * $elements * total_queues
+  Process.waitall
+
+  ending = Time.now - start
+  #Sidekiq.logger.info("Pushed #{queue_sz} in #{ending} secs")
+
+  # Consume
+  (0..number_of_processes-1).each do |idx|
+    Process.fork do
+      # First process only consumes from it's own queue but monitors all queues.
+      # It works as a synchronization point. Once all processes finish
+      # (that is, when all queues are emptied) it prints the the stats.
+      if idx == 0
+        queue = "q#{idx}"
+        consume(queues, queue, true)
+      else
+        queue = "q#{idx % total_queues}"
+        consume(queues, queue, false)
+      end
+    end
+  end
+
+  Process.waitall
+  write_stream.close
+  results = read_stream.read
+  read_stream.close
+end
+
+$total_processes = ENV["PROCESSES"] ? Integer(ENV["PROCESSES"]) : 8;
+$total_queues = ENV["QUEUES"] ? Integer(ENV["QUEUES"]) : 8;
+
+run($total_processes, $total_queues)

data/bin/sidekiqload

@@ -28,7 +28,7 @@ require "ruby-prof" if ENV["PROFILE"]
 require "bundler/setup"
 Bundler.require(:default, :load_test)
 
-latency = Integer(ENV["LATENCY"] || 1)
+latency = Integer(ENV["LATENCY"] || 0)
 if latency > 0
   # brew tap shopify/shopify
   # brew install toxiproxy
@@ -50,7 +50,7 @@ if ENV["AJ"]
   ActiveJob::Base.logger.level = Logger::WARN
 
   class LoadJob < ActiveJob::Base
-    def perform(idx, ts = nil)
+    def perform(string, idx, hash, ts = nil)
       puts(Time.now.to_f - ts) if !ts.nil?
     end
   end
@@ -58,12 +58,21 @@ end
 
 class LoadWorker
   include Sidekiq::Job
+  $count = 0
+  $lock = Mutex.new
+ 
   sidekiq_options retry: 1
   sidekiq_retry_in do |x|
     1
   end
 
-  def perform(idx, ts = nil)
+  def perform(string, idx, hash, ts = nil)
+    $lock.synchronize do
+      $count += 1
+      if $count % 100_000 == 0
+        logger.warn("#{Time.now} Done #{$count}")
+      end
+    end
     puts(Time.now.to_f - ts) if !ts.nil?
     # raise idx.to_s if idx % 100 == 1
   end
@@ -77,7 +86,7 @@ class Loader
   def initialize
     @iter = ENV["GC"] ? 10 : 500
     @count = Integer(ENV["COUNT"] || 1_000)
-    @latency = Integer(ENV["LATENCY"] || 1)
+    @latency = Integer(ENV["LATENCY"] || 0)
   end
 
   def configure
@@ -128,18 +137,18 @@ class Loader
   end
 
   def setup
-    Sidekiq.logger.error("Setup RSS: #{Process.rss}")
+    Sidekiq.logger.warn("Setup RSS: #{Process.rss}")
     Sidekiq.redis { |c| c.flushdb }
     start = Time.now
     if ENV["AJ"]
       @iter.times do
-        @count.times do |idx|
-          LoadJob.perform_later(idx)
-        end
+        ActiveJob.perform_all_later(@count.times.map do |idx|
+          LoadJob.new("mike", idx, {mike: "bob"})
+        end)
       end
     else
       @iter.times do
-        arr = Array.new(@count) { |idx| [idx] }
+        arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
         Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
       end
     end
@@ -158,18 +167,18 @@ class Loader
         if total == 0
           ending = Time.now - @start
           size = @iter * @count
-          Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
-          Sidekiq.logger.error("Ending RSS: #{Process.rss}")
-          Sidekiq.logger.error("Now here's the latency for three jobs")
+          Sidekiq.logger.warn("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
+          Sidekiq.logger.warn("Ending RSS: #{Process.rss}")
+          Sidekiq.logger.warn("Now here's the latency for three jobs")
 
           if ENV["AJ"]
-            LoadJob.perform_later(1, Time.now.to_f)
-            LoadJob.perform_later(2, Time.now.to_f)
-            LoadJob.perform_later(3, Time.now.to_f)
+            LoadJob.perform_later("", 1, {}, Time.now.to_f)
+            LoadJob.perform_later("", 2, {}, Time.now.to_f)
+            LoadJob.perform_later("", 3, {}, Time.now.to_f)
           else
-            LoadWorker.perform_async(1, Time.now.to_f)
-            LoadWorker.perform_async(2, Time.now.to_f)
-            LoadWorker.perform_async(3, Time.now.to_f)
+            LoadWorker.perform_async("", 1, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 2, {}, Time.now.to_f)
+            LoadWorker.perform_async("", 3, {}, Time.now.to_f)
           end
 
           sleep 0.1
@@ -182,7 +191,7 @@ class Loader
   end
 
   def with_latency(latency, &block)
-    Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
+    Sidekiq.logger.warn "Simulating #{latency}ms of latency between Sidekiq and redis"
     if latency > 0
       Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
     else
@@ -195,8 +204,8 @@ class Loader
     monitor
 
     if ENV["PROFILE"]
-      RubyProf.exclude_threads = [@monitor]
-      RubyProf.start
+      $profile = RubyProf::Profile.new(exclude_threads: [@monitor])
+      $profile.start
     elsif ENV["GC"]
       GC.start
       GC.compact
@@ -227,7 +236,7 @@ class Loader
     Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
     if ENV["PROFILE"]
       Sidekiq.logger.error("Profiling...")
-      result = RubyProf.stop
+      result = $profile.stop
       printer = RubyProf::GraphHtmlPrinter.new(result)
       printer.print(File.new("output.html", "w"), min_percent: 1)
     end

data/bin/webload

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.setup
+
+# This skeleton allows you to run Sidekiq::Web page rendering
+# through Vernier for tuning.
+require "sidekiq/web"
+require "rack/test"
+require "vernier"
+
+Sidekiq::Web.configure do |config|
+  config.middlewares.clear # remove csrf
+end
+
+class SomeJob
+  include Sidekiq::Job
+end
+
+class BenchWeb
+  include Rack::Test::Methods
+
+  def app
+    Sidekiq::Web.new
+  end
+
+  def warmup(page = "/scheduled")
+    # Sidekiq.redis {|c| c.flushdb }
+
+    # 100.times do |idx|
+    #   SomeJob.perform_at(idx, 1, 3, "mike", {"foo" => "bar"})
+    # end
+
+    100.times do
+      get page
+    end
+  end
+
+  def load(page = "/scheduled", count = 10_000)
+    profile do
+      count.times do
+        get page
+        raise last_response.inspect unless last_response.status == 200
+      end
+    end
+  end
+
+  def profile(&)
+    if ENV["PROF"]
+      Vernier.profile(out: "profile.json.gz", &)
+    else
+      yield
+    end
+  end
+end
+
+def timer(name="block", count = 10_000)
+  a = Time.now
+  yield count
+  b = Time.now
+  puts "#{name} in #{b - a} sec"
+end
+
+page = "/busy"
+b = BenchWeb.new
+b.warmup(page)
+timer(page) do |count|
+  b.load(page, count)
+end

data/lib/active_job/queue_adapters/sidekiq_adapter.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+begin
+  gem "activejob", ">= 7.0"
+  require "active_job"
+
+  module Sidekiq
+    module ActiveJob
+      # @api private
+      class Wrapper
+        include Sidekiq::Job
+
+        def perform(job_data)
+          ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
+        end
+      end
+    end
+  end
+
+  ActiveSupport.on_load(:active_job) do
+    # By including the Options module, we allow AJs to directly control sidekiq features
+    # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
+    # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
+    # manually retried, don't automatically die, etc.
+    #
+    #   class SomeJob < ActiveJob::Base
+    #     queue_as :default
+    #     sidekiq_options retry: 3, backtrace: 10
+    #     def perform
+    #     end
+    #   end
+    include Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
+  end
+
+  # Patch the ActiveJob module
+  module ActiveJob
+    module QueueAdapters
+      # Explicitly remove the implementation existing in older Rails.
+      remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
+
+      # Sidekiq adapter for Active Job
+      #
+      # To use Sidekiq set the queue_adapter config to +:sidekiq+.
+      #
+      #   Rails.application.config.active_job.queue_adapter = :sidekiq
+      parent = const_defined?(:AbstractAdapter) ? AbstractAdapter : Object
+      class SidekiqAdapter < parent
+        @@stopping = false
+
+        callback = -> { @@stopping = true }
+
+        Sidekiq.configure_client { |config| config.on(:quiet, &callback) }
+        Sidekiq.configure_server { |config| config.on(:quiet, &callback) }
+
+        # Defines whether enqueuing should happen implicitly to after commit when called
+        # from inside a transaction.
+        # @api private
+        def enqueue_after_transaction_commit?
+          true
+        end
+
+        # @api private
+        def enqueue(job)
+          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
+            wrapped: job.class,
+            queue: job.queue_name
+          ).perform_async(job.serialize)
+        end
+
+        # @api private
+        def enqueue_at(job, timestamp)
+          job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
+            wrapped: job.class,
+            queue: job.queue_name
+          ).perform_at(timestamp, job.serialize)
+        end
+
+        # @api private
+        def enqueue_all(jobs)
+          enqueued_count = 0
+          jobs.group_by(&:class).each do |job_class, same_class_jobs|
+            same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
+              immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
+
+              if immediate_jobs.any?
+                jids = Sidekiq::Client.push_bulk(
+                  "class" => Sidekiq::ActiveJob::Wrapper,
+                  "wrapped" => job_class,
+                  "queue" => queue,
+                  "args" => immediate_jobs.map { |job| [job.serialize] }
+                )
+                enqueued_count += jids.compact.size
+              end
+
+              if scheduled_jobs.any?
+                jids = Sidekiq::Client.push_bulk(
+                  "class" => Sidekiq::ActiveJob::Wrapper,
+                  "wrapped" => job_class,
+                  "queue" => queue,
+                  "args" => scheduled_jobs.map { |job| [job.serialize] },
+                  "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
+                )
+                enqueued_count += jids.compact.size
+              end
+            end
+          end
+          enqueued_count
+        end
+
+        # @api private
+        def stopping? = !!@@stopping
+
+        # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
+        # @api private
+        JobWrapper = Sidekiq::ActiveJob::Wrapper
+      end
+    end
+  end
+rescue Gem::LoadError
+  # ActiveJob not available or version requirement not met
+end

data/lib/generators/sidekiq/job_generator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rails/generators/named_base"
 
 module Sidekiq

data/lib/generators/sidekiq/templates/job.rb.erb

@@ -6,4 +6,4 @@ class <%= class_name %>Job
     # Do something
   end
 end
-<% end -%>
+<% end -%>