sidekiq 4.2.4 → 6.4.0

data/bin/sidekiq

@@ -6,13 +6,37 @@ $TESTING = false
 
 require_relative '../lib/sidekiq/cli'
 
+def integrate_with_systemd
+  return unless ENV["NOTIFY_SOCKET"]
+
+  Sidekiq.configure_server do |config|
+    Sidekiq.logger.info "Enabling systemd notification integration"
+    require "sidekiq/sd_notify"
+    config.on(:startup) do
+      Sidekiq::SdNotify.ready
+    end
+    config.on(:shutdown) do
+      Sidekiq::SdNotify.stopping
+    end
+    Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
+  end
+end
+
 begin
   cli = Sidekiq::CLI.instance
   cli.parse
+
+  integrate_with_systemd
+
   cli.run
 rescue => e
   raise e if $DEBUG
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  if Sidekiq.error_handlers.length == 0
+    STDERR.puts e.message
+    STDERR.puts e.backtrace.join("\n")
+  else
+    cli.handle_exception e
+  end
+
   exit 1
 end

data/bin/sidekiqload

@@ -5,29 +5,17 @@
 $TESTING = false
 
 #require 'ruby-prof'
-Bundler.require(:default)
+require 'bundler/setup'
+Bundler.require(:default, :load_test)
 
 require_relative '../lib/sidekiq/cli'
 require_relative '../lib/sidekiq/launcher'
 
 include Sidekiq::Util
 
-# brew tap shopify/shopify
-# brew install toxiproxy
-# gem install toxiproxy
-require 'toxiproxy'
-# simulate a non-localhost network for realer-world conditions.
-# adding 1ms of network latency has an ENORMOUS impact on benchmarks
-Toxiproxy.populate([{
-    "name": "redis",
-    "listen": "127.0.0.1:6380",
-    "upstream": "127.0.0.1:6379"
-}])
-
-
 Sidekiq.configure_server do |config|
-  #config.options[:concurrency] = 1
-  config.redis = { driver: :hiredis, db: 13, port: 6380 }
+  config.options[:concurrency] = 10
+  config.redis = { db: 13, port: 6380, driver: :hiredis }
   config.options[:queues] << 'default'
   config.logger.level = Logger::ERROR
   config.average_scheduled_poll_interval = 2
@@ -41,7 +29,8 @@ class LoadWorker
     1
   end
 
-  def perform(idx)
+  def perform(idx, ts=nil)
+    puts(Time.now.to_f - ts) if ts != nil
     #raise idx.to_s if idx % 100 == 1
   end
 end
@@ -49,6 +38,7 @@ end
 # brew tap shopify/shopify
 # brew install toxiproxy
 # gem install toxiproxy
+# run `toxiproxy-server` in a separate terminal window.
 require 'toxiproxy'
 # simulate a non-localhost network for realer-world conditions.
 # adding 1ms of network latency has an ENORMOUS impact on benchmarks
@@ -59,7 +49,7 @@ Toxiproxy.populate([{
 }])
 
 self_read, self_write = IO.pipe
-%w(INT TERM USR1 USR2 TTIN).each do |sig|
+%w(INT TERM TSTP TTIN).each do |sig|
   begin
     trap sig do
       self_write.puts(sig)
@@ -78,19 +68,14 @@ def handle_signal(launcher, sig)
     # http://jira.codehaus.org/browse/JRUBY-4637
     raise Interrupt
   when 'TERM'
-    # Heroku sends TERM and then waits 10 seconds for process to exit.
+    # Heroku sends TERM and then waits 30 seconds for process to exit.
     raise Interrupt
-  when 'USR1'
-    Sidekiq.logger.info "Received USR1, no longer accepting new work"
+  when 'TSTP'
+    Sidekiq.logger.info "Received TSTP, no longer accepting new work"
     launcher.quiet
-  when 'USR2'
-    if Sidekiq.options[:logfile]
-      Sidekiq.logger.info "Received USR2, reopening log file"
-      Sidekiq::Logging.reopen_logs
-    end
   when 'TTIN'
     Thread.list.each do |thread|
-      Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
       if thread.backtrace
         Sidekiq.logger.warn thread.backtrace.join("\n")
       else
@@ -118,21 +103,26 @@ iter.times do
 end
 Sidekiq.logger.error "Created #{count*iter} jobs"
 
+start = Time.now
+
 Monitoring = Thread.new do
   watchdog("monitor thread") do
     while true
-      sleep 2
-      qsize, retries = Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.llen "queue:default"
-          conn.zcard "retry"
-        end
-      end.map(&:to_i)
-      total = qsize + retries
-      #GC.start
-      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      sleep 0.2
+      qsize = Sidekiq.redis do |conn|
+        conn.llen "queue:default"
+      end
+      total = qsize
+      #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
       if total == 0
-        Sidekiq.logger.error("Done")
+        Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
+        Sidekiq.logger.error("Now here's the latency for three jobs")
+
+        LoadWorker.perform_async(1, Time.now.to_f)
+        LoadWorker.perform_async(2, Time.now.to_f)
+        LoadWorker.perform_async(3, Time.now.to_f)
+
+        sleep 0.2
         exit(0)
       end
     end

data/bin/sidekiqmon

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'sidekiq/monitor'
+
+section = "all"
+section = ARGV[0] if ARGV.size == 1
+
+Sidekiq::Monitor::Status.new.display(section)

data/lib/generators/sidekiq/job_generator.rb

@@ -0,0 +1,57 @@
+require "rails/generators/named_base"
+
+module Sidekiq
+  module Generators # :nodoc:
+    class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
+      desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
+
+      check_class_collision suffix: "Job"
+
+      def self.default_generator_root
+        File.dirname(__FILE__)
+      end
+
+      def create_job_file
+        template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
+      end
+
+      def create_test_file
+        return unless test_framework
+
+        if test_framework == :rspec
+          create_job_spec
+        else
+          create_job_test
+        end
+      end
+
+      private
+
+      def create_job_spec
+        template_file = File.join(
+          "spec/sidekiq",
+          class_path,
+          "#{file_name}_job_spec.rb"
+        )
+        template "job_spec.rb.erb", template_file
+      end
+
+      def create_job_test
+        template_file = File.join(
+          "test/sidekiq",
+          class_path,
+          "#{file_name}_job_test.rb"
+        )
+        template "job_test.rb.erb", template_file
+      end
+
+      def file_name
+        @_file_name ||= super.sub(/_?job\z/i, "")
+      end
+
+      def test_framework
+        ::Rails.application.config.generators.options[:rails][:test_framework]
+      end
+    end
+  end
+end

data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb}

@@ -1,6 +1,6 @@
 <% module_namespacing do -%>
-class <%= class_name %>Worker
-  include Sidekiq::Worker
+class <%= class_name %>Job
+  include Sidekiq::Job
 
   def perform(*args)
     # Do something

data/lib/generators/sidekiq/templates/job_spec.rb.erb

@@ -0,0 +1,6 @@
+require 'rails_helper'
+<% module_namespacing do -%>
+RSpec.describe <%= class_name %>Job, type: :job do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
+<% end -%>

data/lib/generators/sidekiq/templates/job_test.rb.erb

@@ -0,0 +1,8 @@
+require 'test_helper'
+<% module_namespacing do -%>
+class <%= class_name %>JobTest < Minitest::Test
+  def test_example
+    skip "add some examples to (or delete) #{__FILE__}"
+  end
+end
+<% end -%>