sidekiq 2.17.8 → 3.0.0

data/lib/sidekiq/cli.rb

@@ -43,6 +43,13 @@ module Sidekiq
     end
 
     def run
+      # Print logo and banner for development
+      if environment == 'development' && $stdout.tty?
+        puts "\e[#{31}m"
+        puts Sidekiq::BANNER
+        puts "\e[0m"
+      end
+
       self_read, self_write = IO.pipe
 
       %w(INT TERM USR1 USR2 TTIN).each do |sig|
@@ -55,23 +62,19 @@ module Sidekiq
         end
       end
 
-      redis {} # noop to connect redis and print info
       logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.info Sidekiq::LICENSE
 
+      fire_event(:startup)
+
       if !options[:daemon]
         logger.info 'Starting processing, hit Ctrl-C to stop'
       end
 
       require 'sidekiq/launcher'
       @launcher = Sidekiq::Launcher.new(options)
-      launcher.procline(options[:tag] ? "#{options[:tag]} " : '')
 
       begin
-        if options[:profile]
-          require 'ruby-prof'
-          RubyProf.start
-        end
         launcher.run
 
         while readable_io = IO.select([self_read])
@@ -81,6 +84,7 @@ module Sidekiq
       rescue Interrupt
         logger.info 'Shutting down'
         launcher.stop
+        fire_event(:shutdown)
         # Explicitly exit so busy Processor threads can't block
         # process shutdown.
         exit(0)
@@ -89,17 +93,20 @@ module Sidekiq
 
     private
 
+    def fire_event(event)
+      Sidekiq.options[:lifecycle_events][event].each do |block|
+        begin
+          block.call
+        rescue => ex
+          handle_exception(ex, { :event => event })
+        end
+      end
+    end
+
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
       case sig
       when 'INT'
-        if Sidekiq.options[:profile]
-          result = RubyProf.stop
-          printer = RubyProf::GraphHtmlPrinter.new(result)
-          File.open("profile.html", 'w') do |f|
-            printer.print(f, :min_percent => 1)
-          end
-        end
         # Handle Ctrl-C in JRuby like MRI
         # http://jira.codehaus.org/browse/JRUBY-4637
         raise Interrupt
@@ -109,6 +116,7 @@ module Sidekiq
       when 'USR1'
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
         launcher.manager.async.stop
+        fire_event(:quiet)
       when 'USR2'
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
@@ -258,10 +266,6 @@ module Sidekiq
           opts[:index] = Integer(arg.match(/\d+/)[0])
         end
 
-        o.on '-p', '--profile', "Profile all code run by Sidekiq" do |arg|
-          opts[:profile] = arg
-        end
-
         o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
           queue, weight = arg.split(",")
           parse_queue opts, queue, weight
@@ -310,7 +314,7 @@ module Sidekiq
     def initialize_logger
       Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
 
-      Sidekiq.logger.level = Logger::DEBUG if options[:verbose]
+      Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
     end
 
     def write_pid

data/lib/sidekiq/client.rb

@@ -25,6 +25,24 @@ module Sidekiq
       @chain
     end
 
+    attr_accessor :redis_pool
+
+    # Sidekiq::Client normally uses the default Redis pool but you may
+    # pass a custom ConnectionPool if you want to shard your
+    # Sidekiq jobs across several Redis instances (for scalability
+    # reasons, e.g.)
+    #
+    #   Sidekiq::Client.new(ConnectionPool.new { Redis.new })
+    #
+    # Generally this is only needed for very large Sidekiq installs processing
+    # more than thousands jobs per second.  I do not recommend sharding unless
+    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
+    # Some features, like the API, do not support sharding: they are designed to work
+    # against a single Redis instance only.
+    def initialize(redis_pool=nil)
+      @redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
+    end
+
     ##
     # The main method used to push a job to Redis.  Accepts a number of options:
     #
@@ -77,23 +95,35 @@ module Sidekiq
       pushed ? payloads.collect { |payload| payload['jid'] } : nil
     end
 
+    # Allows sharding of jobs across any number of Redis instances.  All jobs
+    # defined within the block will use the given Redis connection pool.
+    #
+    #   pool = ConnectionPool.new { Redis.new }
+    #   Sidekiq::Client.via(pool) do
+    #     SomeWorker.perform_async(1,2,3)
+    #     SomeOtherWorker.perform_async(1,2,3)
+    #   end
+    #
+    # Generally this is only needed for very large Sidekiq installs processing
+    # more than thousands jobs per second.  I do not recommend sharding unless
+    # you truly cannot scale any other way (e.g. splitting your app into smaller apps).
+    # Some features, like the API, do not support sharding: they are designed to work
+    # against a single Redis instance.
+    def self.via(pool)
+      raise ArgumentError, "No pool given" if pool.nil?
+      raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if x = Thread.current[:sidekiq_via_pool] && x != pool
+      Thread.current[:sidekiq_via_pool] = pool
+      yield
+    ensure
+      Thread.current[:sidekiq_via_pool] = nil
+    end
+
     class << self
+
       def default
         @default ||= new
       end
 
-      # deprecated
-      def registered_workers
-        puts "registered_workers is deprecated, please use Sidekiq::Workers.new"
-        Sidekiq.redis { |x| x.smembers('workers') }
-      end
-
-      # deprecated
-      def registered_queues
-        puts "registered_queues is deprecated, please use Sidekiq::Queue.all"
-        Sidekiq::Queue.all.map(&:name)
-      end
-
       def push(item)
         default.push(item)
       end
@@ -147,7 +177,7 @@ module Sidekiq
 
     def raw_push(payloads)
       pushed = false
-      Sidekiq.redis do |conn|
+      @redis_pool.with do |conn|
         if payloads.first['at']
           pushed = conn.zadd('schedule', payloads.map do |hash|
             at = hash.delete('at').to_s
@@ -168,7 +198,7 @@ module Sidekiq
     def process_single(worker_class, item)
       queue = item['queue']
 
-      middleware.invoke(worker_class, item, queue) do
+      middleware.invoke(worker_class, item, queue, @redis_pool) do
         item
       end
     end

data/lib/sidekiq/core_ext.rb

@@ -6,7 +6,6 @@ rescue LoadError
   # Class#class_attribute helper.
   class Class
     def class_attribute(*attrs)
-      instance_reader = true
       instance_writer = true
 
       attrs.each do |name|
@@ -29,14 +28,12 @@ rescue LoadError
             val
           end
 
-          if instance_reader
-            def #{name}
-              defined?(@#{name}) ? @#{name} : self.class.#{name}
-            end
+          def #{name}
+            defined?(@#{name}) ? @#{name} : self.class.#{name}
+          end
 
-            def #{name}?
-              !!#{name}
-            end
+          def #{name}?
+            !!#{name}
           end
         RUBY
 

data/lib/sidekiq/exception_handler.rb

@@ -1,39 +1,30 @@
+require 'sidekiq'
+
 module Sidekiq
   module ExceptionHandler
 
-    def handle_exception(ex, ctxHash={})
-      Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
-      Sidekiq.logger.warn ex
-      Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
-      # This list of services is getting a bit ridiculous.
-      # For future error services, please add your own
-      # middleware like BugSnag does:
-      # https://github.com/bugsnag/bugsnag-ruby/blob/master/lib/bugsnag/sidekiq.rb
-      send_to_airbrake(ctxHash, ex) if defined?(::Airbrake)
-      send_to_honeybadger(ctxHash, ex) if defined?(::Honeybadger)
-      send_to_exceptional(ctxHash, ex) if defined?(::Exceptional)
-      send_to_exception_notifier(ctxHash, ex) if defined?(::ExceptionNotifier)
-    end
-
-    private
-
-    def send_to_airbrake(hash, ex)
-      ::Airbrake.notify_or_ignore(ex, :parameters => hash)
-    end
+    class Logger
+      def call(ex, ctxHash)
+        Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
+        Sidekiq.logger.warn ex
+        Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+      end
 
-    def send_to_honeybadger(hash, ex)
-      ::Honeybadger.notify_or_ignore(ex, :parameters => hash)
+      # Set up default handler which just logs the error
+      Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
     end
 
-    def send_to_exceptional(hash, ex)
-      if ::Exceptional::Config.should_send_to_api?
-        ::Exceptional.context(hash)
-        ::Exceptional::Remote.error(::Exceptional::ExceptionData.new(ex))
+    def handle_exception(ex, ctxHash={})
+      Sidekiq.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Sidekiq.logger.error ex
+          Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
       end
     end
 
-    def send_to_exception_notifier(hash, ex)
-      ::ExceptionNotifier.notify_exception(ex, :data => {:message => hash})
-    end
   end
 end

data/lib/sidekiq/fetch.rb

@@ -140,10 +140,10 @@ module Sidekiq
       end
     end
 
-    # Creating the Redis#blpop command takes into account any
-    # configured queue weights. By default Redis#blpop returns
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
     # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#blpop
+    # recreate the queue command each time we invoke Redis#brpop
     # to honor weights and avoid queue starvation.
     def queues_cmd
       queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq

data/lib/sidekiq/launcher.rb

@@ -36,6 +36,8 @@ module Sidekiq
       watchdog('Launcher#run') do
         manager.async.start
         poller.async.poll(true)
+
+        start_heartbeat
       end
     end
 
@@ -51,12 +53,37 @@ module Sidekiq
 
         # Requeue everything in case there was a worker who grabbed work while stopped
         Sidekiq::Fetcher.strategy.bulk_requeue([], @options)
+
+        stop_heartbeat
       end
     end
 
-    def procline(tag)
-      $0 = manager.procline(tag)
-      manager.after(5) { procline(tag) }
+    private
+
+    def start_heartbeat
+      key = identity
+      data = {
+        'hostname' => hostname,
+        'started_at' => Time.now.to_f,
+        'pid' => $$,
+        'tag' => @options[:tag] || '',
+        'concurrency' => @options[:concurrency],
+        'queues' => @options[:queues].uniq,
+      }
+      Sidekiq.redis do |conn|
+        conn.multi do
+          conn.sadd('processes', key)
+          conn.hset(key, 'info', Sidekiq.dump_json(data))
+          conn.expire(key, 60)
+        end
+      end
+      manager.heartbeat(key, data)
+    end
+
+    def stop_heartbeat
+      Sidekiq.redis do |conn|
+        conn.srem('processes', identity)
+      end
     end
   end
 end

data/lib/sidekiq/logging.rb

@@ -26,7 +26,7 @@ module Sidekiq
     end
 
     def self.initialize_logger(log_target = STDOUT)
-      oldlogger = @logger
+      oldlogger = defined?(@logger) ? @logger : nil
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
       @logger.formatter = Pretty.new
@@ -35,7 +35,7 @@ module Sidekiq
     end
 
     def self.logger
-      @logger || initialize_logger
+      defined?(@logger) ? @logger : initialize_logger
     end
 
     def self.logger=(log)

data/lib/sidekiq/manager.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require 'sidekiq/util'
 require 'sidekiq/actor'
 require 'sidekiq/processor'
@@ -49,7 +50,6 @@ module Sidekiq
         @ready.each { |x| x.terminate if x.alive? }
         @ready.clear
 
-        clear_worker_set
         return if clean_up_for_graceful_shutdown
 
         hard_shutdown_in timeout if should_shutdown
@@ -132,27 +132,30 @@ module Sidekiq
       @threads[proxy_id] = thr
     end
 
-    def procline(tag)
-      "sidekiq #{Sidekiq::VERSION} #{tag}[#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
+    def heartbeat(key, data)
+      return if stopped?
+
+      $0 = "sidekiq #{Sidekiq::VERSION} #{data['tag']}[#{@busy.size} of #{data['concurrency']} busy]#{stopped? ? ' stopping' : ''}"
+      ❤(key)
+      after(5) do
+        heartbeat(key, data)
+      end
     end
 
     private
 
-    def clear_worker_set
-      # Clearing workers in Redis
-      # NOTE: we do this before terminating worker threads because the
-      # process will likely receive a hard shutdown soon anyway, which
-      # means the threads will killed.
-      logger.debug { "Clearing workers in redis" }
-      Sidekiq.redis do |conn|
-        workers = conn.smembers('workers')
-        workers_to_remove = workers.select do |worker_name|
-          worker_name =~ /:#{process_id}-/
+    def ❤(key)
+      begin
+        Sidekiq.redis do |conn|
+          conn.multi do
+            conn.hmset(key, 'busy', @busy.size, 'beat', Time.now.to_f)
+            conn.expire(key, 60)
+          end
         end
-        conn.srem('workers', workers_to_remove) if !workers_to_remove.empty?
+      rescue => e
+        # ignore all redis/network issues
+        logger.error("heartbeat: #{e.message}")
       end
-    rescue => ex
-      Sidekiq.logger.warn("Unable to clear worker set while shutting down: #{ex.message}")
     end
 
     def hard_shutdown_in(delay)

data/lib/sidekiq/middleware/chain.rb

@@ -54,7 +54,7 @@ module Sidekiq
   # to Redis:
   #
   # class MyClientHook
-  #   def call(worker_class, msg, queue)
+  #   def call(worker_class, msg, queue, redis_pool)
   #     puts "Before push"
   #     result = yield
   #     puts "After push"

data/lib/sidekiq/middleware/i18n.rb

@@ -2,7 +2,7 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
-    def call(worker_class, msg, queue)
+    def call(worker_class, msg, queue, redis_pool)
       msg['locale'] ||= I18n.locale
       yield
     end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -108,17 +108,33 @@ module Sidekiq
 
         private
 
+        DEAD_JOB_TIMEOUT = 180 * 24 * 60 * 60 # 6 months
+        MAX_JOBS = 10_000
+
         def retries_exhausted(worker, msg)
           logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
-          if worker.respond_to?(:retries_exhausted)
-            logger.warn { "Defining #{worker.class.name}#retries_exhausted as a method is deprecated, use `sidekiq_retries_exhausted` callback instead http://git.io/Ijju8g" }
-            worker.retries_exhausted(*msg['args'])
-          elsif worker.sidekiq_retries_exhausted_block?
-            worker.sidekiq_retries_exhausted_block.call(msg)
+          begin
+            if worker.sidekiq_retries_exhausted_block?
+              worker.sidekiq_retries_exhausted_block.call(msg)
+            end
+          rescue => e
+            handle_exception(e, { :context => "Error calling retries_exhausted" })
           end
 
-        rescue Exception => e
-          handle_exception(e, { :context => "Error calling retries_exhausted" })
+          send_to_morgue(msg)
+        end
+
+        def send_to_morgue(msg)
+          Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+          payload = Sidekiq.dump_json(msg)
+          now = Time.now.to_f
+          Sidekiq.redis do |conn|
+            conn.multi do
+              conn.zadd('dead', now, payload)
+              conn.zremrangebyscore('dead', '-inf', now - DEAD_JOB_TIMEOUT)
+              conn.zremrangebyrank('dead', 0, -MAX_JOBS)
+            end
+          end
         end
 
         def retry_attempts_from(msg_retry, default)