sidekiq 3.3.0 → 3.3.1

data/lib/sidekiq/cli.rb

@@ -107,17 +107,6 @@ module Sidekiq
          sss }
     end
 
-    private
-
-    def print_banner
-      # Print logo and banner for development
-      if environment == 'development' && $stdout.tty?
-        puts "\e[#{31}m"
-        puts Sidekiq::CLI.banner
-        puts "\e[0m"
-      end
-    end
-
     def handle_signal(sig)
       Sidekiq.logger.debug "Got #{sig} signal"
       case sig
@@ -149,6 +138,17 @@ module Sidekiq
       end
     end
 
+    private
+
+    def print_banner
+      # Print logo and banner for development
+      if environment == 'development' && $stdout.tty?
+        puts "\e[#{31}m"
+        puts Sidekiq::CLI.banner
+        puts "\e[0m"
+      end
+    end
+
     def load_celluloid
       raise "Celluloid cannot be required until here, or it will break Sidekiq's daemonization" if defined?(::Celluloid) && options[:daemon]
 

data/lib/sidekiq/client.rb

@@ -55,7 +55,7 @@ module Sidekiq
     # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
     # symbols in 'args' will be converted to strings.
     #
-    # Returns nil if not pushed to Redis or a unique Job ID if pushed.
+    # Returns a unique Job ID.  If middleware stops the job, nil will be returned instead.
     #
     # Example:
     #   push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
@@ -64,9 +64,10 @@ module Sidekiq
       normed = normalize_item(item)
       payload = process_single(item['class'], normed)
 
-      pushed = false
-      pushed = raw_push([payload]) if payload
-      pushed ? payload['jid'] : nil
+      if payload
+        raw_push([payload])
+        payload['jid']
+      end
     end
 
     ##
@@ -80,9 +81,8 @@ module Sidekiq
     # is run through the client middleware pipeline and each job gets its own Job ID
     # as normal.
     #
-    # Returns an array of the of pushed jobs' jids or nil if the pushed failed.  The number of jobs
-    # pushed can be less than the number given if the middleware stopped processing for one
-    # or more jobs.
+    # Returns an array of the of pushed jobs' jids.  The number of jobs pushed can be less
+    # than the number given if the middleware stopped processing for one or more jobs.
     def push_bulk(items)
       normed = normalize_item(items)
       payloads = items['args'].map do |args|
@@ -90,9 +90,8 @@ module Sidekiq
         process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
       end.compact
 
-      pushed = false
-      pushed = raw_push(payloads) if !payloads.empty?
-      pushed ? payloads.collect { |payload| payload['jid'] } : nil
+      raw_push(payloads) if !payloads.empty?
+      payloads.collect { |payload| payload['jid'] }
     end
 
     # Allows sharding of jobs across any number of Redis instances.  All jobs
@@ -160,7 +159,7 @@ module Sidekiq
         ts = (int < 1_000_000_000 ? now + int : int)
 
         item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
-        item.delete('at') if ts <= now
+        item.delete('at'.freeze) if ts <= now
 
         klass.client_push(item)
       end
@@ -186,14 +185,14 @@ module Sidekiq
 
     def atomic_push(conn, payloads)
       if payloads.first['at']
-        conn.zadd('schedule', payloads.map do |hash|
-          at = hash.delete('at').to_s
+        conn.zadd('schedule'.freeze, payloads.map do |hash|
+          at = hash.delete('at'.freeze).to_s
           [at, Sidekiq.dump_json(hash)]
         end)
       else
         q = payloads.first['queue']
         to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
-        conn.sadd('queues', q)
+        conn.sadd('queues'.freeze, q)
         conn.lpush("queue:#{q}", to_push)
       end
     end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -17,7 +17,11 @@ module Sidekiq
         msg = target.public_send(method_name, *args)
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
-        deliver(msg) if msg && (msg.to || msg.cc || msg.bcc) && msg.from
+        if msg
+          deliver(msg)
+        else
+          raise "#{target.name}##{method_name} returned an undeliverable mail object"
+        end
       end
 
       private

data/lib/sidekiq/launcher.rb

@@ -27,7 +27,9 @@ module Sidekiq
     end
 
     def actor_died(actor, reason)
-      return if @done
+      # https://github.com/mperham/sidekiq/issues/2057#issuecomment-66485477
+      return if @done || !reason
+
       Sidekiq.logger.warn("Sidekiq died due to the following error, cannot recover, process exiting")
       handle_exception(reason)
       exit(1)
@@ -73,6 +75,7 @@ module Sidekiq
         'concurrency' => @options[:concurrency],
         'queues' => @options[:queues].uniq,
         'labels' => Sidekiq.options[:labels],
+        'identity' => identity,
       }
       # this data doesn't change so dump it to a string
       # now so we don't need to dump it every heartbeat.

data/lib/sidekiq/logging.rb

@@ -5,6 +5,8 @@ module Sidekiq
   module Logging
 
     class Pretty < Logger::Formatter
+      SPACE = " "
+
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)
         "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
@@ -12,17 +14,16 @@ module Sidekiq
 
       def context
         c = Thread.current[:sidekiq_context]
-        c ? " #{c}" : ''
+        " #{c.join(SPACE)}" if c && c.any?
       end
     end
 
     def self.with_context(msg)
-      begin
-        Thread.current[:sidekiq_context] = msg
-        yield
-      ensure
-        Thread.current[:sidekiq_context] = nil
-      end
+      Thread.current[:sidekiq_context] ||= []
+      Thread.current[:sidekiq_context] << msg
+      yield
+    ensure
+      Thread.current[:sidekiq_context].pop
     end
 
     def self.initialize_logger(log_target = STDOUT)

data/lib/sidekiq/manager.rb

@@ -21,6 +21,7 @@ module Sidekiq
     attr_accessor :fetcher
 
     SPIN_TIME_FOR_GRACEFUL_SHUTDOWN = 1
+    JVM_RESERVED_SIGNALS = ['USR1', 'USR2'] # Don't Process#kill if we get these signals via the API
 
     def initialize(condvar, options={})
       logger.debug { options.inspect }
@@ -47,7 +48,7 @@ module Sidekiq
 
         @done = true
 
-        logger.info { "Shutting down #{@ready.size} quiet workers" }
+        logger.info { "Terminating #{@ready.size} quiet workers" }
         @ready.each { |x| x.terminate if x.alive? }
         @ready.clear
 
@@ -159,7 +160,13 @@ module Sidekiq
           end
         end
 
-        ::Process.kill(msg, $$) if msg
+        return unless msg
+
+        if JVM_RESERVED_SIGNALS.include?(msg)
+          Sidekiq::CLI.instance.handle_signal(msg)
+        else
+          ::Process.kill(msg, $$)
+        end
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")

data/lib/sidekiq/middleware/server/logging.rb

@@ -4,7 +4,7 @@ module Sidekiq
       class Logging
 
         def call(worker, item, queue)
-          Sidekiq::Logging.with_context("#{worker.class.to_s} JID-#{item['jid']}") do
+          Sidekiq::Logging.with_context("#{worker.class.to_s} JID-#{item['jid']}#{" BID-#{item['bid']}" if item['bid']}") do
             begin
               start = Time.now
               logger.info { "start" }

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -114,10 +114,10 @@ module Sidekiq
 
           if msg['backtrace'] == true
             msg['error_backtrace'] = exception.backtrace
-          elsif msg['backtrace'] == false
+          elsif !msg['backtrace']
             # do nothing
           elsif msg['backtrace'].to_i != 0
-            msg['error_backtrace'] = exception.backtrace[0..msg['backtrace'].to_i]
+            msg['error_backtrace'] = exception.backtrace[0...msg['backtrace'].to_i]
           end
 
           if count < max_retry_attempts
@@ -143,7 +143,7 @@ module Sidekiq
               worker.sidekiq_retries_exhausted_block.call(msg)
             end
           rescue => e
-            handle_exception(e, { :context => "Error calling retries_exhausted for #{worker.class}", :job => msg })
+            handle_exception(e, { context: "Error calling retries_exhausted for #{worker.class}", job: msg })
           end
 
           send_to_morgue(msg) unless msg['dead'] == false
@@ -183,7 +183,7 @@ module Sidekiq
           begin
             worker.sidekiq_retry_in_block.call(count)
           rescue Exception => e
-            handle_exception(e, { :context => "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
+            handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
             nil
           end
         end

data/lib/sidekiq/processor.rb

@@ -98,26 +98,26 @@ module Sidekiq
         yield
       rescue Exception
         retry_and_suppress_exceptions do
+          failed = "stat:failed:#{Time.now.utc.to_date}"
           Sidekiq.redis do |conn|
-            failed = "stat:failed:#{Time.now.utc.to_date}"
-            result = conn.multi do
+            conn.multi do
               conn.incrby("stat:failed", 1)
               conn.incrby(failed, 1)
+              conn.expire(failed, STATS_TIMEOUT)
             end
-            conn.expire(failed, STATS_TIMEOUT) if result.last == 1
           end
         end
         raise
       ensure
         retry_and_suppress_exceptions do
+          processed = "stat:processed:#{Time.now.utc.to_date}"
           Sidekiq.redis do |conn|
-            processed = "stat:processed:#{Time.now.utc.to_date}"
-            result = conn.multi do
+            conn.multi do
               conn.hdel("#{identity}:workers", thread_identity)
               conn.incrby("stat:processed", 1)
               conn.incrby(processed, 1)
+              conn.expire(processed, STATS_TIMEOUT)
             end
-            conn.expire(processed, STATS_TIMEOUT) if result.last == 1
           end
         end
       end

data/lib/sidekiq/testing.rb

@@ -7,12 +7,12 @@ module Sidekiq
     class << self
       attr_accessor :__test_mode
 
-      def __set_test_mode(mode, &block)
-        if block
+      def __set_test_mode(mode)
+        if block_given?
           current_mode = self.__test_mode
           begin
             self.__test_mode = mode
-            block.call
+            yield
           ensure
             self.__test_mode = current_mode
           end
@@ -66,12 +66,11 @@ module Sidekiq
         end
         true
       elsif Sidekiq::Testing.inline?
-        payloads.each do |item|
-          jid = item['jid'] || SecureRandom.hex(12)
-          marshalled = Sidekiq.load_json(Sidekiq.dump_json(item))
-          worker = marshalled['class'].constantize.new
-          worker.jid = jid
-          worker.perform(*marshalled['args'])
+        payloads.each do |job|
+          job['jid'] ||= SecureRandom.hex(12)
+          klass = job['class'].constantize
+          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.perform_one
         end
         true
       else

data/lib/sidekiq/util.rb

@@ -1,4 +1,5 @@
 require 'socket'
+require 'securerandom'
 require 'sidekiq/exception_handler'
 require 'sidekiq/core_ext'
 
@@ -14,7 +15,7 @@ module Sidekiq
     def watchdog(last_words)
       yield
     rescue Exception => ex
-      handle_exception(ex, { :context => last_words })
+      handle_exception(ex, { context: last_words })
       raise ex
     end
 
@@ -30,8 +31,12 @@ module Sidekiq
       ENV['DYNO'] || Socket.gethostname
     end
 
+    def process_nonce
+      @@process_nonce ||= SecureRandom.hex(6)
+    end
+
     def identity
-      @@identity ||= "#{hostname}:#{$$}"
+      @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
     def fire_event(event)
@@ -39,7 +44,7 @@ module Sidekiq
         begin
           block.call
         rescue => ex
-          handle_exception(ex, { :event => event })
+          handle_exception(ex, { event: event })
         end
       end
     end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.3.0"
+  VERSION = "3.3.1"
 end

data/lib/sidekiq/web.rb

@@ -45,12 +45,12 @@ module Sidekiq
     end
 
     post "/busy" do
-      if params['hostname']
-        p = Sidekiq::Process.new('hostname' => params["hostname"], 'pid' => params['pid'])
+      if params['identity']
+        p = Sidekiq::Process.new('identity' => params['identity'])
         p.quiet! if params[:quiet]
         p.stop! if params[:stop]
       else
-        Sidekiq::ProcessSet.new.each do |pro|
+        processes.each do |pro|
           pro.quiet! if params[:quiet]
           pro.stop! if params[:stop]
         end
@@ -69,7 +69,7 @@ module Sidekiq
       @name = params[:name]
       @queue = Sidekiq::Queue.new(@name)
       (@current_page, @total_size, @messages) = page("queue:#{@name}", params[:page], @count)
-      @messages = @messages.map {|msg| Sidekiq::Job.new(msg, @name) }
+      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
       erb :queue
     end
 
@@ -85,8 +85,8 @@ module Sidekiq
 
     get '/morgue' do
       @count = (params[:count] || 25).to_i
-      (@current_page, @total_size, @dead) = page("dead", params[:page], @count, :reverse => true)
-      @dead = @dead.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      (@current_page, @total_size, @dead) = page("dead", params[:page], @count, reverse: true)
+      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
       erb :morgue
     end
 
@@ -128,7 +128,7 @@ module Sidekiq
     get '/retries' do
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @retries) = page("retry", params[:page], @count)
-      @retries = @retries.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
       erb :retries
     end
 
@@ -167,7 +167,7 @@ module Sidekiq
     get '/scheduled' do
       @count = (params[:count] || 25).to_i
       (@current_page, @total_size, @scheduled) = page("schedule", params[:page], @count)
-      @scheduled = @scheduled.map {|msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
       erb :scheduled
     end
 
@@ -205,24 +205,37 @@ module Sidekiq
     REDIS_KEYS = %w(redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human)
 
     get '/dashboard/stats' do
+      redirect "#{root_path}stats"
+    end
+
+    get '/stats' do
       sidekiq_stats = Sidekiq::Stats.new
-      queue         = Sidekiq::Queue.new
-      redis_stats   = redis_info.select{ |k, v| REDIS_KEYS.include? k }
+      redis_stats   = redis_info.select { |k, v| REDIS_KEYS.include? k }
 
       content_type :json
-      Sidekiq.dump_json({
+      Sidekiq.dump_json(
         sidekiq: {
-          processed:  sidekiq_stats.processed,
-          failed:     sidekiq_stats.failed,
-          busy:       workers_size,
-          enqueued:   sidekiq_stats.enqueued,
-          scheduled:  sidekiq_stats.scheduled_size,
-          retries:    sidekiq_stats.retry_size,
-          dead:       sidekiq_stats.dead_size,
-          default_latency: queue.latency,
+          processed:       sidekiq_stats.processed,
+          failed:          sidekiq_stats.failed,
+          busy:            sidekiq_stats.workers_size,
+          processes:       sidekiq_stats.processes_size,
+          enqueued:        sidekiq_stats.enqueued,
+          scheduled:       sidekiq_stats.scheduled_size,
+          retries:         sidekiq_stats.retry_size,
+          dead:            sidekiq_stats.dead_size,
+          default_latency: sidekiq_stats.default_queue_latency
         },
         redis: redis_stats
-      })
+      )
+    end
+
+    get '/stats/queues' do
+      queue_stats = Sidekiq::Stats::Queues.new
+
+      content_type :json
+      Sidekiq.dump_json(
+        queue_stats.lengths
+      )
     end
 
     private