sidekiq 3.5.4 → 5.2.7

data/lib/sidekiq/core_ext.rb

@@ -1,105 +1 @@
-begin
-  require 'active_support/core_ext/class/attribute'
-rescue LoadError
-
-  # A dumbed down version of ActiveSupport's
-  # Class#class_attribute helper.
-  class Class
-    def class_attribute(*attrs)
-      instance_writer = true
-
-      attrs.each do |name|
-        class_eval <<-RUBY, __FILE__, __LINE__ + 1
-          def self.#{name}() nil end
-          def self.#{name}?() !!#{name} end
-
-          def self.#{name}=(val)
-            singleton_class.class_eval do
-              define_method(:#{name}) { val }
-            end
-
-            if singleton_class?
-              class_eval do
-                def #{name}
-                  defined?(@#{name}) ? @#{name} : singleton_class.#{name}
-                end
-              end
-            end
-            val
-          end
-
-          def #{name}
-            defined?(@#{name}) ? @#{name} : self.class.#{name}
-          end
-
-          def #{name}?
-            !!#{name}
-          end
-        RUBY
-
-        attr_writer name if instance_writer
-      end
-    end
-
-    private
-    def singleton_class?
-      ancestors.first != self
-    end
-  end
-end
-
-begin
-  require 'active_support/core_ext/hash/keys'
-  require 'active_support/core_ext/hash/deep_merge'
-rescue LoadError
-  class Hash
-    def stringify_keys
-      keys.each do |key|
-        self[key.to_s] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:stringify_keys)
-
-    def symbolize_keys
-      keys.each do |key|
-        self[(key.to_sym rescue key) || key] = delete(key)
-      end
-      self
-    end if !{}.respond_to?(:symbolize_keys)
-
-    def deep_merge(other_hash, &block)
-      dup.deep_merge!(other_hash, &block)
-    end if !{}.respond_to?(:deep_merge)
-
-    def deep_merge!(other_hash, &block)
-      other_hash.each_pair do |k,v|
-        tv = self[k]
-        if tv.is_a?(Hash) && v.is_a?(Hash)
-          self[k] = tv.deep_merge(v, &block)
-        else
-          self[k] = block && tv ? block.call(k, tv, v) : v
-        end
-      end
-      self
-    end if !{}.respond_to?(:deep_merge!)
-  end
-end
-
-begin
-  require 'active_support/core_ext/string/inflections'
-rescue LoadError
-  class String
-    def constantize
-      names = self.split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
-      end
-      constant
-    end
-  end if !"".respond_to?(:constantize)
-end
-
-
+raise "no longer used, will be removed in 5.1"

data/lib/sidekiq/ctl.rb

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'sidekiq/api'
+
+class Sidekiq::Ctl
+  DEFAULT_KILL_TIMEOUT = 10
+  CMD = File.basename($0)
+
+  attr_reader :stage, :pidfile, :kill_timeout
+
+  def self.print_usage
+    puts "#{CMD} - control Sidekiq from the command line."
+    puts
+    puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
+    puts "       #{CMD} stop <pidfile> <kill_timeout>"
+    puts "       #{CMD} status <section>"
+    puts
+    puts "       <pidfile> is path to a pidfile"
+    puts "       <kill_timeout> is number of seconds to wait until Sidekiq exits"
+    puts "       (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
+    puts
+    puts "       <section> (optional) view a specific section of the status output"
+    puts "       Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
+    puts
+    puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout.  If you want"
+    puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
+    puts " path_to_pidfile 61`"
+    puts
+  end
+
+  def initialize(stage, pidfile, timeout)
+    @stage = stage
+    @pidfile = pidfile
+    @kill_timeout = timeout
+
+    done('No pidfile given', :error) if !pidfile
+    done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
+    done('Invalid pidfile content', :error) if pid == 0
+
+    fetch_process
+
+    begin
+      send(stage)
+    rescue NoMethodError
+      done "Invalid command: #{stage}", :error
+    end
+  end
+
+  def fetch_process
+    Process.kill(0, pid)
+  rescue Errno::ESRCH
+    done "Process doesn't exist", :error
+  # We were not allowed to send a signal, but the process must have existed
+  # when Process.kill() was called.
+  rescue Errno::EPERM
+    return pid
+  end
+
+  def done(msg, error = nil)
+    puts msg
+    exit(exit_signal(error))
+  end
+
+  def exit_signal(error)
+    (error == :error) ? 1 : 0
+  end
+
+  def pid
+    @pid ||= File.read(pidfile).to_i
+  end
+
+  def quiet
+    `kill -TSTP #{pid}`
+  end
+
+  def stop
+    `kill -TERM #{pid}`
+    kill_timeout.times do
+      begin
+        Process.kill(0, pid)
+      rescue Errno::ESRCH
+        FileUtils.rm_f pidfile
+        done 'Sidekiq shut down gracefully.'
+      rescue Errno::EPERM
+        done 'Not permitted to shut down Sidekiq.'
+      end
+      sleep 1
+    end
+    `kill -9 #{pid}`
+    FileUtils.rm_f pidfile
+    done 'Sidekiq shut down forcefully.'
+  end
+  alias_method :shutdown, :stop
+
+  class Status
+    VALID_SECTIONS = %w[all version overview processes queues]
+    def display(section = nil)
+      section ||= 'all'
+      unless VALID_SECTIONS.include? section
+        puts "I don't know how to check the status of '#{section}'!"
+        puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
+        return
+      end
+      send(section)
+    rescue StandardError => e
+      puts "Couldn't get status: #{e}"
+    end
+
+    def all
+      version
+      puts
+      overview
+      puts
+      processes
+      puts
+      queues
+    end
+
+    def version
+      puts "Sidekiq #{Sidekiq::VERSION}"
+      puts Time.now
+    end
+
+    def overview
+      puts '---- Overview ----'
+      puts "  Processed: #{delimit stats.processed}"
+      puts "     Failed: #{delimit stats.failed}"
+      puts "       Busy: #{delimit stats.workers_size}"
+      puts "   Enqueued: #{delimit stats.enqueued}"
+      puts "    Retries: #{delimit stats.retry_size}"
+      puts "  Scheduled: #{delimit stats.scheduled_size}"
+      puts "       Dead: #{delimit stats.dead_size}"
+    end
+
+    def processes
+      puts "---- Processes (#{process_set.size}) ----"
+      process_set.each_with_index do |process, index|
+        puts "#{process['identity']} #{tags_for(process)}"
+        puts "  Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
+        puts "  Threads: #{process['concurrency']} (#{process['busy']} busy)"
+        puts "   Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
+        puts '' unless (index+1) == process_set.size
+      end
+    end
+
+    COL_PAD = 2
+    def queues
+      puts "---- Queues (#{queue_data.size}) ----"
+      columns = {
+        name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
+        size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
+      }
+      columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
+      puts
+      queue_data.each do |q|
+        columns.each do |col, (dir, width)|
+          print q.send(col).public_send(dir, width)
+        end
+        puts
+      end
+    end
+
+    private
+
+    def delimit(number)
+      number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
+    end
+
+    def split_multiline(values, opts = {})
+      return 'none' unless values
+      pad = opts[:pad] || 0
+      max_length = opts[:max_length] || (80 - pad)
+      out = []
+      line = ''
+      values.each do |value|
+        if (line.length + value.length) > max_length
+          out << line
+          line = ' ' * pad
+        end
+        line << value + ', '
+      end
+      out << line[0..-3]
+      out.join("\n")
+    end
+
+    def tags_for(process)
+      tags = [
+        process['tag'],
+        process['labels'],
+        (process['quiet'] == 'true' ? 'quiet' : nil)
+      ].flatten.compact
+      tags.any? ? "[#{tags.join('] [')}]" : nil
+    end
+
+    def time_ago(timestamp)
+      seconds = Time.now - Time.at(timestamp)
+      return 'just now' if seconds < 60
+      return 'a minute ago' if seconds < 120
+      return "#{seconds.floor / 60} minutes ago" if seconds < 3600
+      return 'an hour ago' if seconds < 7200
+      "#{seconds.floor / 60 / 60} hours ago"
+    end
+
+    QUEUE_STRUCT = Struct.new(:name, :size, :latency)
+    def queue_data
+      @queue_data ||= Sidekiq::Queue.all.map do |q|
+        QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
+      end
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+  end
+end

data/lib/sidekiq/delay.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        require 'sidekiq/extensions/active_record'
+        require 'sidekiq/extensions/action_mailer'
+
+        # Need to patch Psych so it can autoload classes whose names are serialized
+        # in the delayed YAML.
+        Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
+
+        ActiveSupport.on_load(:active_record) do
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        # constantize
+        names = klass_name.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      rescue NameError
+        super
+      end
+    end
+  end
+end
+

data/lib/sidekiq/exception_handler.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq'
 
 module Sidekiq
@@ -5,12 +6,11 @@ module Sidekiq
 
     class Logger
       def call(ex, ctxHash)
-        Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
-        Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
-        Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
+        Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
+        Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
       end
 
-      # Set up default handler which just logs the error
       Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
     end
 
@@ -25,6 +25,5 @@ module Sidekiq
         end
       end
     end
-
   end
 end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/active_record.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,7 +1,10 @@
+# frozen_string_literal: true
 require 'yaml'
 
 module Sidekiq
   module Extensions
+    SIZE_LIMIT = 8_192
+
     class Proxy < BasicObject
       def initialize(performable, target, options={})
         @performable = performable
@@ -16,7 +19,11 @@ module Sidekiq
         # to JSON and then deserialized on the other side back into a
         # Ruby object.
         obj = [@target, name, args]
-        @performable.client_push({ 'class' => @performable, 'args' => [::YAML.dump(obj)] }.merge(@opts))
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
       end
     end
 

data/lib/sidekiq/fetch.rb

@@ -1,101 +1,35 @@
+# frozen_string_literal: true
 require 'sidekiq'
-require 'sidekiq/util'
-require 'sidekiq/actor'
 
 module Sidekiq
-  ##
-  # The Fetcher blocks on Redis, waiting for a message to process
-  # from the queues.  It gets the message and hands it to the Manager
-  # to assign to a ready Processor.
-  class Fetcher
-    include Util
-    include Actor
-
-    TIMEOUT = 1
-
-    attr_reader :down
-
-    def initialize(mgr, options)
-      @down = nil
-      @mgr = mgr
-      @strategy = Fetcher.strategy.new(options)
-    end
-
-    # Fetching is straightforward: the Manager makes a fetch
-    # request for each idle processor when Sidekiq starts and
-    # then issues a new fetch request every time a Processor
-    # finishes a message.
-    #
-    # Because we have to shut down cleanly, we can't block
-    # forever and we can't loop forever.  Instead we reschedule
-    # a new fetch if the current fetch turned up nothing.
-    def fetch
-      watchdog('Fetcher#fetch died') do
-        return if Sidekiq::Fetcher.done?
-
-        begin
-          work = @strategy.retrieve_work
-          ::Sidekiq.logger.info("Redis is online, #{Time.now - @down} sec downtime") if @down
-          @down = nil
-
-          if work
-            @mgr.async.assign(work)
-          else
-            after(0) { fetch }
-          end
-        rescue => ex
-          handle_fetch_exception(ex)
-        end
+  class BasicFetch
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down.
+    TIMEOUT = 2
 
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        # nothing to do
       end
-    end
-
-    private
 
-    def pause
-      sleep(TIMEOUT)
-    end
+      def queue_name
+        queue.sub(/.*queue:/, '')
+      end
 
-    def handle_fetch_exception(ex)
-      if !@down
-        logger.error("Error fetching message: #{ex}")
-        ex.backtrace.each do |bt|
-          logger.error(bt)
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.rpush("queue:#{queue_name}", job)
         end
       end
-      @down ||= Time.now
-      pause
-      after(0) { fetch }
-    rescue Celluloid::TaskTerminated
-      # If redis is down when we try to shut down, all the fetch backlog
-      # raises these errors.  Haven't been able to figure out what I'm doing wrong.
-    end
-
-    # Ugh.  Say hello to a bloody hack.
-    # Can't find a clean way to get the fetcher to just stop processing
-    # its mailbox when shutdown starts.
-    def self.done!
-      @done = true
-    end
-
-    def self.reset # testing only
-      @done = nil
     end
 
-    def self.done?
-      defined?(@done) && @done
-    end
-
-    def self.strategy
-      Sidekiq.options[:fetch] || BasicFetch
-    end
-  end
-
-  class BasicFetch
     def initialize(options)
       @strictly_ordered_queues = !!options[:strict]
       @queues = options[:queues].map { |q| "queue:#{q}" }
-      @unique_queues = @queues.uniq
+      if @strictly_ordered_queues
+        @queues = @queues.uniq
+        @queues << TIMEOUT
+      end
     end
 
     def retrieve_work
@@ -103,6 +37,22 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
+
+
     # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
     # an instance method will make it async to the Fetcher actor
     def self.bulk_requeue(inprogress, options)
@@ -112,7 +62,7 @@ module Sidekiq
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
         jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.message
+        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
@@ -122,35 +72,10 @@ module Sidekiq
           end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
-    UnitOfWork = Struct.new(:queue, :message) do
-      def acknowledge
-        # nothing to do
-      end
-
-      def queue_name
-        queue.gsub(/.*queue:/, '')
-      end
-
-      def requeue
-        Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", message)
-        end
-      end
-    end
-
-    # Creating the Redis#brpop command takes into account any
-    # configured queue weights. By default Redis#brpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#brpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
-      queues << Sidekiq::Fetcher::TIMEOUT
-    end
   end
 end