sidekiq_cleaner 5.3.6

data/lib/sidekiq/ctl.rb

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'sidekiq/api'
+
+class Sidekiq::Ctl
+  DEFAULT_KILL_TIMEOUT = 10
+  CMD = File.basename($0)
+
+  attr_reader :stage, :pidfile, :kill_timeout
+
+  def self.print_usage
+    puts "#{CMD} - control Sidekiq from the command line."
+    puts
+    puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
+    puts "       #{CMD} stop <pidfile> <kill_timeout>"
+    puts "       #{CMD} status <section>"
+    puts
+    puts "       <pidfile> is path to a pidfile"
+    puts "       <kill_timeout> is number of seconds to wait until Sidekiq exits"
+    puts "       (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
+    puts
+    puts "       <section> (optional) view a specific section of the status output"
+    puts "       Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
+    puts
+    puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout.  If you want"
+    puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
+    puts " path_to_pidfile 61`"
+    puts
+  end
+
+  def initialize(stage, pidfile, timeout)
+    @stage = stage
+    @pidfile = pidfile
+    @kill_timeout = timeout
+
+    done('No pidfile given', :error) if !pidfile
+    done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
+    done('Invalid pidfile content', :error) if pid == 0
+
+    fetch_process
+
+    begin
+      send(stage)
+    rescue NoMethodError
+      done "Invalid command: #{stage}", :error
+    end
+  end
+
+  def fetch_process
+    Process.kill(0, pid)
+  rescue Errno::ESRCH
+    done "Process doesn't exist", :error
+  # We were not allowed to send a signal, but the process must have existed
+  # when Process.kill() was called.
+  rescue Errno::EPERM
+    return pid
+  end
+
+  def done(msg, error = nil)
+    puts msg
+    exit(exit_signal(error))
+  end
+
+  def exit_signal(error)
+    (error == :error) ? 1 : 0
+  end
+
+  def pid
+    @pid ||= File.read(pidfile).to_i
+  end
+
+  def quiet
+    `kill -TSTP #{pid}`
+  end
+
+  def stop
+    `kill -TERM #{pid}`
+    kill_timeout.times do
+      begin
+        Process.kill(0, pid)
+      rescue Errno::ESRCH
+        FileUtils.rm_f pidfile
+        done 'Sidekiq shut down gracefully.'
+      rescue Errno::EPERM
+        done 'Not permitted to shut down Sidekiq.'
+      end
+      sleep 1
+    end
+    `kill -9 #{pid}`
+    FileUtils.rm_f pidfile
+    done 'Sidekiq shut down forcefully.'
+  end
+  alias_method :shutdown, :stop
+
+  class Status
+    VALID_SECTIONS = %w[all version overview processes queues]
+    def display(section = nil)
+      section ||= 'all'
+      unless VALID_SECTIONS.include? section
+        puts "I don't know how to check the status of '#{section}'!"
+        puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
+        return
+      end
+      send(section)
+    rescue StandardError => e
+      puts "Couldn't get status: #{e}"
+    end
+
+    def all
+      version
+      puts
+      overview
+      puts
+      processes
+      puts
+      queues
+    end
+
+    def version
+      puts "Sidekiq #{Sidekiq::VERSION}"
+      puts Time.now
+    end
+
+    def overview
+      puts '---- Overview ----'
+      puts "  Processed: #{delimit stats.processed}"
+      puts "     Failed: #{delimit stats.failed}"
+      puts "       Busy: #{delimit stats.workers_size}"
+      puts "   Enqueued: #{delimit stats.enqueued}"
+      puts "    Retries: #{delimit stats.retry_size}"
+      puts "  Scheduled: #{delimit stats.scheduled_size}"
+      puts "       Dead: #{delimit stats.dead_size}"
+    end
+
+    def processes
+      puts "---- Processes (#{process_set.size}) ----"
+      process_set.each_with_index do |process, index|
+        puts "#{process['identity']} #{tags_for(process)}"
+        puts "  Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
+        puts "  Threads: #{process['concurrency']} (#{process['busy']} busy)"
+        puts "   Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
+        puts '' unless (index+1) == process_set.size
+      end
+    end
+
+    COL_PAD = 2
+    def queues
+      puts "---- Queues (#{queue_data.size}) ----"
+      columns = {
+        name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
+        size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
+      }
+      columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
+      puts
+      queue_data.each do |q|
+        columns.each do |col, (dir, width)|
+          print q.send(col).public_send(dir, width)
+        end
+        puts
+      end
+    end
+
+    private
+
+    def delimit(number)
+      number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
+    end
+
+    def split_multiline(values, opts = {})
+      return 'none' unless values
+      pad = opts[:pad] || 0
+      max_length = opts[:max_length] || (80 - pad)
+      out = []
+      line = ''
+      values.each do |value|
+        if (line.length + value.length) > max_length
+          out << line
+          line = ' ' * pad
+        end
+        line << value + ', '
+      end
+      out << line[0..-3]
+      out.join("\n")
+    end
+
+    def tags_for(process)
+      tags = [
+        process['tag'],
+        process['labels'],
+        (process['quiet'] == 'true' ? 'quiet' : nil)
+      ].flatten.compact
+      tags.any? ? "[#{tags.join('] [')}]" : nil
+    end
+
+    def time_ago(timestamp)
+      seconds = Time.now - Time.at(timestamp)
+      return 'just now' if seconds < 60
+      return 'a minute ago' if seconds < 120
+      return "#{seconds.floor / 60} minutes ago" if seconds < 3600
+      return 'an hour ago' if seconds < 7200
+      "#{seconds.floor / 60 / 60} hours ago"
+    end
+
+    QUEUE_STRUCT = Struct.new(:name, :size, :latency)
+    def queue_data
+      @queue_data ||= Sidekiq::Queue.all.map do |q|
+        QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
+      end
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+  end
+end

data/lib/sidekiq/delay.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+module Sidekiq
+  module Extensions
+
+    def self.enable_delay!
+      if defined?(::ActiveSupport)
+        require 'sidekiq/extensions/active_record'
+        require 'sidekiq/extensions/action_mailer'
+
+        # Need to patch Psych so it can autoload classes whose names are serialized
+        # in the delayed YAML.
+        Psych::Visitors::ToRuby.prepend(Sidekiq::Extensions::PsychAutoload)
+
+        ActiveSupport.on_load(:active_record) do
+          include Sidekiq::Extensions::ActiveRecord
+        end
+        ActiveSupport.on_load(:action_mailer) do
+          extend Sidekiq::Extensions::ActionMailer
+        end
+      end
+
+      require 'sidekiq/extensions/class_methods'
+      Module.__send__(:include, Sidekiq::Extensions::Klass)
+    end
+
+    module PsychAutoload
+      def resolve_class(klass_name)
+        return nil if !klass_name || klass_name.empty?
+        # constantize
+        names = klass_name.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+      rescue NameError
+        super
+      end
+    end
+  end
+end
+

data/lib/sidekiq/exception_handler.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require 'sidekiq'
+
+module Sidekiq
+  module ExceptionHandler
+
+    class Logger
+      def call(ex, ctxHash)
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
+        Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
+        Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+      end
+
+      Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
+    end
+
+    def handle_exception(ex, ctxHash={})
+      Sidekiq.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Sidekiq.logger.error ex
+          Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds 'delay', 'delay_for' and `delay_until` methods to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.  Example:
+    #
+    #    UserMailer.delay.send_welcome_email(new_user)
+    #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
+    #    UserMailer.delay_until(5.days.from_now).send_welcome_email(new_user)
+    class DelayedMailer
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        msg = target.public_send(method_name, *args)
+        # The email method can return nil, which causes ActionMailer to return
+        # an undeliverable empty message.
+        if msg
+          deliver(msg)
+        else
+          raise "#{target.name}##{method_name} returned an undeliverable mail object"
+        end
+      end
+
+      private
+
+      def deliver(msg)
+        if msg.respond_to?(:deliver_now)
+          # Rails 4.2/5.0
+          msg.deliver_now
+        else
+          # Rails 3.2/4.0/4.1
+          msg.deliver
+        end
+      end
+    end
+
+    module ActionMailer
+      def sidekiq_delay(options={})
+        Proxy.new(DelayedMailer, self, options)
+      end
+      def sidekiq_delay_for(interval, options={})
+        Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
+      end
+      def sidekiq_delay_until(timestamp, options={})
+        Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
+      end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
+    end
+
+  end
+end

data/lib/sidekiq/extensions/active_record.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds 'delay', 'delay_for' and `delay_until` methods to ActiveRecord to offload instance method
+    # execution to Sidekiq.  Examples:
+    #
+    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    #
+    # Please note, this is not recommended as this will serialize the entire
+    # object to Redis.  Your Sidekiq jobs should pass IDs, not entire instances.
+    # This is here for backwards compatibility with Delayed::Job only.
+    class DelayedModel
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.__send__(method_name, *args)
+      end
+    end
+
+    module ActiveRecord
+      def sidekiq_delay(options={})
+        Proxy.new(DelayedModel, self, options)
+      end
+      def sidekiq_delay_for(interval, options={})
+        Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
+      end
+      def sidekiq_delay_until(timestamp, options={})
+        Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
+      end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
+    end
+
+  end
+end

data/lib/sidekiq/extensions/class_methods.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds 'delay', 'delay_for' and `delay_until` methods to all Classes to offload class method
+    # execution to Sidekiq.  Examples:
+    #
+    # User.delay.delete_inactive
+    # Wikipedia.delay.download_changes_for(Date.today)
+    #
+    class DelayedClass
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.__send__(method_name, *args)
+      end
+    end
+
+    module Klass
+      def sidekiq_delay(options={})
+        Proxy.new(DelayedClass, self, options)
+      end
+      def sidekiq_delay_for(interval, options={})
+        Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
+      end
+      def sidekiq_delay_until(timestamp, options={})
+        Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
+      end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
+    end
+
+  end
+end
+
+Module.__send__(:include, Sidekiq::Extensions::Klass) unless defined?(::Rails)

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require 'yaml'
+
+module Sidekiq
+  module Extensions
+    SIZE_LIMIT = 8_192
+
+    class Proxy < BasicObject
+      def initialize(performable, target, options={})
+        @performable = performable
+        @target = target
+        @opts = options
+      end
+
+      def method_missing(name, *args)
+        # Sidekiq has a limitation in that its message must be JSON.
+        # JSON can't round trip real Ruby objects so we use YAML to
+        # serialize the objects to a String.  The YAML will be converted
+        # to JSON and then deserialized on the other side back into a
+        # Ruby object.
+        obj = [@target, name, args]
+        marshalled = ::YAML.dump(obj)
+        if marshalled.size > SIZE_LIMIT
+          ::Sidekiq.logger.warn { "#{@target}.#{name} job argument is #{marshalled.bytesize} bytes, you should refactor it to reduce the size" }
+        end
+        @performable.client_push({ 'class' => @performable, 'args' => [marshalled] }.merge(@opts))
+      end
+    end
+
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+require 'sidekiq'
+
+module Sidekiq
+  class BasicFetch
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down.
+    TIMEOUT = 2
+
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        # nothing to do
+      end
+
+      def queue_name
+        queue.sub(/.*queue:/, '')
+      end
+
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.rpush("queue:#{queue_name}", job)
+        end
+      end
+    end
+
+    def initialize(options)
+      @strictly_ordered_queues = !!options[:strict]
+      @queues = options[:queues].map { |q| "queue:#{q}" }
+      if @strictly_ordered_queues
+        @queues = @queues.uniq
+        @queues << TIMEOUT
+      end
+    end
+
+    def retrieve_work
+      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
+      UnitOfWork.new(*work) if work
+    end
+
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
+
+
+    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+    # an instance method will make it async to the Fetcher actor
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
+      Sidekiq.logger.debug { "Re-queueing terminated jobs" }
+      jobs_to_requeue = {}
+      inprogress.each do |unit_of_work|
+        jobs_to_requeue[unit_of_work.queue_name] ||= []
+        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
+      end
+
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          jobs_to_requeue.each do |queue, jobs|
+            conn.rpush("queue:#{queue}", jobs)
+          end
+        end
+      end
+      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
+    rescue => ex
+      Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
+    end
+
+  end
+end

data/lib/sidekiq/job_logger.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+module Sidekiq
+  class JobLogger
+
+    def call(item, queue)
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      logger.info("start")
+      yield
+      logger.info("done: #{elapsed(start)} sec")
+    rescue Exception
+      logger.info("fail: #{elapsed(start)} sec")
+      raise
+    end
+
+    private
+
+    def elapsed(start)
+      (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
+    end
+
+    def logger
+      Sidekiq.logger
+    end
+  end
+end