sidekiq 2.15.1 → 4.2.10

data/lib/sidekiq/core_ext.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 begin
   require 'active_support/core_ext/class/attribute'
 rescue LoadError
@@ -6,7 +7,6 @@ rescue LoadError
   # Class#class_attribute helper.
   class Class
     def class_attribute(*attrs)
-      instance_reader = true
       instance_writer = true
 
       attrs.each do |name|
@@ -29,14 +29,12 @@ rescue LoadError
             val
           end
 
-          if instance_reader
-            def #{name}
-              defined?(@#{name}) ? @#{name} : self.class.#{name}
-            end
+          def #{name}
+            defined?(@#{name}) ? @#{name} : self.class.#{name}
+          end
 
-            def #{name}?
-              !!#{name}
-            end
+          def #{name}?
+            !!#{name}
           end
         RUBY
 
@@ -53,6 +51,7 @@ end
 
 begin
   require 'active_support/core_ext/hash/keys'
+  require 'active_support/core_ext/hash/deep_merge'
 rescue LoadError
   class Hash
     def stringify_keys
@@ -68,6 +67,22 @@ rescue LoadError
       end
       self
     end if !{}.respond_to?(:symbolize_keys)
+
+    def deep_merge(other_hash, &block)
+      dup.deep_merge!(other_hash, &block)
+    end if !{}.respond_to?(:deep_merge)
+
+    def deep_merge!(other_hash, &block)
+      other_hash.each_pair do |k,v|
+        tv = self[k]
+        if tv.is_a?(Hash) && v.is_a?(Hash)
+          self[k] = tv.deep_merge(v, &block)
+        else
+          self[k] = block && tv ? block.call(k, tv, v) : v
+        end
+      end
+      self
+    end if !{}.respond_to?(:deep_merge!)
   end
 end
 
@@ -89,3 +104,16 @@ rescue LoadError
 end
 
 
+begin
+  require 'active_support/core_ext/kernel/reporting'
+rescue LoadError
+  module Kernel
+    module_function
+    def silence_warnings
+      old_verbose, $VERBOSE = $VERBOSE, nil
+      yield
+    ensure
+      $VERBOSE = old_verbose
+    end
+  end
+end

data/lib/sidekiq/exception_handler.rb

@@ -1,39 +1,31 @@
+# frozen_string_literal: true
+require 'sidekiq'
+
 module Sidekiq
   module ExceptionHandler
 
-    def handle_exception(ex, ctxHash={})
-      Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
-      Sidekiq.logger.warn ex
-      Sidekiq.logger.warn ex.backtrace.join("\n")
-      # This list of services is getting a bit ridiculous.
-      # For future error services, please add your own
-      # middleware like BugSnag does:
-      # https://github.com/bugsnag/bugsnag-ruby/blob/master/lib/bugsnag/sidekiq.rb
-      send_to_airbrake(ctxHash, ex) if defined?(::Airbrake)
-      send_to_honeybadger(ctxHash, ex) if defined?(::Honeybadger)
-      send_to_exceptional(ctxHash, ex) if defined?(::Exceptional)
-      send_to_exception_notifier(ctxHash, ex) if defined?(::ExceptionNotifier)
-    end
-
-    private
-
-    def send_to_airbrake(hash, ex)
-      ::Airbrake.notify_or_ignore(ex, :parameters => hash)
-    end
+    class Logger
+      def call(ex, ctxHash)
+        Sidekiq.logger.warn(Sidekiq.dump_json(ctxHash)) if !ctxHash.empty?
+        Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
+        Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
+      end
 
-    def send_to_honeybadger(hash, ex)
-      ::Honeybadger.notify_or_ignore(ex, :parameters => hash)
+      # Set up default handler which just logs the error
+      Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
     end
 
-    def send_to_exceptional(hash, ex)
-      if ::Exceptional::Config.should_send_to_api?
-        ::Exceptional.context(hash)
-        ::Exceptional::Remote.error(::Exceptional::ExceptionData.new(ex))
+    def handle_exception(ex, ctxHash={})
+      Sidekiq.error_handlers.each do |handler|
+        begin
+          handler.call(ex, ctxHash)
+        rescue => ex
+          Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Sidekiq.logger.error ex
+          Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+        end
       end
     end
 
-    def send_to_exception_notifier(hash, ex)
-      ::ExceptionNotifier.notify_exception(ex, :data => {:message => hash})
-    end
   end
 end

data/lib/sidekiq/extensions/action_mailer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq
@@ -14,23 +15,42 @@ module Sidekiq
 
       def perform(yml)
         (target, method_name, args) = YAML.load(yml)
-        msg = target.send(method_name, *args)
+        msg = target.public_send(method_name, *args)
         # The email method can return nil, which causes ActionMailer to return
         # an undeliverable empty message.
-        msg.deliver if msg && (msg.to || msg.cc || msg.bcc) && msg.from
+        if msg
+          deliver(msg)
+        else
+          raise "#{target.name}##{method_name} returned an undeliverable mail object"
+        end
+      end
+
+      private
+
+      def deliver(msg)
+        if msg.respond_to?(:deliver_now)
+          # Rails 4.2/5.0
+          msg.deliver_now
+        else
+          # Rails 3.2/4.0/4.1
+          msg.deliver
+        end
       end
     end
 
     module ActionMailer
-      def delay(options={})
+      def sidekiq_delay(options={})
         Proxy.new(DelayedMailer, self, options)
       end
-      def delay_for(interval, options={})
+      def sidekiq_delay_for(interval, options={})
         Proxy.new(DelayedMailer, self, options.merge('at' => Time.now.to_f + interval.to_f))
       end
-      def delay_until(timestamp, options={})
+      def sidekiq_delay_until(timestamp, options={})
         Proxy.new(DelayedMailer, self, options.merge('at' => timestamp.to_f))
       end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
     end
 
   end

data/lib/sidekiq/extensions/active_record.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq
@@ -16,20 +17,23 @@ module Sidekiq
 
       def perform(yml)
         (target, method_name, args) = YAML.load(yml)
-        target.send(method_name, *args)
+        target.__send__(method_name, *args)
       end
     end
 
     module ActiveRecord
-      def delay(options={})
+      def sidekiq_delay(options={})
         Proxy.new(DelayedModel, self, options)
       end
-      def delay_for(interval, options={})
+      def sidekiq_delay_for(interval, options={})
         Proxy.new(DelayedModel, self, options.merge('at' => Time.now.to_f + interval.to_f))
       end
-      def delay_until(timestamp, options={})
+      def sidekiq_delay_until(timestamp, options={})
         Proxy.new(DelayedModel, self, options.merge('at' => timestamp.to_f))
       end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
     end
 
   end

data/lib/sidekiq/extensions/class_methods.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'sidekiq/extensions/generic_proxy'
 
 module Sidekiq
@@ -14,23 +15,26 @@ module Sidekiq
 
       def perform(yml)
         (target, method_name, args) = YAML.load(yml)
-        target.send(method_name, *args)
+        target.__send__(method_name, *args)
       end
     end
 
     module Klass
-      def delay(options={})
+      def sidekiq_delay(options={})
         Proxy.new(DelayedClass, self, options)
       end
-      def delay_for(interval, options={})
+      def sidekiq_delay_for(interval, options={})
         Proxy.new(DelayedClass, self, options.merge('at' => Time.now.to_f + interval.to_f))
       end
-      def delay_until(timestamp, options={})
+      def sidekiq_delay_until(timestamp, options={})
         Proxy.new(DelayedClass, self, options.merge('at' => timestamp.to_f))
       end
+      alias_method :delay, :sidekiq_delay
+      alias_method :delay_for, :sidekiq_delay_for
+      alias_method :delay_until, :sidekiq_delay_until
     end
 
   end
 end
 
-Module.send(:include, Sidekiq::Extensions::Klass)
+Module.__send__(:include, Sidekiq::Extensions::Klass) unless defined?(::Rails)

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'yaml'
 
 module Sidekiq

data/lib/sidekiq/fetch.rb

@@ -1,88 +1,35 @@
+# frozen_string_literal: true
 require 'sidekiq'
-require 'sidekiq/actor'
 
 module Sidekiq
-  ##
-  # The Fetcher blocks on Redis, waiting for a message to process
-  # from the queues.  It gets the message and hands it to the Manager
-  # to assign to a ready Processor.
-  class Fetcher
-    include Util
-    include Actor
-
-    TIMEOUT = 1
-
-    def initialize(mgr, options)
-      @down = nil
-      @mgr = mgr
-      @strategy = Fetcher.strategy.new(options)
-    end
-
-    # Fetching is straightforward: the Manager makes a fetch
-    # request for each idle processor when Sidekiq starts and
-    # then issues a new fetch request every time a Processor
-    # finishes a message.
-    #
-    # Because we have to shut down cleanly, we can't block
-    # forever and we can't loop forever.  Instead we reschedule
-    # a new fetch if the current fetch turned up nothing.
-    def fetch
-      watchdog('Fetcher#fetch died') do
-        return if Sidekiq::Fetcher.done?
-
-        begin
-          work = @strategy.retrieve_work
-          ::Sidekiq.logger.info("Redis is online, #{Time.now.to_f - @down.to_f} sec downtime") if @down
-          @down = nil
+  class BasicFetch
+    # We want the fetch operation to timeout every few seconds so the thread
+    # can check if the process is shutting down.
+    TIMEOUT = 2
 
-          if work
-            @mgr.async.assign(work)
-          else
-            after(0) { fetch }
-          end
-        rescue => ex
-          handle_exception(ex)
-        end
+    UnitOfWork = Struct.new(:queue, :job) do
+      def acknowledge
+        # nothing to do
+      end
 
+      def queue_name
+        queue.sub(/.*queue:/, ''.freeze)
       end
-    end
 
-    def handle_exception(ex)
-      if !@down
-        logger.error("Error fetching message: #{ex}")
-        ex.backtrace.each do |bt|
-          logger.error(bt)
+      def requeue
+        Sidekiq.redis do |conn|
+          conn.rpush("queue:#{queue_name}", job)
         end
       end
-      @down ||= Time.now
-      sleep(TIMEOUT)
-      after(0) { fetch }
-    rescue Task::TerminatedError
-      # If redis is down when we try to shut down, all the fetch backlog
-      # raises these errors.  Haven't been able to figure out what I'm doing wrong.
-    end
-
-    # Ugh.  Say hello to a bloody hack.
-    # Can't find a clean way to get the fetcher to just stop processing
-    # its mailbox when shutdown starts.
-    def self.done!
-      @done = true
-    end
-
-    def self.done?
-      @done
     end
 
-    def self.strategy
-      Sidekiq.options[:fetch] || BasicFetch
-    end
-  end
-
-  class BasicFetch
     def initialize(options)
       @strictly_ordered_queues = !!options[:strict]
       @queues = options[:queues].map { |q| "queue:#{q}" }
-      @unique_queues = @queues.uniq
+      if @strictly_ordered_queues
+        @queues = @queues.uniq
+        @queues << TIMEOUT
+      end
     end
 
     def retrieve_work
@@ -90,48 +37,45 @@ module Sidekiq
       UnitOfWork.new(*work) if work
     end
 
-    def self.bulk_requeue(inprogress)
+    # Creating the Redis#brpop command takes into account any
+    # configured queue weights. By default Redis#brpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#brpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      if @strictly_ordered_queues
+        @queues
+      else
+        queues = @queues.shuffle.uniq
+        queues << TIMEOUT
+        queues
+      end
+    end
+
+
+    # By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
+    # an instance method will make it async to the Fetcher actor
+    def self.bulk_requeue(inprogress, options)
+      return if inprogress.empty?
+
       Sidekiq.logger.debug { "Re-queueing terminated jobs" }
       jobs_to_requeue = {}
       inprogress.each do |unit_of_work|
         jobs_to_requeue[unit_of_work.queue_name] ||= []
-        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.message
+        jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
       end
 
       Sidekiq.redis do |conn|
-        jobs_to_requeue.each do |queue, jobs|
-          conn.rpush("queue:#{queue}", jobs)
+        conn.pipelined do
+          jobs_to_requeue.each do |queue, jobs|
+            conn.rpush("queue:#{queue}", jobs)
+          end
         end
       end
-      Sidekiq.logger.info("Pushed #{inprogress.size} messages back to Redis")
+      Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
     rescue => ex
       Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
     end
 
-    UnitOfWork = Struct.new(:queue, :message) do
-      def acknowledge
-        # nothing to do
-      end
-
-      def queue_name
-        queue.gsub(/.*queue:/, '')
-      end
-
-      def requeue
-        Sidekiq.redis do |conn|
-          conn.rpush("queue:#{queue_name}", message)
-        end
-      end
-    end
-
-    # Creating the Redis#blpop command takes into account any
-    # configured queue weights. By default Redis#blpop returns
-    # data from the first queue that has pending elements. We
-    # recreate the queue command each time we invoke Redis#blpop
-    # to honor weights and avoid queue starvation.
-    def queues_cmd
-      queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
-      queues << Sidekiq::Fetcher::TIMEOUT
-    end
   end
 end