sidekiq 3.4.1 → 4.0.0

data/lib/sidekiq/middleware/server/logging.rb

@@ -4,11 +4,7 @@ module Sidekiq
       class Logging
 
         def call(worker, item, queue)
-          # If we're using a wrapper class, like ActiveJob, use the "wrapped"
-          # attribute to expose the underlying thing.
-          klass = item['wrapped'] || worker.class.to_s
-
-          Sidekiq::Logging.with_context("#{klass} JID-#{item['jid']}#{" BID-#{item['bid']}" if item['bid']}") do
+          Sidekiq::Logging.with_context(log_context(worker, item)) do
             begin
               start = Time.now
               logger.info { "start" }
@@ -21,6 +17,15 @@ module Sidekiq
           end
         end
 
+        private
+
+        # If we're using a wrapper class, like ActiveJob, use the "wrapped"
+        # attribute to expose the underlying thing.
+        def log_context(worker, item)
+          klass = item['wrapped'.freeze] || worker.class.to_s
+          "#{klass} JID-#{item['jid'.freeze]}#{" BID-#{item['bid'.freeze]}" if item['bid'.freeze]}"
+        end
+
         def elapsed(start)
           (Time.now - start).round(3)
         end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -121,7 +121,7 @@ module Sidekiq
           end
 
           if count < max_retry_attempts
-            delay = delay_for(worker, count)
+            delay = delay_for(worker, count, exception)
             logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
             retry_at = Time.now.to_f + delay
             payload = Sidekiq.dump_json(msg)
@@ -170,8 +170,8 @@ module Sidekiq
           end
         end
 
-        def delay_for(worker, count)
-          worker.sidekiq_retry_in_block? && retry_in(worker, count) || seconds_to_delay(count)
+        def delay_for(worker, count, exception)
+          worker.sidekiq_retry_in_block? && retry_in(worker, count, exception) || seconds_to_delay(count)
         end
 
         # delayed_job uses the same basic formula
@@ -179,9 +179,9 @@ module Sidekiq
           (count ** 4) + 15 + (rand(30)*(count+1))
         end
 
-        def retry_in(worker, count)
+        def retry_in(worker, count, exception)
           begin
-            worker.sidekiq_retry_in_block.call(count)
+            worker.sidekiq_retry_in_block.call(count, exception).to_i
           rescue Exception => e
             handle_exception(e, { context: "Failure scheduling retry using the defined `sidekiq_retry_in` in #{worker.class.name}, falling back to default" })
             nil

data/lib/sidekiq/processor.rb

@@ -1,155 +1,184 @@
 require 'sidekiq/util'
-require 'sidekiq/actor'
-
-require 'sidekiq/middleware/server/retry_jobs'
-require 'sidekiq/middleware/server/logging'
+require 'sidekiq/fetch'
+require 'thread'
+require 'concurrent/map'
+require 'concurrent/atomic/atomic_fixnum'
 
 module Sidekiq
   ##
-  # The Processor receives a message from the Manager and actually
-  # processes it.  It instantiates the worker, runs the middleware
-  # chain and then calls Sidekiq::Worker#perform.
+  # The Processor is a standalone thread which:
+  #
+  # 1. fetches a job from Redis
+  # 2. executes the job
+  #   a. instantiate the Worker
+  #   b. run the middleware chain
+  #   c. call #perform
+  #
+  # A Processor can exit due to shutdown (processor_stopped)
+  # or due to an error during job execution (processor_died)
+  #
+  # If an error occurs in the job execution, the
+  # Processor calls the Manager to create a new one
+  # to replace itself and exits.
+  #
   class Processor
-    # To prevent a memory leak, ensure that stats expire. However, they should take up a minimal amount of storage
-    # so keep them around for a long time
-    STATS_TIMEOUT = 24 * 60 * 60 * 365 * 5
 
     include Util
-    include Actor
-
-    def self.default_middleware
-      Middleware::Chain.new do |m|
-        m.add Middleware::Server::Logging
-        m.add Middleware::Server::RetryJobs
-        if defined?(::ActiveRecord::Base)
-          require 'sidekiq/middleware/server/active_record'
-          m.add Sidekiq::Middleware::Server::ActiveRecord
+
+    attr_reader :thread
+    attr_reader :job
+
+    def initialize(mgr)
+      @mgr = mgr
+      @down = false
+      @done = false
+      @job = nil
+      @thread = nil
+      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
+    end
+
+    def terminate(wait=false)
+      @done = true
+      return if !@thread
+      @thread.value if wait
+    end
+
+    def kill(wait=false)
+      @done = true
+      return if !@thread
+      # unlike the other actors, terminate does not wait
+      # for the thread to finish because we don't know how
+      # long the job will take to finish.  Instead we
+      # provide a `kill` method to call after the shutdown
+      # timeout passes.
+      @thread.raise ::Sidekiq::Shutdown
+      @thread.value if wait
+    end
+
+    def start
+      @thread ||= safe_thread("processor", &method(:run))
+    end
+
+    private unless $TESTING
+
+    def run
+      begin
+        while !@done
+          process_one
         end
+        @mgr.processor_stopped(self)
+      rescue Sidekiq::Shutdown
+        @mgr.processor_stopped(self)
+      rescue Exception => ex
+        @mgr.processor_died(self, ex)
       end
     end
 
-    attr_accessor :proxy_id
+    def process_one
+      @job = fetch
+      process(@job) if @job
+      @job = nil
+    end
+
+    def get_one
+      begin
+        work = @strategy.retrieve_work
+        (logger.info { "Redis is online, #{Time.now - @down} sec downtime" }; @down = nil) if @down
+        work
+      rescue Sidekiq::Shutdown
+      rescue => ex
+        handle_fetch_exception(ex)
+      end
+    end
 
-    def initialize(boss)
-      @boss = boss
+    def fetch
+      j = get_one
+      if j && @done
+        j.requeue
+        nil
+      else
+        j
+      end
+    end
+
+    def handle_fetch_exception(ex)
+      if !@down
+        @down = Time.now
+        logger.error("Error fetching job: #{ex}")
+        ex.backtrace.each do |bt|
+          logger.error(bt)
+        end
+      end
+      sleep(1)
     end
 
     def process(work)
-      msgstr = work.message
+      jobstr = work.job
       queue = work.queue_name
 
-      @boss.async.real_thread(proxy_id, Thread.current)
-
-      ack = true
+      ack = false
       begin
-        msg = Sidekiq.load_json(msgstr)
-        klass  = msg['class'].constantize
+        job = Sidekiq.load_json(jobstr)
+        klass  = job['class'.freeze].constantize
         worker = klass.new
-        worker.jid = msg['jid']
-
-        stats(worker, msg, queue) do
-          Sidekiq.server_middleware.invoke(worker, msg, queue) do
-            execute_job(worker, cloned(msg['args']))
+        worker.jid = job['jid'.freeze]
+
+        stats(worker, job, queue) do
+          Sidekiq.server_middleware.invoke(worker, job, queue) do
+            # Only ack if we either attempted to start this job or
+            # successfully completed it. This prevents us from
+            # losing jobs if a middleware raises an exception before yielding
+            ack = true
+            execute_job(worker, cloned(job['args'.freeze]))
           end
         end
+        ack = true
       rescue Sidekiq::Shutdown
         # Had to force kill this job because it didn't finish
         # within the timeout.  Don't acknowledge the work since
         # we didn't properly finish it.
         ack = false
       rescue Exception => ex
-        handle_exception(ex, msg || { :message => msgstr })
+        handle_exception(ex, job || { :job => jobstr })
         raise
       ensure
         work.acknowledge if ack
       end
-
-      @boss.async.processor_done(current_actor)
-    end
-
-    def inspect
-      "<Processor##{object_id.to_s(16)}>"
     end
 
     def execute_job(worker, cloned_args)
       worker.perform(*cloned_args)
     end
 
-    private
-
     def thread_identity
       @str ||= Thread.current.object_id.to_s(36)
     end
 
-    def stats(worker, msg, queue)
-      # Do not conflate errors from the job with errors caused by updating
-      # stats so calling code can react appropriately
-      retry_and_suppress_exceptions do
-        hash = Sidekiq.dump_json({:queue => queue, :payload => msg, :run_at => Time.now.to_i })
-        Sidekiq.redis do |conn|
-          conn.multi do
-            conn.hmset("#{identity}:workers", thread_identity, hash)
-            conn.expire("#{identity}:workers", 60*60*4)
-          end
-        end
-      end
+    WORKER_STATE = Concurrent::Map.new
+    PROCESSED = Concurrent::AtomicFixnum.new
+    FAILURE = Concurrent::AtomicFixnum.new
+
+    def stats(worker, job, queue)
+      tid = thread_identity
+      WORKER_STATE[tid] = {:queue => queue, :payload => job, :run_at => Time.now.to_i }
 
       begin
         yield
       rescue Exception
-        retry_and_suppress_exceptions do
-          failed = "stat:failed:#{Time.now.utc.to_date}"
-          Sidekiq.redis do |conn|
-            conn.multi do
-              conn.incrby("stat:failed", 1)
-              conn.incrby(failed, 1)
-              conn.expire(failed, STATS_TIMEOUT)
-            end
-          end
-        end
+        FAILURE.increment
         raise
       ensure
-        retry_and_suppress_exceptions do
-          processed = "stat:processed:#{Time.now.utc.to_date}"
-          Sidekiq.redis do |conn|
-            conn.multi do
-              conn.hdel("#{identity}:workers", thread_identity)
-              conn.incrby("stat:processed", 1)
-              conn.incrby(processed, 1)
-              conn.expire(processed, STATS_TIMEOUT)
-            end
-          end
-        end
+        WORKER_STATE.delete(tid)
+        PROCESSED.increment
       end
     end
 
     # Deep clone the arguments passed to the worker so that if
-    # the message fails, what is pushed back onto Redis hasn't
+    # the job fails, what is pushed back onto Redis hasn't
     # been mutated by the worker.
     def cloned(ary)
       Marshal.load(Marshal.dump(ary))
     end
 
-    # If an exception occurs in the block passed to this method, that block will be retried up to max_retries times.
-    # All exceptions will be swallowed and logged.
-    def retry_and_suppress_exceptions(max_retries = 5)
-      retry_count = 0
-      begin
-        yield
-      rescue => e
-        retry_count += 1
-        if retry_count <= max_retries
-          Sidekiq.logger.debug {"Suppressing and retrying error: #{e.inspect}"}
-          pause_for_recovery(retry_count)
-          retry
-        else
-          handle_exception(e, { :message => "Exhausted #{max_retries} retries"})
-        end
-      end
-    end
-
-    def pause_for_recovery(retry_count)
-      sleep(retry_count)
-    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -9,10 +9,11 @@ module Sidekiq
       def create(options={})
         options[:url] ||= determine_redis_provider
 
-        # need a connection for Fetcher and Retry
-        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 2) : 5)
-        pool_timeout = options[:pool_timeout] || 1
+        size = options[:size] || (Sidekiq.server? ? (Sidekiq.options[:concurrency] + 5) : 5)
+
+        verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
+        pool_timeout = options[:pool_timeout] || 1
         log_info(options)
 
         ConnectionPool.new(:timeout => pool_timeout, :size => size) do
@@ -22,13 +23,30 @@ module Sidekiq
 
       private
 
+      # Sidekiq needs a lot of concurrent Redis connections.
+      #
+      # We need a connection for each Processor.
+      # We need a connection for Pro's real-time change listener
+      # We need a connection to various features to call Redis every few seconds:
+      #   - the process heartbeat.
+      #   - enterprise's leader election
+      #   - enterprise's cron support
+      def verify_sizing(size, concurrency)
+        raise ArgumentError, "Your Redis connection pool is too small for Sidekiq to work, your pool has #{size} connections but really needs to have at least #{concurrency + 2}" if size <= concurrency
+      end
+
       def build_client(options)
         namespace = options[:namespace]
 
         client = Redis.new client_opts(options)
         if namespace
-          require 'redis/namespace'
-          Redis::Namespace.new(namespace, :redis => client)
+          begin
+            require 'redis/namespace'
+            Redis::Namespace.new(namespace, :redis => client)
+          rescue LoadError
+            Sidekiq.logger.error("redis-namespace gem not included in Gemfile, cannot use namespace '#{namespace}'")
+            exit(-127)
+          end
         else
           client
         end

data/lib/sidekiq/scheduled.rb

@@ -1,6 +1,5 @@
 require 'sidekiq'
 require 'sidekiq/util'
-require 'sidekiq/actor'
 require 'sidekiq/api'
 
 module Sidekiq
@@ -17,7 +16,7 @@ module Sidekiq
             # We need to go through the list one at a time to reduce the risk of something
             # going wrong between the time jobs are popped from the scheduled queue and when
             # they are pushed onto a work queue and losing the jobs.
-            while job = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
+            while job = conn.zrangebyscore(sorted_set, '-inf'.freeze, now, :limit => [0, 1]).first do
 
               # Pop item off the queue and add it to the work queue. If the job can't be popped from
               # the queue, it's because another process already popped it so we can move on to the
@@ -39,33 +38,56 @@ module Sidekiq
     # workers can pick it up like any other job.
     class Poller
       include Util
-      include Actor
 
       INITIAL_WAIT = 10
 
       def initialize
         @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+        @sleeper = ConnectionPool::TimedStack.new
+        @done = false
       end
 
-      def poll(first_time=false)
-        watchdog('scheduling poller thread died!') do
-          initial_wait if first_time
-
-          begin
-            @enq.enqueue_jobs
-          rescue => ex
-            # Most likely a problem with redis networking.
-            # Punt and try again at the next interval
-            logger.error ex.message
-            logger.error ex.backtrace.first
+      # Shut down this instance, will pause until the thread is dead.
+      def terminate
+        @done = true
+        if @thread
+          t = @thread
+          @thread = nil
+          @sleeper << 0
+          t.value
+        end
+      end
+
+      def start
+        @thread ||= safe_thread("scheduler") do
+          initial_wait
+
+          while !@done
+            enqueue
+            wait
           end
+          Sidekiq.logger.info("Scheduler exiting...")
+        end
+      end
 
-          after(random_poll_interval) { poll }
+      def enqueue
+        begin
+          @enq.enqueue_jobs
+        rescue => ex
+          # Most likely a problem with redis networking.
+          # Punt and try again at the next interval
+          logger.error ex.message
+          logger.error ex.backtrace.first
         end
       end
 
       private
 
+      def wait
+        @sleeper.pop(random_poll_interval)
+      rescue Timeout::Error
+      end
+
       # Calculates a random interval that is ±50% the desired average.
       def random_poll_interval
         poll_interval_average * rand + poll_interval_average.to_f / 2
@@ -83,7 +105,7 @@ module Sidekiq
       # all your Sidekiq processes at the same time will lead to them all polling at
       # the same time: the thundering herd problem.
       #
-      # We only do this if poll_interval is unset (the default).
+      # We only do this if poll_interval_average is unset (the default).
       def poll_interval_average
         Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
       end
@@ -98,16 +120,15 @@ module Sidekiq
       end
 
       def initial_wait
-        begin
-          # Have all processes sleep between 5-15 seconds.  10 seconds
-          # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
-          # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
-          sleep(INITIAL_WAIT) unless Sidekiq.options[:poll_interval_average]
-          sleep(5 * rand)
-        rescue Celluloid::Task::TerminatedError
-          # Hit Ctrl-C when Sidekiq is finished booting and we have a chance
-          # to get here.
-        end
+        # Have all processes sleep between 5-15 seconds.  10 seconds
+        # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
+        # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
+        total = 0
+        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += (5 * rand)
+
+        @sleeper.pop(total)
+      rescue Timeout::Error
       end
 
     end

data/lib/sidekiq/testing.rb

@@ -48,6 +48,12 @@ module Sidekiq
       def inline?
         self.__test_mode == :inline
       end
+
+      def server_middleware
+        @server_chain ||= Middleware::Chain.new
+        yield @server_chain if block_given?
+        @server_chain
+      end
     end
   end
 
@@ -62,15 +68,15 @@ module Sidekiq
     def raw_push(payloads)
       if Sidekiq::Testing.fake?
         payloads.each do |job|
-          job['class'].constantize.jobs << Sidekiq.load_json(Sidekiq.dump_json(job))
+          Queues.jobs[job['queue']] << Sidekiq.load_json(Sidekiq.dump_json(job))
         end
         true
       elsif Sidekiq::Testing.inline?
         payloads.each do |job|
-          job['jid'] ||= SecureRandom.hex(12)
           klass = job['class'].constantize
-          klass.jobs.unshift Sidekiq.load_json(Sidekiq.dump_json(job))
-          klass.perform_one
+          job['id'] ||= SecureRandom.hex(12)
+          job_hash = Sidekiq.load_json(Sidekiq.dump_json(job))
+          klass.process_job(job_hash)
         end
         true
       else
@@ -79,6 +85,64 @@ module Sidekiq
     end
   end
 
+  module Queues
+    ##
+    # The Queues class is only for testing the fake queue implementation.
+    # The data is structured as a hash with queue name as hash key and array
+    # of job data as the value.
+    #
+    # {
+    #   "default"=>[
+    #     {
+    #       "class"=>"TestTesting::QueueWorker",
+    #       "args"=>[1, 2],
+    #       "retry"=>true,
+    #       "queue"=>"default",
+    #       "jid"=>"abc5b065c5c4b27fc1102833",
+    #       "created_at"=>1447445554.419934
+    #     }
+    #   ]
+    # }
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, Sidekiq::Queues["default"].size
+    #   assert_equal :something, Sidekiq::Queues["default"].first['args'][0]
+    #
+    # You can also clear all workers' jobs:
+    #
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #   HardWorker.perform_async(:something)
+    #   Sidekiq::Queues.clear_all
+    #   assert_equal 0, Sidekiq::Queues["default"].size
+    #
+    # This can be useful to make sure jobs don't linger between tests:
+    #
+    #   RSpec.configure do |config|
+    #     config.before(:each) do
+    #       Sidekiq::Queues.clear_all
+    #     end
+    #   end
+    #
+    class << self
+      def [](queue)
+        jobs[queue.to_s]
+      end
+
+      def jobs
+        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+      end
+
+      def clear_all
+        jobs.clear
+      end
+    end
+  end
+
   module Worker
     ##
     # The Sidekiq testing infrastructure overrides perform_async
@@ -137,34 +201,45 @@ module Sidekiq
     #
     module ClassMethods
 
+      # Queue for this worker
+      def queue
+        self.sidekiq_options["queue"].to_s
+      end
+
       # Jobs queued for this worker
       def jobs
-        Worker.jobs[self]
+        Queues.jobs[queue].select { |job| job["class"] == self.to_s }
       end
 
       # Clear all jobs for this worker
       def clear
-        jobs.clear
+        Queues.jobs[queue].clear
       end
 
       # Drain and run all jobs for this worker
       def drain
-        while job = jobs.shift do
-          worker = new
-          worker.jid = job['jid']
-          worker.bid = job['bid'] if worker.respond_to?(:bid=)
-          execute_job(worker, job['args'])
+        while jobs.any?
+          next_job = jobs.first
+          Queues.jobs[queue].delete_if { |job| job["jid"] == next_job["jid"] }
+          process_job(next_job)
         end
       end
 
       # Pop out a single job and perform it
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
-        job = jobs.shift
+        next_job = jobs.first
+        Queues.jobs[queue].delete_if { |job| job["jid"] == next_job["jid"] }
+        process_job(next_job)
+      end
+
+      def process_job(job)
         worker = new
         worker.jid = job['jid']
         worker.bid = job['bid'] if worker.respond_to?(:bid=)
-        execute_job(worker, job['args'])
+        Sidekiq::Testing.server_middleware.invoke(worker, job, job['queue']) do
+          execute_job(worker, job['args'])
+        end
       end
 
       def execute_job(worker, args)
@@ -174,18 +249,22 @@ module Sidekiq
 
     class << self
       def jobs # :nodoc:
-        @jobs ||= Hash.new { |hash, key| hash[key] = [] }
+        Queues.jobs.values.flatten
       end
 
       # Clear all queued jobs across all workers
       def clear_all
-        jobs.clear
+        Queues.clear_all
       end
 
       # Drain all queued jobs across all workers
       def drain_all
-        until jobs.values.all?(&:empty?) do
-          jobs.keys.each(&:drain)
+        while jobs.any?
+          worker_classes = jobs.map { |job| job["class"] }.uniq
+
+          worker_classes.each do |worker_class|
+            worker_class.constantize.drain
+          end
         end
       end
     end