resque 1.26.0 → 1.27.0

data/lib/resque/thread_signal.rb

@@ -0,0 +1,45 @@
+class Resque::ThreadSignal
+  if RUBY_VERSION <= "1.9"
+    def initialize
+      @signaled = false
+    end
+
+    def signal
+      @signaled = true
+    end
+
+    def wait_for_signal(timeout)
+      (10 * timeout).times do
+        sleep(0.1)
+        return true if @signaled
+      end
+
+      @signaled
+    end
+
+  else
+    def initialize
+      @mutex = Mutex.new
+      @signaled = false
+      @received = ConditionVariable.new
+    end
+
+    def signal
+      @mutex.synchronize do
+        @signaled = true
+        @received.signal
+      end
+    end
+
+    def wait_for_signal(timeout)
+      @mutex.synchronize do
+        unless @signaled
+          @received.wait(@mutex, timeout)
+        end
+
+        @signaled
+      end
+    end
+
+  end
+end

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.26.0'
+  Version = VERSION = '1.27.0'
 end

data/lib/resque/worker.rb

@@ -14,16 +14,25 @@ module Resque
     extend Resque::Helpers
     include Resque::Logging
 
-    WORKER_HEARTBEAT_KEY = "workers:heartbeat"
+    @@all_heartbeat_threads = []
+    def self.kill_all_heartbeat_threads
+      @@all_heartbeat_threads.each(&:kill).each(&:join)
+      @@all_heartbeat_threads = []
+    end
 
     def redis
       Resque.redis
     end
+    alias :data_store :redis
 
     def self.redis
       Resque.redis
     end
 
+    def self.data_store
+      self.redis
+    end
+
     # Given a Ruby object, returns a string suitable for storage in a
     # queue.
     def encode(object)
@@ -48,12 +57,14 @@ module Resque
     # registered in the application. Otherwise, forked workers exit with `exit!`
     attr_accessor :run_at_exit_hooks
 
+    attr_writer :fork_per_job
+    attr_writer :hostname
     attr_writer :to_s
     attr_writer :pid
 
     # Returns an array of all worker objects.
     def self.all
-      Array(redis.smembers(:workers)).map { |id| find(id, :skip_exists => true) }.compact
+      data_store.worker_ids.map { |id| find(id, :skip_exists => true) }.compact
     end
 
     # Returns an array of all worker objects currently processing
@@ -62,17 +73,15 @@ module Resque
       names = all
       return [] unless names.any?
 
-      names.map! { |name| "worker:#{name}" }
-
       reportedly_working = {}
 
       begin
-        reportedly_working = redis.mapped_mget(*names).reject do |key, value|
+        reportedly_working = data_store.workers_map(names).reject do |key, value|
           value.nil? || value.empty?
         end
       rescue Redis::Distributed::CannotDistribute
         names.each do |name|
-          value = redis.get name
+          value = data_store.get_worker_payload(name)
           reportedly_working[name] = value unless value.nil? || value.empty?
         end
       end
@@ -92,6 +101,7 @@ module Resque
         host, pid, queues_raw = worker_id.split(':')
         queues = queues_raw.split(',')
         worker = new(*queues)
+        worker.hostname = host
         worker.to_s = worker_id
         worker.pid = pid.to_i
         worker
@@ -108,7 +118,7 @@ module Resque
     # Given a string worker id, return a boolean indicating whether the
     # worker exists
     def self.exists?(worker_id)
-      redis.sismember(:workers, worker_id)
+      data_store.worker_exists?(worker_id)
     end
 
     # Workers should be initialized with an array of string queue
@@ -122,31 +132,41 @@ module Resque
     # If passed a single "*", this Worker will operate on all queues
     # in alphabetical order. Queues can be dynamically added or
     # removed without needing to restart workers using this method.
+    #
+    # Workers should have `#prepare` called after they are initialized
+    # if you are running work on the worker.
     def initialize(*queues)
       @shutdown = nil
       @paused = nil
       @before_first_fork_hook_ran = false
 
-      self.verbose = ENV['LOGGING'] || ENV['VERBOSE']
-      self.very_verbose = ENV['VVERBOSE']
+      verbose_value = ENV['LOGGING'] || ENV['VERBOSE']
+      self.verbose = verbose_value if verbose_value
+      self.very_verbose = ENV['VVERBOSE'] if ENV['VVERBOSE']
       self.term_timeout = ENV['RESQUE_TERM_TIMEOUT'] || 4.0
       self.term_child = ENV['TERM_CHILD']
       self.graceful_term = ENV['GRACEFUL_TERM']
       self.run_at_exit_hooks = ENV['RUN_AT_EXIT_HOOKS']
 
+      self.queues = queues
+    end
+
+    # Daemonizes the worker if ENV['BACKGROUND'] is set and writes
+    # the process id to ENV['PIDFILE'] if set. Should only be called
+    # once per worker.
+    def prepare
       if ENV['BACKGROUND']
         unless Process.respond_to?('daemon')
             abort "env var BACKGROUND is set, which requires ruby >= 1.9"
         end
         Process.daemon(true)
-        self.reconnect
       end
 
       if ENV['PIDFILE']
         File.open(ENV['PIDFILE'], 'w') { |f| f << pid }
       end
 
-      self.queues = queues
+      self.reconnect if ENV['BACKGROUND']
     end
 
     def queues=(queues)
@@ -200,46 +220,12 @@ module Resque
     # has completed processing. Useful for testing.
     def work(interval = 5.0, &block)
       interval = Float(interval)
-      $0 = "resque: Starting"
       startup
 
       loop do
         break if shutdown?
 
-        if not paused? and job = reserve
-          log_with_severity :info, "got: #{job.inspect}"
-          job.worker = self
-          working_on job
-
-          procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
-          if @child = fork(job)
-            srand # Reseeding
-            procline "Forked #{@child} at #{Time.now.to_i}"
-            begin
-              Process.waitpid(@child)
-            rescue SystemCallError
-              nil
-            end
-            job.fail(DirtyExit.new("Child process received unhandled signal #{$?.stopsig}")) if $?.signaled?
-          else
-            unregister_signal_handlers if will_fork? && term_child
-            begin
-
-              reconnect if will_fork?
-              perform(job, &block)
-
-            rescue Exception => exception
-              report_failed_job(job,exception)
-            end
-
-            if will_fork?
-              run_at_exit_hooks ? exit : exit!
-            end
-          end
-
-          done_working
-          @child = nil
-        else
+        unless work_one_job(&block)
           break if interval.zero?
           log_with_severity :debug, "Sleeping for #{interval} seconds"
           procline paused? ? "Paused" : "Waiting for #{queues.join(',')}"
@@ -249,11 +235,29 @@ module Resque
 
       unregister_worker
     rescue Exception => exception
-      unless exception.class == SystemExit && !@child && run_at_exit_hooks
-        log_with_severity :error, "Failed to start worker : #{exception.inspect}"
+      return if exception.class == SystemExit && !@child && run_at_exit_hooks
+      log_with_severity :error, "Failed to start worker : #{exception.inspect}"
+      unregister_worker(exception)
+    end
+
+    def work_one_job(job = nil, &block)
+      return false if paused?
+      return false unless job ||= reserve
+
+      working_on job
+      procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
 
-        unregister_worker(exception)
+      log_with_severity :info, "got: #{job.inspect}"
+      job.worker = self
+
+      if fork_per_job?
+        perform_with_fork(job, &block)
+      else
+        perform(job, &block)
       end
+
+      done_working
+      true
     end
 
     # DEPRECATED. Processes a single job. If none is given, it will
@@ -283,10 +287,14 @@ module Resque
       end
     end
 
+
     # Processes a given job in the child.
     def perform(job)
       begin
-        run_hook :after_fork, job if will_fork?
+        if fork_per_job?
+          reconnect
+          run_hook :after_fork, job
+        end
         job.perform
       rescue Object => e
         report_failed_job(job,e)
@@ -320,7 +328,7 @@ module Resque
     def reconnect
       tries = 0
       begin
-        redis.client.reconnect
+        data_store.reconnect
       rescue Redis::BaseConnectionError
         if (tries += 1) <= 3
           log_with_severity :error, "Error reconnecting to Redis; retrying"
@@ -333,30 +341,10 @@ module Resque
       end
     end
 
-    # Not every platform supports fork. Here we do our magic to
-    # determine if yours does.
-    def fork(job)
-      return unless will_fork?
-
-      # Only run before_fork hooks if we're actually going to fork
-      # (after checking will_fork?)
-      run_hook :before_fork, job
-
-      begin
-        # IronRuby doesn't support `Kernel.fork` yet
-        if Kernel.respond_to?(:fork)
-          Kernel.fork
-        else
-          raise NotImplementedError
-        end
-      rescue NotImplementedError
-        @cant_fork = true
-        nil
-      end
-    end
-
     # Runs all the methods needed when a worker begins its lifecycle.
     def startup
+      $0 = "resque: Starting"
+
       enable_gc_optimizations
       register_signal_handlers
       start_heartbeat
@@ -468,12 +456,19 @@ module Resque
     end
 
     def heartbeat
-      heartbeat = redis.hget(WORKER_HEARTBEAT_KEY, to_s)
-      heartbeat && Time.parse(heartbeat)
+      data_store.heartbeat(self)
+    end
+
+    def remove_heartbeat
+      data_store.remove_heartbeat(self)
+    end
+
+    def heartbeat!(time = data_store.server_time)
+      data_store.heartbeat!(self, time)
     end
 
     def self.all_heartbeats
-      redis.hgetall(WORKER_HEARTBEAT_KEY)
+      data_store.all_heartbeats
     end
 
     # Returns a list of workers that have sent a heartbeat in the past, but which
@@ -481,13 +476,14 @@ module Resque
     def self.all_workers_with_expired_heartbeats
       workers = Worker.all
       heartbeats = Worker.all_heartbeats
+      now = data_store.server_time
 
       workers.select do |worker|
         id = worker.to_s
         heartbeat = heartbeats[id]
 
         if heartbeat
-          seconds_since_heartbeat = (Time.now - Time.parse(heartbeat)).to_i
+          seconds_since_heartbeat = (now - Time.parse(heartbeat)).to_i
           seconds_since_heartbeat > Resque.prune_interval
         else
           false
@@ -495,18 +491,20 @@ module Resque
       end
     end
 
-    def heartbeat!(time = Time.now)
-      redis.hset(WORKER_HEARTBEAT_KEY, to_s, time.iso8601)
-    end
-
     def start_heartbeat
-      heartbeat!
-      @heart = Thread.new do
+      remove_heartbeat
+
+      @heartbeat_thread_signal = Resque::ThreadSignal.new
+
+      @heartbeat_thread = Thread.new do
         loop do
-          sleep(Resque.heartbeat_interval)
           heartbeat!
+          signaled = @heartbeat_thread_signal.wait_for_signal(Resque.heartbeat_interval)
+          break if signaled
         end
       end
+
+      @@all_heartbeat_threads << @heartbeat_thread
     end
 
     # Kills the forked child immediately with minimal remorse. The job it
@@ -604,10 +602,7 @@ module Resque
     # Registers ourself as a worker. Useful when entering the worker
     # lifecycle on startup.
     def register_worker
-      redis.pipelined do
-        redis.sadd(:workers, self)
-        started!
-      end
+      data_store.register_worker(self)
     end
 
     # Runs a named hook, passing along any arguments.
@@ -625,7 +620,10 @@ module Resque
     end
 
     def kill_background_threads
-      @heart.kill if @heart
+      if @heartbeat_thread
+        @heartbeat_thread_signal.signal
+        @heartbeat_thread.join
+      end
     end
 
     # Unregisters ourself as a worker. Useful when shutting down.
@@ -646,20 +644,15 @@ module Resque
 
       kill_background_threads
 
-      redis.pipelined do
-        redis.srem(:workers, self)
-        redis.del("worker:#{self}")
-        redis.del("worker:#{self}:started")
-        redis.hdel(WORKER_HEARTBEAT_KEY, self.to_s)
-
+      data_store.unregister_worker(self) do
         Stat.clear("processed:#{self}")
         Stat.clear("failed:#{self}")
       end
     rescue Exception => exception_while_unregistering
       message = exception_while_unregistering.message
       if exception
-        message = message + "\nOriginal Exception (#{exception.class}): #{exception.message}\n" +
-                            "  #{exception.backtrace.join("  \n")}"
+        message += "\nOriginal Exception (#{exception.class}): #{exception.message}"
+        message += "\n  #{exception.backtrace.join("  \n")}" if exception.backtrace
       end
       fail(exception_while_unregistering.class,
            message,
@@ -673,15 +666,14 @@ module Resque
         :queue   => job.queue,
         :run_at  => Time.now.utc.iso8601,
         :payload => job.payload
-      redis.set("worker:#{self}", data)
+      data_store.set_worker_payload(self,data)
     end
 
     # Called when we are done working - clears our `working_on` state
     # and tells Redis we processed a job.
     def done_working
-      redis.pipelined do
+      data_store.worker_done_working(self) do
         processed!
-        redis.del("worker:#{self}")
       end
     end
 
@@ -709,18 +701,18 @@ module Resque
 
     # What time did this worker start? Returns an instance of `Time`
     def started
-      redis.get "worker:#{self}:started"
+      data_store.worker_start_time(self)
     end
 
     # Tell Redis we've started
     def started!
-      redis.set("worker:#{self}:started", Time.now.to_s)
+      data_store.worker_started(self)
     end
 
     # Returns a hash explaining the Job we're currently processing, if any.
     def job(reload = true)
       @job = nil if reload
-      @job ||= decode(redis.get("worker:#{self}")) || {}
+      @job ||= decode(data_store.get_worker_payload(self)) || {}
     end
     attr_writer :job
     alias_method :processing, :job
@@ -735,18 +727,15 @@ module Resque
       state == :idle
     end
 
-    def will_fork?
-      !@cant_fork && fork_per_job?
-    end
-
     def fork_per_job?
-      ENV["FORK_PER_JOB"] != 'false'
+      return @fork_per_job if defined?(@fork_per_job)
+      @fork_per_job = ENV["FORK_PER_JOB"] != 'false' && Kernel.respond_to?(:fork)
     end
 
     # Returns a symbol representing the current worker state,
     # which can be either :working or :idle
     def state
-      redis.exists("worker:#{self}") ? :working : :idle
+      data_store.get_worker_payload(self) ? :working : :idle
     end
 
     # Is this worker the same as another worker?
@@ -765,9 +754,9 @@ module Resque
     end
     alias_method :id, :to_s
 
-    # chomp'd hostname of this machine
+    # chomp'd hostname of this worker's machine
     def hostname
-      Socket.gethostname
+      @hostname ||= Socket.gethostname
     end
 
     # Returns Integer PID of running worker
@@ -797,7 +786,7 @@ module Resque
     # Find Resque worker pids on Linux and OS X.
     #
     def linux_worker_pids
-      `ps -A -o pid,command | grep -E "[r]esque:work|[r]esque-[0-9]" | grep -v "resque-web"`.split("\n").map do |line|
+      `ps -A -o pid,command | grep -E "[r]esque:work|[r]esque:\sStarting|[r]esque-[0-9]" | grep -v "resque-web"`.split("\n").map do |line|
         line.split(' ')[0]
       end
     end
@@ -868,6 +857,34 @@ module Resque
 
     private
 
+    def perform_with_fork(job, &block)
+      run_hook :before_fork, job
+
+      begin
+        @child = fork do
+          unregister_signal_handlers if term_child
+          perform(job, &block)
+          exit! unless run_at_exit_hooks
+        end
+      rescue NotImplementedError
+        @fork_per_job = false
+        perform(job, &block)
+        return
+      end
+
+      srand # Reseeding
+      procline "Forked #{@child} at #{Time.now.to_i}"
+
+      begin
+        Process.waitpid(@child)
+      rescue SystemCallError
+        nil
+      end
+
+      job.fail(DirtyExit.new("Child process received unhandled signal #{$?.stopsig}", $?)) if $?.signaled?
+      @child = nil
+    end
+
     def log_with_severity(severity, message)
       Logging.log(severity, message)
     end