resque 1.23.0 → 2.6.0

data/lib/resque/worker.rb

@@ -1,3 +1,7 @@
+require 'time'
+require 'set'
+require 'redis/distributed'
+
 module Resque
   # A Resque Worker processes jobs. On platforms that support fork(2),
   # the worker will fork off a child to process each job. This ensures
@@ -9,27 +13,63 @@ module Resque
   class Worker
     include Resque::Helpers
     extend Resque::Helpers
+    include Resque::Logging
+
+    @@all_heartbeat_threads = []
+    def self.kill_all_heartbeat_threads
+      @@all_heartbeat_threads.each(&:kill).each(&:join)
+      @@all_heartbeat_threads = []
+    end
+
+    def redis
+      Resque.redis
+    end
+    alias :data_store :redis
+
+    def self.redis
+      Resque.redis
+    end
 
-    # Whether the worker should log basic info to STDOUT
-    attr_accessor :verbose
+    def self.data_store
+      self.redis
+    end
 
-    # Whether the worker should log lots of info to STDOUT
-    attr_accessor  :very_verbose
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      Resque.encode(object)
+    end
 
-    # Boolean indicating whether this worker can or can not fork.
-    # Automatically set if a fork(2) fails.
-    attr_accessor :cant_fork
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      Resque.decode(object)
+    end
 
     attr_accessor :term_timeout
 
+    attr_accessor :pre_shutdown_timeout
+
+    attr_accessor :term_child_signal
+
     # decide whether to use new_kill_child logic
     attr_accessor :term_child
 
+    # should term kill workers gracefully (vs. immediately)
+    # Makes SIGTERM work like SIGQUIT
+    attr_accessor :graceful_term
+
+    # When set to true, forked workers will exit with `exit`, calling any `at_exit` code handlers that have been
+    # registered in the application. Otherwise, forked workers exit with `exit!`
+    attr_accessor :run_at_exit_hooks
+
+    attr_writer :fork_per_job
+    attr_writer :hostname
     attr_writer :to_s
+    attr_writer :pid
 
     # Returns an array of all worker objects.
     def self.all
-      Array(redis.smembers(:workers)).map { |id| find(id) }.compact
+      data_store.worker_ids.map { |id| find(id, :skip_exists => true) }.compact
     end
 
     # Returns an array of all worker objects currently processing
@@ -38,32 +78,37 @@ module Resque
       names = all
       return [] unless names.any?
 
-      names.map! { |name| "worker:#{name}" }
-
       reportedly_working = {}
 
       begin
-        reportedly_working = redis.mapped_mget(*names).reject do |key, value|
+        reportedly_working = data_store.workers_map(names).reject do |key, value|
           value.nil? || value.empty?
         end
       rescue Redis::Distributed::CannotDistribute
         names.each do |name|
-          value = redis.get name
+          value = data_store.get_worker_payload(name)
           reportedly_working[name] = value unless value.nil? || value.empty?
         end
       end
 
       reportedly_working.keys.map do |key|
-        find key.sub("worker:", '')
+        worker = find(key.sub("worker:", ''), :skip_exists => true)
+        worker.job = worker.decode(reportedly_working[key])
+        worker
       end.compact
     end
 
     # Returns a single worker object. Accepts a string id.
-    def self.find(worker_id)
-      if exists? worker_id
-        queues = worker_id.split(':')[-1].split(',')
+    def self.find(worker_id, options = {})
+      skip_exists = options[:skip_exists]
+
+      if skip_exists || exists?(worker_id)
+        host, pid, queues_raw = worker_id.split(':', 3)
+        queues = queues_raw.split(',')
         worker = new(*queues)
+        worker.hostname = host
         worker.to_s = worker_id
+        worker.pid = pid.to_i
         worker
       else
         nil
@@ -78,7 +123,7 @@ module Resque
     # Given a string worker id, return a boolean indicating whether the
     # worker exists
     def self.exists?(worker_id)
-      redis.sismember(:workers, worker_id)
+      data_store.worker_exists?(worker_id)
     end
 
     # Workers should be initialized with an array of string queue
@@ -92,10 +137,60 @@ module Resque
     # If passed a single "*", this Worker will operate on all queues
     # in alphabetical order. Queues can be dynamically added or
     # removed without needing to restart workers using this method.
+    #
+    # Workers should have `#prepare` called after they are initialized
+    # if you are running work on the worker.
     def initialize(*queues)
-      @queues = queues.map { |queue| queue.to_s.strip }
       @shutdown = nil
       @paused = nil
+      @before_first_fork_hook_ran = false
+
+      @heartbeat_thread = nil
+      @heartbeat_thread_signal = nil
+
+      @last_state = :idle
+
+      verbose_value = ENV['LOGGING'] || ENV['VERBOSE']
+      self.verbose = verbose_value if verbose_value
+      self.very_verbose = ENV['VVERBOSE'] if ENV['VVERBOSE']
+      self.pre_shutdown_timeout = (ENV['RESQUE_PRE_SHUTDOWN_TIMEOUT'] || 0.0).to_f
+      self.term_timeout = (ENV['RESQUE_TERM_TIMEOUT'] || 4.0).to_f
+      self.term_child = ENV['TERM_CHILD']
+      self.graceful_term = ENV['GRACEFUL_TERM']
+      self.run_at_exit_hooks = ENV['RUN_AT_EXIT_HOOKS']
+
+      self.queues = queues
+    end
+
+    # Daemonizes the worker if ENV['BACKGROUND'] is set and writes
+    # the process id to ENV['PIDFILE'] if set. Should only be called
+    # once per worker.
+    def prepare
+      if ENV['BACKGROUND']
+        Process.daemon(true)
+      end
+
+      if ENV['PIDFILE']
+        File.open(ENV['PIDFILE'], 'w') { |f| f << pid }
+      end
+
+      self.reconnect if ENV['BACKGROUND']
+    end
+
+    WILDCARDS = ['*', '?', '{', '}', '[', ']'].freeze
+
+    def queues=(queues)
+      queues = (ENV["QUEUES"] || ENV['QUEUE']).to_s.split(',') if queues.empty?
+      queues = queues.map { |queue| queue.to_s.strip }
+
+      @skip_queues, @queues = queues.partition { |queue| queue.start_with?('!') }
+      @skip_queues.map! { |queue| queue[1..-1] }
+
+      # The behavior of `queues` is dependent on the value of `@has_dynamic_queues: if it's true, the method returns the result of filtering @queues with `glob_match`
+      # if it's false, the method returns @queues directly. Since `glob_match` will cause skipped queues to be filtered out, we want to make sure it's called if we have @skip_queues.any?
+      @has_dynamic_queues =
+        @skip_queues.any? || WILDCARDS.any? { |char| @queues.join.include?(char) }
+
       validate_queues
     end
 
@@ -109,6 +204,25 @@ module Resque
       end
     end
 
+    # Returns a list of queues to use when searching for a job.
+    # A splat ("*") means you want every queue (in alpha order) - this
+    # can be useful for dynamically adding new queues.
+    def queues
+      if @has_dynamic_queues
+        current_queues = Resque.queues
+        @queues.map { |queue| glob_match(current_queues, queue) }.flatten.uniq
+      else
+        @queues
+      end
+    end
+
+    def glob_match(list, pattern)
+      list.select do |queue|
+        File.fnmatch?(pattern, queue) &&
+          @skip_queues.none? { |skip_pattern| File.fnmatch?(skip_pattern, queue) }
+      end.sort
+    end
+
     # This is the main workhorse method. Called on a Worker instance,
     # it begins the worker life cycle.
     #
@@ -127,46 +241,47 @@ module Resque
     # has completed processing. Useful for testing.
     def work(interval = 5.0, &block)
       interval = Float(interval)
-      $0 = "resque: Starting"
       startup
 
       loop do
         break if shutdown?
 
-        if not paused? and job = reserve
-          log "got: #{job.inspect}"
-          job.worker = self
-          working_on job
-
-          if @child = fork(job)
-            srand # Reseeding
-            procline "Forked #{@child} at #{Time.now.to_i}"
-            begin
-              Process.waitpid(@child)
-            rescue SystemCallError
-              nil
-            end
-          else
-            unregister_signal_handlers if will_fork? && term_child
-            procline "Processing #{job.queue} since #{Time.now.to_i}"
-            reconnect
-            perform(job, &block)
-            exit! if will_fork?
-          end
-
-          done_working
-          @child = nil
-        else
+        unless work_one_job(&block)
+          state_change
           break if interval.zero?
-          log! "Sleeping for #{interval} seconds"
-          procline paused? ? "Paused" : "Waiting for #{@queues.join(',')}"
+          log_with_severity :debug, "Sleeping for #{interval} seconds"
+          procline paused? ? "Paused" : "Waiting for #{queues.join(',')}"
           sleep interval
         end
       end
 
       unregister_worker
+      run_hook :worker_exit
     rescue Exception => exception
+      return if exception.class == SystemExit && !@child && run_at_exit_hooks
+      log_with_severity :error, "Failed to start worker : #{exception.inspect}"
       unregister_worker(exception)
+      run_hook :worker_exit
+    end
+
+    def work_one_job(job = nil, &block)
+      return false if paused?
+      return false unless job ||= reserve
+
+      working_on job
+      procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
+
+      log_with_severity :info, "got: #{job.inspect}"
+      job.worker = self
+
+      if fork_per_job?
+        perform_with_fork(job, &block)
+      else
+        perform(job, &block)
+      end
+
+      done_working
+      true
     end
 
     # DEPRECATED. Processes a single job. If none is given, it will
@@ -181,21 +296,34 @@ module Resque
       done_working
     end
 
+    # Reports the exception and marks the job as failed
+    def report_failed_job(job,exception)
+      log_with_severity :error, "#{job.inspect} failed: #{exception.inspect}"
+      begin
+        job.fail(exception)
+      rescue Object => exception
+        log_with_severity :error, "Received exception when reporting failure: #{exception.inspect}"
+      end
+      begin
+        failed!
+      rescue Object => exception
+        log_with_severity :error, "Received exception when increasing failed jobs counter (redis issue) : #{exception.inspect}"
+      end
+    end
+
+
     # Processes a given job in the child.
     def perform(job)
       begin
-        run_hook :after_fork, job if will_fork?
+        if fork_per_job?
+          reconnect
+          run_hook :after_fork, job
+        end
         job.perform
       rescue Object => e
-        log "#{job.inspect} failed: #{e.inspect}"
-        begin
-          job.fail(e)
-        rescue Object => e
-          log "Received exception when reporting failure: #{e.inspect}"
-        end
-        failed!
+        report_failed_job(job,e)
       else
-        log "done: #{job.inspect}"
+        log_with_severity :info, "done: #{job.inspect}"
       ensure
         yield job if block_given?
       end
@@ -205,17 +333,17 @@ module Resque
     # nil if no job can be found.
     def reserve
       queues.each do |queue|
-        log! "Checking #{queue}"
+        log_with_severity :debug, "Checking #{queue}"
         if job = Resque.reserve(queue)
-          log! "Found job on #{queue}"
+          log_with_severity :debug, "Found job on #{queue}"
           return job
         end
       end
 
       nil
     rescue Exception => e
-      log "Error reserving job: #{e.inspect}"
-      log e.backtrace.join("\n")
+      log_with_severity :error, "Error reserving job: #{e.inspect}"
+      log_with_severity :error, e.backtrace.join("\n")
       raise e
     end
 
@@ -224,53 +352,26 @@ module Resque
     def reconnect
       tries = 0
       begin
-        redis.client.reconnect
+        data_store.reconnect
       rescue Redis::BaseConnectionError
         if (tries += 1) <= 3
-          log "Error reconnecting to Redis; retrying"
+          log_with_severity :error, "Error reconnecting to Redis; retrying"
           sleep(tries)
           retry
         else
-          log "Error reconnecting to Redis; quitting"
+          log_with_severity :error, "Error reconnecting to Redis; quitting"
           raise
         end
       end
     end
 
-    # Returns a list of queues to use when searching for a job.
-    # A splat ("*") means you want every queue (in alpha order) - this
-    # can be useful for dynamically adding new queues.
-    def queues
-      @queues.map {|queue| queue == "*" ? Resque.queues.sort : queue }.flatten.uniq
-    end
-
-    # Not every platform supports fork. Here we do our magic to
-    # determine if yours does.
-    def fork(job)
-      return if @cant_fork
-      
-      # Only run before_fork hooks if we're actually going to fork
-      # (after checking @cant_fork)
-      run_hook :before_fork, job
-
-      begin
-        # IronRuby doesn't support `Kernel.fork` yet
-        if Kernel.respond_to?(:fork)
-          Kernel.fork if will_fork?
-        else
-          raise NotImplementedError
-        end
-      rescue NotImplementedError
-        @cant_fork = true
-        nil
-      end
-    end
-
     # Runs all the methods needed when a worker begins its lifecycle.
     def startup
-      warn "WARNING: This way of doing signal handling is now deprecated. Please see http://hone.heroku.com/resque/2012/08/21/resque-signals.html for more info." unless term_child
+      $0 = "resque: Starting"
+
       enable_gc_optimizations
       register_signal_handlers
+      start_heartbeat
       prune_dead_workers
       run_hook :before_first_fork
       register_worker
@@ -297,7 +398,7 @@ module Resque
     # USR2: Don't process any new jobs
     # CONT: Start processing jobs again after a USR2
     def register_signal_handlers
-      trap('TERM') { shutdown!  }
+      trap('TERM') { graceful_term ? shutdown : shutdown!  }
       trap('INT')  { shutdown!  }
 
       begin
@@ -310,14 +411,21 @@ module Resque
         trap('USR2') { pause_processing }
         trap('CONT') { unpause_processing }
       rescue ArgumentError
-        warn "Signals QUIT, USR1, USR2, and/or CONT not supported."
+        log_with_severity :warn, "Signals QUIT, USR1, USR2, and/or CONT not supported."
       end
 
-      log! "Registered signals"
+      log_with_severity :debug, "Registered signals"
     end
 
     def unregister_signal_handlers
-      trap('TERM') { raise TermException.new("SIGTERM") }
+      trap('TERM') do
+        trap('TERM') do
+          # Ignore subsequent term signals
+        end
+
+        raise TermException.new("SIGTERM")
+      end
+
       trap('INT', 'DEFAULT')
 
       begin
@@ -331,15 +439,24 @@ module Resque
     # Schedule this worker for shutdown. Will finish processing the
     # current job.
     def shutdown
-      log 'Exiting...'
+      log_with_severity :info, 'Exiting...'
       @shutdown = true
     end
 
     # Kill the child and shutdown immediately.
+    # If not forking, abort this process.
     def shutdown!
       shutdown
       if term_child
-        new_kill_child
+        if fork_per_job?
+          new_kill_child
+        else
+          # Raise TermException in the same process
+          trap('TERM') do
+            # ignore subsequent terms
+          end
+          raise TermException.new("SIGTERM")
+        end
       else
         kill_child
       end
@@ -354,54 +471,130 @@ module Resque
     # is processing will not be completed.
     def kill_child
       if @child
-        log! "Killing child at #{@child}"
-        if system("ps -o pid,state -p #{@child}")
+        log_with_severity :debug, "Killing child at #{@child}"
+        if `ps -o pid,state -p #{@child}`
           Process.kill("KILL", @child) rescue nil
         else
-          log! "Child #{@child} not found, restarting."
+          log_with_severity :debug, "Child #{@child} not found, restarting."
           shutdown
         end
       end
     end
 
+    def heartbeat
+      data_store.heartbeat(self)
+    end
+
+    def remove_heartbeat
+      data_store.remove_heartbeat(self)
+    end
+
+    def heartbeat!(time = data_store.server_time)
+      data_store.heartbeat!(self, time)
+    end
+
+    def self.all_heartbeats
+      data_store.all_heartbeats
+    end
+
+    # Returns a list of workers that have sent a heartbeat in the past, but which
+    # already expired (does NOT include workers that have never sent a heartbeat at all).
+    def self.all_workers_with_expired_heartbeats
+      # Use `Worker.all_heartbeats` instead of `Worker.all`
+      # to prune workers which haven't been registered but have set a heartbeat.
+      # https://github.com/resque/resque/pull/1751
+      heartbeats = Worker.all_heartbeats
+      now = data_store.server_time
+
+      heartbeats.select do |id, heartbeat|
+        if heartbeat
+          seconds_since_heartbeat = (now - Time.parse(heartbeat)).to_i
+          seconds_since_heartbeat > Resque.prune_interval
+        else
+          false
+        end
+      end.each_key.map do |id|
+        # skip_exists must be true to include not registered workers
+        find(id, :skip_exists => true)
+      end
+    end
+
+    def start_heartbeat
+      remove_heartbeat
+
+      @heartbeat_thread_signal = Resque::ThreadSignal.new
+
+      @heartbeat_thread = Thread.new do
+        loop do
+          heartbeat!
+          signaled = @heartbeat_thread_signal.wait_for_signal(Resque.heartbeat_interval)
+          break if signaled
+        end
+      end
+
+      @@all_heartbeat_threads << @heartbeat_thread
+    end
+
     # Kills the forked child immediately with minimal remorse. The job it
     # is processing will not be completed. Send the child a TERM signal,
-    # wait 5 seconds, and then a KILL signal if it has not quit
+    # wait <term_timeout> seconds, and then a KILL signal if it has not quit
+    # If pre_shutdown_timeout has been set to a positive number, it will allow
+    # the child that many seconds before sending the aforementioned TERM and KILL.
     def new_kill_child
       if @child
-        unless Process.waitpid(@child, Process::WNOHANG)
-          log! "Sending TERM signal to child #{@child}"
+        unless child_already_exited?
+          if pre_shutdown_timeout && pre_shutdown_timeout > 0.0
+            log_with_severity :debug, "Waiting #{pre_shutdown_timeout.to_f}s for child process to exit"
+            return if wait_for_child_exit(pre_shutdown_timeout)
+          end
+
+          log_with_severity :debug, "Sending TERM signal to child #{@child}"
           Process.kill("TERM", @child)
-          (term_timeout.to_f * 10).round.times do |i|
-            sleep(0.1)
-            return if Process.waitpid(@child, Process::WNOHANG)
+
+          if wait_for_child_exit(term_timeout)
+            return
+          else
+            log_with_severity :debug, "Sending KILL signal to child #{@child}"
+            Process.kill("KILL", @child)
           end
-          log! "Sending KILL signal to child #{@child}"
-          Process.kill("KILL", @child)
         else
-          log! "Child #{@child} already quit."
+          log_with_severity :debug, "Child #{@child} already quit."
         end
       end
     rescue SystemCallError
-      log! "Child #{@child} already quit and reaped."
+      log_with_severity :error, "Child #{@child} already quit and reaped."
+    end
+
+    def child_already_exited?
+      Process.waitpid(@child, Process::WNOHANG)
+    end
+
+    def wait_for_child_exit(timeout)
+      (timeout * 10).round.times do |i|
+        sleep(0.1)
+        return true if child_already_exited?
+      end
+      false
     end
 
     # are we paused?
     def paused?
-      @paused
+      @paused || redis.get('pause-all-workers').to_s.strip.downcase == 'true'
     end
 
     # Stop processing jobs after the current one has completed (if we're
     # currently running one).
     def pause_processing
-      log "USR2 received; pausing job processing"
+      log_with_severity :info, "USR2 received; pausing job processing"
+      run_hook :before_pause, self
       @paused = true
     end
 
     # Start processing jobs again after a pause
     def unpause_processing
-      log "CONT received; resuming job processing"
+      log_with_severity :info, "CONT received; resuming job processing"
       @paused = false
+      run_hook :after_pause, self
     end
 
     # Looks for any workers which should be running on this server
@@ -415,13 +608,45 @@ module Resque
     # By checking the current Redis state against the actual
     # environment, we can determine if Redis is old and clean it up a bit.
     def prune_dead_workers
+      return unless data_store.acquire_pruning_dead_worker_lock(self, Resque.heartbeat_interval)
+
       all_workers = Worker.all
-      known_workers = worker_pids unless all_workers.empty?
+
+      known_workers = worker_pids
+      all_workers_with_expired_heartbeats = Worker.all_workers_with_expired_heartbeats
+      all_workers_with_expired_heartbeats.each do |worker|
+        # If the worker hasn't sent a heartbeat, remove it from the registry.
+        #
+        # If the worker hasn't ever sent a heartbeat, we won't remove it since
+        # the first heartbeat is sent before the worker is registred it means
+        # that this is a worker that doesn't support heartbeats, e.g., another
+        # client library or an older version of Resque. We won't touch these.
+        log_with_severity :info, "Pruning dead worker: #{worker}"
+
+        job_class = worker.job(false)['payload']['class'] rescue nil
+        worker.unregister_worker(PruneDeadWorkerDirtyExit.new(worker.to_s, job_class))
+      end
+
       all_workers.each do |worker|
-        host, pid, queues = worker.id.split(':')
+        if all_workers_with_expired_heartbeats.include?(worker)
+          next
+        end
+
+        host, pid, worker_queues_raw = worker.id.split(':')
+        worker_queues = worker_queues_raw.split(",")
+        unless @queues.include?("*") || (worker_queues.to_set == @queues.to_set)
+          # If the worker we are trying to prune does not belong to the queues
+          # we are listening to, we should not touch it.
+          # Attempt to prune a worker from different queues may easily result in
+          # an unknown class exception, since that worker could easily be even
+          # written in different language.
+          next
+        end
+
         next unless host == hostname
         next if known_workers.include?(pid)
-        log! "Pruning dead worker: #{worker}"
+
+        log_with_severity :debug, "Pruning dead worker: #{worker}"
         worker.unregister_worker
       end
     end
@@ -429,18 +654,29 @@ module Resque
     # Registers ourself as a worker. Useful when entering the worker
     # lifecycle on startup.
     def register_worker
-      redis.sadd(:workers, self)
-      started!
+      data_store.register_worker(self)
     end
 
     # Runs a named hook, passing along any arguments.
     def run_hook(name, *args)
-      return unless hook = Resque.send(name)
-      msg = "Running #{name} hook"
+      hooks = Resque.send(name)
+      return if hooks.empty?
+      return if name == :before_first_fork && @before_first_fork_hook_ran
+      msg = "Running #{name} hooks"
       msg << " with #{args.inspect}" if args.any?
-      log msg
+      log_with_severity :info, msg
+
+      hooks.each do |hook|
+        args.any? ? hook.call(*args) : hook.call
+        @before_first_fork_hook_ran = true if name == :before_first_fork
+      end
+    end
 
-      args.any? ? hook.call(*args) : hook.call
+    def kill_background_threads
+      if @heartbeat_thread
+        @heartbeat_thread_signal.signal
+        @heartbeat_thread.join
+      end
     end
 
     # Unregisters ourself as a worker. Useful when shutting down.
@@ -452,15 +688,28 @@ module Resque
         # Ensure the proper worker is attached to this job, even if
         # it's not the precise instance that died.
         job.worker = self
-        job.fail(exception || DirtyExit.new)
+        begin
+          job.fail(exception || DirtyExit.new("Job still being processed"))
+        rescue RuntimeError => e
+          log_with_severity :error, e.message
+        end
       end
 
-      redis.srem(:workers, self)
-      redis.del("worker:#{self}")
-      redis.del("worker:#{self}:started")
+      kill_background_threads
 
-      Stat.clear("processed:#{self}")
-      Stat.clear("failed:#{self}")
+      data_store.unregister_worker(self) do |**opts|
+        Stat.clear("processed:#{self}", **opts)
+        Stat.clear("failed:#{self}",    **opts)
+      end
+    rescue Exception => exception_while_unregistering
+      message = exception_while_unregistering.message
+      if exception
+        message += "\nOriginal Exception (#{exception.class}): #{exception.message}"
+        message += "\n  #{exception.backtrace.join("  \n")}" if exception.backtrace
+      end
+      fail(exception_while_unregistering.class,
+           message,
+           exception_while_unregistering.backtrace)
     end
 
     # Given a job, tells Redis we're working on it. Useful for seeing
@@ -468,16 +717,26 @@ module Resque
     def working_on(job)
       data = encode \
         :queue   => job.queue,
-        :run_at  => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+        :run_at  => Time.now.utc.iso8601,
         :payload => job.payload
-      redis.set("worker:#{self}", data)
+      data_store.set_worker_payload(self,data)
+      state_change
     end
 
     # Called when we are done working - clears our `working_on` state
     # and tells Redis we processed a job.
     def done_working
-      processed!
-      redis.del("worker:#{self}")
+      data_store.worker_done_working(self) do |**opts|
+        processed!(**opts)
+      end
+    end
+
+    def state_change
+      current_state = state
+      if current_state != @last_state
+        run_hook :queue_empty if current_state == :idle
+        @last_state = current_state
+      end
     end
 
     # How many jobs has this worker processed? Returns an int.
@@ -486,9 +745,9 @@ module Resque
     end
 
     # Tell Redis we've processed a job.
-    def processed!
-      Stat << "processed"
-      Stat << "processed:#{self}"
+    def processed!(**opts)
+      Stat.incr("processed",         1, **opts)
+      Stat.incr("processed:#{self}", 1, **opts)
     end
 
     # How many failed jobs has this worker seen? Returns an int.
@@ -504,18 +763,20 @@ module Resque
 
     # What time did this worker start? Returns an instance of `Time`
     def started
-      redis.get "worker:#{self}:started"
+      data_store.worker_start_time(self)
     end
 
     # Tell Redis we've started
     def started!
-      redis.set("worker:#{self}:started", Time.now.to_s)
+      data_store.worker_started(self)
     end
 
     # Returns a hash explaining the Job we're currently processing, if any.
-    def job
-      decode(redis.get("worker:#{self}")) || {}
+    def job(reload = true)
+      @job = nil if reload
+      @job ||= decode(data_store.get_worker_payload(self)) || {}
     end
+    attr_writer :job
     alias_method :processing, :job
 
     # Boolean - true if working, false if not
@@ -527,15 +788,16 @@ module Resque
     def idle?
       state == :idle
     end
-    
-    def will_fork?
-      !(@cant_fork || $TESTING)
+
+    def fork_per_job?
+      return @fork_per_job if defined?(@fork_per_job)
+      @fork_per_job = ENV["FORK_PER_JOB"] != 'false' && Kernel.respond_to?(:fork)
     end
 
     # Returns a symbol representing the current worker state,
     # which can be either :working or :idle
     def state
-      redis.exists("worker:#{self}") ? :working : :idle
+      data_store.get_worker_payload(self) ? :working : :idle
     end
 
     # Is this worker the same as another worker?
@@ -550,18 +812,18 @@ module Resque
     # The string representation is the same as the id for this worker
     # instance. Can be used with `Worker.find`.
     def to_s
-      @to_s ||= "#{hostname}:#{Process.pid}:#{@queues.join(',')}"
+      @to_s ||= "#{hostname}:#{pid}:#{@queues.join(',')}"
     end
     alias_method :id, :to_s
 
-    # chomp'd hostname of this machine
+    # chomp'd hostname of this worker's machine
     def hostname
-      @hostname ||= `hostname`.chomp
+      @hostname ||= Socket.gethostname
     end
 
     # Returns Integer PID of running worker
     def pid
-      Process.pid
+      @pid ||= Process.pid
     end
 
     # Returns an Array of string pids of all the other workers on this
@@ -569,17 +831,24 @@ module Resque
     def worker_pids
       if RUBY_PLATFORM =~ /solaris/
         solaris_worker_pids
+      elsif RUBY_PLATFORM =~ /mingw32/
+        windows_worker_pids
       else
         linux_worker_pids
       end
     end
 
-    # Find Resque worker pids on Linux and OS X.
-    #
     # Returns an Array of string pids of all the other workers on this
     # machine. Useful when pruning dead workers on startup.
+    def windows_worker_pids
+      tasklist_output = `tasklist /FI "IMAGENAME eq ruby.exe" /FO list`.encode("UTF-8", Encoding.locale_charmap)
+      tasklist_output.split($/).select { |line| line =~ /^PID:/ }.collect { |line| line.gsub(/PID:\s+/, '') }
+    end
+
+    # Find Resque worker pids on Linux and OS X.
+    #
     def linux_worker_pids
-      `ps -A -o pid,command | grep "[r]esque" | grep -v "resque-web"`.split("\n").map do |line|
+      `ps -A -o pid,command | grep -E "[r]esque:work|[r]esque:\sStarting|[r]esque-[0-9]" | grep -v "resque-web"`.split("\n").map do |line|
         line.split(' ')[0]
       end
     end
@@ -592,7 +861,7 @@ module Resque
       `ps -A -o pid,comm | grep "[r]uby" | grep -v "resque-web"`.split("\n").map do |line|
         real_pid = line.split(' ')[0]
         pargs_command = `pargs -a #{real_pid} 2>/dev/null | grep [r]esque | grep -v "resque-web"`
-        if pargs_command.split(':')[1] == " resque-#{Resque::Version}"
+        if pargs_command.split(':')[1] == " resque-#{Resque::VERSION}"
           real_pid
         end
       end.compact
@@ -600,25 +869,80 @@ module Resque
 
     # Given a string, sets the procline ($0) and logs.
     # Procline is always in the format of:
-    #   resque-VERSION: STRING
+    #   RESQUE_PROCLINE_PREFIXresque-VERSION: STRING
     def procline(string)
-      $0 = "resque-#{Resque::Version}: #{string}"
-      log! $0
+      $0 = "#{ENV['RESQUE_PROCLINE_PREFIX']}resque-#{Resque::VERSION}: #{string}"
+      log_with_severity :debug, $0
     end
 
-    # Log a message to STDOUT if we are verbose or very_verbose.
     def log(message)
-      if verbose
-        puts "*** #{message}"
-      elsif very_verbose
-        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
-        puts "** [#{time}] #$$: #{message}"
-      end
+      info(message)
     end
 
-    # Logs a very verbose message to STDOUT.
     def log!(message)
-      log message if very_verbose
+      debug(message)
+    end
+
+
+    attr_reader :verbose, :very_verbose
+
+    def verbose=(value);
+      if value && !very_verbose
+        Resque.logger.formatter = VerboseFormatter.new
+        Resque.logger.level = Logger::INFO
+      elsif !value
+        Resque.logger.formatter = QuietFormatter.new
+      end
+
+      @verbose = value
+    end
+
+    def very_verbose=(value)
+      if value
+        Resque.logger.formatter = VeryVerboseFormatter.new
+        Resque.logger.level = Logger::DEBUG
+      elsif !value && verbose
+        Resque.logger.formatter = VerboseFormatter.new
+        Resque.logger.level = Logger::INFO
+      else
+        Resque.logger.formatter = QuietFormatter.new
+      end
+
+      @very_verbose = value
+    end
+
+    private
+
+    def perform_with_fork(job, &block)
+      run_hook :before_fork, job
+
+      begin
+        @child = fork do
+          unregister_signal_handlers if term_child
+          perform(job, &block)
+          exit! unless run_at_exit_hooks
+        end
+      rescue NotImplementedError
+        @fork_per_job = false
+        perform(job, &block)
+        return
+      end
+
+      srand # Reseeding
+      procline "Forked #{@child} at #{Time.now.to_i}"
+
+      begin
+        Process.waitpid(@child)
+      rescue SystemCallError
+        nil
+      end
+
+      job.fail(DirtyExit.new("Child process received unhandled signal #{$?}", $?)) if $?.signaled?
+      @child = nil
+    end
+
+    def log_with_severity(severity, message)
+      Logging.log(severity, message)
     end
   end
 end