resque 1.23.0 → 1.23.1

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.23.0'
+  Version = VERSION = '1.23.1'
 end

data/lib/resque/worker.rb

@@ -7,14 +7,9 @@ module Resque
   # It also ensures workers are always listening to signals from you,
   # their master, and can react accordingly.
   class Worker
+    extend  Resque::Helpers
     include Resque::Helpers
-    extend Resque::Helpers
-
-    # Whether the worker should log basic info to STDOUT
-    attr_accessor :verbose
-
-    # Whether the worker should log lots of info to STDOUT
-    attr_accessor  :very_verbose
+    include Resque::Logging
 
     # Boolean indicating whether this worker can or can not fork.
     # Automatically set if a fork(2) fails.
@@ -146,12 +141,13 @@ module Resque
             rescue SystemCallError
               nil
             end
+            job.fail(DirtyExit.new($?.to_s)) if $?.signaled?
           else
             unregister_signal_handlers if will_fork? && term_child
-            procline "Processing #{job.queue} since #{Time.now.to_i}"
+            procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class}]"
             reconnect
             perform(job, &block)
-            exit! if will_fork?
+            exit!(true) if will_fork?
           end
 
           done_working
@@ -166,6 +162,8 @@ module Resque
 
       unregister_worker
     rescue Exception => exception
+      log "Failed to start worker : #{exception.inspect}"
+
       unregister_worker(exception)
     end
 
@@ -268,7 +266,7 @@ module Resque
 
     # Runs all the methods needed when a worker begins its lifecycle.
     def startup
-      warn "WARNING: This way of doing signal handling is now deprecated. Please see http://hone.heroku.com/resque/2012/08/21/resque-signals.html for more info." unless term_child
+      Kernel.warn "WARNING: This way of doing signal handling is now deprecated. Please see http://hone.heroku.com/resque/2012/08/21/resque-signals.html for more info." unless term_child
       enable_gc_optimizations
       register_signal_handlers
       prune_dead_workers
@@ -435,12 +433,14 @@ module Resque
 
     # Runs a named hook, passing along any arguments.
     def run_hook(name, *args)
-      return unless hook = Resque.send(name)
-      msg = "Running #{name} hook"
+      return unless hooks = Resque.send(name)
+      msg = "Running #{name} hooks"
       msg << " with #{args.inspect}" if args.any?
       log msg
 
-      args.any? ? hook.call(*args) : hook.call
+      hooks.each do |hook|
+        args.any? ? hook.call(*args) : hook.call
+      end
     end
 
     # Unregisters ourself as a worker. Useful when shutting down.
@@ -468,7 +468,7 @@ module Resque
     def working_on(job)
       data = encode \
         :queue   => job.queue,
-        :run_at  => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+        :run_at  => Time.now.utc.iso8601,
         :payload => job.payload
       redis.set("worker:#{self}", data)
     end
@@ -529,7 +529,7 @@ module Resque
     end
     
     def will_fork?
-      !(@cant_fork || $TESTING)
+      !@cant_fork && !$TESTING && (ENV["FORK_PER_JOB"] != 'false')
     end
 
     # Returns a symbol representing the current worker state,
@@ -606,19 +606,62 @@ module Resque
       log! $0
     end
 
-    # Log a message to STDOUT if we are verbose or very_verbose.
+    # Log a message to Resque.logger
+    # can't use alias_method since info/debug are private methods
     def log(message)
-      if verbose
-        puts "*** #{message}"
-      elsif very_verbose
-        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
-        puts "** [#{time}] #$$: #{message}"
-      end
+      info(message)
     end
 
-    # Logs a very verbose message to STDOUT.
     def log!(message)
-      log message if very_verbose
+      debug(message)
+    end
+    
+    # Deprecated legacy methods for controlling the logging threshhold
+    # Use Resque.logger.level now, e.g.:
+    #
+    #     Resque.logger.level = Logger::DEBUG
+    #
+    def verbose
+      logger_severity_deprecation_warning
+      @verbose
+    end
+
+    def very_verbose
+      logger_severity_deprecation_warning
+      @very_verbose
+    end
+
+    def verbose=(value);
+      logger_severity_deprecation_warning
+
+      if value && !very_verbose
+        Resque.logger.formatter = VerboseFormatter.new
+      elsif !value
+        Resque.logger.formatter = QuietFormatter.new
+      end
+
+      @verbose = value
+    end
+
+    def very_verbose=(value)
+      logger_severity_deprecation_warning
+      if value
+        Resque.logger.formatter = VeryVerboseFormatter.new
+      elsif !value && verbose
+        Resque.logger.formatter = VerboseFormatter.new
+      else
+        Resque.logger.formatter = QuietFormatter.new
+      end
+
+      @very_verbose = value
+    end
+
+    def logger_severity_deprecation_warning
+      return if $warned_logger_severity_deprecation
+      Kernel.warn "*** DEPRECATION WARNING: Resque::Worker#verbose and #very_verbose are deprecated. Please set Resque.logger.level instead"
+      Kernel.warn "Called from: #{caller[0..5].join("\n\t")}"
+      $warned_logger_severity_deprecation = true
+      nil
     end
   end
 end

data/test/logging_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+require 'minitest/mock'
+
+context "Resque::Logging" do
+  teardown { reset_logger }
+
+  test "sets and receives the active logger" do
+    my_logger = Object.new
+    Resque.logger = my_logger
+    assert_equal my_logger, Resque.logger
+  end
+
+  %w(debug info error fatal).each do |severity|
+    test "logs #{severity} messages" do
+      message       = "test message"
+      mock_logger   = MiniTest::Mock.new
+      mock_logger.expect severity.to_sym, nil, [message]
+      Resque.logger = mock_logger
+
+      Resque::Logging.send severity, message
+      mock_logger.verify
+    end
+  end
+end

data/test/redis-test.conf

@@ -112,4 +112,4 @@ databases 16
 # Glue small output buffers together in order to send small replies in a
 # single TCP packet. Uses a bit more CPU but most of the times it is a win
 # in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes
+# glueoutputbuf yes

data/test/resque_failure_redis_test.rb

@@ -3,7 +3,7 @@ require 'resque/failure/redis'
 
 context "Resque::Failure::Redis" do
   setup do
-    @bad_string    = [39, 250, 141, 168, 138, 191, 52, 211, 159, 86, 93, 95, 39].map { |c| c.chr }.join
+    @bad_string    = [39, 52, 127, 86, 93, 95, 39].map { |c| c.chr }.join
     exception      = StandardError.exception(@bad_string)
     worker         = Resque::Worker.new(:test)
     queue          = "queue"

data/test/resque_hook_test.rb

@@ -0,0 +1,153 @@
+require 'test_helper'
+require 'tempfile'
+
+describe "Resque Hooks" do
+  before do
+    Resque.redis.flushall
+
+    Resque.before_first_fork = nil
+    Resque.before_fork = nil
+    Resque.after_fork = nil
+
+    @worker = Resque::Worker.new(:jobs)
+
+    $called = false
+
+    class CallNotifyJob
+      def self.perform
+        $called = true
+      end
+    end
+  end
+
+  it 'retrieving hooks if none have been set' do
+    assert_equal [], Resque.before_first_fork
+    assert_equal [], Resque.before_fork
+    assert_equal [], Resque.after_fork
+  end
+
+  it 'it calls before_first_fork once' do
+    counter = 0
+
+    Resque.before_first_fork { counter += 1 }
+    2.times { Resque::Job.create(:jobs, CallNotifyJob) }
+
+    assert_equal(0, counter)
+    @worker.work(0)
+    assert_equal(1, counter)
+  end
+
+  it 'it calls before_fork before each job' do
+    counter = 0
+
+    Resque.before_fork { counter += 1 }
+    2.times { Resque::Job.create(:jobs, CallNotifyJob) }
+
+    assert_equal(0, counter)
+    @worker.work(0)
+    assert_equal(2, counter)
+  end
+
+  it 'it calls after_fork after each job' do
+    # We have to stub out will_fork? to return true, which is going to cause an actual fork(). As such, the
+    # exit!(true) will be called in Worker#work; to share state, use a tempfile
+    file = Tempfile.new("resque_after_fork")
+
+    begin
+      File.open(file.path, "w") {|f| f.write(0)}
+      Resque.after_fork do
+        val = File.read(file).strip.to_i
+        File.open(file.path, "w") {|f| f.write(val + 1)}
+      end
+      2.times { Resque::Job.create(:jobs, CallNotifyJob) }
+
+      val = File.read(file.path).strip.to_i
+      assert_equal(0, val)
+      @worker.stubs(:will_fork?).returns(true)
+      @worker.work(0)
+      val = File.read(file.path).strip.to_i
+      assert_equal(2, val)
+    ensure
+      file.delete
+    end
+  end
+
+  it 'it calls before_first_fork before forking' do
+    Resque.before_first_fork { assert(!$called) }
+
+    Resque::Job.create(:jobs, CallNotifyJob)
+    @worker.work(0)
+  end
+
+  it 'it calls before_fork before forking' do
+    Resque.before_fork { assert(!$called) }
+
+    Resque::Job.create(:jobs, CallNotifyJob)
+    @worker.work(0)
+  end
+
+  it 'it calls after_fork after forking' do
+    Resque.after_fork { assert($called) }
+
+    Resque::Job.create(:jobs, CallNotifyJob)
+    @worker.work(0)
+  end
+
+  it 'it registers multiple before_first_forks' do
+    first = false
+    second = false
+
+    Resque.before_first_fork { first = true }
+    Resque.before_first_fork { second = true }
+    Resque::Job.create(:jobs, CallNotifyJob)
+
+    assert(!first && !second)
+    @worker.work(0)
+    assert(first && second)
+  end
+
+  it 'it registers multiple before_forks' do
+    first = false
+    second = false
+
+    Resque.before_fork { first = true }
+    Resque.before_fork { second = true }
+    Resque::Job.create(:jobs, CallNotifyJob)
+
+    assert(!first && !second)
+    @worker.work(0)
+    assert(first && second)
+  end
+
+  it 'it registers multiple after_forks' do
+    # We have to stub out will_fork? to return true, which is going to cause an actual fork(). As such, the
+    # exit!(true) will be called in Worker#work; to share state, use a tempfile
+    file = Tempfile.new("resque_after_fork_first")
+    file2 = Tempfile.new("resque_after_fork_second")
+    begin
+      File.open(file.path, "w") {|f| f.write(1)}
+      File.open(file2.path, "w") {|f| f.write(2)}
+
+      Resque.after_fork do
+        val = File.read(file.path).strip.to_i
+        File.open(file.path, "w") {|f| f.write(val + 1)}
+      end
+
+      Resque.after_fork do
+        val = File.read(file2.path).strip.to_i
+        File.open(file2.path, "w") {|f| f.write(val + 1)}
+      end
+      Resque::Job.create(:jobs, CallNotifyJob)
+
+      @worker.stubs(:will_fork?).returns(true)
+      @worker.work(0)
+      val = File.read(file.path).strip.to_i
+      val2 = File.read(file2.path).strip.to_i
+      assert_equal(val, 2)
+      assert_equal(val2, 3)
+    ensure
+      file.delete
+      file2.delete
+    end
+  end
+end

data/test/resque_test.rb

@@ -80,9 +80,9 @@ context "Resque" do
     assert Resque.enqueue(SomeIvarJob, 20, '/tmp')
     assert_equal 5, Resque.size(:ivar)
 
-    assert Resque.dequeue(SomeIvarJob, 30, '/tmp')
+    assert_equal 1, Resque.dequeue(SomeIvarJob, 30, '/tmp')
     assert_equal 4, Resque.size(:ivar)
-    assert Resque.dequeue(SomeIvarJob)
+    assert_equal 3, Resque.dequeue(SomeIvarJob)
     assert_equal 1, Resque.size(:ivar)
   end
 
@@ -199,7 +199,7 @@ context "Resque" do
   test "knows what queues it is managing" do
     assert_equal %w( people ), Resque.queues
     Resque.push(:cars, { 'make' => 'bmw' })
-    assert_equal %w( cars people ), Resque.queues
+    assert_equal %w( cars people ).sort, Resque.queues.sort
   end
 
   test "queues are always a list" do
@@ -209,7 +209,7 @@ context "Resque" do
 
   test "can delete a queue" do
     Resque.push(:cars, { 'make' => 'bmw' })
-    assert_equal %w( cars people ), Resque.queues
+    assert_equal %w( cars people ).sort, Resque.queues.sort
     Resque.remove_queue(:people)
     assert_equal %w( cars ), Resque.queues
     assert_equal nil, Resque.pop(:people)

data/test/test_helper.rb

@@ -1,12 +1,13 @@
 require 'rubygems'
+require 'bundler/setup'
+require 'minitest/autorun'
+require 'redis/namespace'
+require 'mocha'
 
 $dir = File.dirname(File.expand_path(__FILE__))
 $LOAD_PATH.unshift $dir + '/../lib'
-$TESTING = true
-require 'test/unit'
-
-require 'redis/namespace'
 require 'resque'
+$TESTING = true
 
 begin
   require 'leftright'
@@ -30,21 +31,12 @@ end
 # kill it when they end
 #
 
-at_exit do
-  next if $!
-
-  if defined?(MiniTest)
-    exit_code = MiniTest::Unit.new.run(ARGV)
-  else
-    exit_code = Test::Unit::AutoRunner.run
-  end
-
+MiniTest::Unit.after_tests do
   processes = `ps -A -o pid,command | grep [r]edis-test`.split("\n")
   pids = processes.map { |process| process.split(" ")[0] }
   puts "Killing test redis server..."
   pids.each { |pid| Process.kill("TERM", pid.to_i) }
   system("rm -f #{$dir}/dump.rdb #{$dir}/dump-cluster.rdb")
-  exit exit_code
 end
 
 if ENV.key? 'RESQUE_DISTRIBUTED'
@@ -176,3 +168,11 @@ ensure
   $stdout = orig_stdout
   $stderr = orig_stderr
 end
+
+# Log to log/test.log
+def reset_logger
+  $test_logger ||= Logger.new(File.open(File.expand_path("../../log/test.log", __FILE__), "w"))
+  Resque.logger = $test_logger
+end
+
+reset_logger

data/test/worker_test.rb

@@ -31,6 +31,13 @@ context "Resque::Worker" do
       @worker.perform job
     end
   end
+  
+  test "register 'run_at' time on UTC timezone in ISO8601 format" do
+    job = Resque::Job.new(:jobs, {'class' => 'GoodJob', 'args' => "blah"})
+    now = Time.now.utc.iso8601
+    @worker.working_on(job)
+    assert_equal now, @worker.processing['run_at']
+  end
 
   test "fails uncompleted jobs with DirtyExit by default on exit" do
     job = Resque::Job.new(:jobs, {'class' => 'GoodJob', 'args' => "blah"})
@@ -399,22 +406,99 @@ context "Resque::Worker" do
     assert !$BEFORE_FORK_CALLED, "before_fork should not have been called after job runs"
   end
 
+  test "setting verbose to true" do
+    @worker.verbose = true
+
+    assert @worker.verbose
+    assert !@worker.very_verbose
+  end
+
+  test "setting verbose to false" do
+    @worker.verbose = false
+
+    assert !@worker.verbose
+    assert !@worker.very_verbose
+  end
+
+  test "setting very_verbose to true" do
+    @worker.very_verbose = true
+
+    assert !@worker.verbose
+    assert @worker.very_verbose
+  end
+
+  test "setting setting verbose to true and then very_verbose to false" do
+    @worker.very_verbose = true
+    @worker.verbose      = true
+    @worker.very_verbose = false
+
+    assert @worker.verbose
+    assert !@worker.very_verbose
+  end
+
+  test "verbose prints out logs" do
+    messages        = StringIO.new
+    Resque.logger   = Logger.new(messages)
+    @worker.verbose = true
+
+    begin
+      @worker.log("omghi mom")
+    ensure
+      reset_logger
+    end
+
+    assert_equal "*** omghi mom\n", messages.string
+  end
+
+  test "unsetting verbose works" do
+    messages        = StringIO.new
+    Resque.logger   = Logger.new(messages)
+    @worker.verbose = true
+    @worker.verbose = false
+
+    begin
+      @worker.log("omghi mom")
+    ensure
+      reset_logger
+    end
+
+    assert_equal "", messages.string
+  end
+
   test "very verbose works in the afternoon" do
+    messages        = StringIO.new
+    Resque.logger   = Logger.new(messages)
+
     begin
       require 'time'
       last_puts = ""
       Time.fake_time = Time.parse("15:44:33 2011-03-02")
 
-      @worker.extend(Module.new {
-        define_method(:puts) { |thing| last_puts = thing }
-      })
-
       @worker.very_verbose = true
       @worker.log("some log text")
 
-      assert_match /\*\* \[15:44:33 2011-03-02\] \d+: some log text/, last_puts
+      assert_match /\*\* \[15:44:33 2011-03-02\] \d+: some log text/, messages.string
     ensure
       Time.fake_time = nil
+      reset_logger
+    end
+  end
+
+  test "won't fork if ENV['FORK_PER_JOB'] is false" do
+    begin
+      $TESTING = false
+      workerA = Resque::Worker.new(:jobs)
+
+      if workerA.will_fork?
+        begin
+          ENV["FORK_PER_JOB"] = 'false'
+          assert !workerA.will_fork?
+        ensure
+          ENV["FORK_PER_JOB"] = 'true'
+        end
+      end
+    ensure
+      $TESTING = true
     end
   end
 
@@ -491,11 +575,30 @@ context "Resque::Worker" do
         end
       end
 
-      @worker.very_verbose = true
-      stdout, stderr = capture_io { @worker.work(0) }
+      class DummyLogger
+        attr_reader :messages
+
+        def initialize
+          @messages = []
+        end
+
+        def info(message); @messages << message; end
+        alias_method :debug, :info
+        alias_method :warn,  :info
+        alias_method :error, :info
+        alias_method :fatal, :info
+      end
 
-      assert_equal 3, stdout.scan(/retrying/).count
-      assert_equal 1, stdout.scan(/quitting/).count
+      Resque.logger = DummyLogger.new
+      begin
+        @worker.work(0)
+        messages = Resque.logger.messages
+      ensure
+        reset_logger
+      end
+
+      assert_equal 3, messages.grep(/retrying/).count
+      assert_equal 1, messages.grep(/quitting/).count
     ensure
       class Redis::Client
         alias_method :reconnect, :original_reconnect
@@ -653,5 +756,56 @@ context "Resque::Worker" do
 
       assert !stderr.match(/^WARNING:/)
     end
+
+    class SuicidalJob
+      @queue = :jobs
+
+      def self.perform
+        Process.kill('KILL', Process.pid)
+      end
+
+      def self.on_failure_store_exception(exc, *args)
+        @@failure_exception = exc
+      end
+    end
+
+    test "will notify failure hooks when a job is killed by a signal" do
+      begin
+        $TESTING = false
+        Resque.enqueue(SuicidalJob)
+        @worker.work(0)
+        assert_equal Resque::DirtyExit, SuicidalJob.send(:class_variable_get, :@@failure_exception).class
+      ensure
+        $TESTING = true
+      end
+    end
+  end
+
+  test "displays warning when using verbose" do
+    stdout, stderr = capture_io { @worker.verbose }
+    $warned_logger_severity_deprecation = false
+
+    assert stderr.match(/WARNING:/)
+  end
+
+  test "displays warning when using verbose=" do
+    stdout, stderr = capture_io { @worker.verbose = true }
+    $warned_logger_severity_deprecation = false
+
+    assert stderr.match(/WARNING:/)
+  end
+
+  test "displays warning when using very_verbose" do
+    stdout, stderr = capture_io { @worker.very_verbose }
+    $warned_logger_severity_deprecation = false
+
+    assert stderr.match(/WARNING:/)
+  end
+
+  test "displays warning when using very_verbose=" do
+    stdout, stderr = capture_io { @worker.very_verbose = true }
+    $warned_logger_severity_deprecation = false
+
+    assert stderr.match(/WARNING:/)
   end
 end