resque 1.20.0 → 1.21.0

data/HISTORY.md

@@ -1,3 +1,13 @@
+## 1.21.0 (2012-07-02)
+
+* Add a flag to make sure failure hooks are only ran once (jakemack, #546)
+* Support updated MultiJSON API (@twinturbo)
+* Fix worker logging in monit example config (@twinturbo)
+* loosen dependency of redis-namespace to 1.x, support for redis-rb 3.0.x
+* change '%' to '$' in the 'stop program' command for monit
+* UTF8 sanitize exception messages when there's a failure (@brianmario, #507)
+* don't share a redis connection between parent and child (@jsanders, #588)
+
 ## 1.20.0 (2012-02-17)
 
 * Fixed demos for ruby 1.9 (@BMorearty, #445)

data/Rakefile

@@ -20,18 +20,11 @@ require 'rake/testtask'
 
 task :default => :test
 
-if command?(:rg)
-  desc "Run the test suite with rg"
-  task :test do
-    Dir['test/**/*_test.rb'].each do |f|
-      sh("rg #{f}")
-    end
-  end
-else
-  Rake::TestTask.new do |test|
-    test.libs << "test"
-    test.test_files = FileList['test/**/*_test.rb']
-  end
+Rake::TestTask.new do |test|
+  test.verbose = true
+  test.libs << "test"
+  test.libs << "lib"
+  test.test_files = FileList['test/**/*_test.rb']
 end
 
 if command? :kicker
@@ -74,5 +67,4 @@ task :publish do
   sh "git push origin v#{Resque::Version}"
   sh "git push origin master"
   sh "git clean -fd"
-  exec "rake pages"
 end

data/lib/resque.rb

@@ -13,6 +13,8 @@ require 'resque/job'
 require 'resque/worker'
 require 'resque/plugin'
 
+require 'resque/vendor/utf8_util'
+
 module Resque
   include Helpers
   extend self
@@ -77,9 +79,7 @@ module Resque
 
   # Set a proc that will be called in the parent process before the
   # worker forks for the first time.
-  def before_first_fork=(before_first_fork)
-    @before_first_fork = before_first_fork
-  end
+  attr_writer :before_first_fork
 
   # The `before_fork` hook will be run in the **parent** process
   # before every job, so be careful- any changes you make will be
@@ -92,9 +92,7 @@ module Resque
   end
 
   # Set the before_fork proc.
-  def before_fork=(before_fork)
-    @before_fork = before_fork
-  end
+  attr_writer :before_fork
 
   # The `after_fork` hook will be run in the child process and is passed
   # the current job. Any changes you make, therefore, will only live as
@@ -107,25 +105,18 @@ module Resque
   end
 
   # Set the after_fork proc.
-  def after_fork=(after_fork)
-    @after_fork = after_fork
-  end
+  attr_writer :after_fork
 
   def to_s
     "Resque Client connected to #{redis_id}"
   end
 
+  attr_accessor :inline
+
   # If 'inline' is true Resque will call #perform method inline
   # without queuing it into Redis and without any Resque callbacks.
   # The 'inline' is false Resque jobs will be put in queue regularly.
-  def inline?
-    @inline
-  end
-  alias_method :inline, :inline?
-
-  def inline=(inline)
-    @inline = inline
-  end
+  alias :inline? :inline
 
   #
   # queue manipulation

data/lib/resque/failure/redis.rb

@@ -8,7 +8,7 @@ module Resque
           :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
           :payload   => payload,
           :exception => exception.class.to_s,
-          :error     => exception.to_s,
+          :error     => UTF8Util.clean(exception.to_s),
           :backtrace => filter_backtrace(Array(exception.backtrace)),
           :worker    => worker.to_s,
           :queue     => queue

data/lib/resque/helpers.rb

@@ -2,8 +2,10 @@ require 'multi_json'
 
 # OkJson won't work because it doesn't serialize symbols
 # in the same way yajl and json do.
-if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
-  raise "Please install the yajl-ruby or json gem"
+if MultiJson.respond_to?(:adapter)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.adapter.to_s == 'MultiJson::Adapters::OkJson'
+elsif MultiJson.respond_to?(:engine)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
 end
 
 module Resque
@@ -19,7 +21,12 @@ module Resque
     # Given a Ruby object, returns a string suitable for storage in a
     # queue.
     def encode(object)
-      ::MultiJson.encode(object)
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.dump object
+      else
+        MultiJson.encode object
+      end
+
     end
 
     # Given a string, returns a Ruby object.
@@ -27,7 +34,11 @@ module Resque
       return unless object
 
       begin
-        ::MultiJson.decode(object)
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.load object
+        else
+          MultiJson.decode object
+        end
       rescue ::MultiJson::DecodeError => e
         raise DecodeException, e.message, e.backtrace
       end

data/lib/resque/job.rb

@@ -32,6 +32,7 @@ module Resque
     def initialize(queue, payload)
       @queue = queue
       @payload = payload
+      @failure_hooks_ran = false
     end
 
     # Creates a job by placing it on a queue. Expects a string queue
@@ -210,14 +211,17 @@ module Resque
       @after_hooks ||= Plugin.after_hooks(payload_class)
     end
 
-    def failure_hooks 
+    def failure_hooks
       @failure_hooks ||= Plugin.failure_hooks(payload_class)
     end
-    
+
     def run_failure_hooks(exception)
-      job_args = args || []
-      failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) }
+      begin
+        job_args = args || []
+        failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) } unless @failure_hooks_ran
+      ensure
+        @failure_hooks_ran = true
+      end
     end
-
   end
 end

data/lib/resque/vendor/utf8_util.rb

@@ -0,0 +1,26 @@
+module UTF8Util
+  # use '?' intsead of the unicode replace char, since that is 3 bytes
+  # and can increase the string size if it's done a lot
+  REPLACEMENT_CHAR = "?"
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns self as valid UTF-8.
+  def self.clean!(str)
+    raise NotImplementedError
+  end
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns a copy of this String as valid UTF-8.
+  def self.clean(str)
+    clean!(str.dup)
+  end
+
+end
+
+if RUBY_VERSION <= '1.9'
+  require 'resque/vendor/utf8_util/utf8_util_18'
+else
+  require 'resque/vendor/utf8_util/utf8_util_19'
+end

data/lib/resque/vendor/utf8_util/utf8_util_18.rb

@@ -0,0 +1,91 @@
+require 'strscan'
+
+module UTF8Util
+  HIGH_BIT_RANGE = /[\x80-\xff]/
+
+  # Check if this String is valid UTF-8
+  #
+  # Returns true or false.
+  def self.valid?(str)
+    sc = StringScanner.new(str)
+
+    while sc.skip_until(HIGH_BIT_RANGE)
+      sc.pos -= 1
+
+      if !sequence_length(sc)
+        return false
+      end
+    end
+
+    true
+  end
+
+  # Replace invalid UTF-8 character sequences with a replacement character
+  #
+  # Returns self as valid UTF-8.
+  def self.clean!(str)
+    sc = StringScanner.new(str)
+    while sc.skip_until(HIGH_BIT_RANGE)
+      pos = sc.pos = sc.pos-1
+
+      if !sequence_length(sc)
+        str[pos] = REPLACEMENT_CHAR
+      end
+    end
+
+    str
+  end
+
+  # Validate the UTF-8 sequence at the current scanner position.
+  #
+  # scanner - StringScanner instance so we can advance the pointer as we verify.
+  #
+  # Returns The length in bytes of this UTF-8 sequence, false if invalid.
+  def self.sequence_length(scanner)
+    leader = scanner.get_byte[0]
+
+    if (leader >> 5) == 0x6
+      if check_next_sequence(scanner)
+        return 2
+      else
+        scanner.pos -= 1
+      end
+    elsif (leader >> 4) == 0x0e
+      if check_next_sequence(scanner)
+        if check_next_sequence(scanner)
+          return 3
+        else
+          scanner.pos -= 2
+        end
+      else
+        scanner.pos -= 1
+      end
+    elsif (leader >> 3) == 0x1e
+      if check_next_sequence(scanner)
+        if check_next_sequence(scanner)
+          if check_next_sequence(scanner)
+            return 4
+          else
+            scanner.pos -= 3
+          end
+        else
+          scanner.pos -= 2
+        end
+      else
+        scanner.pos -= 1
+      end
+    end
+
+    false
+  end
+
+  private
+
+  # Read another byte off the scanner oving the scan position forward one place
+  #
+  # Returns nothing.
+  def self.check_next_sequence(scanner)
+    byte = scanner.get_byte[0]
+    (byte >> 6) == 0x2
+  end
+end

data/lib/resque/vendor/utf8_util/utf8_util_19.rb

@@ -0,0 +1,5 @@
+module UTF8Util
+  def self.clean!(str)
+    str.force_encoding("binary").encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => REPLACEMENT_CHAR)
+  end
+end

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.20.0'
+  Version = VERSION = '1.21.0'
 end

data/lib/resque/worker.rb

@@ -89,6 +89,8 @@ module Resque
     # removed without needing to restart workers using this method.
     def initialize(*queues)
       @queues = queues.map { |queue| queue.to_s.strip }
+      @shutdown = nil
+      @paused = nil
       validate_queues
     end
 
@@ -138,6 +140,7 @@ module Resque
             Process.wait(@child)
           else
             procline "Processing #{job.queue} since #{Time.now.to_i}"
+            redis.client.reconnect # Don't share connection with parent
             perform(job, &block)
             exit! unless @cant_fork
           end

data/test/resque_failure_redis_test.rb

@@ -0,0 +1,23 @@
+require 'test_helper'
+require 'resque/failure/redis'
+
+context "Resque::Failure::Redis" do
+  setup do
+    @bad_string    = [39, 250, 141, 168, 138, 191, 52, 211, 159, 86, 93, 95, 39].map { |c| c.chr }.join
+    exception      = StandardError.exception(@bad_string)
+    worker         = Resque::Worker.new(:test)
+    queue          = "queue"
+    payload        = { "class" => Object, "args" => 3 }
+    @redis_backend = Resque::Failure::Redis.new(exception, worker, queue, payload)
+  end
+
+  test 'cleans up bad strings before saving the failure, in order to prevent errors on the resque UI' do
+    # test assumption: the bad string should not be able to round trip though JSON
+    assert_raises(MultiJson::DecodeError) {
+      MultiJson.decode(MultiJson.encode(@bad_string))
+    }
+
+    @redis_backend.save
+    Resque::Failure::Redis.all # should not raise an error
+  end
+end

data/test/test_helper.rb

@@ -1,13 +1,13 @@
 require 'rubygems'
-require 'bundler'
-Bundler.setup(:default, :test)
-Bundler.require(:default, :test)
 
 dir = File.dirname(File.expand_path(__FILE__))
 $LOAD_PATH.unshift dir + '/../lib'
 $TESTING = true
 require 'test/unit'
 
+require 'redis/namespace'
+require 'resque'
+
 begin
   require 'leftright'
 rescue LoadError
@@ -147,12 +147,14 @@ end
 class Time
   # Thanks, Timecop
   class << self
+    attr_accessor :fake_time
+
     alias_method :now_without_mock_time, :now
 
-    def now_with_mock_time
-      $fake_time || now_without_mock_time
+    def now
+      fake_time || now_without_mock_time
     end
-
-    alias_method :now, :now_with_mock_time
   end
+
+  self.fake_time = nil
 end

data/test/worker_test.rb

@@ -43,7 +43,7 @@ context "Resque::Worker" do
     def self.on_failure_record_failure(exception, *job_args)
       @@exception = exception
     end
-    
+
     def self.exception
       @@exception
     end
@@ -57,6 +57,29 @@ context "Resque::Worker" do
     assert(SimpleJobWithFailureHandling.exception.kind_of?(Resque::DirtyExit))
   end
 
+  class ::SimpleFailingJob
+    @@exception_count = 0
+
+    def self.on_failure_record_failure(exception, *job_args)
+      @@exception_count += 1
+    end
+
+    def self.exception_count
+      @@exception_count
+    end
+
+    def self.perform
+      raise Exception.new
+    end
+  end
+
+  test "only calls failure hook once on exception" do
+    job = Resque::Job.new(:jobs, {'class' => 'SimpleFailingJob', 'args' => ""})
+    @worker.perform(job)
+    assert_equal 1, Resque::Failure.count
+    assert_equal 1, SimpleFailingJob.exception_count
+  end
+
   test "can peek at failed jobs" do
     10.times { Resque::Job.create(:jobs, BadJob) }
     @worker.work(0)
@@ -355,16 +378,22 @@ context "Resque::Worker" do
   end
 
   test "very verbose works in the afternoon" do
-    require 'time'
-    $last_puts = ""
-    $fake_time = Time.parse("15:44:33 2011-03-02")
-    singleton = class << @worker; self end
-    singleton.send :define_method, :puts, lambda { |thing| $last_puts = thing }
+    begin
+      require 'time'
+      last_puts = ""
+      Time.fake_time = Time.parse("15:44:33 2011-03-02")
 
-    @worker.very_verbose = true
-    @worker.log("some log text")
+      @worker.extend(Module.new {
+        define_method(:puts) { |thing| last_puts = thing }
+      })
 
-    assert_match /\*\* \[15:44:33 2011-03-02\] \d+: some log text/, $last_puts
+      @worker.very_verbose = true
+      @worker.log("some log text")
+
+      assert_match /\*\* \[15:44:33 2011-03-02\] \d+: some log text/, last_puts
+    ensure
+      Time.fake_time = nil
+    end
   end
 
   test "Will call an after_fork hook after forking" do
@@ -382,7 +411,7 @@ context "Resque::Worker" do
   test "returns PID of running process" do
     assert_equal @worker.to_s.split(":")[1].to_i, @worker.pid
   end
-  
+
   test "requeue failed queue" do
     queue = 'good_job'
     Resque::Failure.create(:exception => Exception.new, :worker => Resque::Worker.new(queue), :queue => queue, :payload => {'class' => 'GoodJob'})
@@ -402,4 +431,10 @@ context "Resque::Worker" do
     assert_equal queue2, Resque::Failure.all(0)['queue']
     assert_equal 1, Resque::Failure.count
   end
+
+  test "reconnects to redis after fork" do
+    original_connection = Resque.redis.client.connection.instance_variable_get("@sock")
+    @worker.work(0)
+    assert_not_equal original_connection, Resque.redis.client.connection.instance_variable_get("@sock")
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: resque
 version: !ruby/object:Gem::Version 
-  hash: 71
+  hash: 67
   prerelease: 
   segments: 
   - 1
-  - 20
+  - 21
   - 0
-  version: 1.20.0
+  version: 1.21.0
 platform: ruby
 authors: 
 - Chris Wanstrath
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-17 00:00:00 Z
+date: 2012-07-03 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: redis-namespace
@@ -26,12 +26,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 15
         segments: 
         - 1
         - 0
-        - 2
-        version: 1.0.2
+        version: "1.0"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -127,6 +126,9 @@ files:
 - lib/resque/plugin.rb
 - lib/resque/server.rb
 - lib/resque/helpers.rb
+- lib/resque/vendor/utf8_util.rb
+- lib/resque/vendor/utf8_util/utf8_util_19.rb
+- lib/resque/vendor/utf8_util/utf8_util_18.rb
 - lib/resque/tasks.rb
 - lib/resque/stat.rb
 - lib/resque/failure/hoptoad.rb
@@ -141,6 +143,7 @@ files:
 - test/airbrake_test.rb
 - test/job_hooks_test.rb
 - test/plugin_test.rb
+- test/resque_failure_redis_test.rb
 - test/redis-test-cluster.conf
 - test/resque-web_test.rb
 - test/redis-test.conf
@@ -178,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Resque is a Redis-backed queueing system.