resque 1.24.1 → 1.25.0.pre

data/lib/resque/server.rb

@@ -90,7 +90,9 @@ module Resque
       end
 
       def show_args(args)
-        Array(args).map { |a| a.inspect }.join("\n")
+        Array(args).map do |a|
+          a.to_yaml
+        end.join("\n")
       end
 
       def worker_hosts

data/lib/resque/server/helpers.rb

@@ -32,10 +32,14 @@ Resque::Server.helpers do
     if failed_start_at + failed_per_page > failed_size
       failed_size
     else
-      failed_start_at + failed_per_page
+      failed_start_at  + failed_per_page
     end
   end
 
+  def failed_order
+    params[:order] || 'desc'
+  end
+
   def failed_class_counts(queue = params[:queue])
     classes = Hash.new(0)
     Resque::Failure.each(0, Resque::Failure.count(queue), queue) do |_, item|
@@ -45,4 +49,4 @@ Resque::Server.helpers do
     end
     classes
   end
-end
+end

data/lib/resque/server/views/failed.erb

@@ -18,8 +18,9 @@
 <% else %>
 <p class='sub'>Showing <%= failed_start_at %> to <%= failed_end_at %> of <b><%= failed_size %></b> jobs</p>
 
+
 <ul class='failed'>
-  <% Resque::Failure.each(failed_start_at, failed_per_page, params[:queue], params[:class]) do |id, job| %>
+  <% Resque::Failure.each(failed_start_at, failed_per_page, params[:queue], params[:class], failed_order) do |id, job| %>
     <%= partial :failed_job, :id => id, :job => job %>
   <% end %>
 </ul>

data/lib/resque/server/views/layout.erb

@@ -36,9 +36,9 @@
 </div>
 
 <div id="footer">
-  <p>Powered by <a href="http://github.com/defunkt/resque">Resque</a> v<%=Resque::Version%></p>
+  <p>Powered by <a href="http://github.com/resque/resque">Resque</a> v<%=Resque::Version%></p>
   <p>Connected to Redis namespace <%= Resque.redis.namespace %> on <%=Resque.redis_id%></p>
 </div>
 
 </body>
-</html>
+</html>

data/lib/resque/server/views/next_more.erb

@@ -3,20 +3,20 @@
 <% per_page ||= 20 %>
 <%if  start - per_page >= 0 || start + per_page <= size%>
 <p class='pagination'>
-  <% if start - per_page >= 0 %>
-    <a href="<%= current_page %>?start=<%= start - per_page %>" class='less'>&laquo; Previous</a>
+  <% if start + per_page <= size %>
+    <a href="<%= current_page %>?start=<%= start + per_page %>" class='more'>&laquo; Next</a>
   <% end %>
 
-  <% (0...(size / per_page.to_f).ceil).each do |page_num| %>
+  <% (size / per_page.to_f - 1).ceil.downto(0).each do |page_num| %>
     <% if start == page_num * per_page %>
-       <%= page_num + 1 %>
+       <%= page_num %>
     <% else %>
-      <a href="<%= current_page %>?start=<%= page_num * per_page %>"> <%= page_num + 1 %></a>
+      <a href="<%= current_page %>?start=<%= page_num * per_page %>"> <%= page_num %></a>
     <% end %>
   <% end %>
 
-  <% if start + per_page <= size %>
-    <a href="<%= current_page %>?start=<%= start + per_page %>" class='more'>Next &raquo;</a>
+  <% if start - per_page >= 0 %>
+    <a href="<%= current_page %>?start=<%= start - per_page %>" class='less'>Previous &raquo;</a>
   <% end %>
 </p>
 <%end%>

data/lib/resque/stat.rb

@@ -7,7 +7,11 @@ module Resque
   #   Kill a stat: Stat.clear(name)
   module Stat
     extend self
-    extend Helpers
+    
+    # Direct access to the Redis instance.
+    def redis
+      Resque.redis
+    end
 
     # Returns the int value of a stat, given a string stat name.
     def get(stat)

data/lib/resque/tasks.rb

@@ -29,7 +29,7 @@ namespace :resque do
       unless Process.respond_to?('daemon')
           abort "env var BACKGROUND is set, which requires ruby >= 1.9"
       end
-      Process.daemon(true)
+      Process.daemon(true, true)
     end
 
     if ENV['PIDFILE']

data/lib/resque/version.rb

@@ -1,3 +1,3 @@
 module Resque
-  Version = VERSION = '1.24.1'
+  Version = VERSION = '1.25.0.pre'
 end

data/lib/resque/worker.rb

@@ -1,4 +1,5 @@
 require 'time'
+require 'set'
 
 module Resque
   # A Resque Worker processes jobs. On platforms that support fork(2),
@@ -9,10 +10,41 @@ module Resque
   # It also ensures workers are always listening to signals from you,
   # their master, and can react accordingly.
   class Worker
-    extend  Resque::Helpers
-    include Resque::Helpers
     include Resque::Logging
 
+    def redis
+      Resque.redis
+    end
+
+    def self.redis
+      Resque.redis
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.dump object
+      else
+        MultiJson.encode object
+      end
+    end
+
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      return unless object
+
+      begin
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.load object
+        else
+          MultiJson.decode object
+        end
+      rescue ::MultiJson::DecodeError => e
+        raise DecodeException, e.message, e.backtrace
+      end
+    end
+
     # Boolean indicating whether this worker can or can not fork.
     # Automatically set if a fork(2) fails.
     attr_accessor :cant_fork
@@ -139,14 +171,8 @@ module Resque
           job.worker = self
           working_on job
 
-          procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class}]"
-          if @child = fork(job) do
-              unregister_signal_handlers if term_child
-              reconnect
-              perform(job, &block)
-              exit! unless run_at_exit_hooks
-            end
-
+          procline "Processing #{job.queue} since #{Time.now.to_i} [#{job.payload_class_name}]"
+          if @child = fork(job)
             srand # Reseeding
             procline "Forked #{@child} at #{Time.now.to_i}"
             begin
@@ -156,9 +182,21 @@ module Resque
             end
             job.fail(DirtyExit.new($?.to_s)) if $?.signaled?
           else
-            reconnect
-            perform(job, &block)
+            unregister_signal_handlers if will_fork? && term_child
+            begin
+
+              reconnect
+              perform(job, &block)
+
+            rescue Exception => exception
+              report_failed_job(job,exception)
+            end
+
+            if will_fork?
+              run_at_exit_hooks ? exit : exit!
+            end
           end
+
           done_working
           @child = nil
         else
@@ -171,9 +209,11 @@ module Resque
 
       unregister_worker
     rescue Exception => exception
-      log "Failed to start worker : #{exception.inspect}"
+      unless exception.class == SystemExit && !@child && run_at_exit_hooks
+        log "Failed to start worker : #{exception.inspect}"
 
-      unregister_worker(exception)
+        unregister_worker(exception)
+      end
     end
 
     # DEPRECATED. Processes a single job. If none is given, it will
@@ -188,19 +228,28 @@ module Resque
       done_working
     end
 
+    # Reports the exception and marks the job as failed
+    def report_failed_job(job,exception)
+      log "#{job.inspect} failed: #{exception.inspect}"
+      begin
+        job.fail(exception)
+      rescue Object => exception
+        log "Received exception when reporting failure: #{exception.inspect}"
+      end
+      begin
+        failed!
+      rescue Object => exception
+        log "Received exception when increasing failed jobs counter (redis issue) : #{exception.inspect}"
+      end
+    end
+
     # Processes a given job in the child.
     def perform(job)
       begin
         run_hook :after_fork, job if will_fork?
         job.perform
       rescue Object => e
-        log "#{job.inspect} failed: #{e.inspect}"
-        begin
-          job.fail(e)
-        rescue Object => e
-          log "Received exception when reporting failure: #{e.inspect}"
-        end
-        failed!
+        report_failed_job(job,e)
       else
         log "done: #{job.inspect}"
       ensure
@@ -248,12 +297,25 @@ module Resque
     # A splat ("*") means you want every queue (in alpha order) - this
     # can be useful for dynamically adding new queues.
     def queues
-      @queues.map {|queue| queue == "*" ? Resque.queues.sort : queue }.flatten.uniq
+      @queues.map do |queue|
+        queue.strip!
+        if (matched_queues = glob_match(queue)).empty?
+          queue
+        else
+          matched_queues
+        end
+      end.flatten.uniq
+    end
+
+    def glob_match(pattern)
+      Resque.queues.select do |queue|
+        File.fnmatch?(pattern, queue)
+      end.sort
     end
 
     # Not every platform supports fork. Here we do our magic to
     # determine if yours does.
-    def fork(job,&block)
+    def fork(job)
       return if @cant_fork
 
       # Only run before_fork hooks if we're actually going to fork
@@ -263,7 +325,7 @@ module Resque
       begin
         # IronRuby doesn't support `Kernel.fork` yet
         if Kernel.respond_to?(:fork)
-          Kernel.fork &block if will_fork?
+          Kernel.fork if will_fork?
         else
           raise NotImplementedError
         end
@@ -324,7 +386,12 @@ module Resque
     end
 
     def unregister_signal_handlers
-      trap('TERM') { raise TermException.new("SIGTERM") }
+      trap('TERM') do
+        trap ('TERM') do 
+          # ignore subsequent terms               
+        end  
+        raise TermException.new("SIGTERM") 
+      end 
       trap('INT', 'DEFAULT')
 
       begin
@@ -425,7 +492,16 @@ module Resque
       all_workers = Worker.all
       known_workers = worker_pids unless all_workers.empty?
       all_workers.each do |worker|
-        host, pid, queues = worker.id.split(':')
+        host, pid, worker_queues_raw = worker.id.split(':')
+        worker_queues = worker_queues_raw.split(",")
+        unless @queues.include?("*") || (worker_queues.to_set == @queues.to_set)
+          # If the worker we are trying to prune does not belong to the queues
+          # we are listening to, we should not touch it. 
+          # Attempt to prune a worker from different queues may easily result in
+          # an unknown class exception, since that worker could easily be even 
+          # written in different language.
+          next
+        end
         next unless host == hostname
         next if known_workers.include?(pid)
         log! "Pruning dead worker: #{worker}"
@@ -436,8 +512,10 @@ module Resque
     # Registers ourself as a worker. Useful when entering the worker
     # lifecycle on startup.
     def register_worker
-      redis.sadd(:workers, self)
-      started!
+      redis.pipelined do
+        redis.sadd(:workers, self)
+        started!
+      end
     end
 
     # Runs a named hook, passing along any arguments.
@@ -464,12 +542,14 @@ module Resque
         job.fail(exception || DirtyExit.new)
       end
 
-      redis.srem(:workers, self)
-      redis.del("worker:#{self}")
-      redis.del("worker:#{self}:started")
+      redis.pipelined do
+        redis.srem(:workers, self)
+        redis.del("worker:#{self}")
+        redis.del("worker:#{self}:started")
 
-      Stat.clear("processed:#{self}")
-      Stat.clear("failed:#{self}")
+        Stat.clear("processed:#{self}")
+        Stat.clear("failed:#{self}")
+      end
     end
 
     # Given a job, tells Redis we're working on it. Useful for seeing
@@ -485,8 +565,10 @@ module Resque
     # Called when we are done working - clears our `working_on` state
     # and tells Redis we processed a job.
     def done_working
-      processed!
-      redis.del("worker:#{self}")
+      redis.pipelined do
+        processed!
+        redis.del("worker:#{self}")
+      end
     end
 
     # How many jobs has this worker processed? Returns an int.
@@ -578,6 +660,8 @@ module Resque
     def worker_pids
       if RUBY_PLATFORM =~ /solaris/
         solaris_worker_pids
+      elsif RUBY_PLATFORM =~ /mingw32/
+        windows_worker_pids
       else
         linux_worker_pids
       end

data/test/failure_base_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+require 'minitest/mock'
+
+require 'resque/failure/base'
+
+class TestFailure < Resque::Failure::Base
+end
+
+describe "Base failure class" do
+  it "allows calling all without throwing" do
+    with_failure_backend TestFailure do
+      assert_empty Resque::Failure.all
+    end
+  end
+end

data/test/test_helper.rb

@@ -3,6 +3,8 @@ require 'bundler/setup'
 require 'minitest/autorun'
 require 'redis/namespace'
 
+require 'mocha/setup'
+
 $dir = File.dirname(File.expand_path(__FILE__))
 $LOAD_PATH.unshift $dir + '/../lib'
 require 'resque'

data/test/worker_test.rb

@@ -72,6 +72,38 @@ context "Resque::Worker" do
 
   end
 
+  class ::RaiseExceptionOnFailure
+
+    def self.on_failure_trhow_exception(exception,*args)
+      $TESTING = true
+      raise "The worker threw an exception"
+    end
+
+    def self.perform
+      ""
+    end
+  end
+
+  test "should not treat SystemExit as an exception in the child with run_at_exit_hooks == true" do
+
+    if worker_pid = Kernel.fork
+      Process.waitpid(worker_pid)
+    else
+      # ensure we actually fork
+      $TESTING = false
+      Resque.redis.client.reconnect
+      Resque::Job.create(:not_failing_job, RaiseExceptionOnFailure)
+      worker = Resque::Worker.new(:not_failing_job)
+      worker.run_at_exit_hooks = true
+      suppress_warnings do
+        worker.work(0)
+      end
+      exit
+    end
+
+  end
+
+
   test "does not execute at_exit hooks by default" do
     tmpfile = File.join(Dir.tmpdir, "resque_at_exit_test_file")
     FileUtils.rm_f tmpfile
@@ -93,6 +125,12 @@ context "Resque::Worker" do
 
   end
 
+  test "does report failure for jobs with invalid payload" do
+    job = Resque::Job.new(:jobs, { 'class' => 'NotAValidJobClass', 'args' => '' })
+    @worker.perform job
+    assert_equal 1, Resque::Failure.count, 'failure not reported'
+  end
+
   test "register 'run_at' time on UTC timezone in ISO8601 format" do
     job = Resque::Job.new(:jobs, {'class' => 'GoodJob', 'args' => "blah"})
     now = Time.now.utc.iso8601
@@ -260,6 +298,19 @@ context "Resque::Worker" do
     assert_equal %w( jobs high critical blahblah ).sort, processed_queues
   end
 
+  test "works with globs" do
+    Resque::Job.create(:critical, GoodJob)
+    Resque::Job.create(:test_one, GoodJob)
+    Resque::Job.create(:test_two, GoodJob)
+
+    worker = Resque::Worker.new("test_*")
+
+    worker.work(0)
+    assert_equal 1, Resque.size(:critical)
+    assert_equal 0, Resque.size(:test_one)
+    assert_equal 0, Resque.size(:test_two)
+  end
+
   test "has a unique id" do
     assert_equal "#{`hostname`.chomp}:#{$$}:jobs", @worker.to_s
   end
@@ -453,7 +504,7 @@ context "Resque::Worker" do
     workerA.work(0)
     assert $BEFORE_FORK_CALLED
   end
-  
+
   test "Will not call a before_fork hook when the worker can't fork" do
     Resque.redis.flushall
     $BEFORE_FORK_CALLED = false
@@ -618,7 +669,7 @@ context "Resque::Worker" do
     assert_not_equal original_connection, Resque.redis.client.connection.instance_variable_get("@sock")
   end
 
-  test "tries to reconnect three times before giving up" do
+  test "tries to reconnect three times before giving up and the failure does not unregister the parent" do
     begin
       class Redis::Client
         alias_method :original_reconnect, :reconnect
@@ -643,6 +694,9 @@ context "Resque::Worker" do
 
       assert_equal 3, stdout.scan(/retrying/).count
       assert_equal 1, stdout.scan(/quitting/).count
+      assert_equal 0, stdout.scan(/Failed to start worker/).count
+      assert_equal 1, stdout.scan(/Redis::BaseConnectionError: Redis::BaseConnectionError/).count
+
     ensure
       class Redis::Client
         alias_method :reconnect, :original_reconnect