resque 1.23.0 → 2.6.0

data/lib/resque/failure/redis.rb

@@ -3,9 +3,18 @@ module Resque
     # A Failure backend that stores exceptions in Redis. Very simple but
     # works out of the box, along with support in the Resque web app.
     class Redis < Base
+
+      def data_store
+        Resque.data_store
+      end
+
+      def self.data_store
+        Resque.data_store
+      end
+
       def save
         data = {
-          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+          :failed_at => UTF8Util.clean(Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")),
           :payload   => payload,
           :exception => exception.class.to_s,
           :error     => UTF8Util.clean(exception.to_s),
@@ -14,38 +23,106 @@ module Resque
           :queue     => queue
         }
         data = Resque.encode(data)
-        Resque.redis.rpush(:failed, data)
+        data_store.push_to_failed_queue(data)
+      end
+
+      def self.count(queue = nil, class_name = nil)
+        check_queue(queue)
+
+        if class_name
+          n = 0
+          each(0, count(queue), queue, class_name) { n += 1 }
+          n
+        else
+          data_store.num_failed
+        end
+      end
+
+      def self.queues
+        data_store.failed_queue_names
       end
 
-      def self.count
-        Resque.redis.llen(:failed).to_i
+      def self.all(offset = 0, limit = 1, queue = nil)
+        check_queue(queue)
+        Resque.list_range(:failed, offset, limit)
       end
 
-      def self.all(start = 0, count = 1)
-        Resque.list_range(:failed, start, count)
+      def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
+        if class_name
+          original_limit = limit
+          limit = count
+        end
+        all_items = limit == 1 ? [all(offset,limit,queue)] : Array(all(offset, limit, queue))
+        reversed = false
+        if order.eql? 'desc'
+          all_items.reverse!
+          reversed = true
+        end
+        all_items.each_with_index do |item, i|
+          if !class_name || (item['payload'] && item['payload']['class'] == class_name && (original_limit -= 1) >= 0)
+            if reversed
+              id = (all_items.length - 1) + (offset - i)
+            else
+              id = offset + i
+            end
+            yield id, item
+          end
+        end
       end
 
-      def self.clear
-        Resque.redis.del(:failed)
+      def self.clear(queue = nil)
+        check_queue(queue)
+        data_store.clear_failed_queue
       end
 
-      def self.requeue(index)
-        item = all(index)
+      def self.requeue(id, queue = nil)
+        check_queue(queue)
+        item = all(id)
         item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
-        Resque.redis.lset(:failed, index, Resque.encode(item))
+        data_store.update_item_in_failed_queue(id,Resque.encode(item))
         Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
       end
 
-      def self.remove(index)
-        id = rand(0xffffff)
-        Resque.redis.lset(:failed, index, id)
-        Resque.redis.lrem(:failed, 1, id)
+      def self.remove(id, queue = nil)
+        check_queue(queue)
+        data_store.remove_from_failed_queue(id, queue)
+      end
+
+      def self.requeue_queue(queue)
+        i = 0
+        while job = all(i)
+           requeue(i) if job['queue'] == queue
+           i += 1
+        end
+      end
+
+      def self.requeue_all
+        count.times do |num|
+          requeue(num)
+        end
+      end
+
+      def self.remove_queue(queue)
+        i = 0
+        while job = all(i)
+          if job['queue'] == queue
+            # This will remove the failure from the array so do not increment the index.
+            remove(i)
+          else
+            i += 1
+          end
+        end
+      end
+
+      def self.check_queue(queue)
+        raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s != "failed"
       end
 
       def filter_backtrace(backtrace)
         index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
         backtrace.first(index.to_i)
       end
+
     end
   end
 end

data/lib/resque/failure/redis_multi_queue.rb

@@ -0,0 +1,104 @@
+module Resque
+  module Failure
+    # A Failure backend that stores exceptions in Redis. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class RedisMultiQueue < Base
+
+      def data_store
+        Resque.data_store
+      end
+
+      def self.data_store
+        Resque.data_store
+      end
+
+      def save
+        data = {
+          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => UTF8Util.clean(exception.to_s),
+          :backtrace => filter_backtrace(Array(exception.backtrace)),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        data = Resque.encode(data)
+        data_store.push_to_failed_queue(data,Resque::Failure.failure_queue_name(queue))
+      end
+
+      def self.count(queue = nil, class_name = nil)
+        if queue
+          if class_name
+            n = 0
+            each(0, count(queue), queue, class_name) { n += 1 }
+            n
+          else
+            data_store.num_failed(queue).to_i
+          end
+        else
+          total = 0
+          queues.each { |q| total += count(q) }
+          total
+        end
+      end
+
+      def self.all(offset = 0, limit = 1, queue = :failed)
+        Resque.list_range(queue, offset, limit)
+      end
+
+      def self.queues
+        data_store.failed_queue_names(:failed_queues)
+      end
+
+      def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
+        items = all(offset, limit, queue)
+        items = [items] unless items.is_a? Array
+        reversed = false
+        if order.eql? 'desc'
+          items.reverse!
+          reversed = true
+        end
+        items.each_with_index do |item, i|
+          if !class_name || (item['payload'] && item['payload']['class'] == class_name)
+            id = reversed ? (items.length - 1) + (offset - i) : offset + i
+            yield id, item
+          end
+        end
+      end
+
+      def self.clear(queue = :failed)
+        queues = queue ? Array(queue) : self.queues
+        queues.each { |queue| data_store.clear_failed_queue(queue) }
+      end
+
+      def self.requeue(id, queue = :failed)
+        item = all(id, 1, queue)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        data_store.update_item_in_failed_queue(id,Resque.encode(item),queue)
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+
+      def self.remove(id, queue = :failed)
+        data_store.remove_from_failed_queue(id,queue)
+      end
+
+      def self.requeue_queue(queue)
+        failure_queue = Resque::Failure.failure_queue_name(queue)
+        each(0, count(failure_queue), failure_queue) { |id, _| requeue(id, failure_queue) }
+      end
+
+      def self.requeue_all
+        queues.each { |queue| requeue_queue(Resque::Failure.job_queue_name(queue)) }
+      end
+
+      def self.remove_queue(queue)
+        data_store.remove_failed_queue(Resque::Failure.failure_queue_name(queue))
+      end
+
+      def filter_backtrace(backtrace)
+        index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
+        backtrace.first(index.to_i)
+      end
+    end
+  end
+end

data/lib/resque/failure.rb

@@ -22,31 +22,63 @@ module Resque
     # `Resque::Failure::Base`.
     #
     # Example use:
-    #   require 'resque/failure/hoptoad'
-    #   Resque::Failure.backend = Resque::Failure::Hoptoad
+    #   require 'resque/failure/airbrake'
+    #   Resque::Failure.backend = Resque::Failure::Airbrake
     def self.backend=(backend)
       @backend = backend
     end
+    self.backend = nil
 
     # Returns the current backend class. If none has been set, falls
     # back to `Resque::Failure::Redis`
     def self.backend
       return @backend if @backend
-      require 'resque/failure/redis'
-      @backend = Failure::Redis
+
+      case ENV['FAILURE_BACKEND']
+      when 'redis_multi_queue'
+        require 'resque/failure/redis_multi_queue'
+        @backend = Failure::RedisMultiQueue
+      when 'redis', nil
+        require 'resque/failure/redis'
+        @backend = Failure::Redis
+      else
+        raise ArgumentError, "invalid failure backend: #{FAILURE_BACKEND}"
+      end
+    end
+
+    # Obtain the failure queue name for a given job queue
+    def self.failure_queue_name(job_queue_name)
+      name = "#{job_queue_name}_failed"
+      Resque.data_store.add_failed_queue(name)
+      name
+    end
+
+    # Obtain the job queue name for a given failure queue
+    def self.job_queue_name(failure_queue_name)
+      failure_queue_name.sub(/_failed$/, '')
+    end
+
+    # Returns an array of all the failed queues in the system
+    def self.queues
+      backend.queues
     end
 
     # Returns the int count of how many failures we have seen.
-    def self.count
-      backend.count
+    def self.count(queue = nil, class_name = nil)
+      backend.count(queue, class_name)
     end
 
     # Returns an array of all the failures, paginated.
     #
-    # `start` is the int of the first item in the page, `count` is the
+    # `offset` is the int of the first item in the page, `limit` is the
     # number of items to return.
-    def self.all(start = 0, count = 1)
-      backend.all(start, count)
+    def self.all(offset = 0, limit = 1, queue = nil)
+      backend.all(offset, limit, queue)
+    end
+
+    # Iterate across all failures with the given options
+    def self.each(offset = 0, limit = self.count, queue = nil, class_name = nil, order = 'desc', &block)
+      backend.each(offset, limit, queue, class_name, order, &block)
     end
 
     # The string url of the backend's web interface, if any.
@@ -55,42 +87,40 @@ module Resque
     end
 
     # Clear all failure jobs
-    def self.clear
-      backend.clear
+    def self.clear(queue = nil)
+      backend.clear(queue)
     end
 
-    def self.requeue(index)
-      backend.requeue(index)
+    def self.clear_retried
+      each do |index, job|
+        remove(index) unless job['retried_at'].nil?
+      end
+    end
+
+    def self.requeue(id, queue = nil)
+      backend.requeue(id, queue)
     end
 
-    def self.remove(index)
-      backend.remove(index)
+    def self.remove(id, queue = nil)
+      backend.remove(id, queue)
     end
-    
+
     # Requeues all failed jobs in a specific queue.
     # Queue name should be a string.
     def self.requeue_queue(queue)
-      i=0
-      while job = Resque::Failure.all(i)
-        if job['queue'] == queue
-          Resque::Failure.requeue(i)
-        end  
-        i+=1
-      end
+      backend.requeue_queue(queue)
+    end
+
+    # Requeues all failed jobs
+    def self.requeue_all
+      backend.requeue_all
     end
 
     # Removes all failed jobs in a specific queue.
     # Queue name should be a string.
     def self.remove_queue(queue)
-      i=0
-      while job = Resque::Failure.all(i)
-        if job['queue'] == queue
-          # This will remove the failure from the array so do not increment the index.
-          Resque::Failure.remove(i)
-        else
-          i+=1    
-        end
-      end
+      backend.remove_queue(queue)
     end
+
   end
 end

data/lib/resque/helpers.rb

@@ -15,80 +15,34 @@ module Resque
 
     # Direct access to the Redis instance.
     def redis
+      # No infinite recursions, please.
+      # Some external libraries depend on Resque::Helpers being mixed into
+      # Resque, but this method causes recursions. If we have a super method,
+      # assume it is canonical. (see #1150)
+      return super if defined?(super)
+
       Resque.redis
     end
 
     # Given a Ruby object, returns a string suitable for storage in a
     # queue.
     def encode(object)
-      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
-        MultiJson.dump object
-      else
-        MultiJson.encode object
-      end
-
+      Resque.encode(object)
     end
 
     # Given a string, returns a Ruby object.
     def decode(object)
-      return unless object
-
-      begin
-        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
-          MultiJson.load object
-        else
-          MultiJson.decode object
-        end
-      rescue ::MultiJson::DecodeError => e
-        raise DecodeException, e.message, e.backtrace
-      end
+      Resque.decode(object)
     end
 
     # Given a word with dashes, returns a camel cased version of it.
-    #
-    # classify('job-name') # => 'JobName'
     def classify(dashed_word)
-      dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
+      Resque.classify(dashed_word)
     end
 
-    # Tries to find a constant with the name specified in the argument string:
-    #
-    # constantize("Module") # => Module
-    # constantize("Test::Unit") # => Test::Unit
-    #
-    # The name is assumed to be the one of a top-level constant, no matter
-    # whether it starts with "::" or not. No lexical context is taken into
-    # account:
-    #
-    # C = 'outside'
-    # module M
-    #   C = 'inside'
-    #   C # => 'inside'
-    #   constantize("C") # => 'outside', same as ::C
-    # end
-    #
-    # NameError is raised when the constant is unknown.
+    # Tries to find a constant with the name specified in the argument string
     def constantize(camel_cased_word)
-      camel_cased_word = camel_cased_word.to_s
-
-      if camel_cased_word.include?('-')
-        camel_cased_word = classify(camel_cased_word)
-      end
-
-      names = camel_cased_word.split('::')
-      names.shift if names.empty? || names.first.empty?
-
-      constant = Object
-      names.each do |name|
-        args = Module.method(:const_get).arity != 1 ? [false] : []
-
-        if constant.const_defined?(name, *args)
-          constant = constant.const_get(name)
-        else
-          constant = constant.const_missing(name)
-        end
-      end
-      constant
+      Resque.constantize(camel_cased_word)
     end
   end
 end

data/lib/resque/job.rb

@@ -14,6 +14,50 @@ module Resque
   class Job
     include Helpers
     extend Helpers
+    def redis
+      Resque.redis
+    end
+    alias :data_store :redis
+
+    def self.redis
+      Resque.redis
+    end
+
+    def self.data_store
+      self.redis
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      Resque.encode(object)
+    end
+
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      Resque.decode(object)
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def self.encode(object)
+      Resque.encode(object)
+    end
+
+    # Given a string, returns a Ruby object.
+    def self.decode(object)
+      Resque.decode(object)
+    end
+
+    # Given a word with dashes, returns a camel cased version of it.
+    def classify(dashed_word)
+      Resque.classify(dashed_word)
+    end
+
+    # Tries to find a constant with the name specified in the argument string
+    def constantize(camel_cased_word)
+      Resque.constantize(camel_cased_word)
+    end
 
     # Raise Resque::Job::DontPerform from a before_perform hook to
     # abort the job.
@@ -78,17 +122,16 @@ module Resque
     # a Ruby array before processing.
     def self.destroy(queue, klass, *args)
       klass = klass.to_s
-      queue = "queue:#{queue}"
       destroyed = 0
 
       if args.empty?
-        redis.lrange(queue, 0, -1).each do |string|
+        data_store.everything_in_queue(queue).each do |string|
           if decode(string)['class'] == klass
-            destroyed += redis.lrem(queue, 0, string).to_i
+            destroyed += data_store.remove_from_queue(queue,string).to_i
           end
         end
       else
-        destroyed += redis.lrem(queue, 0, encode(:class => klass, :args => args))
+        destroyed += data_store.remove_from_queue(queue, encode(:class => klass, :args => args))
       end
 
       destroyed
@@ -111,7 +154,7 @@ module Resque
 
       begin
         # Execute before_perform hook. Abort the job gracefully if
-        # Resque::DontPerform is raised.
+        # Resque::Job::DontPerform is raised.
         begin
           before_hooks.each do |hook|
             job.send(hook, *job_args)
@@ -166,6 +209,19 @@ module Resque
       @payload_class ||= constantize(@payload['class'])
     end
 
+    # Returns the payload class as a string without raising NameError
+    def payload_class_name
+      payload_class.to_s
+    rescue NameError
+      'No Name'
+    end
+
+    def has_payload_class?
+      payload_class != Object
+    rescue NameError
+      false
+    end
+
     # Returns an array of args represented in this job's payload.
     def args
       @payload['args']
@@ -174,12 +230,17 @@ module Resque
     # Given an exception object, hands off the needed parameters to
     # the Failure module.
     def fail(exception)
-      run_failure_hooks(exception)
-      Failure.create \
-        :payload   => payload,
-        :exception => exception,
-        :worker    => worker,
-        :queue     => queue
+      begin
+        run_failure_hooks(exception)
+      rescue Exception => e
+        raise e
+      ensure
+        Failure.create \
+          :payload   => payload,
+          :exception => exception,
+          :worker    => worker,
+          :queue     => queue
+      end
     end
 
     # Creates an identical job, essentially placing this job back on
@@ -220,7 +281,13 @@ module Resque
     def run_failure_hooks(exception)
       begin
         job_args = args || []
-        failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) } unless @failure_hooks_ran
+        if has_payload_class?
+          failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) } unless @failure_hooks_ran
+        end
+      rescue Exception => e
+        error_message = "Additional error (#{e.class}: #{e}) occurred in running failure hooks for job #{inspect}\n" \
+                        "Original error that caused job failure was #{e.class}: #{exception.class}: #{exception.message}"
+        raise RuntimeError.new(error_message)
       ensure
         @failure_hooks_ran = true
       end

data/lib/resque/log_formatters/quiet_formatter.rb

@@ -0,0 +1,7 @@
+module Resque
+  class QuietFormatter
+    def call(serverity, datetime, progname, msg)
+      ""
+    end
+  end
+end

data/lib/resque/log_formatters/verbose_formatter.rb

@@ -0,0 +1,7 @@
+module Resque
+  class VerboseFormatter
+    def call(serverity, datetime, progname, msg)
+      "*** #{msg}\n"
+    end
+  end
+end

data/lib/resque/log_formatters/very_verbose_formatter.rb

@@ -0,0 +1,8 @@
+module Resque
+  class VeryVerboseFormatter
+    def call(serverity, datetime, progname, msg)
+      time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+      "** [#{time}] #$$: #{msg}\n"
+    end
+  end
+end

data/lib/resque/logging.rb

@@ -0,0 +1,18 @@
+module Resque
+  # Include this module in classes you wish to have logging facilities
+  module Logging
+    module_function
+
+    # Thunk to the logger's own log method (if configured)
+    def self.log(severity, message)
+      Resque.logger.__send__(severity, message) if Resque.logger
+    end
+
+    # Log level aliases
+    def debug(message); Logging.log :debug, message; end
+    def info(message);  Logging.log :info,  message; end
+    def warn(message);  Logging.log :warn,  message; end
+    def error(message); Logging.log :error, message; end
+    def fatal(message); Logging.log :fatal, message; end
+  end
+end