mongo-resque 1.17.1

data/lib/resque/failure/base.rb

@@ -0,0 +1,64 @@
+module Resque
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count
+        0
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        []
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+      end
+      
+      def self.requeue(index)
+      end
+
+      def self.remove(index)
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque/failure/hoptoad.rb

@@ -0,0 +1,133 @@
+require 'net/https'
+require 'builder'
+require 'uri'
+
+module Resque
+  module Failure
+    # A Failure backend that sends exceptions raised by jobs to Hoptoad.
+    #
+    # To use it, put this code in an initializer, Rake task, or wherever:
+    #
+    #   require 'resque/failure/hoptoad'
+    #
+    #   Resque::Failure::Hoptoad.configure do |config|
+    #     config.api_key = 'blah'
+    #     config.secure = true
+    #
+    #     # optional proxy support
+    #     config.proxy_host = 'x.y.z.t'
+    #     config.proxy_port = 8080
+    #
+    #     # server env support, defaults to RAILS_ENV or RACK_ENV
+    #     config.server_environment = "test"
+    #   end
+    class Hoptoad < Base
+      # From the hoptoad plugin
+      INPUT_FORMAT = /^([^:]+):(\d+)(?::in `([^']+)')?$/
+
+      class << self
+        attr_accessor :secure, :api_key, :proxy_host, :proxy_port
+        attr_accessor :server_environment
+      end
+
+      def self.count
+        # We can't get the total # of errors from Hoptoad so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      def save
+        http = use_ssl? ? :https : :http
+        url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
+
+        request = Net::HTTP::Proxy(self.class.proxy_host, self.class.proxy_port)
+        http = request.new(url.host, url.port)
+        headers = {
+          'Content-type' => 'text/xml',
+          'Accept' => 'text/xml, application/xml'
+        }
+
+        http.read_timeout = 5 # seconds
+        http.open_timeout = 2 # seconds
+
+        http.use_ssl = use_ssl?
+
+        begin
+          response = http.post(url.path, xml, headers)
+        rescue TimeoutError => e
+          log "Timeout while contacting the Hoptoad server."
+        end
+
+        case response
+        when Net::HTTPSuccess then
+          log "Hoptoad Success: #{response.class}"
+        else
+          body = response.body if response.respond_to? :body
+          log "Hoptoad Failure: #{response.class}\n#{body}"
+        end
+      end
+
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.tag! "class", exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("project-root", "RAILS_ROOT")
+            x.tag!("environment-name",server_environment)
+          end
+
+        end
+      end
+
+      def fill_in_backtrace_lines(x)
+        Array(exception.backtrace).each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file => file,:number => number
+        end
+      end
+
+      def use_ssl?
+        self.class.secure
+      end
+
+      def api_key
+        self.class.api_key
+      end
+
+      def server_environment
+        return self.class.server_environment if self.class.server_environment
+        defined?(RAILS_ENV) ? RAILS_ENV : (ENV['RACK_ENV'] || 'development')
+      end
+    end
+  end
+end

data/lib/resque/failure/mongo.rb

@@ -0,0 +1,45 @@
+module Resque
+  module Failure
+    # A Failure backend that stores exceptions in Mongo. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class Mongo < Base
+      def save
+        data = {
+          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => exception.to_s,
+          :backtrace => Array(exception.backtrace),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        Resque.mongo_failures << data
+      end
+
+      def self.count
+        Resque.mongo_failures.count
+      end
+
+      def self.all(start = 0, count = 1)
+        all_failures = Resque.mongo_failures.find().skip(start.to_i).limit(count.to_i).to_a
+        all_failures.size == 1 ? all_failures.first : all_failures        
+      end
+
+      def self.clear
+        Resque.mongo_failures.remove
+      end
+
+      def self.requeue(index)
+        item = all(index)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        Resque.mongo_failures.update({ :_id => item['_id']}, item)
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+
+      def self.remove(index)
+        item = all(index)
+        Resque.mongo_failures.remove(:_id => item['_id'])
+      end
+    end
+  end
+end

data/lib/resque/failure/multiple.rb

@@ -0,0 +1,54 @@
+module Resque
+  module Failure
+    # A Failure backend that uses multiple backends
+    # delegates all queries to the first backend
+    class Multiple < Base
+
+      class << self
+        attr_accessor :classes
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      def initialize(*args)
+        super
+        @backends = self.class.classes.map {|klass| klass.new(*args)}
+      end
+
+      def save
+        @backends.each(&:save)
+      end
+
+      # The number of failures.
+      def self.count
+        classes.first.count
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        classes.first.all(start,count)
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+        classes.first.url
+      end
+
+      # Clear all failure objects
+      def self.clear
+        classes.first.clear
+      end
+
+      def self.requeue(*args)
+        classes.first.requeue(*args)
+      end
+
+      def self.remove(index)
+        classes.each { |klass| klass.remove(index) }
+      end
+    end
+  end
+end

data/lib/resque/failure/redis.rb

@@ -0,0 +1,46 @@
+module Resque
+  module Failure
+    # A Failure backend that stores exceptions in Mongo. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class Redis < Base
+      def save
+        data = {
+          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => exception.to_s,
+          :backtrace => Array(exception.backtrace),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        Resque.mongo_failures << data
+      end
+
+      def self.count
+        Resque.mongo_failures.count
+      end
+
+      def self.all(start = 0, count = 1)
+        all_failures = Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
+       # all_failures.size == 1 ? all_failures.first : all_failures        
+      end
+
+      def self.clear
+        Resque.mongo_failures.remove
+      end
+
+      def self.requeue(index)
+        item = all(index)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        Resque.redis.lset(:failed, index, Resque.encode(item))
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+
+      def self.remove(index)
+        id = rand(0xffffff)
+        Resque.redis.lset(:failed, index, id)
+        Resque.redis.lrem(:failed, 1, id)
+      end
+    end
+  end
+end

data/lib/resque/failure.rb

@@ -0,0 +1,70 @@
+module Resque
+  # The Failure module provides an interface for working with different
+  # failure backends.
+  #
+  # You can use it to query the failure backend without knowing which specific
+  # backend is being used. For instance, the Resque web app uses it to display
+  # stats and other information.
+  module Failure
+    # Creates a new failure, which is delegated to the appropriate backend.
+    #
+    # Expects a hash with the following keys:
+    #   :exception - The Exception object
+    #   :worker    - The Worker object who is reporting the failure
+    #   :queue     - The string name of the queue from which the job was pulled
+    #   :payload   - The job's payload
+    def self.create(options = {})
+      backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
+    end
+
+    #
+    # Sets the current backend. Expects a class descendent of
+    # `Resque::Failure::Base`.
+    #
+    # Example use:
+    #   require 'resque/failure/hoptoad'
+    #   Resque::Failure.backend = Resque::Failure::Hoptoad
+    def self.backend=(backend)
+      @backend = backend
+    end
+
+    # Returns the current backend class. If none has been set, falls
+    # back to `Resque::Failure::Redis`
+    def self.backend
+      return @backend if @backend
+      require 'resque/failure/mongo'
+      @backend = Failure::Mongo
+    end
+
+    # Returns the int count of how many failures we have seen.
+    def self.count
+      backend.count
+    end
+
+    # Returns an array of all the failures, paginated.
+    #
+    # `start` is the int of the first item in the page, `count` is the
+    # number of items to return.
+    def self.all(start = 0, count = 1)
+      backend.all(start, count)
+    end
+
+    # The string url of the backend's web interface, if any.
+    def self.url
+      backend.url
+    end
+
+    # Clear all failure jobs
+    def self.clear
+      backend.clear
+    end
+
+    def self.requeue(index)
+      backend.requeue(index)
+    end
+
+    def self.remove(index)
+      backend.remove(index)
+    end
+  end
+end

data/lib/resque/helpers.rb

@@ -0,0 +1,75 @@
+module Resque
+  # Methods used by various classes in Resque.
+  module Helpers
+    class DecodeException < StandardError; end
+
+    # Direct access to the Redis instance.
+    #def mongo
+    #  Resque.mongo
+    #end
+
+    def mongo_workers
+      Resque.mongo_workers
+    end
+
+    def mongo_stats
+      Resque.mongo_stats
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      if defined? Yajl
+        Yajl::Encoder.encode(object)
+      else
+        object.to_json
+      end
+    end
+    
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      return unless object
+
+      if defined? Yajl
+        begin
+          Yajl::Parser.parse(object, :check_utf8 => false)
+        rescue Yajl::ParseError => e
+          raise DecodeException, e
+        end
+      else
+        begin
+          JSON.parse(object)
+        rescue JSON::ParserError => e
+          raise DecodeException, e
+        end
+      end
+    end
+
+    # Given a word with dashes, returns a camel cased version of it.
+    #
+    # classify('job-name') # => 'JobName'
+    def classify(dashed_word)
+      dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
+    end
+
+    # Given a camel cased word, returns the constant it represents
+    #
+    # constantize('JobName') # => JobName
+    def constantize(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_get(name) || constant.const_missing(name)
+      end
+      constant
+    end
+  end
+end

data/lib/resque/job.rb

@@ -0,0 +1,215 @@
+module Resque
+  # A Resque::Job represents a unit of work. Each job lives on a
+  # single queue and has an associated payload object. The payload
+  # is a hash with two attributes: `class` and `args`. The `class` is
+  # the name of the Ruby class which should be used to run the
+  # job. The `args` are an array of arguments which should be passed
+  # to the Ruby class's `perform` class-level method.
+  #
+  # You can manually run a job using this code:
+  #
+  #   job = Resque::Job.reserve(:high)
+  #   klass = Resque::Job.constantize(job.payload['class'])
+  #   klass.perform(*job.payload['args'])
+  class Job
+    include Helpers
+    extend Helpers
+
+    # Raise Resque::Job::DontPerform from a before_perform hook to
+    # abort the job.
+    DontPerform = Class.new(StandardError)
+
+    # The worker object which is currently processing this job.
+    attr_accessor :worker
+
+    # The name of the queue from which this job was pulled (or is to be
+    # placed)
+    attr_reader :queue
+
+    # This job's associated payload object.
+    attr_reader :payload
+
+    def initialize(queue, payload)
+      @queue = queue
+      @payload = payload
+    end
+
+    # Creates a job by placing it on a queue. Expects a string queue
+    # name, a string class name, and an optional array of arguments to
+    # pass to the class' `perform` method.
+    #
+    # Raises an exception if no queue or class is given.
+    def self.create(queue, klass, *args)
+      Resque.validate(klass, queue)
+
+      item = { :class => klass.to_s, :args => args}
+
+      #are we trying to put a non-delayed job into a delayed queue?
+      if Resque.queue_allows_delayed(queue)
+        if Resque.allows_delayed_jobs(klass)
+          if args[0].is_a?(Hash) && args[0].has_key?(:delay_until)
+            item[:delay_until] = args[0][:delay_until]
+          else
+            raise QueueError.new 'trying to insert delayed job without delay_until'
+          end
+        else
+          raise QueueError.new 'trying to insert non-delayed job into delayed queue'
+        end
+      else
+        if Resque.allows_delayed_jobs(klass)
+          raise QueueError.new 'trying to insert a delayed job into a non-delayed queue'
+        end
+      end
+
+      if Resque.inline?
+        constantize(klass).perform(*decode(encode(args)))
+      else
+        Resque.push(queue, item)
+      end
+    end
+
+    # Removes a job from a queue. Expects a string queue name, a
+    # string class name, and, optionally, args.
+    #
+    # Returns the number of jobs destroyed.
+    #
+    # If no args are provided, it will remove all jobs of the class
+    # provided.
+    #
+    # That is, for these two jobs:
+    #
+    # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
+    # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
+    #
+    # The following call will remove both:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph')
+    #
+    # Whereas specifying args will only remove the 2nd job:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
+    #
+    # This method can be potentially very slow and memory intensive,
+    # depending on the size of your queue, as it loads all jobs into
+    # a Ruby array before processing.
+    def self.destroy(queue, klass, *args)
+      collection = Resque.collection_for_queue(queue)
+      selector = {'class' => klass.to_s}
+      selector['args'] = args unless args.empty?
+      destroyed = collection.find(selector).count
+      collection.remove(selector, :safe => true)
+      destroyed
+    end
+
+    # Given a string queue name, returns an instance of Resque::Job
+    # if any jobs are available. If not, returns nil.
+    def self.reserve(queue)
+      return unless payload = Resque.pop(queue)
+      new(queue, payload)
+    end
+
+    # Attempts to perform the work represented by this job instance.
+    # Calls #perform on the class given in the payload with the
+    # arguments given in the payload.
+    def perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      before_hooks  = Plugin.before_hooks(job)
+      around_hooks  = Plugin.around_hooks(job)
+      after_hooks   = Plugin.after_hooks(job)
+      failure_hooks = Plugin.failure_hooks(job)
+
+      begin
+        # Execute before_perform hook. Abort the job gracefully if
+        # Resque::DontPerform is raised.
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        # Execute the job. Do it in an around_perform hook if available.
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          # We want to nest all around_perform plugins, with the last one
+          # finally calling perform
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  result = job.perform(*job_args)
+                  job_was_performed = true
+                  result
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        # Execute after_perform hook
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        # Return true if the job was performed
+        return job_was_performed
+
+      # If an exception occurs during the job execution, look for an
+      # on_failure hook then re-raise.
+      rescue Object => e
+        failure_hooks.each { |hook| job.send(hook, e, *job_args) }
+        raise e
+      end
+    end
+
+    # Returns the actual class constant represented in this job's payload.
+    def payload_class
+      @payload_class ||= constantize(@payload['class'])
+    end
+
+    # Returns an array of args represented in this job's payload.
+    def args
+      @payload['args']
+    end
+
+    # Given an exception object, hands off the needed parameters to
+    # the Failure module.
+    def fail(exception)
+      Failure.create \
+        :payload   => payload,
+        :exception => exception,
+        :worker    => worker,
+        :queue     => queue
+    end
+
+    # Creates an identical job, essentially placing this job back on
+    # the queue.
+    def recreate
+      self.class.create(queue, payload_class, *args)
+    end
+
+    # String representation
+    def inspect
+      obj = @payload
+      "(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
+    end
+
+    # Equality
+    def ==(other)
+      queue == other.queue &&
+        payload_class == other.payload_class &&
+        args == other.args
+    end
+  end
+end