resque-mongo 1.4.0 → 1.8.1

data/examples/demo/Rakefile

@@ -1,3 +1,8 @@
 $LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
 require 'resque/tasks'
 require 'job'
+
+desc "Start the demo using `rackup`"
+task :start do 
+  exec "rackup config.ru"
+end

data/examples/monit/resque.monit

@@ -0,0 +1,6 @@
+check process resque_worker_QUEUE
+  with pidfile /data/APP_NAME/current/tmp/pids/resque_worker_QUEUE.pid
+  start program = "/bin/sh -c 'cd /data/APP_NAME/current; RAILS_ENV=production QUEUE=queue_name VERBOSE=1 nohup rake resque:work& &> log/resque_worker_QUEUE.log && echo $! > tmp/pids/resque_worker_QUEUE.pid'" as uid deploy and gid deploy
+  stop program = "/bin/sh -c 'cd /data/APP_NAME/current && kill -s QUIT `cat tmp/pids/resque_worker_QUEUE.pid` && rm -f tmp/pids/resque_worker_QUEUE.pid; exit 0;'"
+  if totalmem is greater than 300 MB for 10 cycles then restart  # eating up memory?
+  group resque_workers

data/lib/resque.rb

@@ -6,6 +6,8 @@ rescue LoadError
   require 'json'
 end
 
+require 'resque/version'
+
 require 'resque/errors'
 
 require 'resque/failure'
@@ -15,6 +17,7 @@ require 'resque/helpers'
 require 'resque/stat'
 require 'resque/job'
 require 'resque/worker'
+require 'resque/plugin'
 
 module Resque
   include Helpers
@@ -65,6 +68,53 @@ module Resque
     @stats
   end
   
+  # The `before_first_fork` hook will be run in the **parent** process
+  # only once, before forking to run the first job. Be careful- any
+  # changes you make will be permanent for the lifespan of the
+  # worker.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def before_first_fork(&block)
+    block ? (@before_first_fork = block) : @before_first_fork
+  end
+
+  # Set a proc that will be called in the parent process before the
+  # worker forks for the first time.
+  def before_first_fork=(before_first_fork)
+    @before_first_fork = before_first_fork
+  end
+
+  # The `before_fork` hook will be run in the **parent** process
+  # before every job, so be careful- any changes you make will be
+  # permanent for the lifespan of the worker.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def before_fork(&block)
+    block ? (@before_fork = block) : @before_fork
+  end
+
+  # Set the before_fork proc.
+  def before_fork=(before_fork)
+    @before_fork = before_fork
+  end
+
+  # The `after_fork` hook will be run in the child process and is passed
+  # the current job. Any changes you make, therefor, will only live as
+  # long as the job currently being processes.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def after_fork(&block)
+    block ? (@after_fork = block) : @after_fork
+  end
+
+  # Set the after_fork proc.
+  def after_fork=(after_fork)
+    @after_fork = after_fork
+  end
+
   def to_s
     "Mongo Client connected to #{@con.host}"
   end
@@ -98,9 +148,9 @@ module Resque
   #
   # Returns a Ruby object.
   def pop(queue)
-    doc = mongo.find_modify( :query => { :queue => queue },
-                             :sort => [:natural, :desc],
-                             :remove => true )
+    doc = mongo.find_and_modify( :query => { :queue => queue },
+                                 :sort => [:natural, :desc],
+                                 :remove => true )
     decode doc['item']
   rescue Mongo::OperationFailure => e
     return nil if e.message =~ /No matching object/
@@ -122,6 +172,7 @@ module Resque
   # To get the 3rd page of a 30 item, paginatied list one would use:
   #   Resque.peek('my_list', 59, 30)
   def peek(queue, start = 0, count = 1)
+    start, count = [start, count].map { |n| Integer(n) }
     res = mongo.find(:queue => queue).sort([:natural, :desc]).skip(start).limit(count).to_a
     res.collect! { |doc| decode(doc['item']) }
     
@@ -165,13 +216,49 @@ module Resque
   # If either of those conditions are met, it will use the value obtained
   # from performing one of the above operations to determine the queue.
   #
-  # If no queue can be inferred this method will return a non-true value.
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
   #
   # This method is considered part of the `stable` API.
   def enqueue(klass, *args)
-    queue = klass.instance_variable_get(:@queue)
-    queue ||= klass.queue if klass.respond_to?(:queue)
-    Job.create(queue, klass, *args)
+    Job.create(queue_from_class(klass), klass, *args)
+  end
+
+  # This method can be used to conveniently remove a job from a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
+  #
+  # If no args are given, this method will dequeue *all* jobs matching
+  # the provided class. See `Resque::Job.destroy` for more
+  # information.
+  #
+  # Returns the number of jobs destroyed.
+  #
+  # Example:
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph`
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph)
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph` with matching args.
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph, 'repo:135325')
+  #
+  # This method is considered part of the `stable` API.
+  def dequeue(klass, *args)
+    Job.destroy(queue_from_class(klass), klass, *args)
+  end
+
+  # Given a class, try to extrapolate an appropriate queue based on a
+  # class instance variable or `queue` method.
+  def queue_from_class(klass)
+    klass.instance_variable_get(:@queue) ||
+      (klass.respond_to?(:queue) and klass.queue)
   end
 
   # This method will return a `Resque::Job` object or a non-true value

data/lib/resque/errors.rb

@@ -4,4 +4,7 @@ module Resque
 
   # Raised when trying to create a job without a class
   class NoClassError < RuntimeError; end
+  
+  # Raised when a worker was killed while processing a job.
+  class DirtyExit < RuntimeError; end
 end

data/lib/resque/failure.rb

@@ -59,5 +59,8 @@ module Resque
       backend.clear
     end
     
+    def self.requeue(index)
+      backend.requeue(index)
+    end
   end
 end

data/lib/resque/failure/base.rb

@@ -48,6 +48,9 @@ module Resque
       # Clear all failure objects
       def self.clear
       end
+      
+      def self.requeue(index)
+      end
 
       # Logging!
       def log(message)

data/lib/resque/failure/hoptoad.rb

@@ -1,5 +1,6 @@
-require 'net/http'
+require 'net/https'
 require 'builder'
+require 'uri'
 
 module Resque
   module Failure
@@ -7,21 +8,26 @@ module Resque
     #
     # To use it, put this code in an initializer, Rake task, or wherever:
     #
+    #   require 'resque/failure/hoptoad'
+    #
     #   Resque::Failure::Hoptoad.configure do |config|
     #     config.api_key = 'blah'
     #     config.secure = true
-    #     config.subdomain = 'your_hoptoad_subdomain'
+    #
+    #     # optional proxy support
+    #     config.proxy_host = 'x.y.z.t'
+    #     config.proxy_port = 8080
+    #
+    #     # server env support, defaults to RAILS_ENV or RACK_ENV
+    #     config.server_environment = "test"
     #   end
     class Hoptoad < Base
-      #from the hoptoad plugin
-      INPUT_FORMAT = %r{^([^:]+):(\d+)(?::in `([^']+)')?$}.freeze
-      
-      class << self
-        attr_accessor :secure, :api_key, :subdomain
-      end
+      # From the hoptoad plugin
+      INPUT_FORMAT = /^([^:]+):(\d+)(?::in `([^']+)')?$/
 
-      def self.url
-        "http://#{subdomain}.hoptoadapp.com/" if subdomain
+      class << self
+        attr_accessor :secure, :api_key, :proxy_host, :proxy_port
+        attr_accessor :server_environment
       end
 
       def self.count
@@ -35,13 +41,12 @@ module Resque
         Resque::Failure.backend = self
       end
 
-      
-
       def save
         http = use_ssl? ? :https : :http
         url = URI.parse("#{http}://hoptoadapp.com/notifier_api/v2/notices")
 
-        http = Net::HTTP.new(url.host, url.port)
+        request = Net::HTTP::Proxy(self.class.proxy_host, self.class.proxy_port)
+        http = request.new(url.host, url.port)
         headers = {
           'Content-type' => 'text/xml',
           'Accept' => 'text/xml, application/xml'
@@ -49,7 +54,7 @@ module Resque
 
         http.read_timeout = 5 # seconds
         http.open_timeout = 2 # seconds
-        
+
         http.use_ssl = use_ssl?
 
         begin
@@ -66,7 +71,7 @@ module Resque
           log "Hoptoad Failure: #{response.class}\n#{body}"
         end
       end
-      
+
       def xml
         x = Builder::XmlMarkup.new
         x.instruct!
@@ -97,16 +102,16 @@ module Resque
             end
           end
           x.tag!("server-environment") do
-            x.tag!("environment-name",RAILS_ENV)
+            x.tag!("environment-name",server_environment)
           end
-          
+
         end
       end
-      
+
       def fill_in_backtrace_lines(x)
         exception.backtrace.each do |unparsed_line|
           _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
-          x.line :file=>file,:number=>number
+          x.line :file => file,:number => number
         end
       end
 
@@ -117,6 +122,11 @@ module Resque
       def api_key
         self.class.api_key
       end
+
+      def server_environment
+        return self.class.server_environment if self.class.server_environment
+        defined?(RAILS_ENV) ? RAILS_ENV : (ENV['RACK_ENV'] || 'development')
+      end
     end
   end
 end

data/lib/resque/failure/mongo.rb

@@ -7,6 +7,7 @@ module Resque
         data = {
           :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
           :payload   => payload,
+          :exception => exception.class.to_s,
           :error     => exception.to_s,
           :backtrace => exception.backtrace,
           :worker    => worker.to_s,
@@ -20,13 +21,21 @@ module Resque
       end
 
       def self.all(start = 0, count = 1)
-        Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
+        start, count = [start, count].map { |n| Integer(n) }
+        all_failures = Resque.mongo_failures.find().sort([:natural, :desc]).skip(start).limit(count).to_a
+        all_failures.size == 1 ? all_failures.first : all_failures
       end
       
       def self.clear
         Resque.mongo_failures.remove
       end
       
+      def self.requeue(index)
+        item = all(index)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        Resque.mongo_failures.update({:_id => item['_id']}, item)
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
     end
   end
 end

data/lib/resque/helpers.rb

@@ -29,9 +29,15 @@ module Resque
       return unless object
 
       if defined? Yajl
-        Yajl::Parser.parse(object, :check_utf8 => false)
+        begin
+          Yajl::Parser.parse(object, :check_utf8 => false)
+        rescue Yajl::ParseError
+        end
       else
-        JSON.parse(object)
+        begin
+          JSON.parse(object)
+        rescue JSON::ParserError
+        end
       end
     end
 

data/lib/resque/job.rb

@@ -15,6 +15,10 @@ module Resque
     include Helpers
     extend Helpers
 
+    # Raise Resque::Job::DontPerform from a before_perform hook to
+    # abort the job.
+    DontPerform = Class.new(StandardError)
+
     # The worker object which is currently processing this job.
     attr_accessor :worker
 
@@ -36,7 +40,7 @@ module Resque
     #
     # Raises an exception if no queue or class is given.
     def self.create(queue, klass, *args)
-      if queue.to_s.empty?
+      if !queue
         raise NoQueueError.new("Jobs must be placed onto a queue.")
       end
 
@@ -47,6 +51,50 @@ module Resque
       Resque.push(queue, :class => klass.to_s, :args => args)
     end
 
+    # Removes a job from a queue. Expects a string queue name, a
+    # string class name, and, optionally, args.
+    #
+    # Returns the number of jobs destroyed.
+    #
+    # If no args are provided, it will remove all jobs of the class
+    # provided.
+    #
+    # That is, for these two jobs:
+    #
+    # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
+    # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
+    #
+    # The following call will remove both:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph')
+    #
+    # Whereas specifying args will only remove the 2nd job:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
+    #
+    # This method can be potentially very slow and memory intensive,
+    # depending on the size of your queue, as it loads all jobs into
+    # a Ruby array before processing.
+    def self.destroy(queue, klass, *args)
+      klass = klass.to_s
+
+      destroyed = 0
+
+      mongo.find(:queue => queue).each do |rec|
+        json   = decode(rec['item'])
+
+        match  = json['class'] == klass
+        match &= json['args'] == args unless args.empty?
+
+        if match
+          destroyed += 1
+          mongo.remove(:_id => rec['_id'])
+        end
+      end
+
+      destroyed
+    end
+
     # Given a string queue name, returns an instance of Resque::Job
     # if any jobs are available. If not, returns nil.
     def self.reserve(queue)
@@ -58,7 +106,64 @@ module Resque
     # Calls #perform on the class given in the payload with the
     # arguments given in the payload.
     def perform
-      args ? payload_class.perform(*args) : payload_class.perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      before_hooks  = Plugin.before_hooks(job)
+      around_hooks  = Plugin.around_hooks(job)
+      after_hooks   = Plugin.after_hooks(job)
+      failure_hooks = Plugin.failure_hooks(job)
+
+      begin
+        # Execute before_perform hook. Abort the job gracefully if
+        # Resque::DontPerform is raised.
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        # Execute the job. Do it in an around_perform hook if available.
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          # We want to nest all around_perform plugins, with the last one
+          # finally calling perform
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  job.perform(*job_args)
+                  job_was_performed = true
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        # Execute after_perform hook
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        # Return true if the job was performed
+        return job_was_performed
+
+      # If an exception occurs during the job execution, look for an
+      # on_failure hook then re-raise.
+      rescue Object => e
+        failure_hooks.each { |hook| job.send(hook, e, *job_args) }
+        raise e
+      end
     end
 
     # Returns the actual class constant represented in this job's payload.