resque-cedar 1.20.0

data/lib/resque/job.rb

@@ -0,0 +1,223 @@
+module Resque
+  # A Resque::Job represents a unit of work. Each job lives on a
+  # single queue and has an associated payload object. The payload
+  # is a hash with two attributes: `class` and `args`. The `class` is
+  # the name of the Ruby class which should be used to run the
+  # job. The `args` are an array of arguments which should be passed
+  # to the Ruby class's `perform` class-level method.
+  #
+  # You can manually run a job using this code:
+  #
+  #   job = Resque::Job.reserve(:high)
+  #   klass = Resque::Job.constantize(job.payload['class'])
+  #   klass.perform(*job.payload['args'])
+  class Job
+    include Helpers
+    extend Helpers
+
+    # Raise Resque::Job::DontPerform from a before_perform hook to
+    # abort the job.
+    DontPerform = Class.new(StandardError)
+
+    # The worker object which is currently processing this job.
+    attr_accessor :worker
+
+    # The name of the queue from which this job was pulled (or is to be
+    # placed)
+    attr_reader :queue
+
+    # This job's associated payload object.
+    attr_reader :payload
+
+    def initialize(queue, payload)
+      @queue = queue
+      @payload = payload
+    end
+
+    # Creates a job by placing it on a queue. Expects a string queue
+    # name, a string class name, and an optional array of arguments to
+    # pass to the class' `perform` method.
+    #
+    # Raises an exception if no queue or class is given.
+    def self.create(queue, klass, *args)
+      Resque.validate(klass, queue)
+
+      if Resque.inline?
+        constantize(klass).perform(*decode(encode(args)))
+      else
+        Resque.push(queue, :class => klass.to_s, :args => args)
+      end
+    end
+
+    # Removes a job from a queue. Expects a string queue name, a
+    # string class name, and, optionally, args.
+    #
+    # Returns the number of jobs destroyed.
+    #
+    # If no args are provided, it will remove all jobs of the class
+    # provided.
+    #
+    # That is, for these two jobs:
+    #
+    # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
+    # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
+    #
+    # The following call will remove both:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph')
+    #
+    # Whereas specifying args will only remove the 2nd job:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
+    #
+    # This method can be potentially very slow and memory intensive,
+    # depending on the size of your queue, as it loads all jobs into
+    # a Ruby array before processing.
+    def self.destroy(queue, klass, *args)
+      klass = klass.to_s
+      queue = "queue:#{queue}"
+      destroyed = 0
+
+      if args.empty?
+        redis.lrange(queue, 0, -1).each do |string|
+          if decode(string)['class'] == klass
+            destroyed += redis.lrem(queue, 0, string).to_i
+          end
+        end
+      else
+        destroyed += redis.lrem(queue, 0, encode(:class => klass, :args => args))
+      end
+
+      destroyed
+    end
+
+    # Given a string queue name, returns an instance of Resque::Job
+    # if any jobs are available. If not, returns nil.
+    def self.reserve(queue)
+      return unless payload = Resque.pop(queue)
+      new(queue, payload)
+    end
+
+    # Attempts to perform the work represented by this job instance.
+    # Calls #perform on the class given in the payload with the
+    # arguments given in the payload.
+    def perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      begin
+        # Execute before_perform hook. Abort the job gracefully if
+        # Resque::DontPerform is raised.
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        # Execute the job. Do it in an around_perform hook if available.
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          # We want to nest all around_perform plugins, with the last one
+          # finally calling perform
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  result = job.perform(*job_args)
+                  job_was_performed = true
+                  result
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        # Execute after_perform hook
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        # Return true if the job was performed
+        return job_was_performed
+
+      # If an exception occurs during the job execution, look for an
+      # on_failure hook then re-raise.
+      rescue Object => e
+        run_failure_hooks(e)
+        raise e
+      end
+    end
+
+    # Returns the actual class constant represented in this job's payload.
+    def payload_class
+      @payload_class ||= constantize(@payload['class'])
+    end
+
+    # Returns an array of args represented in this job's payload.
+    def args
+      @payload['args']
+    end
+
+    # Given an exception object, hands off the needed parameters to
+    # the Failure module.
+    def fail(exception)
+      run_failure_hooks(exception)
+      Failure.create \
+        :payload   => payload,
+        :exception => exception,
+        :worker    => worker,
+        :queue     => queue
+    end
+
+    # Creates an identical job, essentially placing this job back on
+    # the queue.
+    def recreate
+      self.class.create(queue, payload_class, *args)
+    end
+
+    # String representation
+    def inspect
+      obj = @payload
+      "(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
+    end
+
+    # Equality
+    def ==(other)
+      queue == other.queue &&
+        payload_class == other.payload_class &&
+        args == other.args
+    end
+
+    def before_hooks
+      @before_hooks ||= Plugin.before_hooks(payload_class)
+    end
+
+    def around_hooks
+      @around_hooks ||= Plugin.around_hooks(payload_class)
+    end
+
+    def after_hooks
+      @after_hooks ||= Plugin.after_hooks(payload_class)
+    end
+
+    def failure_hooks 
+      @failure_hooks ||= Plugin.failure_hooks(payload_class)
+    end
+    
+    def run_failure_hooks(exception)
+      job_args = args || []
+      failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) }
+    end
+
+  end
+end

data/lib/resque/multi_json_coder.rb

@@ -0,0 +1,37 @@
+require 'multi_json'
+require 'resque/coder'
+
+# OkJson won't work because it doesn't serialize symbols
+# in the same way yajl and json do.
+
+if MultiJson.respond_to?(:adapter)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.adapter.to_s == 'MultiJson::Adapters::OkJson'
+elsif MultiJson.respond_to?(:engine)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
+end
+
+module Resque
+  class MultiJsonCoder < Coder
+    def encode(object)
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.dump object
+      else
+        MultiJson.encode object
+      end
+    end
+
+    def decode(object)
+      return unless object
+
+      begin
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.load object
+        else
+          MultiJson.decode object
+        end
+      rescue ::MultiJson::DecodeError => e
+        raise DecodeException, e.message, e.backtrace
+      end
+    end
+  end
+end

data/lib/resque/multi_queue.rb

@@ -0,0 +1,73 @@
+require 'redis'
+require 'redis-namespace'
+require 'thread'
+require 'mutex_m'
+
+module Resque
+  ###
+  # Holds multiple queues, allowing you to pop the first available job
+  class MultiQueue
+    include Mutex_m
+
+    ###
+    # Create a new MultiQueue using the +queues+ from the +redis+ connection
+    def initialize(queues, redis)
+      super()
+
+      @queues     = queues # since ruby 1.8 doesn't have Ordered Hashes
+      @queue_hash = {}
+      @redis      = redis
+
+      queues.each do |queue|
+        key = @redis.is_a?(Redis::Namespace) ? "#{@redis.namespace}:" : ""
+        key += queue.redis_name
+        @queue_hash[key] = queue
+      end
+    end
+
+    # Pop an item off one of the queues.  This method will block until an item
+    # is available. This method returns a tuple of the queue object and job.
+    #
+    # Pass +true+ for a non-blocking pop.  If nothing is read on a non-blocking
+    # pop, a ThreadError is raised.
+    def pop(non_block = false)
+      if non_block
+        synchronize do
+          value = nil
+
+          @queues.each do |queue|
+            begin
+              return [queue, queue.pop(true)]
+            rescue ThreadError
+            end
+          end
+
+          raise ThreadError
+        end
+      else
+        queue_names = @queues.map {|queue| queue.redis_name }
+        synchronize do
+          value = @redis.blpop(*(queue_names + [1])) until value
+          queue_name, payload = value
+          queue = @queue_hash[queue_name]
+          [queue, queue.decode(payload)]
+        end
+      end
+    end
+
+    # Retrieves data from the queue head, and removes it.
+    #
+    # Blocks for +timeout+ seconds if the queue is empty, and returns nil if
+    # the timeout expires.
+    def poll(timeout)
+      queue_names = @queues.map {|queue| queue.redis_name }
+      queue_name, payload = @redis.blpop(*(queue_names + [timeout]))
+      return unless payload
+
+      synchronize do
+        queue = @queue_hash[queue_name]
+        [queue, queue.decode(payload)]
+      end
+    end
+  end
+end

data/lib/resque/plugin.rb

@@ -0,0 +1,66 @@
+module Resque
+  module Plugin
+    extend self
+
+    LintError = Class.new(RuntimeError)
+
+    # Ensure that your plugin conforms to good hook naming conventions.
+    #
+    #   Resque::Plugin.lint(MyResquePlugin)
+    def lint(plugin)
+      hooks = before_hooks(plugin) + around_hooks(plugin) + after_hooks(plugin)
+
+      hooks.each do |hook|
+        if hook =~ /perform$/
+          raise LintError, "#{plugin}.#{hook} is not namespaced"
+        end
+      end
+
+      failure_hooks(plugin).each do |hook|
+        if hook =~ /failure$/
+          raise LintError, "#{plugin}.#{hook} is not namespaced"
+        end
+      end
+    end
+
+    # Given an object, returns a list `before_perform` hook names.
+    def before_hooks(job)
+      job.methods.grep(/^before_perform/).sort
+    end
+
+    # Given an object, returns a list `around_perform` hook names.
+    def around_hooks(job)
+      job.methods.grep(/^around_perform/).sort
+    end
+
+    # Given an object, returns a list `after_perform` hook names.
+    def after_hooks(job)
+      job.methods.grep(/^after_perform/).sort
+    end
+
+    # Given an object, returns a list `on_failure` hook names.
+    def failure_hooks(job)
+      job.methods.grep(/^on_failure/).sort
+    end
+
+    # Given an object, returns a list `after_enqueue` hook names.
+    def after_enqueue_hooks(job)
+      job.methods.grep(/^after_enqueue/).sort
+    end
+
+    # Given an object, returns a list `before_enqueue` hook names.
+    def before_enqueue_hooks(job)
+      job.methods.grep(/^before_enqueue/).sort
+    end
+
+    # Given an object, returns a list `after_dequeue` hook names.
+    def after_dequeue_hooks(job)
+      job.methods.grep(/^after_dequeue/).sort
+    end
+
+    # Given an object, returns a list `before_dequeue` hook names.
+    def before_dequeue_hooks(job)
+      job.methods.grep(/^before_dequeue/).sort
+    end
+  end
+end

data/lib/resque/queue.rb

@@ -0,0 +1,117 @@
+require 'redis'
+require 'redis-namespace'
+require 'thread'
+require 'mutex_m'
+
+module Resque
+  ###
+  # Exception raised when trying to access a queue that's already destroyed
+  class QueueDestroyed < RuntimeError; end
+
+  ###
+  # A queue interface that quacks like Queue from Ruby's stdlib.
+  class Queue
+    include Mutex_m
+
+    attr_reader :name, :redis_name
+
+    ###
+    # Create a new Queue object with +name+ on +redis+ connection, and using
+    # the +coder+ for encoding and decoding objects that are stored in redis.
+    def initialize name, redis, coder = Marshal
+      super()
+      @name       = name
+      @redis_name = "queue:#{@name}"
+      @redis      = redis
+      @coder      = coder
+      @destroyed  = false
+
+      @redis.sadd(:queues, @name)
+    end
+
+    # Add +object+ to the queue
+    # If trying to push to an already destroyed queue, it will raise a Resque::QueueDestroyed exception
+    def push object
+      raise QueueDestroyed if destroyed?
+
+      synchronize do
+        @redis.rpush @redis_name, encode(object)
+      end
+    end
+
+    alias :<< :push
+    alias :enq :push
+
+    # Returns a list of objects in the queue.  This method is *not* available
+    # on the stdlib Queue.
+    def slice start, length
+      if length == 1
+        synchronize do
+          decode @redis.lindex @redis_name, start
+        end
+      else
+        synchronize do
+          Array(@redis.lrange(@redis_name, start, start + length - 1)).map do |item|
+            decode item
+          end
+        end
+      end
+    end
+
+    # Pop an item off the queue.  This method will block until an item is
+    # available.
+    #
+    # Pass +true+ for a non-blocking pop.  If nothing is read on a non-blocking
+    # pop, a ThreadError is raised.
+    def pop non_block = false
+      if non_block
+        synchronize do
+          value = @redis.lpop(@redis_name)
+          raise ThreadError unless value
+          decode value
+        end
+      else
+        synchronize do
+          value = @redis.blpop(@redis_name, 1) until value
+          decode value.last
+        end
+      end
+    end
+
+    # Get the length of the queue
+    def length
+      @redis.llen @redis_name
+    end
+    alias :size :length
+
+    # Is the queue empty?
+    def empty?
+      size == 0
+    end
+
+    # Deletes this Queue from redis. This method is *not* available on the
+    # stdlib Queue.
+    #
+    # If there are multiple queue objects of the same name, Queue A and Queue
+    # B and you delete Queue A, pushing to Queue B will have unknown side
+    # effects. Queue A will be marked destroyed, but Queue B will not.
+    def destroy
+      @redis.del @redis_name
+      @redis.srem(:queues, @name)
+      @destroyed = true
+    end
+
+    # returns +true+ if the queue is destroyed and +false+ if it isn't
+    def destroyed?
+      @destroyed
+    end
+
+    def encode object
+      @coder.dump object
+    end
+
+    def decode object
+      @coder.load object
+    end
+  end
+end