resqueue 1.0.0

data/lib/resque/job.rb

@@ -0,0 +1,296 @@
+module Resque
+  # A Resque::Job represents a unit of work. Each job lives on a
+  # single queue and has an associated payload object. The payload
+  # is a hash with two attributes: `class` and `args`. The `class` is
+  # the name of the Ruby class which should be used to run the
+  # job. The `args` are an array of arguments which should be passed
+  # to the Ruby class's `perform` class-level method.
+  #
+  # You can manually run a job using this code:
+  #
+  #   job = Resque::Job.reserve(:high)
+  #   klass = Resque::Job.constantize(job.payload['class'])
+  #   klass.perform(*job.payload['args'])
+  class Job
+    include Helpers
+    extend Helpers
+    def redis
+      Resque.redis
+    end
+    alias :data_store :redis
+
+    def self.redis
+      Resque.redis
+    end
+
+    def self.data_store
+      self.redis
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      Resque.encode(object)
+    end
+
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      Resque.decode(object)
+    end
+
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def self.encode(object)
+      Resque.encode(object)
+    end
+
+    # Given a string, returns a Ruby object.
+    def self.decode(object)
+      Resque.decode(object)
+    end
+    
+    # Given a word with dashes, returns a camel cased version of it.
+    def classify(dashed_word)
+      Resque.classify(dashed_word)
+    end
+
+    # Tries to find a constant with the name specified in the argument string
+    def constantize(camel_cased_word)
+      Resque.constantize(camel_cased_word)
+    end
+
+    # Raise Resque::Job::DontPerform from a before_perform hook to
+    # abort the job.
+    DontPerform = Class.new(StandardError)
+
+    # The worker object which is currently processing this job.
+    attr_accessor :worker
+
+    # The name of the queue from which this job was pulled (or is to be
+    # placed)
+    attr_reader :queue
+
+    # This job's associated payload object.
+    attr_reader :payload
+
+    def initialize(queue, payload)
+      @queue = queue
+      @payload = payload
+      @failure_hooks_ran = false
+    end
+
+    # Creates a job by placing it on a queue. Expects a string queue
+    # name, a string class name, and an optional array of arguments to
+    # pass to the class' `perform` method.
+    #
+    # Raises an exception if no queue or class is given.
+    def self.create(queue, klass, *args)
+      Resque.validate(klass, queue)
+
+      if Resque.inline?
+        # Instantiating a Resque::Job and calling perform on it so callbacks run
+        # decode(encode(args)) to ensure that args are normalized in the same manner as a non-inline job
+        new(:inline, {'class' => klass, 'args' => decode(encode(args))}).perform
+      else
+        Resque.push(queue, :class => klass.to_s, :args => args)
+      end
+    end
+
+    # Removes a job from a queue. Expects a string queue name, a
+    # string class name, and, optionally, args.
+    #
+    # Returns the number of jobs destroyed.
+    #
+    # If no args are provided, it will remove all jobs of the class
+    # provided.
+    #
+    # That is, for these two jobs:
+    #
+    # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
+    # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
+    #
+    # The following call will remove both:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph')
+    #
+    # Whereas specifying args will only remove the 2nd job:
+    #
+    #   Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
+    #
+    # This method can be potentially very slow and memory intensive,
+    # depending on the size of your queue, as it loads all jobs into
+    # a Ruby array before processing.
+    def self.destroy(queue, klass, *args)
+      klass = klass.to_s
+      destroyed = 0
+
+      if args.empty?
+        data_store.everything_in_queue(queue).each do |string|
+          if decode(string)['class'] == klass
+            destroyed += data_store.remove_from_queue(queue,string).to_i
+          end
+        end
+      else
+        destroyed += data_store.remove_from_queue(queue, encode(:class => klass, :args => args))
+      end
+
+      destroyed
+    end
+
+    # Given a string queue name, returns an instance of Resque::Job
+    # if any jobs are available. If not, returns nil.
+    def self.reserve(queue)
+      return unless payload = Resque.pop(queue)
+      new(queue, payload)
+    end
+
+    # Attempts to perform the work represented by this job instance.
+    # Calls #perform on the class given in the payload with the
+    # arguments given in the payload.
+    def perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      begin
+        # Execute before_perform hook. Abort the job gracefully if
+        # Resque::DontPerform is raised.
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        # Execute the job. Do it in an around_perform hook if available.
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          # We want to nest all around_perform plugins, with the last one
+          # finally calling perform
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  result = job.perform(*job_args)
+                  job_was_performed = true
+                  result
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        # Execute after_perform hook
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        # Return true if the job was performed
+        return job_was_performed
+
+      # If an exception occurs during the job execution, look for an
+      # on_failure hook then re-raise.
+      rescue Object => e
+        run_failure_hooks(e)
+        raise e
+      end
+    end
+
+    # Returns the actual class constant represented in this job's payload.
+    def payload_class
+      @payload_class ||= constantize(@payload['class'])
+    end
+
+    # Returns the payload class as a string without raising NameError
+    def payload_class_name
+      payload_class.to_s
+    rescue NameError
+      'No Name'
+    end
+
+    def has_payload_class?
+      payload_class != Object
+    rescue NameError
+      false
+    end
+
+    # Returns an array of args represented in this job's payload.
+    def args
+      @payload['args']
+    end
+
+    # Given an exception object, hands off the needed parameters to
+    # the Failure module.
+    def fail(exception)
+      begin
+        run_failure_hooks(exception)
+      rescue Exception => e
+        raise e
+      ensure
+        Failure.create \
+          :payload   => payload,
+          :exception => exception,
+          :worker    => worker,
+          :queue     => queue
+      end
+    end
+
+    # Creates an identical job, essentially placing this job back on
+    # the queue.
+    def recreate
+      self.class.create(queue, payload_class, *args)
+    end
+
+    # String representation
+    def inspect
+      obj = @payload
+      "(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
+    end
+
+    # Equality
+    def ==(other)
+      queue == other.queue &&
+        payload_class == other.payload_class &&
+        args == other.args
+    end
+
+    def before_hooks
+      @before_hooks ||= Plugin.before_hooks(payload_class)
+    end
+
+    def around_hooks
+      @around_hooks ||= Plugin.around_hooks(payload_class)
+    end
+
+    def after_hooks
+      @after_hooks ||= Plugin.after_hooks(payload_class)
+    end
+
+    def failure_hooks
+      @failure_hooks ||= Plugin.failure_hooks(payload_class)
+    end
+
+    def run_failure_hooks(exception)
+      begin
+        job_args = args || []
+        if has_payload_class?
+          failure_hooks.each { |hook| payload_class.send(hook, exception, *job_args) } unless @failure_hooks_ran
+        end
+      rescue Exception => e
+        error_message = "Additional error (#{e.class}: #{e}) occurred in running failure hooks for job #{inspect}\n" \
+                        "Original error that caused job failure was #{e.class}: #{exception.class}: #{exception.message}"
+        raise RuntimeError.new(error_message)
+      ensure
+        @failure_hooks_ran = true
+      end
+    end
+  end
+end

data/lib/resque/log_formatters/quiet_formatter.rb

@@ -0,0 +1,7 @@
+module Resque
+  class QuietFormatter
+    def call(serverity, datetime, progname, msg)
+      ""
+    end
+  end
+end

data/lib/resque/log_formatters/verbose_formatter.rb

@@ -0,0 +1,7 @@
+module Resque
+  class VerboseFormatter
+    def call(serverity, datetime, progname, msg)
+      "*** #{msg}\n"
+    end
+  end
+end

data/lib/resque/log_formatters/very_verbose_formatter.rb

@@ -0,0 +1,8 @@
+module Resque
+  class VeryVerboseFormatter
+    def call(serverity, datetime, progname, msg)
+      time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
+      "** [#{time}] #$$: #{msg}\n"
+    end
+  end
+end

data/lib/resque/logging.rb

@@ -0,0 +1,18 @@
+module Resque
+  # Include this module in classes you wish to have logging facilities
+  module Logging
+    module_function
+
+    # Thunk to the logger's own log method (if configured)
+    def self.log(severity, message)
+      Resque.logger.__send__(severity, message) if Resque.logger
+    end
+    
+    # Log level aliases
+    def debug(message); Logging.log :debug, message; end
+    def info(message);  Logging.log :info,  message; end
+    def warn(message);  Logging.log :warn,  message; end
+    def error(message); Logging.log :error, message; end
+    def fatal(message); Logging.log :fatal, message; end
+  end
+end

data/lib/resque/plugin.rb

@@ -0,0 +1,78 @@
+module Resque
+  module Plugin
+    extend self
+
+    LintError = Class.new(RuntimeError)
+
+    # Ensure that your plugin conforms to good hook naming conventions.
+    #
+    #   Resque::Plugin.lint(MyResquePlugin)
+    def lint(plugin)
+      hooks = before_hooks(plugin) + around_hooks(plugin) + after_hooks(plugin)
+
+      hooks.each do |hook|
+        if hook.to_s.end_with?("perform")
+          raise LintError, "#{plugin}.#{hook} is not namespaced"
+        end
+      end
+
+      failure_hooks(plugin).each do |hook|
+        if hook.to_s.end_with?("failure")
+          raise LintError, "#{plugin}.#{hook} is not namespaced"
+        end
+      end
+    end
+
+    @job_methods = {}
+    def job_methods(job)
+      @job_methods[job] ||= job.methods.collect{|m| m.to_s}
+    end
+
+    # Given an object, and a method prefix, returns a list of methods prefixed
+    # with that name (hook names).
+    def get_hook_names(job, hook_method_prefix)
+      methods = (job.respond_to?(:hooks) && job.hooks) || job_methods(job)
+      methods.select{|m| m.start_with?(hook_method_prefix)}.sort
+    end
+
+    # Given an object, returns a list `before_perform` hook names.
+    def before_hooks(job)
+      get_hook_names(job, 'before_perform')
+    end
+
+    # Given an object, returns a list `around_perform` hook names.
+    def around_hooks(job)
+      get_hook_names(job, 'around_perform')
+    end
+
+    # Given an object, returns a list `after_perform` hook names.
+    def after_hooks(job)
+      get_hook_names(job, 'after_perform')
+    end
+
+    # Given an object, returns a list `on_failure` hook names.
+    def failure_hooks(job)
+      get_hook_names(job, 'on_failure')
+    end
+
+    # Given an object, returns a list `after_enqueue` hook names.
+    def after_enqueue_hooks(job)
+      get_hook_names(job, 'after_enqueue')
+    end
+
+    # Given an object, returns a list `before_enqueue` hook names.
+    def before_enqueue_hooks(job)
+      get_hook_names(job, 'before_enqueue')
+    end
+
+    # Given an object, returns a list `after_dequeue` hook names.
+    def after_dequeue_hooks(job)
+      get_hook_names(job, 'after_dequeue')
+    end
+
+    # Given an object, returns a list `before_dequeue` hook names.
+    def before_dequeue_hooks(job)
+      get_hook_names(job, 'before_dequeue')
+    end
+  end
+end

data/lib/resque/server.rb

@@ -0,0 +1,299 @@
+require 'sinatra/base'
+require 'tilt/erb'
+require 'resque'
+require 'resque/version'
+require 'time'
+require 'yaml'
+
+if defined? Encoding
+  Encoding.default_external = Encoding::UTF_8
+end
+
+module Resque
+  class Server < Sinatra::Base
+    require 'resque/server/helpers'
+
+    dir = File.dirname(File.expand_path(__FILE__))
+
+    set :views,  "#{dir}/server/views"
+
+    if respond_to? :public_folder
+      set :public_folder, "#{dir}/server/public"
+    else
+      set :public, "#{dir}/server/public"
+    end
+
+    set :static, true
+
+    helpers do
+      include Rack::Utils
+      alias_method :h, :escape_html
+
+      def current_section
+        url_path request.path_info.sub('/','').split('/')[0].downcase
+      end
+
+      def current_page
+        url_path request.path_info.sub('/','')
+      end
+
+      def url_path(*path_parts)
+        [ url_prefix, path_prefix, path_parts ].join("/").squeeze('/')
+      end
+      alias_method :u, :url_path
+
+      def redirect_url_path(*path_parts)
+        [ path_prefix, path_parts ].join("/").squeeze('/')
+      end
+
+      def path_prefix
+        request.env['SCRIPT_NAME']
+      end
+
+      def class_if_current(path = '')
+        'class="current"' if current_page[0, path.size] == path
+      end
+
+      def tab(name)
+        dname = name.to_s.downcase
+        path = url_path(dname)
+        "<li #{class_if_current(path)}><a href='#{path}'>#{name}</a></li>"
+      end
+
+      def tabs
+        Resque::Server.tabs
+      end
+
+      def url_prefix
+        Resque::Server.url_prefix
+      end
+
+      def redis_get_size(key)
+        case Resque.redis.type(key)
+        when 'none'
+          []
+        when 'list'
+          Resque.redis.llen(key)
+        when 'set'
+          Resque.redis.scard(key)
+        when 'string'
+          Resque.redis.get(key).length
+        when 'zset'
+          Resque.redis.zcard(key)
+        end
+      end
+
+      def redis_get_value_as_array(key, start=0)
+        case Resque.redis.type(key)
+        when 'none'
+          []
+        when 'list'
+          Resque.redis.lrange(key, start, start + 20)
+        when 'set'
+          Resque.redis.smembers(key)[start..(start + 20)]
+        when 'string'
+          [Resque.redis.get(key)]
+        when 'zset'
+          Resque.redis.zrange(key, start, start + 20)
+        end
+      end
+
+      def show_args(args)
+        Array(args).map do |a|
+          a.to_yaml
+        end.join("\n")
+      end
+
+      def worker_hosts
+        @worker_hosts ||= worker_hosts!
+      end
+
+      def worker_hosts!
+        hosts = Hash.new { [] }
+
+        Resque.workers.each do |worker|
+          host, _ = worker.to_s.split(':')
+          hosts[host] += [worker.to_s]
+        end
+
+        hosts
+      end
+
+      def partial?
+        @partial
+      end
+
+      def partial(template, local_vars = {})
+        @partial = true
+        erb(template.to_sym, {:layout => false}, local_vars)
+      ensure
+        @partial = false
+      end
+
+      def poll
+        if @polling
+          text = "Last Updated: #{Time.now.strftime("%H:%M:%S")}"
+        else
+          text = "<a href='#{u(request.path_info)}.poll' rel='poll'>Live Poll</a>"
+        end
+        "<p class='poll'>#{text}</p>"
+      end
+
+    end
+
+    def show(page, layout = true)
+      response["Cache-Control"] = "max-age=0, private, must-revalidate"
+      begin
+        erb page.to_sym, {:layout => layout}, :resque => Resque
+      rescue Errno::ECONNREFUSED
+        erb :error, {:layout => false}, :error => "Can't connect to Redis! (#{Resque.redis_id})"
+      end
+    end
+
+    def show_for_polling(page)
+      content_type "text/html"
+      @polling = true
+      show(page.to_sym, false).gsub(/\s{1,}/, ' ')
+    end
+
+    # to make things easier on ourselves
+    get "/?" do
+      redirect redirect_url_path(:overview)
+    end
+
+    %w( overview workers ).each do |page|
+      get "/#{page}.poll/?" do
+        show_for_polling(page)
+      end
+
+      get "/#{page}/:id.poll/?" do
+        show_for_polling(page)
+      end
+    end
+
+    %w( overview queues working workers key ).each do |page|
+      get "/#{page}/?" do
+        show page
+      end
+
+      get "/#{page}/:id/?" do
+        show page
+      end
+    end
+
+    post "/queues/:id/remove" do
+      Resque.remove_queue(params[:id])
+      redirect u('queues')
+    end
+
+    get "/failed/?" do
+      if Resque::Failure.url
+        redirect Resque::Failure.url
+      else
+        show :failed
+      end
+    end
+
+    get "/failed/:queue" do
+      if Resque::Failure.url
+        redirect Resque::Failure.url
+      else
+        show :failed
+      end
+    end
+
+    post "/failed/clear" do
+      Resque::Failure.clear
+      redirect u('failed')
+    end
+
+    post "/failed/:queue/clear" do
+      Resque::Failure.clear params[:queue]
+      redirect u('failed')
+    end
+
+    post "/failed/requeue/all" do
+      Resque::Failure.requeue_all
+      redirect u('failed')
+    end
+
+    post "/failed/:queue/requeue/all" do
+      Resque::Failure.requeue_queue Resque::Failure.job_queue_name(params[:queue])
+      redirect redirect_url_path("/failed/#{params[:queue]}")
+    end
+
+    get "/failed/requeue/:index/?" do
+      Resque::Failure.requeue(params[:index])
+      if request.xhr?
+        return Resque::Failure.all(params[:index])['retried_at']
+      else
+        redirect u('failed')
+      end
+    end
+
+    get "/failed/:queue/requeue/:index/?" do
+      Resque::Failure.requeue(params[:index], params[:queue])
+      if request.xhr?
+        return Resque::Failure.all(params[:index],1,params[:queue])['retried_at']
+      else
+        redirect url_path("/failed/#{params[:queue]}")
+      end
+    end
+
+    get "/failed/remove/:index/?" do
+      Resque::Failure.remove(params[:index])
+      redirect u('failed')
+    end
+
+    get "/failed/:queue/remove/:index/?" do
+      Resque::Failure.remove(params[:index], params[:queue])
+      redirect url_path("/failed/#{params[:queue]}")
+    end
+
+    get "/stats/?" do
+      redirect redirect_url_path("/stats/resque")
+    end
+
+    get "/stats/:id/?" do
+      show :stats
+    end
+
+    get "/stats/keys/:key/?" do
+      show :stats
+    end
+
+    get "/stats.txt/?" do
+      info = Resque.info
+
+      stats = []
+      stats << "resque.pending=#{info[:pending]}"
+      stats << "resque.processed+=#{info[:processed]}"
+      stats << "resque.failed+=#{info[:failed]}"
+      stats << "resque.workers=#{info[:workers]}"
+      stats << "resque.working=#{info[:working]}"
+
+      Resque.queues.each do |queue|
+        stats << "queues.#{queue}=#{Resque.size(queue)}"
+      end
+
+      content_type 'text/html'
+      stats.join "\n"
+    end
+
+    def resque
+      Resque
+    end
+
+    def self.tabs
+      @tabs ||= ["Overview", "Working", "Failed", "Queues", "Workers", "Stats"]
+    end
+
+    def self.url_prefix=(url_prefix)
+      @url_prefix = url_prefix
+    end
+
+    def self.url_prefix
+      (@url_prefix.nil? || @url_prefix.empty?) ? '' : @url_prefix + '/'
+    end
+  end
+end