nfo-resque-mongo 1.15.0

data/Rakefile

@@ -0,0 +1,73 @@
+load 'tasks/redis.rake'
+
+$LOAD_PATH.unshift 'lib'
+require 'resque/tasks'
+
+def command?(command)
+  system("type #{command} > /dev/null 2>&1")
+end
+
+
+#
+# Tests
+#
+
+require 'rake/testtask'
+
+task :default => :test
+
+if command?(:rg)
+  desc "Run the test suite with rg"
+  task :test do
+    Dir['test/**/*_test.rb'].each do |f|
+      sh("rg #{f}")
+    end
+  end
+else
+  Rake::TestTask.new do |test|
+    test.libs << "test"
+    test.test_files = FileList['test/**/*_test.rb']
+  end  
+end
+
+if command? :kicker
+  desc "Launch Kicker (like autotest)"
+  task :kicker do
+    puts "Kicking... (ctrl+c to cancel)"
+    exec "kicker -e rake test lib examples"
+  end
+end
+
+#
+# Install
+#
+
+task :install => [ 'redis:install', 'dtach:install' ]
+
+
+#
+# Documentation
+#
+
+begin
+  require 'sdoc_helpers'
+rescue LoadError
+end
+
+
+#
+# Publishing
+#
+
+desc "Push a new version to Gemcutter"
+task :publish do
+  require 'resque/version'
+
+  sh "gem build resque.gemspec"
+  sh "gem push resque-#{Resque::Version}.gem"
+  sh "git tag v#{Resque::Version}"
+  sh "git push origin v#{Resque::Version}"
+  sh "git push origin master"
+  sh "git clean -fd"
+  exec "rake pages"
+end

data/bin/resque

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'resque'
+require 'optparse'
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: resque [options] COMMAND"
+
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-m", "--mongo [HOST:PORT]", "Redis connection string") do |host|
+    Resque.mongo = host
+  end
+
+  opts.on("-N", "--namespace [NAMESPACE]", "Redis namespace") do |namespace|
+    Resque.redis.namespace = namespace
+  end
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.separator ""
+  opts.separator "Commands:"
+  opts.separator "  remove WORKER   Removes a worker"
+  opts.separator "  kill WORKER     Kills a worker"
+  opts.separator "  list            Lists known workers"
+end
+
+def kill(worker)
+  abort "** resque kill WORKER_ID" if worker.nil?
+  pid = worker.split(':')[1].to_i
+
+  begin
+    Process.kill("KILL", pid)
+    puts "** killed #{worker}"
+  rescue Errno::ESRCH
+    puts "** worker #{worker} not running"
+  end
+
+  remove worker
+end
+
+def remove(worker)
+  abort "** resque remove WORKER_ID" if worker.nil?
+
+  Resque.remove_worker(worker)
+  puts "** removed #{worker}"
+end
+
+def list
+  if Resque.workers.any?
+    Resque.workers.each do |worker|
+      puts "#{worker} (#{worker.state})"
+    end
+  else
+    puts "None"
+  end
+end
+
+parser.parse!
+
+case ARGV[0]
+when 'kill'
+  kill ARGV[1]
+when 'remove'
+  remove ARGV[1]
+when 'list'
+  list
+else
+  puts parser.help
+end

data/bin/resque-web

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'vegas'
+rescue LoadError
+  require 'rubygems'
+  require 'vegas'
+end  
+require 'resque/server'
+
+
+Vegas::Runner.new(Resque::Server, 'resque-web', {
+  :before_run => lambda {|v|
+    path = (ENV['RESQUECONFIG'] || v.args.first)
+    load path.to_s.strip if path
+  }
+}) do |runner, opts, app|
+  opts.on('-N NAMESPACE', "--namespace NAMESPACE", "set the Redis namespace") {|namespace|
+    runner.logger.info "Using Redis namespace '#{namespace}'"
+    Resque.redis.namespace = namespace
+  }
+end

data/lib/resque/errors.rb

@@ -0,0 +1,10 @@
+module Resque
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+
+  # Raised when a worker was killed while processing a job.
+  class DirtyExit < RuntimeError; end
+end

data/lib/resque/failure/base.rb

@@ -0,0 +1,74 @@
+module Resque
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count
+        0
+      end
+
+      # The number of failures after a search.
+      def self.search_count
+        0
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(start = 0, count = 1)
+        []
+      end
+
+      # Returns a paginated array of failure objects after a search.
+      def self.search_results(query, start = 0, count = 1)
+        []
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear
+      end
+      
+      def self.requeue(index)
+      end
+
+      def self.remove(index)
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque/failure/hoptoad.rb

@@ -0,0 +1,139 @@
+require 'net/https'
+require 'builder'
+require 'uri'
+
+module Resque
+  module Failure
+    # A Failure backend that sends exceptions raised by jobs to Hoptoad.
+    #
+    # To use it, put this code in an initializer, Rake task, or wherever:
+    #
+    #   require 'resque/failure/hoptoad'
+    #
+    #   Resque::Failure::Hoptoad.configure do |config|
+    #     config.api_key = 'blah'
+    #     config.secure = true
+    #
+    #     # optional proxy support
+    #     config.proxy_host = 'x.y.z.t'
+    #     config.proxy_port = 8080
+    #
+    #     # server env support, defaults to RAILS_ENV or RACK_ENV
+    #     config.server_environment = "test"
+    #   end
+    class Hoptoad < Base
+      # From the hoptoad plugin
+      INPUT_FORMAT = /^([^:]+):(\d+)(?::in `([^']+)')?$/
+
+      class << self
+        attr_accessor :secure, :api_key
+        attr_accessor :proxy_host, :proxy_port, :proxy_user, :proxy_pass
+        attr_accessor :server_environment
+        attr_accessor :host, :port
+        attr_accessor :http_read_timeout, :http_open_timeout
+      end
+
+      def self.count
+        # We can't get the total # of errors from Hoptoad so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def self.configure
+        yield self
+        Resque::Failure.backend = self
+      end
+
+      def save
+        http = use_ssl? ? :https : :http
+        host = self.class.host || 'hoptoadapp.com'
+        port = self.class.port
+        url = URI.parse("#{http}://#{host}:#{port}/notifier_api/v2/notices/")
+
+        request = Net::HTTP::Proxy self.class.proxy_host, self.class.proxy_port,
+                                   self.class.proxy_user, self.class.proxy_pass
+        http = request.new(url.host, url.port)
+        headers = {
+          'Content-type' => 'text/xml',
+          'Accept' => 'text/xml, application/xml'
+        }
+
+        http.read_timeout = self.class.http_read_timeout || 5 # seconds
+        http.open_timeout = self.class.http_open_timeout || 2 # seconds
+
+        http.use_ssl = use_ssl?
+
+        begin
+          response = http.post(url.path, xml, headers)
+        rescue TimeoutError => e
+          log "Timeout while contacting the Hoptoad server."
+        end
+
+        case response
+        when Net::HTTPSuccess then
+          log "Hoptoad Success: #{response.class}"
+        else
+          body = response.body if response.respond_to? :body
+          log "Hoptoad Failure: #{response.class}\n#{body}"
+        end
+      end
+
+      def xml
+        x = Builder::XmlMarkup.new
+        x.instruct!
+        x.notice :version=>"2.0" do
+          x.tag! "api-key", api_key
+          x.notifier do
+            x.name "Resqueue"
+            x.version "0.1"
+            x.url "http://github.com/defunkt/resque"
+          end
+          x.error do
+            x.tag! "class", exception.class.name
+            x.message "#{exception.class.name}: #{exception.message}"
+            x.backtrace do
+              fill_in_backtrace_lines(x)
+            end
+          end
+          x.request do
+            x.url queue.to_s
+            x.component worker.to_s
+            x.params do
+              x.var :key=>"payload_class" do
+                x.text! payload["class"].to_s
+              end
+              x.var :key=>"payload_args" do
+                x.text! payload["args"].to_s
+              end
+            end
+          end
+          x.tag!("server-environment") do
+            x.tag!("environment-name",server_environment)
+            x.tag!("project-root", "RAILS_ROOT")
+          end
+
+        end
+      end
+
+      def fill_in_backtrace_lines(x)
+        Array(exception.backtrace).each do |unparsed_line|
+          _, file, number, method = unparsed_line.match(INPUT_FORMAT).to_a
+          x.line :file => file,:number => number
+        end
+      end
+
+      def use_ssl?
+        self.class.secure
+      end
+
+      def api_key
+        self.class.api_key
+      end
+
+      def server_environment
+        return self.class.server_environment if self.class.server_environment
+        defined?(RAILS_ENV) ? RAILS_ENV : (ENV['RACK_ENV'] || 'development')
+      end
+    end
+  end
+end

data/lib/resque/failure/mongo.rb

@@ -0,0 +1,92 @@
+require 'set'
+
+module Resque
+  module Failure
+    # A Failure backend that stores exceptions in Mongo. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class Mongo < Base
+      def save
+        data = {
+          :failed_at => Time.now.utc,
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => exception.to_s,
+          :backtrace => Array(exception.backtrace),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        Resque.mongo_failures << data
+      end
+
+      def self.count
+        Resque.mongo_failures.count
+      end
+      
+      def self.search_count
+        @@search_results.count
+      end
+
+      def self.all(start = 0, count = 1)
+        start, count = [start, count].map { |n| Integer(n) }
+        all_failures = Resque.mongo_failures.find().sort([:failed_at, :desc]).skip(start).limit(count).to_a
+        all_failures.size == 1 ? all_failures.first : all_failures
+      end
+      
+      # Looks for failed jobs who match a particular search string
+      def self.search_results(query, start = 0, count = 1)
+        # If the search query is nil or empty, return an empty array
+        if query.nil? || query.empty?
+          @@search_results = []
+          return []
+        end
+        
+        start, count = [start, count].map { |n| Integer(n) }
+        set_results = Set.new
+        
+        # For each search term, retrieve the failed jobs that contain at least one relevant field matching the regexp defined by that search term
+        query.split.each do |term|
+          partial_results = Resque.mongo_failures.find(
+            {"$or" => [ 
+              {"exception" => /#{term}/i},
+              {"error" => /#{term}/i},
+              {"payload.class" => /#{term}/i},
+              {"payload.args" => /#{term}/i},
+              {"worker" => /#{term}/i},
+              {"queue" => /#{term}/i},
+              {"backtrace" => /#{term}/i}
+            ] }
+          ).sort([:failed_at, :desc]).to_a
+            
+          # If the set was empty, merge the first results, else intersect it with the current results
+          if set_results.empty?
+            set_results.merge(partial_results)
+          else
+            set_results = set_results & partial_results
+          end
+        end
+        
+        # search_res will be an array containing 'count' values, starting with 'start', sorted in descending order
+        @@search_results = set_results.to_a || []
+        search_results = set_results.to_a[start, count]
+        
+        search_results || []
+      end
+
+      def self.clear
+        Resque.mongo_failures.remove
+      end
+
+      def self.requeue(index)
+        item = all(index)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        Resque.mongo_failures.update({:_id => item['_id']}, item)
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+      
+      def self.remove(index)
+        item = all(index)
+        Resque.mongo_failures.remove(:_id => item['_id'])
+      end
+    end
+  end
+end