resque_sqs 1.25.2

data/Rakefile

@@ -0,0 +1,70 @@
+#
+# Setup
+#
+
+load 'lib/tasks/redis_sqs.rake'
+
+$LOAD_PATH.unshift 'lib'
+require 'resque_sqs/tasks'
+
+def command?(command)
+  system("type #{command} > /dev/null 2>&1")
+end
+
+
+#
+# Tests
+#
+
+require 'rake/testtask'
+
+task :default => :test
+
+Rake::TestTask.new do |test|
+  test.verbose = true
+  test.libs << "test"
+  test.libs << "lib"
+  test.test_files = FileList['test/**/*_test.rb']
+end
+
+if command? :kicker
+  desc "Launch Kicker (like autotest)"
+  task :kicker do
+    puts "Kicking... (ctrl+c to cancel)"
+    exec "kicker -e rake test lib examples"
+  end
+end
+
+
+#
+# Install
+#
+
+task :install => [ 'redis_sqs:install', 'dtach_sqs:install' ]
+
+
+#
+# Documentation
+#
+
+begin
+  require 'sdoc_helpers'
+rescue LoadError
+end
+
+
+#
+# Publishing
+#
+
+desc "Push a new version to Gemcutter"
+task :sqs_publish do
+  require 'resque_sqs/version'
+
+  sh "gem build resque_sqs.gemspec"
+  sh "gem push resque_sqs-#{ResqueSqs::Version}.gem"
+  sh "git tag v#{ResqueSqs::Version}"
+  sh "git push origin v#{ResqueSqs::Version}"
+  sh "git push origin master"
+  sh "git clean -fd"
+end

data/bin/resque-sqs

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'redis-namespace'
+rescue LoadError
+  require 'rubygems'
+  require 'redis-namespace'
+end
+require 'resque_sqs'
+require 'optparse'
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: resque [options] COMMAND"
+
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-r", "--redis [HOST:PORT]", "Redis connection string") do |host|
+    ResqueSqs.redis = host
+  end
+
+  opts.on("-N", "--namespace [NAMESPACE]", "Redis namespace") do |namespace|
+    ResqueSqs.redis.namespace = namespace
+  end
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+  opts.separator ""
+  opts.separator "Commands:"
+  opts.separator "  remove WORKER   Removes a worker"
+  opts.separator "  kill WORKER     Kills a worker"
+  opts.separator "  list            Lists known workers"
+end
+
+def kill(worker)
+  abort "** resque kill WORKER_ID" if worker.nil?
+  pid = worker.split(':')[1].to_i
+
+  begin
+    Process.kill("KILL", pid)
+    puts "** killed #{worker}"
+  rescue Errno::ESRCH
+    puts "** worker #{worker} not running"
+  end
+
+  remove worker
+end
+
+def remove(worker)
+  abort "** resque remove WORKER_ID" if worker.nil?
+
+  ResqueSqs.remove_worker(worker)
+  puts "** removed #{worker}"
+end
+
+def list
+  if ResqueSqs.workers.any?
+    ResqueSqs.workers.each do |worker|
+      puts "#{worker} (#{worker.state})"
+    end
+  else
+    puts "None"
+  end
+end
+
+parser.parse!
+
+case ARGV[0]
+when 'kill'
+  kill ARGV[1]
+when 'remove'
+  remove ARGV[1]
+when 'list'
+  list
+else
+  puts parser.help
+end

data/bin/resque-sqs-web

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+begin
+  require 'vegas'
+rescue LoadError
+  require 'rubygems'
+  require 'vegas'
+end  
+require 'resque_sqs/server'
+
+
+Vegas::Runner.new(ResqueSqs::Server, 'resque-web', {
+  :before_run => lambda {|v|
+    path = (ENV['RESQUECONFIG'] || v.args.first)
+    load path.to_s.strip if path
+  }
+}) do |runner, opts, app|
+  opts.on('-N NAMESPACE', "--namespace NAMESPACE", "set the Redis namespace") {|namespace|
+    runner.logger.info "Using Redis namespace '#{namespace}'"
+    ResqueSqs.redis.namespace = namespace
+  }
+  opts.on('-r redis-connection', "--redis redis-connection", "set the Redis connection string") {|redis_conf|
+    runner.logger.info "Using Redis connection '#{redis_conf}'"
+    ResqueSqs.redis = redis_conf
+  }
+end

data/lib/resque_sqs/errors.rb

@@ -0,0 +1,13 @@
+module ResqueSqs
+  # Raised whenever we need a queue but none is provided.
+  class NoQueueError < RuntimeError; end
+
+  # Raised when trying to create a job without a class
+  class NoClassError < RuntimeError; end
+
+  # Raised when a worker was killed while processing a job.
+  class DirtyExit < RuntimeError; end
+
+  # Raised when child process is TERM'd so job can rescue this to do shutdown work.
+  class TermException < SignalException; end
+end

data/lib/resque_sqs/failure/airbrake.rb

@@ -0,0 +1,33 @@
+begin
+  require 'airbrake'
+rescue LoadError
+  raise "Can't find 'airbrake' gem. Please add it to your Gemfile or install it."
+end
+
+module ResqueSqs
+  module Failure
+    class Airbrake < Base
+      def self.configure(&block)
+        ResqueSqs.logger.warn "This actually sets global Airbrake configuration, " \
+          "which is probably not what you want. This will be gone in 2.0."
+        ResqueSqs::Failure.backend = self
+        ::Airbrake.configure(&block)
+      end
+
+      def self.count(queue = nil, class_name = nil)
+        # We can't get the total # of errors from Airbrake so we fake it
+        # by asking Resque how many errors it has seen.
+        Stat[:failed]
+      end
+
+      def save
+        ::Airbrake.notify_or_ignore(exception,
+            :parameters => {
+            :payload_class => payload['class'].to_s,
+            :payload_args => payload['args'].inspect
+            }
+          )
+      end
+    end
+  end
+end

data/lib/resque_sqs/failure/base.rb

@@ -0,0 +1,73 @@
+module ResqueSqs
+  module Failure
+    # All Failure classes are expected to subclass Base.
+    #
+    # When a job fails, a new instance of your Failure backend is created
+    # and #save is called.
+    class Base
+      # The exception object raised by the failed job
+      attr_accessor :exception
+
+      # The worker object who detected the failure
+      attr_accessor :worker
+
+      # The string name of the queue from which the failed job was pulled
+      attr_accessor :queue
+
+      # The payload object associated with the failed job
+      attr_accessor :payload
+
+      def initialize(exception, worker, queue, payload)
+        @exception = exception
+        @worker    = worker
+        @queue     = queue
+        @payload   = payload
+      end
+
+      # When a job fails, a new instance of your Failure backend is created
+      # and #save is called.
+      #
+      # This is where you POST or PUT or whatever to your Failure service.
+      def save
+      end
+
+      # The number of failures.
+      def self.count(queue = nil, class_name = nil)
+        0
+      end
+
+      # Returns an array of all available failure queues
+      def self.queues
+        []
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(offset = 0, limit = 1, queue = nil)
+        []
+      end
+
+      # Iterate across failed objects
+      def self.each(*args)
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+      end
+      
+      # Clear all failure objects
+      def self.clear(*args)
+      end
+      
+      def self.requeue(index)
+      end
+
+      def self.remove(index)
+      end
+
+      # Logging!
+      def log(message)
+        @worker.log(message)
+      end
+    end
+  end
+end

data/lib/resque_sqs/failure/multiple.rb

@@ -0,0 +1,59 @@
+module ResqueSqs
+  module Failure
+    # A Failure backend that uses multiple backends
+    # delegates all queries to the first backend
+    class Multiple < Base
+
+      class << self
+        attr_accessor :classes
+      end
+
+      def self.configure
+        yield self
+        ResqueSqs::Failure.backend = self
+      end
+
+      def initialize(*args)
+        super
+        @backends = self.class.classes.map {|klass| klass.new(*args)}
+      end
+
+      def save
+        @backends.each(&:save)
+      end
+
+      # The number of failures.
+      def self.count(*args)
+        classes.first.count(*args)
+      end
+
+      # Returns a paginated array of failure objects.
+      def self.all(*args)
+        classes.first.all(*args)
+      end
+
+      # Iterate across failed objects
+      def self.each(*args, &block)
+        classes.first.each(*args, &block)
+      end
+
+      # A URL where someone can go to view failures.
+      def self.url
+        classes.first.url
+      end
+
+      # Clear all failure objects
+      def self.clear(*args)
+        classes.first.clear(*args)
+      end
+
+      def self.requeue(*args)
+        classes.first.requeue(*args)
+      end
+
+      def self.remove(index)
+        classes.each { |klass| klass.remove(index) }
+      end
+    end
+  end
+end

data/lib/resque_sqs/failure/redis.rb

@@ -0,0 +1,108 @@
+module ResqueSqs
+  module Failure
+    # A Failure backend that stores exceptions in Redis. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class Redis < Base
+      def save
+        data = {
+          :failed_at => UTF8Util.clean(Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")),
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => UTF8Util.clean(exception.to_s),
+          :backtrace => filter_backtrace(Array(exception.backtrace)),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        data = ResqueSqs.encode(data)
+        ResqueSqs.redis.rpush(:failed, data)
+      end
+
+      def self.count(queue = nil, class_name = nil)
+        check_queue(queue)
+
+        if class_name
+          n = 0
+          each(0, count(queue), queue, class_name) { n += 1 } 
+          n
+        else
+          ResqueSqs.redis.llen(:failed).to_i
+        end
+      end
+
+      def self.queues
+        [:failed]
+      end
+
+      def self.all(offset = 0, limit = 1, queue = nil)
+        check_queue(queue)
+        ResqueSqs.list_range(:failed, offset, limit)
+      end
+
+      def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
+        all_items = Array(all(offset, limit, queue))
+        reversed = false
+        if order.eql? 'desc'
+          all_items.reverse!
+          reversed = true
+        end
+        all_items.each_with_index do |item, i|
+          if !class_name || (item['payload'] && item['payload']['class'] == class_name)
+            if reversed
+              id = (all_items.length - 1) - (offset + i)
+            else
+              id = offset + i
+            end
+            yield id, item
+          end
+        end
+      end
+
+      def self.clear(queue = nil)
+        check_queue(queue)
+        ResqueSqs.redis.del(:failed)
+      end
+
+      def self.requeue(id)
+        item = all(id)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        ResqueSqs.redis.lset(:failed, id, ResqueSqs.encode(item))
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+
+      def self.remove(id)
+        sentinel = ""
+        ResqueSqs.redis.lset(:failed, id, sentinel)
+        ResqueSqs.redis.lrem(:failed, 1,  sentinel)
+      end
+
+      def self.requeue_queue(queue)
+        i = 0
+        while job = all(i)
+           requeue(i) if job['queue'] == queue
+           i += 1
+        end
+      end
+
+      def self.remove_queue(queue)
+        i = 0
+        while job = all(i)
+          if job['queue'] == queue
+            # This will remove the failure from the array so do not increment the index.
+            remove(i)
+          else
+            i += 1
+          end
+        end
+      end
+
+      def self.check_queue(queue)
+        raise ArgumentError, "invalid queue: #{queue}" if queue && queue.to_s != "failed"
+      end
+
+      def filter_backtrace(backtrace)
+        index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
+        backtrace.first(index.to_i)
+      end
+    end
+  end
+end

data/lib/resque_sqs/failure/redis_multi_queue.rb

@@ -0,0 +1,89 @@
+module ResqueSqs
+  module Failure
+    # A Failure backend that stores exceptions in Redis. Very simple but
+    # works out of the box, along with support in the Resque web app.
+    class RedisMultiQueue < Base
+      def save
+        data = {
+          :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
+          :payload   => payload,
+          :exception => exception.class.to_s,
+          :error     => UTF8Util.clean(exception.to_s),
+          :backtrace => filter_backtrace(Array(exception.backtrace)),
+          :worker    => worker.to_s,
+          :queue     => queue
+        }
+        data = ResqueSqs.encode(data)
+        ResqueSqs.redis.rpush(ResqueSqs::Failure.failure_queue_name(queue), data)
+      end
+
+      def self.count(queue = nil, class_name = nil)
+        if queue
+          if class_name
+            n = 0
+            each(0, count(queue), queue, class_name) { n += 1 } 
+            n
+          else
+            ResqueSqs.redis.llen(queue).to_i
+          end
+        else
+          total = 0
+          queues.each { |q| total += count(q) }
+          total
+        end
+      end
+
+      def self.all(offset = 0, limit = 1, queue = :failed, order = 'desc')
+        ResqueSqs.list_range(queue, offset, limit, order)
+      end
+
+      def self.queues
+        Array(ResqueSqs.redis.smembers(:failed_queues))
+      end
+
+      def self.each(offset = 0, limit = self.count, queue = :failed, class_name = nil, order = 'desc')
+        items = all(offset, limit, queue, order)
+        items = [items] unless items.is_a? Array
+        if order.eql? 'desc'
+          items.reverse!
+        end
+        items.each_with_index do |item, i|
+          if !class_name || (item['payload'] && item['payload']['class'] == class_name)
+            yield offset + i, item
+          end
+        end
+      end
+
+      def self.clear(queue = :failed)
+        ResqueSqs.redis.del(queue)
+      end
+
+      def self.requeue(id, queue = :failed)
+        item = all(id, 1, queue)
+        item['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
+        ResqueSqs.redis.lset(queue, id, ResqueSqs.encode(item))
+        Job.create(item['queue'], item['payload']['class'], *item['payload']['args'])
+      end
+
+      def self.remove(id, queue = :failed)
+        sentinel = ""
+        ResqueSqs.redis.lset(queue, id, sentinel)
+        ResqueSqs.redis.lrem(queue, 1,  sentinel)
+      end
+
+      def self.requeue_queue(queue)
+        failure_queue = ResqueSqs::Failure.failure_queue_name(queue)
+        each(0, count(failure_queue), failure_queue) { |id, _| requeue(id, failure_queue) }
+      end
+
+      def self.remove_queue(queue)
+        ResqueSqs.redis.del(ResqueSqs::Failure.failure_queue_name(queue))
+      end
+
+      def filter_backtrace(backtrace)
+        index = backtrace.index { |item| item.include?('/lib/resque/job.rb') }
+        backtrace.first(index.to_i)
+      end
+    end
+  end
+end

data/lib/resque_sqs/failure.rb

@@ -0,0 +1,113 @@
+module ResqueSqs
+  # The Failure module provides an interface for working with different
+  # failure backends.
+  #
+  # You can use it to query the failure backend without knowing which specific
+  # backend is being used. For instance, the Resque web app uses it to display
+  # stats and other information.
+  module Failure
+    # Creates a new failure, which is delegated to the appropriate backend.
+    #
+    # Expects a hash with the following keys:
+    #   :exception - The Exception object
+    #   :worker    - The Worker object who is reporting the failure
+    #   :queue     - The string name of the queue from which the job was pulled
+    #   :payload   - The job's payload
+    def self.create(options = {})
+      backend.new(*options.values_at(:exception, :worker, :queue, :payload)).save
+    end
+
+    #
+    # Sets the current backend. Expects a class descendent of
+    # `ResqueSqs::Failure::Base`.
+    #
+    # Example use:
+    #   require 'resque_sqs/failure/airbrake'
+    #   ResqueSqs::Failure.backend = ResqueSqs::Failure::Airbrake
+    def self.backend=(backend)
+      @backend = backend
+    end
+
+    # Returns the current backend class. If none has been set, falls
+    # back to `ResqueSqs::Failure::Redis`
+    def self.backend
+      return @backend if @backend
+
+      case ENV['FAILURE_BACKEND']
+      when 'redis_multi_queue'
+        require 'resque_sqs/failure/redis_multi_queue'
+        @backend = Failure::RedisMultiQueue
+      when 'redis', nil
+        require 'resque_sqs/failure/redis'
+        @backend = Failure::Redis
+      else
+        raise ArgumentError, "invalid failure backend: #{FAILURE_BACKEND}"
+      end
+    end
+
+    # Obtain the failure queue name for a given job queue
+    def self.failure_queue_name(job_queue_name)
+      name = "#{job_queue_name}_failed"
+      ResqueSqs.redis.sadd(:failed_queues, name)
+      name
+    end
+
+    # Obtain the job queue name for a given failure queue
+    def self.job_queue_name(failure_queue_name)
+      failure_queue_name.sub(/_failed$/, '')
+    end
+
+    # Returns an array of all the failed queues in the system
+    def self.queues
+      backend.queues
+    end
+
+    # Returns the int count of how many failures we have seen.
+    def self.count(queue = nil, class_name = nil)
+      backend.count(queue, class_name)
+    end
+
+    # Returns an array of all the failures, paginated.
+    #
+    # `offset` is the int of the first item in the page, `limit` is the
+    # number of items to return.
+    def self.all(offset = 0, limit = 1, queue = nil)
+      backend.all(offset, limit, queue)
+    end
+
+    # Iterate across all failures with the given options
+    def self.each(offset = 0, limit = self.count, queue = nil, class_name = nil, order = 'desc', &block)
+      backend.each(offset, limit, queue, class_name, order, &block)
+    end
+
+    # The string url of the backend's web interface, if any.
+    def self.url
+      backend.url
+    end
+
+    # Clear all failure jobs
+    def self.clear(queue = nil)
+      backend.clear(queue)
+    end
+
+    def self.requeue(id)
+      backend.requeue(id)
+    end
+
+    def self.remove(id)
+      backend.remove(id)
+    end
+    
+    # Requeues all failed jobs in a specific queue.
+    # Queue name should be a string.
+    def self.requeue_queue(queue)
+      backend.requeue_queue(queue)
+    end
+
+    # Removes all failed jobs in a specific queue.
+    # Queue name should be a string.
+    def self.remove_queue(queue)
+      backend.remove_queue(queue)
+    end
+  end
+end