resque_sqs 1.25.2

data/lib/resque_sqs.rb

@@ -0,0 +1,479 @@
+require 'mono_logger'
+require 'redis/namespace'
+require 'forwardable'
+
+require 'resque_sqs/version'
+
+require 'resque_sqs/errors'
+
+require 'resque_sqs/failure'
+require 'resque_sqs/failure/base'
+
+require 'resque_sqs/helpers'
+require 'resque_sqs/stat'
+require 'resque_sqs/logging'
+require 'resque_sqs/log_formatters/quiet_formatter'
+require 'resque_sqs/log_formatters/verbose_formatter'
+require 'resque_sqs/log_formatters/very_verbose_formatter'
+require 'resque_sqs/job'
+require 'resque_sqs/worker'
+require 'resque_sqs/plugin'
+
+require 'resque_sqs/vendor/utf8_util'
+
+module ResqueSqs
+  extend self
+
+  # Given a Ruby object, returns a string suitable for storage in a
+  # queue.
+  def encode(object)
+    if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+      MultiJson.dump object
+    else
+      MultiJson.encode object
+    end
+  end
+
+  # Given a string, returns a Ruby object.
+  def decode(object)
+    return unless object
+
+    begin
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.load object
+      else
+        MultiJson.decode object
+      end
+    rescue ::MultiJson::DecodeError => e
+      raise Helpers::DecodeException, e.message, e.backtrace
+    end
+  end
+
+  extend ::Forwardable
+
+  def self.config=(options = {})
+    @config = Config.new(options)
+  end
+
+  def self.config
+    @config ||= Config.new
+  end
+
+  def self.configure
+    yield config
+  end
+
+  # Accepts:
+  #   1. A 'hostname:port' String
+  #   2. A 'hostname:port:db' String (to select the Redis db)
+  #   3. A 'hostname:port/namespace' String (to set the Redis namespace)
+  #   4. A Redis URL String 'redis://host:port'
+  #   5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
+  #      or `Redis::Namespace`.
+  #   6. An Hash of a redis connection {:host => 'localhost', :port => 6379, :db => 0}
+  def redis=(server)
+    case server
+    when String
+      if server =~ /redis\:\/\//
+        redis = Redis.connect(:url => server, :thread_safe => true)
+      else
+        server, namespace = server.split('/', 2)
+        host, port, db = server.split(':')
+        redis = Redis.new(:host => host, :port => port,
+          :thread_safe => true, :db => db)
+      end
+      namespace ||= :resque
+
+      @redis = Redis::Namespace.new(namespace, :redis => redis)
+    when Redis::Namespace
+      @redis = server
+    when Hash
+      @redis = Redis::Namespace.new(:resque, :redis => Redis.new(server))
+    else
+      @redis = Redis::Namespace.new(:resque, :redis => server)
+    end
+  end
+
+  # Returns the current Redis connection. If none has been created, will
+  # create a new one.
+  def redis
+    return @redis if @redis
+    self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
+    self.redis
+  end
+
+  def redis_id
+    # support 1.x versions of redis-rb
+    if redis.respond_to?(:server)
+      redis.server
+    elsif redis.respond_to?(:nodes) # distributed
+      redis.nodes.map { |n| n.id }.join(', ')
+    else
+      redis.client.id
+    end
+  end
+
+  # Set or retrieve the current logger object
+  attr_accessor :logger
+
+  # The `before_first_fork` hook will be run in the **parent** process
+  # only once, before forking to run the first job. Be careful- any
+  # changes you make will be permanent for the lifespan of the
+  # worker.
+  #
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
+  def before_first_fork(&block)
+    block ? register_hook(:before_first_fork, block) : hooks(:before_first_fork)
+  end
+
+  # Register a before_first_fork proc.
+  def before_first_fork=(block)
+    register_hook(:before_first_fork, block)
+  end
+
+  # The `before_fork` hook will be run in the **parent** process
+  # before every job, so be careful- any changes you make will be
+  # permanent for the lifespan of the worker.
+  #
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
+  def before_fork(&block)
+    block ? register_hook(:before_fork, block) : hooks(:before_fork)
+  end
+
+  # Register a before_fork proc.
+  def before_fork=(block)
+    register_hook(:before_fork, block)
+  end
+
+  # The `after_fork` hook will be run in the child process and is passed
+  # the current job. Any changes you make, therefore, will only live as
+  # long as the job currently being processed.
+  #
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
+  def after_fork(&block)
+    block ? register_hook(:after_fork, block) : hooks(:after_fork)
+  end
+
+  # Register an after_fork proc.
+  def after_fork=(block)
+    register_hook(:after_fork, block)
+  end
+
+  # The `before_pause` hook will be run in the parent process before the
+  # worker has paused processing (via #pause_processing or SIGUSR2).
+  def before_pause(&block)
+    block ? register_hook(:before_pause, block) : hooks(:before_pause)
+  end
+
+  # Set the after_pause proc.
+  attr_writer :before_pause
+
+  # The `after_pause` hook will be run in the parent process after the
+  # worker has paused (via SIGCONT).
+  def after_pause(&block)
+    block ? register_hook(:after_pause, block) : hooks(:after_pause)
+  end
+
+  # Set the after_continue proc.
+  attr_writer :after_pause
+
+  def to_s
+    "Resque Client connected to #{redis_id}"
+  end
+
+  attr_accessor :inline
+
+  # If 'inline' is true Resque will call #perform method inline
+  # without queuing it into Redis and without any Resque callbacks.
+  # The 'inline' is false Resque jobs will be put in queue regularly.
+  alias :inline? :inline
+
+  #
+  # queue manipulation
+  #
+
+  # Pushes a job onto a queue. Queue name should be a string and the
+  # item should be any JSON-able Ruby object.
+  #
+  # Resque works generally expect the `item` to be a hash with the following
+  # keys:
+  #
+  #   class - The String name of the job to run.
+  #    args - An Array of arguments to pass the job. Usually passed
+  #           via `class.to_class.perform(*args)`.
+  #
+  # Example
+  #
+  #   ResqueSqs.push('archive', :class => 'Archive', :args => [ 35, 'tar' ])
+  #
+  # Returns nothing
+  def push(queue, item)
+    redis.pipelined do
+      watch_queue(queue)
+      redis.rpush "queue:#{queue}", encode(item)
+    end
+  end
+
+  # Pops a job off a queue. Queue name should be a string.
+  #
+  # Returns a Ruby object.
+  def pop(queue)
+    decode redis.lpop("queue:#{queue}")
+  end
+
+  # Returns an integer representing the size of a queue.
+  # Queue name should be a string.
+  def size(queue)
+    redis.llen("queue:#{queue}").to_i
+  end
+
+  # Returns an array of items currently queued. Queue name should be
+  # a string.
+  #
+  # start and count should be integer and can be used for pagination.
+  # start is the item to begin, count is how many items to return.
+  #
+  # To get the 3rd page of a 30 item, paginatied list one would use:
+  #   ResqueSqs.peek('my_list', 59, 30)
+  def peek(queue, start = 0, count = 1)
+    list_range("queue:#{queue}", start, count)
+  end
+
+  # Does the dirty work of fetching a range of items from a Redis list
+  # and converting them into Ruby objects.
+  def list_range(key, start = 0, count = 1)
+    if count == 1
+      decode redis.lindex(key, start)
+    else
+      Array(redis.lrange(key, start, start+count-1)).map do |item|
+        decode item
+      end
+    end
+  end
+
+  # Returns an array of all known Resque queues as strings.
+  def queues
+    Array(redis.smembers(:queues))
+  end
+
+  # Given a queue name, completely deletes the queue.
+  def remove_queue(queue)
+    redis.pipelined do
+      redis.srem(:queues, queue.to_s)
+      redis.del("queue:#{queue}")
+    end
+  end
+
+  # Used internally to keep track of which queues we've created.
+  # Don't call this directly.
+  def watch_queue(queue)
+    redis.sadd(:queues, queue.to_s)
+  end
+
+
+  #
+  # job shortcuts
+  #
+
+  # This method can be used to conveniently add a job to a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `ResqueSqs::NoQueueError`
+  #
+  # Returns true if the job was queued, nil if the job was rejected by a
+  # before_enqueue hook.
+  #
+  # This method is considered part of the `stable` API.
+  def enqueue(klass, *args)
+    enqueue_to(queue_from_class(klass), klass, *args)
+  end
+
+  # Just like `enqueue` but allows you to specify the queue you want to
+  # use. Runs hooks.
+  #
+  # `queue` should be the String name of the queue you're targeting.
+  #
+  # Returns true if the job was queued, nil if the job was rejected by a
+  # before_enqueue hook.
+  #
+  # This method is considered part of the `stable` API.
+  def enqueue_to(queue, klass, *args)
+    # Perform before_enqueue hooks. Don't perform enqueue if any hook returns false
+    before_hooks = Plugin.before_enqueue_hooks(klass).collect do |hook|
+      klass.send(hook, *args)
+    end
+    return nil if before_hooks.any? { |result| result == false }
+
+    Job.create(queue, klass, *args)
+
+    Plugin.after_enqueue_hooks(klass).each do |hook|
+      klass.send(hook, *args)
+    end
+
+    return true
+  end
+
+  # This method can be used to conveniently remove a job from a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `ResqueSqs::NoQueueError`
+  #
+  # If no args are given, this method will dequeue *all* jobs matching
+  # the provided class. See `ResqueSqs::Job.destroy` for more
+  # information.
+  #
+  # Returns the number of jobs destroyed.
+  #
+  # Example:
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph`
+  #   ResqueSqs.dequeue(GitHub::Jobs::UpdateNetworkGraph)
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph` with matching args.
+  #   ResqueSqs.dequeue(GitHub::Jobs::UpdateNetworkGraph, 'repo:135325')
+  #
+  # This method is considered part of the `stable` API.
+  def dequeue(klass, *args)
+    # Perform before_dequeue hooks. Don't perform dequeue if any hook returns false
+    before_hooks = Plugin.before_dequeue_hooks(klass).collect do |hook|
+      klass.send(hook, *args)
+    end
+    return if before_hooks.any? { |result| result == false }
+
+    destroyed = Job.destroy(queue_from_class(klass), klass, *args)
+
+    Plugin.after_dequeue_hooks(klass).each do |hook|
+      klass.send(hook, *args)
+    end
+
+    destroyed
+  end
+
+  # Given a class, try to extrapolate an appropriate queue based on a
+  # class instance variable or `queue` method.
+  def queue_from_class(klass)
+    klass.instance_variable_get(:@queue) ||
+      (klass.respond_to?(:queue) and klass.queue)
+  end
+
+  # This method will return a `ResqueSqs::Job` object or a non-true value
+  # depending on whether a job can be obtained. You should pass it the
+  # precise name of a queue: case matters.
+  #
+  # This method is considered part of the `stable` API.
+  def reserve(queue)
+    Job.reserve(queue)
+  end
+
+  # Validates if the given klass could be a valid Resque job
+  #
+  # If no queue can be inferred this method will raise a `ResqueSqs::NoQueueError`
+  #
+  # If given klass is nil this method will raise a `ResqueSqs::NoClassError`
+  def validate(klass, queue = nil)
+    queue ||= queue_from_class(klass)
+
+    if !queue
+      raise NoQueueError.new("Jobs must be placed onto a queue.")
+    end
+
+    if klass.to_s.empty?
+      raise NoClassError.new("Jobs must be given a class.")
+    end
+  end
+
+
+  #
+  # worker shortcuts
+  #
+
+  # A shortcut to Worker.all
+  def workers
+    Worker.all
+  end
+
+  # A shortcut to Worker.working
+  def working
+    Worker.working
+  end
+
+  # A shortcut to unregister_worker
+  # useful for command line tool
+  def remove_worker(worker_id)
+    worker = ResqueSqs::Worker.find(worker_id)
+    worker.unregister_worker
+  end
+
+  #
+  # stats
+  #
+
+  # Returns a hash, similar to redis-rb's #info, of interesting stats.
+  def info
+    return {
+      :pending   => queues.inject(0) { |m,k| m + size(k) },
+      :processed => Stat[:processed],
+      :queues    => queues.size,
+      :workers   => workers.size.to_i,
+      :working   => working.size,
+      :failed    => ResqueSqs.redis.llen(:failed).to_i,
+      :servers   => [redis_id],
+      :environment  => ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    }
+  end
+
+  # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+  # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+  def keys
+    redis.keys("*").map do |key|
+      key.sub("#{redis.namespace}:", '')
+    end
+  end
+
+  private
+
+  # Register a new proc as a hook. If the block is nil this is the
+  # equivalent of removing all hooks of the given name.
+  #
+  # `name` is the hook that the block should be registered with.
+  def register_hook(name, block)
+    return clear_hooks(name) if block.nil?
+
+    @hooks ||= {}
+    @hooks[name] ||= []
+
+    block = Array(block)
+    @hooks[name].concat(block)
+  end
+
+  # Clear all hooks given a hook name.
+  def clear_hooks(name)
+    @hooks && @hooks[name] = []
+  end
+
+  # Retrieve all hooks of a given name.
+  def hooks(name)
+    (@hooks && @hooks[name]) || []
+  end
+end
+
+# Log to STDOUT by default
+ResqueSqs.logger           = MonoLogger.new(STDOUT)
+ResqueSqs.logger.formatter = ResqueSqs::QuietFormatter.new

data/lib/tasks/redis_sqs.rake

@@ -0,0 +1,161 @@
+# Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master
+require 'fileutils'
+require 'open-uri'
+require 'pathname'
+
+class RedisSqsRunner
+  def self.redis_dir
+    @redis_dir ||= if ENV['PREFIX']
+                     Pathname.new(ENV['PREFIX'])
+                   else
+                     Pathname.new(`which redis-server`) + '..' + '..'
+                   end
+  end
+
+  def self.bin_dir
+    redis_dir + 'bin'
+  end
+
+  def self.config
+    @config ||= if File.exists?(redis_dir + 'etc/redis.conf')
+                  redis_dir + 'etc/redis.conf'
+                else
+                  redis_dir + '../etc/redis.conf'
+                end
+  end
+
+  def self.dtach_socket
+    '/tmp/redis.dtach'
+  end
+
+  # Just check for existance of dtach socket
+  def self.running?
+    File.exists? dtach_socket
+  end
+
+  def self.start
+    puts 'Detach with Ctrl+\  Re-attach with rake redis_sqs:attach'
+    sleep 1
+    command = "#{bin_dir}/dtach -A #{dtach_socket} #{bin_dir}/redis-server #{config}"
+    sh command
+  end
+
+  def self.attach
+    exec "#{bin_dir}/dtach -a #{dtach_socket}"
+  end
+
+  def self.stop
+    sh 'echo "SHUTDOWN" | nc localhost 6379'
+  end
+end
+
+INSTALL_DIR = ENV['INSTALL_DIR'] || '/tmp/redis'
+
+namespace :sqs_redis do
+  desc 'About redis'
+  task :about do
+    puts "\nSee http://code.google.com/p/redis/ for information about redis.\n\n"
+  end
+
+  desc 'Start redis'
+  task :start do
+    RedisSqsRunner.start
+  end
+
+  desc 'Stop redis'
+  task :stop do
+    RedisSqsRunner.stop
+  end
+
+  desc 'Restart redis'
+  task :restart do
+    RedisSqsRunner.stop
+    RedisSqsRunner.start
+  end
+
+  desc 'Attach to redis dtach socket'
+  task :attach do
+    RedisSqsRunner.attach
+  end
+
+  desc <<-DOC
+  Install the latest verison of Redis from Github (requires git, duh).
+    Use INSTALL_DIR env var like "rake redis_sqs:install INSTALL_DIR=~/tmp"
+    in order to get an alternate location for your install files.
+  DOC
+
+  task :install => [:about, :download, :make] do
+    bin_dir = '/usr/bin'
+    conf_dir = '/etc'
+
+    if ENV['PREFIX']
+      bin_dir = "#{ENV['PREFIX']}/bin"
+      sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
+
+      conf_dir = "#{ENV['PREFIX']}/etc"
+      sh "mkdir -p #{conf_dir}" unless File.exists?("#{conf_dir}")
+    end
+
+    %w(redis-benchmark redis-cli redis-server).each do |bin|
+      sh "cp #{INSTALL_DIR}/src/#{bin} #{bin_dir}"
+    end
+
+    puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
+
+    unless File.exists?("#{conf_dir}/redis.conf")
+      sh "cp #{INSTALL_DIR}/redis.conf #{conf_dir}/redis.conf"
+      puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
+    end
+  end
+
+  task :make do
+    sh "cd #{INSTALL_DIR}/src && make clean"
+    sh "cd #{INSTALL_DIR}/src && make"
+  end
+
+  desc "Download package"
+  task :download do
+    sh "rm -rf #{INSTALL_DIR}/" if File.exists?("#{INSTALL_DIR}/.svn")
+    sh "git clone git://github.com/antirez/redis.git #{INSTALL_DIR}" unless File.exists?(INSTALL_DIR)
+    sh "cd #{INSTALL_DIR} && git pull" if File.exists?("#{INSTALL_DIR}/.git")
+  end
+end
+
+namespace :dtach_sqs do
+  desc 'About dtach'
+  task :about do
+    puts "\nSee http://dtach.sourceforge.net/ for information about dtach.\n\n"
+  end
+
+  desc 'Install dtach 0.8 from source'
+  task :install => [:about, :download, :make] do
+
+    bin_dir = "/usr/bin"
+
+
+    if ENV['PREFIX']
+      bin_dir = "#{ENV['PREFIX']}/bin"
+      sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
+    end
+
+    sh "cp #{INSTALL_DIR}/dtach-0.8/dtach #{bin_dir}"
+  end
+
+  task :make do
+    sh "cd #{INSTALL_DIR}/dtach-0.8/ && ./configure && make"
+  end
+
+  desc "Download package"
+  task :download do
+    unless File.exists?("#{INSTALL_DIR}/dtach-0.8.tar.gz")
+      require 'net/http'
+
+      url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'
+      open("#{INSTALL_DIR}/dtach-0.8.tar.gz", 'wb') do |file| file.write(open(url).read) end
+    end
+
+    unless File.directory?("#{INSTALL_DIR}/dtach-0.8")
+      sh "cd #{INSTALL_DIR} && tar xzf dtach-0.8.tar.gz"
+    end
+  end
+end

data/lib/tasks/resque_sqs.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'resque_sqs/tasks'

data/test/airbrake_test.rb

@@ -0,0 +1,27 @@
+
+require 'test_helper'
+
+begin
+  require 'airbrake'
+rescue LoadError
+  warn "Install airbrake gem to run Airbrake tests."
+end
+
+if defined? Airbrake
+  require 'resque_sqs/failure/airbrake'
+  context "Airbrake" do
+    test "should be notified of an error" do
+      exception = StandardError.new("BOOM")
+      worker = ResqueSqs::Worker.new(:test)
+      queue = "test"
+      payload = {'class' => Object, 'args' => 66}
+
+      Airbrake.expects(:notify_or_ignore).with(
+        exception,
+        :parameters => {:payload_class => 'Object', :payload_args => '66'})
+
+      backend = ResqueSqs::Failure::Airbrake.new(exception, worker, queue, payload)
+      backend.save
+    end
+  end
+end

data/test/failure_base_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+require 'minitest/mock'
+
+require 'resque_sqs/failure/base'
+
+class TestFailure < ResqueSqs::Failure::Base
+end
+
+describe "Base failure class" do
+  it "allows calling all without throwing" do
+    with_failure_backend TestFailure do
+      assert_empty ResqueSqs::Failure.all
+    end
+  end
+end