resque 1.23.0 → 2.6.0

data/lib/resque.rb

@@ -1,4 +1,6 @@
+require 'mono_logger'
 require 'redis/namespace'
+require 'forwardable'
 
 require 'resque/version'
 
@@ -9,16 +11,98 @@ require 'resque/failure/base'
 
 require 'resque/helpers'
 require 'resque/stat'
+require 'resque/logging'
+require 'resque/log_formatters/quiet_formatter'
+require 'resque/log_formatters/verbose_formatter'
+require 'resque/log_formatters/very_verbose_formatter'
 require 'resque/job'
 require 'resque/worker'
 require 'resque/plugin'
+require 'resque/data_store'
+require 'resque/thread_signal'
 
 require 'resque/vendor/utf8_util'
 
+require 'resque/railtie' if defined?(Rails::Railtie)
+
 module Resque
   include Helpers
   extend self
 
+  # Given a Ruby object, returns a string suitable for storage in a
+  # queue.
+  def encode(object)
+    if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+      MultiJson.dump object
+    else
+      MultiJson.encode object
+    end
+  end
+
+  # Given a string, returns a Ruby object.
+  def decode(object)
+    return unless object
+
+    begin
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.load object
+      else
+        MultiJson.decode object
+      end
+    rescue ::MultiJson::DecodeError => e
+      raise Helpers::DecodeException, e.message, e.backtrace
+    end
+  end
+
+  # Given a word with dashes, returns a camel cased version of it.
+  #
+  # classify('job-name') # => 'JobName'
+  def classify(dashed_word)
+    dashed_word.split('-').map(&:capitalize).join
+  end
+
+  # Tries to find a constant with the name specified in the argument string:
+  #
+  # constantize("Module") # => Module
+  # constantize("Test::Unit") # => Test::Unit
+  #
+  # The name is assumed to be the one of a top-level constant, no matter
+  # whether it starts with "::" or not. No lexical context is taken into
+  # account:
+  #
+  # C = 'outside'
+  # module M
+  #   C = 'inside'
+  #   C # => 'inside'
+  #   constantize("C") # => 'outside', same as ::C
+  # end
+  #
+  # NameError is raised when the constant is unknown.
+  def constantize(camel_cased_word)
+    camel_cased_word = camel_cased_word.to_s
+
+    if camel_cased_word.include?('-')
+      camel_cased_word = classify(camel_cased_word)
+    end
+
+    names = camel_cased_word.split('::')
+    names.shift if names.empty? || names.first.empty?
+
+    constant = Object
+    names.each do |name|
+      args = Module.method(:const_get).arity != 1 ? [false] : []
+
+      if constant.const_defined?(name, *args)
+        constant = constant.const_get(name)
+      else
+        constant = constant.const_missing(name)
+      end
+    end
+    constant
+  end
+
+  extend ::Forwardable
+
   # Accepts:
   #   1. A 'hostname:port' String
   #   2. A 'hostname:port:db' String (to select the Redis db)
@@ -26,43 +110,113 @@ module Resque
   #   4. A Redis URL String 'redis://host:port'
   #   5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
   #      or `Redis::Namespace`.
+  #   6. An Hash of a redis connection {:host => 'localhost', :port => 6379, :db => 0}
   def redis=(server)
     case server
     when String
-      if server =~ /redis\:\/\//
-        redis = Redis.connect(:url => server, :thread_safe => true)
+      if server =~ /rediss?\:\/\//
+        redis = Redis.new(:url => server)
       else
         server, namespace = server.split('/', 2)
         host, port, db = server.split(':')
-        redis = Redis.new(:host => host, :port => port,
-          :thread_safe => true, :db => db)
+        redis = Redis.new(:host => host, :port => port, :db => db)
       end
       namespace ||= :resque
 
-      @redis = Redis::Namespace.new(namespace, :redis => redis)
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(namespace, :redis => redis))
     when Redis::Namespace
-      @redis = server
+      @data_store = Resque::DataStore.new(server)
+    when Resque::DataStore
+      @data_store = server
+    when Hash
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(:resque, :redis => Redis.new(server)))
     else
-      @redis = Redis::Namespace.new(:resque, :redis => server)
+      @data_store = Resque::DataStore.new(Redis::Namespace.new(:resque, :redis => server))
     end
   end
 
   # Returns the current Redis connection. If none has been created, will
   # create a new one.
   def redis
-    return @redis if @redis
+    return @data_store if @data_store
     self.redis = Redis.respond_to?(:connect) ? Redis.connect : "localhost:6379"
     self.redis
   end
+  alias :data_store :redis
 
   def redis_id
-    # support 1.x versions of redis-rb
-    if redis.respond_to?(:server)
-      redis.server
-    elsif redis.respond_to?(:nodes) # distributed
-      redis.nodes.map { |n| n.id }.join(', ')
+    data_store.identifier
+  end
+
+  # Set the data store for the processed and failed statistics.
+  #
+  # By default it uses the same as `Resque.redis`, but different stores can be used.
+  #
+  # A custom store needs to obey the following API to work correctly
+  #
+  # class NullDataStore
+  #   # Returns the current value for the given stat.
+  #   def stat(stat)
+  #   end
+  #
+  #   # Increments the stat by the given value.
+  #   def increment_stat(stat, by)
+  #   end
+  #
+  #   # Decrements the stat by the given value.
+  #   def decrement_stat(stat, by)
+  #   end
+  #
+  #   # Clear the values for the given stat.
+  #   def clear_stat(stat)
+  #   end
+  # end
+  def stat_data_store=(stat_data_store)
+    Resque::Stat.data_store = stat_data_store
+  end
+
+  # Returns the data store for the statistics module.
+  def stat_data_store
+    Resque::Stat.data_store
+  end
+
+  # Set or retrieve the current logger object
+  attr_accessor :logger
+
+  DEFAULT_HEARTBEAT_INTERVAL = 60
+  DEFAULT_PRUNE_INTERVAL = DEFAULT_HEARTBEAT_INTERVAL * 5
+
+  # Defines how often a Resque worker updates the heartbeat key. Must be less
+  # than the prune interval.
+  attr_writer :heartbeat_interval
+  def heartbeat_interval
+    if defined? @heartbeat_interval
+      @heartbeat_interval
     else
-      redis.client.id
+      DEFAULT_HEARTBEAT_INTERVAL
+    end
+  end
+
+  # Defines how often Resque checks for dead workers.
+  attr_writer :prune_interval
+  def prune_interval
+    if defined? @prune_interval
+      @prune_interval
+    else
+      DEFAULT_PRUNE_INTERVAL
+    end
+  end
+
+  # By default, jobs are pushed to the back of the queue and popped from
+  # the front, resulting in "first in, first out" (FIFO) execution order.
+  # Set to true to push jobs to the front of the queue instead, resulting
+  # in "last in, first out" (LIFO) execution order.
+  attr_writer :enqueue_front
+  def enqueue_front
+    if defined? @enqueue_front
+      @enqueue_front
+    else
+      @enqueue_front = false
     end
   end
 
@@ -71,41 +225,96 @@ module Resque
   # changes you make will be permanent for the lifespan of the
   # worker.
   #
-  # Call with a block to set the hook.
-  # Call with no arguments to return the hook.
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
   def before_first_fork(&block)
-    block ? (@before_first_fork = block) : @before_first_fork
+    block ? register_hook(:before_first_fork, block) : hooks(:before_first_fork)
   end
 
-  # Set a proc that will be called in the parent process before the
-  # worker forks for the first time.
-  attr_writer :before_first_fork
+  # Register a before_first_fork proc.
+  def before_first_fork=(block)
+    register_hook(:before_first_fork, block)
+  end
 
   # The `before_fork` hook will be run in the **parent** process
   # before every job, so be careful- any changes you make will be
   # permanent for the lifespan of the worker.
   #
-  # Call with a block to set the hook.
-  # Call with no arguments to return the hook.
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
   def before_fork(&block)
-    block ? (@before_fork = block) : @before_fork
+    block ? register_hook(:before_fork, block) : hooks(:before_fork)
   end
 
-  # Set the before_fork proc.
-  attr_writer :before_fork
+  # Register a before_fork proc.
+  def before_fork=(block)
+    register_hook(:before_fork, block)
+  end
 
   # The `after_fork` hook will be run in the child process and is passed
   # the current job. Any changes you make, therefore, will only live as
   # long as the job currently being processed.
   #
-  # Call with a block to set the hook.
-  # Call with no arguments to return the hook.
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
   def after_fork(&block)
-    block ? (@after_fork = block) : @after_fork
+    block ? register_hook(:after_fork, block) : hooks(:after_fork)
+  end
+
+  # Register an after_fork proc.
+  def after_fork=(block)
+    register_hook(:after_fork, block)
+  end
+
+  # The `before_pause` hook will be run in the parent process before the
+  # worker has paused processing (via #pause_processing or SIGUSR2).
+  def before_pause(&block)
+    block ? register_hook(:before_pause, block) : hooks(:before_pause)
+  end
+
+  # Register a before_pause proc.
+  def before_pause=(block)
+    register_hook(:before_pause, block)
+  end
+
+  # The `after_pause` hook will be run in the parent process after the
+  # worker has paused (via SIGCONT).
+  def after_pause(&block)
+    block ? register_hook(:after_pause, block) : hooks(:after_pause)
+  end
+
+  # Register an after_pause proc.
+  def after_pause=(block)
+    register_hook(:after_pause, block)
+  end
+
+  # The `queue_empty` hook will be run in the **parent** process when
+  # the worker finds no more jobs in the queue and becomes idle.
+  #
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
+  def queue_empty(&block)
+    block ? register_hook(:queue_empty, block) : hooks(:queue_empty)
+  end
+
+  # Register a queue_empty proc.
+  def queue_empty=(block)
+    register_hook(:queue_empty, block)
   end
 
-  # Set the after_fork proc.
-  attr_writer :after_fork
+  # The `worker_exit` hook will be run in the **parent** process
+  # after the worker has existed (via SIGQUIT, SIGTERM, SIGINT, etc.).
+  #
+  # Call with a block to register a hook.
+  # Call with no arguments to return all registered hooks.
+  def worker_exit(&block)
+    block ? register_hook(:worker_exit, block) : hooks(:worker_exit)
+  end
+
+  # Register a worker_exit proc.
+  def worker_exit=(block)
+    register_hook(:worker_exit, block)
+  end
 
   def to_s
     "Resque Client connected to #{redis_id}"
@@ -138,25 +347,24 @@ module Resque
   #
   # Returns nothing
   def push(queue, item)
-    watch_queue(queue)
-    redis.rpush "queue:#{queue}", encode(item)
+    data_store.push_to_queue(queue,encode(item))
   end
 
   # Pops a job off a queue. Queue name should be a string.
   #
   # Returns a Ruby object.
   def pop(queue)
-    decode redis.lpop("queue:#{queue}")
+    decode(data_store.pop_from_queue(queue))
   end
 
   # Returns an integer representing the size of a queue.
   # Queue name should be a string.
   def size(queue)
-    redis.llen("queue:#{queue}").to_i
+    data_store.queue_size(queue)
   end
 
-  # Returns an array of items currently queued. Queue name should be
-  # a string.
+  # Returns an array of items currently queued, or the item itself
+  # if count = 1. Queue name should be a string.
   #
   # start and count should be integer and can be used for pagination.
   # start is the item to begin, count is how many items to return.
@@ -164,36 +372,39 @@ module Resque
   # To get the 3rd page of a 30 item, paginatied list one would use:
   #   Resque.peek('my_list', 59, 30)
   def peek(queue, start = 0, count = 1)
-    list_range("queue:#{queue}", start, count)
+    results = data_store.peek_in_queue(queue,start,count)
+    if count == 1
+      decode(results)
+    else
+      results.map { |result| decode(result) }
+    end
   end
 
   # Does the dirty work of fetching a range of items from a Redis list
   # and converting them into Ruby objects.
   def list_range(key, start = 0, count = 1)
+    results = data_store.list_range(key, start, count)
     if count == 1
-      decode redis.lindex(key, start)
+      decode(results)
     else
-      Array(redis.lrange(key, start, start+count-1)).map do |item|
-        decode item
-      end
+      results.map { |result| decode(result) }
     end
   end
 
   # Returns an array of all known Resque queues as strings.
   def queues
-    Array(redis.smembers(:queues))
+    data_store.queue_names
   end
 
   # Given a queue name, completely deletes the queue.
   def remove_queue(queue)
-    redis.srem(:queues, queue.to_s)
-    redis.del("queue:#{queue}")
+    data_store.remove_queue(queue)
   end
 
   # Used internally to keep track of which queues we've created.
   # Don't call this directly.
   def watch_queue(queue)
-    redis.sadd(:queues, queue.to_s)
+    data_store.watch_queue(queue)
   end
 
 
@@ -280,17 +491,19 @@ module Resque
     end
     return if before_hooks.any? { |result| result == false }
 
-    Job.destroy(queue_from_class(klass), klass, *args)
+    destroyed = Job.destroy(queue_from_class(klass), klass, *args)
 
     Plugin.after_dequeue_hooks(klass).each do |hook|
       klass.send(hook, *args)
     end
+
+    destroyed
   end
 
   # Given a class, try to extrapolate an appropriate queue based on a
   # class instance variable or `queue` method.
   def queue_from_class(klass)
-    klass.instance_variable_get(:@queue) ||
+    (klass.instance_variable_defined?(:@queue) && klass.instance_variable_get(:@queue)) ||
       (klass.respond_to?(:queue) and klass.queue)
   end
 
@@ -312,7 +525,7 @@ module Resque
     queue ||= queue_from_class(klass)
 
     if !queue
-      raise NoQueueError.new("Jobs must be placed onto a queue.")
+      raise NoQueueError.new("Jobs must be placed onto a queue. No queue could be inferred for class #{klass}")
     end
 
     if klass.to_s.empty?
@@ -349,12 +562,12 @@ module Resque
   # Returns a hash, similar to redis-rb's #info, of interesting stats.
   def info
     return {
-      :pending   => queues.inject(0) { |m,k| m + size(k) },
+      :pending   => queue_sizes.inject(0) { |sum, (_queue_name, queue_size)| sum + queue_size },
       :processed => Stat[:processed],
       :queues    => queues.size,
       :workers   => workers.size.to_i,
       :working   => working.size,
-      :failed    => Stat[:failed],
+      :failed    => data_store.num_failed,
       :servers   => [redis_id],
       :environment  => ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     }
@@ -363,9 +576,76 @@ module Resque
   # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
   # is O(N) for the keyspace, so be careful - this can be slow for big databases.
   def keys
-    redis.keys("*").map do |key|
-      key.sub("#{redis.namespace}:", '')
+    data_store.all_resque_keys
+  end
+
+  # Returns a hash, mapping queue names to queue sizes
+  def queue_sizes
+    queue_names = queues
+
+    sizes = redis.pipelined do |piped|
+      queue_names.each do |name|
+        piped.llen("queue:#{name}")
+      end
     end
+
+    Hash[queue_names.zip(sizes)]
+  end
+
+  # Returns a hash, mapping queue names to (up to `sample_size`) samples of jobs in that queue
+  def sample_queues(sample_size = 1000)
+    queue_names = queues
+
+    samples = redis.pipelined do |piped|
+      queue_names.each do |name|
+        key = "queue:#{name}"
+        piped.llen(key)
+        piped.lrange(key, 0, sample_size - 1)
+      end
+    end
+
+    hash = {}
+
+    queue_names.zip(samples.each_slice(2).to_a) do |queue_name, (queue_size, serialized_samples)|
+      samples = serialized_samples.map do |serialized_sample|
+        Job.decode(serialized_sample)
+      end
+
+      hash[queue_name] = {
+        :size => queue_size,
+        :samples => samples
+      }
+    end
+
+    hash
+  end
+
+  private
+
+  @hooks = Hash.new { |h, k| h[k] = [] }
+
+  # Register a new proc as a hook. If the block is nil this is the
+  # equivalent of removing all hooks of the given name.
+  #
+  # `name` is the hook that the block should be registered with.
+  def register_hook(name, block)
+    return clear_hooks(name) if block.nil?
+
+    block = Array(block)
+    @hooks[name].concat(block)
+  end
+
+  # Clear all hooks given a hook name.
+  def clear_hooks(name)
+    @hooks[name] = []
+  end
+
+  # Retrieve all hooks of a given name.
+  def hooks(name)
+    @hooks[name]
   end
 end
 
+# Log to STDOUT by default
+Resque.logger           = MonoLogger.new(STDOUT)
+Resque.logger.formatter = Resque::QuietFormatter.new

data/lib/tasks/redis.rake

@@ -17,7 +17,7 @@ class RedisRunner
   end
 
   def self.config
-    @config ||= if File.exists?(redis_dir + 'etc/redis.conf')
+    @config ||= if File.exist?(redis_dir + 'etc/redis.conf')
                   redis_dir + 'etc/redis.conf'
                 else
                   redis_dir + '../etc/redis.conf'
@@ -30,7 +30,7 @@ class RedisRunner
 
   # Just check for existance of dtach socket
   def self.running?
-    File.exists? dtach_socket
+    File.exist? dtach_socket
   end
 
   def self.start
@@ -79,7 +79,7 @@ namespace :redis do
   end
 
   desc <<-DOC
-  Install the latest verison of Redis from Github (requires git, duh).
+  Install the latest version of Redis from Github (requires git, duh).
     Use INSTALL_DIR env var like "rake redis:install INSTALL_DIR=~/tmp"
     in order to get an alternate location for your install files.
   DOC
@@ -90,10 +90,10 @@ namespace :redis do
 
     if ENV['PREFIX']
       bin_dir = "#{ENV['PREFIX']}/bin"
-      sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
+      sh "mkdir -p #{bin_dir}" unless File.exist?("#{bin_dir}")
 
       conf_dir = "#{ENV['PREFIX']}/etc"
-      sh "mkdir -p #{conf_dir}" unless File.exists?("#{conf_dir}")
+      sh "mkdir -p #{conf_dir}" unless File.exist?("#{conf_dir}")
     end
 
     %w(redis-benchmark redis-cli redis-server).each do |bin|
@@ -102,7 +102,7 @@ namespace :redis do
 
     puts "Installed redis-benchmark, redis-cli and redis-server to #{bin_dir}"
 
-    unless File.exists?("#{conf_dir}/redis.conf")
+    unless File.exist?("#{conf_dir}/redis.conf")
       sh "cp #{INSTALL_DIR}/redis.conf #{conf_dir}/redis.conf"
       puts "Installed redis.conf to #{conf_dir} \n You should look at this file!"
     end
@@ -115,9 +115,9 @@ namespace :redis do
 
   desc "Download package"
   task :download do
-    sh "rm -rf #{INSTALL_DIR}/" if File.exists?("#{INSTALL_DIR}/.svn")
-    sh "git clone git://github.com/antirez/redis.git #{INSTALL_DIR}" unless File.exists?(INSTALL_DIR)
-    sh "cd #{INSTALL_DIR} && git pull" if File.exists?("#{INSTALL_DIR}/.git")
+    sh "rm -rf #{INSTALL_DIR}/" if File.exist?("#{INSTALL_DIR}/.svn")
+    sh "git clone git://github.com/antirez/redis.git #{INSTALL_DIR}" unless File.exist?(INSTALL_DIR)
+    sh "cd #{INSTALL_DIR} && git pull" if File.exist?("#{INSTALL_DIR}/.git")
   end
 end
 
@@ -135,7 +135,7 @@ namespace :dtach do
 
     if ENV['PREFIX']
       bin_dir = "#{ENV['PREFIX']}/bin"
-      sh "mkdir -p #{bin_dir}" unless File.exists?("#{bin_dir}")
+      sh "mkdir -p #{bin_dir}" unless File.exist?("#{bin_dir}")
     end
 
     sh "cp #{INSTALL_DIR}/dtach-0.8/dtach #{bin_dir}"
@@ -147,7 +147,7 @@ namespace :dtach do
 
   desc "Download package"
   task :download do
-    unless File.exists?("#{INSTALL_DIR}/dtach-0.8.tar.gz")
+    unless File.exist?("#{INSTALL_DIR}/dtach-0.8.tar.gz")
       require 'net/http'
 
       url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'