jerefrer-resque 1.1.0

data/lib/resque/worker.rb

@@ -0,0 +1,406 @@
+module Resque
+  # A Resque Worker processes jobs. On platforms that support fork(2),
+  # the worker will fork off a child to process each job. This ensures
+  # a clean slate when beginning the next job and cuts down on gradual
+  # memory growth as well as low level failures.
+  #
+  # It also ensures workers are always listening to signals from you,
+  # their master, and can react accordingly.
+  class Worker
+    include Resque::Helpers
+    extend Resque::Helpers
+
+    # Whether the worker should log basic info to STDOUT
+    attr_accessor :verbose
+
+    # Whether the worker should log lots of info to STDOUT
+    attr_accessor  :very_verbose
+
+    # Boolean indicating whether this worker can or can not fork.
+    # Automatically set if a fork(2) fails.
+    attr_accessor :cant_fork
+
+    attr_writer :to_s
+
+    # Returns an array of all worker objects.
+    def self.all
+      redis.smembers(:workers).map { |id| find(id) }
+    end
+
+    # Returns an array of all worker objects currently processing
+    # jobs.
+    def self.working
+      names = all
+      return [] unless names.any?
+      names.map! { |name| "worker:#{name}" }
+      redis.mapped_mget(*names).keys.map do |key|
+        find key.sub("worker:", '')
+      end
+    end
+
+    # Returns a single worker object. Accepts a string id.
+    def self.find(worker_id)
+      if exists? worker_id
+        queues = worker_id.split(':')[-1].split(',')
+        worker = new(*queues)
+        worker.to_s = worker_id
+        worker
+      else
+        nil
+      end
+    end
+
+    # Alias of `find`
+    def self.attach(worker_id)
+      find(worker_id)
+    end
+
+    # Given a string worker id, return a boolean indicating whether the
+    # worker exists
+    def self.exists?(worker_id)
+      redis.sismember(:workers, worker_id)
+    end
+
+    # Workers should be initialized with an array of string queue
+    # names. The order is important: a Worker will check the first
+    # queue given for a job. If none is found, it will check the
+    # second queue name given. If a job is found, it will be
+    # processed. Upon completion, the Worker will again check the
+    # first queue given, and so forth. In this way the queue list
+    # passed to a Worker on startup defines the priorities of queues.
+    #
+    # If passed a single "*", this Worker will operate on all queues
+    # in alphabetical order. Queues can be dynamically added or
+    # removed without needing to restart workers using this method.
+    def initialize(*queues)
+      @queues = queues
+      validate_queues
+    end
+
+    # A worker must be given a queue, otherwise it won't know what to
+    # do with itself.
+    #
+    # You probably never need to call this.
+    def validate_queues
+      if @queues.nil? || @queues.empty?
+        raise NoQueueError.new("Please give each worker at least one queue.")
+      end
+    end
+
+    # This is the main workhorse method. Called on a Worker instance,
+    # it begins the worker life cycle.
+    #
+    # The following events occur during a worker's life cycle:
+    #
+    # 1. startup: Signals are registered, dead workers are pruned,
+    #             and this worker is registered.
+    # 2. work loop: Jobs are pulled from a queue and processed
+    # 3. teardown: This worker is unregistered.
+    #
+    # Can be passed an integered representing the polling
+    # frequency. The default is 5 seconds, but for a semi-active site
+    # you may want to use a smaller value.
+    #
+    # Also accepts a block which will be passed the job as soon as it
+    # has completed processing. Useful for testing.
+    def work(interval = 5, &block)
+      $0 = "resque: Starting"
+      startup
+
+      loop do
+        break if @shutdown
+
+        if job = reserve
+          log "got: #{job.inspect}"
+
+          if @child = fork
+            procline = "resque: Forked #{@child} at #{Time.now.to_i}"
+            $0 = procline
+            log! procline
+            Process.wait
+          else
+            procline = "resque: Processing #{job.queue} since #{Time.now.to_i}"
+            $0 = procline
+            log! procline
+            process(job, &block)
+            exit! unless @cant_fork
+          end
+
+          @child = nil
+        else
+          break if interval.to_i == 0
+          log! "Sleeping for #{interval.to_i}"
+          $0 = "resque: Waiting for #{@queues.join(',')}"
+          sleep interval.to_i
+        end
+      end
+
+    ensure
+      unregister_worker
+    end
+
+    # Processes a single job. If none is given, it will try to produce
+    # one.
+    def process(job = nil)
+      return unless job ||= reserve
+
+      begin
+        working_on job
+        job.perform
+      rescue Object => e
+        log "#{job.inspect} failed: #{e.inspect}"
+        job.fail(e)
+        failed!
+      else
+        log "done: #{job.inspect}"
+      ensure
+        yield job if block_given?
+        done_working
+      end
+    end
+
+    # Attempts to grab a job off one of the provided queues. Returns
+    # nil if no job can be found.
+    def reserve
+      queues.each do |queue|
+        log! "Checking #{queue}"
+        if job = Resque::Job.reserve(queue)
+          log! "Found job on #{queue}"
+          return job
+        end
+      end
+
+      nil
+    end
+
+    # Returns a list of queues to use when searching for a job.
+    # A splat ("*") means you want every queue (in alpha order) - this
+    # can be useful for dynamically adding new queues.
+    def queues
+      @queues[0] == "*" ? Resque.queues.sort : @queues
+    end
+
+    # Not every platform supports fork. Here we do our magic to
+    # determine if yours does.
+    def fork
+      @cant_fork = true if $TESTING
+
+      return if @cant_fork
+
+      begin
+        Kernel.fork
+      rescue NotImplementedError
+        @cant_fork = true
+        nil
+      end
+    end
+
+    # Runs all the methods needed when a worker begins its lifecycle.
+    def startup
+      enable_gc_optimizations
+      register_signal_handlers
+      prune_dead_workers
+      register_worker
+    end
+
+    # Enables GC Optimizations if you're running REE.
+    # http://www.rubyenterpriseedition.com/faq.html#adapt_apps_for_cow
+    def enable_gc_optimizations
+      if GC.respond_to?(:copy_on_write_friendly=)
+        GC.copy_on_write_friendly = true
+      end
+    end
+
+    # Registers the various signal handlers a worker responds to.
+    #
+    # TERM: Shutdown immediately, stop processing jobs.
+    #  INT: Shutdown immediately, stop processing jobs.
+    # QUIT: Shutdown after the current job has finished processing.
+    # USR1: Kill the forked child immediately, continue processing jobs.
+    def register_signal_handlers
+      trap('TERM') { shutdown!  }
+      trap('INT')  { shutdown!  }
+      unless defined? JRUBY_VERSION
+        trap('QUIT') { shutdown   }
+        trap('USR1') { kill_child }
+      end
+      log! "Registered signals"
+    end
+
+    # Schedule this worker for shutdown. Will finish processing the
+    # current job.
+    def shutdown
+      log 'Exiting...'
+      @shutdown = true
+    end
+
+    # Kill the child and shutdown immediately.
+    def shutdown!
+      shutdown
+      kill_child
+    end
+
+    # Kills the forked child immediately, without remorse. The job it
+    # is processing will not be completed.
+    def kill_child
+      if @child
+        log! "Killing child at #{@child}"
+        Process.kill("KILL", @child) rescue nil
+      end
+    end
+
+    # Looks for any workers which should be running on this server
+    # and, if they're not, removes them from Redis.
+    #
+    # This is a form of garbage collection. If a server is killed by a
+    # hard shutdown, power failure, or something else beyond our
+    # control, the Resque workers will not die gracefully and therefor
+    # will leave stale state information in Redis.
+    #
+    # By checking the current Redis state against the actual
+    # environment, we can determine if Redis is old and clean it up a bit.
+    def prune_dead_workers
+      Worker.all.each do |worker|
+        host, pid, queues = worker.id.split(':')
+        next unless host == hostname
+        next if worker_pids.include?(pid)
+        log! "Pruning dead worker: #{worker}"
+        worker.unregister_worker
+      end
+    end
+
+    # Registers ourself as a worker. Useful when entering the worker
+    # lifecycle on startup.
+    def register_worker
+      redis.sadd(:workers, self)
+      started!
+    end
+
+    # Unregisters ourself as a worker. Useful when shutting down.
+    def unregister_worker
+      done_working
+
+      redis.srem(:workers, self)
+      redis.del("worker:#{self}:started")
+
+      Stat.clear("processed:#{self}")
+      Stat.clear("failed:#{self}")
+    end
+
+    # Given a job, tells Redis we're working on it. Useful for seeing
+    # what workers are doing and when.
+    def working_on(job)
+      job.worker = self
+      data = encode \
+        :queue   => job.queue,
+        :run_at  => Time.now.to_s,
+        :payload => job.payload
+      redis.set("worker:#{self}", data)
+    end
+
+    # Called when we are done working - clears our `working_on` state
+    # and tells Redis we processed a job.
+    def done_working
+      processed!
+      redis.del("worker:#{self}")
+    end
+
+    # How many jobs has this worker processed? Returns an int.
+    def processed
+      Stat["processed:#{self}"]
+    end
+
+    # Tell Redis we've processed a job.
+    def processed!
+      Stat << "processed"
+      Stat << "processed:#{self}"
+    end
+
+    # How many failed jobs has this worker seen? Returns an int.
+    def failed
+      Stat["failed:#{self}"]
+    end
+
+    # Tells Redis we've failed a job.
+    def failed!
+      Stat << "failed"
+      Stat << "failed:#{self}"
+    end
+
+    # What time did this worker start? Returns an instance of `Time`
+    def started
+      redis.get "worker:#{self}:started"
+    end
+
+    # Tell Redis we've started
+    def started!
+      redis.set("worker:#{self}:started", Time.now.to_s)
+    end
+
+    # Returns a hash explaining the Job we're currently processing, if any.
+    def job
+      decode(redis.get("worker:#{self}")) || {}
+    end
+    alias_method :processing, :job
+
+    # Boolean - true if working, false if not
+    def working?
+      state == :working
+    end
+
+    # Boolean - true if idle, false if not
+    def idle?
+      state == :idle
+    end
+
+    # Returns a symbol representing the current worker state,
+    # which can be either :working or :idle
+    def state
+      redis.exists("worker:#{self}") ? :working : :idle
+    end
+
+    # Is this worker the same as another worker?
+    def ==(other)
+      to_s == other.to_s
+    end
+
+    def inspect
+      "#<Worker #{to_s}>"
+    end
+
+    # The string representation is the same as the id for this worker
+    # instance. Can be used with `Worker.find`.
+    def to_s
+      @to_s ||= "#{hostname}:#{Process.pid}:#{@queues.join(',')}"
+    end
+    alias_method :id, :to_s
+
+    # chomp'd hostname of this machine
+    def hostname
+      @hostname ||= `hostname`.chomp
+    end
+
+    # Returns an array of string pids of all the other workers on this
+    # machine. Useful when pruning dead workers on startup.
+    def worker_pids
+      `ps -A -o pid,command | grep [r]esque`.split("\n").map do |line|
+        line.split(' ')[0]
+      end
+    end
+
+    # Log a message to STDOUT if we are verbose or very_verbose.
+    def log(message)
+      if verbose
+        puts "*** #{message}"
+      elsif very_verbose
+        time = Time.now.strftime('%I:%M:%S %Y-%m-%d')
+        puts "** [#{time}] #$$: #{message}"
+      end
+    end
+
+    # Logs a very verbose message to STDOUT.
+    def log!(message)
+      log message if very_verbose
+    end
+  end
+end

data/lib/resque.rb

@@ -0,0 +1,184 @@
+require 'redis/namespace'
+
+begin
+  require 'yajl'
+rescue LoadError
+  require 'json'
+end
+
+require 'resque/errors'
+
+require 'resque/failure'
+require 'resque/failure/base'
+
+require 'resque/helpers'
+require 'resque/stat'
+require 'resque/job'
+require 'resque/worker'
+
+module Resque
+  include Helpers
+  extend self
+
+  # Accepts a 'hostname:port' string or a Redis server.
+  def redis=(server)
+    case server
+    when String
+      host, port = server.split(':')
+      redis = Redis.new(:host => host, :port => port, :thread_safe => true)
+      @redis = Redis::Namespace.new(:resque, :redis => redis)
+    when Redis
+      @redis = Redis::Namespace.new(:resque, :redis => server)
+    else
+      raise "I don't know what to do with #{server.inspect}"
+    end
+  end
+
+  # Returns the current Redis connection. If none has been created, will
+  # create a new one.
+  def redis
+    return @redis if @redis
+    self.redis = 'localhost:6379'
+    self.redis
+  end
+
+  def to_s
+    "Resque Client connected to #{redis.server}"
+  end
+
+
+  #
+  # queue manipulation
+  #
+
+  # Pushes a job onto a queue. Queue name should be a string and the
+  # item should be any JSON-able Ruby object.
+  def push(queue, item)
+    watch_queue(queue)
+    redis.rpush "queue:#{queue}", encode(item)
+  end
+
+  # Pops a job off a queue. Queue name should be a string.
+  #
+  # Returns a Ruby object.
+  def pop(queue)
+    decode redis.lpop("queue:#{queue}")
+  end
+
+  # Returns an int representing the size of a queue.
+  # Queue name should be a string.
+  def size(queue)
+    redis.llen("queue:#{queue}").to_i
+  end
+
+  # Returns an array of items currently queued. Queue name should be
+  # a string.
+  #
+  # start and count should be integer and can be used for pagination.
+  # start is the item to begin, count is how many items to return.
+  #
+  # To get the 3rd page of a 30 item, paginatied list one would use:
+  #   Resque.peek('my_list', 59, 30)
+  def peek(queue, start = 0, count = 1)
+    list_range("queue:#{queue}", start, count)
+  end
+
+  # Does the dirty work of fetching a range of items from a Redis list
+  # and converting them into Ruby objects.
+  def list_range(key, start = 0, count = 1)
+    if count == 1
+      decode redis.lindex(key, start)
+    else
+      Array(redis.lrange(key, start, start+count-1)).map do |item|
+        decode item
+      end
+    end
+  end
+
+  # Returns an array of all known Resque queues as strings.
+  def queues
+    redis.smembers(:queues)
+  end
+
+  # Used internally to keep track of which queues we've created.
+  # Don't call this directly.
+  def watch_queue(queue)
+    @watched_queues ||= {}
+    return if @watched_queues[queue]
+    redis.sadd(:queues, queue.to_s)
+  end
+
+
+  #
+  # job shortcuts
+  #
+
+  # This method can be used to conveniently add a job to a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will return a non-true value.
+  #
+  # This method is considered part of the `stable` API.
+  def enqueue(klass, *args)
+    queue = klass.instance_variable_get(:@queue)
+    queue ||= klass.queue if klass.respond_to?(:queue)
+    Job.create(queue, klass, *args)
+  end
+
+  # This method will return a `Resque::Job` object or a non-true value
+  # depending on whether a job can be obtained. You should pass it the
+  # precise name of a queue: case matters.
+  #
+  # This method is considered part of the `stable` API.
+  def reserve(queue)
+    Job.reserve(queue)
+  end
+
+
+  #
+  # worker shortcuts
+  #
+
+  # A shortcut to Worker.all
+  def workers
+    Worker.all
+  end
+
+  # A shortcut to Worker.working
+  def working
+    Worker.working
+  end
+
+
+  #
+  # stats
+  #
+
+  # Returns a hash, similar to redis-rb's #info, of interesting stats.
+  def info
+    return {
+      :pending   => queues.inject(0) { |m,k| m + size(k) },
+      :processed => Stat[:processed],
+      :queues    => queues.size,
+      :workers   => workers.size.to_i,
+      :working   => working.size,
+      :failed    => Stat[:failed],
+      :servers   => [redis.server]
+    }
+  end
+
+  # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+  # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+  def keys
+    redis.keys("*").map do |key|
+      key.sub('resque:', '')
+    end
+  end
+end

data/tasks/redis.rake

@@ -0,0 +1,125 @@
+# Inspired by rabbitmq.rake the Redbox project at http://github.com/rick/redbox/tree/master
+require 'fileutils'
+require 'open-uri'
+
+class RedisRunner
+
+  def self.redisdir
+    "/tmp/redis/"
+  end
+
+  def self.redisconfdir
+    '/etc/redis.conf'
+  end
+
+  def self.dtach_socket
+    '/tmp/redis.dtach'
+  end
+
+  # Just check for existance of dtach socket
+  def self.running?
+    File.exists? dtach_socket
+  end
+
+  def self.start
+    puts 'Detach with Ctrl+\  Re-attach with rake redis:attach'
+    sleep 1
+    exec "dtach -A #{dtach_socket} redis-server #{redisconfdir}"
+  end
+
+  def self.attach
+    exec "dtach -a #{dtach_socket}"
+  end
+
+  def self.stop
+    sh 'echo "SHUTDOWN" | nc localhost 6379'
+  end
+
+end
+
+namespace :redis do
+
+  desc 'About redis'
+  task :about do
+    puts "\nSee http://code.google.com/p/redis/ for information about redis.\n\n"
+  end
+
+  desc 'Start redis'
+  task :start do
+    RedisRunner.start
+  end
+
+  desc 'Stop redis'
+  task :stop do
+    RedisRunner.stop
+  end
+
+  desc 'Restart redis'
+  task :restart do
+    RedisRunner.stop
+    RedisRunner.start
+  end
+
+  desc 'Attach to redis dtach socket'
+  task :attach do
+    RedisRunner.attach
+  end
+
+  desc 'Install the lastest verison of Redis from Github (requires git, duh)'
+  task :install => [:about, :download, :make] do
+    %w(redis-benchmark redis-cli redis-server).each do |bin|
+      sh "sudo cp /tmp/redis/#{bin} /usr/bin/"
+    end
+
+    puts "Installed redis-benchmark, redis-cli and redis-server to /usr/bin/"
+
+    unless File.exists?('/etc/redis.conf')
+      sh 'sudo cp /tmp/redis/redis.conf /etc/'
+      puts "Installed redis.conf to /etc/ \n You should look at this file!"
+    end
+  end
+
+  task :make do
+    sh "cd #{RedisRunner.redisdir} && make clean"
+    sh "cd #{RedisRunner.redisdir} && make"
+  end
+
+  desc "Download package"
+  task :download do
+    sh 'rm -rf /tmp/redis/' if File.exists?("#{RedisRunner.redisdir}/.svn")
+    sh 'git clone git://github.com/antirez/redis.git /tmp/redis' unless File.exists?(RedisRunner.redisdir)
+    sh "cd #{RedisRunner.redisdir} && git pull" if File.exists?("#{RedisRunner.redisdir}/.git")
+  end
+
+end
+
+namespace :dtach do
+
+  desc 'About dtach'
+  task :about do
+    puts "\nSee http://dtach.sourceforge.net/ for information about dtach.\n\n"
+  end
+
+  desc 'Install dtach 0.8 from source'
+  task :install => [:about] do
+
+    Dir.chdir('/tmp/')
+    unless File.exists?('/tmp/dtach-0.8.tar.gz')
+      require 'net/http'
+
+      url = 'http://downloads.sourceforge.net/project/dtach/dtach/0.8/dtach-0.8.tar.gz'
+      open('/tmp/dtach-0.8.tar.gz', 'wb') do |file| file.write(open(url).read) end
+    end
+
+    unless File.directory?('/tmp/dtach-0.8')
+      system('tar xzf dtach-0.8.tar.gz')
+    end
+
+    Dir.chdir('/tmp/dtach-0.8/')
+    sh 'cd /tmp/dtach-0.8/ && ./configure && make'
+    sh 'sudo cp /tmp/dtach-0.8/dtach /usr/bin/'
+
+    puts 'Dtach successfully installed to /usr/bin.'
+  end
+end
+

data/tasks/resque.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'resque/tasks'