vitobotta-resque-pool 0.3.0.dev

data/features/step_definitions/resque-pool_steps.rb

@@ -0,0 +1,147 @@
+def process_should_exist(pid)
+  lambda { Process.kill(0, pid) }.should_not raise_error(Errno::ESRCH)
+end
+
+def process_should_not_exist(pid)
+  lambda { Process.kill(0, pid) }.should raise_error(Errno::ESRCH)
+end
+
+def grab_worker_pids(count, str)
+  announce "TODO: check output_or_log for #{count} worker started messages"
+  pid_regex = (1..count).map { '(\d+)' }.join ', '
+  full_regex = /resque-pool-manager\[\d+\]: Pool contains worker PIDs: \[#{pid_regex}\]/m
+  str.should =~ full_regex
+  @worker_pids = full_regex.match(str).captures.map {|pid| pid.to_i }
+end
+
+def output_or_logfiles_string(report_log)
+  case report_log
+  when "report", "output"
+    "output"
+  when "log", "logfiles"
+    "logfiles"
+  else
+    raise ArgumentError
+  end
+end
+
+def output_or_log(report_log)
+  case report_log
+  when "report", "output"
+    all_output
+  when "log", "logfiles"
+    in_current_dir do
+      File.read("log/resque-pool.stdout.log") << File.read("log/resque-pool.stderr.log")
+    end
+  else
+    raise ArgumentError
+  end
+end
+
+def children_of(ppid)
+  ps = `ps -eo ppid,pid,cmd | grep '^ *#{ppid} '`
+  ps.split(/\s*\n/).map do |line|
+    _, pid, cmd = line.strip.split(/\s+/, 3)
+    [pid, cmd]
+  end
+end
+
+When /^I run the pool manager as "([^"]*)"$/ do |cmd|
+  @pool_manager_process = run_background(unescape(cmd))
+end
+
+When /^I send the pool manager the "([^"]*)" signal$/ do |signal|
+  Process.kill signal, background_pid
+  output_logfiles = @pid_from_pidfile ? "logfiles" : "output"
+  case signal
+  when "QUIT"
+    keep_trying do
+      Then "the #{output_logfiles} should contain the following lines (with interpolated $PID):", <<-EOF
+resque-pool-manager[$PID]: QUIT: graceful shutdown, waiting for children
+      EOF
+    end
+  else
+    raise ArgumentError
+  end
+end
+
+Then /^the pool manager should record its pid in "([^"]*)"$/ do |pidfile|
+  in_current_dir do
+    keep_trying do
+      File.should be_file(pidfile)
+      @pid_from_pidfile = File.read(pidfile).to_i
+      @pid_from_pidfile.should_not == 0
+      process_should_exist(@pid_from_pidfile)
+    end
+  end
+end
+
+Then /^the pool manager should daemonize$/ do
+  @pool_manager_process.stop
+end
+
+Then /^the pool manager daemon should finish$/ do
+  keep_trying do
+    process_should_not_exist(@pid_from_pidfile)
+  end
+end
+
+# nomenclature: "report" => output to stdout/stderr
+#               "log"    => output to default logfile
+
+Then /^the pool manager should (report|log) that it has started up$/ do |report_log|
+  keep_trying do
+    Then "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
+resque-pool-manager[$PID]: Resque Pool running in development environment
+resque-pool-manager[$PID]: started manager
+    EOF
+  end
+end
+
+Then /^the pool manager should (report|log) that the pool is empty$/ do |report_log|
+  Then "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
+resque-pool-manager[$PID]: Pool is empty
+  EOF
+end
+
+Then /^the pool manager should (report|log) that (\d+) workers are in the pool$/ do |report_log, count|
+  grab_worker_pids Integer(count), output_or_log(report_log)
+end
+
+Then /^the resque workers should all shutdown$/ do
+  @worker_pids.each do |pid|
+    keep_trying do
+      process_should_not_exist(pid)
+    end
+  end
+end
+
+Then "the pool manager should have no child processes" do
+  children_of(background_pid).should have(:no).keys
+end
+
+Then /^the pool manager should have (\d+) "([^"]*)" worker child processes$/ do |count, queues|
+  children_of(background_pid).select do |pid, cmd|
+    cmd =~ /^resque-\d+.\d+.\d+: Waiting for #{queues}$/
+  end.should have(Integer(count)).members
+end
+
+Then "the pool manager should finish" do
+  # assuming there will not be multiple processes running
+  processes.each { |cmd, p| p.stop }
+end
+
+Then /^the pool manager should (report|log) that it is finished$/ do |report_log|
+  Then "the #{output_or_logfiles_string(report_log)} should contain the following lines (with interpolated $PID):", <<-EOF
+resque-pool-manager[$PID]: manager finished
+  EOF
+end
+
+Then /^the pool manager should (report|log) that a "([^"]*)" worker has been reaped$/ do |report_log, worker_type|
+  And 'the '+ output_or_logfiles_string(report_log) +' should match /Reaped resque worker\[\d+\] \(status: 0\) queues: '+ worker_type + '/'
+end
+
+Then /^the logfiles should match \/([^\/]*)\/$/ do |partial_output|
+  output_or_log("log").should =~ /#{partial_output}/
+end
+ 

data/features/support/aruba_daemon_support.rb

@@ -0,0 +1,60 @@
+require 'aruba/api'
+require 'aruba/process'
+
+module Aruba
+
+  module Api
+
+    # this is a horrible hack, to make sure that it's done what it needs to do
+    # before we do our next step
+    def keep_trying(timeout=10, tries=0)
+      announce "Try: #{tries}" if @announce_env
+      yield
+    rescue RSpec::Expectations::ExpectationNotMetError
+      if tries < timeout
+        sleep 1
+        tries += 1
+        retry
+      else
+        raise
+      end
+    end
+
+    def run_background(cmd)
+      @background = run(cmd)
+    end
+
+    def send_signal(cmd, signal)
+      announce_or_puts "$ kill -#{signal} #{processes[cmd].pid}" if @announce_env
+      processes[cmd].send_signal signal
+    end
+
+    def background_pid
+      @pid_from_pidfile || @background.pid
+    end
+
+    def interpolate_background_pid(string)
+      interpolated = string.gsub('$PID', background_pid.to_s)
+      announce_or_puts interpolated if @announce_env
+      interpolated
+    end
+
+    def kill_all_processes!
+      stop_processes!
+    rescue
+      processes.each {|cmd,process| send_signal(cmd, 'KILL') }
+      raise
+    end
+
+  end
+
+  class Process
+    def pid
+      @process.pid
+    end
+    def send_signal signal
+      @process.send :send_signal, signal
+    end
+  end
+
+end

data/features/support/env.rb

@@ -0,0 +1 @@
+require 'aruba/cucumber'

data/lib/resque/pool/cli.rb

@@ -0,0 +1,116 @@
+require 'trollop'
+require 'resque/pool'
+
+module Resque
+  class Pool
+    module CLI
+      extend Logging
+      extend self
+
+      def run
+        opts = parse_options
+        daemonize if opts[:daemon]
+        manage_pidfile opts[:pidfile]
+        redirect opts
+        setup_environment opts
+        start_pool
+      end
+
+      def parse_options
+        opts = Trollop::options do
+          version "resque-pool #{VERSION} (c) nicholas a. evans"
+          banner <<-EOS
+resque-pool is the best way to manage a group (pool) of resque workers
+
+When daemonized, stdout and stderr default to resque-pool.stdxxx.log files in
+the log directory and pidfile defaults to resque-pool.pid in the current dir.
+
+Usage:
+   resque-pool [options]
+where [options] are:
+          EOS
+          opt :config, "Alternate path to config file", :type => String, :short => "-c"
+          opt :daemon, "Run as a background daemon", :default => false,  :short => "-d"
+          opt :stdout, "Redirect stdout to logfile", :type => String,    :short => '-o'
+          opt :stderr, "Redirect stderr to logfile", :type => String,    :short => '-e'
+          opt :nosync, "Don't sync logfiles on every write"
+          opt :pidfile, "PID file location",         :type => String,    :short => "-p"
+          opt :environment, "Set RAILS_ENV/RACK_ENV/RESQUE_ENV", :type => String, :short => "-E"
+        end
+        if opts[:daemon]
+          opts[:stdout]  ||= "log/resque-pool.stdout.log"
+          opts[:stderr]  ||= "log/resque-pool.stderr.log"
+          opts[:pidfile] ||= "tmp/pids/resque-pool.pid"
+        end
+        opts
+      end
+
+      def daemonize
+        raise 'First fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+        Process.setsid
+        raise 'Second fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+      end
+
+      def manage_pidfile(pidfile)
+        return unless pidfile
+        pid = Process.pid
+        if File.exist? pidfile
+          if process_still_running? pidfile
+            raise "Pidfile already exists at #{pidfile} and process is still running."
+          else
+            File.delete pidfile
+          end
+        end
+        File.open pidfile, "w" do |f|
+          f.write pid
+        end
+        at_exit do
+          if Process.pid == pid
+            File.delete pidfile
+          end
+        end
+      end
+
+      def process_still_running?(pidfile)
+        old_pid = open(pidfile).read.strip.to_i
+        Process.kill 0, old_pid
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      rescue ::Exception
+        $stderr.puts "While checking if PID #{old_pid} is running, unexpected #{e.class}: #{e}"
+        true
+      end
+
+      def redirect(opts)
+        $stdin.reopen  '/dev/null'        if opts[:daemon]
+        # need to reopen as File, or else Resque::Pool::Logging.reopen_logs! won't work
+        out = File.new(opts[:stdout], "a") if opts[:stdout] && !opts[:stdout].empty?
+        err = File.new(opts[:stderr], "a") if opts[:stderr] && !opts[:stderr].empty?
+        $stdout.reopen out if out
+        $stderr.reopen err if err
+        $stdout.sync = $stderr.sync = true unless opts[:nosync]
+      end
+
+      def setup_environment(opts)
+        ENV["RACK_ENV"] = ENV["RAILS_ENV"] = ENV["RESQUE_ENV"] = opts[:environment] if opts[:environment]
+        log "Resque Pool running in #{ENV["RAILS_ENV"] || "development"} environment"
+        ENV["RESQUE_POOL_CONFIG"] = opts[:config] if opts[:config]
+      end
+
+      def start_pool
+        require 'rake'
+        require 'resque/pool/tasks'
+        Rake.application.init
+        Rake.application.load_rakefile
+        Rake.application["resque:pool"].invoke
+      end
+
+    end
+  end
+end
+

data/lib/resque/pool/logging.rb

@@ -0,0 +1,58 @@
+module Resque
+  class Pool
+    module Logging
+      extend self
+
+      # more than a little bit complicated...
+      # copied this from Unicorn.
+      def self.reopen_logs!
+        log "Flushing logs"
+        [$stdout, $stderr].each do |fd|
+          if fd.instance_of? File
+            # skip if the file is the exact same inode and device
+            orig_st = fd.stat
+            begin
+              cur_st = File.stat(fd.path)
+              next if orig_st.ino == cur_st.ino && orig_st.dev == cur_st.dev
+            rescue Errno::ENOENT
+            end
+            # match up the encoding
+            open_arg = 'a'
+            if fd.respond_to?(:external_encoding) && enc = fd.external_encoding
+              open_arg << ":#{enc.to_s}"
+              enc = fd.internal_encoding and open_arg << ":#{enc.to_s}"
+            end
+            # match up buffering (does reopen reset this?)
+            sync = fd.sync
+            # sync to disk
+            fd.fsync
+            # reopen, and set ruby buffering appropriately
+            fd.reopen fd.path, open_arg
+            fd.sync = sync
+            log "Reopened logfile: #{fd.path}"
+          end
+        end
+      end
+
+      # Given a string, sets the procline ($0)
+      # Procline is always in the format of:
+      #   resque-pool-master: STRING
+      def procline(string)
+        $0 = "resque-pool-master: #{string}"
+      end
+
+      # TODO: make this use an actual logger
+      def log(message)
+        puts "resque-pool-manager[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # TODO: make this use an actual logger
+      def log_worker(message)
+        puts "resque-pool-worker[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+    end
+  end
+end

data/lib/resque/pool/pooled_worker.rb

@@ -0,0 +1,21 @@
+class Resque::Pool
+
+  class PooledWorker < ::Resque::Worker
+
+    def initialize(*args)
+      @pool_master_pid = Process.pid
+      super
+    end
+
+    def pool_master_has_gone_away?
+      @pool_master_pid && @pool_master_pid != Process.ppid
+    end
+
+    # override +shutdown?+ method
+    def shutdown?
+      super || pool_master_has_gone_away?
+    end
+
+  end
+
+end

data/lib/resque/pool/tasks.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require 'resque/tasks'
+require 'resque/pool'
+
+namespace :resque do
+
+  # resque worker config (not pool related).  e.g. hoptoad, rails environment
+  task :setup
+
+  namespace :pool do
+     # resque pool config.  e.g. after_prefork connection handling
+    task :setup
+  end
+
+  desc "Launch a pool of resque workers"
+  task :pool => %w[resque:setup resque:pool:setup] do
+    Resque::Pool.run
+  end
+
+end

data/lib/resque/pool/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  class Pool
+    VERSION = "0.3.0.dev"
+  end
+end