heroku-resque-pool 0.0.0

data/lib/resque/pool/cli.rb

@@ -0,0 +1,184 @@
+require 'optparse'
+require 'resque/pool'
+require 'resque/pool/logging'
+require 'fileutils'
+
+module Resque
+  class Pool
+    module CLI
+      include Logging
+      extend  Logging
+      extend self
+
+      def run
+        opts = parse_options
+        obtain_shared_lock opts[:lock_file]
+        daemonize if opts[:daemon]
+        manage_pidfile opts[:pidfile]
+        redirect opts
+        setup_environment opts
+        set_pool_options opts
+        start_pool
+      end
+
+      def parse_options(argv=nil)
+        opts = {}
+        parser = OptionParser.new do |opt|
+          opt.banner = <<-EOS.gsub(/^            /, '')
+            resque-pool is the best way to manage a group (pool) of resque workers
+
+            When daemonized, stdout and stderr default to resque-pool.stdxxx.log files in
+            the log directory and pidfile defaults to resque-pool.pid in the current dir.
+
+            Usage:
+               resque-pool [options]
+
+            where [options] are:
+          EOS
+          opt.on('-c', '--config PATH', "Alternate path to config file") { |c| opts[:config] = c }
+          opt.on('-a', '--appname NAME', "Alternate appname") { |c| opts[:appname] = c }
+          opt.on("-d", '--daemon', "Run as a background daemon") {
+            opts[:daemon] = true
+            opts[:stdout]  ||= "log/resque-pool.stdout.log"
+            opts[:stderr]  ||= "log/resque-pool.stderr.log"
+            opts[:pidfile] ||= "tmp/pids/resque-pool.pid" unless opts[:no_pidfile]
+          }
+          opt.on("-k", '--kill-others', "Shutdown any other Resque Pools on startup") { opts[:killothers] = true }
+          opt.on('-o', '--stdout FILE', "Redirect stdout to logfile") { |c| opts[:stdout] = c }
+          opt.on('-e', '--stderr FILE', "Redirect stderr to logfile") { |c| opts[:stderr] = c }
+          opt.on('--nosync', "Don't sync logfiles on every write") { opts[:nosync] = true }
+          opt.on("-p", '--pidfile FILE', "PID file location") { |c|
+            opts[:pidfile] = c
+            opts[:no_pidfile] = false
+          }
+          opt.on('--no-pidfile', "Force no pidfile, even if daemonized") {
+            opts[:pidfile] = nil
+            opts[:no_pidfile] = true
+          }
+          opt.on('-l', '--lock FILE' "Open a shared lock on a file") { |c| opts[:lock_file] = c }
+          opt.on("-H", "--hot-swap", "Set appropriate defaults to hot-swap a new pool for a running pool") {|c|
+            opts[:pidfile] = nil
+            opts[:no_pidfile] = true
+            opts[:lock_file] ||= "tmp/resque-pool.lock"
+            opts[:killothers] = true
+          }
+          opt.on("-E", '--environment ENVIRONMENT', "Set RAILS_ENV/RACK_ENV/RESQUE_ENV") { |c| opts[:environment] = c }
+          opt.on("-s", '--spawn-delay MS', Integer, "Delay in milliseconds between spawning missing workers") { |c| opts[:spawn_delay] = c }
+          opt.on('--term-graceful-wait', "On TERM signal, wait for workers to shut down gracefully") { opts[:term_graceful_wait] = true }
+          opt.on('--term-graceful',      "On TERM signal, shut down workers gracefully") { opts[:term_graceful] = true }
+          opt.on('--term-immediate',     "On TERM signal, shut down workers immediately (default)") { opts[:term_immediate] = true }
+          opt.on('--single-process-group', "Workers remain in the same process group as the master") { opts[:single_process_group] = true }
+          opt.on("-h", "--help", "Show this.") { puts opt; exit }
+          opt.on("-v", "--version", "Show Version"){ puts "resque-pool #{VERSION} (c) nicholas a. evans"; exit}
+        end
+        parser.parse!(argv || parser.default_argv)
+
+        opts
+      end
+
+      def daemonize
+        raise 'First fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+        Process.setsid
+        raise 'Second fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+      end
+
+      # Obtain a lock on a file that will be held for the lifetime of
+      # the process.  This aids in concurrent daemonized deployment with
+      # process managers like upstart since multiple pools can share a
+      # lock, but not a pidfile.
+      def obtain_shared_lock(lock_path)
+        return unless lock_path
+        @lock_file = File.open(lock_path, 'w')
+        unless @lock_file.flock(File::LOCK_SH)
+          fail "unable to obtain shared lock on #{@lock_file}"
+        end
+      end
+
+      def manage_pidfile(pidfile)
+        return unless pidfile
+        pid = Process.pid
+        if File.exist? pidfile
+          if process_still_running? pidfile
+            raise "Pidfile already exists at #{pidfile} and process is still running."
+          else
+            File.delete pidfile
+          end
+        else
+          FileUtils.mkdir_p File.dirname(pidfile)
+        end
+        File.open pidfile, "w" do |f|
+          f.write pid
+        end
+        at_exit do
+          if Process.pid == pid
+            File.delete pidfile
+          end
+        end
+      end
+
+      def process_still_running?(pidfile)
+        old_pid = open(pidfile).read.strip.to_i
+        old_pid > 0 && Process.kill(0, old_pid)
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      rescue ::Exception => e
+        $stderr.puts "While checking if PID #{old_pid} is running, unexpected #{e.class}: #{e}"
+        true
+      end
+
+      def redirect(opts)
+        $stdin.reopen  '/dev/null'        if opts[:daemon]
+        # need to reopen as File, or else Resque::Pool::Logging.reopen_logs! won't work
+        out = File.new(opts[:stdout], "a") if opts[:stdout] && !opts[:stdout].empty?
+        err = File.new(opts[:stderr], "a") if opts[:stderr] && !opts[:stderr].empty?
+        $stdout.reopen out if out
+        $stderr.reopen err if err
+        $stdout.sync = $stderr.sync = true unless opts[:nosync]
+      end
+
+      # TODO: global variables are not the best way
+      def set_pool_options(opts)
+        if opts[:daemon]
+          Resque::Pool.handle_winch = true
+        end
+        if opts[:term_graceful_wait]
+          Resque::Pool.term_behavior = "graceful_worker_shutdown_and_wait"
+        elsif opts[:term_graceful]
+          Resque::Pool.term_behavior = "graceful_worker_shutdown"
+        elsif ENV["TERM_CHILD"]
+          log "TERM_CHILD enabled, so will user 'term-graceful-and-wait' behaviour"
+          Resque::Pool.term_behavior = "graceful_worker_shutdown_and_wait"
+        end
+        if ENV.include?("DYNO") && !ENV["TERM_CHILD"]
+          log "WARNING: Are you running on Heroku? You should probably set TERM_CHILD=1"
+        end
+        if opts[:spawn_delay]
+          Resque::Pool.spawn_delay = opts[:spawn_delay] * 0.001
+        end
+        Resque::Pool.kill_other_pools = !!opts[:killothers]
+      end
+
+      def setup_environment(opts)
+        Resque::Pool.app_name = opts[:appname]    if opts[:appname]
+        ENV["RACK_ENV"] = ENV["RAILS_ENV"] = ENV["RESQUE_ENV"] = opts[:environment] if opts[:environment]
+        Resque::Pool.log "Resque Pool running in #{ENV["RAILS_ENV"] || "development"} environment"
+        ENV["RESQUE_POOL_CONFIG"] = opts[:config] if opts[:config]
+        Resque::Pool.single_process_group = opts[:single_process_group]
+      end
+
+      def start_pool
+        require 'rake'
+        require 'resque/pool/tasks'
+        Rake.application.init
+        Rake.application.load_rakefile
+        Rake.application["resque:pool"].invoke
+      end
+
+    end
+  end
+end
+

data/lib/resque/pool/file_or_hash_loader.rb

@@ -0,0 +1,59 @@
+module Resque
+  class Pool
+    class FileOrHashLoader
+      def initialize(filename_or_hash=nil)
+        case filename_or_hash
+        when String, nil
+          @filename = filename_or_hash
+        when Hash
+          @static_config = filename_or_hash.dup
+        else
+          raise "#{self.class} cannot be initialized with #{filename_or_hash.inspect}"
+        end
+      end
+
+      def call(environment)
+        @config ||= load_config_from_file(environment)
+      end
+
+      def reset!
+        @config = nil
+      end
+
+      private
+
+      def load_config_from_file(environment)
+        if @static_config
+          new_config = @static_config
+        else
+          filename = config_filename
+          new_config = load_config filename
+        end
+        apply_environment new_config, environment
+      end
+
+      def apply_environment(config, environment)
+        environment and config[environment] and config.merge!(config[environment])
+        config.delete_if {|key, value| value.is_a? Hash }
+      end
+
+      def config_filename
+        @filename || choose_config_file
+      end
+
+      def load_config(filename)
+        return {} unless filename
+        YAML.load(ERB.new(IO.read(filename)).result)
+      end
+
+      CONFIG_FILES = ["resque-pool.yml", "config/resque-pool.yml"]
+      def choose_config_file
+        if ENV["RESQUE_POOL_CONFIG"]
+          ENV["RESQUE_POOL_CONFIG"]
+        else
+          CONFIG_FILES.detect { |f| File.exist?(f) }
+        end
+      end
+    end
+  end
+end

data/lib/resque/pool/killer.rb

@@ -0,0 +1,40 @@
+module Resque
+  class Pool
+    class Killer
+      include Logging
+
+      GRACEFUL_SHUTDOWN_SIGNAL=:INT
+
+      def self.run
+        new.run
+      end
+
+      def run
+        my_pid = Process.pid
+        pool_pids = all_resque_pool_processes
+        pids_to_kill = pool_pids.reject{|pid| pid == my_pid}
+        pids_to_kill.each do |pid|
+          log "Pool (#{my_pid}) in kill-others mode: killing pool with pid (#{pid})"
+          Process.kill(GRACEFUL_SHUTDOWN_SIGNAL, pid)
+        end
+      end
+
+
+      def all_resque_pool_processes
+        out = `ps -e -o pid= -o command= 2>&1`
+        raise "Unable to identify other pools: #{out}" unless $?.success?
+        parse_pids_from_output out
+      end
+
+      RESQUE_POOL_PIDS = /
+        ^\s*(\d+)                         # PID digits, optional leading spaces
+        \s+                               # column divider
+        #{Regexp.escape(PROCLINE_PREFIX)} # exact match at start of command
+      /x
+
+      def parse_pids_from_output(output)
+        output.scan(RESQUE_POOL_PIDS).flatten.map(&:to_i)
+      end
+    end
+  end
+end

data/lib/resque/pool/logging.rb

@@ -0,0 +1,124 @@
+module Resque
+  class Pool
+    module Logging
+      extend self
+
+      # This reopens ALL logfiles in the process that have been rotated
+      # using logrotate(8) (without copytruncate) or similar tools.
+      # A +File+ object is considered for reopening if it is:
+      #   1) opened with the O_APPEND and O_WRONLY flags
+      #   2) the current open file handle does not match its original open path
+      #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+      # Returns the number of files reopened
+      #
+      # This was mostly copied from Unicorn 4.8.2 to simplify reopening
+      # logs in the same way that Unicorn does.  Original comments and 
+      # explanations are left intact.
+      def self.reopen_logs!
+        to_reopen      = [ ]
+        reopened_count = 0
+
+        ObjectSpace.each_object(File) { |fp| is_log?(fp) and to_reopen << fp }
+        log "Flushing #{to_reopen.length} logs"
+
+        to_reopen.each do |fp|
+          orig_st = begin
+            fp.stat
+          rescue IOError, Errno::EBADF # race
+            next
+          end
+
+          begin
+            b = File.stat(fp.path)
+            # Skip if reopening wouldn't do anything
+            next if orig_st.ino == b.ino && orig_st.dev == b.dev
+          rescue Errno::ENOENT
+          end
+
+          begin
+            # stdin, stdout, stderr are special.  The following dance should
+            # guarantee there is no window where `fp' is unwritable in MRI
+            # (or any correct Ruby implementation).
+            #
+            # Fwiw, GVL has zero bearing here.  This is tricky because of
+            # the unavoidable existence of stdio FILE * pointers for
+            # std{in,out,err} in all programs which may use the standard C library
+            if fp.fileno <= 2
+              # We do not want to hit fclose(3)->dup(2) window for std{in,out,err}
+              # MRI will use freopen(3) here internally on std{in,out,err}
+              fp.reopen(fp.path, "a")
+            else
+              # We should not need this workaround, Ruby can be fixed:
+              #    http://bugs.ruby-lang.org/issues/9036
+              # MRI will not call call fclose(3) or freopen(3) here
+              # since there's no associated std{in,out,err} FILE * pointer
+              # This should atomically use dup3(2) (or dup2(2)) syscall
+              File.open(fp.path, "a") { |tmpfp| fp.reopen(tmpfp) }
+            end
+
+            fp.sync = true
+            fp.flush # IO#sync=true may not implicitly flush
+            new_st = fp.stat
+
+            # this should only happen in the master:
+            if orig_st.uid != new_st.uid || orig_st.gid != new_st.gid
+              fp.chown(orig_st.uid, orig_st.gid)
+            end
+
+            log "Reopened logfile: #{fp.path}"
+            reopened_count += 1
+          rescue IOError, Errno::EBADF
+            # not much we can do...
+          end
+        end
+
+        reopened_count
+      end
+
+      PROCLINE_PREFIX="resque-pool-master"
+
+      # Given a string, sets the procline ($0)
+      # Procline is always in the format of:
+      #   resque-pool-master: STRING
+      def procline(string)
+        $0 = "#{PROCLINE_PREFIX}#{app}: #{string}"
+      end
+
+      # TODO: make this use an actual logger
+      def log(message)
+        return if $skip_logging
+        puts "resque-pool-manager#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # TODO: make this use an actual logger
+      def log_worker(message)
+        return if $skip_logging
+        puts "resque-pool-worker#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # Include optional app name in procline
+      def app
+        app_name   = self.respond_to?(:app_name)       && self.app_name
+        app_name ||= self.class.respond_to?(:app_name) && self.class.app_name
+        app_name ? "[#{app_name}]" : ""
+      end
+
+      private
+
+      # Used by reopen_logs, borrowed from Unicorn...
+      def self.is_log?(fp)
+        append_flags = File::WRONLY | File::APPEND
+
+        ! fp.closed? &&
+          fp.stat.file? &&
+          fp.sync &&
+          (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+        rescue IOError, Errno::EBADF
+          false
+      end
+
+    end
+  end
+end

data/lib/resque/pool/pooled_worker.rb

@@ -0,0 +1,41 @@
+require 'resque/worker'
+
+class Resque::Pool
+  module PooledWorker
+    attr_accessor :pool_master_pid
+    attr_accessor :worker_parent_pid
+
+    # We will return false if there are no potential_parent_pids, because that
+    # means we aren't even running inside resque-pool.
+    #
+    # We can't just check if we've been re-parented to PID 1 (init) because we
+    # want to support docker (which will make the pool master PID 1).
+    #
+    # We also check the worker_parent_pid, because resque-multi-jobs-fork calls
+    # Worker#shutdown? from inside the worker child process.
+    def pool_master_has_gone_away?
+      pids = potential_parent_pids
+      pids.any? && !pids.include?(Process.ppid)
+    end
+
+    def potential_parent_pids
+      [pool_master_pid, worker_parent_pid].compact
+    end
+
+    def shutdown_with_pool?
+      shutdown_without_pool? || pool_master_has_gone_away?
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        alias_method :shutdown_without_pool?, :shutdown?
+        alias_method :shutdown?, :shutdown_with_pool?
+      end
+    end
+
+  end
+end
+
+Resque::Worker.class_eval do
+  include Resque::Pool::PooledWorker
+end