resque-pool-vinted 0.4.0.rc1

data/lib/resque/pool/cli.rb

@@ -0,0 +1,138 @@
+require 'trollop'
+require 'resque/pool'
+require 'fileutils'
+
+module Resque
+  class Pool
+    module CLI
+      extend self
+
+      def run
+        opts = parse_options
+        daemonize if opts[:daemon]
+        manage_pidfile opts[:pidfile]
+        redirect opts
+        setup_environment opts
+        set_pool_options opts
+        start_pool
+      end
+
+      def parse_options
+        opts = Trollop::options do
+          version "resque-pool #{VERSION} (c) nicholas a. evans"
+          banner <<-EOS
+resque-pool is the best way to manage a group (pool) of resque workers
+
+When daemonized, stdout and stderr default to resque-pool.stdxxx.log files in
+the log directory and pidfile defaults to resque-pool.pid in the current dir.
+
+Usage:
+   resque-pool [options]
+where [options] are:
+          EOS
+          opt :config, "Alternate path to config file", :type => String, :short => "-c"
+          opt :appname, "Alternate appname",         :type => String,    :short => "-a"
+          opt :daemon, "Run as a background daemon", :default => false,  :short => "-d"
+          opt :stdout, "Redirect stdout to logfile", :type => String,    :short => '-o'
+          opt :stderr, "Redirect stderr to logfile", :type => String,    :short => '-e'
+          opt :nosync, "Don't sync logfiles on every write"
+          opt :pidfile, "PID file location",         :type => String,    :short => "-p"
+          opt :environment, "Set RAILS_ENV/RACK_ENV/RESQUE_ENV", :type => String, :short => "-E"
+          opt :term_graceful_wait, "On TERM signal, wait for workers to shut down gracefully"
+          opt :term_graceful,      "On TERM signal, shut down workers gracefully"
+          opt :term_immediate,     "On TERM signal, shut down workers immediately (default)"
+          opt :single_process_group, "Workers remain in the same process group as the master", :default => false
+        end
+        if opts[:daemon]
+          opts[:stdout]  ||= "log/resque-pool.stdout.log"
+          opts[:stderr]  ||= "log/resque-pool.stderr.log"
+          opts[:pidfile] ||= "tmp/pids/resque-pool.pid"
+        end
+        opts
+      end
+
+      def daemonize
+        raise 'First fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+        Process.setsid
+        raise 'Second fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+      end
+
+      def manage_pidfile(pidfile)
+        return unless pidfile
+        pid = Process.pid
+        if File.exist? pidfile
+          if process_still_running? pidfile
+            raise "Pidfile already exists at #{pidfile} and process is still running."
+          else
+            File.delete pidfile
+          end
+        else
+          FileUtils.mkdir_p File.dirname(pidfile)
+        end
+        File.open pidfile, "w" do |f|
+          f.write pid
+        end
+        at_exit do
+          if Process.pid == pid
+            File.delete pidfile
+          end
+        end
+      end
+
+      def process_still_running?(pidfile)
+        old_pid = open(pidfile).read.strip.to_i
+        Process.kill 0, old_pid
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      rescue ::Exception => e
+        $stderr.puts "While checking if PID #{old_pid} is running, unexpected #{e.class}: #{e}"
+        true
+      end
+
+      def redirect(opts)
+        $stdin.reopen  '/dev/null'        if opts[:daemon]
+        # need to reopen as File, or else Resque::Pool::Logging.reopen_logs! won't work
+        out = File.new(opts[:stdout], "a") if opts[:stdout] && !opts[:stdout].empty?
+        err = File.new(opts[:stderr], "a") if opts[:stderr] && !opts[:stderr].empty?
+        $stdout.reopen out if out
+        $stderr.reopen err if err
+        $stdout.sync = $stderr.sync = true unless opts[:nosync]
+      end
+
+      # TODO: global variables are not the best way
+      def set_pool_options(opts)
+        if opts[:daemon]
+          Resque::Pool.handle_winch = true
+        end
+        if opts[:term_graceful_wait]
+          Resque::Pool.term_behavior = "graceful_worker_shutdown_and_wait"
+        elsif opts[:term_graceful]
+          Resque::Pool.term_behavior = "graceful_worker_shutdown"
+        end
+      end
+
+      def setup_environment(opts)
+        Resque::Pool.app_name = opts[:appname]    if opts[:appname]
+        ENV["RACK_ENV"] = ENV["RAILS_ENV"] = ENV["RESQUE_ENV"] = opts[:environment] if opts[:environment]
+        Resque::Pool.log "Resque Pool running in #{ENV["RAILS_ENV"] || "development"} environment"
+        ENV["RESQUE_POOL_CONFIG"] = opts[:config] if opts[:config]
+        Resque::Pool.single_process_group = opts[:single_process_group]
+      end
+
+      def start_pool
+        require 'rake'
+        require 'resque/pool/tasks'
+        Rake.application.init
+        Rake.application.load_rakefile
+        Rake.application["resque:pool"].invoke
+      end
+
+    end
+  end
+end
+

data/lib/resque/pool/logging.rb

@@ -0,0 +1,65 @@
+module Resque
+  class Pool
+    module Logging
+      extend self
+
+      # more than a little bit complicated...
+      # copied this from Unicorn.
+      def self.reopen_logs!
+        log "Flushing logs"
+        [$stdout, $stderr].each do |fd|
+          if fd.instance_of? File
+            # skip if the file is the exact same inode and device
+            orig_st = fd.stat
+            begin
+              cur_st = File.stat(fd.path)
+              next if orig_st.ino == cur_st.ino && orig_st.dev == cur_st.dev
+            rescue Errno::ENOENT
+            end
+            # match up the encoding
+            open_arg = 'a'
+            if fd.respond_to?(:external_encoding) && enc = fd.external_encoding
+              open_arg << ":#{enc.to_s}"
+              enc = fd.internal_encoding and open_arg << ":#{enc.to_s}"
+            end
+            # match up buffering (does reopen reset this?)
+            sync = fd.sync
+            # sync to disk
+            fd.fsync
+            # reopen, and set ruby buffering appropriately
+            fd.reopen fd.path, open_arg
+            fd.sync = sync
+            log "Reopened logfile: #{fd.path}"
+          end
+        end
+      end
+
+      # Given a string, sets the procline ($0)
+      # Procline is always in the format of:
+      #   resque-pool-master: STRING
+      def procline(string)
+        $0 = "resque-pool-master#{app}: #{string}"
+      end
+
+      # TODO: make this use an actual logger
+      def log(message)
+        puts "resque-pool-manager#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # TODO: make this use an actual logger
+      def log_worker(message)
+        puts "resque-pool-worker#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # Include optional app name in procline
+      def app
+        app_name   = self.respond_to?(:app_name)       && self.app_name
+        app_name ||= self.class.respond_to?(:app_name) && self.class.app_name
+        app_name ? "[#{app_name}]" : ""
+      end
+
+    end
+  end
+end

data/lib/resque/pool/pooled_worker.rb

@@ -0,0 +1,21 @@
+require 'resque/worker'
+
+class Resque::Pool
+  module PooledWorker
+    def shutdown_with_pool
+      shutdown_without_pool || Process.ppid == 1
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        alias_method :shutdown_without_pool, :shutdown?
+        alias_method :shutdown?, :shutdown_with_pool
+      end
+    end
+
+  end
+end
+
+Resque::Worker.class_eval do
+  include Resque::Pool::PooledWorker
+end

data/lib/resque/pool/tasks.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require 'resque/tasks'
+
+namespace :resque do
+
+  # resque worker config (not pool related).  e.g. hoptoad, rails environment
+  task :setup
+
+  namespace :pool do
+     # resque pool config.  e.g. after_prefork connection handling
+    task :setup
+  end
+
+  desc "Launch a pool of resque workers"
+  task :pool => %w[resque:preload resque:setup resque:pool:setup] do
+    require 'resque/pool'
+    Resque::Pool.run
+  end
+
+end

data/lib/resque/pool/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  class Pool
+    VERSION = "0.4.0.rc1"
+  end
+end

data/man/resque-pool.1

@@ -0,0 +1,88 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "RESQUE\-POOL" "1" "June 2012" "RESQUE-POOL 0.4.0.DEV" "RESQUE-POOL"
+.
+.SH "NAME"
+\fBresque\-pool\fR \- resque worker pool management
+.
+.SH "SYNOPSIS"
+\fBresque\-pool\fR \fIoptions\fR
+.
+.SH "DESCRIPTION"
+\fBResque\-pool\fR is the best way to manage a group (pool) of resque workers\.
+.
+.P
+When resque\-pool(1) is daemonized the \fBstdout\fR and \fBstderr\fR output streams are redirected to \fBresque\-pool\.stdxxx\.log\fR log files in the \fBlog\fR directory\. Additionally the PID file defaults to \fBresque\-pool\.pid\fR in the \fBtmp/pids\fR directory\.
+.
+.SH "OPTIONS"
+.
+.IP "\(bu" 4
+\fB\-c, \-\-config\fR \fIfile\fR: Uses the configuration specified in the \fIfile\fR provided instead of searching in the current and \fBconfig\fR directories for \fBresque\-pool\.yml\fR\.
+.
+.IP "\(bu" 4
+\fB\-a, \-\-appname\fR \fIname\fR: Specifies the app name to be used for logging and procline\. If not specified, this defaults to the current working directory\.
+.
+.IP "\(bu" 4
+\fB\-d, \-\-daemon\fR: Runs \fBresque\-pool\fR in the background as a daemon process\. This will redirect \fBstdout\fR and \fBstderr\fR to log files and write a PID file\.
+.
+.IP "\(bu" 4
+\fB\-o, \-\-stdout\fR \fIfile\fR: Writes the normal log output to \fIfile\fR instead of printing to the terminal\. When running as a daemon this defaults to the path \fBlog/resque\-pool\.stdout\.log\fR\.
+.
+.IP "\(bu" 4
+\fB\-e, \-\-stderr\fR \fIfile\fR: Writes the standard error output to \fIfile\fR instead of printing to the terminal\. When running as a daemon this defaults to the path \fBlog/resque\-pool\.stderr\.log\fR\.
+.
+.IP "\(bu" 4
+\fB\-\-nosync\fR Allows writes to \fBstdout\fR and \fBstderr\fR to be buffered\.
+.
+.IP "\(bu" 4
+\fB\-p, \-\-pidfile\fR \fIfile\fR: Writes the PID to the \fIfile\fR\. When running as a daemon this defaults to \fBtmp/pids/resque\-pool\.pid\fR\.
+.
+.IP "\(bu" 4
+\fB\-E, \-\-environment\fR \fIname\fR: Specifies the environment \fIname\fR to be set for \fBRAILS_ENV\fR, \fBRACK_ENV\fR and \fBRESQUE_ENV\fR which will be passed on to the pooled resque workers\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-graceful\-wait\fR: Configure TERM signal handling so the master will gracefully request worker shut downs (via \fBQUIT\fR) and wait for the workers to quit before shutting down itself\. This is the same behavior as the \fBQUIT\fR signal\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-graceful\fR: Configure TERM signal handling so the master will gracefully request worker shut downs (via \fBQUIT\fR) but shut itself down immediately\. This the same behavior as the \fBINT\fR signal\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-immediate\fR: Configure TERM signal handling so the master will request imediate worker shut downs (via \fBINT\fR) and shut itself down immediately\. This is the default \fBTERM\fR signal behavior\.
+.
+.IP "" 0
+.
+.SH "HISTORY"
+.
+.TP
+\fBv0\.3\.0\fR
+Support for ruby 1\.9, resque 1\.20\.
+.
+.br
+Added appname for logging\.
+.
+.br
+Minor bugfixes\.
+.
+.TP
+\fBv0\.2\.0\fR
+Support for reloading logs and workers with \fBHUP\fR signal
+.
+.br
+Cleans up PID file on startup
+.
+.br
+Fixed \fB\-c, \-\-config\fR option\.
+.
+.TP
+\fBv0\.1\.0\fR
+\fBresque\-pool\fR command line interface added
+.
+.SH "AUTHOR"
+Nicholas Evans
+.
+.SH "COPYRIGHT"
+Copyright (C) 2010 by Nicholas Evans \fInick@ekenosen\.net\fR, et al\.
+.
+.SH "SEE ALSO"
+resque\-pool\.yml(5)

data/man/resque-pool.1.ronn

@@ -0,0 +1,92 @@
+resque-pool(1) -- resque worker pool management
+================================================
+
+## SYNOPSIS
+
+`resque-pool` [options]
+
+## DESCRIPTION
+
+**Resque-pool** is the best way to manage a group (pool) of resque workers.
+
+When resque-pool(1) is daemonized the `stdout` and `stderr` output streams
+are redirected to `resque-pool.stdxxx.log` log files in the `log` directory.
+Additionally the PID file defaults to `resque-pool.pid` in the `tmp/pids`
+directory.
+
+## OPTIONS
+
+  * `-c, --config` <file>:
+    Uses the configuration specified in the <file> provided instead of
+    searching in the current and `config` directories for `resque-pool.yml`.
+
+  * `-a, --appname` <name>:
+    Specifies the app name to be used for logging and procline.  If not
+    specified, this defaults to the current working directory.
+
+  * `-d, --daemon`:
+    Runs `resque-pool` in the background as a daemon process. This will
+    redirect `stdout` and `stderr` to log files and write a PID file.
+
+  * `-o, --stdout` <file>:
+    Writes the normal log output to <file> instead of printing to the
+    terminal. When running as a daemon this defaults to the path
+    `log/resque-pool.stdout.log`.
+
+  * `-e, --stderr` <file>:
+    Writes the standard error output to <file> instead of printing to the
+    terminal. When running as a daemon this defaults to the path
+    `log/resque-pool.stderr.log`.
+
+  * `--nosync`
+    Allows writes to `stdout` and `stderr` to be buffered.
+
+  * `-p, --pidfile` <file>:
+    Writes the PID to the <file>. When running as a daemon this defaults
+    to `tmp/pids/resque-pool.pid`.
+
+  * `-E, --environment` <name>:
+    Specifies the environment <name> to be set for `RAILS_ENV`, `RACK_ENV`
+    and `RESQUE_ENV` which will be passed on to the pooled resque workers.
+
+  * `--term-graceful-wait`:
+    Configure TERM signal handling so the master will gracefully request worker
+    shut downs (via `QUIT`) and wait for the workers to quit before shutting
+    down itself.  This is the same behavior as the `QUIT` signal.
+
+  * `--term-graceful`:
+    Configure TERM signal handling so the master will gracefully request worker
+    shut downs (via `QUIT`) but shut itself down immediately.  This the same
+    behavior as the `INT` signal.
+
+  * `--term-immediate`:
+    Configure TERM signal handling so the master will request imediate worker
+    shut downs (via `INT`) and shut itself down immediately.  This is the
+    default `TERM` signal behavior.
+
+## HISTORY
+
+  * `v0.3.0`:
+    Support for ruby 1.9, resque 1.20.<br>
+    Added appname for logging.<br>
+    Minor bugfixes.
+
+  * `v0.2.0`:
+    Support for reloading logs and workers with `HUP` signal<br>
+    Cleans up PID file on startup<br>
+    Fixed `-c, --config` option.
+
+  * `v0.1.0`:
+    `resque-pool` command line interface added
+
+## AUTHOR
+
+Nicholas Evans
+
+## COPYRIGHT
+
+Copyright (C) 2010 by Nicholas Evans <nick@ekenosen.net>, et al.
+
+## SEE ALSO
+
+resque-pool.yml(5)

data/man/resque-pool.yml.5

@@ -0,0 +1,46 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "RESQUE\-POOL\.YML" "5" "May 2012" "RESQUE-POOL 0.3.0.DEV" "RESQUE-POOL"
+.
+.SH "NAME"
+\fBresque\-pool\.yml\fR \- resque\-pool pool configuration
+.
+.SH "SYNOPSIS"
+\fBresque\-pool\.yml\fR
+.
+.br
+\fBconfig/resque\-pool\.yml\fR
+.
+.SH "DESCRIPTION"
+resque\-pool(1) reads pool configuration data from \fBresque\-pool\.yml\fR (or the file specified with \fB\-c\fR on the command line)\. The file contains queue\-worker\-count pairs, one per line\. The configuration file supports both using root level defaults as well as environment specific overrides (\fBRACK_ENV\fR, \fBRAILS_ENV\fR, and \fBRESQUE_ENV\fR environment variables can be used to determine environment)\.
+.
+.P
+An example configuration
+.
+.IP "" 4
+.
+.nf
+
+foo: 1
+bar: 2
+"foo,bar,baz": 1
+
+production:
+  "foo,bar,baz": 4
+.
+.fi
+.
+.IP "" 0
+.
+.P
+will create 7 workers in production and 4 in other environment configurations\. The simpler worker definition \fBfoo: 1\fR will create 1 worker for the \fBfoo\fR queue, the more complicated \fBfoo,bar,baz: 1\fR will create 1 worker for the queues \fBfoo\fR, \fBbar\fR and \fBbaz\fR\.
+.
+.SH "AUTHOR"
+Nicholas Evans
+.
+.SH "COPYRIGHT"
+Copyright (C) 2010 by Nicholas Evans \fInick@ekenosen\.net\fR, et al\.
+.
+.SH "SEE ALSO"
+resque\-pool(1)