qless-pool 0.1.0

data/lib/qless/pool/cli.rb

@@ -0,0 +1,136 @@
+require 'trollop'
+require 'qless/pool'
+require 'fileutils'
+
+module Qless
+  class Pool
+    module CLI
+      extend self
+
+      def run
+        opts = parse_options
+        daemonize if opts[:daemon]
+        manage_pidfile opts[:pidfile]
+        redirect opts
+        setup_environment opts
+        set_pool_options opts
+        start_pool
+      end
+
+      def parse_options
+        opts = Trollop::options do
+          version "qless-pool #{VERSION} (c) nicholas a. evans"
+          banner <<-EOS
+qless-pool is the best way to manage a group (pool) of qless workers
+
+When daemonized, stdout and stderr default to qless-pool.stdxxx.log files in
+the log directory and pidfile defaults to qless-pool.pid in the current dir.
+
+Usage:
+   qless-pool [options]
+where [options] are:
+          EOS
+          opt :config, "Alternate path to config file", :type => String, :short => "-c"
+          opt :appname, "Alternate appname",         :type => String,    :short => "-a"
+          opt :daemon, "Run as a background daemon", :default => false,  :short => "-d"
+          opt :stdout, "Redirect stdout to logfile", :type => String,    :short => '-o'
+          opt :stderr, "Redirect stderr to logfile", :type => String,    :short => '-e'
+          opt :nosync, "Don't sync logfiles on every write"
+          opt :pidfile, "PID file location",         :type => String,    :short => "-p"
+          opt :environment, "Set RAILS_ENV/RACK_ENV/QLESS_ENV", :type => String, :short => "-E"
+          opt :term_graceful_wait, "On TERM signal, wait for workers to shut down gracefully"
+          opt :term_graceful,      "On TERM signal, shut down workers gracefully"
+          opt :term_immediate,     "On TERM signal, shut down workers immediately (default)"
+        end
+        if opts[:daemon]
+          opts[:stdout]  ||= "log/qless-pool.stdout.log"
+          opts[:stderr]  ||= "log/qless-pool.stderr.log"
+          opts[:pidfile] ||= "tmp/pids/qless-pool.pid"
+        end
+        opts
+      end
+
+      def daemonize
+        raise 'First fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+        Process.setsid
+        raise 'Second fork failed' if (pid = fork) == -1
+        exit unless pid.nil?
+      end
+
+      def manage_pidfile(pidfile)
+        return unless pidfile
+        pid = Process.pid
+        if File.exist? pidfile
+          if process_still_running? pidfile
+            raise "Pidfile already exists at #{pidfile} and process is still running."
+          else
+            File.delete pidfile
+          end
+        else
+          FileUtils.mkdir_p File.dirname(pidfile)
+        end
+        File.open pidfile, "w" do |f|
+          f.write pid
+        end
+        at_exit do
+          if Process.pid == pid
+            File.delete pidfile
+          end
+        end
+      end
+
+      def process_still_running?(pidfile)
+        old_pid = open(pidfile).read.strip.to_i
+        Process.kill 0, old_pid
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      rescue ::Exception => e
+        $stderr.puts "While checking if PID #{old_pid} is running, unexpected #{e.class}: #{e}"
+        true
+      end
+
+      def redirect(opts)
+        $stdin.reopen  '/dev/null'        if opts[:daemon]
+        # need to reopen as File, or else Qless::Pool::Logging.reopen_logs! won't work
+        out = File.new(opts[:stdout], "a") if opts[:stdout] && !opts[:stdout].empty?
+        err = File.new(opts[:stderr], "a") if opts[:stderr] && !opts[:stderr].empty?
+        $stdout.reopen out if out
+        $stderr.reopen err if err
+        $stdout.sync = $stderr.sync = true unless opts[:nosync]
+      end
+
+      # TODO: global variables are not the best way
+      def set_pool_options(opts)
+        if opts[:daemon]
+          Qless::Pool.handle_winch = true
+        end
+        if opts[:term_graceful_wait]
+          Qless::Pool.term_behavior = "graceful_worker_shutdown_and_wait"
+        elsif opts[:term_graceful]
+          Qless::Pool.term_behavior = "graceful_worker_shutdown"
+        end
+      end
+
+      def setup_environment(opts)
+        Qless::Pool.app_name = opts[:appname]    if opts[:appname]
+        ENV["RACK_ENV"] = ENV["RAILS_ENV"] = ENV["QLESS_ENV"] = opts[:environment] if opts[:environment]
+        Qless::Pool.log "Qless Pool running in #{ENV["RAILS_ENV"] || "development"} environment"
+        ENV["QLESS_POOL_CONFIG"] = opts[:config] if opts[:config]
+      end
+
+      def start_pool
+        require 'rake'
+        require 'qless/pool/tasks'
+        Rake.application.init
+        Rake.application.load_rakefile
+        Rake.application["qless:pool"].invoke
+      end
+
+    end
+  end
+end
+

data/lib/qless/pool/logging.rb

@@ -0,0 +1,65 @@
+module Qless
+  class Pool
+    module Logging
+      extend self
+
+      # more than a little bit complicated...
+      # copied this from Unicorn.
+      def self.reopen_logs!
+        log "Flushing logs"
+        [$stdout, $stderr].each do |fd|
+          if fd.instance_of? File
+            # skip if the file is the exact same inode and device
+            orig_st = fd.stat
+            begin
+              cur_st = File.stat(fd.path)
+              next if orig_st.ino == cur_st.ino && orig_st.dev == cur_st.dev
+            rescue Errno::ENOENT
+            end
+            # match up the encoding
+            open_arg = 'a'
+            if fd.respond_to?(:external_encoding) && enc = fd.external_encoding
+              open_arg << ":#{enc.to_s}"
+              enc = fd.internal_encoding and open_arg << ":#{enc.to_s}"
+            end
+            # match up buffering (does reopen reset this?)
+            sync = fd.sync
+            # sync to disk
+            fd.fsync
+            # reopen, and set ruby buffering appropriately
+            fd.reopen fd.path, open_arg
+            fd.sync = sync
+            log "Reopened logfile: #{fd.path}"
+          end
+        end
+      end
+
+      # Given a string, sets the procline ($0)
+      # Procline is always in the format of:
+      #   qless-pool-master: STRING
+      def procline(string)
+        $0 = "qless-pool-master#{app}: #{string}"
+      end
+
+      # TODO: make this use an actual logger
+      def log(message)
+        puts "qless-pool-manager#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # TODO: make this use an actual logger
+      def log_worker(message)
+        puts "qless-pool-worker#{app}[#{Process.pid}]: #{message}"
+        #$stdout.fsync
+      end
+
+      # Include optional app name in procline
+      def app
+        app_name   = self.respond_to?(:app_name)       && self.app_name
+        app_name ||= self.class.respond_to?(:app_name) && self.class.app_name
+        app_name ? "[#{app_name}]" : ""
+      end
+
+    end
+  end
+end

data/lib/qless/pool/pool_factory.rb

@@ -0,0 +1,43 @@
+module Qless
+    class PoolFactory
+      
+      def initialize(options={})
+        @options = {
+            :term_timeout => ENV['TERM_TIMEOUT'] || 4.0,
+            :verbose => !!ENV['VERBOSE'],
+            :very_verbose => !!ENV['VVERBOSE'],
+           :run_as_single_process => !!ENV['RUN_AS_SINGLE_PROCESS']
+        }.merge(options)
+      end
+      
+      def client
+        @qless_client ||= Qless::Client.new
+      end
+      
+      def client=(client)
+        @qless_client = client
+      end
+          
+      def reserver_class
+        @reserver_class ||= Qless::JobReservers.const_get(ENV.fetch('JOB_RESERVER', 'Ordered'))
+      end
+      
+      def reserver_class=(reserver_class)
+        @reserver_class = reserver_class
+      end
+
+      def reserver(queues)
+        reserver_class.new(queues)
+      end
+      
+      def worker(queues)
+        queues = queues.to_s.split(',').map { |q| client.queues[q.strip] }
+        if queues.none?
+          raise "No queues provided"
+        end
+  
+        Qless::Worker.new(reserver(queues), @options)
+      end
+            
+    end
+end

data/lib/qless/pool/pooled_worker.rb

@@ -0,0 +1,21 @@
+require 'qless/worker'
+
+class Qless::Pool
+  module PooledWorker
+    def shutdown_with_pool
+      shutdown_without_pool || Process.ppid == 1
+    end
+
+    def self.included(base)
+      base.instance_eval do
+        alias_method :shutdown_without_pool, :shutdown?
+        alias_method :shutdown?, :shutdown_with_pool
+      end
+    end
+
+  end
+end
+
+Qless::Worker.class_eval do
+  include Qless::Pool::PooledWorker
+end

data/lib/qless/pool/tasks.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require 'qless/tasks'
+
+namespace :qless do
+
+  # qless worker config (not pool related).  e.g. hoptoad, rails environment
+  task :setup
+
+  namespace :pool do
+     # qless pool config.  e.g. after_prefork connection handling
+    task :setup
+  end
+
+  desc "Launch a pool of qless workers"
+  task :pool => %w[qless:setup qless:pool:setup] do
+    require 'qless/pool'
+    Qless::Pool.run
+  end
+
+end

data/lib/qless/pool/version.rb

@@ -0,0 +1,5 @@
+module Qless
+  class Pool
+    VERSION = "0.1.0"
+  end
+end

data/man/qless-pool.1

@@ -0,0 +1,88 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "QLESS\-POOL" "1" "June 2012" "QLESS-POOL 0.4.0.DEV" "QLESS-POOL"
+.
+.SH "NAME"
+\fBqless\-pool\fR \- qless worker pool management
+.
+.SH "SYNOPSIS"
+\fBqless\-pool\fR \fIoptions\fR
+.
+.SH "DESCRIPTION"
+\fBQless\-pool\fR is the best way to manage a group (pool) of qless workers\.
+.
+.P
+When qless\-pool(1) is daemonized the \fBstdout\fR and \fBstderr\fR output streams are redirected to \fBqless\-pool\.stdxxx\.log\fR log files in the \fBlog\fR directory\. Additionally the PID file defaults to \fBqless\-pool\.pid\fR in the \fBtmp/pids\fR directory\.
+.
+.SH "OPTIONS"
+.
+.IP "\(bu" 4
+\fB\-c, \-\-config\fR \fIfile\fR: Uses the configuration specified in the \fIfile\fR provided instead of searching in the current and \fBconfig\fR directories for \fBqless\-pool\.yml\fR\.
+.
+.IP "\(bu" 4
+\fB\-a, \-\-appname\fR \fIname\fR: Specifies the app name to be used for logging and procline\. If not specified, this defaults to the current working directory\.
+.
+.IP "\(bu" 4
+\fB\-d, \-\-daemon\fR: Runs \fBqless\-pool\fR in the background as a daemon process\. This will redirect \fBstdout\fR and \fBstderr\fR to log files and write a PID file\.
+.
+.IP "\(bu" 4
+\fB\-o, \-\-stdout\fR \fIfile\fR: Writes the normal log output to \fIfile\fR instead of printing to the terminal\. When running as a daemon this defaults to the path \fBlog/qless\-pool\.stdout\.log\fR\.
+.
+.IP "\(bu" 4
+\fB\-e, \-\-stderr\fR \fIfile\fR: Writes the standard error output to \fIfile\fR instead of printing to the terminal\. When running as a daemon this defaults to the path \fBlog/qless\-pool\.stderr\.log\fR\.
+.
+.IP "\(bu" 4
+\fB\-\-nosync\fR Allows writes to \fBstdout\fR and \fBstderr\fR to be buffered\.
+.
+.IP "\(bu" 4
+\fB\-p, \-\-pidfile\fR \fIfile\fR: Writes the PID to the \fIfile\fR\. When running as a daemon this defaults to \fBtmp/pids/qless\-pool\.pid\fR\.
+.
+.IP "\(bu" 4
+\fB\-E, \-\-environment\fR \fIname\fR: Specifies the environment \fIname\fR to be set for \fBRAILS_ENV\fR, \fBRACK_ENV\fR and \fBQLESS_ENV\fR which will be passed on to the pooled qless workers\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-graceful\-wait\fR: Configure TERM signal handling so the master will gracefully request worker shut downs (via \fBQUIT\fR) and wait for the workers to quit before shutting down itself\. This is the same behavior as the \fBQUIT\fR signal\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-graceful\fR: Configure TERM signal handling so the master will gracefully request worker shut downs (via \fBQUIT\fR) but shut itself down immediately\. This the same behavior as the \fBINT\fR signal\.
+.
+.IP "\(bu" 4
+\fB\-\-term\-immediate\fR: Configure TERM signal handling so the master will request imediate worker shut downs (via \fBINT\fR) and shut itself down immediately\. This is the default \fBTERM\fR signal behavior\.
+.
+.IP "" 0
+.
+.SH "HISTORY"
+.
+.TP
+\fBv0\.3\.0\fR
+Support for ruby 1\.9, qless 1\.20\.
+.
+.br
+Added appname for logging\.
+.
+.br
+Minor bugfixes\.
+.
+.TP
+\fBv0\.2\.0\fR
+Support for reloading logs and workers with \fBHUP\fR signal
+.
+.br
+Cleans up PID file on startup
+.
+.br
+Fixed \fB\-c, \-\-config\fR option\.
+.
+.TP
+\fBv0\.1\.0\fR
+\fBqless\-pool\fR command line interface added
+.
+.SH "AUTHOR"
+Nicholas Evans
+.
+.SH "COPYRIGHT"
+Copyright (C) 2010 by Nicholas Evans \fInick@ekenosen\.net\fR, et al\.
+.
+.SH "SEE ALSO"
+qless\-pool\.yml(5)

data/man/qless-pool.1.ronn

@@ -0,0 +1,92 @@
+qless-pool(1) -- qless worker pool management
+================================================
+
+## SYNOPSIS
+
+`qless-pool` [options]
+
+## DESCRIPTION
+
+**Qless-pool** is the best way to manage a group (pool) of qless workers.
+
+When qless-pool(1) is daemonized the `stdout` and `stderr` output streams
+are redirected to `qless-pool.stdxxx.log` log files in the `log` directory.
+Additionally the PID file defaults to `qless-pool.pid` in the `tmp/pids`
+directory.
+
+## OPTIONS
+
+  * `-c, --config` <file>:
+    Uses the configuration specified in the <file> provided instead of
+    searching in the current and `config` directories for `qless-pool.yml`.
+
+  * `-a, --appname` <name>:
+    Specifies the app name to be used for logging and procline.  If not
+    specified, this defaults to the current working directory.
+
+  * `-d, --daemon`:
+    Runs `qless-pool` in the background as a daemon process. This will
+    redirect `stdout` and `stderr` to log files and write a PID file.
+
+  * `-o, --stdout` <file>:
+    Writes the normal log output to <file> instead of printing to the
+    terminal. When running as a daemon this defaults to the path
+    `log/qless-pool.stdout.log`.
+
+  * `-e, --stderr` <file>:
+    Writes the standard error output to <file> instead of printing to the
+    terminal. When running as a daemon this defaults to the path
+    `log/qless-pool.stderr.log`.
+
+  * `--nosync`
+    Allows writes to `stdout` and `stderr` to be buffered.
+
+  * `-p, --pidfile` <file>:
+    Writes the PID to the <file>. When running as a daemon this defaults
+    to `tmp/pids/qless-pool.pid`.
+
+  * `-E, --environment` <name>:
+    Specifies the environment <name> to be set for `RAILS_ENV`, `RACK_ENV`
+    and `QLESS_ENV` which will be passed on to the pooled qless workers.
+
+  * `--term-graceful-wait`:
+    Configure TERM signal handling so the master will gracefully request worker
+    shut downs (via `QUIT`) and wait for the workers to quit before shutting
+    down itself.  This is the same behavior as the `QUIT` signal.
+
+  * `--term-graceful`:
+    Configure TERM signal handling so the master will gracefully request worker
+    shut downs (via `QUIT`) but shut itself down immediately.  This the same
+    behavior as the `INT` signal.
+
+  * `--term-immediate`:
+    Configure TERM signal handling so the master will request imediate worker
+    shut downs (via `INT`) and shut itself down immediately.  This is the
+    default `TERM` signal behavior.
+
+## HISTORY
+
+  * `v0.3.0`:
+    Support for ruby 1.9, qless 1.20.<br>
+    Added appname for logging.<br>
+    Minor bugfixes.
+
+  * `v0.2.0`:
+    Support for reloading logs and workers with `HUP` signal<br>
+    Cleans up PID file on startup<br>
+    Fixed `-c, --config` option.
+
+  * `v0.1.0`:
+    `qless-pool` command line interface added
+
+## AUTHOR
+
+Nicholas Evans
+
+## COPYRIGHT
+
+Copyright (C) 2010 by Nicholas Evans <nick@ekenosen.net>, et al.
+
+## SEE ALSO
+
+qless-pool.yml(5)