pastry 0.1.3 → 0.3.0

data/README.rdoc

@@ -21,7 +21,6 @@ It's fairly pre-alpha, so use at your own peril
 
 == TODO
 
-* graceful stop / restart
 * increase or decrease worker count
 
 == License

data/bin/pastry

@@ -3,33 +3,41 @@
 require 'pastry'
 require 'optparse'
 
-options = {}
+def start_command
+  $start_command ||= "#{$0} #{ARGV.join(' ')}".freeze
+end
+
+options = {start_command: start_command}
 parser  = OptionParser.new do |opts|
   opts.banner = "pastry [options]"
 
-  opts.on('-n', '--servers number',   'worker count')     {|value| options[:workers]    = value }
-  opts.on('-E', '--environment name', 'rack environment') {|value| options[:env]        = value }
-  opts.on('-R', '--rackup file',      'rackup file')      {|value| options[:rackup]     = value }
-  opts.on('-a', '--address name',     'bind ip/host')     {|value| options[:host]       = value }
-  opts.on('-p', '--port name',        'bind port')        {|value| options[:port]       = value }
-  opts.on('-s', '--socket file',      'unix socket')      {|value| options[:socket]     = value }
-  opts.on('-d', '--[no-]daemon',      'daemonize')        {|value| options[:daemonize]  = value }
-  opts.on('-l', '--logfile file',     'logfile')          {|value| options[:logfile]    = value }
-  opts.on('-P', '--pidfile file',     'pidfile')          {|value| options[:pidfile]    = value }
-  opts.on('-c', '--connections num',  'max connections')  {|value| options[:maxconn]    = value }
-  opts.on('-t', '--timeout secs',     'read timeout')     {|value| options[:timeout]    = value }
-  opts.on('-A', '--application name', 'app name')         {|value| options[:name]       = value }
+  opts.on('-n', '--servers number',   'worker count')     {|value| options[:workers]          = value }
+  opts.on('-E', '--environment name', 'rack environment') {|value| options[:env]              = value }
+  opts.on('-R', '--rackup file',      'rackup file')      {|value| options[:rackup]           = value }
+  opts.on('-a', '--address name',     'bind ip/host')     {|value| options[:host]             = value }
+  opts.on('-p', '--port name',        'bind port')        {|value| options[:port]             = value }
+  opts.on('-s', '--socket file',      'unix socket')      {|value| options[:socket]           = value }
+  opts.on('-d', '--[no-]daemon',      'daemonize')        {|value| options[:daemonize]        = value }
+  opts.on('-l', '--logfile file',     'logfile')          {|value| options[:logfile]          = value }
+  opts.on('-P', '--pidfile file',     'pidfile')          {|value| options[:pidfile]          = value }
+  opts.on('-q', '--queue size',       'pending requests') {|value| options[:queue]            = value }
+  opts.on('-c', '--connections num',  'max connections')  {|value| options[:max_connections]  = value }
+  opts.on('-t', '--timeout secs',     'read timeout')     {|value| options[:timeout]          = value }
+  opts.on('-A', '--application name', 'app name')         {|value| options[:name]             = value }
 end
 
 parser.parse!
 
-%w(workers port maxconn timeout).map(&:to_sym).each do |name|
+%w(workers port max_connections queue timeout).map(&:to_sym).each do |name|
   options[name] = options[name].to_i if options.key?(name)
 end
 
-app  = Rack::Builder.parse_file(options.delete(:rackup) || 'config.ru').first
-size = options.delete(:workers) || 2
-env  = options.delete(:env)     || 'development'
+# setup environment before loading any app library
+ENV['RACK_ENV'] = options.delete(:env) || 'development'
+
+app    = Rack::Builder.parse_file(options.delete(:rackup) || 'config.ru').first
+size   = options.delete(:workers) || 2
+master = Pastry.new(size, app, options)
 
-ENV['RACK_ENV'] = env
-Pastry.new(size, app, options).start
+master.parse_config(ARGV.shift) if ARGV.first && File.exists?(ARGV.first)
+master.start

data/lib/pastry.rb

@@ -2,45 +2,80 @@ require 'fileutils'
 require 'logger'
 require 'socket'
 require 'thin'
+require 'thin/server'
 
 class Pastry
-  attr_reader :pool, :unix, :host, :port, :pidfile, :logfile, :daemon
+  # have defaults
+  attr_accessor :pool, :host, :port, :queue, :max_connections, :timeout, :daemonize, :pidfile
+
+  # no defaults
+  attr_accessor :name, :socket, :logfile, :start_command, :cwd
 
   def initialize pool, app, options = {}
-    @pool    = pool
-    @app     = app
-    @host    = options.fetch :host,       '127.0.0.1'
-    @unix    = options.fetch :socket,     nil
-    @logfile = options.fetch :logfile,    nil
-    @daemon  = options.fetch :daemonize,  false
-    @pidfile = options.fetch :pidfile,    '/tmp/pastry.pid'
-    @name    = options.fetch :name,       nil
+    @pool             = pool
+    @app              = app
+    @host             = options.fetch :host,            '127.0.0.1'
+    @port             = options.fetch :port,            3000
+    @queue            = options.fetch :queue,           1024
+    @max_connections  = options.fetch :max_connections, 1024
+    @timeout          = options.fetch :timeout,           30
+    @daemonize        = options.fetch :daemonize,       false
+    @pidfile          = options.fetch :pidfile,         '/tmp/pastry.pid'
+    @name             = options.fetch :name,            nil
+    @socket           = options.fetch :socket,          nil
+    @logfile          = options.fetch :logfile,         nil
+    @start_command    = options.fetch :start_command,   nil
+    @cwd              = File.expand_path(ENV['PWD'] || Dir.pwd)
+
+    @before_fork      = nil
+    @after_fork       = nil
+  end
+
+  def before_fork &block
+    raise ArgumentError, 'missing callback' unless block
+    @before_fork = block
+  end
+
+  def after_fork &block
+    raise ArgumentError, 'missing callback' unless block
+    @after_fork = block
+  end
 
-    @port    = options.fetch(:port,    3000).to_i
-    @queue   = options.fetch(:queue,   1024).to_i
-    @maxconn = options.fetch(:maxconn, 1024).to_i
-    @timeout = options.fetch(:timeout,   30).to_i
+  def parse_config file
+    instance_eval File.read(file)
   end
 
   def start
+    do_sanity_checks
     ensure_not_running!
-    Process.daemon if daemon
 
-    if daemon || logfile
+    Process.daemon(true, true) if daemonize
+
+    if daemonize || logfile
       STDOUT.reopen(logfile || '/tmp/pastry.log', 'a')
       STDERR.reopen(logfile || '/tmp/pastry.log', 'a')
       STDOUT.sync = true
       STDERR.sync = true
+      STDOUT.binmode
+      STDERR.binmode
     end
 
     start!
   end
 
+  private
+
+  attr_accessor :pids
+
+  def do_sanity_checks
+    %w(port queue max_connections timeout).each {|var| send("#{var}=", send(var).to_i)}
+  end
+
   def ensure_not_running!
     if File.exists?(pidfile) && pid = File.read(pidfile).to_i
       running = Process.kill(0, pid) rescue nil
       raise "already running with pid #{pid}" if running
-      FileUtils.rm_f(pidfile)
+      FileUtils.rm_f([pidfile, socket.to_s])
     end
   end
 
@@ -48,62 +83,150 @@ class Pastry
     File.open(pidfile, 'w') {|fh| fh.write(Process.pid)}
   end
 
-  def name
-    '%s master' % (@name || 'pastry')
+  def master_name
+    '%s master' % (name || 'pastry')
   end
 
   def motd
-    "starting #{name} with #{pool} minions listening on #{unix ? 'socket %s ' % unix : 'port %d' % port}"
+    "starting #{master_name} with #{pool} minions listening on #{socket ? 'socket %s ' % socket : 'port %d' % port}"
+  end
+
+  def logger
+    @logger ||= Logger.new(logfile || (daemonize ? '/tmp/pastry.log' : $stdout), 0)
   end
 
   def start!
     create_pidfile
-    server = unix ? UNIXServer.new(unix) : TCPServer.new(host, port)
+    server   = ENV['PASTRY_FD'] && Socket.for_fd(ENV['PASTRY_FD'].to_i)
+    server ||= socket ? UNIXServer.new(socket) : TCPServer.new(host, port)
 
-    server.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true) unless unix
+    server.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true) unless socket
+    server.fcntl(Fcntl::F_SETFD, server.fcntl(Fcntl::F_GETFD) | Fcntl::FD_CLOEXEC)
     server.fcntl(Fcntl::F_SETFL, server.fcntl(Fcntl::F_GETFL) | Fcntl::O_NONBLOCK)
+    server.autoclose = false
 
-    server.listen(@queue)
+    server.listen(queue)
     server.extend(PastryServer)
 
     server.app = @app
-    logger     = Logger.new(logfile || (daemon ? '/tmp/pastry.log' : $stdout), 0)
-    options    = {timeout: @timeout, maximum_connections: @maxconn}
-
     logger.info motd
 
-    $0         = name if @name
+    # make this world readable.
+    FileUtils.chmod(0777, socket) if socket
+
+    # pre-fork cleanups, let user cleanup any leaking fds.
+    @before_fork && @before_fork.call
+
+    $0         = "#{name} master (started: #{Time.now})" if name
     @running   = true
-    pids       = pool.times.map {|n| run(server, options, n) }
+    @pids      = pool.times.map {|idx| run(server, idx) }
 
     Signal.trap('CHLD') do
       if @running
-        died  = pids.reject {|pid| Process.kill(0, pid) rescue nil}
-        died.each do |pid|
-          logger.info "process #{pid} died, starting a new one"
-          idx       = pids.index(pid)
-          pids[idx] = run(server, options, idx)
+        Thread.exclusive do
+          died = pids.select {|pid| Process.waitpid(pid, Process::WNOHANG) rescue 0} # find dead pids
+          died = died.reject {|pid| Process.kill(0, pid) rescue nil} # make sure what's dead is dead
+
+          died.each do |pid|
+            logger.info "process #{pid} died, starting a new one"
+            if idx = pids.index(pid)
+              pids[idx] = run(server, idx)
+            end
+          end
         end
       end
     end
 
-    %w(INT TERM HUP).each do |signal|
+    signals =  %w(INT TERM QUIT)
+    signals << %q(HUP) unless start_command
+
+    signals.each do |signal|
       Signal.trap(signal) do
         @running = false
         logger.info "caught #{signal}, closing time for the bakery -- no more pastries!"
-        pids.each {|pid| Process.kill(signal, pid) rescue nil}
-        exit
+        stop_workers(signal)
+        FileUtils.rm_f([pidfile, socket.to_s])
+        Kernel.exit!
       end
     end
 
-    at_exit { FileUtils.rm_f(pidfile); FileUtils.rm_f(unix.to_s) }
+    Signal.trap('HUP') { graceful_restart(server) } if start_command
     Process.waitall rescue nil
   end
 
-  def run server, options, worker
+  def graceful_restart server
+    @running = false
+    logger.info "caught SIGHUP, restarting gracefully"
+
+    FileUtils.mv pidfile, "#{pidfile}.old"
+
+    pair = UNIXSocket.pair
+    data = Socket::AncillaryData.unix_rights(server)
+    pid  = fork do
+      mesg, addr, rflags, *controls = pair[0].recvmsg(scm_rights: true)
+      ENV['PASTRY_FD'] = controls.first.unix_rights[0].fileno.to_s
+      pair.each(&:close)
+      Dir.chdir(cwd) do
+        Kernel.exec(start_command)
+      end
+    end
+
+    pair[1].sendmsg "*", 0, nil, data
+    pair.each(&:close)
+
+    # TODO signal parent that all is good and the new master is good to roll on its own ?
+    #      1. the new master failed to start
+    #      2. something in the after_fork bit crapped out
+    #      3. something else in the new code barfs during request
+    #
+    # A way to do this is provide pastry with a test route to hit after spawning the new master.
+    Process.detach(pid)
+
+    begin
+      Timeout.timeout(timeout) { sleep 0.5 until File.exists?(pidfile) }
+    rescue Timeout::Error => e
+      Process.kill('TERM', pid) rescue nil
+      logger.error "new master failed to spawn within #{timeout} secs, check logs"
+      @running = true
+      FileUtils.mv "#{pidfile}.old", pidfile
+    else
+      # finish exiting requests
+      stop_workers('HUP')
+      server.close
+      FileUtils.rm_f "#{pidfile}.old"
+      Kernel.exit
+    end
+  end
+
+  def stop_workers signal
+    logger.info "stopping workers"
+    pids.each {|pid| Process.kill(signal, pid) rescue nil}
+    return if signal == 'KILL'
+
+    logger.info "waiting up to #{timeout} seconds"
+    begin
+      Timeout.timeout(timeout) do
+        alive = pids
+        until alive.empty?
+          sleep 0.1
+          alive = pids.reject {|pid| Process.waitpid(pid, Process::WNOHANG) rescue 0}
+        end
+      end
+    rescue Timeout::Error => e
+      logger.info "killing stray pastry chefs with butcher knife (SIGKILL)"
+      pids.each {|pid| Process.kill('KILL', pid) rescue nil}
+    end
+
+    logger.info "all stop - ok"
+  end
+
+  def run server, worker
     fork do
-      $0 = "#{@name ? "%s worker" % @name : "pastry chef"} #{worker} (started: #{Time.now})"
-      EM.run { Backend.new(options).start(server) }
+      @after_fork && @after_fork.call(Process.pid, worker)
+      $0 = "#{name ? "%s worker" % name : "pastry chef"} #{worker} (started: #{Time.now})"
+      EM.epoll
+      EM.set_descriptor_table_size(max_connections)
+      EM.run { Backend.new.start(server) }
     end
   end
 
@@ -112,24 +235,38 @@ class Pastry
   end
 
   class Backend < Thin::Backends::Base
-    def initialize options = {}
-      super()
-      options.each {|key, value| send("#{key}=", value)}
-    end
-
     def start server
       @stopping = false
       @running  = true
       @server   = server
 
-      config
       trap_signals!
-
-      EM.attach_server_socket(server, Thin::Connection, &method(:initialize_connection))
+      @signature = EM.attach_server_socket(server, Thin::Connection, &method(:initialize_connection))
     end
 
     def trap_signals!
-      %w(INT TERM HUP CHLD).each {|signal| Signal.trap(signal) { exit }}
+      # ignore SIGCHLD, can be overriden by app.
+      Signal.trap('CHLD', 'IGNORE')
+
+      # close connections and stop gracefully
+      %w(HUP QUIT).each do |signal|
+        Signal.trap(signal) do
+          stop
+          EM.add_periodic_timer(1) { Kernel.exit! if @connections.empty? }
+        end
+      end
+
+      # hard stop, discard requests and quit
+      %w(INT TERM).each do |signal|
+        Signal.trap(signal) do
+          stop!
+          Kernel.exit!
+        end
+      end
+    end
+
+    def disconnect
+      EM.stop_server(@signature)
     end
   end # Backend
 end # Pastry

metadata

@@ -1,78 +1,68 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: pastry
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 1
-  - 3
-  version: 0.1.3
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Bharanee Rathna
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-11-27 00:00:00 +11:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: thin
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+date: 2013-03-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
   type: :runtime
-  version_requirements: *id001
+  name: thin
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+    none: false
+  prerelease: false
 description: thin runner that forks and supports binding to single socket
 email: deepfryed@gmail.com
-executables: 
+executables:
 - pastry
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
+files:
 - lib/pastry.rb
 - README.rdoc
 - bin/pastry
-has_rdoc: true
 homepage: http://github.com/deepfryed/pastry
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      hash: 3318781981847808244
+      version: '0'
+      segments:
       - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: thin runner that supports forking
 test_files: []
-