pipemaster 0.3.1 → 0.4.0

data/CHANGELOG

@@ -0,0 +1,7 @@
+0.4.0 (2010-02-19)
+* Added: pipe command (client).
+* Added: Pipemaster::Client gets pipe and capture methods, global address setting.
+* Added: Command line client gets --tty option, when used, captures stdin when running in terminal.
+* Added: Configurator command method now takes proc/method as second argument.
+* Changed: Unicorn is no longer a dependency, code merged instead.
+* Fixed: USR2 signal now properly restarts server.

data/Gemfile

@@ -1,5 +1,4 @@
 source "http://gemcutter.org"
 gem "gemcutter"
 gem "rake"
-gem "unicorn", "0.96.1"
 gem "yard"

data/README.rdoc

@@ -48,8 +48,8 @@ Step 3:  For a new shell, execute a command:
 
 == License
 
-Pipemaster is copyright of Assaf Arkin.  It is based on the awesome Unicorn Web
-server and therefore uses the same license.
+Pipemaster is copyright of Assaf Arkin.  It is heavily based on the awesome
+Unicorn Web server and therefore uses the same license.
 
 Unicorn is copyright 2009 by all contributors (see logs in git).
 It is based on Mongrel and carries the same license.

data/Rakefile

@@ -13,9 +13,13 @@ task :push do
   sh "gem push #{spec.name}-#{spec.version}.gem"
 end
 
-desc "Install #{spec.name} locally"
-task :install do
+desc "Build #{spec.name}-#{spec.version}.gem"
+task :build=>:test do
   sh "gem build #{spec.name}.gemspec"
+end
+
+desc "Install #{spec.name} locally"
+task :install=>:build do
   sudo = "sudo" unless File.writable?( Gem::ConfigMap[:bindir])
   sh "#{sudo} gem install #{spec.name}-#{spec.version}.gem"
 end

data/bin/pipe

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# -*- encoding: binary -*-
+require "pipemaster/client"
+require "optparse"
+
+ENV["PIPE_ENV"] ||= "development"
+address = nil
+tty = false
+
+opts = OptionParser.new("", 24, '  ') do |opts|
+  opts.banner = "Usage: #{File.basename($0)} [options] command [args]\n"
+
+  opts.on("-t", "--tty", "read input from terminal (default: false)") do |t|
+    tty = t ? true : false
+  end
+
+  opts.on("-s", "--socket {HOST:PORT|PATH}",
+          "communicate on HOST:PORT or PATH",
+          "(default: #{Pipemaster::DEFAULT_LISTEN})") do |a|
+    address = a
+  end
+
+  opts.on("-h", "--help", "Show this message") do
+    puts opts.to_s.gsub(/^.*DEPRECATED.*$/s, '')
+    exit
+  end
+
+  opts.on("-v", "--version", "Show version") do
+    puts "Pipemaster v#{Pipemaster::VERSION}"
+    exit
+  end
+
+  opts.parse! ARGV
+end
+
+input = $stdin if tty || !$stdin.isatty 
+exit Pipemaster::Client.new(address).capture(input, $stdout).run(*ARGV)

data/bin/pipemaster

@@ -1,6 +1,7 @@
-#!/this/will/be/overwritten/or/wrapped/anyways/do/not/worry/ruby
+#!/usr/bin/env ruby
 # -*- encoding: binary -*-
-require "pipemaster"
+require "pipemaster/server"
+require "pipemaster/launcher"
 require "optparse"
 
 ENV["PIPE_ENV"] ||= "development"
@@ -8,13 +9,12 @@ server = false
 daemonize = false
 listeners = []
 options = { :listeners => listeners }
+tty = false
 
 opts = OptionParser.new("", 24, '  ') do |opts|
-  opts.banner = "Usage: #{File.basename($0)}\n" \
-                "[ruby options] [pipemaster options] command [args]\n" \
-                "[ruby options] [pipemaster options] --server [Pipefile]"
+  opts.banner = "Usage: #{File.basename($0)} [ruby options] [pipemaster options] [Pipefile]"
 
-  opts.separator "Ruby options:"
+  opts.separator "\nRuby options:"
 
   lineno = 1
   opts.on("-e", "--eval LINE", "evaluate a LINE of code") do |line|
@@ -40,61 +40,41 @@ opts = OptionParser.new("", 24, '  ') do |opts|
     require library
   end
 
-  opts.separator "Pipemaster options:"
-
-  opts.on("-s", "--socket {HOST:PORT|PATH}",
-          "communicate on HOST:PORT or PATH",
-          "for server, this may be specified multiple times",
-          "(default: #{Pipemaster::DEFAULT_LISTEN})") do |address|
-    listeners << address
-  end
+  opts.separator "\nPipemaster options:"
 
   opts.on("-E", "--env ENVIRONMENT",
           "use ENVIRONMENT for defaults (default: development)") do |e|
     ENV["PIPE_ENV"] = e
   end
 
-  opts.on("-S", "--server", "run as server (default: client)") do |s|
-    server = s ? true : false
-  end
-
   opts.on("-D", "--daemonize", "run daemonized in the background") do |d|
     daemonize = d ? true : false
   end
 
-  opts.separator "Common options:"
+  opts.on("-s", "--socket {HOST:PORT|PATH}",
+          "communicate on HOST:PORT or PATH",
+          "for server, this may be specified multiple times",
+          "(default: #{Pipemaster::DEFAULT_LISTEN})") do |address|
+    listeners << address
+  end
 
-  opts.on_tail("-h", "--help", "Show this message") do
+  opts.on("-h", "--help", "Show this message") do
     puts opts.to_s.gsub(/^.*DEPRECATED.*$/s, '')
     exit
   end
 
-  opts.on_tail("-v", "--version", "Show version") do
+  opts.on("-v", "--version", "Show version") do
     puts "Pipemaster v#{Pipemaster::VERSION}"
     exit
   end
 
+
   opts.parse! ARGV
 end
 
-if server
-
-  gem "unicorn"
-  require "pipemaster/server"
-  require "unicorn/launcher"
-  config = ARGV[0] || "Pipefile"
-  abort "configuration file #{config} not found" unless File.exist?(config)
-  options[:config_file] = config
-  options[:working_directory] = File.dirname(config)
-  Unicorn::Launcher.daemonize!(options) if daemonize
-  Pipemaster.run(options)
-
-else
-
-  require "pipemaster/client"
-  client = Pipemaster::Client.new(listeners.first)
-  client.input = $stdin unless $stdin.isatty
-  client.output = $stdout
-  exit client.request(*ARGV)
-
-end
+config = ARGV[0] || "Pipefile"
+abort "configuration file #{config} not found" unless File.exist?(config)
+options[:config_file] = config
+options[:working_directory] = File.dirname(config)
+Pipemaster::Launcher.daemonize!(options)
+Pipemaster::Server.run(options)

data/lib/pipemaster/client.rb

@@ -1,24 +1,44 @@
 require "socket"
+require "pipemaster"
 
 module Pipemaster
   # Pipemaster client.  Use this to send commands to the Pipemaster server.
   #
   # For example:
   #   c = Pipemaster::Client.new
-  #   c.request "motd"
+  #   c.run :motd
   #   puts c.output.string
   #   => "Toilet out of order please use floor below."
   #
   #   c = Pipemaster::Client.new(9988)
   #   c.input = File.open("image.png")
   #   c.output = File.open("thumbnail.png", "w")
-  #   c.request "transform", "thumbnail"
+  #   c.run :transform, "thumbnail"
   class Client
     BUFFER_SIZE = 16 * 1024
 
-    # Address can be "x.x.x.x:port", ":port" or UNIX socket file name.
-    def initialize(address)
-      @address = expand_addr(address || DEFAULT_LISTEN)
+    class << self
+      # Pipe stdin/stdout into the command.  For example:
+      #   exit Pipemaster.pipe(:echo)
+      def pipe(command, *args)
+        new.capture($stdin, $stdout).run(command, *args)
+      end
+
+      # Address to use by default for new clients.  Leave as nil to use the
+      # default (Pipemaster::DEFAULT_LISTEN).
+      attr_accessor :address
+    end
+
+    # Creates a new client.  Accepts optional address.  If missing, uses the
+    # global address (see Pipemaster::Client::address).  Address can be port
+    # number, hostname (default port), "host:port" or socket file name.
+    #
+    # Examples:
+    #   Pipemaster::Client.new 7890
+    #   Pipemaster::Client.new "localhost:7890"
+    #   Pipemaster::Client.new
+    def initialize(address = nil)
+      @address = expand_addr(address || Client.address || DEFAULT_LISTEN)
     end
 
     # Set this to supply an input stream (for commands that read from $stdin).
@@ -28,11 +48,18 @@ module Pipemaster
     # StringIO).
     attr_accessor :output
 
+    # Captures input and output.  For example:
+    #   Pipemaster.new(7890).capture($stdin, $stdout).run(:echo)
+    def capture(input, output)
+      self.input, self.output = input, output
+      self
+    end
+
     # Make a request.  First argument is the command name.  All other arguments
     # are optional.  Returns the exit code (usually 0).  Will raise IOError if
     # it can't talk to the server, or the server closed the connection
     # prematurely.
-    def request(command, *args)
+    def run(command, *args)
       # Connect and send arguments.
       socket = connect(@address)
       socket.sync = true

data/lib/pipemaster/configurator.rb

@@ -1,13 +1,15 @@
 require 'socket'
 require 'logger'
+require 'pipemaster/util'
 
 module Pipemaster
 
   # Implements a simple DSL for configuring a Pipemaster server.
-  class Configurator < Unicorn::Configurator
+  class Configurator < Struct.new(:set, :config_file)
 
     # Default settings for Pipemaster
     DEFAULTS = {
+      :timeout => 60,
       :logger => Logger.new($stderr),
       :after_fork => lambda { |server, worker|
           server.logger.info("spawned pid=#{$$}")
@@ -19,8 +21,7 @@ module Pipemaster
           server.logger.info("forked child re-executing...")
         },
       :pid => nil,
-      :commands => {},
-      :timeout => 60
+      :commands => {}
     }
 
     def initialize(defaults = {}) #:nodoc:
@@ -35,20 +36,52 @@ module Pipemaster
       reload
     end
 
+    def reload #:nodoc:
+      instance_eval(File.read(config_file), config_file) if config_file
+
+      # working_directory binds immediately (easier error checking that way),
+      # now ensure any paths we changed are correctly set.
+      [ :pid, :stderr_path, :stdout_path ].each do |var|
+        String === (path = set[var]) or next
+        path = File.expand_path(path)
+        test(?w, path) || test(?w, File.dirname(path)) or \
+              raise ArgumentError, "directory for #{var}=#{path} not writable"
+      end
+    end
+
+    def commit!(server, options = {}) #:nodoc:
+      skip = options[:skip] || []
+      set.each do |key, value|
+        value == :unset and next
+        skip.include?(key) and next
+        server.__send__("#{key}=", value)
+      end
+    end
+
+    def [](key) # :nodoc:
+      set[key]
+    end
+
     # Sets object to the +new+ Logger-like object.  The new logger-like
     # object must respond to the following methods:
     #  +debug+, +info+, +warn+, +error+, +fatal+, +close+
     def logger(new)
-      super
+      %w(debug info warn error fatal close).each do |m|
+        new.respond_to?(m) and next
+        raise ArgumentError, "logger=#{new} does not respond to method=#{m}"
+      end
+
+      set[:logger] = new
     end
 
+    # Set commands from a hash (name=>proc).
     def commands(hash)
       set[:commands] = hash
     end
 
     # Defines a command.
-    def command(name, &block)
-      set[:commands][name.to_sym] = block
+    def command(name, a_proc = nil, &block)
+      set[:commands][name.to_sym] = a_proc || block
     end
 
     autoload :Etc, 'etc'
@@ -59,7 +92,7 @@ module Pipemaster
       # capabilities.  Let the caller handle any and all errors.
       uid = Etc.getpwnam(user).uid
       gid = Etc.getgrnam(group).gid if group
-      Unicorn::Util.chown_logs(uid, gid)
+      Pipemaster::Util.chown_logs(uid, gid)
       if gid && Process.egid != gid
         Process.initgroups(user, gid)
         Process::GID.change_privilege(gid)
@@ -116,10 +149,15 @@ module Pipemaster
 
     # Does nothing interesting for now.
     def timeout(seconds)
+      # Not implemented yet.
     end
 
-    # Does nothing interesting for now.
-    def worker_processes(nr)
+    # This is for internal API use only, do not use it in your Pipemaster
+    # config file.  Use listen instead.
+    def listeners(addresses) # :nodoc:
+      Array === addresses or addresses = Array(addresses)
+      addresses.map! { |addr| expand_addr(addr) }
+      set[:listeners] = addresses
     end
 
     # Adds an +address+ to the existing listener set.
@@ -195,7 +233,26 @@ module Pipemaster
     #
     # Default: 0 (world read/writable)
     def listen(address, opt = {})
-      super
+      address = expand_addr(address)
+      if String === address
+        [ :umask, :backlog, :sndbuf, :rcvbuf, :tries ].each do |key|
+          value = opt[key] or next
+          Integer === value or
+            raise ArgumentError, "not an integer: #{key}=#{value.inspect}"
+        end
+        [ :tcp_nodelay, :tcp_nopush ].each do |key|
+          (value = opt[key]).nil? and next
+          TrueClass === value || FalseClass === value or
+            raise ArgumentError, "not boolean: #{key}=#{value.inspect}"
+        end
+        unless (value = opt[:delay]).nil?
+          Numeric === value or
+            raise ArgumentError, "not numeric: delay=#{value.inspect}"
+        end
+        set[:listener_opts][address].merge!(opt)
+      end
+
+      set[:listeners] << address
     end
 
     # Sets the +path+ for the PID file of the Pipemaster master process
@@ -218,9 +275,54 @@ module Pipemaster
       set_path(:stdout_path, path)
     end
 
+    # expands "unix:path/to/foo" to a socket relative to the current path
+    # expands pathnames of sockets if relative to "~" or "~username"
+    # expands "*:port and ":port" to "0.0.0.0:port"
+    def expand_addr(address) #:nodoc
+      return "0.0.0.0:#{address}" if Integer === address
+      return address unless String === address
+
+      case address
+      when %r{\Aunix:(.*)\z}
+        File.expand_path($1)
+      when %r{\A~}
+        File.expand_path(address)
+      when %r{\A(?:\*:)?(\d+)\z}
+        "0.0.0.0:#$1"
+      when %r{\A(.*):(\d+)\z}
+        # canonicalize the name
+        packed = Socket.pack_sockaddr_in($2.to_i, $1)
+        Socket.unpack_sockaddr_in(packed).reverse!.join(':')
+      else
+        address
+      end
+    end
+
   private
 
-    def preload_app(bool)
+    def set_path(var, path) #:nodoc:
+      case path
+      when NilClass, String
+        set[var] = path
+      else
+        raise ArgumentError
+      end
+    end
+
+    def set_hook(var, my_proc, req_arity = 2) #:nodoc:
+      case my_proc
+      when Proc
+        arity = my_proc.arity
+        (arity == req_arity) or \
+          raise ArgumentError,
+                "#{var}=#{my_proc.inspect} has invalid arity: " \
+                "#{arity} (need #{req_arity})"
+      when NilClass
+        my_proc = DEFAULTS[var]
+      else
+        raise ArgumentError, "invalid type: #{var}=#{my_proc.inspect}"
+      end
+      set[var] = my_proc
     end
 
   end

data/lib/pipemaster/launcher.rb

@@ -0,0 +1,67 @@
+# -*- encoding: binary -*-
+
+$stdout.sync = $stderr.sync = true
+$stdin.binmode
+$stdout.binmode
+$stderr.binmode
+
+require "pipemaster/configurator"
+
+module Pipemaster
+  class Launcher
+
+    # We don't do a lot of standard daemonization stuff:
+    #   * umask is whatever was set by the parent process at startup
+    #     and can be set in config.ru and config_file, so making it
+    #     0000 and potentially exposing sensitive log data can be bad
+    #     policy.
+    #   * don't bother to chdir("/") here since unicorn is designed to
+    #     run inside APP_ROOT.  Pipemaster will also re-chdir() to
+    #     the directory it was started in when being re-executed
+    #     to pickup code changes if the original deployment directory
+    #     is a symlink or otherwise got replaced.
+    def self.daemonize!(options = nil)
+      $stdin.reopen("/dev/null")
+
+      # We only start a new process group if we're not being reexecuted
+      # and inheriting file descriptors from our parent
+      unless ENV['PIPEMASTER_FD']
+        if options
+          # grandparent - reads pipe, exits when master is ready
+          #  \_ parent  - exits immediately ASAP
+          #      \_ unicorn master - writes to pipe when ready
+
+          rd, wr = IO.pipe
+          grandparent = $$
+          if fork
+            wr.close # grandparent does not write
+          else
+            rd.close # unicorn master does not read
+            Process.setsid
+            exit if fork # parent dies now
+          end
+
+          if grandparent == $$
+            # this will block until HttpServer#join runs (or it dies)
+            master_pid = (rd.readpartial(16) rescue nil).to_i
+            unless master_pid > 1
+              warn "master failed to start, check stderr log for details"
+              exit!(1)
+            end
+            exit 0
+          else # unicorn master process
+            options[:ready_pipe] = wr
+          end
+        else # backwards compat
+          exit if fork
+          Process.setsid
+          exit if fork
+        end
+        # $stderr/$stderr can/will be redirected separately in the Pipemaster config
+        Pipemaster::Configurator::DEFAULTS[:stderr_path] = "/dev/null"
+        Pipemaster::Configurator::DEFAULTS[:stdout_path] = "/dev/null"
+      end
+    end
+
+  end
+end

data/lib/pipemaster/server.rb

@@ -1,16 +1,71 @@
-require "unicorn"
+require "pipemaster"
 require "pipemaster/worker"
+require 'pipemaster/util'
 require "pipemaster/configurator"
+require "pipemaster/socket_helper"
 
 module Pipemaster
 
-  class << self
-    def run(options = {})
-      Server.new(options).start.join
-    end
-  end
+  class Server < Struct.new(:listener_opts, :timeout, :logger,
+                            :before_fork, :after_fork, :before_exec,
+                            :pid, :reexec_pid, :init_listeners,
+                            :master_pid, :config, :ready_pipe)
+
+    include SocketHelper
+
+    # prevents IO objects in here from being GC-ed
+    IO_PURGATORY = []
+
+    # all bound listener sockets
+    LISTENERS = []
+
+    # This hash maps PIDs to Workers
+    WORKERS = {}
+
+    # We populate this at startup so we can figure out how to reexecute
+    # and upgrade the currently running instance of Pipemaster
+    # This Hash is considered a stable interface and changing its contents
+    # will allow you to switch between different installations of Pipemaster
+    # or even different installations of the same applications without
+    # downtime.  Keys of this constant Hash are described as follows:
+    #
+    # * 0 - the path to the pipemaster executable
+    # * :argv - a deep copy of the ARGV array the executable originally saw
+    # * :cwd - the working directory of the application, this is where
+    # you originally started Pipemaster.
+    #
+    # The following example may be used in your Pipemaster config file to
+    # change your working directory during a config reload (HUP) without
+    # upgrading or restarting:
+    #
+    #   Dir.chdir(Pipemaster::Server::START_CTX[:cwd] = path)
+    #
+    # To change your Pipemaster executable to a different path without downtime,
+    # you can set the following in your Pipemaster config file, HUP and then
+    # continue with the traditional USR2 + QUIT upgrade steps:
+    #
+    #   Pipemaster::Server::START_CTX[0] = "/home/bofh/1.9.2/bin/pipemaster"
+    START_CTX = {
+      :argv => ARGV.map { |arg| arg.dup },
+      :cwd => lambda {
+          # favor ENV['PWD'] since it is (usually) symlink aware for
+          # Capistrano and like systems
+          begin
+            a = File.stat(pwd = ENV['PWD'])
+            b = File.stat(Dir.pwd)
+            a.ino == b.ino && a.dev == b.dev ? pwd : Dir.pwd
+          rescue
+            Dir.pwd
+          end
+        }.call,
+      0 => $0.dup,
+    }
 
-  class Server < Unicorn::HttpServer
+    class << self
+      def run(options = {})
+        Server.new(options).start.join
+      end
+    end
 
     def initialize(options = {})
       self.reexec_pid = 0
@@ -38,22 +93,22 @@ module Pipemaster
       trap(:CHLD) { reap_all_workers }
       trap :USR1 do
         logger.info "master reopening logs..."
-        Unicorn::Util.reopen_logs
+        Pipemaster::Util.reopen_logs
         logger.info "master done reopening logs"
       end
       reloaded = nil
       trap(:HUP)  { reloaded = true ; load_config! }
       trap(:USR2) { reexec }
 
-      begin
-        config_listeners = config[:listeners].dup
-        if config_listeners.empty? && LISTENERS.empty?
-          config_listeners << DEFAULT_LISTEN
-          init_listeners << DEFAULT_LISTEN
-          START_CTX[:argv] << "-s#{DEFAULT_LISTEN}"
-        end
-        config_listeners.each { |addr| listen(addr) }
+      config_listeners = config[:listeners].dup
+      if config_listeners.empty? && LISTENERS.empty?
+        config_listeners << DEFAULT_LISTEN
+        init_listeners << DEFAULT_LISTEN
+        START_CTX[:argv] << "-s#{DEFAULT_LISTEN}"
+      end
+      config_listeners.each { |addr| listen(addr) }
 
+      begin
         reloaded = false
         while selected = Kernel.select(LISTENERS)
           selected.first.each do |socket|
@@ -81,6 +136,139 @@ module Pipemaster
       unlink_pid_safe(pid) if pid
     end
 
+    # replaces current listener set with +listeners+.  This will
+    # close the socket if it will not exist in the new listener set
+    def listeners=(listeners)
+      cur_names, dead_names = [], []
+      listener_names.each do |name|
+        if ?/ == name[0]
+          # mark unlinked sockets as dead so we can rebind them
+          (File.socket?(name) ? cur_names : dead_names) << name
+        else
+          cur_names << name
+        end
+      end
+      set_names = listener_names(listeners)
+      dead_names.concat(cur_names - set_names).uniq!
+
+      LISTENERS.delete_if do |io|
+        if dead_names.include?(sock_name(io))
+          IO_PURGATORY.delete_if do |pio|
+            pio.fileno == io.fileno && (pio.close rescue nil).nil? # true
+          end
+          (io.close rescue nil).nil? # true
+        else
+          set_server_sockopt(io, listener_opts[sock_name(io)])
+          false
+        end
+      end
+
+      (set_names - cur_names).each { |addr| listen(addr) }
+    end
+
+    # returns an array of string names for the given listener array
+    def listener_names(listeners = LISTENERS)
+      listeners.map { |io| sock_name(io) }
+    end
+
+    def stdout_path=(path); redirect_io($stdout, path); end
+    def stderr_path=(path); redirect_io($stderr, path); end
+
+    def redirect_io(io, path)
+      File.open(path, 'ab') { |fp| io.reopen(fp) } if path
+      io.sync = true
+    end
+
+    # sets the path for the PID file of the master process
+    def pid=(path)
+      if path
+        if x = valid_pid?(path)
+          return path if pid && path == pid && x == $$
+          raise ArgumentError, "Already running on PID:#{x} " \
+                               "(or pid=#{path} is stale)"
+        end
+      end
+      unlink_pid_safe(pid) if pid
+
+      if path
+        fp = begin
+          tmp = "#{File.dirname(path)}/#{rand}.#$$"
+          File.open(tmp, File::RDWR|File::CREAT|File::EXCL, 0644)
+        rescue Errno::EEXIST
+          retry
+        end
+        fp.syswrite("#$$\n")
+        File.rename(fp.path, path)
+        fp.close
+      end
+      super(path)
+    end
+
+    # unlinks a PID file at given +path+ if it contains the current PID
+    # still potentially racy without locking the directory (which is
+    # non-portable and may interact badly with other programs), but the
+    # window for hitting the race condition is small
+    def unlink_pid_safe(path)
+      (File.read(path).to_i == $$ and File.unlink(path)) rescue nil
+    end
+
+    # returns a PID if a given path contains a non-stale PID file,
+    # nil otherwise.
+    def valid_pid?(path)
+      wpid = File.read(path).to_i
+      wpid <= 0 and return nil
+      begin
+        Process.kill(0, wpid)
+        wpid
+      rescue Errno::ESRCH
+        # don't unlink stale pid files, racy without non-portable locking...
+      end
+      rescue Errno::ENOENT
+    end
+
+    def proc_name(tag)
+      $0 = ([ File.basename(START_CTX[0]), tag
+            ]).concat(START_CTX[:argv]).join(' ')
+    end
+
+    # add a given address to the +listeners+ set, idempotently
+    # Allows workers to add a private, per-process listener via the
+    # after_fork hook.  Very useful for debugging and testing.
+    # +:tries+ may be specified as an option for the number of times
+    # to retry, and +:delay+ may be specified as the time in seconds
+    # to delay between retries.
+    # A negative value for +:tries+ indicates the listen will be
+    # retried indefinitely, this is useful when workers belonging to
+    # different masters are spawned during a transparent upgrade.
+    def listen(address, opt = {}.merge(listener_opts[address] || {}))
+      address = config.expand_addr(address)
+      return if String === address && listener_names.include?(address)
+
+      delay = opt[:delay] || 0.5
+      tries = opt[:tries] || 5
+      begin
+        io = bind_listen(address, opt)
+        unless TCPServer === io || UNIXServer === io
+          IO_PURGATORY << io
+          io = server_cast(io)
+        end
+        logger.info "listening on addr=#{sock_name(io)} fd=#{io.fileno}"
+        LISTENERS << io
+        io
+      rescue Errno::EADDRINUSE => err
+        logger.error "adding listener failed addr=#{address} (in use)"
+        raise err if tries == 0
+        tries -= 1
+        logger.error "retrying in #{delay} seconds " \
+                     "(#{tries < 0 ? 'infinite' : tries} tries left)"
+        sleep(delay)
+        retry
+      rescue => err
+        logger.fatal "error adding listener addr=#{address}"
+        raise err
+      end
+    end
+
     def kill_worker(signal, wpid)
       begin
         Process.kill(signal, wpid)
@@ -94,15 +282,15 @@ module Pipemaster
         loop do
           wpid, status = Process.waitpid2(-1, Process::WNOHANG)
           wpid or break
-          #if reexec_pid == wpid
-          #  logger.error "reaped #{status.inspect} exec()-ed"
-          #  self.reexec_pid = 0
-          #  self.pid = pid.chomp('.oldbin') if pid
-          #  proc_name 'master'
-          #else
+          if reexec_pid == wpid
+            logger.error "reaped #{status.inspect} exec()-ed"
+            self.reexec_pid = 0
+            self.pid = pid.chomp('.oldbin') if pid
+            proc_name 'master'
+          else
             worker = WORKERS.delete(wpid) rescue nil
             logger.info "reaped #{status.inspect} "
-          #end
+          end
         end
       rescue Errno::ECHILD
       end
@@ -114,7 +302,7 @@ module Pipemaster
         config[:listeners].replace(init_listeners)
         config.reload
         config.commit!(self)
-        Unicorn::Util.reopen_logs
+        Pipemaster::Util.reopen_logs
         logger.info "done reloading pipefile=#{config.config_file}"
       rescue => e
         logger.error "error reloading pipefile=#{config.config_file}: " \
@@ -122,6 +310,56 @@ module Pipemaster
       end
     end
 
+    def reexec
+      if reexec_pid > 0
+        begin
+          Process.kill(0, reexec_pid)
+          logger.error "reexec-ed child already running PID:#{reexec_pid}"
+          return
+        rescue Errno::ESRCH
+          self.reexec_pid = 0
+        end
+      end
+
+      if pid
+        old_pid = "#{pid}.oldbin"
+        prev_pid = pid.dup
+        begin
+          self.pid = old_pid  # clear the path for a new pid file
+        rescue ArgumentError
+          logger.error "old PID:#{valid_pid?(old_pid)} running with " \
+                       "existing pid=#{old_pid}, refusing rexec"
+          return
+        rescue => e
+          logger.error "error writing pid=#{old_pid} #{e.class} #{e.message}"
+          return
+        end
+      end
+
+      listener_fds = LISTENERS.map { |sock| sock.fileno }
+      LISTENERS.delete_if { |s| s.close rescue nil ; true }
+      self.reexec_pid = fork do
+        ENV['PIPEMASTER_FD'] = listener_fds.join(',')
+        Dir.chdir(START_CTX[:cwd])
+        cmd = [ START_CTX[0] ].concat(START_CTX[:argv])
+
+        # avoid leaking FDs we don't know about, but let before_exec
+        # unset FD_CLOEXEC, if anything else in the app eventually
+        # relies on FD inheritence.
+        (3..1024).each do |io|
+          next if listener_fds.include?(io)
+          io = IO.for_fd(io) rescue nil
+          io or next
+          IO_PURGATORY << io
+          io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+        end
+        logger.info "executing #{cmd.inspect} (in #{Dir.pwd})"
+        before_exec.call(self)
+        exec(*cmd)
+      end
+      proc_name 'master (old)'
+    end
+
     def process_request(socket, worker)
       trap(:QUIT) { exit }
       [:TERM, :INT].each { |sig| trap(sig) { exit! } }

data/lib/pipemaster/socket_helper.rb

@@ -0,0 +1,149 @@
+require 'socket'
+
+module Pipemaster
+  module SocketHelper
+    include Socket::Constants
+
+    # configure platform-specific options (only tested on Linux 2.6 so far)
+    case RUBY_PLATFORM
+    when /linux/
+      # from /usr/include/linux/tcp.h
+      TCP_DEFER_ACCEPT = 9 unless defined?(TCP_DEFER_ACCEPT)
+
+      # do not send out partial frames (Linux)
+      TCP_CORK = 3 unless defined?(TCP_CORK)
+    when /freebsd(([1-4]\..{1,2})|5\.[0-4])/
+      # Do nothing for httpready, just closing a bug when freebsd <= 5.4
+      TCP_NOPUSH = 4 unless defined?(TCP_NOPUSH) # :nodoc:
+    when /freebsd/
+      # do not send out partial frames (FreeBSD)
+      TCP_NOPUSH = 4 unless defined?(TCP_NOPUSH)
+
+      # Use the HTTP accept filter if available.
+      # The struct made by pack() is defined in /usr/include/sys/socket.h
+      # as accept_filter_arg
+      unless `/sbin/sysctl -nq net.inet.accf.http`.empty?
+        # set set the "httpready" accept filter in FreeBSD if available
+        # if other protocols are to be supported, this may be
+        # String#replace-d with "dataready" arguments instead
+        FILTER_ARG = ['httpready', nil].pack('a16a240')
+      end
+    end
+
+    def set_tcp_sockopt(sock, opt)
+
+      # highly portable, but off by default because we don't do keepalive
+      if defined?(TCP_NODELAY) && ! (val = opt[:tcp_nodelay]).nil?
+        sock.setsockopt(IPPROTO_TCP, TCP_NODELAY, val ? 1 : 0)
+      end
+
+      unless (val = opt[:tcp_nopush]).nil?
+        val = val ? 1 : 0
+        if defined?(TCP_CORK) # Linux
+          sock.setsockopt(IPPROTO_TCP, TCP_CORK, val)
+        elsif defined?(TCP_NOPUSH) # TCP_NOPUSH is untested (FreeBSD)
+          sock.setsockopt(IPPROTO_TCP, TCP_NOPUSH, val)
+        end
+      end
+
+      # No good reason to ever have deferred accepts off
+      if defined?(TCP_DEFER_ACCEPT)
+        sock.setsockopt(SOL_TCP, TCP_DEFER_ACCEPT, 1)
+      elsif defined?(SO_ACCEPTFILTER) && defined?(FILTER_ARG)
+        sock.setsockopt(SOL_SOCKET, SO_ACCEPTFILTER, FILTER_ARG)
+      end
+    end
+
+    def set_server_sockopt(sock, opt)
+      opt ||= {}
+
+      TCPSocket === sock and set_tcp_sockopt(sock, opt)
+
+      if opt[:rcvbuf] || opt[:sndbuf]
+        log_buffer_sizes(sock, "before: ")
+        sock.setsockopt(SOL_SOCKET, SO_RCVBUF, opt[:rcvbuf]) if opt[:rcvbuf]
+        sock.setsockopt(SOL_SOCKET, SO_SNDBUF, opt[:sndbuf]) if opt[:sndbuf]
+        log_buffer_sizes(sock, " after: ")
+      end
+      sock.listen(opt[:backlog] || 1024)
+      rescue => e
+        if respond_to?(:logger)
+          logger.error "error setting socket options: #{e.inspect}"
+          logger.error e.backtrace.join("\n")
+        end
+    end
+
+    def log_buffer_sizes(sock, pfx = '')
+      respond_to?(:logger) or return
+      rcvbuf = sock.getsockopt(SOL_SOCKET, SO_RCVBUF).unpack('i')
+      sndbuf = sock.getsockopt(SOL_SOCKET, SO_SNDBUF).unpack('i')
+      logger.info "#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}"
+    end
+
+    # creates a new server, socket. address may be a HOST:PORT or
+    # an absolute path to a UNIX socket.  address can even be a Socket
+    # object in which case it is immediately returned
+    def bind_listen(address = '0.0.0.0:8080', opt = {})
+      return address unless String === address
+
+      sock = if address[0] == ?/
+        if File.exist?(address)
+          if File.socket?(address)
+            if self.respond_to?(:logger)
+              logger.info "unlinking existing socket=#{address}"
+            end
+            File.unlink(address)
+          else
+            raise ArgumentError,
+                  "socket=#{address} specified but it is not a socket!"
+          end
+        end
+        old_umask = File.umask(opt[:umask] || 0)
+        begin
+          UNIXServer.new(address)
+        ensure
+          File.umask(old_umask)
+        end
+      elsif address =~ /^(\d+\.\d+\.\d+\.\d+):(\d+)$/
+        TCPServer.new($1, $2.to_i)
+      else
+        raise ArgumentError, "Don't know how to bind: #{address}"
+      end
+      set_server_sockopt(sock, opt)
+      sock
+    end
+
+    # Returns the configuration name of a socket as a string.  sock may
+    # be a string value, in which case it is returned as-is
+    # Warning: TCP sockets may not always return the name given to it.
+    def sock_name(sock)
+      case sock
+      when String then sock
+      when UNIXServer
+        Socket.unpack_sockaddr_un(sock.getsockname)
+      when TCPServer
+        Socket.unpack_sockaddr_in(sock.getsockname).reverse!.join(':')
+      when Socket
+        begin
+          Socket.unpack_sockaddr_in(sock.getsockname).reverse!.join(':')
+        rescue ArgumentError
+          Socket.unpack_sockaddr_un(sock.getsockname)
+        end
+      else
+        raise ArgumentError, "Unhandled class #{sock.class}: #{sock.inspect}"
+      end
+    end
+
+    # casts a given Socket to be a TCPServer or UNIXServer
+    def server_cast(sock)
+      begin
+        Socket.unpack_sockaddr_in(sock.getsockname)
+        TCPServer.for_fd(sock.fileno)
+      rescue ArgumentError
+        UNIXServer.for_fd(sock.fileno)
+      end
+    end
+
+  end # module SocketHelper
+end # module Pipemaster
+

data/lib/pipemaster/util.rb

@@ -0,0 +1,67 @@
+# -*- encoding: binary -*-
+
+require 'fcntl'
+require 'tmpdir'
+
+module Pipemaster
+
+  class Util
+    class << self
+
+      def is_log?(fp)
+        append_flags = File::WRONLY | File::APPEND
+
+        ! fp.closed? &&
+          fp.sync &&
+          fp.path[0] == ?/ &&
+          (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+      end
+
+      def chown_logs(uid, gid)
+        ObjectSpace.each_object(File) do |fp|
+          is_log?(fp) or next
+          fp.chown(uid, gid)
+        end
+      end
+
+      # This reopens ALL logfiles in the process that have been rotated
+      # using logrotate(8) (without copytruncate) or similar tools.
+      # A +File+ object is considered for reopening if it is:
+      #   1) opened with the O_APPEND and O_WRONLY flags
+      #   2) opened with an absolute path (starts with "/")
+      #   3) the current open file handle does not match its original open path
+      #   4) unbuffered (as far as userspace buffering goes, not O_SYNC)
+      # Returns the number of files reopened
+      def reopen_logs
+        nr = 0
+
+        ObjectSpace.each_object(File) do |fp|
+          is_log?(fp) or next
+          orig_st = fp.stat
+          begin
+            b = File.stat(fp.path)
+            next if orig_st.ino == b.ino && orig_st.dev == b.dev
+          rescue Errno::ENOENT
+          end
+
+          open_arg = 'a'
+          if fp.respond_to?(:external_encoding) && enc = fp.external_encoding
+            open_arg << ":#{enc.to_s}"
+            enc = fp.internal_encoding and open_arg << ":#{enc.to_s}"
+          end
+          fp.reopen(fp.path, open_arg)
+          fp.sync = true
+          new_st = fp.stat
+          if orig_st.uid != new_st.uid || orig_st.gid != new_st.gid
+            fp.chown(orig_st.uid, orig_st.gid)
+          end
+          nr += 1
+        end # each_object
+        nr
+      end
+
+    end
+
+  end
+end
+

data/pipemaster.gemspec

@@ -1,18 +1,17 @@
 Gem::Specification.new do |spec|
   spec.name           = "pipemaster"
-  spec.version        = "0.3.1"
+  spec.version        = "0.4.0"
   spec.author         = "Assaf Arkin"
   spec.email          = "assaf@labnotes.org"
-  spec.homepage       = "http://labnotes.org"
+  spec.homepage       = "http://github.com/assaf/pipemaster"
   spec.summary        = "Use the fork"
   spec.post_install_message = "To get started run pipemaster --help"
 
   spec.files          = Dir["{bin,lib,test}/**/*", "CHANGELOG", "LICENSE", "README.rdoc", "Rakefile", "Gemfile", "pipemaster.gemspec"]
-  spec.executable     = "pipemaster"
+  spec.executables    = %w{pipe pipemaster}
 
   spec.has_rdoc         = true
   spec.extra_rdoc_files = "README.rdoc", "CHANGELOG"
   spec.rdoc_options     = "--title", "Pipemaster #{spec.version}", "--main", "README.rdoc",
                           "--webcvs", "http://github.com/assaf/#{spec.name}"
-  spec.add_dependency("unicorn", ["~> 0.96"])
 end

data/test/test_helper.rb

@@ -70,7 +70,7 @@ end
 
 def hit(address, *args)
   client = Pipemaster::Client.new(address)
-  code = client.request(*args)
+  code = client.run(*args)
   [code, client.output.string]
 end
 
@@ -90,7 +90,7 @@ def unused_port(addr = '127.0.0.1')
   begin
     begin
       port = base + rand(32768 - base)
-      while port == Unicorn::Const::DEFAULT_PORT
+      while port == Pipemaster::DEFAULT_PORT
         port = base + rand(32768 - base)
       end
 
@@ -106,7 +106,7 @@ def unused_port(addr = '127.0.0.1')
     # condition could allow the random port we just chose to reselect itself
     # when running tests in parallel with gmake.  Create a lock file while
     # we have the port here to ensure that does not happen .
-    lock_path = "#{Dir::tmpdir}/unicorn_test.#{addr}:#{port}.lock"
+    lock_path = "#{Dir::tmpdir}/pipemaster_test.#{addr}:#{port}.lock"
     lock = File.open(lock_path, File::WRONLY|File::CREAT|File::EXCL, 0600)
     at_exit { File.unlink(lock_path) rescue nil }
   rescue Errno::EEXIST
@@ -126,20 +126,6 @@ def try_require(lib)
   end
 end
 
-# sometimes the server may not come up right away
-def retry_hit(uris = [])
-  tries = DEFAULT_TRIES
-  begin
-    hit(uris)
-  rescue Errno::EINVAL, Errno::ECONNREFUSED => err
-    if (tries -= 1) > 0
-      sleep DEFAULT_RES
-      retry
-    end
-    raise err
-  end
-end
-
 def assert_shutdown(pid)
   wait_master_ready("test_stderr.#{pid}.log")
   assert_nothing_raised { Process.kill(:QUIT, pid) }

data/test/unit/test_server.rb

@@ -1,11 +1,5 @@
 # -*- encoding: binary -*-
 
-# Copyright (c) 2005 Zed A. Shaw 
-# You can redistribute it and/or modify it under the same terms as Ruby.
-#
-# Additional work donated by contributors.  See http://mongrel.rubyforge.org/attributions.html
-# for more information.
-
 require 'test/test_helper'
 
 class ServerTest < Test::Unit::TestCase
@@ -72,10 +66,8 @@ class ServerTest < Test::Unit::TestCase
     tmp.write "foo bar"
     tmp.flush
     tmp.rewind
-    client.input = tmp
-    output = StringIO.new
-    client.output = output
-    client.request :reverse
+    client.capture tmp, StringIO.new
+    client.run :reverse
     assert_equal "rab oof", client.output.string
   ensure
     tmp.close!

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: pipemaster
 version: !ruby/object:Gem::Version 
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors: 
 - Assaf Arkin
@@ -9,22 +9,14 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-18 00:00:00 -08:00
+date: 2010-02-19 00:00:00 -08:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: unicorn
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        version: "0.96"
-    version: 
+dependencies: []
+
 description: 
 email: assaf@labnotes.org
 executables: 
+- pipe
 - pipemaster
 extensions: []
 
@@ -32,13 +24,19 @@ extra_rdoc_files:
 - README.rdoc
 - CHANGELOG
 files: 
+- bin/pipe
 - bin/pipemaster
+- lib/pipemaster
 - lib/pipemaster/client.rb
 - lib/pipemaster/configurator.rb
+- lib/pipemaster/launcher.rb
 - lib/pipemaster/server.rb
+- lib/pipemaster/socket_helper.rb
+- lib/pipemaster/util.rb
 - lib/pipemaster/worker.rb
 - lib/pipemaster.rb
 - test/test_helper.rb
+- test/unit
 - test/unit/test_configurator.rb
 - test/unit/test_server.rb
 - CHANGELOG
@@ -48,13 +46,11 @@ files:
 - Gemfile
 - pipemaster.gemspec
 has_rdoc: true
-homepage: http://labnotes.org
-licenses: []
-
+homepage: http://github.com/assaf/pipemaster
 post_install_message: To get started run pipemaster --help
 rdoc_options: 
 - --title
-- Pipemaster 0.3.1
+- Pipemaster 0.4.0
 - --main
 - README.rdoc
 - --webcvs
@@ -76,9 +72,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.5
+rubygems_version: 1.3.1
 signing_key: 
-specification_version: 3
+specification_version: 2
 summary: Use the fork
 test_files: []