unicorn 5.2.0 → 5.3.1

data/lib/unicorn/http_server.rb

@@ -15,6 +15,7 @@ class Unicorn::HttpServer
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
                 :orig_app, :config, :ready_pipe, :user
+  attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
   include Unicorn::SocketHelper
@@ -88,6 +89,7 @@ def initialize(app, options = {})
     @self_pipe = []
     @workers = {} # hash maps PIDs to Workers
     @sig_queue = [] # signal queue used for self-piping
+    @pid = nil
 
     # we try inheriting listeners first, so we bind them later.
     # we don't write the pid file until we've bound listeners in case
@@ -104,6 +106,14 @@ def initialize(app, options = {})
     # list of signals we care about and trap in master.
     @queue_sigs = [
       :WINCH, :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, :TTIN, :TTOU ]
+
+    @worker_data = if worker_data = ENV['UNICORN_WORKER']
+      worker_data = worker_data.split(',').map!(&:to_i)
+      worker_data[1] = worker_data.slice!(1..2).map do |i|
+        Kgio::Pipe.for_fd(i)
+      end
+      worker_data
+    end
   end
 
   # Runs the thing.  Returns self so you can run join on it
@@ -112,7 +122,7 @@ def start
     # this pipe is used to wake us up from select(2) in #join when signals
     # are trapped.  See trap_deferred.
     @self_pipe.replace(Unicorn.pipe)
-    @master_pid = $$
+    @master_pid = @worker_data ? Process.ppid : $$
 
     # setup signal handlers before writing pid file in case people get
     # trigger happy and send signals as soon as the pid file exists.
@@ -395,8 +405,7 @@ def reap_all_workers
         proc_name 'master'
       else
         worker = @workers.delete(wpid) and worker.close rescue nil
-        m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
-        status.success? ? logger.info(m) : logger.error(m)
+        @after_worker_exit.call(self, worker, status)
       end
     rescue Errno::ECHILD
       break
@@ -430,11 +439,7 @@ def reexec
     end
 
     @reexec_pid = fork do
-      listener_fds = {}
-      LISTENERS.each do |sock|
-        sock.close_on_exec = false
-        listener_fds[sock.fileno] = sock
-      end
+      listener_fds = listener_sockets
       ENV['UNICORN_FD'] = listener_fds.keys.join(',')
       Dir.chdir(START_CTX[:cwd])
       cmd = [ START_CTX[0] ].concat(START_CTX[:argv])
@@ -442,12 +447,7 @@ def reexec
       # avoid leaking FDs we don't know about, but let before_exec
       # unset FD_CLOEXEC, if anything else in the app eventually
       # relies on FD inheritence.
-      (3..1024).each do |io|
-        next if listener_fds.include?(io)
-        io = IO.for_fd(io) rescue next
-        io.autoclose = false
-        io.close_on_exec = true
-      end
+      close_sockets_on_exec(listener_fds)
 
       # exec(command, hash) works in at least 1.9.1+, but will only be
       # required in 1.9.4/2.0.0 at earliest.
@@ -459,6 +459,40 @@ def reexec
     proc_name 'master (old)'
   end
 
+  def worker_spawn(worker)
+    listener_fds = listener_sockets
+    env = {}
+    env['UNICORN_FD'] = listener_fds.keys.join(',')
+
+    listener_fds[worker.to_io.fileno] = worker.to_io
+    listener_fds[worker.master.fileno] = worker.master
+
+    worker_info = [worker.nr, worker.to_io.fileno, worker.master.fileno]
+    env['UNICORN_WORKER'] = worker_info.join(',')
+
+    close_sockets_on_exec(listener_fds)
+
+    Process.spawn(env, START_CTX[0], *START_CTX[:argv], listener_fds)
+  end
+
+  def listener_sockets
+    listener_fds = {}
+    LISTENERS.each do |sock|
+      sock.close_on_exec = false
+      listener_fds[sock.fileno] = sock
+    end
+    listener_fds
+  end
+
+  def close_sockets_on_exec(sockets)
+    (3..1024).each do |io|
+      next if sockets.include?(io)
+      io = IO.for_fd(io) rescue next
+      io.autoclose = false
+      io.close_on_exec = true
+    end
+  end
+
   # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
   def murder_lazy_workers
     next_sleep = @timeout - 1
@@ -495,19 +529,29 @@ def after_fork_internal
   end
 
   def spawn_missing_workers
+    if @worker_data
+      worker = Unicorn::Worker.new(*@worker_data)
+      after_fork_internal
+      worker_loop(worker)
+      exit
+    end
+
     worker_nr = -1
     until (worker_nr += 1) == @worker_processes
       @workers.value?(worker_nr) and next
       worker = Unicorn::Worker.new(worker_nr)
       before_fork.call(self, worker)
-      if pid = fork
-        @workers[pid] = worker
-        worker.atfork_parent
-      else
+
+      pid = @worker_exec ? worker_spawn(worker) : fork
+
+      unless pid
         after_fork_internal
         worker_loop(worker)
         exit
       end
+
+      @workers[pid] = worker
+      worker.atfork_parent
     end
     rescue => e
       @logger.error(e) rescue nil
@@ -644,7 +688,7 @@ def worker_loop(worker)
     trap(:USR1) { nr = -65536 }
 
     ready = readers.dup
-    @logger.info "worker=#{worker.nr} ready"
+    @after_worker_ready.call(self, worker)
 
     begin
       nr < 0 and reopen_worker_logs(worker.nr)

data/lib/unicorn/oob_gc.rb

@@ -66,10 +66,9 @@ def self.new(app, interval = 5, path = %r{\A/})
   end
 
   #:stopdoc:
-  PATH_INFO = "PATH_INFO"
   def process_client(client)
     super(client) # Unicorn::HttpServer#process_client
-    if OOBGC_PATH =~ OOBGC_ENV[PATH_INFO] && ((@@nr -= 1) <= 0)
+    if OOBGC_PATH =~ OOBGC_ENV['PATH_INFO'] && ((@@nr -= 1) <= 0)
       @@nr = OOBGC_INTERVAL
       OOBGC_ENV.clear
       disabled = GC.enable

data/lib/unicorn/socket_helper.rb

@@ -3,6 +3,18 @@
 require 'socket'
 
 module Unicorn
+
+  # Instead of using a generic Kgio::Socket for everything,
+  # tag TCP sockets so we can use TCP_INFO under Linux without
+  # incurring extra syscalls for Unix domain sockets.
+  # TODO: remove these when we remove kgio
+  TCPClient = Class.new(Kgio::Socket) # :nodoc:
+  class TCPSrv < Kgio::TCPServer # :nodoc:
+    def kgio_tryaccept # :nodoc:
+      super(TCPClient)
+    end
+  end
+
   module SocketHelper
 
     # internal interface
@@ -63,12 +75,15 @@ def set_tcp_sockopt(sock, opt)
       elsif respond_to?(:accf_arg)
         name = opt[:accept_filter]
         name = DEFAULTS[:accept_filter] if name.nil?
+        sock.listen(opt[:backlog])
+        got = (sock.getsockopt(:SOL_SOCKET, :SO_ACCEPTFILTER) rescue nil).to_s
+        arg = accf_arg(name)
         begin
-          sock.setsockopt(:SOL_SOCKET, :SO_ACCEPTFILTER, accf_arg(name))
+          sock.setsockopt(:SOL_SOCKET, :SO_ACCEPTFILTER, arg)
         rescue => e
           logger.error("#{sock_name(sock)} " \
                        "failed to set accept_filter=#{name} (#{e.inspect})")
-        end
+        end if arg != got
       end
     end
 
@@ -148,7 +163,7 @@ def new_tcp_server(addr, port, opt)
       end
       sock.bind(Socket.pack_sockaddr_in(port, addr))
       sock.autoclose = false
-      Kgio::TCPServer.for_fd(sock.fileno)
+      TCPSrv.for_fd(sock.fileno)
     end
 
     # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6
@@ -185,7 +200,7 @@ def sock_name(sock)
     def server_cast(sock)
       begin
         Socket.unpack_sockaddr_in(sock.getsockname)
-        Kgio::TCPServer.for_fd(sock.fileno)
+        TCPSrv.for_fd(sock.fileno)
       rescue ArgumentError
         Kgio::UNIXServer.for_fd(sock.fileno)
       end

data/lib/unicorn/stream_input.rb

@@ -1,16 +1,17 @@
 # -*- encoding: binary -*-
 
-# When processing uploads, Unicorn may expose a StreamInput object under
-# "rack.input" of the (future) Rack (2.x) environment.
+# When processing uploads, unicorn may expose a StreamInput object under
+# "rack.input" of the Rack environment when
+# Unicorn::Configurator#rewindable_input is set to +false+
 class Unicorn::StreamInput
   # The I/O chunk size (in +bytes+) for I/O operations where
   # the size cannot be user-specified when a method is called.
   # The default is 16 kilobytes.
-  @@io_chunk_size = Unicorn::Const::CHUNK_SIZE
+  @@io_chunk_size = Unicorn::Const::CHUNK_SIZE # :nodoc:
 
   # Initializes a new StreamInput object.  You normally do not have to call
   # this unless you are writing an HTTP server.
-  def initialize(socket, request)
+  def initialize(socket, request) # :nodoc:
     @chunked = request.content_length.nil?
     @socket = socket
     @parser = request

data/lib/unicorn/tee_input.rb

@@ -1,6 +1,6 @@
 # -*- encoding: binary -*-
 
-# acts like tee(1) on an input input to provide a input-like stream
+# Acts like tee(1) on an input input to provide a input-like stream
 # while providing rewindable semantics through a File/StringIO backing
 # store.  On the first pass, the input is only read on demand so your
 # Rack application can use input notification (upload progress and
@@ -9,22 +9,22 @@
 # strict interpretation of Rack::Lint::InputWrapper functionality and
 # will not support any deviations from it.
 #
-# When processing uploads, Unicorn exposes a TeeInput object under
-# "rack.input" of the Rack environment.
+# When processing uploads, unicorn exposes a TeeInput object under
+# "rack.input" of the Rack environment by default.
 class Unicorn::TeeInput < Unicorn::StreamInput
   # The maximum size (in +bytes+) to buffer in memory before
   # resorting to a temporary file.  Default is 112 kilobytes.
-  @@client_body_buffer_size = Unicorn::Const::MAX_BODY
+  @@client_body_buffer_size = Unicorn::Const::MAX_BODY # :nodoc:
 
   # sets the maximum size of request bodies to buffer in memory,
   # amounts larger than this are buffered to the filesystem
-  def self.client_body_buffer_size=(bytes)
+  def self.client_body_buffer_size=(bytes) # :nodoc:
     @@client_body_buffer_size = bytes
   end
 
   # returns the maximum size of request bodies to buffer in memory,
   # amounts larger than this are buffered to the filesystem
-  def self.client_body_buffer_size
+  def self.client_body_buffer_size # :nodoc:
     @@client_body_buffer_size
   end
 
@@ -37,7 +37,7 @@ def new_tmpio # :nodoc:
 
   # Initializes a new TeeInput object.  You normally do not have to call
   # this unless you are writing an HTTP server.
-  def initialize(socket, request)
+  def initialize(socket, request) # :nodoc:
     @len = request.content_length
     super
     @tmp = @len && @len <= @@client_body_buffer_size ?
@@ -125,9 +125,7 @@ def consume!
   end
 
   def tee(buffer)
-    if buffer && buffer.size > 0
-      @tmp.write(buffer)
-    end
+    @tmp.write(buffer) if buffer
     buffer
   end
 end

data/lib/unicorn/version.rb

@@ -1 +1 @@
-Unicorn::Const::UNICORN_VERSION = '5.2.0'
+Unicorn::Const::UNICORN_VERSION = '5.3.1'

data/lib/unicorn/worker.rb

@@ -12,18 +12,19 @@ class Unicorn::Worker
   # :stopdoc:
   attr_accessor :nr, :switched
   attr_reader :to_io # IO.select-compatible
+  attr_reader :master
 
   PER_DROP = Raindrops::PAGE_SIZE / Raindrops::SIZE
   DROPS = []
 
-  def initialize(nr)
+  def initialize(nr, pipe=nil)
     drop_index = nr / PER_DROP
     @raindrop = DROPS[drop_index] ||= Raindrops.new(PER_DROP)
     @offset = nr % PER_DROP
     @raindrop[@offset] = 0
     @nr = nr
     @switched = false
-    @to_io, @master = Unicorn.pipe
+    @to_io, @master = pipe || Unicorn.pipe
   end
 
   def atfork_child # :nodoc:
@@ -111,9 +112,11 @@ def close # :nodoc:
   # In most cases, you should be using the Unicorn::Configurator#user
   # directive instead.  This method should only be used if you need
   # fine-grained control of exactly when you want to change permissions
-  # in your after_fork hooks.
+  # in your after_fork or after_worker_ready hooks, or if you want to
+  # use the chroot support.
   #
-  # Changes the worker process to the specified +user+ and +group+
+  # Changes the worker process to the specified +user+ and +group+,
+  # and chroots to the current working directory if +chroot+ is set.
   # This is only intended to be called from within the worker
   # process from the +after_fork+ hook.  This should be called in
   # the +after_fork+ hook after any privileged functions need to be
@@ -122,8 +125,11 @@ def close # :nodoc:
   # Any and all errors raised within this method will be propagated
   # directly back to the caller (usually the +after_fork+ hook.
   # These errors commonly include ArgumentError for specifying an
-  # invalid user/group and Errno::EPERM for insufficient privileges
-  def user(user, group = nil)
+  # invalid user/group and Errno::EPERM for insufficient privileges.
+  #
+  # chroot support is only available in unicorn 5.3.0+
+  # user and group switching appeared in unicorn 0.94.0 (2009-11-05)
+  def user(user, group = nil, chroot = false)
     # we do not protect the caller, checking Process.euid == 0 is
     # insufficient because modern systems have fine-grained
     # capabilities.  Let the caller handle any and all errors.
@@ -134,6 +140,11 @@ def user(user, group = nil)
       Process.initgroups(user, gid)
       Process::GID.change_privilege(gid)
     end
+    if chroot
+      chroot = Dir.pwd if chroot == true
+      Dir.chroot(chroot)
+      Dir.chdir('/')
+    end
     Process.euid != uid and Process::UID.change_privilege(uid)
     @switched = true
   end

data/t/t0011-active-unix-socket.sh

@@ -52,7 +52,7 @@ t_begin "worker pid unchanged (again)" && {
 }
 
 t_begin "nuking the existing Unicorn succeeds" && {
-	kill -9 $unicorn_pid $worker_pid
+	kill -9 $unicorn_pid
 	while kill -0 $unicorn_pid
 	do
 		sleep 1

data/t/t0012-reload-empty-config.sh

@@ -18,7 +18,8 @@ after_fork { |s,w| }
     next if key =~ %r{\Astd(?:err|out)_path\z}
     key = key.to_sym
     def_value = defaults[key]
-    srv_value = srv.__send__(key)
+    srv_value = srv.respond_to?(key) ? srv.__send__(key)
+                                     : srv.instance_variable_get("@#{key}")
     fp << "#{key}|#{srv_value}|#{def_value}\\n"
   end
 }

data/t/test-lib.sh

@@ -106,8 +106,8 @@ check_stderr () {
 # unicorn_setup
 unicorn_setup () {
 	eval $(unused_listen)
-	port=$(expr $listen : '[^:]*:\([0-9]\+\)')
-	host=$(expr $listen : '\([^:]*\):[0-9]\+')
+	port=$(expr $listen : '[^:]*:\([0-9]*\)')
+	host=$(expr $listen : '\([^:][^:]*\):[0-9][0-9]*')
 
 	rtmpfiles unicorn_config pid r_err r_out fifo tmp ok
 	cat > $unicorn_config <<EOF

data/test/exec/test_exec.rb

@@ -97,6 +97,9 @@ def teardown
   end
 
   def test_sd_listen_fds_emulation
+    # [ruby-core:69895] [Bug #11336] fixed by r51576
+    return if RUBY_VERSION.to_f < 2.3
+
     File.open("config.ru", "wb") { |fp| fp.write(HI) }
     sock = TCPServer.new(@addr, @port)
 
@@ -119,14 +122,12 @@ def test_sd_listen_fds_emulation
       res = hit(["http://#@addr:#@port/"])
       assert_equal [ "HI\n" ], res
       assert_shutdown(pid)
-      assert_equal 1, sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).int,
+      assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
                   'unicorn should always set SO_KEEPALIVE on inherited sockets'
     end
   ensure
     sock.close if sock
-    # disabled test on old Rubies: https://bugs.ruby-lang.org/issues/11336
-    # [ruby-core:69895] [Bug #11336] fixed by r51576
-  end if RUBY_VERSION.to_f >= 2.3
+  end
 
   def test_inherit_listener_unspecified
     File.open("config.ru", "wb") { |fp| fp.write(HI) }
@@ -142,7 +143,7 @@ def test_inherit_listener_unspecified
     res = hit(["http://#@addr:#@port/"])
     assert_equal [ "HI\n" ], res
     assert_shutdown(pid)
-    assert_equal 1, sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).int,
+    assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
                 'unicorn should always set SO_KEEPALIVE on inherited sockets'
   ensure
     sock.close if sock

data/test/unit/test_ccc.rb

@@ -0,0 +1,90 @@
+require 'socket'
+require 'unicorn'
+require 'io/wait'
+require 'tempfile'
+require 'test/unit'
+
+class TestCccTCPI < Test::Unit::TestCase
+  def test_ccc_tcpi
+    start_pid = $$
+    host = '127.0.0.1'
+    srv = TCPServer.new(host, 0)
+    port = srv.addr[1]
+    err = Tempfile.new('unicorn_ccc')
+    rd, wr = IO.pipe
+    sleep_pipe = IO.pipe
+    pid = fork do
+      sleep_pipe[1].close
+      reqs = 0
+      rd.close
+      worker_pid = nil
+      app = lambda do |env|
+        worker_pid ||= begin
+          at_exit { wr.write(reqs.to_s) if worker_pid == $$ }
+          $$
+        end
+        reqs += 1
+
+        # will wake up when writer closes
+        sleep_pipe[0].read if env['PATH_INFO'] == '/sleep'
+
+        [ 200, [ %w(Content-Length 0),  %w(Content-Type text/plain) ], [] ]
+      end
+      ENV['UNICORN_FD'] = srv.fileno.to_s
+      opts = {
+        listeners: [ "#{host}:#{port}" ],
+        stderr_path: err.path,
+        check_client_connection: true,
+      }
+      uni = Unicorn::HttpServer.new(app, opts)
+      uni.start.join
+    end
+    wr.close
+
+    # make sure the server is running, at least
+    client = TCPSocket.new(host, port)
+    client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+    assert client.wait_readable(10), 'never got response from server'
+    res = client.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of response, server is ready'
+    client.close
+
+    # start a slow request...
+    sleeper = TCPSocket.new(host, port)
+    sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n")
+
+    # and a bunch of aborted ones
+    nr = 100
+    nr.times do |i|
+      client = TCPSocket.new(host, port)
+      client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \
+                   "Host: example.com\r\n\r\n")
+      client.close
+    end
+    sleep_pipe[1].close # wake up the reader in the worker
+    res = sleeper.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first sleeper response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of sleeper response'
+    sleeper.close
+    kpid = pid
+    pid = nil
+    Process.kill(:QUIT, kpid)
+    _, status = Process.waitpid2(kpid)
+    assert status.success?
+    reqs = rd.read.to_i
+    warn "server got #{reqs} requests with #{nr} CCC aborted\n" if $DEBUG
+    assert_operator reqs, :<, nr
+    assert_operator reqs, :>=, 2, 'first 2 requests got through, at least'
+  ensure
+    return if start_pid != $$
+    srv.close if srv
+    if pid
+      Process.kill(:QUIT, pid)
+      _, status = Process.waitpid2(pid)
+      assert status.success?
+    end
+    err.close! if err
+    rd.close if rd
+  end
+end