passenger 2.0.6 → 2.1.2

data/lib/{passenger → phusion_passenger}/abstract_request_handler.rb

@@ -18,9 +18,9 @@
 
 require 'socket'
 require 'fcntl'
-require 'passenger/utils'
-require 'passenger/native_support'
-module Passenger
+require 'phusion_passenger/message_channel'
+require 'phusion_passenger/utils'
+module PhusionPassenger
 
 # The request handler is the layer which connects Apache with the underlying application's
 # request dispatcher (i.e. either Rails's Dispatcher class or Rack).
@@ -39,30 +39,20 @@ module Passenger
 # administrator maintenance overhead. These decisions are documented
 # in this section.
 #
-# === Abstract namespace Unix sockets
-#
-# AbstractRequestHandler listens on a Unix socket for incoming requests. If possible,
-# AbstractRequestHandler will try to create a Unix socket on the _abstract namespace_,
-# instead of on the filesystem. If the RoR application crashes (segfault),
-# or if it gets killed by SIGKILL, or if the system loses power, then there
-# will be no stale socket files left on the filesystem.
-# Unfortunately, abstract namespace Unix sockets are only supported by Linux.
-# On systems that do not support abstract namespace Unix sockets,
-# AbstractRequestHandler will automatically fallback to using regular Unix socket files.
-#
-# It is possible to force AbstractRequestHandler to use regular Unix socket files by
-# setting the environment variable PASSENGER_NO_ABSTRACT_NAMESPACE_SOCKETS
-# to 1.
-#
 # === Owner pipes
 #
 # Because only the web server communicates directly with a request handler,
 # we want the request handler to exit if the web server has also exited.
 # This is implemented by using a so-called _owner pipe_. The writable part
-# of the pipe will be owned by the web server. AbstractRequestHandler will
-# continuously check whether the other side of the pipe has been closed. If
-# so, then it knows that the web server has exited, and so the request handler
-# will exit as well. This works even if the web server gets killed by SIGKILL.
+# of the pipe will be passed to the web server* via a Unix socket, and the web
+# server will own that part of the pipe, while AbstractRequestHandler owns
+# the readable part of the pipe. AbstractRequestHandler will continuously
+# check whether the other side of the pipe has been closed. If so, then it
+# knows that the web server has exited, and so the request handler will exit
+# as well. This works even if the web server gets killed by SIGKILL.
+#
+# * It might also be passed to the ApplicationPoolServerExecutable, if the web
+#   server's using ApplicationPoolServer instead of StandardApplicationPool.
 #
 #
 # == Request format
@@ -94,7 +84,7 @@ class AbstractRequestHandler
 	HARD_TERMINATION_SIGNAL = "SIGTERM"
 	# Signal which will cause the Rails application to exit as soon as it's done processing a request.
 	SOFT_TERMINATION_SIGNAL = "SIGUSR1"
-	BACKLOG_SIZE    = 50
+	BACKLOG_SIZE    = 100
 	MAX_HEADER_SIZE = 128 * 1024
 	
 	# String constants which exist to relieve Ruby's garbage collector.
@@ -104,72 +94,117 @@ class AbstractRequestHandler
 	CONTENT_LENGTH      = 'CONTENT_LENGTH'      # :nodoc:
 	HTTP_CONTENT_LENGTH = 'HTTP_CONTENT_LENGTH' # :nodoc:
 	X_POWERED_BY        = 'X-Powered-By'        # :nodoc:
+	REQUEST_METHOD      = 'REQUEST_METHOD'      # :nodoc:
+	PING                = 'ping'                # :nodoc:
 	
 	# The name of the socket on which the request handler accepts
-	# new connections. This is either a Unix socket filename, or
-	# the name for an abstract namespace Unix socket.
+	# new connections. At this moment, this value is always the filename
+	# of a Unix domain socket.
 	#
-	# If +socket_name+ refers to an abstract namespace Unix socket,
-	# then the name does _not_ contain a leading null byte.
-	#
-	# See also using_abstract_namespace?
+	# See also #socket_type.
 	attr_reader :socket_name
-
+	
+	# The type of socket that #socket_name refers to. At the moment, the
+	# value is always 'unix', which indicates a Unix domain socket.
+	attr_reader :socket_type
+	
+	# Specifies the maximum allowed memory usage, in MB. If after having processed
+	# a request AbstractRequestHandler detects that memory usage has risen above
+	# this limit, then it will gracefully exit (that is, exit after having processed
+	# all pending requests).
+	#
+	# A value of 0 (the default) indicates that there's no limit.
+	attr_accessor :memory_limit
+	
+	# The number of times the main loop has iterated so far. Mostly useful
+	# for unit test assertions.
+	attr_reader :iterations
+	
+	# Number of requests processed so far. This includes requests that raised
+	# exceptions.
+	attr_reader :processed_requests
+	
 	# Create a new RequestHandler with the given owner pipe.
 	# +owner_pipe+ must be the readable part of a pipe IO object.
-	def initialize(owner_pipe)
-		if abstract_namespace_sockets_allowed?
-			@using_abstract_namespace = create_unix_socket_on_abstract_namespace
-		else
-			@using_abstract_namespace = false
-		end
-		if !@using_abstract_namespace
+	#
+	# Additionally, the following options may be given:
+	# - memory_limit: Used to set the +memory_limit+ attribute.
+	def initialize(owner_pipe, options = {})
+		if should_use_unix_sockets?
 			create_unix_socket_on_filesystem
+		else
+			create_tcp_socket
 		end
-		@socket.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+		@socket.close_on_exec!
 		@owner_pipe = owner_pipe
 		@previous_signal_handlers = {}
+		@main_loop_thread_lock = Mutex.new
+		@main_loop_thread_cond = ConditionVariable.new
+		@memory_limit = options["memory_limit"] || 0
+		@iterations = 0
+		@processed_requests = 0
 	end
 	
 	# Clean up temporary stuff created by the request handler.
-	# This method should be called after the main loop has exited.
+	#
+	# If the main loop was started by #main_loop, then this method may only
+	# be called after the main loop has exited.
+	#
+	# If the main loop was started by #start_main_loop_thread, then this method
+	# may be called at any time, and it will stop the main loop thread.
 	def cleanup
+		if @main_loop_thread
+			@main_loop_thread.raise(Interrupt.new("Cleaning up"))
+			@main_loop_thread.join
+		end
 		@socket.close rescue nil
 		@owner_pipe.close rescue nil
-		if !using_abstract_namespace?
-			File.unlink(@socket_name) rescue nil
-		end
+		File.unlink(@socket_name) rescue nil
 	end
 	
-	# Returns whether socket_name refers to an abstract namespace Unix socket.
-	def using_abstract_namespace?
-		return @using_abstract_namespace
+	# Check whether the main loop's currently running.
+	def main_loop_running?
+		return @main_loop_running
 	end
 	
 	# Enter the request handler's main loop.
 	def main_loop
 		reset_signal_handlers
 		begin
-			done = false
-			while !done
+			@graceful_termination_pipe = IO.pipe
+			@graceful_termination_pipe[0].close_on_exec!
+			@graceful_termination_pipe[1].close_on_exec!
+			
+			@main_loop_thread_lock.synchronize do
+				@main_loop_running = true
+				@main_loop_thread_cond.broadcast
+			end
+			
+			install_useful_signal_handlers
+			
+			while true
+				@iterations += 1
 				client = accept_connection
 				if client.nil?
 					break
 				end
-				trap SOFT_TERMINATION_SIGNAL do
-					done = true
-				end
 				begin
 					headers, input = parse_request(client)
 					if headers
-						process_request(headers, input, client)
+						if headers[REQUEST_METHOD] == PING
+							process_ping(headers, input, client)
+						else
+							process_request(headers, input, client)
+						end
 					end
 				rescue IOError, SocketError, SystemCallError => e
 					print_exception("Passenger RequestHandler", e)
 				ensure
+					# 'input' is the same as 'client' so we don't
+					# need to close that.
 					client.close rescue nil
 				end
-				trap SOFT_TERMINATION_SIGNAL, DEFAULT
+				@processed_requests += 1
 			end
 		rescue EOFError
 			# Exit main loop.
@@ -181,47 +216,54 @@ class AbstractRequestHandler
 				raise
 			end
 		ensure
+			@graceful_termination_pipe[0].close rescue nil
+			@graceful_termination_pipe[1].close rescue nil
 			revert_signal_handlers
+			@main_loop_thread_lock.synchronize do
+				@main_loop_running = false
+				@main_loop_thread_cond.broadcast
+			end
 		end
 	end
-
-private
-	include Utils
-
-	def create_unix_socket_on_abstract_namespace
-		while true
-			begin
-				# I have no idea why, but using base64-encoded IDs
-				# don't pass the unit tests. I couldn't find the cause
-				# of the problem. The system supports base64-encoded
-				# names for abstract namespace unix sockets just fine.
-				@socket_name = generate_random_id(:hex)
-				@socket_name = @socket_name.slice(0, NativeSupport::UNIX_PATH_MAX - 2)
-				fd = NativeSupport.create_unix_socket("\x00#{socket_name}", BACKLOG_SIZE)
-				@socket = IO.new(fd)
-				@socket.instance_eval do
-					def accept
-						fd = NativeSupport.accept(fileno)
-						return IO.new(fd)
-					end
-				end
-				return true
-			rescue Errno::EADDRINUSE
-				# Do nothing, try again with another name.
-			rescue Errno::ENOENT
-				# Abstract namespace sockets not supported on this system.
-				return false
+	
+	# Start the main loop in a new thread. This thread will be stopped by #cleanup.
+	def start_main_loop_thread
+		@main_loop_thread = Thread.new do
+			main_loop
+		end
+		@main_loop_thread_lock.synchronize do
+			while !@main_loop_running
+				@main_loop_thread_cond.wait(@main_loop_thread_lock)
 			end
 		end
 	end
+
+private
+	include Utils
 	
+	def should_use_unix_sockets?
+		# There seems to be a bug in MacOS X w.r.t. Unix sockets.
+		# When the Unix socket subsystem is under high stress, a
+		# recv()/read() on a Unix socket can return 0 even when EOF is
+		# not reached. We work around this by using TCP sockets on
+		# MacOS X.
+		return RUBY_PLATFORM !~ /darwin/
+	end
+
 	def create_unix_socket_on_filesystem
 		done = false
 		while !done
 			begin
-				@socket_name = "/tmp/passenger.#{generate_random_id(:base64)}"
-				@socket_name = @socket_name.slice(0, NativeSupport::UNIX_PATH_MAX - 1)
+				if defined?(NativeSupport)
+					unix_path_max = NativeSupport::UNIX_PATH_MAX
+				else
+					unix_path_max = 100
+				end
+				@socket_name = "#{passenger_tmpdir}/passenger_backend.#{generate_random_id(:base64)}"
+				@socket_name = @socket_name.slice(0, unix_path_max - 1)
 				@socket = UNIXServer.new(@socket_name)
+				@socket.listen(BACKLOG_SIZE)
+				@socket_type = "unix"
 				File.chmod(0600, @socket_name)
 				done = true
 			rescue Errno::EADDRINUSE
@@ -229,12 +271,21 @@ private
 			end
 		end
 	end
+	
+	def create_tcp_socket
+		# We use "127.0.0.1" as address in order to force
+		# TCPv4 instead of TCPv6.
+		@socket = TCPServer.new('127.0.0.1', 0)
+		@socket.listen(BACKLOG_SIZE)
+		@socket_name = "127.0.0.1:#{@socket.addr[1]}"
+		@socket_type = "tcp"
+	end
 
 	# Reset signal handlers to their default handler, and install some
 	# special handlers for a few signals. The previous signal handlers
 	# will be put back by calling revert_signal_handlers.
 	def reset_signal_handlers
-		Signal.list.each_key do |signal|
+		Signal.list_trappable.each_key do |signal|
 			begin
 				prev_handler = trap(signal, DEFAULT)
 				if prev_handler != DEFAULT
@@ -245,9 +296,45 @@ private
 			end
 		end
 		trap('HUP', IGNORE)
+	end
+	
+	def install_useful_signal_handlers
+		trappable_signals = Signal.list_trappable
+		
+		trap(SOFT_TERMINATION_SIGNAL) do
+			@graceful_termination_pipe[1].close rescue nil
+		end if trappable_signals.has_key?(SOFT_TERMINATION_SIGNAL.sub(/^SIG/, ''))
+		
 		trap('ABRT') do
 			raise SignalException, "SIGABRT"
-		end
+		end if trappable_signals.has_key?('ABRT')
+		
+		trap('QUIT') do
+			if Kernel.respond_to?(:caller_for_all_threads)
+				output = "========== Process #{Process.pid}: backtrace dump ==========\n"
+				caller_for_all_threads.each_pair do |thread, stack|
+					output << ("-" * 60) << "\n"
+					output << "# Thread: #{thread.inspect}, "
+					if thread == Thread.main
+						output << "[main thread], "
+					else
+						output << "[current thread], "
+					end
+					output << "alive = #{thread.alive?}\n"
+					output << ("-" * 60) << "\n"
+					output << "    " << stack.join("\n    ")
+					output << "\n\n"
+				end
+			else
+				output = "========== Process #{Process.pid}: backtrace dump ==========\n"
+				output << ("-" * 60) << "\n"
+				output << "# Current thread: #{Thread.current.inspect}\n"
+				output << ("-" * 60) << "\n"
+				output << "    " << caller.join("\n    ")
+			end
+			STDERR.puts(output)
+			STDERR.flush
+		end if trappable_signals.has_key?('QUIT')
 	end
 	
 	def revert_signal_handlers
@@ -257,10 +344,38 @@ private
 	end
 	
 	def accept_connection
-		ios = select([@socket, @owner_pipe])[0]
+		ios = select([@socket, @owner_pipe, @graceful_termination_pipe[0]]).first
 		if ios.include?(@socket)
 			client = @socket.accept
-			client.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+			client.close_on_exec!
+			
+			# Some people report that sometimes their Ruby (MRI/REE)
+			# processes get stuck with 100% CPU usage. Upon further
+			# inspection with strace, it turns out that these Ruby
+			# processes are continuously calling lseek() on a socket,
+			# which of course returns ESPIPE as error. gdb reveals
+			# lseek() is called by fwrite(), which in turn is called
+			# by rb_fwrite(). The affected socket is the
+			# AbstractRequestHandler client socket.
+			#
+			# I inspected the MRI source code and didn't find
+			# anything that would explain this behavior. This makes
+			# me think that it's a glibc bug, but that's very
+			# unlikely.
+			#
+			# The rb_fwrite() implementation takes an entirely
+			# different code path if I set 'sync' to true: it will
+			# skip fwrite() and use write() instead. So here we set
+			# 'sync' to true in the hope that this will work around
+			# the problem.
+			client.sync = true
+			
+			# We monkeypatch the 'sync=' method to a no-op so that
+			# sync mode can't be disabled.
+			client.instance_eval do
+				def client.sync=(value)
+				end
+			end
 			
 			# The real input stream is not seekable (calling _seek_
 			# or _rewind_ on it will raise an exception). But some
@@ -274,8 +389,10 @@ private
 			
 			return client
 		else
-			# The other end of the pipe has been closed.
-			# So we know all owning processes have quit.
+			# The other end of the owner pipe has been closed, or the
+			# graceful termination pipe has been closed. This is our
+			# call to gracefully terminate (after having processed all
+			# incoming requests).
 			return nil
 		end
 	end
@@ -301,6 +418,10 @@ private
 		print_exception("Passenger RequestHandler", e)
 	end
 	
+	def process_ping(env, input, output)
+		output.write("pong")
+	end
+	
 	# Generate a long, cryptographically secure random ID string, which
 	# is also a valid filename.
 	def generate_random_id(method)
@@ -318,11 +439,6 @@ private
 		return data
 	end
 	
-	def abstract_namespace_sockets_allowed?
-		return !ENV['PASSENGER_NO_ABSTRACT_NAMESPACE_SOCKETS'] ||
-			ENV['PASSENGER_NO_ABSTRACT_NAMESPACE_SOCKETS'].empty?
-	end
-
 	def self.determine_passenger_version
 		rakefile = "#{File.dirname(__FILE__)}/../../Rakefile"
 		if File.exist?(rakefile)
@@ -347,4 +463,4 @@ public
 	PASSENGER_HEADER = determine_passenger_header
 end
 
-end # module Passenger
+end # module PhusionPassenger

data/lib/{passenger → phusion_passenger}/abstract_server.rb

@@ -19,10 +19,9 @@
 require 'socket'
 require 'set'
 require 'timeout'
-require 'passenger/message_channel'
-require 'passenger/utils'
-require 'passenger/native_support'
-module Passenger
+require 'phusion_passenger/message_channel'
+require 'phusion_passenger/utils'
+module PhusionPassenger
 
 # An abstract base class for a server, with the following properties:
 #
@@ -43,7 +42,7 @@ module Passenger
 #
 # Here's an example on using AbstractServer:
 #
-#  class MyServer < Passenger::AbstractServer
+#  class MyServer < PhusionPassenger::AbstractServer
 #     def initialize
 #        super()
 #        define_message_handler(:hello, :handle_hello)
@@ -88,11 +87,25 @@ class AbstractServer
 	class ServerError < StandardError
 	end
 	
+	# The last time when this AbstractServer had processed a message.
+	attr_accessor :last_activity_time
+	
+	# The maximum time that this AbstractServer may be idle. Used by
+	# AbstractServerCollection to determine when this object should
+	# be cleaned up. nil or 0 indicate that this object should never
+	# be idle cleaned.
+	attr_accessor :max_idle_time
+	
+	# Used by AbstractServerCollection to remember when this AbstractServer
+	# should be idle cleaned.
+	attr_accessor :next_cleaning_time
+	
 	def initialize
 		@done = false
 		@message_handlers = {}
 		@signal_handlers = {}
 		@orig_signal_handlers = {}
+		@last_activity_time = Time.now
 	end
 	
 	# Start the server. This method does not block since the server runs
@@ -109,7 +122,8 @@ class AbstractServer
 	
 		@parent_socket, @child_socket = UNIXSocket.pair
 		before_fork
-		@pid = fork do
+		@pid = fork
+		if @pid.nil?
 			begin
 				STDOUT.sync = true
 				STDERR.sync = true
@@ -278,7 +292,7 @@ private
 	# Reset all signal handlers to default. This is called in the child process,
 	# before entering the main loop.
 	def reset_signal_handlers
-		Signal.list.each_key do |signal|
+		Signal.list_trappable.each_key do |signal|
 			begin
 				@orig_signal_handlers[signal] = trap(signal, 'DEFAULT')
 			rescue ArgumentError
@@ -311,6 +325,7 @@ private
 		while !@done
 			begin
 				name, *args = channel.read
+				@last_activity_time = Time.now
 				if name.nil?
 					@done = true
 				elsif @message_handlers.has_key?(name)
@@ -331,4 +346,4 @@ private
 	end
 end
 
-end # module Passenger
+end # module PhusionPassenger