passenger 2.2.15 → 3.0.0.pre1

data/{misc/rake/extensions.rb → build/rake_extensions.rb}

@@ -1,5 +1,5 @@
 #  Phusion Passenger - http://www.modrails.com/
-#  Copyright (c) 2008, 2009 Phusion
+#  Copyright (c) 2010 Phusion
 #
 #  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
 #

data/build/ruby_extension.rb

@@ -0,0 +1,46 @@
+#  Phusion Passenger - http://www.modrails.com/
+#  Copyright (C) 2010  Phusion
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; version 2 of the License.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+native_support_archdir = PlatformInfo.ruby_extension_binary_compatibility_ids.join("-")
+output_dir  = "ext/phusion_passenger/#{native_support_archdir}"
+output_name = "native_support.#{LIBEXT}"
+source_dir  = "ext/phusion_passenger"
+
+task :native_support => "#{output_dir}/#{output_name}"
+task :clean => 'native_support:clean'
+
+dependencies = [
+	"#{output_dir}/Makefile",
+	"#{source_dir}/native_support.c"
+]
+file("#{output_dir}/#{output_name}" => dependencies) do
+	sh "mkdir -p '#{output_dir}'" if !File.exist?(output_dir)
+	sh "cd '#{output_dir}' && make"
+end
+
+file "#{output_dir}/Makefile" => "#{source_dir}/extconf.rb" do
+	sh "mkdir -p '#{output_dir}'" if !File.exist?(output_dir)
+	extconf_rb = File.expand_path("#{source_dir}/extconf.rb")
+	sh "cd '#{output_dir}' && #{PlatformInfo.ruby_command} '#{extconf_rb}'"
+end
+
+task 'native_support:clean' do
+	Dir["ext/phusion_passenger/*"].each do |entry|
+		if File.exist?("#{entry}/Makefile")
+			sh "rm -rf #{entry}"
+		end
+	end
+end

data/build/ruby_tests.rb

@@ -0,0 +1,42 @@
+#  Phusion Passenger - http://www.modrails.com/
+#  Copyright (C) 2008, 2009  Phusion
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; version 2 of the License.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+### Ruby components tests ###
+
+desc "Run unit tests for the Ruby libraries"
+task 'test:ruby' => [:native_support, 'agents/PassengerLoggingAgent'] do
+	if PlatformInfo.rspec.nil?
+		abort "RSpec is not installed for Ruby interpreter '#{PlatformInfo.ruby_command}'. Please install it."
+	else
+		Dir.chdir("test") do
+			ruby "#{PlatformInfo.rspec} -c -f s ruby/*_spec.rb ruby/*/*_spec.rb"
+		end
+	end
+end
+
+desc "Run coverage tests for the Ruby libraries"
+task 'test:rcov' => :native_support do
+	if PlatformInfo.rspec.nil?
+		abort "RSpec is not installed for Ruby interpreter '#{PlatformInfo.ruby_command}'. Please install it."
+	else
+		Dir.chdir("test") do
+			sh "rcov", "--exclude",
+				"lib\/spec,\/spec$,_spec\.rb$,support\/,platform_info,integration_tests",
+				PlatformInfo.rspec, "--", "-c", "-f", "s",
+				*Dir["ruby/*.rb", "ruby/*/*.rb", "integration_tests.rb"]
+		end
+	end
+end

data/build/test_basics.rb

@@ -0,0 +1,31 @@
+#  Phusion Passenger - http://www.modrails.com/
+#  Copyright (C) 2008, 2009  Phusion
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; version 2 of the License.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+TEST_BOOST_OXT_LIBRARY = LIBBOOST_OXT
+TEST_COMMON_LIBRARY    = LIBCOMMON
+
+TEST_COMMON_CFLAGS = "-DTESTING_APPLICATION_POOL " <<
+	"#{PlatformInfo.portability_cflags} #{EXTRA_CXXFLAGS}"
+
+desc "Run all unit tests and integration tests"
+task :test => ['test:oxt', 'test:cxx', 'test:ruby', 'test:integration']
+
+desc "Clean all compiled test files"
+task 'test:clean' do
+	sh("rm -rf test/oxt/oxt_test_main test/oxt/*.o test/cxx/CxxTestMain test/cxx/*.o")
+end
+
+task :clean => 'test:clean'

data/doc/ApplicationPool algorithm.txt

@@ -21,8 +21,8 @@ The algorithm should strive to keep spawning to a minimum.
   "app/", "public/", etc. For a Rack application, this is the directory that
   contains "config.ru".
 
-- "Active application instance":
-  An application instance that has more than 0 active sessions.
+- "Active application process":
+  An application process that has more than 0 sessions.
 
 === Types
 
@@ -34,83 +34,145 @@ explicitly define some special types:
   all the usual list operations that one can expect from a linked list, like
   add_to_back(), etc.
   
+  We assume that operations that insert an element into the list return an
+  iterator object. An iterator object is an opaque object which represents a
+  specific position in the list; it probably contains the links to the previous
+  and the next iterator, as well as a reference to the actual list element,
+  depending on the list implementation.
+  
   The following operations deserve special mention:
   * remove(iterator)
-    Removes the specified element from the list. _iterator_ is a linked list
-    iterator: it probably contains the links and a reference to the actual
-    list element, depending on the list implementation. This operation can be
-    done in O(1) time.
+    Removes the specified element from the list, as represented by the given
+    iterator. This operation can be done in O(1) time.
   
   * move_to_front(iterator)
-    Moves the specified element to the front of the list. _iterator_ is an
-    iterator, as described earlier.
+    Moves the specified element - as represented by the given iterator - to
+    the front of the list. This operation can be done in O(1) time.
   
-- Domain
+- Group
   A compound type (class) which contains information about an application root,
-  such as the application instances that have been spawned for this application
+  such as the application processes that have been spawned for this application
   root.
   
-  A Domain has the following members:
-  * instances (list<AppContainer>) - a list of AppContainer objects.
+  A Group has the following members:
+  * name (string):
+    This group's key in the _groups_ map of the application pool.
+
+  * app_root (string):
+    This group's application root. Note that it is *not* guaranteed that all
+    ProcessInfo objects in _processes_ have the same application root.
+
+  * processes (list<ProcessInfo>):
+    A list of ProcessInfo objects.
     
     Invariant:
-        containers is non-empty.
-        for all 0 <= i < containers.size() - 1:
-           if containers[i].app is active:
-              containers[i + 1].app is active
-
-  * size (unsigned integer): The number of items in _instances_.
+        processes is non-empty.
+        for all 0 <= i < processes.size() - 1:
+           processes[i].group_name == name
+           if processes[i].process is active:
+              processes[i + 1].process is active
+
+  * size (unsigned integer):
+    The number of items in _processes_.
+    Invariant:
+        if !detached:
+           size == processes.size()
   
-  * max_requests (unsigned integer): The maximum number of requests that each
-    application instance in this domain may process. After having processed this
-    many requests, the application instance will be shut down.
+  * max_requests (unsigned integer):
+    The maximum number of requests that each application process in
+    this group may process. After having processed this
+    many requests, the application process will be shut down.
     A value of 0 indicates that there is no maximum.
 
-- AppContainer
-  A compound type (class) which contains an application instance, as well as
-  iterators for various linked lists. These iterators make it possible to
-  perform actions on the linked list in O(1) time.
+  * min_processes (unsigned integer):
+    The minimum number of processes that the cleaner thread should keep in
+    this group. Defaults to 0.
+
+  * spawning (boolean): Whether a background thread is currently spawning
+    a new process for this group.
+
+  * spawner_thread: A handle to the background thread that is currently
+    spawning a new process. Only valid if _spawning_ is true.
+
+  * detached (boolean): If true, then it indicates that this Group is
+    no longer accessible via _groups_.
+    Set to false by the constructor.
+    Invariant:
+       (detached) == (This Group is accessible via _groups_.)
+       if detached:
+          for all process_info objects p that have once been in this.processes:
+              p.detached
+
+  * environment (string): Does nothing. Data is stored in memory for analytics
+    purposes.
+
+- ProcessInfo
+  A compound type (class) which contains a reference to an application process
+  object, as well as various metadata, such as iterators for various linked
+  lists. These iterators make it possible to perform actions on the linked
+  lists in O(1) time.
   
-  An AppContainer has the following members:
-  * app - An Application object, representing an application instance.
-  * start_time (time) - The time at which this application instance was
-    started. It's set to the current time by AppContainer's constructor.
+  A ProcessInfo has the following members:
+  * process - A process object, representing an application process.
+  * group_name (string) - The name of the group that this ProcessInfo belongs
+    to.
+  * identifier (string) - A key that uniquely identifies this ProcessInfo in
+    this application pool. This key allows external processes to refer to a
+    specific ProcessInfo object without knowing its memory pointer. It's set
+    to a random string by ProcessInfo's constructor.
+  * start_time (timestamp with milisecond resolution) - The time at which this
+    application process was started. It's set to the current time by the
+    constructor.
   * processed_requests (integer) - The number of requests processed by this
-    application instance so far.
-  * last_used (time) - The last time a session for this application instance
+    application instance so far. Set to 0 by the constructor.
+  * last_used (time) - The last time a session for this application process
     was opened or closed.
   * sessions (integer) - The number of open sessions for this application
-    instance.
+    process. It's set to 0 by the constructor.
     Invariant:
-       (sessions == 0) == (This AppContainer is in inactive_apps.)
-  * iterator - The iterator for this AppContainer in the linked list
-    domains[app.app_root].instances
-  * ia_iterator - The iterator for this AppContainer in the linked list
-    inactive_apps. This iterator is only valid if this AppContainer really is
+       (sessions == 0 && !detached) == (This ProcessInfo is in inactive_apps.)
+  * iterator - The iterator for this ProcessInfo in the linked list
+    groups[process_info.group_name].processes
+  * ia_iterator - The iterator for this ProcessInfo in the linked list
+    inactive_apps. This iterator is only valid if this ProcessInfo really is
     in that list.
+  * detached (boolean) - If true, then it indicates that this ProcessInfo is
+    no longer accessible via _groups_; this implies that it's no longer
+    contained in its associated Group's _processes_ member, a), and that
+    _iterator_ and _ia_iterator_ are no longer valid.
+    Set to false by the constructor.
+    Invariant:
+       (detached) == (This ProcessInfo is accessible via _groups_.)
 
 - PoolOptions
   A structure containing additional information used by the spawn manager's
   spawning process, as well as by the get() function.
   
   A PoolOptions has at least the following members:
+  * app_group_name (string) - A name which is used to group application
+    processes together.
   * max_requests (unsigned integer) - The maximum number of requests that the
-    application instance may process. After having processed this many requests,
-    the application instance will be shut down. A value of 0 indicates that there
+    application process may process. After having processed this many requests,
+    the application process will be shut down. A value of 0 indicates that there
     is no maximum.
+  * min_processes (unsigned integer) - The minimum number of processes for the
+    current group that the cleaner thread should keep around.
   * use_global_queue (boolean) - Whether to use a global queue for all
-    application instances, or a queue that's private to the application instance.
+    application processes, or a queue that's private to the application process.
     The users guide explains this feature in more detail.
   * restart_dir (string) - The directory in which the algorithm should look for
     restart.txt and always_restart.txt. The existance and modification times of
     these files tell the algorithm whether an application should be restarted.
+  * environment (string) - The environment (RAILS_ENV/RACK_ENV) in which the app
+    should run.
 
 === Special functions
 
-- spawn(app_root)
-  Spawns a new instance of the application at the given application root.
-  Throws an exception if something went wrong. This function is thread-safe.
-  Note that application initialization can take an arbitrary amount of time.
+- spawn(app_root, options)
+  Spawns a new application process at the given application root with the given
+  spawn options. Throws an exception if something went wrong. This function is
+  thread-safe. Note that application process initialization can take an arbitrary
+  amount of time.
 
 === Instance variables
 
@@ -122,41 +184,44 @@ information:
   is non-recursive, i.e. if a thread locks a mutex that it has already locked,
   then it will result in a deadlock.
 
-- domains: map[string => Domain]
-  Maps an application root to its Domain object. This map contains all
-  application instances in the pool.
+- groups: map[string => Group]
+  Maps an application root to its Group object. This map contains all
+  application processes in the pool.
   
   Invariant:
-     for all values d in domains:
-        d.size <= count
-     (sum of all d.size in domains) == count
+     for all values g in groups:
+        !g.detached
+        g.size <= count
+	for all i in g.processes:
+           !i.detached
+     (sum of all g.size in groups) == count
   
 - max: integer
-  The maximum number of AppContainer objects that may exist in the pool.
+  The maximum number of ProcessInfo objects that may exist in the pool.
 
 - max_per_app: integer
-  The maximum number of AppContainer objects that may be simultaneously alive
-  for a single Domain.
+  The maximum number of ProcessInfo objects that may be simultaneously alive
+  for a single Group.
 
 - count: integer
-  The current number of AppContainer objects in the pool.
+  The current number of ProcessInfo objects in the pool.
   Since 'max' can be set dynamically during the life time of an application
   pool, 'count > max' is possible.
 
 - active: integer
-  The number of application instances in the pool that are active.
+  The number of application processes in the pool that are active.
   Invariant:
      active <= count
 
-- inactive_apps: list<AppContainer>
-  A linked list of AppContainer objects. All application instances in this list
+- inactive_apps: list<ProcessInfo>
+  A linked list of ProcessInfo objects. All application processes in this list
   are inactive.
   
   Invariant:
      inactive_apps.size() == count - active
-     for all c in inactive_apps:
-        c can be found in _domains_.
-        c.sessions == 0
+     for all x in inactive_apps:
+        x can be accessed from _groups_.
+        x.sessions == 0
 
 - waiting_on_global_queue: integer
   If global queuing mode is enabled, then when get() is waiting for a backend
@@ -168,12 +233,12 @@ information:
 
 Here's an UML diagram in ASCII art:
 
-[AppContainer] 1..* -------+
+[ProcessInfo] 1..* --------+
                            |
                            |
                            
                            1
-[ApplicationPool]       [Domain]
+[ApplicationPool]       [Group]
        1                  0..*
 
        |                   |
@@ -185,181 +250,156 @@ Here's an UML diagram in ASCII art:
 # Thread-safetiness notes:
 # - All wait commands are to unlock the lock during waiting.
 
-# Connect to an existing application instance or to a newly spawned application instance.
-# 'app_root' specifies the application root folder of the application. 'options' is an
-# object of type 'PoolOptions', which contains additional information which may be
-# relevant for spawning.
+
+# Connect to an existing application process, or spawn a new application process
+# and connect to that if necessary.
+# 'app_root' refers to an application root.
+# 'options' is an object of type 'PoolOptions', which contains additional
+# information which may be relevant for spawning.
+#
+# Returns a Session object, representing a single HTTP request/response pair.
 function get(app_root, options):
 	MAX_ATTEMPTS = 10
 	attempt = 0
-	time_limit = now() + 5 seconds
+	while (true):
+		attempt++
+		lock.synchronize:
+			process_info, group = checkout_without_lock(app_root, options)
+		try:
+			return process_info.process.connect()
+		on exception:
+			# The app process seems to have crashed.
+			# So we remove this process from our data
+			# structures.
+			lock.synchronize:
+				detach_without_lock(process_info.identifier)
+				process_info.sessions--
+			if (attempt == MAX_ATTEMPTS):
+				propagate exception
+
+
+# Detach the process with the given identifier from the pool's data structures.
+function detach(identifier):
 	lock.synchronize:
-		while (true):
-			attempt++
-			container, domain = spawn_or_use_existing(app_root, options)
-			container.last_used = current_time()
-			container.sessions++
-			try:
-				return container.app.connect()
-			on exception:
-				# The app instance seems to have crashed.
-				# So we remove this instance from our data
-				# structures.
-				container.sessions--
-				instances = domain.instances
-				instances.remove(container.iterator)
-				domain.size--
-				if instances.empty():
-					domains.remove(app_root)
-				count--
-				active--
-				if (attempt == MAX_ATTEMPTS):
-					propagate exception
-
-
-# Returns a pair of [AppContainer, Domain] that matches the given application
-# root. If no such AppContainer exists, then it is created and a new
-# application instance is spawned. All exceptions that occur are propagated.
-function spawn_or_use_existing(app_root, options):
-	domain = domains[app_root]
+		return detach_without_lock(identifier)
+
+
+# Checkout a process from the application pool and mark it as being used.
+# If there's no appropriate process in the pool, or if there are not
+# enough processes, then one will be spawned.
+#
+# Returns a pair of [ProcessInfo, Group].
+# All exceptions that occur are propagated.
+private function checkout_without_lock(app_root, options):
+	group = groups[options.app_group_name]
 	
 	if needs_restart(app_root, options):
-		if (domain != nil):
-			for all container in domain.instances:
-				if container.sessions == 0:
-					inactive_apps.remove(container.ia_iterator)
-				else:
-					active--
-				domain.instances.remove(container.iterator)
-				count--
-			domains.remove(app_root)
-		Tell spawn server to reload code for app_root.
-		domain = nil
+		Tell spawn server to reload code for options.app_group_name.
+		if (group != null):
+			detach_group_without_lock(group)
+			group = null
 	
-	if domain != nil:
-		# There are apps for this app root.
-		instances = domain.instances
+	if (group != null):
+		# There are existing processes for this app group.
+		processes = group.processes
 		
-		if (instances.front.sessions == 0):
-			# There is an inactive app, so we use it.
-			container = instances.front
-			instances.move_to_back(container.iterator)
-			inactive_apps.remove(container.ia_iterator)
-			active++
-		else if	(count >= max) or (
-			(max_per_app != 0) and (domain.size >= max_per_app)
-			):
-			# All apps are active, and the pool is full. 
-			#  -OR-
-			# All apps are active and the number of max instances
-			# spawned for this application domain has been reached.
-			#
-			# We're not allowed to spawn a new application instance.
-			if options.use_global_queue:
-				# So we wait until _active_ has changed, then
-				# we restart this function and try again.
-				waiting_on_global_queue++
-				wait until _active_ has changed
-				waiting_on_global_queue--
-				goto beginning of function
-			else:
-				# So we connect to an already active application.
-				# This connection will be put into that
-				# application's private queue.
-				container = a container in _instances_ with the smallest _session_ value
-				instances.move_to_back(container.iterator)
+		if (processes.front.sessions == 0):
+			# There is an inactive process, so we use it.
+			process_info = processes.front
+			processes.move_to_back(process_info.iterator)
+			inactive_apps.remove(process_info.ia_iterator)
+			mutate_max(active + 1)
 		else:
-			# All apps are active, but the pool hasn't reached its
-			# maximum yet. So we spawn a new app.
-			container = new AppContainer
-			# TODO: we should add some kind of timeout check for spawning.
-			container.app = spawn(app_root)
-			container.sessions = 0
-			iterator = instances.add_to_back(container)
-			container.iterator = iterator
-			domain.size++
-			count++
-			active++
+			# All existing processes are active. We either use
+			# one of them now or we wait until one of them becomes
+			# available. And, if we're allowed to, we spawn an
+			# extra process in the background.
+			if spawning_allowed(group, options) and !group.spawning:
+				spawn_in_background(group, options)
+			process_info = select_process(processes, options)
+			if (process_info == null):
+				goto beginning of function
 	else:
-		# There are no apps for this app root. Wait until there's at
-		# least 1 idle instance, or until there's an empty slot in the
-		# pool, then restart this function. Restarting is necessary,
-		# because after waiting and reacquiring the lack, some other
-		# thread might already have spawned instances for this app root.
+		# There are no processes for this app group.
 		if (active >= max):
-			wait until _active_ has changed
+			# Looks like the pool is full and all processes are busy.
+			# Wait until the pool appears to have changed in such a
+			# way that we can spawn a new app group, and restart
+			# this function.
+			new_app_group_creatable.wait
 			goto beginning of function
 		elsif count == max:
-			# Here we are in a though situation. There are several
-			# apps which are inactive, and none of them have
-			# application root _app_root_, so we must kill one of
-			# them in order to free a spot in the pool. But which
+			# The pool is full, and not all processes are busy, but
+			# we're in a though situation nevertheless: there are
+			# several processes which are inactive, and none of them
+			# belong to our current app group, so we must kill one
+			# of them in order to free a spot in the pool. But which
 			# one do we kill? We want to minimize spawning.
 			#
 			# It's probably a good idea to keep some kind of
 			# statistics in order to decide this. We want the
 			# application root that gets the least traffic to be
 			# killed. But for now, we kill a random application
-			# instance.
-			container = inactive_apps.pop_front
-			domain = domains[container.app.app_root]
-			instances = domain.instances
-			instances.remove(container.iterator)
-			if instances.empty():
-				domains.remove(container.app.app_root)
+			# process.
+			process_info = inactive_apps.pop_front
+			process_info.detached = true
+			group = groups[process_info.group_name]
+			processes = group.processes
+			processes.remove(process_info.iterator)
+			if processes.empty():
+				detach_group_without_lock(group)
 			else:
-				domain.size--
-			count--
-		container = new AppContainer
-		# TODO: we should add some kind of timeout check for spawning.
-		container.app = spawn(app_root)
-		container.sessions = 0
-		domain = domains[app_root]
-		if domain == nil:
-			domain = new Domain
-			initialize domain.instances
-			domain.size = 1
-			domain.max_requests = options.max_requests
-			domains[app_root] = domain
-		else:
-			domain.size++
-		iterator = domain.instances.add_to_back(container)
-		container.iterator = iterator
-		count++
-		active++
-	return [container, domain]
+				group.size--
+			mutate_count(count - 1)
+		process_info = new ProcessInfo
+		process_info.process = spawn(app_root, options)
+		process_info.group_name = options.app_group_name
+		group = new Group
+		group.name = options.app_group_name
+		group.app_root = app_root
+		group.size = 1
+		groups[options.app_group_name] = group
+		iterator = group.processes.add_to_back(process_info)
+		process_info.iterator = iterator
+		mutate_count(count + 1)
+		mutate_active(active + 1)
+		if (options.min_processes > 1) and spawning_allowed(group, options):
+			spawn_in_background(group, options)
+	
+	group.max_requests = options.max_requests
+	group.min_processes = options.min_processes
+	group.environment = options.environment
+	
+	process_info.last_used = current_time()
+	process_info.sessions++
+	
+	return [process_info, group]
 
 
-# The following function is to be called when a session has been closed.
-# _container_ is the AppContainer that contains the application for which a
-# session has been closed.
-function session_has_been_closed(container):
-	lock.synchronize:
-		domain = domains[container.app.app_root]
-		if domain != nil:
-			instances = domain.instances
-			container.processed++
-			
-			if (domain.max_requests) > 0 and (container.processed >= domain.max_requests):
-				# The application instance has processed its maximum allowed
-				# number of requests, so we shut it down.
-				instances.remove(container.iterator)
-				domain.size--
-				if instances.empty():
-					domains.remove(app_root)
-				count--
-				active--
-			else:
-				container.last_used = current_time()
-				container.sessions--
-				container.processed++
-				if container.sessions == 0:
-					instances.move_to_front(container.iterator)
-					container.ia_iterator = inactive_apps.add_to_back(container.app)
-					active--
+private function mutate_active(value):
+	if (value < active):
+		new_app_group_creatable.notify_all
+		global_queue_position_became_available.notify_all
+	active = value
+
+private function mutate_count(value):
+	# No point in notifying new_app_group_creatable here;
+	# if _count_ is being increased then that means the pool
+	# isn't full, and nobody is waiting on
+	# new_app_group_creatable.
+	global_queue_position_became_available.notify_all
+	count = value
 
+private function mutate_max(value):
+	if (value > max):
+		new_app_group_creatable.notify_all
+		# We will want any code waiting on the global queue
+		# to go ahead and spawn another process.
+		global_queue_position_became_available.notify_all
+	max = value
 
-function needs_restart(app_root, options):
+
+private function needs_restart(app_root, options):
 	if (options.restart_dir is not set):
 		restart_dir = app_root + "/tmp"
 	else if (options.restart_dir is an absolute path):
@@ -372,31 +412,204 @@ function needs_restart(app_root, options):
 	       ("$restart_dir/restart.txt" changed since the last time we checked)
 
 
+private function spawning_allowed(group, options):
+	return ( count < max ) and
+		( (max_per_app == 0) or (group.size < max_per_app) )
+
+
+# Precondition: !group.detached
+private function detach_group_without_lock(group):
+	for all process_info in group.processes:
+		if (process_info.sessions == 0):
+			inactive_apps.remove(process_info.ia_iterator)
+		else:
+			mutate_active(active - 1)
+		group.processes.remove(process_info.iterator)
+		process_info.detached = true
+		mutate_count(count - 1)
+	if (group.spawning):
+		group.spawner_thread.interrupt_and_join
+		group.spawner_thread = null
+		group.spawning = false
+	group.detached = true
+	groups.remove(options.app_group_name)
+
+
+private function select_process(processes, options):
+	if options.use_global_queue:
+		# So we wait until _active_ has changed, then
+		# we restart this function and try again.
+		waiting_on_global_queue++
+		global_queue_position_became_available.wait
+		waiting_on_global_queue--
+		return null
+	else:
+		# So we connect to an already active process.
+		# This connection will be put into that
+		# process's private queue.
+		process_info = an element in _processes_ with the smallest _session_ value
+		processes.move_to_back(process_info.iterator)
+		return process_info
+
+
+# Preconditions:
+#   !group.detached
+#   !group.spawning
+private function spawn_in_background(group, options):
+	group.spawning = true
+	group.spawner_thread = new thread(spawner_thread_callback,
+		with these arguments to the thread function:
+		group, options)
+
+
+private function spawner_thread_callback(group, options):
+	Ignore thread interruptions in this function
+	while true:
+		try:
+			Allow thread interruptions in this block
+			process = spawn(app_root, options)
+		on thread interruption:
+			lock.synchronize:
+				group.spawning = false
+				group.spawner_thread = null
+				return
+		on exception:
+			lock.synchronize:
+				if (!group.detached):
+					group.spawning = false
+					group.spawner_thread = null
+					# We want to report the error to the browser
+					# but there's no way to do this in this thread, so
+					# we just remove the entire group and have the next
+					# get() call spawn the process and display the
+					# error.
+					remove_group_without_lock(group)
+				return
+		
+		lock.synchronize:
+			if (group.detached):
+				return
+			else:
+				process_info = new ProcessInfo
+				process_info.process = process
+				process_info.group_name = options.app_group_name
+				process_info.iterator = group.processes.add_to_front(process_info)
+				process_info.ia_iterator = inactive_apps.add_to_back(process_info)
+				group.size++
+				mutate_count(count + 1)
+				if (group.size >= options.min_processes) or
+				   (!spawning_allowed(group, options)):
+					group.spawning = false
+					group.spawner_thread = null
+					return
+
+
+private function detach_without_lock(identifier):
+	for group in groups:
+		processes = group.processes
+		for process_info in processes:
+			if process_info.identifier == identifier:
+				# Found a matching process.
+				process_info.detached = true
+				processes.remove(process_info.iterator)
+				group.size--
+				if processes.empty():
+					detach_group_without_lock(group)
+				if process_info.sessions == 0:
+					inactive_apps.remove(process_info.ia_iterator)
+				else:
+					mutate_active(active - 1)
+				mutate_count(count - 1)
+				return true
+	return false
+
+
+# The following function is to be called when a session has been closed.
+# _process_info_ is a weak reference to the ProcessInfo that belongs to
+# the process whose session has been closed; it evaluates to NULL if the
+# ProcessInfo object that it belongs to has been destroyed.
+callback session_has_been_closed(session, process_info):
+	Convert process_info into a normal reference.
+	
+	# We check process_info.detached without locking. This should be safe:
+	# even if a boolean update isn't atomic on the current CPU, a non-zero
+	# value evaluates to true. Once true, _detached_ will never become false,
+	# so instruction reorderings by the compiler or CPU won't cause any
+	# damage.
+	if (process_info == null) or (process_info.detached):
+		return
+	
+	lock.synchronize:
+		if process_info.detached:
+			return
+		
+		group = groups[process_info.group_name]
+		processes = group.processes
+		process_info.processed++
+		
+		if (group.max_requests > 0) and (process_info.processed >= group.max_requests):
+			# The application process has processed its maximum allowed
+			# number of requests, so we shut it down.
+			process_info.detached = true
+			processes.remove(process_info.iterator)
+			group.size--
+			if processes.empty():
+				detach_group_without_lock(group)
+			mutate_count(count - 1)
+			if (process_info.sessions == 0):
+				inactive_apps.remove(process_info.ia_iterator)
+			else:
+				mutate_active(active - 1)
+		else:
+			process_info.last_used = current_time()
+			process_info.sessions--
+			if (process_info.sessions == 0):
+				processes.move_to_front(process_info.iterator)
+				process_info.ia_iterator = inactive_apps.add_to_back(process_info)
+				mutate_active(active - 1)
+
+
 # The following thread will be responsible for cleaning up idle application
-# instances, i.e. instances that haven't been used for a while.
+# process, i.e. processes that haven't been used for a while.
 # This can be disabled per app when setting it's maxIdleTime to 0.
 thread cleaner:
 	lock.synchronize:
-		done = false
-		while !done:
-			Wait until CLEAN_INTERVAL seconds have expired, or until the thread has been signalled to quit.
-			if thread has been signalled to quit:
-				done = true
-				break
+		while true:
+			# If MAX_IDLE_TIME is 0 we don't clean up any processes,
+			# giving us the option to persist the processes
+			# forever unless it's killed in order to free up space
+			# for another process.
+			if (MAX_IDLE_TIME == 0):
+				Wait until the thread has been signalled to quit
+				  or until MAX_IDLE_TIME changed.
+				if thread has been signalled to quit:
+					return
+				else:
+					restart loop
+			else:
+				Wait until MAX_IDLE_TIME seconds have passed,
+				  or until the thread has been signalled to quit,
+				  or until MAX_IDLE_TIME changed.
+				if thread has been signalled to quit:
+					return
+				else if MAX_IDLE_TIME changed:
+					restart loop
+			
+			# Invariant:
+			# From this point on, MAX_IDLE_TIME > 0
 			
 			now = current_time()
-			for all container in inactive_apps:
-				app = container.app
-				domain = domains[app.app_root]
-				instances = domain.instances
-				# If MAX_IDLE_TIME is 0 we don't clean up the instance,
-				# giving us the option to persist the app container
-				# forever unless it's killed by another app.
-				if (MAX_IDLE_TIME > 0) and (now - container.last_used > MAX_IDLE_TIME):
-					instances.remove(container.iterator)
-					inactive_apps.remove(container.iterator)
-					domain.size--
-					count--
-				if instances.empty():
-					domains.remove(app.app_root)
+			for all process_info in inactive_apps:
+				if (now - process_info.last_used > MAX_IDLE_TIME):
+					process = process_info.process
+					group = groups[process_info.group_name]
+					if (group.size > group.min_processes):
+						processes = group.processes
+						processes.remove(process_info.iterator)
+						process_info.detached = true
+						inactive_apps.remove(process_info.ia_iterator)
+						group.size--
+						mutate_count(count - 1)
+						if processes.empty():
+							detach_group_without_lock(group)