rockdog-passenger 0.0.1

data/bin/passenger-stress-test

@@ -0,0 +1,339 @@
+#!/usr/bin/env ruby
+#  Phusion Passenger - http://www.modrails.com/
+#  Copyright (C) 2008  Phusion
+#
+#  Phusion Passenger is a trademark of Hongli Lai & Ninh Bui.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; version 2 of the License.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+$LOAD_PATH << File.expand_path(File.dirname(__FILE__) + "/../lib")
+$LOAD_PATH << File.expand_path(File.dirname(__FILE__) + "/../ext")
+require 'rubygems'
+require 'optparse'
+require 'socket'
+require 'thread'
+require 'phusion_passenger/platform_info'
+require 'phusion_passenger/message_channel'
+require 'phusion_passenger/utils'
+
+include PhusionPassenger
+include PhusionPassenger::Utils
+include PlatformInfo
+
+# A thread or a process, depending on the Ruby VM implementation.
+class Subprocess
+	attr_accessor :channel
+	
+	def initialize(name, &block)
+		if RUBY_PLATFORM == "java"
+			a, b = UNIXSocket.pair
+			@thread = Thread.new do
+				block.call(true, MessageChannel.new(b))
+			end
+			@channel = MessageChannel.new(a)
+			@thread_channel = b
+		else
+			a, b = UNIXSocket.pair
+			@pid = safe_fork(name) do
+				a.close
+				$0 = name
+				Process.setsid
+				block.call(false, MessageChannel.new(b))
+			end
+			b.close
+			@channel = MessageChannel.new(a)
+		end
+	end
+	
+	def stop
+		if RUBY_PLATFORM == "java"
+			@thread.terminate
+			@channel.close
+			@thread_channel.close
+		else
+			Process.kill('SIGKILL', @pid) rescue nil
+			Process.waitpid(@pid) rescue nil
+			@channel.close
+		end
+	end
+end
+
+class StressTester
+	def start
+		@options = parse_options
+		load_hawler
+		
+		Thread.abort_on_exception = true
+		if GC.respond_to?(:copy_on_write_friendly=)
+			GC.copy_on_write_friendly = true
+		end
+		@terminal_height = ENV['LINES'] ? ENV['LINES'].to_i : 24
+		@terminal_width = ENV['COLUMNS'] ? ENV['COLUMNS'].to_i : 80
+		
+		if Process.euid != 0
+			puts "*** WARNING: This program might not be able to restart " <<
+				"Apache because it's not running as root. Please run " <<
+				"this tool as root."
+			puts
+			puts "Press Enter to continue..."
+			begin
+				STDIN.readline
+			rescue Interrupt
+				exit 1
+			end
+		end
+		
+		run_crawlers
+	end
+
+	def parse_options
+		options = {
+			:concurrency => 20,
+			:depth => 20,
+			:nice => true,
+			:apache_restart_interval => 24 * 60,
+			:app_restart_interval => 55
+		}
+		parser = OptionParser.new do |opts|
+			opts.banner = "Usage: passenger-stress-test <hostname> <app_root> [options]\n\n" <<
+				"Stress test the given (Passenger-powered) website by:\n" <<
+				"  * crawling it with multiple concurrently running crawlers.\n" <<
+				"  * gracefully restarting Apache at random times (please point the 'APXS2'\n" <<
+				"    variable to your Apache's 'apxs' binary).\n" <<
+				"  * restarting the target (Passenger-powered) application at random time.\n" <<
+				"\n" <<
+				"Example:\n" <<
+				"  passenger-stress-test mywebsite.com /webapps/mywebsite\n" <<
+				"\n"
+		
+			opts.separator "Options:"
+			opts.on("-c", "--concurrency N", Integer,
+				"Number of crawlers to start (default = #{options[:concurrency]})") do |v|
+				options[:concurrency] = v
+			end
+			opts.on("-p", "--apache-restart-interval N", Integer,
+				"Gracefully restart Apache after N minutes\n" <<
+				(" " * 37) << "(default = #{options[:apache_restart_interval]})") do |v|
+				options[:apache_restart_interval] = v
+			end
+			opts.on("-a", "--app-restart-interval N", Integer,
+				"Restart the application after N minutes\n" <<
+				(" " * 37) << "(default = #{options[:app_restart_interval]})") do |v|
+				options[:app_restart_interval] = v
+			end
+			opts.on("-h", "--help", "Show this message") do
+				puts opts
+				exit
+			end
+		end
+		parser.parse!
+	
+		options[:host] = ARGV[0]
+		options[:app_root] = ARGV[1]
+		if !options[:host] || !options[:app_root]
+			puts parser
+			exit 1
+		end
+		return options
+	end
+
+	def load_hawler
+		begin
+			require 'hawler'
+		rescue LoadError
+			STDERR.puts "This tool requires Hawler (http://tinyurl.com/ywgk6x). Please install it with:"
+			STDERR.puts
+			STDERR.puts "  gem install --source http://spoofed.org/files/hawler/ hawler"
+			exit 1
+		end
+	end
+
+	def run_crawlers
+		@started = false
+		@crawlers = []
+		
+		# Start crawler processes.
+		GC.start if GC.copy_on_write_friendly?
+		@options[:concurrency].times do |i|
+			STDOUT.write("Starting crawler #{i + 1} of #{@options[:concurrency]}...\n")
+			STDOUT.flush
+			process = Subprocess.new("crawler #{i + 1}") do |is_thread, channel|
+				if !is_thread && @options[:nice]
+					system("renice 1 #{Process.pid} >/dev/null 2>/dev/null")
+				end
+				while true
+					crawl!(i + 1, channel)
+				end
+			end
+			@crawlers << {
+				:id => i + 1,
+				:process => process,
+				:channel => process.channel,
+				:mutex => Mutex.new,
+				:current_uri => nil,
+				:crawled => 0
+			}
+		end
+		
+		puts
+		if RUBY_PLATFORM != "java"
+			# 'sleep' b0rks when running in JRuby?
+			sleep 1
+		end
+		begin
+			$0 = "Passenger Crawler: control process"
+			io_to_crawler = {}
+			ios = []
+			@crawlers.each do |crawler|
+				io_to_crawler[crawler[:channel].io] = crawler
+				ios << crawler[:channel].io
+			end
+			
+			# Tell each crawler to start crawling.
+			@crawlers.each do |crawler|
+				crawler[:channel].write("start")
+			end
+			
+			# Show progress periodically.
+			@start_time = Time.now
+			progress_reporter = Thread.new(&method(:report_progress))
+			@next_apache_restart = Time.now + @options[:apache_restart_interval] * 60
+			apache_restarter = Thread.new(&method(:restart_apache))
+			@next_app_restart = Time.now + @options[:app_restart_interval] * 60
+			app_restarter = Thread.new(&method(:restart_app))
+			
+			while true
+				note_progress(ios, io_to_crawler)
+			end
+		rescue Interrupt
+			trap('SIGINT') {}
+			puts "Shutting down..."
+			@done = true
+			@crawlers.each do |crawler|
+				STDOUT.write("Stopping crawler #{crawler[:id]} of #{@options[:concurrency]}...\r")
+				STDOUT.flush
+				crawler[:process].stop
+			end
+			progress_reporter.join if progress_reporter
+			apache_restarter.join if apache_restarter
+			app_restarter.join if app_restarter
+			puts
+		end
+	end
+	
+	def note_progress(ios, io_to_crawler)
+		select(ios)[0].each do |io|
+			crawler = io_to_crawler[io]
+			uri = crawler[:channel].read[0]
+			crawler[:mutex].synchronize do
+				crawler[:current_uri] = uri
+				crawler[:crawled] += 1
+			end
+		end
+	end
+	
+	def report_progress
+		while !@done
+			output = "\n" * @terminal_height
+			output << "### Running for #{duration(Time.now.to_i - @start_time.to_i)}\n"
+			@crawlers.each do |crawler|
+				crawler[:mutex].synchronize do
+					line = sprintf("Crawler %-2d: %-3d -> %s",
+						crawler[:id],
+						crawler[:crawled],
+						crawler[:current_uri])
+					output << sprintf("%-#{@terminal_width}s\n", line)
+				end
+			end
+			output << "Next Apache restart: in #{duration(@next_apache_restart.to_i - Time.now.to_i)}\n"
+			output << "Next app restart   : in #{duration(@next_app_restart.to_i - Time.now.to_i)}\n"
+			STDOUT.write(output)
+			sleep 0.5
+		end
+	end
+	
+	def restart_apache
+		while !@done
+			if Time.now > @next_apache_restart
+				@next_apache_restart = Time.now + @options[:apache_restart_interval] * 60
+				system("#{HTTPD} -k graceful")
+			end
+		end
+	end
+	
+	def restart_app
+		while !@done
+			if Time.now > @next_app_restart
+				@next_app_restart = Time.now + @options[:app_restart_interval] * 60
+				system("touch #{@options[:app_root]}/tmp/restart.txt")
+			end
+		end
+	end
+	
+	def duration(seconds)
+		result = ""
+		if seconds >= 60
+			minutes = (seconds / 60)
+			if minutes >= 60
+				hours = minutes / 60
+				minutes = minutes % 60
+				if hours == 1
+					result << "#{hours} hour "
+				else
+					result << "#{hours} hours "
+				end
+			end
+			
+			seconds = seconds % 60
+			if minutes == 1
+				result << "#{minutes} minute "
+			else
+				result << "#{minutes} minutes "
+			end
+		end
+		result << "#{seconds} seconds"
+		return result
+	end
+
+	def crawl!(id, channel)
+		progress_reporter = lambda do |uri, referer, response|
+			begin
+				if !@started
+					# At the beginning, wait until the control process
+					# tells us to start.
+					@started = true
+					channel.read
+				end
+				channel.write(uri, referer, response)
+			rescue
+				if RUBY_PLATFORM == "java"
+					Thread.current.terminate
+				else
+					Process.kill('SIGKILL', Process.pid)
+				end
+			end
+		end
+		crawler = Hawler.new(@options[:host], progress_reporter)
+		if RUBY_PLATFORM == "java"
+			trap('SIGINT') do
+				raise Interrupt, "Interrupted"
+			end
+		end
+		crawler.recurse = true
+		crawler.depth = @options[:depth]
+		crawler.start
+	end
+end
+
+StressTester.new.start

data/debian/compat

@@ -0,0 +1 @@
+5

data/debian/control

@@ -0,0 +1,21 @@
+Package: passenger
+Version: 1.1.0
+Section: net
+Priority: optional
+Architecture: any
+Essential: no
+Depends: 
+Pre-Depends: 
+XSB-Comment: We put dependencies in Recommends because a lot of people
+ have Ruby/RubyGems/Apache installed from source instead of from apt.
+Recommends: ruby1.8, rubygems, apache2
+Suggests: 
+Maintainer: Hongli Lai <hongli@phusion.nl>
+Conflicts: 
+Replaces: 
+Provides: 
+Installed-Size: 1741
+Description: Phusion Passenger, Ruby web application deployment module for Apache.
+ Phusion Passenger makes deploying Ruby and Ruby on Rails web applications on Apache
+ a breeze. Please don't forget to run 'passenger-install-apache2-module' after the
+ installation.

data/debian/postinst

@@ -0,0 +1,24 @@
+#!/bin/sh
+if ruby -e ''; then
+	if ! ruby -rpassenger/native_support -e '' 2>/dev/null; then
+		export RUBYLIB=/usr/lib/ruby/1.8
+	fi
+	passenger-install-apache2-module
+	if ! ruby -rubygems -rfastthread -e '' 2>/dev/null; then
+		echo
+		echo
+		echo "****** WARNING WARNING WARNING WARNING WARNING WARNING WARNING ******"
+		echo
+		echo "Passenger requires the Ruby library 'fastthread', which does not seem to be installed. Since there's no native package for it in the Debian repository, you must install it via RubyGems:"
+		echo
+		echo "  gem install fastthread"
+		echo
+		echo "Please install fastthread, otherwise Passenger won't work."
+		echo
+		echo "****** WARNING WARNING WARNING WARNING WARNING WARNING WARNING ******"
+	fi
+else
+	echo "********* NOTE ***********"
+	echo "Ruby doesn't seem to be installed. Please install Ruby and RubyGems, because Passenger needs them."
+fi
+exit 0

data/debian/prerm

@@ -0,0 +1,2 @@
+#!/bin/sh
+

data/doc/ApplicationPool algorithm.txt

@@ -0,0 +1,398 @@
+= ApplicationPool algorithm
+
+
+== Introduction
+
+For efficiency reasons, Passenger keeps a pool spawned Rails/Ruby applications.
+Please read the C++ API documentation for the ApplicationPool class for a full
+introduction. This document describes an algorithm for managing the pool, in a
+high-level way.
+
+The algorithm should strive to keep spawning to a minimum.
+
+
+== Definitions
+
+=== Vocabulary
+
+- "Application root":
+  The toplevel directory in which an application is contained. For Rails
+  application, this is the same as RAILS_ROOT, i.e. the directory that contains
+  "app/", "public/", etc. For a Rack application, this is the directory that
+  contains "config.ru".
+
+- "Active application instance":
+  An application instance that has more than 0 active sessions.
+
+=== Types
+
+Most of the types that we use in this document are pretty standard. But we
+explicitly define some special types:
+
+- list<SomeType>
+  A doubly linked list which contains elements of type SomeType. It supports
+  all the usual list operations that one can expect from a linked list, like
+  add_to_back(), etc.
+  
+  The following operations deserve special mention:
+  * remove(iterator)
+    Removes the specified element from the list. _iterator_ is a linked list
+    iterator: it probably contains the links and a reference to the actual
+    list element, depending on the list implementation. This operation can be
+    done in O(1) time.
+  
+  * move_to_front(iterator)
+    Moves the specified element to the front of the list. _iterator_ is an
+    iterator, as described earlier.
+  
+- Domain
+  A compound type (class) which contains information about an application root,
+  such as the application instances that have been spawned for this application
+  root.
+  
+  A Domain has the following members:
+  * instances (list<AppContainer>) - a list of AppContainer objects.
+    
+    Invariant:
+        containers is non-empty.
+        for all 0 <= i < containers.size() - 1:
+           if containers[i].app is active:
+              containers[i + 1].app is active
+
+  * size (unsigned integer): The number of items in _instances_.
+  
+  * max_requests (unsigned integer): The maximum number of requests that each
+    application instance in this domain may process. After having processed this
+    many requests, the application instance will be shut down.
+    A value of 0 indicates that there is no maximum.
+  
+  * restart_file_checker
+    An object which monitors the restart.txt file, which belongs to this
+    application root, for changes. This object has the method changed(),
+    which rteurns whether restart.txt's timestamp has changed since the last
+    check.
+
+- AppContainer
+  A compound type (class) which contains an application instance, as well as
+  iterators for various linked lists. These iterators make it possible to
+  perform actions on the linked list in O(1) time.
+  
+  An AppContainer has the following members:
+  * app - An Application object, representing an application instance.
+  * start_time (time) - The time at which this application instance was
+    started. It's set to the current time by AppContainer's constructor.
+  * processed_requests (integer) - The number of requests processed by this
+    application instance so far.
+  * last_used (time) - The last time a session for this application instance
+    was opened or closed.
+  * sessions (integer) - The number of open sessions for this application
+    instance.
+    Invariant:
+       (sessions == 0) == (This AppContainer is in inactive_apps.)
+  * iterator - The iterator for this AppContainer in the linked list
+    domains[app.app_root].instances
+  * ia_iterator - The iterator for this AppContainer in the linked list
+    inactive_apps. This iterator is only valid if this AppContainer really is
+    in that list.
+
+- PoolOptions
+  A structure containing additional information used by the spawn manager's
+  spawning process, as well as by the get() function.
+  
+  A PoolOptions has the following members:
+  * max_requests (unsigned integer) - The maximum number of requests that the
+    application instance may process. After having processed this many requests,
+    the application instance will be shut down. A value of 0 indicates that there
+    is no maximum.
+  * use_global_queue (boolean) - Whether to use a global queue for all
+    application instances, or a queue that's private to the application instance.
+    The users guide explains this feature in more detail.
+
+=== Special functions
+
+- spawn(app_root)
+  Spawns a new instance of the application at the given application root.
+  Throws an exception if something went wrong. This function is thread-safe.
+  Note that application initialization can take an arbitrary amount of time.
+
+=== Instance variables
+
+The algorithm requires the following instance variables for storing state
+information:
+
+- lock: mutex
+  This lock is used for implementing thread-safetiness. We assume that it
+  is non-recursive, i.e. if a thread locks a mutex that it has already locked,
+  then it will result in a deadlock.
+
+- domains: map[string => Domain]
+  Maps an application root to its Domain object. This map contains all
+  application instances in the pool.
+  
+  Invariant:
+     for all values d in domains:
+        d.size <= count
+     (sum of all d.size in domains) == count
+  
+- max: integer
+  The maximum number of AppContainer objects that may exist in the pool.
+
+- max_per_app: integer
+  The maximum number of AppContainer objects that may be simultaneously alive
+  for a single Domain.
+
+- count: integer
+  The current number of AppContainer objects in the pool.
+  Since 'max' can be set dynamically during the life time of an application
+  pool, 'count > max' is possible.
+
+- active: integer
+  The number of application instances in the pool that are active.
+  Invariant:
+     active <= count
+
+- inactive_apps: list<AppContainer>
+  A linked list of AppContainer objects. All application instances in this list
+  are inactive.
+  
+  Invariant:
+     inactive_apps.size() == count - active
+     for all c in inactive_apps:
+        c can be found in _domains_.
+        c.sessions == 0
+
+- waiting_on_global_queue: integer
+  If global queuing mode is enabled, then when get() is waiting for a backend
+  process to become idle, this variable will be incremented. When get() is done
+  waiting, this variable will be decremented.
+
+
+== Class relations
+
+Here's an UML diagram in ASCII art:
+
+[AppContainer] 1..* -------+
+                           |
+                           |
+                           
+                           1
+[ApplicationPool]       [Domain]
+       1                  0..*
+
+       |                   |
+       +-------------------+
+
+
+== Algorithm in pseudo code
+
+# Thread-safetiness notes:
+# - All wait commands are to unlock the lock during waiting.
+
+# Connect to an existing application instance or to a newly spawned application instance.
+# 'app_root' specifies the application root folder of the application. 'options' is an
+# object of type 'PoolOptions', which contains additional information which may be
+# relevant for spawning.
+function get(app_root, options):
+	MAX_ATTEMPTS = 10
+	attempt = 0
+	time_limit = now() + 5 seconds
+	lock.synchronize:
+		while (true):
+			attempt++
+			container, domain = spawn_or_use_existing(app_root, options)
+			container.last_used = current_time()
+			container.sessions++
+			try:
+				return container.app.connect()
+			on exception:
+				# The app instance seems to have crashed.
+				# So we remove this instance from our data
+				# structures.
+				container.sessions--
+				instances = domain.instances
+				instances.remove(container.iterator)
+				domain.size--
+				if instances.empty():
+					domains.remove(app_root)
+				count--
+				active--
+				if (attempt == MAX_ATTEMPTS):
+					propagate exception
+
+
+# Returns a pair of [AppContainer, Domain] that matches the given application
+# root. If no such AppContainer exists, then it is created and a new
+# application instance is spawned. All exceptions that occur are propagated.
+function spawn_or_use_existing(app_root, options):
+	domain = domains[app_root]
+	
+	if (domain != nil) and (needs_restart(app_root, domain)):
+		for all container in domain.instances:
+			if container.sessions == 0:
+				inactive_apps.remove(container.ia_iterator)
+			else:
+				active--
+			domain.instances.remove(container.iterator)
+			count--
+		domains.remove(app_root)
+		list = nil
+		Tell spawn server to reload code for app_root.
+	
+	if domain != nil:
+		# There are apps for this app root.
+		instances = domain.instances
+		
+		if (instances.front.sessions == 0):
+			# There is an inactive app, so we use it.
+			container = instances.front
+			instances.move_to_back(container.iterator)
+			inactive_apps.remove(container.ia_iterator)
+			active++
+		else if	(count >= max) or (
+			(max_per_app != 0) and (domain.size >= max_per_app)
+			):
+			# All apps are active, and the pool is full. 
+			#  -OR-
+			# All apps are active and the number of max instances
+			# spawned for this application domain has been reached.
+			#
+			# We're not allowed to spawn a new application instance.
+			if options.use_global_queue:
+				# So we wait until _active_ has changed, then
+				# we restart this function and try again.
+				waiting_on_global_queue++
+				wait until _active_ has changed
+				waiting_on_global_queue--
+				goto beginning of function
+			else:
+				# So we connect to an already active application.
+				# This connection will be put into that
+				# application's private queue.
+				container = a container in _instances_ with the smallest _session_ value
+				instances.move_to_back(container.iterator)
+		else:
+			# All apps are active, but the pool hasn't reached its
+			# maximum yet. So we spawn a new app.
+			container = new AppContainer
+			# TODO: we should add some kind of timeout check for spawning.
+			container.app = spawn(app_root)
+			container.sessions = 0
+			iterator = instances.add_to_back(container)
+			container.iterator = iterator
+			domain.size++
+			count++
+			active++
+	else:
+		# There are no apps for this app root. Wait until there's at
+		# least 1 idle instance, or until there's an empty slot in the
+		# pool, then restart this function. Restarting is necessary,
+		# because after waiting and reacquiring the lack, some other
+		# thread might already have spawned instances for this app root.
+		if (active >= max):
+			wait until _active_ has changed
+			goto beginning of function
+		elsif count == max:
+			# Here we are in a though situation. There are several
+			# apps which are inactive, and none of them have
+			# application root _app_root_, so we must kill one of
+			# them in order to free a spot in the pool. But which
+			# one do we kill? We want to minimize spawning.
+			#
+			# It's probably a good idea to keep some kind of
+			# statistics in order to decide this. We want the
+			# application root that gets the least traffic to be
+			# killed. But for now, we kill a random application
+			# instance.
+			container = inactive_apps.pop_front
+			domain = domains[container.app.app_root]
+			instances = domain.instances
+			instances.remove(container.iterator)
+			if instances.empty():
+				domains.remove(container.app.app_root)
+			else:
+				domain.size--
+			count--
+		container = new AppContainer
+		# TODO: we should add some kind of timeout check for spawning.
+		container.app = spawn(app_root)
+		container.sessions = 0
+		domain = domains[app_root]
+		if domain == nil:
+			domain = new Domain
+			initialize domain.instances
+			initialize domain.restart_file_checker with "$app_root/tmp/restart.txt"
+			domain.size = 1
+			domain.max_requests = options.max_requests
+			domains[app_root] = domain
+		else:
+			domain.size++
+		iterator = domain.instances.add_to_back(container)
+		container.iterator = iterator
+		count++
+		active++
+	return [container, domain]
+
+
+# The following function is to be called when a session has been closed.
+# _container_ is the AppContainer that contains the application for which a
+# session has been closed.
+function session_has_been_closed(container):
+	lock.synchronize:
+		domain = domains[container.app.app_root]
+		if domain != nil:
+			instances = domain.instances
+			container.processed++
+			
+			if (domain.max_requests) > 0 and (container.processed >= domain.max_requests):
+				# The application instance has processed its maximum allowed
+				# number of requests, so we shut it down.
+				instances.remove(container.iterator)
+				domain.size--
+				if instances.empty():
+					domains.remove(app_root)
+				count--
+				active--
+			else:
+				container.last_used = current_time()
+				container.sessions--
+				container.processed++
+				if container.sessions == 0:
+					instances.move_to_front(container.iterator)
+					container.ia_iterator = inactive_apps.add_to_back(container.app)
+					active--
+
+
+function needs_restart(app_root, domain):
+	always_restart_file = "$app_root/tmp/always_restart.txt"
+	return (file_exists(always_restart_file)) or
+	       (domain.restart_file_checker.changed())
+
+
+# The following thread will be responsible for cleaning up idle application
+# instances, i.e. instances that haven't been used for a while.
+# This can be disabled per app when setting it's maxIdleTime to 0.
+thread cleaner:
+	lock.synchronize:
+		done = false
+		while !done:
+			Wait until CLEAN_INTERVAL seconds have expired, or until the thread has been signalled to quit.
+			if thread has been signalled to quit:
+				done = true
+				break
+			
+			now = current_time()
+			for all container in inactive_apps:
+				app = container.app
+				domain = domains[app.app_root]
+				instances = domain.instances
+				# If MAX_IDLE_TIME is 0 we don't clean up the instance,
+				# giving us the option to persist the app container
+				# forever unless it's killed by another app.
+				if (MAX_IDLE_TIME > 0) and (now - container.last_used > MAX_IDLE_TIME):
+					instances.remove(container.iterator)
+					inactive_apps.remove(container.iterator)
+					domain.size--
+					count--
+				if instances.empty():
+					domains.remove(app.app_root)
+