passenger 3.9.2.beta → 4.0.0.rc4

data/ext/common/ApplicationPool2/Options.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010, 2011, 2012 Phusion
+ *  Copyright (c) 2010-2013 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -29,6 +29,7 @@
 #include <vector>
 #include <utility>
 #include <boost/shared_array.hpp>
+#include <ApplicationPool2/AppTypes.h>
 #include <Account.h>
 #include <UnionStation.h>
 #include <Constants.h>
@@ -98,6 +99,7 @@ private:
 		result.push_back(&postexecChroot);
 		
 		result.push_back(&ruby);
+		result.push_back(&python);
 		result.push_back(&loggingAgentAddress);
 		result.push_back(&loggingAgentUsername);
 		result.push_back(&loggingAgentPassword);
@@ -163,10 +165,10 @@ public:
 	
 	/** The application's type, used for determining the command to invoke to
 	 * spawn an application process as well as determining the startup file's
-	 * filename. Either "classic-rails", "rack", "wsgi" or the empty string (default).
-	 * In case of the latter, 'startCommand' and 'startupFile' (which MUST
-	 * be set) will dictate the startup command and the startup file's
-	 * filename. */
+	 * filename. It can be one of the app type names in AppType.cpp, or the
+	 * empty string (default). In case of the latter, 'startCommand' and
+	 * 'startupFile' (which MUST be set) will dictate the startup command
+	 * and the startup file's filename. */
 	StaticString appType;
 	
 	/** The command for spawning the application process. This is a list of
@@ -236,6 +238,12 @@ public:
 	 * is a Ruby app.
 	 */
 	StaticString ruby;
+
+	/**
+	 * Path to the Python interpreter to use, in case the application to spawn
+	 * is a Python app.
+	 */
+	StaticString python;
 	
 	/**
 	 * Any rights that the spawned application process may have. The SpawnManager
@@ -362,6 +370,7 @@ public:
 		spawnMethod             = "smart";
 		defaultUser             = "nobody";
 		ruby                    = "ruby";
+		python                  = "python";
 		rights                  = DEFAULT_BACKEND_ACCOUNT_RIGHTS;
 		debugger                = false;
 		loadShellEnvvars        = true;
@@ -498,6 +507,7 @@ public:
 		appendKeyValue (vec, "preexec_chroot",     preexecChroot);
 		appendKeyValue (vec, "postexec_chroot",    postexecChroot);
 		appendKeyValue (vec, "ruby",               ruby);
+		appendKeyValue (vec, "python",             python);
 		appendKeyValue (vec, "logging_agent_address",  loggingAgentAddress);
 		appendKeyValue (vec, "logging_agent_username", loggingAgentUsername);
 		appendKeyValue (vec, "logging_agent_password", loggingAgentPassword);
@@ -528,39 +538,27 @@ public:
 		} else if (appType == "rack") {
 			return ruby + "\1" + resourceLocator.getHelperScriptsDir() + "/rack-loader.rb";
 		} else if (appType == "wsgi") {
-			return "python\1" + resourceLocator.getHelperScriptsDir() + "/wsgi-loader.py";
-		} else if (appType == "node") {
-			return "node\1" + resourceLocator.getHelperScriptsDir() + "/node-loader.js";
+			return python + "\1" + resourceLocator.getHelperScriptsDir() + "/wsgi-loader.py";
 		} else {
 			return startCommand;
 		}
 	}
 	
 	StaticString getStartupFile() const {
-		if (appType == "classic-rails") {
-			return "config/environment.rb";
-		} else if (appType == "rack") {
-			return "config.ru";
-		} else if (appType == "wsgi") {
-			return "passenger_wsgi.py";
-		} else if (appType == "node") {
-			return "passenger_node.js";
-		} else {
+		const char *result = getAppTypeStartupFile(getAppType(appType));
+		if (result == NULL) {
 			return startupFile;
+		} else {
+			return result;
 		}
 	}
 	
 	StaticString getProcessTitle() const {
-		if (appType == "classic-rails") {
-			return "Passenger RailsApp";
-		} else if (appType == "rack") {
-			return "Passenger RackApp";
-		} else if (appType == "wsgi") {
-			return "Passenger WsgiApp";
-		} else if (appType == "node") {
-			return "Passenger NodeJsApp";
-		} else {
+		const char *result = getAppTypeProcessTitle(getAppType(appType));
+		if (result == NULL) {
 			return processTitle;
+		} else {
+			return result;
 		}
 	}
 	

data/ext/common/ApplicationPool2/PipeWatcher.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2012 Phusion
+ *  Copyright (c) 2012-2013 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -27,9 +27,10 @@
 
 #include <boost/shared_ptr.hpp>
 #include <boost/enable_shared_from_this.hpp>
-#include <SafeLibev.h>
+#include <boost/thread.hpp>
+#include <boost/function.hpp>
+#include <sys/types.h>
 #include <FileDescriptor.h>
-#include <ev++.h>
 
 namespace Passenger {
 namespace ApplicationPool2 {
@@ -37,19 +38,25 @@ namespace ApplicationPool2 {
 using namespace boost;
 
 
+/** A PipeWatcher lives until the file descriptor is closed. */
 struct PipeWatcher: public enable_shared_from_this<PipeWatcher> {
-	SafeLibevPtr libev;
+	// For unit tests.
+	typedef function<void (const char *data, unsigned int size)> DataCallback;
+	static DataCallback onData;
+
 	FileDescriptor fd;
-	ev::io watcher;
-	shared_ptr<PipeWatcher> selfPointer;
-	int fdToForwardTo;
-
-	PipeWatcher(const SafeLibevPtr &_libev,
-		const FileDescriptor &_fd,
-		int _fdToForwardTo);
-	~PipeWatcher();
+	const char *name;
+	pid_t pid;
+	bool print;
+	bool started;
+	boost::mutex startSyncher;
+	condition_variable startCond;
+
+	PipeWatcher(const FileDescriptor &_fd, const char *name, pid_t pid, bool _print);
+	void initialize();
 	void start();
-	void onReadable(ev::io &io, int revents);
+	static void threadMain(shared_ptr<PipeWatcher> self);
+	void threadMain();
 };
 
 typedef shared_ptr<PipeWatcher> PipeWatcherPtr;

data/ext/common/ApplicationPool2/Pool.h

@@ -36,6 +36,7 @@
 #include <boost/make_shared.hpp>
 #include <boost/function.hpp>
 #include <boost/foreach.hpp>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
 #include <oxt/dynamic_thread_group.hpp>
 #include <oxt/backtrace.hpp>
 #include <ApplicationPool2/Common.h>
@@ -43,15 +44,15 @@
 #include <ApplicationPool2/Group.h>
 #include <ApplicationPool2/SuperGroup.h>
 #include <ApplicationPool2/Session.h>
-#include <ApplicationPool2/Spawner.h>
+#include <ApplicationPool2/SpawnerFactory.h>
 #include <ApplicationPool2/Options.h>
 #include <UnionStation.h>
 #include <Logging.h>
 #include <SafeLibev.h>
-#include <AnsiColorConstants.h>
 #include <Exceptions.h>
 #include <RandomGenerator.h>
 #include <Utils/Lock.h>
+#include <Utils/AnsiColorConstants.h>
 #include <Utils/SystemTime.h>
 #include <Utils/MessagePassing.h>
 #include <Utils/VariantMap.h>
@@ -94,6 +95,11 @@ public:
 		MessageBoxPtr debugger;
 		MessageBoxPtr messages;
 
+		// Choose aspects to debug.
+		bool restarting;
+		bool spawning;
+		bool superGroup;
+
 		// The following fields may only be accessed by Pool.
 		boost::mutex syncher;
 		unsigned int spawnLoopIteration;
@@ -101,6 +107,9 @@ public:
 		DebugSupport() {
 			debugger = make_shared<MessageBox>();
 			messages = make_shared<MessageBox>();
+			restarting = true;
+			spawning   = true;
+			superGroup = false;
 			spawnLoopIteration = 0;
 		}
 	};
@@ -116,8 +125,7 @@ public:
 	unsigned int max;
 	unsigned long long maxIdleTime;
 	
-	ev::timer garbageCollectionTimer;
-	ev::timer analyticsCollectionTimer;
+	condition_variable garbageCollectionCond;
 	
 	/**
 	 * Code can register background threads in one of these dynamic thread groups
@@ -130,6 +138,12 @@ public:
 	 */
 	dynamic_thread_group interruptableThreads;
 	dynamic_thread_group nonInterruptableThreads;
+
+	enum LifeStatus {
+		ALIVE,
+		SHUTTING_DOWN,
+		SHUT_DOWN
+	} lifeStatus;
 	
 	SuperGroupMap superGroups;
 	
@@ -209,8 +223,11 @@ public:
 	}
 
 	void fullVerifyInvariants() const {
+		TRACE_POINT();
 		verifyInvariants();
+		UPDATE_TRACE_POINT();
 		verifyExpensiveInvariants();
+		UPDATE_TRACE_POINT();
 		StringMap<SuperGroupPtr>::const_iterator sg_it, sg_end = superGroups.end();
 		for (sg_it = superGroups.begin(); sg_it != sg_end; sg_it++) {
 			pair<StaticString, SuperGroupPtr> p = *sg_it;
@@ -308,6 +325,19 @@ public:
 		
 		getWaitlist = newWaitlist;
 	}
+
+	template<typename Queue>
+	static void assignExceptionToGetWaiters(Queue &getWaitlist,
+		const ExceptionPtr &exception,
+		vector<Callback> &postLockActions)
+	{
+		while (!getWaitlist.empty()) {
+			postLockActions.push_back(boost::bind(
+				getWaitlist.front().callback, SessionPtr(),
+				exception));
+			getWaitlist.pop();
+		}
+	}
 	
 	void possiblySpawnMoreProcessesForExistingGroups() {
 		StringMap<SuperGroupPtr>::const_iterator sg_it, sg_end = superGroups.end();
@@ -343,14 +373,6 @@ public:
 		}
 	}
 	
-	void migrateGroupGetWaitlistToPool(const GroupPtr &group) {
-		getWaitlist.reserve(getWaitlist.size() + group->getWaitlist.size());
-		while (!group->getWaitlist.empty()) {
-			getWaitlist.push_back(group->getWaitlist.front());
-			group->getWaitlist.pop();
-		}
-	}
-	
 	void migrateSuperGroupGetWaitlistToPool(const SuperGroupPtr &superGroup) {
 		getWaitlist.reserve(getWaitlist.size() + superGroup->getWaitlist.size());
 		while (!superGroup->getWaitlist.empty()) {
@@ -364,24 +386,26 @@ public:
 	 * the SuperGroup may have a non-empty getWaitlist so be sure to do
 	 * something with it.
 	 *
-	 * 'superGroup' is a non-const non-reference smart pointer so that
-	 * it does not get destroy immediately after the 'superGroups.remove()'
-	 * call.
+	 * Also, one of the post lock actions can potentially perform a long-running
+	 * operation, so running them in a thread is advised.
 	 */
-	void forceDetachSuperGroup(SuperGroupPtr superGroup, vector<Callback> &postLockActions) {
+	void forceDetachSuperGroup(const SuperGroupPtr &superGroup,
+		vector<Callback> &postLockActions,
+		const SuperGroup::ShutdownCallback &callback)
+	{
+		const SuperGroupPtr sp = superGroup; // Prevent premature destruction.
 		bool removed = superGroups.remove(superGroup->name);
 		assert(removed);
 		(void) removed; // Shut up compiler warning.
-		superGroup->destroy(postLockActions, false);
-		superGroup->setPool(PoolPtr());
+		superGroup->destroy(false, postLockActions, callback);
 	}
 
 	bool detachProcessUnlocked(const ProcessPtr &process, vector<Callback> &postLockActions) {
-		GroupPtr group = process->getGroup();
-		if (group != NULL && group->getPool().get() == this) {
+		if (OXT_LIKELY(process->isAlive())) {
 			verifyInvariants();
 			
-			SuperGroupPtr superGroup = group->getSuperGroup();
+			const GroupPtr group = process->getGroup();
+			const SuperGroupPtr superGroup = group->getSuperGroup();
 			assert(superGroup->state != SuperGroup::INITIALIZING);
 			assert(superGroup->getWaitlist.empty());
 			
@@ -431,6 +455,17 @@ public:
 		}
 	}
 
+	struct DetachSuperGroupWaitTicket {
+		boost::mutex syncher;
+		condition_variable cond;
+		SuperGroup::ShutdownResult result;
+		bool done;
+
+		DetachSuperGroupWaitTicket() {
+			done = false;
+		}
+	};
+
 	struct DisableWaitTicket {
 		boost::mutex syncher;
 		condition_variable cond;
@@ -442,10 +477,26 @@ public:
 		}
 	};
 
+	static void syncDetachSuperGroupCallback(SuperGroup::ShutdownResult result,
+		shared_ptr<DetachSuperGroupWaitTicket> ticket)
+	{
+		LockGuard l(ticket->syncher);
+		ticket->done = true;
+		ticket->result = result;
+		ticket->cond.notify_one();
+	}
+
+	static void waitDetachSuperGroupCallback(shared_ptr<DetachSuperGroupWaitTicket> ticket) {
+		ScopedLock l(ticket->syncher);
+		while (!ticket->done) {
+			ticket->cond.wait(l);
+		}
+	}
+
 	static void syncDisableProcessCallback(const ProcessPtr &process, DisableResult result,
 		shared_ptr<DisableWaitTicket> ticket)
 	{
-		ScopedLock l(ticket->syncher);
+		LockGuard l(ticket->syncher);
 		ticket->done = true;
 		ticket->result = result;
 		ticket->cond.notify_one();
@@ -465,17 +516,37 @@ public:
 		return superGroups.get(options.getAppGroupName()).get();
 	}
 
-	void garbageCollect(ev::timer &timer, int revents) {
-		PoolPtr self = shared_from_this(); // Keep pool object alive.
+	static void garbageCollect(PoolPtr self) {
+		TRACE_POINT();
+		{
+			ScopedLock lock(self->syncher);
+			self->garbageCollectionCond.timed_wait(lock,
+				posix_time::seconds(5));
+		}
+		while (!this_thread::interruption_requested()) {
+			try {
+				UPDATE_TRACE_POINT();
+				unsigned long long sleepTime = self->realGarbageCollect();
+				ScopedLock lock(self->syncher);
+				self->garbageCollectionCond.timed_wait(lock,
+					posix_time::microseconds(sleepTime));
+			} catch (const thread_interrupted &) {
+				break;
+			} catch (const tracable_exception &e) {
+				P_WARN("ERROR: " << e.what() << "\n  Backtrace:\n" << e.backtrace());
+			}
+		}
+	}
+
+	unsigned long long realGarbageCollect() {
 		TRACE_POINT();
 		ScopedLock lock(syncher);
 		SuperGroupMap::iterator it, end = superGroups.end();
-		vector<SuperGroupPtr> superGroupsToDetach;
 		vector<Callback> actions;
 		unsigned long long now = SystemTime::getUsec();
 		unsigned long long nextGcRunTime = 0;
 		
-		P_DEBUG("Garbage collection time");
+		P_DEBUG("Garbage collection time...");
 		verifyInvariants();
 		
 		// For all supergroups and groups...
@@ -521,7 +592,7 @@ public:
 						group->options.getMaxPreloaderIdleTime() * 1000000;
 					if (now >= spawnerGcTime) {
 						P_DEBUG("Garbage collect idle spawner: group=" << group->name);
-						group->asyncCleanupSpawner();
+						group->cleanupSpawner(actions);
 					} else if (nextGcRunTime == 0
 					        || spawnerGcTime < nextGcRunTime) {
 						nextGcRunTime = spawnerGcTime;
@@ -529,36 +600,27 @@ public:
 				}
 			}
 			
-			// ...remove entire supergroup if it has become garbage
-			// collectable after detaching idle processes.
-			if (superGroup->garbageCollectable(now)) {
-				superGroupsToDetach.push_back(superGroup);
-			}
-			
 			superGroup->verifyInvariants();
 		}
 		
-		vector<SuperGroupPtr>::const_iterator it2;
-		for (it2 = superGroupsToDetach.begin(); it2 != superGroupsToDetach.end(); it2++) {
-			P_DEBUG("Garbage collect SuperGroup: " << (*it2)->inspect());
-			detachSuperGroup(*it2, false, &actions);
-		}
-		
 		verifyInvariants();
+		lock.unlock();
 
 		// Schedule next garbage collection run.
-		ev_tstamp tstamp;
+		unsigned long long sleepTime;
 		if (nextGcRunTime == 0 || nextGcRunTime <= now) {
-			tstamp = maxIdleTime / 1000000.0;
+			sleepTime = maxIdleTime;
 		} else {
-			tstamp = (nextGcRunTime - now) / 1000000.0;
+			sleepTime = nextGcRunTime - now;
 		}
 		P_DEBUG("Garbage collection done; next garbage collect in " <<
-			std::fixed << std::setprecision(3) << tstamp << " sec");
+			std::fixed << std::setprecision(3) << (sleepTime / 1000000.0) << " sec");
 
-		lock.unlock();
+		UPDATE_TRACE_POINT();
 		runAllActions(actions);
-		timer.start(tstamp, 0.0);
+		UPDATE_TRACE_POINT();
+		actions.clear();
+		return sleepTime;
 	}
 
 	struct ProcessAnalyticsLogEntry {
@@ -569,22 +631,56 @@ public:
 
 	typedef shared_ptr<ProcessAnalyticsLogEntry> ProcessAnalyticsLogEntryPtr;
 
-	void collectAnalytics(ev::timer &timer, int revents) {
-		try {
-			realCollectAnalytics(timer);
-		} catch (const tracable_exception &e) {
-			P_WARN("ERROR: " << e.what() << "\n  Backtrace:\n" << e.backtrace());
+	static void collectAnalytics(PoolPtr self) {
+		TRACE_POINT();
+		syscalls::usleep(3000000);
+		while (!this_thread::interruption_requested()) {
+			try {
+				UPDATE_TRACE_POINT();
+				unsigned long long sleepTime = self->realCollectAnalytics();
+				syscalls::usleep(sleepTime);
+			} catch (const thread_interrupted &) {
+				break;
+			} catch (const tracable_exception &e) {
+				P_WARN("ERROR: " << e.what() << "\n  Backtrace:\n" << e.backtrace());
+			}
+		}
+	}
+
+	static void collectPids(const ProcessList &processes, vector<pid_t> &pids) {
+		foreach (const ProcessPtr &process, processes) {
+			pids.push_back(process->pid);
+		}
+	}
+
+	static void updateProcessMetrics(const ProcessList &processes,
+		const ProcessMetricMap &allMetrics,
+		vector<ProcessPtr> &processesToDetach)
+	{
+		foreach (const ProcessPtr &process, processes) {
+			ProcessMetricMap::const_iterator metrics_it =
+				allMetrics.find(process->pid);
+			if (metrics_it != allMetrics.end()) {
+				process->metrics = metrics_it->second;
+			// If the process is missing from 'allMetrics' then either 'ps'
+			// failed or the process really is gone. We double check by sending
+			// it a signal.
+			} else if (!process->dummy && !process->osProcessExists()) {
+				P_WARN("Process " << process->inspect() << " no longer exists! "
+					"Detaching it from the pool.");
+				processesToDetach.push_back(process);
+			}
 		}
 	}
 
-	void realCollectAnalytics(ev::timer &timer) {
-		PoolPtr self = shared_from_this(); // Keep pool object alive.
+	unsigned long long realCollectAnalytics() {
 		TRACE_POINT();
 		this_thread::disable_interruption di;
 		this_thread::disable_syscall_interruption dsi;
 		vector<pid_t> pids;
 		unsigned int max;
 		
+		P_DEBUG("Analytics collection time...");
 		// Collect all the PIDs.
 		{
 			UPDATE_TRACE_POINT();
@@ -603,24 +699,9 @@ public:
 
 				for (g_it = superGroup->groups.begin(); g_it != g_end; g_it++) {
 					const GroupPtr &group = *g_it;
-					ProcessList::const_iterator p_it, p_end = group->enabledProcesses.end();
-
-					for (p_it = group->enabledProcesses.begin(); p_it != p_end; p_it++) {
-						const ProcessPtr &process = *p_it;
-						pids.push_back(process->pid);
-					}
-
-					p_end = group->disablingProcesses.end();
-					for (p_it = group->disablingProcesses.begin(); p_it != p_end; p_it++) {
-						const ProcessPtr &process = *p_it;
-						pids.push_back(process->pid);
-					}
-
-					p_end = group->disabledProcesses.end();
-					for (p_it = group->disabledProcesses.begin(); p_it != p_end; p_it++) {
-						const ProcessPtr &process = *p_it;
-						pids.push_back(process->pid);
-					}
+					collectPids(group->enabledProcesses, pids);
+					collectPids(group->disablingProcesses, pids);
+					collectPids(group->disabledProcesses, pids);
 				}
 			}
 		}
@@ -639,6 +720,8 @@ public:
 		{
 			UPDATE_TRACE_POINT();
 			vector<ProcessAnalyticsLogEntryPtr> logEntries;
+			vector<ProcessPtr> processesToDetach;
+			vector<Callback> actions;
 			ScopedLock l(syncher);
 			SuperGroupMap::iterator sg_it, sg_end = superGroups.end();
 			
@@ -649,36 +732,10 @@ public:
 
 				for (g_it = superGroup->groups.begin(); g_it != g_end; g_it++) {
 					const GroupPtr &group = *g_it;
-					ProcessList::iterator p_it, p_end = group->enabledProcesses.end();
-
-					for (p_it = group->enabledProcesses.begin(); p_it != p_end; p_it++) {
-						ProcessPtr &process = *p_it;
-						ProcessMetricMap::const_iterator metrics_it =
-							allMetrics.find(process->pid);
-						if (metrics_it != allMetrics.end()) {
-							process->metrics = metrics_it->second;
-						}
-					}
 
-					p_end = group->disablingProcesses.end();
-					for (p_it = group->disablingProcesses.begin(); p_it != p_end; p_it++) {
-						ProcessPtr &process = *p_it;
-						ProcessMetricMap::const_iterator metrics_it =
-							allMetrics.find(process->pid);
-						if (metrics_it != allMetrics.end()) {
-							process->metrics = metrics_it->second;
-						}
-					}
-
-					p_end = group->disabledProcesses.end();
-					for (p_it = group->disabledProcesses.begin(); p_it != p_end; p_it++) {
-						ProcessPtr &process = *p_it;
-						ProcessMetricMap::const_iterator metrics_it =
-							allMetrics.find(process->pid);
-						if (metrics_it != allMetrics.end()) {
-							process->metrics = metrics_it->second;
-						}
-					}
+					updateProcessMetrics(group->enabledProcesses, allMetrics, processesToDetach);
+					updateProcessMetrics(group->disablingProcesses, allMetrics, processesToDetach);
+					updateProcessMetrics(group->disabledProcesses, allMetrics, processesToDetach);
 
 					// Log to Union Station.
 					if (group->options.analytics && loggerFactory != NULL) {
@@ -694,7 +751,15 @@ public:
 				}
 			}
 
+			UPDATE_TRACE_POINT();
+			foreach (const ProcessPtr process, processesToDetach) {
+				detachProcessUnlocked(process, actions);
+			}
+			UPDATE_TRACE_POINT();
+			processesToDetach.clear();
+
 			l.unlock();
+			UPDATE_TRACE_POINT();
 			while (!logEntries.empty()) {
 				ProcessAnalyticsLogEntryPtr entry = logEntries.back();
 				logEntries.pop_back();
@@ -702,16 +767,24 @@ public:
 					"processes", entry->key);
 				logger->message(entry->data.str());
 			}
+
+			UPDATE_TRACE_POINT();
+			runAllActions(actions);
+			UPDATE_TRACE_POINT();
+			// Run destructors with updated trace point.
+			actions.clear();
 		}
 		
 		end:
 		// Sleep for about 4 seconds, aligned to seconds boundary
 		// for saving power on laptops.
-		ev_now_update(libev->getLoop());
 		unsigned long long currentTime = SystemTime::getUsec();
 		unsigned long long deadline =
 			roundUp<unsigned long long>(currentTime, 1000000) + 4000000;
-		timer.start((deadline - currentTime) / 1000000.0, 0.0);
+		P_DEBUG("Analytics collection done; next analytics collection in " <<
+			std::fixed << std::setprecision(3) << ((deadline - currentTime) / 1000000.0) <<
+			" sec");
+		return deadline - currentTime;
 	}
 	
 	SuperGroupPtr createSuperGroup(const Options &options) {
@@ -752,16 +825,10 @@ public:
 			this->randomGenerator = make_shared<RandomGenerator>();
 		}
 		
+		lifeStatus  = ALIVE;
 		max         = 6;
 		maxIdleTime = 60 * 1000000;
 		
-		garbageCollectionTimer.set<Pool, &Pool::garbageCollect>(this);
-		garbageCollectionTimer.set(maxIdleTime / 1000000.0, 0.0);
-		libev->start(garbageCollectionTimer);
-		analyticsCollectionTimer.set<Pool, &Pool::collectAnalytics>(this);
-		analyticsCollectionTimer.set(3.0, 0.0);
-		libev->start(analyticsCollectionTimer);
-
 		// The following code only serve to instantiate certain inline methods
 		// so that they can be invoked from gdb.
 		(void) SuperGroupPtr().get();
@@ -771,8 +838,23 @@ public:
 	}
 	
 	~Pool() {
-		TRACE_POINT();
-		destroy();
+		if (lifeStatus != SHUT_DOWN) {
+			P_BUG("You must call Pool::destroy() before actually destroying the Pool object!");
+		}
+	}
+
+	void initialize() {
+		LockGuard l(syncher);
+		interruptableThreads.create_thread(
+			boost::bind(collectAnalytics, shared_from_this()),
+			"Pool analytics collector",
+			POOL_HELPER_THREAD_STACK_SIZE
+		);
+		interruptableThreads.create_thread(
+			boost::bind(garbageCollect, shared_from_this()),
+			"Pool garbage collector",
+			POOL_HELPER_THREAD_STACK_SIZE
+		);
 	}
 
 	void initDebugging() {
@@ -782,34 +864,29 @@ public:
 
 	void destroy() {
 		TRACE_POINT();
-		libev->stop(garbageCollectionTimer);
-		libev->stop(analyticsCollectionTimer);
+		ScopedLock lock(syncher);
+		assert(lifeStatus == ALIVE);
+
+		lifeStatus = SHUTTING_DOWN;
+
+		while (!superGroups.empty()) {
+			string name = superGroups.begin()->second->name;
+			lock.unlock();
+			detachSuperGroupByName(name);
+			lock.lock();
+		}
 
 		UPDATE_TRACE_POINT();
+		lock.unlock();
 		interruptableThreads.interrupt_and_join_all();
 		nonInterruptableThreads.join_all();
+		lock.lock();
+
+		lifeStatus = SHUT_DOWN;
 
 		UPDATE_TRACE_POINT();
-		ScopedLock l(syncher);
-		SuperGroupMap::iterator it;
-		vector<SuperGroupPtr>::iterator it2;
-		vector<SuperGroupPtr> superGroupsToDetach;
-		vector<Callback> actions;
-		for (it = superGroups.begin(); it != superGroups.end(); it++) {
-			superGroupsToDetach.push_back(it->second);
-		}
-		for (it2 = superGroupsToDetach.begin(); it2 != superGroupsToDetach.end(); it2++) {
-			detachSuperGroup(*it2, false, &actions);
-		}
-		
 		verifyInvariants();
 		verifyExpensiveInvariants();
-		l.unlock();
-		runAllActions(actions);
-
-		// detachSuperGroup() may launch additional threads, so wait for them.
-		interruptableThreads.interrupt_and_join_all();
-		nonInterruptableThreads.join_all();
 	}
 
 	// 'lockNow == false' may only be used during unit tests. Normally we
@@ -817,6 +894,7 @@ public:
 	void asyncGet(const Options &options, const GetCallback &callback, bool lockNow = true) {
 		DynamicScopedLock lock(syncher, lockNow);
 		
+		assert(lifeStatus == ALIVE);
 		verifyInvariants();
 		P_TRACE(2, "asyncGet(appRoot=" << options.appRoot << ")");
 		
@@ -965,7 +1043,7 @@ public:
 				createSuperGroup(options);
 			}
 		}
-		return get(options2, &ticket)->getProcess()->getGroup();
+		return get(options2, &ticket)->getGroup();
 	}
 	
 	void setMax(unsigned int max) {
@@ -996,18 +1074,10 @@ public:
 		}
 	}
 
-	void activateNewMaxIdleTime() {
-		LockGuard l(syncher);
-		garbageCollectionTimer.stop();
-		garbageCollectionTimer.start(maxIdleTime / 1000000.0, 0.0);
-	}
-
 	void setMaxIdleTime(unsigned long long value) {
-		{
-			LockGuard l(syncher);
-			maxIdleTime = value;
-		}
-		libev->run(boost::bind(&Pool::activateNewMaxIdleTime, this));
+		LockGuard l(syncher);
+		maxIdleTime = value;
+		garbageCollectionCond.notify_all();
 	}
 	
 	unsigned int utilization(bool lock = true) const {
@@ -1062,6 +1132,11 @@ public:
 		}
 		return result;
 	}
+
+	unsigned int getSuperGroupCount() const {
+		LockGuard l(syncher);
+		return superGroups.size();
+	}
 	
 	SuperGroupPtr findSuperGroupBySecret(const string &secret, bool lock = true) const {
 		DynamicScopedLock l(syncher, lock);
@@ -1086,21 +1161,29 @@ public:
 		}
 		return ProcessPtr();
 	}
-	
-	bool detachSuperGroup(const SuperGroupPtr &superGroup, bool lock = true,
-		vector<Callback> *postLockActions = NULL)
-	{
-		assert(lock || postLockActions != NULL);
-		DynamicScopedLock l(syncher, lock);
+
+	bool detachSuperGroupByName(const string &name) {
+		TRACE_POINT();
+		ScopedLock l(syncher);
 		
-		if (OXT_LIKELY(superGroup->getPool().get() == this)) {
+		SuperGroupPtr superGroup = superGroups.get(name);
+		if (OXT_LIKELY(superGroup != NULL)) {
 			if (OXT_LIKELY(superGroups.get(superGroup->name) != NULL)) {
+				UPDATE_TRACE_POINT();
 				verifyInvariants();
 				verifyExpensiveInvariants();
 				
 				vector<Callback> actions;
-				
-				forceDetachSuperGroup(superGroup, actions);
+				shared_ptr<DetachSuperGroupWaitTicket> ticket =
+					make_shared<DetachSuperGroupWaitTicket>();
+				ExceptionPtr exception = copyException(
+					GetAbortedException("The containg SuperGroup was detached."));
+
+				forceDetachSuperGroup(superGroup, actions,
+					boost::bind(syncDetachSuperGroupCallback, _1, ticket));
+				assignExceptionToGetWaiters(superGroup->getWaitlist,
+					exception, actions);
+				#if 0
 				/* If this SuperGroup had get waiters, either
 				 * on itself or in one of its groups, then we must
 				 * reprocess them immediately. Detaching such a
@@ -1108,26 +1191,25 @@ public:
 				 */
 				migrateSuperGroupGetWaitlistToPool(superGroup);
 				
+				UPDATE_TRACE_POINT();
 				assignSessionsToGetWaiters(actions);
+				#endif
 				possiblySpawnMoreProcessesForExistingGroups();
 				
 				verifyInvariants();
 				verifyExpensiveInvariants();
 				
-				if (lock) {
-					l.unlock();
-					runAllActions(actions);
-				} else if (postLockActions->empty()) {
-					*postLockActions = actions;
-				} else {
-					postLockActions->reserve(postLockActions->size() +
-						actions.size());
-					for (unsigned int i = 0; i < actions.size(); i++) {
-						postLockActions->push_back(actions[i]);
-					}
+				l.unlock();
+				UPDATE_TRACE_POINT();
+				runAllActions(actions);
+				actions.clear();
+
+				UPDATE_TRACE_POINT();
+				ScopedLock l2(ticket->syncher);
+				while (!ticket->done) {
+					ticket->cond.wait(l2);
 				}
-				
-				return true;
+				return ticket->result == SuperGroup::SUCCESS;
 			} else {
 				return false;
 			}
@@ -1136,11 +1218,14 @@ public:
 		}
 	}
 	
-	bool detachSuperGroup(const string &superGroupSecret) {
-		LockGuard l(syncher);
+	bool detachSuperGroupBySecret(const string &superGroupSecret) {
+		ScopedLock l(syncher);
 		SuperGroupPtr superGroup = findSuperGroupBySecret(superGroupSecret, false);
 		if (superGroup != NULL) {
-			return detachSuperGroup(superGroup, false);
+			string name = superGroup->name;
+			superGroup.reset();
+			l.unlock();
+			return detachSuperGroupByName(name);
 		} else {
 			return false;
 		}
@@ -1174,23 +1259,63 @@ public:
 	DisableResult disableProcess(const string &gupid) {
 		ScopedLock l(syncher);
 		ProcessPtr process = findProcessByGupid(gupid, false);
-		GroupPtr group = process->getGroup();
-		// Must be a shared_ptr to be interruption-safe.
-		shared_ptr<DisableWaitTicket> ticket = make_shared<DisableWaitTicket>();
-		DisableResult result = group->disable(process,
-			boost::bind(syncDisableProcessCallback, _1, _2, ticket));
-		group->verifyInvariants();
-		group->verifyExpensiveInvariants();
-		if (result == DR_DEFERRED) {
-			l.unlock();
-			ScopedLock l2(ticket->syncher);
-			while (!ticket->done) {
-				ticket->cond.wait(l2);
+		if (process != NULL) {
+			GroupPtr group = process->getGroup();
+			// Must be a shared_ptr to be interruption-safe.
+			shared_ptr<DisableWaitTicket> ticket = make_shared<DisableWaitTicket>();
+			DisableResult result = group->disable(process,
+				boost::bind(syncDisableProcessCallback, _1, _2, ticket));
+			group->verifyInvariants();
+			group->verifyExpensiveInvariants();
+			if (result == DR_DEFERRED) {
+				l.unlock();
+				ScopedLock l2(ticket->syncher);
+				while (!ticket->done) {
+					ticket->cond.wait(l2);
+				}
+				return ticket->result;
+			} else {
+				return result;
 			}
-			return ticket->result;
 		} else {
-			return result;
+			return DR_NOOP;
+		}
+	}
+
+	unsigned int restartGroupsByAppRoot(const string &appRoot) {
+		ScopedLock l(syncher);
+		SuperGroupMap::iterator sg_it, sg_end = superGroups.end();
+		unsigned int result = 0;
+
+		for (sg_it = superGroups.begin(); sg_it != sg_end; sg_it++) {
+			const SuperGroupPtr &superGroup = sg_it->second;
+			foreach (const GroupPtr &group, superGroup->groups) {
+				if (group->options.appRoot == appRoot) {
+					result++;
+					if (!group->restarting()) {
+						group->restart(group->options);
+					}
+				}
+			}
 		}
+
+		return result;
+	}
+
+	unsigned int restartSuperGroupsByAppRoot(const string &appRoot) {
+		ScopedLock l(syncher);
+		SuperGroupMap::iterator sg_it, sg_end = superGroups.end();
+		unsigned int result = 0;
+
+		for (sg_it = superGroups.begin(); sg_it != sg_end; sg_it++) {
+			const SuperGroupPtr &superGroup = sg_it->second;
+			if (superGroup->options.appRoot == appRoot) {
+				result++;
+				superGroup->restart(superGroup->options);
+			}
+		}
+
+		return result;
 	}
 
 	/**
@@ -1238,6 +1363,9 @@ public:
 			if (group != NULL) {
 				result << group->name << ":" << endl;
 				result << "  App root: " << group->options.appRoot << endl;
+				if (group->restarting()) {
+					result << "  (restarting...)" << endl;
+				}
 				if (group->spawning()) {
 					result << "  (spawning new process...)" << endl;
 				}
@@ -1251,8 +1379,8 @@ public:
 		return result.str();
 	}
 
-	string toXml(bool includeSecrets = true) const {
-		LockGuard l(syncher);
+	string toXml(bool includeSecrets = true, bool lock = true) const {
+		DynamicScopedLock l(syncher, lock);
 		stringstream result;
 		SuperGroupMap::const_iterator sg_it;
 		vector<GroupPtr>::const_iterator g_it;
@@ -1265,11 +1393,25 @@ public:
 		result << "<max>" << max << "</max>";
 		result << "<utilization>" << utilization(false) << "</utilization>";
 		result << "<get_wait_list_size>" << getWaitlist.size() << "</get_wait_list_size>";
+
+		if (includeSecrets) {
+			vector<GetWaiter>::const_iterator w_it, w_end = getWaitlist.end();
+
+			result << "<get_wait_list>";
+			for (w_it = getWaitlist.begin(); w_it != w_end; w_it++) {
+				const GetWaiter &waiter = *w_it;
+				result << "<item>";
+				result << "<app_group_name>" << escapeForXml(waiter.options.getAppGroupName()) << "</app_group_name>";
+				result << "</item>";
+			}
+			result << "</get_wait_list>";
+		}
 		
 		result << "<supergroups>";
 		for (sg_it = superGroups.begin(); sg_it != superGroups.end(); sg_it++) {
 			const SuperGroupPtr &superGroup = sg_it->second;
 
+			result << "<supergroup>";
 			result << "<name>" << escapeForXml(superGroup->name) << "</name>";
 			result << "<state>" << superGroup->getStateName() << "</state>";
 			result << "<get_wait_list_size>" << superGroup->getWaitlist.size() << "</get_wait_list_size>";
@@ -1289,6 +1431,7 @@ public:
 				group->inspectXml(result, includeSecrets);
 				result << "</group>";
 			}
+			result << "</supergroup>";
 		}
 		result << "</supergroups>";