passenger 5.0.9 → 5.0.10

data/ext/common/ApplicationPool2/Options.h

@@ -111,7 +111,7 @@ private:
 		result.push_back(&options.loggingAgentAddress);
 		result.push_back(&options.loggingAgentUsername);
 		result.push_back(&options.loggingAgentPassword);
-		result.push_back(&options.groupSecret);
+		result.push_back(&options.apiKey);
 		result.push_back(&options.hostName);
 		result.push_back(&options.uri);
 		result.push_back(&options.unionStationKey);
@@ -417,7 +417,7 @@ public:
 	unsigned long long currentTime;
 
 	/** When true, Pool::get() and Pool::asyncGet() will create the necessary
-	 * SuperGroup and Group structures just as normally, and will even handle
+	 * Group structure just as normally, and will even handle
 	 * restarting logic, but will not actually spawn any processes and will not
 	 * open a session with an existing process. Instead, a fake Session object
 	 * is returned which points to a Process object that isn't stored anywhere
@@ -436,8 +436,8 @@ public:
 	 * see these values.
 	 */
 
-	/** The secret key of the pool group that the spawned process is to belong to. */
-	StaticString groupSecret;
+	/** The API key of the pool group that the spawned process is to belong to. */
+	StaticString apiKey;
 
 	/**
 	 * A UUID that's generated on Group initialization, and changes every time
@@ -603,7 +603,7 @@ public:
 			appendKeyValue4(vec, "debugger",           debugger);
 			appendKeyValue4(vec, "analytics",          analytics);
 
-			appendKeyValue (vec, "group_secret",       groupSecret);
+			appendKeyValue (vec, "api_key",            apiKey);
 
 			/*********************************/
 		}

data/ext/common/ApplicationPool2/Pool.h

@@ -45,13 +45,14 @@
 #include <boost/date_time/posix_time/posix_time_types.hpp>
 #include <oxt/dynamic_thread_group.hpp>
 #include <oxt/backtrace.hpp>
+#include <sys/types.h>
 #include <ApplicationPool2/Common.h>
+#include <ApplicationPool2/Context.h>
 #include <ApplicationPool2/Process.h>
 #include <ApplicationPool2/Group.h>
-#include <ApplicationPool2/SuperGroup.h>
 #include <ApplicationPool2/Session.h>
-#include <ApplicationPool2/SpawnerFactory.h>
 #include <ApplicationPool2/Options.h>
+#include <SpawningKit/Factory.h>
 #include <MemoryKit/palloc.h>
 #include <Logging.h>
 #include <Exceptions.h>
@@ -74,22 +75,93 @@ using namespace oxt;
 
 class Pool: public boost::enable_shared_from_this<Pool> {
 public:
-	typedef void (*AbortLongRunningConnectionsCallback)(const ProcessPtr &process);
+	struct AuthenticationOptions {
+		uid_t uid;
+		ApiKey apiKey;
+
+		AuthenticationOptions()
+			: uid(-1)
+			{ }
+
+		static AuthenticationOptions makeAuthorized() {
+			AuthenticationOptions options;
+			options.apiKey = ApiKey::makeSuper();
+			return options;
+		}
+	};
+
+
+	/****** Group data structure utilities ******/
+
+	struct RestartOptions: public AuthenticationOptions {
+		RestartMethod method;
+
+		RestartOptions()
+			: method(RM_DEFAULT)
+			{ }
+
+		static RestartOptions makeAuthorized() {
+			RestartOptions options;
+			options.apiKey = ApiKey::makeSuper();
+			return options;
+		}
+	};
+
+	/****** State inspection ******/
+
+	struct InspectOptions: public AuthenticationOptions {
+		bool colorize;
+		bool verbose;
+
+		InspectOptions()
+			: colorize(false),
+			  verbose(false)
+			{ }
+
+		InspectOptions(const VariantMap &options)
+			: colorize(options.getBool("colorize", false, false)),
+			  verbose(options.getBool("verbose", false, false))
+			{ }
+
+		static InspectOptions makeAuthorized() {
+			InspectOptions options;
+			options.apiKey = ApiKey::makeSuper();
+			return options;
+		}
+	};
+
+	struct ToXmlOptions: public AuthenticationOptions {
+		bool secrets;
+
+		ToXmlOptions()
+			: secrets(true)
+			{ }
+
+		ToXmlOptions(const VariantMap &options)
+			: secrets(options.getBool("secrets", false, false))
+			{ }
+
+		static ToXmlOptions makeAuthorized() {
+			ToXmlOptions options;
+			options.apiKey = ApiKey::makeSuper();
+			return options;
+		}
+	};
+
 
 // Actually private, but marked public so that unit tests can access the fields.
 public:
-	friend class SuperGroup;
 	friend class Group;
 	friend class Process;
 	friend struct tut::ApplicationPool2_PoolTest;
 
-	SpawnerFactoryPtr spawnerFactory;
-
 	mutable boost::mutex syncher;
 	unsigned int max;
 	unsigned long long maxIdleTime;
 	bool selfchecking;
 
+	Context context;
+
 	/**
 	 * Code can register background threads in one of these dynamic thread groups
 	 * to ensure that threads are interrupted and/or joined properly upon Pool
@@ -109,9 +181,7 @@ public:
 		SHUT_DOWN
 	} lifeStatus;
 
-	mutable SuperGroupMap superGroups;
-	boost::mutex sessionObjectPoolSyncher;
-	object_pool<Session> sessionObjectPool;
+	mutable GroupMap groups;
 	psg_pool_t *palloc;
 
 	/**
@@ -132,16 +202,16 @@ public:
 	 *   then be killed to free capacity.
 	 * - A process has failed to spawn, resulting in capacity to
 	 *   become free.
-	 * - A SuperGroup failed to initialize, resulting in free capacity.
+	 * - A Group failed to initialize, resulting in free capacity.
 	 * - Someone commanded Pool to detach a process, resulting in free
 	 *   capacity.
-	 * - Someone commanded Pool to detach a SuperGroup, resulting in
+	 * - Someone commanded Pool to detach a Group, resulting in
 	 *   free capacity.
 	 * - The 'max' option has been increased, resulting in free capacity.
 	 *
 	 * Invariant 1:
 	 *    for all options in getWaitlist:
-	 *       options.getAppGroupName() is not in 'superGroups'.
+	 *       options.getAppGroupName() is not in 'groups'.
 	 *
 	 * Invariant 2:
 	 *    if getWaitlist is non-empty:
@@ -154,215 +224,136 @@ public:
 
 	const VariantMap *agentsOptions;
 
-	#include <ApplicationPool2/Pool/AnalyticsCollection.h>
-	#include <ApplicationPool2/Pool/GarbageCollection.h>
-	#include <ApplicationPool2/Pool/GeneralUtils.h>
-	#include <ApplicationPool2/Pool/ProcessUtils.h>
-	#include <ApplicationPool2/Pool/Inspection.h>
-	#include <ApplicationPool2/Pool/Debug.h>
-
 // Actually private, but marked public so that unit tests can access the fields.
 public:
-
-	/** Process all waiters on the getWaitlist. Call when capacity has become free.
-	 * This function assigns sessions to them by calling get() on the corresponding
-	 * SuperGroups, or by creating more SuperGroups, in so far the new capacity allows.
-	 */
-	void assignSessionsToGetWaiters(boost::container::vector<Callback> &postLockActions) {
-		bool done = false;
-		vector<GetWaiter>::iterator it, end = getWaitlist.end();
-		vector<GetWaiter> newWaitlist;
-
-		for (it = getWaitlist.begin(); it != end && !done; it++) {
-			GetWaiter &waiter = *it;
-
-			SuperGroup *superGroup = findMatchingSuperGroup(waiter.options);
-			if (superGroup != NULL) {
-				SessionPtr session = superGroup->get(waiter.options, waiter.callback,
-					postLockActions);
-				if (session != NULL) {
-					postLockActions.push_back(boost::bind(GetCallback::call,
-						waiter.callback, session, ExceptionPtr()));
-				}
-				/* else: the callback has now been put in
-				 *       the group's get wait list.
-				 */
-			} else if (!atFullCapacityUnlocked()) {
-				createSuperGroupAndAsyncGetFromIt(waiter.options, waiter.callback,
-					postLockActions);
-			} else {
-				/* Still cannot satisfy this get request. Keep it on the get
-				 * wait list and try again later.
-				 */
-				newWaitlist.push_back(waiter);
-			}
+	/****** Debugging support *******/
+
+	struct DebugSupport {
+		/** Mailbox for the unit tests to receive messages on. */
+		MessageBoxPtr debugger;
+		/** Mailbox for the ApplicationPool code to receive messages on. */
+		MessageBoxPtr messages;
+
+		// Choose aspects to debug.
+		bool restarting;
+		bool spawning;
+		bool oobw;
+		bool testOverflowRequestQueue;
+		bool detachedProcessesChecker;
+
+		// The following fields may only be accessed by Pool.
+		boost::mutex syncher;
+		unsigned int spawnLoopIteration;
+
+		DebugSupport() {
+			debugger = boost::make_shared<MessageBox>();
+			messages = boost::make_shared<MessageBox>();
+			restarting = true;
+			spawning   = true;
+			oobw       = false;
+			detachedProcessesChecker = false;
+			testOverflowRequestQueue = false;
+			spawnLoopIteration = 0;
 		}
+	};
 
-		std::swap(getWaitlist, newWaitlist);
-	}
+	typedef boost::shared_ptr<DebugSupport> DebugSupportPtr;
 
-	template<typename Queue>
-	static void assignExceptionToGetWaiters(Queue &getWaitlist,
-		const ExceptionPtr &exception,
-		boost::container::vector<Callback> &postLockActions)
-	{
-		while (!getWaitlist.empty()) {
-			postLockActions.push_back(boost::bind(GetCallback::call,
-				getWaitlist.front().callback, SessionPtr(),
-				exception));
-			getWaitlist.pop_front();
-		}
-	}
-
-	void possiblySpawnMoreProcessesForExistingGroups() {
-		/* Looks for Groups that are waiting for capacity to become available,
-		 * and spawn processes in those groups.
-		 */
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			foreach (GroupPtr group, superGroup->groups) {
-				if (group->isWaitingForCapacity()) {
-					P_DEBUG("Group " << group->name << " is waiting for capacity");
-					group->spawn();
-					if (atFullCapacityUnlocked()) {
-						return;
-					}
-				}
-			}
-			sg_it.next();
-		}
-		/* Now look for Groups that haven't maximized their allowed capacity
-		 * yet, and spawn processes in those groups.
-		 */
-		sg_it = SuperGroupMap::ConstIterator(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			foreach (GroupPtr group, superGroup->groups) {
-				if (group->shouldSpawn()) {
-					P_DEBUG("Group " << group->name << " requests more processes to be spawned");
-					group->spawn();
-					if (atFullCapacityUnlocked()) {
-						return;
-					}
-				}
-			}
-			sg_it.next();
-		}
-	}
+	DebugSupportPtr debugSupport;
 
-	void migrateSuperGroupGetWaitlistToPool(const SuperGroupPtr &superGroup) {
-		getWaitlist.reserve(getWaitlist.size() + superGroup->getWaitlist.size());
-		while (!superGroup->getWaitlist.empty()) {
-			getWaitlist.push_back(superGroup->getWaitlist.front());
-			superGroup->getWaitlist.pop_front();
-		}
-	}
-
-	unsigned int capacityUsedUnlocked() const {
-		if (superGroups.size() == 1) {
-			SuperGroupPtr *superGroup;
-			superGroups.lookupRandom(NULL, &superGroup);
-			return (*superGroup)->capacityUsed();
-		} else {
-			SuperGroupMap::ConstIterator sg_it(superGroups);
-			int result = 0;
-			while (*sg_it != NULL) {
-				const SuperGroupPtr &superGroup = sg_it.getValue();
-				result += superGroup->capacityUsed();
-				sg_it.next();
-			}
-			return result;
-		}
-	}
 
-	bool atFullCapacityUnlocked() const {
-		return capacityUsedUnlocked() >= max;
-	}
+	/****** Analytics collection ******/
 
-	/**
-	 * Calls Group::detach() so be sure to fix up the invariants afterwards.
-	 * See the comments for Group::detach() and the code for detachProcessUnlocked().
-	 */
-	ProcessPtr forceFreeCapacity(const Group *exclude,
-		boost::container::vector<Callback> &postLockActions)
-	{
-		ProcessPtr process = findOldestIdleProcess(exclude);
-		if (process != NULL) {
-			P_DEBUG("Forcefully detaching process " << process->inspect() <<
-				" in order to free capacity in the pool");
-
-			Group *group = process->getGroup();
-			assert(group != NULL);
-			assert(group->getWaitlist.empty());
-
-			SuperGroup *superGroup = group->getSuperGroup();
-			assert(superGroup != NULL);
-			(void) superGroup;
-
-			group->detach(process, postLockActions);
-		}
-		return process;
-	}
+	struct UnionStationLogEntry {
+		string groupName;
+		const char *category;
+		string key;
+		string data;
+	};
 
-	/**
-	 * Forcefully destroys and detaches the given SuperGroup. After detaching
-	 * the SuperGroup may have a non-empty getWaitlist so be sure to do
-	 * something with it.
-	 *
-	 * Also, one of the post lock actions can potentially perform a long-running
-	 * operation, so running them in a thread is advised.
-	 */
-	void forceDetachSuperGroup(const SuperGroupPtr &superGroup,
-		boost::container::vector<Callback> &postLockActions,
-		const SuperGroup::ShutdownCallback &callback)
-	{
-		const SuperGroupPtr sp = superGroup; // Prevent premature destruction.
-		bool removed = superGroups.erase(superGroup->name);
-		assert(removed);
-		(void) removed; // Shut up compiler warning.
-		superGroup->destroy(false, postLockActions, callback);
-	}
+	SystemMetricsCollector systemMetricsCollector;
+	SystemMetrics systemMetrics;
 
-	bool detachProcessUnlocked(const ProcessPtr &process,
-		boost::container::vector<Callback> &postLockActions)
-	{
-		if (OXT_LIKELY(process->isAlive())) {
-			verifyInvariants();
-
-			Group *group = process->getGroup();
-			SuperGroup *superGroup = group->getSuperGroup();
-			assert(superGroup->state != SuperGroup::INITIALIZING);
-			assert(superGroup->getWaitlist.empty());
-			(void) superGroup;
-
-			group->detach(process, postLockActions);
-			// 'process' may now be a stale pointer so don't use it anymore.
-			assignSessionsToGetWaiters(postLockActions);
-			possiblySpawnMoreProcessesForExistingGroups();
-
-			group->verifyInvariants();
-			superGroup->verifyInvariants();
-			verifyInvariants();
-			verifyExpensiveInvariants();
-
-			return true;
-		} else {
-			return false;
-		}
-	}
+	void initializeAnalyticsCollection();
+	static void collectAnalytics(PoolPtr self);
+	static void collectPids(const ProcessList &processes, vector<pid_t> &pids);
+	static void updateProcessMetrics(const ProcessList &processes,
+		const ProcessMetricMap &allMetrics,
+		vector<ProcessPtr> &processesToDetach);
+	void prepareUnionStationProcessStateLogs(vector<UnionStationLogEntry> &logEntries,
+		const GroupPtr &group) const;
+	void prepareUnionStationSystemMetricsLogs(vector<UnionStationLogEntry> &logEntries,
+		const GroupPtr &group) const;
+	void realCollectAnalytics();
 
-	struct DetachSuperGroupWaitTicket {
+
+	/****** Garbage collection ******/
+
+	struct GarbageCollectorState {
+		unsigned long long now;
+		unsigned long long nextGcRunTime;
+		boost::container::vector<Callback> actions;
+	};
+
+	boost::condition_variable garbageCollectionCond;
+
+	void initializeGarbageCollection();
+	static void garbageCollect(PoolPtr self);
+	void maybeUpdateNextGcRuntime(GarbageCollectorState &state, unsigned long candidate);
+	void checkWhetherProcessCanBeGarbageCollected(GarbageCollectorState &state,
+		const GroupPtr &group, const ProcessPtr &process, ProcessList &output);
+	void garbageCollectProcessesInGroup(GarbageCollectorState &state,
+		const GroupPtr &group);
+	void maybeCleanPreloader(GarbageCollectorState &state, const GroupPtr &group);
+	unsigned long long realGarbageCollect();
+	void wakeupGarbageCollector();
+
+
+	/****** General utilities ******/
+
+	static const char *maybeColorize(const InspectOptions &options, const char *color);
+	static const char *maybePluralize(unsigned int count, const char *singular, const char *plural);
+	static void runAllActions(const boost::container::vector<Callback> &actions);
+	static void runAllActionsWithCopy(boost::container::vector<Callback> actions);
+	bool runHookScripts(const char *name,
+		const boost::function<void (HookScriptOptions &)> &setup) const;
+	void verifyInvariants() const;
+	void verifyExpensiveInvariants() const;
+	void fullVerifyInvariants() const;
+	void assignSessionsToGetWaiters(boost::container::vector<Callback> &postLockActions);
+	template<typename Queue> static void assignExceptionToGetWaiters(Queue &getWaitlist,
+		const ExceptionPtr &exception,
+		boost::container::vector<Callback> &postLockActions);
+	static void syncGetCallback(const SessionPtr &session, const ExceptionPtr &e,
+		void *userData);
+
+
+	/****** Group data structure utilities ******/
+
+	struct DetachGroupWaitTicket {
 		boost::mutex syncher;
 		boost::condition_variable cond;
-		SuperGroup::ShutdownResult result;
 		bool done;
 
-		DetachSuperGroupWaitTicket() {
+		DetachGroupWaitTicket() {
 			done = false;
 		}
 	};
 
+	const GroupPtr getGroup(const char *name);
+	Group *findMatchingGroup(const Options &options);
+	GroupPtr createGroup(const Options &options);
+	GroupPtr createGroupAndAsyncGetFromIt(const Options &options,
+		const GetCallback &callback, boost::container::vector<Callback> &postLockActions);
+	void forceDetachGroup(const GroupPtr &group,
+		const Callback &callback,
+		boost::container::vector<Callback> &postLockActions);
+	static void syncDetachGroupCallback(boost::shared_ptr<DetachGroupWaitTicket> ticket);
+	static void waitDetachGroupCallback(boost::shared_ptr<DetachGroupWaitTicket> ticket);
+
+
+	/****** Process data structure utilities ******/
+
 	struct DisableWaitTicket {
 		boost::mutex syncher;
 		boost::condition_variable cond;
@@ -374,641 +365,95 @@ public:
 		}
 	};
 
-	static void syncDetachSuperGroupCallback(SuperGroup::ShutdownResult result,
-		boost::shared_ptr<DetachSuperGroupWaitTicket> ticket)
-	{
-		LockGuard l(ticket->syncher);
-		ticket->done = true;
-		ticket->result = result;
-		ticket->cond.notify_one();
-	}
-
-	static void waitDetachSuperGroupCallback(boost::shared_ptr<DetachSuperGroupWaitTicket> ticket) {
-		ScopedLock l(ticket->syncher);
-		while (!ticket->done) {
-			ticket->cond.wait(l);
-		}
-	}
-
+	ProcessPtr findOldestIdleProcess(const Group *exclude = NULL) const;
+	ProcessPtr findBestProcessToTrash() const;
+	ProcessPtr forceFreeCapacity(const Group *exclude,
+		boost::container::vector<Callback> &postLockActions);
+	bool detachProcessUnlocked(const ProcessPtr &process,
+		boost::container::vector<Callback> &postLockActions);
 	static void syncDisableProcessCallback(const ProcessPtr &process, DisableResult result,
-		boost::shared_ptr<DisableWaitTicket> ticket)
-	{
-		LockGuard l(ticket->syncher);
-		ticket->done = true;
-		ticket->result = result;
-		ticket->cond.notify_one();
-	}
+		boost::shared_ptr<DisableWaitTicket> ticket);
+	void possiblySpawnMoreProcessesForExistingGroups();
 
-	static void syncGetCallback(const SessionPtr &session, const ExceptionPtr &e,
-		void *userData)
-	{
-		Ticket *ticket = static_cast<Ticket *>(userData);
-		ScopedLock lock(ticket->syncher);
-		if (OXT_LIKELY(session != NULL)) {
-			ticket->session = session;
-		} else {
-			ticket->exception = e;
-		}
-		ticket->cond.notify_one();
-	}
-
-	SuperGroup *findMatchingSuperGroup(const Options &options) {
-		SuperGroupPtr *superGroup;
-		if (superGroups.lookup(options.getAppGroupName(), &superGroup)) {
-			return superGroup->get();
-		} else {
-			return NULL;
-		}
-	}
-
-	SuperGroupPtr createSuperGroup(const Options &options) {
-		SuperGroupPtr superGroup = boost::make_shared<SuperGroup>(this,
-			options);
-		superGroup->initialize();
-		superGroups.insert(options.getAppGroupName(), superGroup);
-		wakeupGarbageCollector();
-		return superGroup;
-	}
-
-	SuperGroupPtr createSuperGroupAndAsyncGetFromIt(const Options &options,
-		const GetCallback &callback, boost::container::vector<Callback> &postLockActions)
-	{
-		SuperGroupPtr superGroup = createSuperGroup(options);
-		SessionPtr session = superGroup->get(options, callback,
-			postLockActions);
-		/* Callback should now have been put on the wait list,
-		 * unless something has changed and we forgot to update
-		 * some code here...
-		 */
-		assert(session == NULL);
-		return superGroup;
-	}
-
-	// Debugging helper function, implemented in .cpp file so that GDB can access it.
-	const SuperGroupPtr getSuperGroup(const char *name);
+
+	/****** State inspection ******/
+
+	unsigned int capacityUsedUnlocked() const;
+	bool atFullCapacityUnlocked() const;
+	void inspectProcessList(const InspectOptions &options, stringstream &result,
+		const Group *group, const ProcessList &processes) const;
 
 public:
+	typedef void (*AbortLongRunningConnectionsCallback)(const ProcessPtr &process);
 	AbortLongRunningConnectionsCallback abortLongRunningConnectionsCallback;
 
-	Pool(const SpawnerFactoryPtr &spawnerFactory,
-		const VariantMap *agentsOptions = NULL)
-		: sessionObjectPool(64, 1024),
-		  abortLongRunningConnectionsCallback(NULL)
-	{
-		this->spawnerFactory = spawnerFactory;
-		this->agentsOptions = agentsOptions;
-
-		try {
-			systemMetricsCollector.collect(systemMetrics);
-		} catch (const RuntimeException &e) {
-			P_WARN("Unable to collect system metrics: " << e.what());
-		}
 
-		lifeStatus   = ALIVE;
-		max          = 6;
-		maxIdleTime  = 60 * 1000000;
-		selfchecking = true;
-		palloc       = psg_create_pool(PSG_DEFAULT_POOL_SIZE);
-
-		// The following code only serve to instantiate certain inline methods
-		// so that they can be invoked from gdb.
-		(void) SuperGroupPtr().get();
-		(void) GroupPtr().get();
-		(void) ProcessPtr().get();
-		(void) SessionPtr().get();
-	}
-
-	~Pool() {
-		if (lifeStatus != SHUT_DOWN) {
-			P_BUG("You must call Pool::destroy() before actually destroying the Pool object!");
-		}
-		psg_destroy_pool(palloc);
-	}
-
-	/** Must be called right after construction. */
-	void initialize() {
-		LockGuard l(syncher);
-		initializeAnalyticsCollection();
-		initializeGarbageCollection();
-	}
-
-	void initDebugging() {
-		LockGuard l(syncher);
-		debugSupport = boost::make_shared<DebugSupport>();
-	}
-
-	/** Should be called right after the agent has received
-	 * the message to exit gracefully. This will tell processes to
-	 * abort any long-running connections, e.g. WebSocket connections,
-	 * because the RequestHandler has to wait until all connections are
-	 * finished before proceeding with shutdown.
-	 */
-	void prepareForShutdown() {
-		TRACE_POINT();
-		ScopedLock lock(syncher);
-		assert(lifeStatus == ALIVE);
-		lifeStatus = PREPARED_FOR_SHUTDOWN;
-		if (abortLongRunningConnectionsCallback != NULL) {
-			vector<ProcessPtr> processes = getProcesses(false);
-			foreach (ProcessPtr process, processes) {
-				// Ensure that the process is not immediately respawned.
-				process->getGroup()->options.minProcesses = 0;
-				abortLongRunningConnectionsCallback(process);
-			}
-		}
-	}
-
-	/** Must be called right before destruction. */
-	void destroy() {
-		TRACE_POINT();
-		ScopedLock lock(syncher);
-		assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
-
-		lifeStatus = SHUTTING_DOWN;
-
-		while (!superGroups.empty()) {
-			SuperGroupPtr *superGroup;
-			superGroups.lookupRandom(NULL, &superGroup);
-			string name = superGroup->get()->name;
-			lock.unlock();
-			detachSuperGroupByName(name);
-			lock.lock();
-		}
+	/****** Initialization and shutdown ******/
 
-		UPDATE_TRACE_POINT();
-		lock.unlock();
-		interruptableThreads.interrupt_and_join_all();
-		nonInterruptableThreads.join_all();
-		lock.lock();
+	Pool(const SpawningKit::FactoryPtr &spawningKitFactory,
+		const VariantMap *agentsOptions = NULL);
+	~Pool();
+	void initialize();
+	void initDebugging();
+	void prepareForShutdown();
+	void destroy();
 
-		lifeStatus = SHUT_DOWN;
 
-		UPDATE_TRACE_POINT();
-		verifyInvariants();
-		verifyExpensiveInvariants();
-	}
+	/****** General utilities ******/
 
-	// 'lockNow == false' may only be used during unit tests. Normally we
-	// should never call the callback while holding the lock.
-	void asyncGet(const Options &options, const GetCallback &callback, bool lockNow = true) {
-		DynamicScopedLock lock(syncher, lockNow);
+	Context *getContext();
+	const SpawningKit::ConfigPtr &getSpawningKitConfig() const;
+	const UnionStation::CorePtr &getUnionStationCore() const;
+	const RandomGeneratorPtr &getRandomGenerator() const;
 
-		assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
-		verifyInvariants();
-		P_TRACE(2, "asyncGet(appGroupName=" << options.getAppGroupName() << ")");
-		boost::container::vector<Callback> actions;
 
-		SuperGroup *existingSuperGroup = findMatchingSuperGroup(options);
-		if (OXT_LIKELY(existingSuperGroup != NULL)) {
-			/* Best case: the app super group is already in the pool. Let's use it. */
-			P_TRACE(2, "Found existing SuperGroup");
-			existingSuperGroup->verifyInvariants();
-			SessionPtr session = existingSuperGroup->get(options, callback, actions);
-			existingSuperGroup->verifyInvariants();
-			verifyInvariants();
-			P_TRACE(2, "asyncGet() finished");
-			if (lockNow) {
-				lock.unlock();
-			}
-			if (session != NULL) {
-				callback(session, ExceptionPtr());
-			}
-
-		} else if (!atFullCapacityUnlocked()) {
-			/* The app super group isn't in the pool and we have enough free
-			 * resources to make a new one.
-			 */
-			P_DEBUG("Spawning new SuperGroup");
-			SuperGroupPtr superGroup = createSuperGroupAndAsyncGetFromIt(options,
-				callback, actions);
-			superGroup->verifyInvariants();
-			verifyInvariants();
-			P_DEBUG("asyncGet() finished");
-
-		} else {
-			/* Uh oh, the app super group isn't in the pool but we don't
-			 * have the resources to make a new one. The sysadmin should
-			 * configure the system to let something like this happen
-			 * as least as possible, but let's try to handle it as well
-			 * as we can.
-			 */
-			ProcessPtr freedProcess = forceFreeCapacity(NULL, actions);
-			if (freedProcess == NULL) {
-				/* No process is eligible for killing. This could happen if, for example,
-				 * all (super)groups are currently initializing/restarting/spawning/etc.
-				 * We have no choice but to satisfy this get() action later when resources
-				 * become available.
-				 */
-				P_DEBUG("Could not free a process; putting request to top-level getWaitlist");
-				getWaitlist.push_back(GetWaiter(
-					options.copyAndPersist().detachFromUnionStationTransaction(),
-					callback));
-			} else {
-				/* Now that a process has been trashed we can create
-				 * the missing SuperGroup.
-				 */
-				P_DEBUG("Creating new SuperGroup");
-				SuperGroupPtr superGroup;
-				superGroup = boost::make_shared<SuperGroup>(this, options);
-				superGroup->initialize();
-				superGroups.insert(options.getAppGroupName(), superGroup);
-				wakeupGarbageCollector();
-				SessionPtr session = superGroup->get(options, callback,
-					actions);
-				/* The SuperGroup is still initializing so the callback
-				 * should now have been put on the wait list,
-				 * unless something has changed and we forgot to update
-				 * some code here...
-				 */
-				assert(session == NULL);
-				freedProcess->getGroup()->verifyInvariants();
-				superGroup->verifyInvariants();
-			}
-
-			assert(atFullCapacityUnlocked());
-			verifyInvariants();
-			verifyExpensiveInvariants();
-			P_TRACE(2, "asyncGet() finished");
-		}
+	/****** Group manipulation ******/
 
-		if (!actions.empty()) {
-			if (lockNow) {
-				if (lock.owns_lock()) {
-					lock.unlock();
-				}
-				runAllActions(actions);
-			} else {
-				// This state is not allowed. If we reach
-				// here then it probably indicates a bug in
-				// the test suite.
-				abort();
-			}
-		}
-	}
+	GroupPtr findOrCreateGroup(const Options &options);
+	GroupPtr findGroupByApiKey(const StaticString &value, bool lock = true) const;
+	bool detachGroupByName(const HashedStaticString &name);
+	bool detachGroupByApiKey(const StaticString &value);
+	bool restartGroupByName(const StaticString &name,
+		const RestartOptions &options = RestartOptions::makeAuthorized());
+	unsigned int restartGroupsByAppRoot(const StaticString &appRoot,
+		const RestartOptions &options = RestartOptions::makeAuthorized());
 
-	// TODO: 'ticket' should be a boost::shared_ptr for interruption-safety.
-	SessionPtr get(const Options &options, Ticket *ticket) {
-		ticket->session.reset();
-		ticket->exception.reset();
 
-		GetCallback callback;
-		callback.func = syncGetCallback;
-		callback.userData = ticket;
-		asyncGet(options, callback);
+	/***** Process manipulation ******/
 
-		ScopedLock lock(ticket->syncher);
-		while (ticket->session == NULL && ticket->exception == NULL) {
-			ticket->cond.wait(lock);
-		}
-		lock.unlock();
-
-		if (OXT_LIKELY(ticket->session != NULL)) {
-			SessionPtr session = ticket->session;
-			ticket->session.reset();
-			return session;
-		} else {
-			rethrowException(ticket->exception);
-			return SessionPtr(); // Shut up compiler warning.
-		}
-	}
-
-	GroupPtr findOrCreateGroup(const Options &options) {
-		Options options2 = options;
-		options2.noop = true;
-
-		Ticket ticket;
-		{
-			LockGuard l(syncher);
-			SuperGroupPtr *superGroup;
-			if (!superGroups.lookup(options.getAppGroupName(), &superGroup)) {
-				// Forcefully create SuperGroup, don't care whether resource limits
-				// actually allow it.
-				createSuperGroup(options);
-			}
-		}
-		return get(options2, &ticket)->getGroup()->shared_from_this();
-	}
-
-	void setMax(unsigned int max) {
-		ScopedLock l(syncher);
-		assert(max > 0);
-		fullVerifyInvariants();
-		bool bigger = max > this->max;
-		this->max = max;
-		if (bigger) {
-			/* If there are clients waiting for resources
-			 * to become free, spawn more processes now that
-			 * we have the capacity.
-			 *
-			 * We favor waiters on the pool over waiters on the
-			 * the groups because the latter already have the
-			 * resources to eventually complete. Favoring waiters
-			 * on the pool should be fairer.
-			 */
-			boost::container::vector<Callback> actions;
-			assignSessionsToGetWaiters(actions);
-			possiblySpawnMoreProcessesForExistingGroups();
-
-			fullVerifyInvariants();
-			l.unlock();
-			runAllActions(actions);
-		} else {
-			fullVerifyInvariants();
-		}
-	}
-
-	void setMaxIdleTime(unsigned long long value) {
-		LockGuard l(syncher);
-		maxIdleTime = value;
-		wakeupGarbageCollector();
-	}
-
-	void enableSelfChecking(bool enabled) {
-		LockGuard l(syncher);
-		selfchecking = enabled;
-	}
-
-	unsigned int capacityUsed() const {
-		LockGuard l(syncher);
-		return capacityUsedUnlocked();
-	}
-
-	bool atFullCapacity() const {
-		LockGuard l(syncher);
-		return atFullCapacityUnlocked();
-	}
-
-	vector<ProcessPtr> getProcesses(bool lock = true) const {
-		DynamicScopedLock l(syncher, lock);
-		vector<ProcessPtr> result;
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			SuperGroup::GroupList &groups = superGroup->groups;
-			SuperGroup::GroupList::const_iterator g_it, g_end = groups.end();
-			for (g_it = groups.begin(); g_it != g_end; g_it++) {
-				const GroupPtr &group = *g_it;
-				ProcessList::const_iterator p_it;
-
-				for (p_it = group->enabledProcesses.begin(); p_it != group->enabledProcesses.end(); p_it++) {
-					result.push_back(*p_it);
-				}
-				for (p_it = group->disablingProcesses.begin(); p_it != group->disablingProcesses.end(); p_it++) {
-					result.push_back(*p_it);
-				}
-				for (p_it = group->disabledProcesses.begin(); p_it != group->disabledProcesses.end(); p_it++) {
-					result.push_back(*p_it);
-				}
-			}
-			sg_it.next();
-		}
-		return result;
-	}
+	vector<ProcessPtr> getProcesses(bool lock = true) const;
+	ProcessPtr findProcessByGupid(const StaticString &gupid, bool lock = true) const;
+	ProcessPtr findProcessByPid(pid_t pid, bool lock = true) const;
+	bool detachProcess(const ProcessPtr &process);
+	bool detachProcess(pid_t pid,
+		const AuthenticationOptions &options = AuthenticationOptions::makeAuthorized());
+	bool detachProcess(const string &gupid,
+		const AuthenticationOptions &options = AuthenticationOptions::makeAuthorized());
+	DisableResult disableProcess(const StaticString &gupid);
 
-	/**
-	 * Returns the total number of processes in the pool, including all disabling and
-	 * disabled processes, but excluding processes that are shutting down and excluding
-	 * processes that are being spawned.
-	 */
-	unsigned int getProcessCount(bool lock = true) const {
-		DynamicScopedLock l(syncher, lock);
-		unsigned int result = 0;
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			result += superGroup->getProcessCount();
-			sg_it.next();
-		}
-		return result;
-	}
-
-	unsigned int getSuperGroupCount() const {
-		LockGuard l(syncher);
-		return superGroups.size();
-	}
-
-	SuperGroupPtr findSuperGroupBySecret(const string &secret, bool lock = true) const {
-		DynamicScopedLock l(syncher, lock);
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			if (superGroup->secret == secret) {
-				return superGroup;
-			}
-			sg_it.next();
-		}
-		return SuperGroupPtr();
-	}
-
-	ProcessPtr findProcessByGupid(const StaticString &gupid, bool lock = true) const {
-		vector<ProcessPtr> processes = getProcesses(lock);
-		vector<ProcessPtr>::const_iterator it, end = processes.end();
-		for (it = processes.begin(); it != end; it++) {
-			const ProcessPtr &process = *it;
-			if (StaticString(process->gupid, process->gupidSize) == gupid) {
-				return process;
-			}
-		}
-		return ProcessPtr();
-	}
-
-	ProcessPtr findProcessByPid(pid_t pid, bool lock = true) const {
-		vector<ProcessPtr> processes = getProcesses(lock);
-		vector<ProcessPtr>::const_iterator it, end = processes.end();
-		for (it = processes.begin(); it != end; it++) {
-			const ProcessPtr &process = *it;
-			if (process->pid == pid) {
-				return process;
-			}
-		}
-		return ProcessPtr();
-	}
-
-	bool detachSuperGroupByName(const string &name) {
-		TRACE_POINT();
-		ScopedLock l(syncher);
-		SuperGroupPtr superGroup = superGroups.lookupCopy(name);
-
-		if (OXT_LIKELY(superGroup != NULL)) {
-			P_ASSERT_EQ(superGroup->name, name);
-			UPDATE_TRACE_POINT();
-			verifyInvariants();
-			verifyExpensiveInvariants();
-
-			boost::container::vector<Callback> actions;
-			boost::shared_ptr<DetachSuperGroupWaitTicket> ticket =
-				boost::make_shared<DetachSuperGroupWaitTicket>();
-			ExceptionPtr exception = copyException(
-				GetAbortedException("The containing SuperGroup was detached."));
-
-			forceDetachSuperGroup(superGroup, actions,
-				boost::bind(syncDetachSuperGroupCallback, _1, ticket));
-			assignExceptionToGetWaiters(superGroup->getWaitlist,
-				exception, actions);
-			#if 0
-			/* If this SuperGroup had get waiters, either
-			 * on itself or in one of its groups, then we must
-			 * reprocess them immediately. Detaching such a
-			 * SuperGroup is essentially the same as restarting it.
-			 */
-			migrateSuperGroupGetWaitlistToPool(superGroup);
-
-			UPDATE_TRACE_POINT();
-			assignSessionsToGetWaiters(actions);
-			#endif
-			possiblySpawnMoreProcessesForExistingGroups();
-
-			verifyInvariants();
-			verifyExpensiveInvariants();
-
-			l.unlock();
-			UPDATE_TRACE_POINT();
-			runAllActions(actions);
-			actions.clear();
-
-			UPDATE_TRACE_POINT();
-			ScopedLock l2(ticket->syncher);
-			while (!ticket->done) {
-				ticket->cond.wait(l2);
-			}
-			return ticket->result == SuperGroup::SUCCESS;
-		} else {
-			return false;
-		}
-	}
-
-	bool detachSuperGroupBySecret(const string &superGroupSecret) {
-		ScopedLock l(syncher);
-		SuperGroupPtr superGroup = findSuperGroupBySecret(superGroupSecret, false);
-		if (superGroup != NULL) {
-			string name = superGroup->name;
-			superGroup.reset();
-			l.unlock();
-			return detachSuperGroupByName(name);
-		} else {
-			return false;
-		}
-	}
 
-	bool detachProcess(const ProcessPtr &process) {
-		ScopedLock l(syncher);
-		boost::container::vector<Callback> actions;
-		bool result = detachProcessUnlocked(process, actions);
-		fullVerifyInvariants();
-		l.unlock();
-		runAllActions(actions);
-		return result;
-	}
-
-	bool detachProcess(pid_t pid) {
-		ScopedLock l(syncher);
-		ProcessPtr process = findProcessByPid(pid, false);
-		if (process != NULL) {
-			boost::container::vector<Callback> actions;
-			bool result = detachProcessUnlocked(process, actions);
-			fullVerifyInvariants();
-			l.unlock();
-			runAllActions(actions);
-			return result;
-		} else {
-			return false;
-		}
-	}
-
-	bool detachProcess(const string &gupid) {
-		ScopedLock l(syncher);
-		ProcessPtr process = findProcessByGupid(gupid, false);
-		if (process != NULL) {
-			boost::container::vector<Callback> actions;
-			bool result = detachProcessUnlocked(process, actions);
-			fullVerifyInvariants();
-			l.unlock();
-			runAllActions(actions);
-			return result;
-		} else {
-			return false;
-		}
-	}
-
-	DisableResult disableProcess(const StaticString &gupid) {
-		ScopedLock l(syncher);
-		ProcessPtr process = findProcessByGupid(gupid, false);
-		if (process != NULL) {
-			Group *group = process->getGroup();
-			// Must be a boost::shared_ptr to be interruption-safe.
-			boost::shared_ptr<DisableWaitTicket> ticket = boost::make_shared<DisableWaitTicket>();
-			DisableResult result = group->disable(process,
-				boost::bind(syncDisableProcessCallback, _1, _2, ticket));
-			group->verifyInvariants();
-			group->verifyExpensiveInvariants();
-			if (result == DR_DEFERRED) {
-				l.unlock();
-				ScopedLock l2(ticket->syncher);
-				while (!ticket->done) {
-					ticket->cond.wait(l2);
-				}
-				return ticket->result;
-			} else {
-				return result;
-			}
-		} else {
-			return DR_NOOP;
-		}
-	}
-
-	bool restartGroupByName(const StaticString &name, RestartMethod method = RM_DEFAULT) {
-		ScopedLock l(syncher);
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			foreach (const GroupPtr &group, superGroup->groups) {
-				if (name == group->name) {
-					if (!group->restarting()) {
-						group->restart(group->options, method);
-					}
-					return true;
-				}
-			}
-			sg_it.next();
-		}
+	/****** State inspection ******/
 
-		return false;
-	}
-
-	unsigned int restartSuperGroupsByAppRoot(const StaticString &appRoot) {
-		ScopedLock l(syncher);
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		unsigned int result = 0;
-
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			if (appRoot == superGroup->options.appRoot) {
-				result++;
-				superGroup->restart(superGroup->options);
-			}
-			sg_it.next();
-		}
+	unsigned int capacityUsed() const;
+	bool atFullCapacity() const;
+	unsigned int getProcessCount(bool lock = true) const;
+	unsigned int getGroupCount() const;
+	string inspect(const InspectOptions &options = InspectOptions::makeAuthorized(),
+		bool lock = true) const;
+	string toXml(const ToXmlOptions &options = ToXmlOptions::makeAuthorized(),
+		bool lock = true) const;
 
-		return result;
-	}
 
-	/**
-	 * Checks whether at least one process is being spawned.
-	 */
-	bool isSpawning(bool lock = true) const {
-		DynamicScopedLock l(syncher, lock);
-		SuperGroupMap::ConstIterator sg_it(superGroups);
-		while (*sg_it != NULL) {
-			const SuperGroupPtr &superGroup = sg_it.getValue();
-			foreach (GroupPtr group, superGroup->groups) {
-				if (group->spawning()) {
-					return true;
-				}
-			}
-			sg_it.next();
-		}
-		return false;
-	}
+	/****** Miscellaneous ******/
+
+	void asyncGet(const Options &options, const GetCallback &callback, bool lockNow = true);
+	SessionPtr get(const Options &options, Ticket *ticket);
+	void setMax(unsigned int max);
+	void setMaxIdleTime(unsigned long long value);
+	void enableSelfChecking(bool enabled);
+	bool isSpawning(bool lock = true) const;
+	bool authorizeByApiKey(const ApiKey &key, bool lock = true) const;
+	bool authorizeByUid(uid_t uid, bool lock = true) const;
 };