passenger 5.0.9 → 5.0.10

data/ext/common/ApplicationPool2/Pool/GroupUtils.cpp

@@ -0,0 +1,276 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2011-2015 Phusion
+ *
+ *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#include <ApplicationPool2/Pool.h>
+
+/*************************************************************************
+ *
+ * Group data structure utility functions for ApplicationPool2::Pool
+ *
+ *************************************************************************/
+
+namespace Passenger {
+namespace ApplicationPool2 {
+
+using namespace std;
+using namespace boost;
+
+
+/****************************
+ *
+ * Private methods
+ *
+ ****************************/
+
+
+const GroupPtr
+Pool::getGroup(const char *name) {
+	GroupPtr *group;
+	if (groups.lookup(name, &group)) {
+		return *group;
+	} else {
+		return GroupPtr();
+	}
+}
+
+Group *
+Pool::findMatchingGroup(const Options &options) {
+	GroupPtr *group;
+	if (groups.lookup(options.getAppGroupName(), &group)) {
+		return group->get();
+	} else {
+		return NULL;
+	}
+}
+
+GroupPtr
+Pool::createGroup(const Options &options) {
+	GroupPtr group = boost::make_shared<Group>(this, options);
+	group->initialize();
+	groups.insert(options.getAppGroupName(), group);
+	wakeupGarbageCollector();
+	return group;
+}
+
+GroupPtr
+Pool::createGroupAndAsyncGetFromIt(const Options &options,
+	const GetCallback &callback, boost::container::vector<Callback> &postLockActions)
+{
+	GroupPtr group = createGroup(options);
+	SessionPtr session = group->get(options, callback,
+		postLockActions);
+	/* If !options.noop, then the callback should now have been put on the
+	 * wait list, unless something has changed and we forgot to update
+	 * some code here...
+	 */
+	if (session != NULL) {
+		assert(options.noop);
+		postLockActions.push_back(boost::bind(GetCallback::call,
+			callback, session, ExceptionPtr()));
+	}
+	return group;
+}
+
+/**
+ * Forcefully destroys and detaches the given Group. After detaching
+ * the Group may have a non-empty getWaitlist so be sure to do
+ * something with it.
+ *
+ * Also, one of the post lock actions can potentially perform a long-running
+ * operation, so running them in a thread is advised.
+ */
+void
+Pool::forceDetachGroup(const GroupPtr &group,
+	const Callback &callback,
+	boost::container::vector<Callback> &postLockActions)
+{
+	assert(group->getWaitlist.empty());
+	const GroupPtr p = group; // Prevent premature destruction.
+	bool removed = groups.erase(group->getName());
+	assert(removed);
+	(void) removed; // Shut up compiler warning.
+	group->shutdown(callback, postLockActions);
+}
+
+void
+Pool::syncDetachGroupCallback(boost::shared_ptr<DetachGroupWaitTicket> ticket) {
+	LockGuard l(ticket->syncher);
+	ticket->done = true;
+	ticket->cond.notify_one();
+}
+
+void
+Pool::waitDetachGroupCallback(boost::shared_ptr<DetachGroupWaitTicket> ticket) {
+	ScopedLock l(ticket->syncher);
+	while (!ticket->done) {
+		ticket->cond.wait(l);
+	}
+}
+
+
+/****************************
+ *
+ * Public methods
+ *
+ ****************************/
+
+
+GroupPtr
+Pool::findOrCreateGroup(const Options &options) {
+	Options options2 = options;
+	options2.noop = true;
+
+	Ticket ticket;
+	{
+		LockGuard l(syncher);
+		GroupPtr *group;
+		if (!groups.lookup(options.getAppGroupName(), &group)) {
+			// Forcefully create Group, don't care whether resource limits
+			// actually allow it.
+			createGroup(options);
+		}
+	}
+	return get(options2, &ticket)->getGroup()->shared_from_this();
+}
+
+GroupPtr
+Pool::findGroupByApiKey(const StaticString &value, bool lock) const {
+	DynamicScopedLock l(syncher, lock);
+	GroupMap::ConstIterator g_it(groups);
+	while (*g_it != NULL) {
+		const GroupPtr &group = g_it.getValue();
+		if (group->getApiKey() == value) {
+			return group;
+		}
+		g_it.next();
+	}
+	return GroupPtr();
+}
+
+bool
+Pool::detachGroupByName(const HashedStaticString &name) {
+	TRACE_POINT();
+	ScopedLock l(syncher);
+	GroupPtr group = groups.lookupCopy(name);
+
+	if (OXT_LIKELY(group != NULL)) {
+		P_ASSERT_EQ(group->getName(), name);
+		UPDATE_TRACE_POINT();
+		verifyInvariants();
+		verifyExpensiveInvariants();
+
+		boost::container::vector<Callback> actions;
+		boost::shared_ptr<DetachGroupWaitTicket> ticket =
+			boost::make_shared<DetachGroupWaitTicket>();
+		ExceptionPtr exception = copyException(
+			GetAbortedException("The containing Group was detached."));
+
+		assignExceptionToGetWaiters(group->getWaitlist,
+			exception, actions);
+		forceDetachGroup(group,
+			boost::bind(syncDetachGroupCallback, ticket),
+			actions);
+		possiblySpawnMoreProcessesForExistingGroups();
+
+		verifyInvariants();
+		verifyExpensiveInvariants();
+
+		l.unlock();
+		UPDATE_TRACE_POINT();
+		runAllActions(actions);
+		actions.clear();
+
+		UPDATE_TRACE_POINT();
+		ScopedLock l2(ticket->syncher);
+		while (!ticket->done) {
+			ticket->cond.wait(l2);
+		}
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool
+Pool::detachGroupByApiKey(const StaticString &value) {
+	ScopedLock l(syncher);
+	GroupPtr group = findGroupByApiKey(value, false);
+	if (group != NULL) {
+		string name = group->getName();
+		group.reset();
+		l.unlock();
+		return detachGroupByName(name);
+	} else {
+		return false;
+	}
+}
+
+bool
+Pool::restartGroupByName(const StaticString &name, const RestartOptions &options) {
+	ScopedLock l(syncher);
+	GroupMap::ConstIterator g_it(groups);
+	while (*g_it != NULL) {
+		const GroupPtr &group = g_it.getValue();
+		if (name == group->getName()) {
+			if (!group->authorizeByUid(options.uid)
+			 && !group->authorizeByApiKey(options.apiKey))
+			{
+				throw SecurityException("Operation unauthorized");
+			}
+			if (!group->restarting()) {
+				group->restart(group->options, options.method);
+			}
+			return true;
+		}
+		g_it.next();
+	}
+
+	return false;
+}
+
+unsigned int
+Pool::restartGroupsByAppRoot(const StaticString &appRoot, const RestartOptions &options) {
+	ScopedLock l(syncher);
+	GroupMap::ConstIterator g_it(groups);
+	unsigned int result = 0;
+
+	while (*g_it != NULL) {
+		const GroupPtr &group = g_it.getValue();
+		if (appRoot == group->options.appRoot) {
+			if (group->authorizeByUid(options.uid)
+			 || group->authorizeByApiKey(options.apiKey))
+			{
+				result++;
+				group->restart(group->options, options.method);
+			}
+		}
+		g_it.next();
+	}
+
+	return result;
+}
+
+
+} // namespace ApplicationPool2
+} // namespace Passenger

data/ext/common/ApplicationPool2/Pool/InitializationAndShutdown.cpp

@@ -0,0 +1,145 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2011-2015 Phusion
+ *
+ *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#include <ApplicationPool2/Pool.h>
+
+/*************************************************************************
+ *
+ * Initialization and shutdown-related code for ApplicationPool2::Pool
+ *
+ *************************************************************************/
+
+namespace Passenger {
+namespace ApplicationPool2 {
+
+using namespace std;
+using namespace boost;
+
+
+Pool::Pool(const SpawningKit::FactoryPtr &spawningKitFactory,
+	const VariantMap *agentsOptions)
+	: abortLongRunningConnectionsCallback(NULL)
+{
+	context.setSpawningKitFactory(spawningKitFactory);
+	context.finalize();
+
+	this->agentsOptions = agentsOptions;
+
+	try {
+		systemMetricsCollector.collect(systemMetrics);
+	} catch (const RuntimeException &e) {
+		P_WARN("Unable to collect system metrics: " << e.what());
+	}
+
+	lifeStatus   = ALIVE;
+	max          = 6;
+	maxIdleTime  = 60 * 1000000;
+	selfchecking = true;
+	palloc       = psg_create_pool(PSG_DEFAULT_POOL_SIZE);
+
+	// The following code only serve to instantiate certain inline methods
+	// so that they can be invoked from gdb.
+	(void) GroupPtr().get();
+	(void) ProcessPtr().get();
+	(void) SessionPtr().get();
+}
+
+Pool::~Pool() {
+	if (lifeStatus != SHUT_DOWN) {
+		P_BUG("You must call Pool::destroy() before actually destroying the Pool object!");
+	}
+	psg_destroy_pool(palloc);
+}
+
+/** Must be called right after construction. */
+void
+Pool::initialize() {
+	LockGuard l(syncher);
+	initializeAnalyticsCollection();
+	initializeGarbageCollection();
+}
+
+void
+Pool::initDebugging() {
+	LockGuard l(syncher);
+	debugSupport = boost::make_shared<DebugSupport>();
+}
+
+/**
+ * Should be called right after the agent has received
+ * the message to exit gracefully. This will tell processes to
+ * abort any long-running connections, e.g. WebSocket connections,
+ * because the RequestHandler has to wait until all connections are
+ * finished before proceeding with shutdown.
+ */
+void
+Pool::prepareForShutdown() {
+	TRACE_POINT();
+	ScopedLock lock(syncher);
+	assert(lifeStatus == ALIVE);
+	lifeStatus = PREPARED_FOR_SHUTDOWN;
+	if (abortLongRunningConnectionsCallback != NULL) {
+		vector<ProcessPtr> processes = getProcesses(false);
+		foreach (ProcessPtr process, processes) {
+			// Ensure that the process is not immediately respawned.
+			process->getGroup()->options.minProcesses = 0;
+			abortLongRunningConnectionsCallback(process);
+		}
+	}
+}
+
+/** Must be called right before destruction. */
+void
+Pool::destroy() {
+	TRACE_POINT();
+	ScopedLock lock(syncher);
+	assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
+
+	lifeStatus = SHUTTING_DOWN;
+
+	while (!groups.empty()) {
+		GroupPtr *group;
+		groups.lookupRandom(NULL, &group);
+		string name = group->get()->getName().toString();
+		lock.unlock();
+		detachGroupByName(name);
+		lock.lock();
+	}
+
+	UPDATE_TRACE_POINT();
+	lock.unlock();
+	interruptableThreads.interrupt_and_join_all();
+	nonInterruptableThreads.join_all();
+	lock.lock();
+
+	lifeStatus = SHUT_DOWN;
+
+	UPDATE_TRACE_POINT();
+	verifyInvariants();
+	verifyExpensiveInvariants();
+}
+
+
+} // namespace ApplicationPool2
+} // namespace Passenger

data/ext/common/ApplicationPool2/Pool/Miscellaneous.cpp

@@ -0,0 +1,244 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2011-2015 Phusion
+ *
+ *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#include <ApplicationPool2/Pool.h>
+
+namespace Passenger {
+namespace ApplicationPool2 {
+
+using namespace std;
+using namespace boost;
+
+
+// 'lockNow == false' may only be used during unit tests. Normally we
+// should never call the callback while holding the lock.
+void
+Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow) {
+	DynamicScopedLock lock(syncher, lockNow);
+
+	assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
+	verifyInvariants();
+	P_TRACE(2, "asyncGet(appGroupName=" << options.getAppGroupName() << ")");
+	boost::container::vector<Callback> actions;
+
+	Group *existingGroup = findMatchingGroup(options);
+	if (OXT_LIKELY(existingGroup != NULL)) {
+		/* Best case: the app group is already in the pool. Let's use it. */
+		P_TRACE(2, "Found existing Group");
+		existingGroup->verifyInvariants();
+		SessionPtr session = existingGroup->get(options, callback, actions);
+		existingGroup->verifyInvariants();
+		verifyInvariants();
+		P_TRACE(2, "asyncGet() finished");
+		if (lockNow) {
+			lock.unlock();
+		}
+		if (session != NULL) {
+			callback(session, ExceptionPtr());
+		}
+
+	} else if (!atFullCapacityUnlocked()) {
+		/* The app super group isn't in the pool and we have enough free
+		 * resources to make a new one.
+		 */
+		P_DEBUG("Spawning new Group");
+		GroupPtr group = createGroupAndAsyncGetFromIt(options,
+			callback, actions);
+		group->verifyInvariants();
+		verifyInvariants();
+		P_DEBUG("asyncGet() finished");
+
+	} else {
+		/* Uh oh, the app super group isn't in the pool but we don't
+		 * have the resources to make a new one. The sysadmin should
+		 * configure the system to let something like this happen
+		 * as least as possible, but let's try to handle it as well
+		 * as we can.
+		 */
+		ProcessPtr freedProcess = forceFreeCapacity(NULL, actions);
+		if (freedProcess == NULL) {
+			/* No process is eligible for killing. This could happen if, for example,
+			 * all (super)groups are currently initializing/restarting/spawning/etc.
+			 * We have no choice but to satisfy this get() action later when resources
+			 * become available.
+			 */
+			P_DEBUG("Could not free a process; putting request to top-level getWaitlist");
+			getWaitlist.push_back(GetWaiter(
+				options.copyAndPersist().detachFromUnionStationTransaction(),
+				callback));
+		} else {
+			/* Now that a process has been trashed we can create
+			 * the missing Group.
+			 */
+			P_DEBUG("Creating new Group");
+			GroupPtr group = createGroup(options);
+			SessionPtr session = group->get(options, callback,
+				actions);
+			/* The Group is now spawning a process so the callback
+			 * should now have been put on the wait list,
+			 * unless something has changed and we forgot to update
+			 * some code here or if options.noop...
+			 */
+			if (session != NULL) {
+				assert(options.noop);
+				actions.push_back(boost::bind(GetCallback::call,
+					callback, session, ExceptionPtr()));
+			}
+			freedProcess->getGroup()->verifyInvariants();
+			group->verifyInvariants();
+		}
+
+		assert(atFullCapacityUnlocked());
+		verifyInvariants();
+		verifyExpensiveInvariants();
+		P_TRACE(2, "asyncGet() finished");
+	}
+
+	if (!actions.empty()) {
+		if (lockNow) {
+			if (lock.owns_lock()) {
+				lock.unlock();
+			}
+			runAllActions(actions);
+		} else {
+			// This state is not allowed. If we reach
+			// here then it probably indicates a bug in
+			// the test suite.
+			abort();
+		}
+	}
+}
+
+// TODO: 'ticket' should be a boost::shared_ptr for interruption-safety.
+SessionPtr
+Pool::get(const Options &options, Ticket *ticket) {
+	ticket->session.reset();
+	ticket->exception.reset();
+
+	GetCallback callback;
+	callback.func = syncGetCallback;
+	callback.userData = ticket;
+	asyncGet(options, callback);
+
+	ScopedLock lock(ticket->syncher);
+	while (ticket->session == NULL && ticket->exception == NULL) {
+		ticket->cond.wait(lock);
+	}
+	lock.unlock();
+
+	if (OXT_LIKELY(ticket->session != NULL)) {
+		SessionPtr session = ticket->session;
+		ticket->session.reset();
+		return session;
+	} else {
+		rethrowException(ticket->exception);
+		return SessionPtr(); // Shut up compiler warning.
+	}
+}
+
+void
+Pool::setMax(unsigned int max) {
+	ScopedLock l(syncher);
+	assert(max > 0);
+	fullVerifyInvariants();
+	bool bigger = max > this->max;
+	this->max = max;
+	if (bigger) {
+		/* If there are clients waiting for resources
+		 * to become free, spawn more processes now that
+		 * we have the capacity.
+		 *
+		 * We favor waiters on the pool over waiters on the
+		 * the groups because the latter already have the
+		 * resources to eventually complete. Favoring waiters
+		 * on the pool should be fairer.
+		 */
+		boost::container::vector<Callback> actions;
+		assignSessionsToGetWaiters(actions);
+		possiblySpawnMoreProcessesForExistingGroups();
+
+		fullVerifyInvariants();
+		l.unlock();
+		runAllActions(actions);
+	} else {
+		fullVerifyInvariants();
+	}
+}
+
+void
+Pool::setMaxIdleTime(unsigned long long value) {
+	LockGuard l(syncher);
+	maxIdleTime = value;
+	wakeupGarbageCollector();
+}
+
+void
+Pool::enableSelfChecking(bool enabled) {
+	LockGuard l(syncher);
+	selfchecking = enabled;
+}
+
+/**
+ * Checks whether at least one process is being spawned.
+ */
+bool
+Pool::isSpawning(bool lock) const {
+	DynamicScopedLock l(syncher, lock);
+	GroupMap::ConstIterator g_it(groups);
+	while (*g_it != NULL) {
+		const GroupPtr &group = g_it.getValue();
+		if (group->spawning()) {
+			return true;
+		}
+		g_it.next();
+	}
+	return false;
+}
+
+bool
+Pool::authorizeByApiKey(const ApiKey &key, bool lock) const {
+	return key.isSuper() || findGroupByApiKey(key.toStaticString(), lock) != NULL;
+}
+
+bool
+Pool::authorizeByUid(uid_t uid, bool lock) const {
+	if (uid == 0 || uid == geteuid()) {
+		return true;
+	}
+
+	DynamicScopedLock l(syncher, lock);
+	GroupMap::ConstIterator g_it(groups);
+	while (*g_it != NULL) {
+		const GroupPtr &group = g_it.getValue();
+		if (group->authorizeByUid(uid)) {
+			return true;
+		}
+		g_it.next();
+	}
+	return false;
+}
+
+
+} // namespace ApplicationPool2
+} // namespace Passenger