passenger 4.0.27 → 4.0.28

data/ext/common/ApplicationPool2/Implementation.cpp

@@ -24,6 +24,7 @@
  */
 #include <typeinfo>
 #include <algorithm>
+#include <utility>
 #include <boost/make_shared.hpp>
 #include <boost/date_time/posix_time/posix_time_types.hpp>
 #include <oxt/backtrace.hpp>
@@ -156,6 +157,23 @@ SuperGroup::generateSecret() const {
 	return getPool()->randomGenerator->generateAsciiString(43);
 }
 
+void
+SuperGroup::runInitializationHooks() const {
+	getPool()->runHookScripts("after_initialize_supergroup",
+		boost::bind(&SuperGroup::setupInitializationOrDestructionHook, this, _1));
+}
+
+void
+SuperGroup::runDestructionHooks() const {
+	getPool()->runHookScripts("before_destroy_supergroup",
+		boost::bind(&SuperGroup::setupInitializationOrDestructionHook, this, _1));
+}
+
+void
+SuperGroup::setupInitializationOrDestructionHook(HookScriptOptions &options) const {
+	options.environment.push_back(make_pair("PASSENGER_APP_ROOT", this->options.appRoot));
+}
+
 void
 SuperGroup::createInterruptableThread(const boost::function<void ()> &func, const string &name,
 	unsigned int stackSize)
@@ -216,7 +234,7 @@ SuperGroup::realDoInitialize(const Options &options, unsigned int generation) {
 				const GetWaiter &waiter = getWaitlist.front();
 				actions.push_back(boost::bind(waiter.callback,
 					SessionPtr(), exception));
-				getWaitlist.pop();
+				getWaitlist.pop_front();
 			}
 		} else {
 			for (it = componentInfos.begin(); it != componentInfos.end(); it++) {
@@ -236,9 +254,11 @@ SuperGroup::realDoInitialize(const Options &options, unsigned int generation) {
 		verifyInvariants();
 		P_TRACE(2, "Done initializing SuperGroup " << inspect());
 	}
+
 	this_thread::disable_interruption di;
 	this_thread::disable_syscall_interruption dsi;
 	runAllActions(actions);
+	runInitializationHooks();
 }
 
 void
@@ -861,7 +881,8 @@ Group::spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options, un
 		m_spawning = false;
 		
 		done = done
-			|| ((unsigned long) getProcessCount() >= options.minProcesses && getWaitlist.empty())
+			|| (getProcessCount() >= options.minProcesses && getWaitlist.empty())
+			|| (options.maxProcesses != 0 && getProcessCount() >= options.maxProcesses)
 			|| pool->atFullCapacity(false);
 		m_spawning = !done;
 		if (done) {
@@ -899,7 +920,9 @@ Group::shouldSpawnForGetAction() const {
 
 bool
 Group::allowSpawn() const {
-	return isAlive() && !poolAtFullCapacity();
+	return isAlive()
+		&& !poolAtFullCapacity()
+		&& (options.maxProcesses == 0 || getProcessCount() < options.maxProcesses);
 }
 
 void
@@ -1154,6 +1177,26 @@ Group::getResourceLocator() const {
 	return getPool()->spawnerFactory->getResourceLocator();
 }
 
+// 'process' is not a reference so that bind(runAttachHooks, ...) causes the shared
+// pointer reference to increment.
+void
+Group::runAttachHooks(const ProcessPtr process) const {
+	getPool()->runHookScripts("attached_process",
+		boost::bind(&Group::setupAttachOrDetachHook, this, process, _1));
+}
+
+void
+Group::runDetachHooks(const ProcessPtr process) const {
+	getPool()->runHookScripts("detached_process",
+		boost::bind(&Group::setupAttachOrDetachHook, this, process, _1));
+}
+
+void
+Group::setupAttachOrDetachHook(const ProcessPtr process, HookScriptOptions &options) const {
+	options.environment.push_back(make_pair("PASSENGER_PROCESS_PID", toString(process->pid)));
+	options.environment.push_back(make_pair("PASSENGER_APP_ROOT", this->options.appRoot));
+}
+
 string
 Group::generateSecret(const SuperGroupPtr &superGroup) {
 	return superGroup->getPool()->randomGenerator->generateAsciiString(43);
@@ -1196,6 +1239,11 @@ Session::getGupid() const {
 	return getProcess()->gupid;
 }
 
+int
+Session::getStickySessionId() const {
+	return getProcess()->stickySessionId;
+}
+
 const GroupPtr
 Session::getGroup() const {
 	return getProcess()->getGroup();
@@ -1216,11 +1264,10 @@ Session::kill(int signo) {
 
 PipeWatcher::DataCallback PipeWatcher::onData;
 
-PipeWatcher::PipeWatcher(const FileDescriptor &_fd, const char *_name, pid_t _pid, bool _print)
+PipeWatcher::PipeWatcher(const FileDescriptor &_fd, const char *_name, pid_t _pid)
 	: fd(_fd),
 	  name(_name),
-	  pid(_pid),
-	  print(_print)
+	  pid(_pid)
 {
 	started = false;
 }
@@ -1276,8 +1323,7 @@ PipeWatcher::threadMain() {
 			}
 		} else if (ret == 1 && buf[0] == '\n') {
 			UPDATE_TRACE_POINT();
-			P_LOG(print ? LVL_INFO : LVL_DEBUG,
-				"[App " << pid << " " << name << "] ");
+			printAppOutput(pid, name, "", 0);
 		} else {
 			UPDATE_TRACE_POINT();
 			vector<StaticString> lines;
@@ -1287,8 +1333,7 @@ PipeWatcher::threadMain() {
 			}
 			split(StaticString(buf, ret2), '\n', lines);
 			foreach (const StaticString line, lines) {
-				P_LOG(print ? LVL_INFO : LVL_DEBUG,
-					"[App " << pid << " " << name << "] " << line);
+				printAppOutput(pid, name, line.data(), line.size());
 			}
 		}
 

data/ext/common/ApplicationPool2/Options.h

@@ -146,7 +146,10 @@ private:
 	}
 	
 public:
-	/*********** Spawn options that should be set manually ***********/
+	/*********** Spawn options that should be set manually ***********
+	 * These are the options that are relevant while spawning an application
+	 * process. These options are only used during spawning.
+	 */
 	
 	/**
 	 * The root directory of the application to spawn. In case of a Ruby on Rails
@@ -294,13 +297,25 @@ public:
 	bool raiseInternalError;
 	
 	
-	/*********** Per-group pool options that should be set manually ***********/
+	/*********** Per-group pool options that should be set manually ***********
+	 * These options dictate how Pool will manage processes, routing, etc. within
+	 * a single Group. These options are not process-specific, only group-specific.
+	 */
 	
 	/**
 	 * The minimum number of processes for the current group that the application
 	 * pool's cleaner thread should keep around.
 	 */
-	unsigned long minProcesses;
+	unsigned int minProcesses;
+
+	/**
+	 * The maximum number of processes that may be spawned
+	 * for this app root. This option only has effect if it's lower than
+	 * the pool size.
+	 *
+	 * A value of 0 means unspecified, and has no effect.
+	 */
+	unsigned int maxProcesses;
 	
 	/** The number of seconds that preloader processes may stay alive idling. */
 	long maxPreloaderIdleTime;
@@ -317,10 +332,23 @@ public:
 	 */
 	unsigned int maxRequestQueueSize;
 
+	/**
+	 * The Union Station key to use in case analytics logging is enabled.
+	 * It is used by Pool::collectAnalytics() and other administrative
+	 * functions which are called periodically. Because they do not belong
+	 * to any request, and they may still want to log to Union Station,
+	 * this key is stored in the per-group options structure.
+	 *
+	 * It is not used on a per-request basis. Per-request analytics logging
+	 * (and Union Station logging) uses the logger object in the `logger` field
+	 * instead.
+	 */
+	StaticString unionStationKey;
+
 	/*-----------------*/
 	
 	
-	/*********** Per-request options that should be set manually and that only matter to Pool ***********/
+	/*********** Per-request options that should be set manually ***********/
 	
 	/** Current request host name. */
 	StaticString hostName;
@@ -335,9 +363,9 @@ public:
 	UnionStation::LoggerPtr logger;
 
 	/**
-	 * The Union Station key to use in case analytics logging is enabled.
+	 * A sticky session ID for routing to a specific process.
 	 */
-	StaticString unionStationKey;
+	unsigned int stickySessionId;
 	
 	/**
 	 * A throttling rate for file stats. When set to a non-zero value N,
@@ -372,7 +400,10 @@ public:
 	/*-----------------*/
 	
 	
-	/*********** Spawn options automatically set by Pool ***********/
+	/*********** Spawn options automatically set by Pool ***********
+	 * These options are passed to the Spawner. The Pool::get() caller may not
+	 * see these values.
+	 */
 	
 	/** The secret key of the pool group that the spawned process is to belong to. */
 	StaticString groupSecret;
@@ -401,10 +432,12 @@ public:
 		raiseInternalError      = false;
 		
 		minProcesses            = 1;
+		maxProcesses            = 0;
 		maxPreloaderIdleTime    = -1;
 		maxOutOfBandWorkInstances = 1;
 		maxRequestQueueSize     = 100;
 		
+		stickySessionId         = 0;
 		statThrottleRate        = 0;
 		maxRequests             = 0;
 		noop                    = false;
@@ -494,8 +527,9 @@ public:
 	}
 	
 	Options &clearPerRequestFields() {
-		hostName = string();
-		uri      = string();
+		hostName = StaticString();
+		uri      = StaticString();
+		stickySessionId = 0;
 		noop     = false;
 		return clearLogger();
 	}
@@ -546,14 +580,15 @@ public:
 			appendKeyValue (vec, "logging_agent_password", loggingAgentPassword);
 			appendKeyValue4(vec, "debugger",           debugger);
 			appendKeyValue4(vec, "analytics",          analytics);
-			appendKeyValue (vec, "union_station_key",  unionStationKey);
 
 			appendKeyValue (vec, "group_secret",       groupSecret);
 		}
 		if (fields & PER_GROUP_POOL_OPTIONS) {
 			appendKeyValue3(vec, "min_processes",       minProcesses);
+			appendKeyValue3(vec, "max_processes",       maxProcesses);
 			appendKeyValue2(vec, "max_preloader_idle_time", maxPreloaderIdleTime);
 			appendKeyValue3(vec, "max_out_of_band_work_instances", maxOutOfBandWorkInstances);
+			appendKeyValue (vec, "union_station_key",   unionStationKey);
 		}
 		
 		/*********************************/

data/ext/common/ApplicationPool2/PipeWatcher.h

@@ -47,12 +47,11 @@ struct PipeWatcher: public boost::enable_shared_from_this<PipeWatcher> {
 	FileDescriptor fd;
 	const char *name;
 	pid_t pid;
-	bool print;
 	bool started;
 	boost::mutex startSyncher;
 	boost::condition_variable startCond;
 
-	PipeWatcher(const FileDescriptor &_fd, const char *name, pid_t pid, bool _print);
+	PipeWatcher(const FileDescriptor &_fd, const char *name, pid_t pid);
 	void initialize();
 	void start();
 	static void threadMain(boost::shared_ptr<PipeWatcher> self);

data/ext/common/ApplicationPool2/Pool.h

@@ -48,9 +48,9 @@
 #include <ApplicationPool2/Options.h>
 #include <UnionStation.h>
 #include <Logging.h>
-#include <SafeLibev.h>
 #include <Exceptions.h>
 #include <RandomGenerator.h>
+#include <Hooks.h>
 #include <Utils/Lock.h>
 #include <Utils/AnsiColorConstants.h>
 #include <Utils/SystemTime.h>
@@ -129,7 +129,6 @@ public:
 	RandomGeneratorPtr randomGenerator;
 
 	mutable boost::mutex syncher;
-	SafeLibev *libev;
 	unsigned int max;
 	unsigned long long maxIdleTime;
 	
@@ -193,6 +192,7 @@ public:
 	 */
 	vector<GetWaiter> getWaitlist;
 
+	const VariantMap *agentsOptions;
 	DebugSupportPtr debugSupport;
 	
 	static void runAllActions(const vector<Callback> &actions) {
@@ -247,6 +247,27 @@ public:
 		}
 	}
 	
+	bool runHookScripts(const char *name,
+		const boost::function<void (HookScriptOptions &)> &setup) const
+	{
+		if (agentsOptions != NULL) {
+			string hookName = string("hook_") + name;
+			string spec = agentsOptions->get(hookName, false);
+			if (!spec.empty()) {
+				HookScriptOptions options;
+				options.agentsOptions = agentsOptions;
+				options.name = name;
+				options.spec = spec;
+				setup(options);
+				return Passenger::runHookScripts(options);
+			} else {
+				return true;
+			}
+		} else {
+			return true;
+		}
+	}
+
 	ProcessPtr findOldestIdleProcess() const {
 		ProcessPtr oldestIdleProcess;
 		
@@ -343,7 +364,7 @@ public:
 			postLockActions.push_back(boost::bind(
 				getWaitlist.front().callback, SessionPtr(),
 				exception));
-			getWaitlist.pop();
+			getWaitlist.pop_front();
 		}
 	}
 	
@@ -385,7 +406,7 @@ public:
 		getWaitlist.reserve(getWaitlist.size() + superGroup->getWaitlist.size());
 		while (!superGroup->getWaitlist.empty()) {
 			getWaitlist.push_back(superGroup->getWaitlist.front());
-			superGroup->getWaitlist.pop();
+			superGroup->getWaitlist.pop_front();
 		}
 	}
 	
@@ -861,11 +882,11 @@ public:
 	const SuperGroupPtr getSuperGroup(const char *name);
 	
 public:
-	Pool(SafeLibev *libev, const SpawnerFactoryPtr &spawnerFactory,
+	Pool(const SpawnerFactoryPtr &spawnerFactory,
 		const LoggerFactoryPtr &loggerFactory = LoggerFactoryPtr(),
-		const RandomGeneratorPtr &randomGenerator = RandomGeneratorPtr())
+		const RandomGeneratorPtr &randomGenerator = RandomGeneratorPtr(),
+		const VariantMap *agentsOptions = NULL)
 	{
-		this->libev = libev;
 		this->spawnerFactory = spawnerFactory;
 		this->loggerFactory = loggerFactory;
 		if (randomGenerator != NULL) {
@@ -873,6 +894,7 @@ public:
 		} else {
 			this->randomGenerator = boost::make_shared<RandomGenerator>();
 		}
+		this->agentsOptions = agentsOptions;
 		
 		lifeStatus  = ALIVE;
 		max         = 6;

data/ext/common/ApplicationPool2/Process.h

@@ -209,7 +209,11 @@ public:
 	SafeLibev * const libev;
 	/** Process PID. */
 	pid_t pid;
-	/** UUID for this process, randomly generated and will never appear again. */
+	/** An ID that uniquely identifies this Process in the Group, for
+	 * use in implementing sticky sessions. Set by Group::attach(). */
+	unsigned int stickySessionId;
+	/** UUID for this process, randomly generated and extremely unlikely to ever
+	 * appear again in this universe. */
 	string gupid;
 	string connectPassword;
 	/** Admin socket, see class description. */
@@ -326,6 +330,7 @@ public:
 		: pqHandle(NULL),
 		  libev(_libev.get()),
 		  pid(_pid),
+		  stickySessionId(0),
 		  gupid(_gupid),
 		  connectPassword(_connectPassword),
 		  adminSocket(_adminSocket),
@@ -352,13 +357,13 @@ public:
 
 		if (_adminSocket != -1) {
 			PipeWatcherPtr watcher = boost::make_shared<PipeWatcher>(_adminSocket,
-				"stdout", pid, config->forwardStdout);
+				"stdout", pid);
 			watcher->initialize();
 			watcher->start();
 		}
 		if (_errorPipe != -1) {
 			PipeWatcherPtr watcher = boost::make_shared<PipeWatcher>(_errorPipe,
-				"stderr", pid, config->forwardStderr);
+				"stderr", pid);
 			watcher->initialize();
 			watcher->start();
 		}
@@ -537,9 +542,22 @@ public:
 		return atFullUtilization();
 	}
 
+	/**
+	 * Whether we've reached the maximum number of concurrent sessions for this
+	 * process.
+	 */
 	bool atFullUtilization() const {
 		return concurrency != 0 && sessions >= concurrency;
 	}
+
+	/**
+	 * Whether a get() request can be routed to this process, assuming that
+	 * the sticky session ID (if any) matches. This is only not the case
+	 * if this process is at full utilization.
+	 */
+	bool canBeRoutedTo() const {
+		return !atFullUtilization();
+	}
 	
 	/**
 	 * Create a new communication session with this process. This will connect to one
@@ -588,6 +606,7 @@ public:
 	template<typename Stream>
 	void inspectXml(Stream &stream, bool includeSockets = true) const {
 		stream << "<pid>" << pid << "</pid>";
+		stream << "<sticky_session_id>" << stickySessionId << "</sticky_session_id>";
 		stream << "<gupid>" << gupid << "</gupid>";
 		stream << "<connect_password>" << connectPassword << "</connect_password>";
 		stream << "<concurrency>" << concurrency << "</concurrency>";

data/ext/common/ApplicationPool2/Session.h

@@ -108,6 +108,7 @@ public:
 	const string &getConnectPassword() const;
 	pid_t getPid() const;
 	const string &getGupid() const;
+	int getStickySessionId() const;
 	const GroupPtr getGroup() const;
 	void requestOOBW();
 	int kill(int signo);