passenger 4.0.45 → 4.0.46

data/ext/common/Constants.h

@@ -90,7 +90,7 @@
 
 	#define NGINX_DOC_URL "https://www.phusionpassenger.com/documentation/Users%20guide%20Nginx.html"
 
-	#define PASSENGER_VERSION "4.0.45"
+	#define PASSENGER_VERSION "4.0.46"
 
 	#define POOL_HELPER_THREAD_STACK_SIZE 262144
 

data/ext/common/Exceptions.h

@@ -255,17 +255,26 @@ class SpawnException: public oxt::tracable_exception {
 public:
 	enum ErrorKind {
 		UNDEFINED_ERROR,
-		/** The preloader failed to start, not due to a wrong protocol message. */
+		/** The preloader failed to start, not due to a wrong protocol message.
+		 * It did not explicitly supply an error message.
+		 */
 		PRELOADER_STARTUP_ERROR,
 		/** The preloader sent a wrong protocol message during startup. */
 		PRELOADER_STARTUP_PROTOCOL_ERROR,
+		/** The preloader timed out during startup. */
 		PRELOADER_STARTUP_TIMEOUT,
+		/** The preloader failed to start, not due to a wrong protocol message.
+		 * It DID explicitly supply an error message. */
 		PRELOADER_STARTUP_EXPLAINABLE_ERROR,
-		/** The application failed to start, not due to a wrong protocol message. */
+		/** The application failed to start, not due to a wrong protocol message.
+		 * It did not explicitly supply an error message. */
 		APP_STARTUP_ERROR,
 		/** The application sent a wrong protocol message during startup. */
 		APP_STARTUP_PROTOCOL_ERROR,
+		/** The application timed out during startup. */
 		APP_STARTUP_TIMEOUT,
+		/** The application failed to start, not due to a wrong protocol message.
+		 * It DID explicitly supply an error message. */
 		APP_STARTUP_EXPLAINABLE_ERROR
 	};
 	
@@ -342,6 +351,10 @@ public:
 		return get(name);
 	}
 
+	void set(const string &name, const string &value) {
+		annotations[name] = value;
+	}
+
 	string get(const string &name) const {
 		map<string, string>::const_iterator it = annotations.find(name);
 		if (it == annotations.end()) {

data/ext/common/UnionStation/Core.h

@@ -381,6 +381,9 @@ public:
 		// Get a connection to the logging server.
 		ConnectionPtr connection = checkoutConnection();
 		if (connection == NULL) {
+			P_TRACE(2, "Created NULL Union Station transaction: group=" << groupName <<
+				", category=" << category << ", txnId=" <<
+				StaticString(txnId, txnIdEnd - txnId));
 			return createNullTransaction();
 		}
 
@@ -395,8 +398,14 @@ public:
 				category,
 				unionStationKey);
 			guard.clear();
+			P_TRACE(2, "Created new Union Station transaction: group=" << groupName <<
+				", category=" << category << ", txnId=" <<
+				StaticString(txnId, txnIdEnd - txnId));
 			return transaction;
 		} else {
+			P_TRACE(2, "Created NULL Union Station transaction: group=" << groupName <<
+				", category=" << category << ", txnId=" <<
+				StaticString(txnId, txnIdEnd - txnId));
 			return createNullTransaction();
 		}
 	}

data/ext/common/Utils/JsonUtils.h

@@ -0,0 +1,53 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2014 Phusion
+ *
+ *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#ifndef _PASSENGER_UTILS_JSON_UTILS_H_
+#define _PASSENGER_UTILS_JSON_UTILS_H_
+
+#include <string>
+#include <StaticString.h>
+#include <Utils/json.h>
+#include <Utils/StrIntUtils.h>
+
+namespace Passenger {
+
+using namespace std;
+
+inline string
+stringifyJson(const Json::Value &value) {
+	Json::FastWriter writer;
+	string str = writer.write(value);
+	str.erase(str.size() - 1, 1);
+	return str;
+}
+
+/** `str` MUST be NULL-terminated! */
+inline string
+jsonString(const Passenger::StaticString &str) {
+	return stringifyJson(Json::Value(Json::StaticString(str.data())));
+}
+
+} // namespace Passenger
+
+#endif /* _PASSENGER_UTILS_JSON_UTILS_H_ */

data/ext/common/Utils/ProcessMetricsCollector.h

@@ -92,7 +92,7 @@ struct ProcessMetrics {
 	 */
 	ssize_t  swap;
 	/** OS X Snow Leopard does not report the VM size correctly, so don't use this. */
-	size_t  vmsize;
+	ssize_t  vmsize;
 	pid_t   processGroupId;
 	uid_t   uid;
 	string  command;

data/ext/common/Utils/SpeedMeter.h

@@ -191,7 +191,7 @@ public:
 		return count;
 	}
 
-	/** Current speed over the configured window. Returns NaN if less than 2
+	/** Current speed over the configured window. Returns unknownSpeed() if less than 2
 	 * samples have been collected so far.
 	 */
 	double currentSpeed() const {
@@ -221,16 +221,20 @@ public:
 		}
 		avgWeight /= std::max(1, (int) count - 1 - begin);
 
-		interval = getSample(count - 1).timestamp - getSample(begin).timestamp;
+		interval = getSample((int) count - 1).timestamp - getSample(begin).timestamp;
 		if (interval > 0) {
 			// sum / interval is the speed per average delta interval,
 			// so we extrapolate that over the entire window interval.
 			return (sum / interval) * (window / avgWeight);
 		} else {
-			return numeric_limits<double>::quiet_NaN();
+			return unknownSpeed();
 		}
 	}
 
+	static ValueType unknownSpeed() {
+		return numeric_limits<ValueType>::max();
+	}
+
 	#if 1
 		void debug() const {
 			const unsigned long long timeThreshold = getTimeThreshold();

data/ext/common/Utils/SystemMetricsCollector.h

@@ -45,7 +45,6 @@
 	#include <sys/sysinfo.h>
 	#include <Exceptions.h>
 	#include <Utils/StringScanning.h>
-	#include <Utils/SpeedMeter.h>
 	#include <Utils/IOUtils.h>
 #endif
 #ifdef __APPLE__
@@ -69,6 +68,7 @@
 #include <Utils/StrIntUtils.h>
 #include <Utils/SystemTime.h>
 #include <Utils/AnsiColorConstants.h>
+#include <Utils/SpeedMeter.h>
 
 /*
  * Useful resources
@@ -346,13 +346,15 @@ public:
 	time_t boottime;
 
 	/** Speed at which processes are created per second.
-	 * NaN if it's not yet known (because too few samples have been taken so far).
+	 * SpeedMeter<unsigned long long>::unknownSpeed() if it's not yet known (because too
+	 * few samples have been taken so far).
 	 * -1 if there was an error querying this information.
 	 * -2 if the OS does not support this metric.
 	 */
 	double forkRate;
 	/** Speed at which the OS swaps in and swaps out data, in KB/sec.
-	 * NaN if it's not yet known (because too few samples have been taken so far).
+	 * SpeedMeter<size_t>::unknownSpeed() if it's not yet known
+	 * (because too few samples have been taken so far).
 	 * -1 if there was an error querying this information.
 	 * -2 if the OS does not support this metric.
 	 */
@@ -537,7 +539,7 @@ public:
 
 			if (forkRate != -2) {
 				stream << "Fork rate         : ";
-				if (std::isnan(forkRate) || forkRate < 0) {
+				if (forkRate == SpeedMeter<unsigned long long>::unknownSpeed() || forkRate < 0) {
 					if (options.colors) {
 						stream << ANSI_COLOR_DGRAY;
 					}
@@ -648,7 +650,7 @@ public:
 
 			if (swapInRate != -2) {
 				stream << "Swap in           : ";
-				if (std::isnan(swapInRate) || swapInRate < 0) {
+				if (swapInRate == SpeedMeter<size_t>::unknownSpeed() || swapInRate < 0) {
 					if (options.colors) {
 						stream << ANSI_COLOR_DGRAY;
 					}
@@ -668,7 +670,7 @@ public:
 
 			if (swapOutRate != -2) {
 				stream << "Swap out          : ";
-				if (std::isnan(swapOutRate) || swapOutRate < 0) {
+				if (swapOutRate == SpeedMeter<size_t>::unknownSpeed() || swapOutRate < 0) {
 					if (options.colors) {
 						stream << ANSI_COLOR_DGRAY;
 					}

data/ext/common/agents/HelperAgent/Main.cpp

@@ -462,10 +462,10 @@ public:
 		UPDATE_TRACE_POINT();
 		unionStationCore = boost::make_shared<UnionStation::Core>(options.loggingAgentAddress,
 			"logging", options.loggingAgentPassword);
-		spawnerFactory = boost::make_shared<SpawnerFactory>(resourceLocator,
-			generation, boost::make_shared<SpawnerConfig>(randomGenerator));
-		pool = boost::make_shared<Pool>(spawnerFactory, unionStationCore,
-			randomGenerator, &options);
+		spawnerFactory = boost::make_shared<SpawnerFactory>(generation,
+			boost::make_shared<SpawnerConfig>(resourceLocator, unionStationCore,
+				randomGenerator));
+		pool = boost::make_shared<Pool>(spawnerFactory, &options);
 		pool->initialize();
 		pool->setMax(options.maxPoolSize);
 		pool->setMaxIdleTime(options.poolIdleTime * 1000000);

data/ext/common/agents/HelperAgent/RequestHandler.h

@@ -112,6 +112,7 @@
 #include <boost/shared_ptr.hpp>
 #include <boost/weak_ptr.hpp>
 #include <boost/make_shared.hpp>
+#include <boost/regex.hpp>
 #include <ev++.h>
 
 #if defined(__GLIBCXX__) || defined(__APPLE__)
@@ -135,6 +136,7 @@
 #include <UnionStation/Transaction.h>
 #include <UnionStation/ScopeLog.h>
 #include <ApplicationPool2/Pool.h>
+#include <ApplicationPool2/ErrorRenderer.h>
 #include <Utils/StrIntUtils.h>
 #include <Utils/IOUtils.h>
 #include <Utils/HttpHeaderBufferer.h>
@@ -225,6 +227,8 @@ private:
 		sessionCheckoutTry = 0;
 		responseHeaderSeen = false;
 		chunkedResponse = false;
+		responseContentLength = -1;
+		responseBodyAlreadyRead = 0;
 		appRoot.clear();
 	}
 
@@ -336,6 +340,21 @@ public:
 
 	bool responseHeaderSeen;
 	bool chunkedResponse;
+	/** The size of the response body, set based on the values of
+	 * the Content-Length and Transfer-Encoding response headers.
+	 * Possible values:
+	 *
+	 * -1: infinite. Should keep forwarding response body until end of stream.
+	 *     This is the case for WebSockets or for responses without Content-Length.
+	 *     Responses with "Transfer-Encoding: chunked" also fall under this
+	 *     category, though in this case encountering the zero-length chunk is
+	 *     treated the same as end of stream.
+	 * 0 : no client body. Should immediately close connection after forwarding
+	 *     headers.
+	 * >0: Should forward exactly this many bytes of the response body.
+	 */
+	long long responseContentLength;
+	unsigned long long responseBodyAlreadyRead;
 	HttpHeaderBufferer responseHeaderBufferer;
 	Dechunker responseDechunker;
 
@@ -586,6 +605,8 @@ public:
 			<< indent << "requestIsChunked            = " << boolStr(requestIsChunked) << "\n"
 			<< indent << "requestBodyLength           = " << requestBodyLength << "\n"
 			<< indent << "requestBodyAlreadyRead      = " << requestBodyAlreadyRead << "\n"
+			<< indent << "responseContentLength       = " << responseContentLength << "\n"
+			<< indent << "responseBodyAlreadyRead     = " << responseBodyAlreadyRead << "\n"
 			<< indent << "clientInput                 = " << clientInput.get() <<  " " << clientInput->inspect() << "\n"
 			<< indent << "clientInput started         = " << boolStr(clientInput->isStarted()) << "\n"
 			<< indent << "clientBodyBuffer started    = " << boolStr(clientBodyBuffer->isStarted()) << "\n"
@@ -629,6 +650,7 @@ private:
 	HashMap<int, ClientPtr> clients;
 	Timer inactivityTimer;
 	bool accept4Available;
+	boost::regex upgradeHeaderRegex;
 
 
 	void disconnect(const ClientPtr &client) {
@@ -774,47 +796,12 @@ private:
 		assert(client->state < Client::FORWARDING_BODY_TO_APP);
 		client->state = Client::WRITING_SIMPLE_RESPONSE;
 
-		string templatesDir = resourceLocator.getResourcesDir() + "/templates";
+		ErrorRenderer renderer(resourceLocator);
 		string data;
 
 		if (friendlyErrorPagesEnabled(client)) {
 			try {
-				string cssFile = templatesDir + "/error_layout.css";
-				string errorLayoutFile = templatesDir + "/error_layout.html.template";
-				string generalErrorFile =
-					(e != NULL && e->isHTML())
-					? templatesDir + "/general_error_with_html.html.template"
-					: templatesDir + "/general_error.html.template";
-				string css = readAll(cssFile);
-				StringMap<StaticString> params;
-
-				params.set("CSS", css);
-				params.set("APP_ROOT", client->options.appRoot);
-				params.set("RUBY", client->options.ruby);
-				params.set("ENVIRONMENT", client->options.environment);
-				params.set("MESSAGE", message);
-				params.set("IS_RUBY_APP",
-					(client->options.appType == "classic-rails" || client->options.appType == "rack")
-					? "true" : "false");
-				if (e != NULL) {
-					params.set("TITLE", "Web application could not be started");
-					// Store all SpawnException annotations into 'params',
-					// but convert its name to uppercase.
-					const map<string, string> &annotations = e->getAnnotations();
-					map<string, string>::const_iterator it, end = annotations.end();
-					for (it = annotations.begin(); it != end; it++) {
-						string name = it->first;
-						for (string::size_type i = 0; i < name.size(); i++) {
-							name[i] = toupper(name[i]);
-						}
-						params.set(name, it->second);
-					}
-				} else {
-					params.set("TITLE", "Internal server error");
-				}
-				string content = Template::apply(readAll(generalErrorFile), params);
-				params.set("CONTENT", content);
-				data = Template::apply(readAll(errorLayoutFile), params);
+				data = renderer.renderWithDetails(message, client->options, e);
 			} catch (const SystemException &e2) {
 				P_ERROR("Cannot render an error page: " << e2.what() << "\n" <<
 					e2.backtrace());
@@ -822,13 +809,7 @@ private:
 			}
 		} else {
 			try {
-				StringMap<StaticString> params;
-				params.set("PROGRAM_NAME", PROGRAM_NAME);
-				params.set("NGINX_DOC_URL", NGINX_DOC_URL);
-				params.set("APACHE2_DOC_URL", APACHE2_DOC_URL);
-				params.set("STANDALONE_DOC_URL", STANDALONE_DOC_URL);
-				data = Template::apply(readAll(templatesDir + "/undisclosed_error.html.template"),
-					params);
+				data = renderer.renderWithoutDetails();
 			} catch (const SystemException &e2) {
 				P_ERROR("Cannot render an error page: " << e2.what() << "\n" <<
 					e2.backtrace());
@@ -1121,6 +1102,20 @@ private:
 			RH_TRACE(client, 3, "Response with chunked transfer encoding detected.");
 			client->chunkedResponse = true;
 			removeHeader(headerData, transferEncoding);
+		} else {
+			Header contentLength = lookupHeader(headerData, "Content-Length", "content-length");
+			if (!contentLength.empty()) {
+				client->responseContentLength = stringToLL(contentLength.value);
+			}
+		}
+
+		Header connection = lookupHeader(headerData, "Connection", "connection");
+		if (!connection.empty() && (connection.value == "keep-alive"
+			|| connection.value == "Keep-Alive"))
+		{
+			RH_TRACE(client, 3, "Keep-alive response detected. Changing to non-keep alive.");
+			removeHeader(headerData, connection);
+			headerData.append("Connection: close\r\n");
 		}
 
 		// Add X-Powered-By.
@@ -1132,6 +1127,11 @@ private:
 
 		// Add sticky session ID.
 		if (client->stickySession && client->session != NULL) {
+			StaticString baseURI = client->scgiParser.getHeader("SCRIPT_NAME");
+			if (baseURI.empty()) {
+				baseURI = StaticString("/", 1);
+			}
+
 			StaticString cookieName = getStickySessionCookieName(client);
 			// Note that we do NOT set HttpOnly. If we set that flag then Chrome
 			// doesn't send cookies over WebSocket handshakes. Confirmed on Chrome 25.
@@ -1139,6 +1139,8 @@ private:
 			headerData.append(cookieName.data(), cookieName.size());
 			headerData.append("=");
 			headerData.append(toString(client->session->getStickySessionId()));
+			headerData.append("; Path=");
+			headerData.append(baseURI.data(), baseURI.size());
 			headerData.append("\r\n");
 
 			// Invalidate all cookies with a different route.
@@ -1160,7 +1162,9 @@ private:
 						headerData.append(cookie.first.data(), cookie.first.size());
 						headerData.append("=");
 						headerData.append(cookie.second.data(), cookie.second.size());
-						headerData.append("; path=/; expires=Thu, 01 Jan 1970 00:00:00 GMT\r\n");
+						headerData.append("; Path=");
+						headerData.append(baseURI.data(), baseURI.size());
+						headerData.append("; Expires=Thu, 01 Jan 1970 00:00:00 GMT\r\n");
 					}
 				}
 			}
@@ -1191,6 +1195,7 @@ private:
 			removeHeader(headerData, oobw);
 		}
 
+		P_TRACE(2, "Fowarding response header from app client: " << headerData);
 		headerData.append("\r\n");
 		writeToClientOutputPipe(client, headerData);
 		return true;
@@ -1234,6 +1239,10 @@ private:
 						client->responseHeaderSeen = true;
 						StaticString header = client->responseHeaderBufferer.getData();
 						if (processResponseHeader(client, header)) {
+							if (client->responseContentLength == 0) {
+								RH_TRACE(client, 3, "Disconnecting client because response Content-Length = 0");
+								onAppInputEof(client);
+							}
 							return consumed;
 						} else {
 							assert(!client->connected());
@@ -1258,7 +1267,40 @@ private:
 
 	void onAppInputChunk(const ClientPtr &client, const StaticString &data) {
 		RH_LOG_EVENT(client, "onAppInputChunk");
-		writeToClientOutputPipe(client, data);
+		StaticString data2;
+
+		if (client->responseContentLength == -1) {
+			data2 = data;
+		} else {
+			size_t rest = client->responseContentLength -
+				client->responseBodyAlreadyRead;
+			data2 = StaticString(data.data(), std::min<size_t>(
+				rest, data.size()));
+		}
+
+		client->responseBodyAlreadyRead += data2.size();
+		assert(client->responseContentLength == -1 || client->responseBodyAlreadyRead <=
+			(unsigned long long) client->responseContentLength);
+		if (data2.empty()) {
+			// Client sent more data than was advertised through
+			// Content-Length. Ignore them.
+			return;
+		}
+
+		writeToClientOutputPipe(client, data2);
+
+		if (client->responseContentLength > 0) {
+			RH_TRACE(client, 3, client->responseBodyAlreadyRead << "/" <<
+				client->responseContentLength <<
+				" bytes of application data forwarded so far.");
+
+			if (client->connected() && (unsigned long long) client->responseContentLength
+				== client->responseBodyAlreadyRead)
+			{
+				RH_TRACE(client, 3, "Disconnecting client because application data has been fully forwarded.");
+				onAppInputEof(client);
+			}
+		}
 	}
 
 	void onAppInputChunkEnd(const ClientPtr &client) {
@@ -1270,11 +1312,15 @@ private:
 		RH_LOG_EVENT(client, "onAppInputEof");
 		// Check for session == NULL in order to avoid executing the code twice on
 		// responses with chunked encoding.
+		// This also ensures that when onAppInputEof() is called twice (e.g. because
+		// additional data was received after onAppInputChunk has already called onAppInputEof()),
+		// we don't do things twice.
 		if (!client->connected() || client->session == NULL) {
 			return;
 		}
 
 		RH_DEBUG(client, "Application sent EOF");
+		client->appInput->stop();
 		client->session.reset();
 		client->endScopeLog(&client->scopeLogs.requestProxying);
 		client->clientOutputPipe->end();
@@ -2254,14 +2300,10 @@ private:
 	}
 
 	void writeSpawnExceptionErrorResponse(const ClientPtr &client, const boost::shared_ptr<SpawnException> &e) {
-		if (strip(e->getErrorPage()).empty()) {
-			RH_WARN(client, "Cannot checkout session. " << e->what());
-			writeErrorResponse(client, e->what());
-		} else {
-			RH_WARN(client, "Cannot checkout session.\nError page:\n" <<
-				e->getErrorPage());
-			writeErrorResponse(client, e->getErrorPage(), e.get());
-		}
+		RH_ERROR(client, "Cannot checkout session because a spawning error occurred. " <<
+			"The identifier of the error is " << e->get("ERROR_ID") << ". Please see earlier logs for " <<
+			"details about the error.");
+		writeErrorResponse(client, e->getErrorPage(), e.get());
 	}
 
 	void writeOtherExceptionErrorResponse(const ClientPtr &client, const ExceptionPtr &e) {
@@ -2418,7 +2460,9 @@ private:
 			}
 
 			StaticString connection = parser.getHeader("HTTP_CONNECTION");
-			if (connection == "upgrade" || connection == "Upgrade") {
+			if (regex_match(connection.data(), connection.data() + connection.size(),
+				upgradeHeaderRegex))
+			{
 				data.append("Connection: ");
 				data.append(connection.data(), connection.size());
 				data.append("\r\n");
@@ -2440,12 +2484,25 @@ private:
 				data.append("\r\n");
 			}
 
+			header = parser.getHeader("HTTPS");
+			if (!header.empty()) {
+				data.append("X-Forwarded-Proto: https\r\n");
+			}
+
+			header = parser.getHeader("REMOTE_ADDR");
+			if (!header.empty()) {
+				data.append("X-Forwarded-For: ");
+				data.append(header);
+				data.append("\r\n");
+			}
+
 			if (client->options.analytics) {
 				data.append("Passenger-Txn-Id: ");
 				data.append(client->options.transaction->getTxnId());
 				data.append("\r\n");
 			}
 
+			P_TRACE(3, "Sending headers to application: " << data);
 			data.append("\r\n");
 
 			StaticString datas[] = { data };
@@ -2650,11 +2707,13 @@ public:
 		  pool(_pool),
 		  options(_options),
 		  resourceLocator(_options.passengerRoot),
+		  upgradeHeaderRegex("(keep-alive, *)?upgrade(, *keep-alive)?",
+		      boost::regex::perl | boost::regex::icase),
 		  benchmarkPoint(getDefaultBenchmarkPoint())
 	{
 		accept4Available = true;
 		connectPasswordTimeout = 15000;
-		unionStationCore = pool->unionStationCore;
+		unionStationCore = pool->getUnionStationCore();
 
 		requestSocketWatcher.set(_requestSocket, ev::READ);
 		requestSocketWatcher.set(_libev->getLoop());