passenger 5.0.23 → 5.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9860ce906b0de7200f82e30d96333f500e57dd64
-  data.tar.gz: 68ecdbe231a4e6484ec1336ab395858bcb54e19a
+  metadata.gz: 0af1aea8f91dd9fc4b54c4c25c3ae15e36aabc62
+  data.tar.gz: 9f6890815b77b348c99a81b5dd3c1c28b9226520
 SHA512:
-  metadata.gz: 8ba93ff84710a7f93351551340173219326ce2b8672172980ccc9f29dea0e5fb8120533fc1abea5a3f71bca3895e19c2813e5872ad22aef6cf4dd4d719d542be
-  data.tar.gz: a4293240d2581fa7d2f6b8abc2ebee0c7eedf338aff91b65a18e7d6ae2c924ef81fe1fa4fd3d15dc1ff70b09ee67513aff04daee0f706fd83d4370b5fb2cf632
+  metadata.gz: e59f5e66d3f9bf483ab9e4e32570b76ff9a63ac0c38e0c3bc2b8a161068012663d7ecc6b0ac0335017ea4da01bdf2b4b2db2bbe65fe462d889f508bccca31d40
+  data.tar.gz: d05d4492c67ba8d0135b4ae5ab6e81cdaa907ffe0457d996b604be838b8322007935fe42091ace80eec96eeecd25490d997962673beb954e051a4a6f3b87c018

data/CHANGELOG

@@ -1,3 +1,18 @@
+Release 5.0.24
+--------------
+
+ * Fixes a crash when the new `force_max_concurrent_requests_per_process` option (5.0.22) was used for non-Node.js apps (e.g. Ruby). Closes GH-1720.
+ * Fixes Solaris compilation. This was a regression due to the patch for GH-1643 in 5.0.22. Closes GH-1694, GH-1701.
+ * Logs for [Union Station](https://www.unionstationapp.com) provide more information about request queueing. Closes GH-1633.
+ * Also log HTTP headers to Union Station for HTTP 4xx responses (extends the header logging for HTTP 5xx that was added in 5.0.22)
+ * Fixes cases where compilation failure of (optional) native utils was not reported.
+ * On Ruby, no longer traps SIGEXIT. This fixes erroneously setting `$ERROR_INFO` in `at_exit` callbacks. Closes GH-1730.
+ * Fixes a wrong loop exit condition that could cause a deadlock with 100% CPU usage by Passenger core. Closes GH-1709, GH-1732.
+ * Adds `socket_backlog` option to configure the Passenger Core socket backlog. For use with e.g. "Resource temporarily unavailable while connecting to upstream" errors. Closes GH-1726.
+ * [Nginx] The preferred Nginx version is now 1.8.1 (previously 1.8.0).
+ * [Standalone] Fixes the default value of the `load_shell_envvars` option. It's supposed to be disabled by default, but due to a typo it was enabled by default.
+
+
 Release 5.0.23
 --------------
 
@@ -12,7 +27,7 @@ Release 5.0.23
 Release 5.0.22
 --------------
 
- * Fixes a header collision vulnerability (CVE-2015-7519, medium severity). Please see our blog for detailed vulnerability description and advisory. Thanks to the SUSE security team for reporting this issue.
+ * Fixes a header collision vulnerability (CVE-2015-7519, medium severity). Note that this fix involves filtering request headers containing underscores. Please see our blog for detailed vulnerability description and advisory. Thanks to the SUSE security team for reporting this issue.
  * [Apache] Fixes compatibility with Apache 2.4.17's mod_autoindex. Fix contributed by Eric Covener. Closes GH-1642.
  * [Standalone] Passenger Standalone now [accepts configuration options from environment variables](https://www.phusionpassenger.com/library/config/standalone/intro.html). This makes using Passenger Standalone significantly easier on Heroku or on systems that follow the 12-factor principle. Closes GH-1661.
  * [Standalone] The Nginx configuration template has been cleaned up. It is now significantly easier to edit the Nginx configuration template without breaking compatibility with future versions.

data/bin/passenger-install-apache2-module

@@ -154,7 +154,7 @@ private
 
   def check_selinux_and_suggest_rpm
     return if !PhusionPassenger.originally_packaged? ||
-      PlatformInfo.os_name != "linux" ||
+      PlatformInfo.os_name_simple != "linux" ||
       !(PlatformInfo.linux_distro_tags.include?(:rhel) ||
           PlatformInfo.linux_distro_tags.include?(:centos))
       !PlatformInfo.find_command('sestatus')

data/build/basics.rb

@@ -182,7 +182,7 @@ AGENT_LDFLAGS << " -lselinux" if USE_SELINUX
 # Extra linker flags for backtrace_symbols() to generate useful output (see agent/Base.cpp).
 AGENT_LDFLAGS << " #{PlatformInfo.export_dynamic_flags}"
 # Enable dead symbol elimination on OS X.
-AGENT_LDFLAGS << " -Wl,-dead_strip" if PlatformInfo.os_name == "macosx"
+AGENT_LDFLAGS << " -Wl,-dead_strip" if PlatformInfo.os_name_simple == "macosx"
 AGENT_LDFLAGS.strip!
 
 # Extra compiler flags that should always be passed to the C/C++ compiler.

data/build/cplusplus_support.rb

@@ -131,7 +131,7 @@ def create_static_library(target, objects)
 end
 
 def create_shared_library(target, objects, options_or_flags = nil)
-  if PlatformInfo.os_name == "macosx"
+  if PlatformInfo.os_name_simple == "macosx"
     shlib_flag = "-flat_namespace -bundle -undefined dynamic_lookup"
   else
     shlib_flag = "-shared"

data/build/integration_tests.rb

@@ -89,7 +89,7 @@ task 'test:integration:native_packaging' do
     require 'shellwords'
     command << " -e #{Shellwords.escape(grep)}"
   end
-  case PlatformInfo.os_name
+  case PlatformInfo.os_name_simple
   when "linux"
     if PlatformInfo.linux_distro_tags.include?(:debian)
       rubylibdir = RbConfig::CONFIG["vendordir"]

data/build/packaging.rb

@@ -283,7 +283,7 @@ task 'package:tarball' => Packaging::PREGENERATED_FILES do
   if ENV['OFFICIAL_RELEASE']
     File.open("#{PKG_DIR}/#{basename}/resources/release.txt", "w").close
   end
-  if PlatformInfo.os_name == "macosx"
+  if PlatformInfo.os_name_simple == "macosx"
     sh "cd #{PKG_DIR}/#{basename} && find . -print0 | xargs -0 touch -t '201310270000'"
   else
     sh "cd #{PKG_DIR}/#{basename} && find . -print0 | xargs -0 touch -d '2013-10-27 00:00:00 UTC'"

data/resources/templates/standalone/http.erb

@@ -23,6 +23,7 @@ passenger_user_switching off;
     passenger_analytics_log_user <%= current_user %>;
 <% end %>
 
+<%= nginx_http_option(:socket_backlog) %>
 <%= nginx_http_option(:python) %>
 <%= nginx_http_option(:nodejs) %>
 <%= nginx_http_option(:log_level) %>

data/src/agent/Core/ApiServer.h

@@ -61,6 +61,8 @@ public:
 	string body;
 	Json::Value jsonBody;
 	Authorization authorization;
+	unsigned int controllerStatesGathered;
+	vector<Json::Value> controllerStates;
 
 	DEFINE_SERVER_KIT_BASE_HTTP_REQUEST_FOOTER(Passenger::Core::ApiServer::Request);
 };
@@ -172,30 +174,56 @@ private:
 		}
 	}
 
-	static void inspectControllerState(Controller *controller, Json::Value *json) {
-		*json = controller->inspectStateAsJson();
+	void gatherControllerState(Client *client, Request *req,
+		Controller *controller, unsigned int i)
+	{
+		Json::Value state = controller->inspectStateAsJson();
+		getContext()->libev->runLater(boost::bind(&ApiServer::controllerStateGathered,
+			this, client, req, i, state));
 	}
 
-	void processServerStatus(Client *client, Request *req) {
-		if (authorizeStateInspectionOperation(this, client, req)) {
+	void controllerStateGathered(Client *client, Request *req,
+		unsigned int i, Json::Value state)
+	{
+		if (req->ended()) {
+			unrefRequest(req, __FILE__, __LINE__);
+			return;
+		}
+
+		req->controllerStatesGathered++;
+		req->controllerStates[i] = state;
+
+		if (req->controllerStatesGathered == controllers.size()) {
 			HeaderTable headers;
 			headers.insert(req->pool, "Content-Type", "application/json");
 
-			Json::Value doc;
-			doc["threads"] = (Json::UInt) controllers.size();
+			Json::Value response;
+			response["threads"] = (Json::UInt) controllers.size();
+
 			for (unsigned int i = 0; i < controllers.size(); i++) {
-				Json::Value json;
 				string key = "thread" + toString(i + 1);
-
-				controllers[i]->getContext()->libev->runSync(boost::bind(
-					inspectControllerState, controllers[i], &json));
-				doc[key] = json;
+				response[key] = req->controllerStates[i];
 			}
 
 			writeSimpleResponse(client, 200, &headers,
-				psg_pstrdup(req->pool, doc.toStyledString()));
+				psg_pstrdup(req->pool, response.toStyledString()));
 			if (!req->ended()) {
-				endRequest(&client, &req);
+				Request *req2 = req;
+				endRequest(&client, &req2);
+			}
+		}
+
+		unrefRequest(req, __FILE__, __LINE__);
+	}
+
+	void processServerStatus(Client *client, Request *req) {
+		if (authorizeStateInspectionOperation(this, client, req)) {
+			req->controllerStates.resize(controllers.size());
+			for (unsigned int i = 0; i < controllers.size(); i++) {
+				refRequest(req, __FILE__, __LINE__);
+				controllers[i]->getContext()->libev->runLater(boost::bind(
+					&ApiServer::gatherControllerState, this,
+					client, req, controllers[i], i));
 			}
 		} else {
 			apiServerRespondWith401(this, client, req);
@@ -407,37 +435,16 @@ private:
 		}
 	}
 
-	static void getControllerConfig(Controller *controller, Json::Value *json) {
-		*json = controller->getConfigAsJson();
-	}
-
 	void processConfig(Client *client, Request *req) {
 		if (req->method == HTTP_GET) {
 			if (!authorizeStateInspectionOperation(this, client, req)) {
 				apiServerRespondWith401(this, client, req);
 			}
 
-			HeaderTable headers;
-			string logFile = getLogFile();
-			string fileDescriptorLogFile = getFileDescriptorLogFile();
-
-			headers.insert(req->pool, "Content-Type", "application/json");
-			Json::Value doc;
-			controllers[0]->getContext()->libev->runSync(boost::bind(
-				getControllerConfig, controllers[0], &doc));
-			doc["log_level"] = getLogLevel();
-			if (!logFile.empty()) {
-				doc["log_file"] = logFile;
-			}
-			if (!fileDescriptorLogFile.empty()) {
-				doc["file_descriptor_log_file"] = fileDescriptorLogFile;
-			}
-
-			writeSimpleResponse(client, 200, &headers,
-				psg_pstrdup(req->pool, doc.toStyledString()));
-			if (!req->ended()) {
-				endRequest(&client, &req);
-			}
+			refRequest(req, __FILE__, __LINE__);
+			controllers[0]->getContext()->libev->runLater(boost::bind(
+				&ApiServer::processConfig_getControllerConfig, this,
+				client, req, controllers[0]));
 		} else if (req->method == HTTP_PUT) {
 			if (!authorizeAdminOperation(this, client, req)) {
 				apiServerRespondWith401(this, client, req);
@@ -450,6 +457,46 @@ private:
 		}
 	}
 
+	void processConfig_getControllerConfig(Client *client, Request *req,
+		Controller *controller)
+	{
+		Json::Value config = controller->getConfigAsJson();
+		getContext()->libev->runLater(boost::bind(
+			&ApiServer::processConfig_controllerConfigGathered, this,
+			client, req, controller, config));
+	}
+
+	void processConfig_controllerConfigGathered(Client *client, Request *req,
+		Controller *controller, Json::Value config)
+	{
+		if (req->ended()) {
+			unrefRequest(req, __FILE__, __LINE__);
+			return;
+		}
+
+		HeaderTable headers;
+		string logFile = getLogFile();
+		string fileDescriptorLogFile = getFileDescriptorLogFile();
+
+		headers.insert(req->pool, "Content-Type", "application/json");
+		config["log_level"] = getLogLevel();
+		if (!logFile.empty()) {
+			config["log_file"] = logFile;
+		}
+		if (!fileDescriptorLogFile.empty()) {
+			config["file_descriptor_log_file"] = fileDescriptorLogFile;
+		}
+
+		writeSimpleResponse(client, 200, &headers,
+			psg_pstrdup(req->pool, config.toStyledString()));
+		if (!req->ended()) {
+			Request *req2 = req;
+			endRequest(&client, &req2);
+		}
+
+		unrefRequest(req, __FILE__, __LINE__);
+	}
+
 	static void configureController(Controller *controller, Json::Value json) {
 		controller->configure(json);
 	}
@@ -575,12 +622,18 @@ protected:
 		return ServerKit::Channel::Result(buffer.size(), false);
 	}
 
+	virtual void reinitializeRequest(Client *client, Request *req) {
+		ParentClass::reinitializeRequest(client, req);
+		req->controllerStatesGathered = 0;
+	}
+
 	virtual void deinitializeRequest(Client *client, Request *req) {
 		req->body.clear();
 		if (!req->jsonBody.isNull()) {
 			req->jsonBody = Json::Value();
 		}
 		req->authorization = Authorization();
+		req->controllerStates.clear();
 		ParentClass::deinitializeRequest(client, req);
 	}
 

data/src/agent/Core/ApplicationPool/Pool.h

@@ -58,6 +58,7 @@
 #include <Utils/VariantMap.h>
 #include <Utils/ProcessMetricsCollector.h>
 #include <Utils/SystemMetricsCollector.h>
+#include <Core/UnionStation/StopwatchLog.h>
 #include <Core/ApplicationPool/Common.h>
 #include <Core/ApplicationPool/Context.h>
 #include <Core/ApplicationPool/Process.h>
@@ -448,7 +449,7 @@ public:
 
 	/****** Miscellaneous ******/
 
-	void asyncGet(const Options &options, const GetCallback &callback, bool lockNow = true);
+	void asyncGet(const Options &options, const GetCallback &callback, bool lockNow = true, UnionStation::StopwatchLog **stopwatchLog = NULL);
 	SessionPtr get(const Options &options, Ticket *ticket);
 	void setMax(unsigned int max);
 	void setMaxIdleTime(unsigned long long value);

data/src/agent/Core/ApplicationPool/Pool/GroupUtils.cpp

@@ -40,7 +40,8 @@ using namespace boost;
 
 /****************************
  *
- * Private methods
+ * Consider these to be private methods,
+ * they are only marked public for unit testing!
  *
  ****************************/
 

data/src/agent/Core/ApplicationPool/Pool/Miscellaneous.cpp

@@ -41,7 +41,7 @@ using namespace boost;
 // 'lockNow == false' may only be used during unit tests. Normally we
 // should never call the callback while holding the lock.
 void
-Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow) {
+Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow, UnionStation::StopwatchLog **stopwatchLog) {
 	DynamicScopedLock lock(syncher, lockNow);
 
 	assert(lifeStatus == ALIVE || lifeStatus == PREPARED_FOR_SHUTDOWN);
@@ -50,6 +50,33 @@ Pool::asyncGet(const Options &options, const GetCallback &callback, bool lockNow
 	boost::container::vector<Callback> actions;
 
 	Group *existingGroup = findMatchingGroup(options);
+	if (stopwatchLog != NULL) {
+		// Log some essentials stats about what this request is facing in its upcoming journey through the queue:
+		// 1) position in the queue upon entry, and 2) whether spawning activity is occurring (which takes cycles
+		// but also indicates the server has headroom to handle the load).
+		Json::Value data;
+		if (!existingGroup) {
+			data["message"] = "spawning.."; // the first of this group, so keep it simple (also: we don't know maxQ yet)
+		} else {
+			char queueMaxStr[10];
+			int queueMax = existingGroup->options.maxRequestQueueSize;
+			if (queueMax > 0) {
+				snprintf(queueMaxStr, 10, "%d", queueMax);
+			}
+			char message[50];
+			snprintf(message, 100, "queue: %zu / %s, spawning: %s", existingGroup->getWaitlist.size(),
+					(queueMax == 0 ? "inf" : queueMaxStr),
+					(existingGroup->processesBeingSpawned == 0 ? "no" : "yes"));
+			data["message"] = message;
+		}
+		Json::Value json;
+		json["data"] = data;
+		json["data_type"] = "generic";
+		json["name"] = "Await available process";
+
+		*stopwatchLog = new UnionStation::StopwatchLog(options.transaction, "Pool::asyncGet", stringifyJson(json).c_str());
+	}
+
 	if (OXT_LIKELY(existingGroup != NULL)) {
 		/* Best case: the app group is already in the pool. Let's use it. */
 		P_TRACE(2, "Found existing Group");

data/src/agent/Core/ApplicationPool/Process.h

@@ -483,6 +483,9 @@ public:
 	void forceMaxConcurrency(int value) {
 		assert(value >= 0);
 		concurrency = value;
+		for (unsigned i = 0; i < sessionSocketCount; i++) {
+			sessionSockets[i]->concurrency = concurrency;
+		}
 	}
 
 	void shutdownNotRequired() {

data/src/agent/Core/Controller/CheckoutSession.cpp

@@ -54,7 +54,7 @@ Controller::checkoutSession(Client *client, Request *req) {
 	CC_BENCHMARK_POINT(client, req, BM_BEFORE_CHECKOUT);
 	SKC_TRACE(client, 2, "Checking out session: appRoot=" << options.appRoot);
 	req->state = Request::CHECKING_OUT_SESSION;
-	req->beginStopwatchLog(&req->stopwatchLogs.getFromPool, "get from pool");
+
 	if (req->requestBodyBuffering) {
 		assert(!req->bodyBuffer.isStarted());
 	} else {
@@ -70,7 +70,7 @@ Controller::checkoutSession(Client *client, Request *req) {
 	#ifdef DEBUG_CC_EVENT_LOOP_BLOCKING
 		req->timeBeforeAccessingApplicationPool = ev_now(getLoop());
 	#endif
-	appPool->asyncGet(options, callback);
+	appPool->asyncGet(options, callback, true, req->useUnionStation() ? &req->stopwatchLogs.getFromPool : NULL);
 	#ifdef DEBUG_CC_EVENT_LOOP_BLOCKING
 		if (!req->timedAppPoolGet) {
 			req->timedAppPoolGet = true;

data/src/agent/Core/Controller/ForwardResponse.cpp

@@ -895,7 +895,7 @@ Controller::logResponseHeaders(Client *client, Request *req, struct iovec *buffe
 			req->logMessage("Status: " + toString(req->appResponse.statusCode));
 		}
 
-		if (req->appResponse.statusCode >= 500 && req->appResponse.statusCode <= 599) {
+		if (req->appResponse.statusCode >= 400 && req->appResponse.statusCode <= 599) {
 			// Log the request headers like Request headers: { header1: values1-concatenated, header2: values2-concatenated } (single line)
 			// Concatenation was already done by HeaderTable.h:insert (using a comma ',' for joining, or a semicolon ';' for Cookie headers
 			ServerKit::HeaderTable::Iterator it(req->headers);

data/src/agent/Core/Controller/Request.h

@@ -140,9 +140,9 @@ public:
 		return options.transaction != NULL;
 	}
 
-	void beginStopwatchLog(UnionStation::StopwatchLog **stopwatchLog, const char *name) {
+	void beginStopwatchLog(UnionStation::StopwatchLog **stopwatchLog, const char *id, const char *nameAndData = NULL) {
 		if (options.transaction != NULL) {
-			*stopwatchLog = new UnionStation::StopwatchLog(options.transaction, name);
+			*stopwatchLog = new UnionStation::StopwatchLog(options.transaction, id, nameAndData);
 		}
 	}
 

data/src/agent/Core/CoreMain.cpp

@@ -323,7 +323,7 @@ startListening() {
 	#endif
 
 	for (unsigned int i = 0; i < addresses.size(); i++) {
-		wo->serverFds[i] = createServer(addresses[i], 0, true,
+		wo->serverFds[i] = createServer(addresses[i], agentsOptions->getInt("socket_backlog"), true,
 			__FILE__, __LINE__);
 		#ifdef USE_SELINUX
 			resetSelinuxSocketContext();
@@ -790,7 +790,7 @@ static void
 abortLongRunningConnections(const ApplicationPool2::ProcessPtr &process) {
 	// We are inside the ApplicationPool lock. Be very careful here.
 	WorkingObjects *wo = workingObjects;
-	P_NOTICE("Disconnecting long-running connections for process " <<
+	P_NOTICE("Checking whether to disconnect long-running connections for process " <<
 		process->getPid() << ", application " << process->getGroup()->getName());
 	for (unsigned int i = 0; i < wo->threadWorkingObjects.size(); i++) {
 		wo->threadWorkingObjects[i].bgloop->safe->runLater(
@@ -1048,6 +1048,7 @@ setAgentsOptionsDefaults() {
 			inferDefaultGroup(options.get("default_user")));
 	}
 	options.setDefaultStrSet("core_addresses", defaultAddress);
+	options.setDefaultInt("socket_backlog", DEFAULT_SOCKET_BACKLOG);
 	options.setDefaultBool("multi_app", false);
 	options.setDefault("environment", DEFAULT_APP_ENV);
 	options.setDefault("spawn_method", DEFAULT_SPAWN_METHOD);

data/src/agent/Core/OptionParser.h

@@ -65,6 +65,8 @@ coreUsage() {
 	printf("      --api-listen ADDRESS  Listen on the given address for API commands.\n");
 	printf("                            The same syntax and limitations as with --listen\n");
 	printf("                            are applicable\n");
+	printf("      --socket-backlog      Override size of the socket backlog.\n");
+	printf("                            Default: %d\n", DEFAULT_SOCKET_BACKLOG);
 	printf("\n");
 	printf("Daemon options (optional):\n");
 	printf("      --pid-file PATH       Store the core's PID in the given file. The file\n");
@@ -250,6 +252,9 @@ parseCoreOption(int argc, const char *argv[], int &i, VariantMap &options) {
 		authorizations.push_back(argv[i + 1]);
 		options.setStrSet("core_authorizations", authorizations);
 		i += 2;
+	} else if (p.isValueFlag(argc, i, argv[i], '\0', "--socket-backlog")) {
+		options.setInt("socket_backlog", atoi(argv[i + 1]));
+		i += 2;
 	} else if (p.isFlag(argv[i], '\0', "--no-user-switching")) {
 		options.setBool("user_switching", false);
 		i++;