passenger 5.0.25 → 5.0.26

data/src/agent/Core/Controller.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -119,6 +119,8 @@ private:
 	typedef ServerKit::FileBufferedChannel FileBufferedChannel;
 	typedef ServerKit::FileBufferedFdSinkChannel FileBufferedFdSinkChannel;
 
+	// If you change this value, make sure that Request::sessionCheckoutTry
+	// has enough bits.
 	static const unsigned int MAX_SESSION_CHECKOUT_TRY = 10;
 
 	unsigned int statThrottleRate;
@@ -218,12 +220,12 @@ private:
 	/****** Stage: checkout session ******/
 
 	void checkoutSession(Client *client, Request *req);
-	static void sessionCheckedOut(const SessionPtr &session,
+	static void sessionCheckedOut(const AbstractSessionPtr &session,
 		const ExceptionPtr &e, void *userData);
 	void sessionCheckedOutFromAnotherThread(Client *client, Request *req,
-		SessionPtr session, ExceptionPtr e);
+		AbstractSessionPtr session, ExceptionPtr e);
 	void sessionCheckedOutFromEventLoopThread(Client *client, Request *req,
-		const SessionPtr &session, const ExceptionPtr &e);
+		const AbstractSessionPtr &session, const ExceptionPtr &e);
 	void maybeSend100Continue(Client *client, Request *req);
 	void initiateSession(Client *client, Request *req);
 	static void checkoutSessionLater(Request *req);
@@ -261,7 +263,7 @@ private:
 	void sendHeaderToAppWithHttpProtocolWithBuffering(Request *req, unsigned int offset,
 		HttpHeaderConstructionCache &cache);
 	void sendBodyToApp(Client *client, Request *req);
-	void halfCloseAppSink(Client *client, Request *req);
+	void maybeHalfCloseAppSinkBecauseRequestBodyEndReached(Client *client, Request *req);
 	Channel::Result whenSendingRequest_onRequestBody(Client *client, Request *req,
 		const MemoryKit::mbuf &buffer, int errcode);
 	static void resumeRequestBodyChannelWhenAppSinkIdle(Channel *_channel,
@@ -376,10 +378,17 @@ protected:
 	void deinitializeAppResponse(Client *client, Request *req);
 	virtual Channel::Result onRequestBody(Client *client, Request *req,
 		const MemoryKit::mbuf &buffer, int errcode);
+	virtual void onNextRequestEarlyReadError(Client *client, Request *req, int errcode);
 	virtual bool shouldDisconnectClientOnShutdown(Client *client);
 	virtual bool supportsUpgrade(Client *client, Request *req);
 
 
+	/****** Marked virtual so that unit tests can mock these ******/
+
+	virtual void asyncGetFromApplicationPool(Request *req,
+		ApplicationPool2::GetCallback callback);
+
+
 public:
 	ResourceLocator *resourceLocator;
 	PoolPtr appPool;
@@ -390,7 +399,7 @@ public:
 
 	Controller(ServerKit::Context *context, const VariantMap *_agentsOptions,
 		unsigned int _threadNumber = 1);
-	~Controller();
+	virtual ~Controller();
 	void initialize();
 
 

data/src/agent/Core/Controller/CheckoutSession.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -70,7 +70,7 @@ Controller::checkoutSession(Client *client, Request *req) {
 	#ifdef DEBUG_CC_EVENT_LOOP_BLOCKING
 		req->timeBeforeAccessingApplicationPool = ev_now(getLoop());
 	#endif
-	appPool->asyncGet(options, callback, true, req->useUnionStation() ? &req->stopwatchLogs.getFromPool : NULL);
+	asyncGetFromApplicationPool(req, callback);
 	#ifdef DEBUG_CC_EVENT_LOOP_BLOCKING
 		if (!req->timedAppPoolGet) {
 			req->timedAppPoolGet = true;
@@ -82,7 +82,15 @@ Controller::checkoutSession(Client *client, Request *req) {
 }
 
 void
-Controller::sessionCheckedOut(const SessionPtr &session, const ExceptionPtr &e,
+Controller::asyncGetFromApplicationPool(Request *req, ApplicationPool2::GetCallback callback) {
+	appPool->asyncGet(req->options, callback, true,
+		req->useUnionStation()
+		? &req->stopwatchLogs.getFromPool
+		: NULL);
+}
+
+void
+Controller::sessionCheckedOut(const AbstractSessionPtr &session, const ExceptionPtr &e,
 	void *userData)
 {
 	Request *req = static_cast<Request *>(userData);
@@ -101,7 +109,7 @@ Controller::sessionCheckedOut(const SessionPtr &session, const ExceptionPtr &e,
 
 void
 Controller::sessionCheckedOutFromAnotherThread(Client *client, Request *req,
-	SessionPtr session, ExceptionPtr e)
+	AbstractSessionPtr session, ExceptionPtr e)
 {
 	SKC_LOG_EVENT(Controller, client, "sessionCheckedOutFromAnotherThread");
 	sessionCheckedOutFromEventLoopThread(client, req, session, e);
@@ -110,7 +118,7 @@ Controller::sessionCheckedOutFromAnotherThread(Client *client, Request *req,
 
 void
 Controller::sessionCheckedOutFromEventLoopThread(Client *client, Request *req,
-	const SessionPtr &session, const ExceptionPtr &e)
+	const AbstractSessionPtr &session, const ExceptionPtr &e)
 {
 	if (req->ended()) {
 		return;

data/src/agent/Core/Controller/ForwardResponse.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -295,6 +295,7 @@ Controller::onAppSourceData(Client *client, Request *req, const MemoryKit::mbuf
 			// EOF
 			UPDATE_TRACE_POINT();
 			SKC_TRACE(client, 2, "Application sent EOF");
+			SKC_TRACE(client, 2, "Not keep-aliving application session connection");
 			req->session->close(true, false);
 			endRequest(&client, &req);
 			return Channel::Result(0, false);
@@ -1078,7 +1079,24 @@ Controller::handleAppResponseBodyEnd(Client *client, Request *req) {
 
 OXT_FORCE_INLINE void
 Controller::keepAliveAppConnection(Client *client, Request *req) {
-	req->session->close(true, req->appResponse.wantKeepAlive);
+	if (req->halfClosePolicy == Request::HALF_CLOSE_PERFORMED) {
+		SKC_TRACE(client, 2, "Not keep-aliving application session connection"
+			" because it had been half-closed before");
+		req->session->close(true, false);
+	} else {
+		// halfClosePolicy is initialized in sendHeaderToApp(). That method is
+		// called immediately after checking out a session, before any events
+		// from the appSource channel can be received.
+		assert(req->halfClosePolicy != Request::HALF_CLOSE_POLICY_UNINITIALIZED);
+		if (req->appResponse.wantKeepAlive) {
+			SKC_TRACE(client, 2, "Keep-aliving application session connection");
+			req->session->close(true, true);
+		} else {
+			SKC_TRACE(client, 2, "Not keep-aliving application session connection"
+				" because application did not allow it");
+			req->session->close(true, false);
+		}
+	}
 }
 
 void

data/src/agent/Core/Controller/Hooks.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -129,8 +129,8 @@ Controller::reinitializeRequest(Client *client, Request *req) {
 	req->requestBodyBuffering = false;
 	req->https = false;
 	req->stickySession = false;
-	req->halfCloseAppConnection = false;
 	req->sessionCheckoutTry = 0;
+	req->halfClosePolicy = Request::HALF_CLOSE_POLICY_UNINITIALIZED;
 	req->appResponseInitialized = false;
 	req->strip100ContinueHeader = false;
 	req->hasPragmaHeader = false;
@@ -258,6 +258,19 @@ Controller::onRequestBody(Client *client, Request *req, const MemoryKit::mbuf &b
 	}
 }
 
+void
+Controller::onNextRequestEarlyReadError(Client *client, Request *req, int errcode) {
+	ParentClass::onNextRequestEarlyReadError(client, req, errcode);
+	if (req->halfClosePolicy == Request::HALF_CLOSE_UPON_NEXT_REQUEST_EARLY_READ_ERROR) {
+		SKC_TRACE(client, 3, "Half-closing application socket with SHUT_WR"
+			" because the next request's early read error has been detected: "
+			<< ServerKit::getErrorDesc(errcode) << " (errno=" << errcode << ")");
+		req->halfClosePolicy = Request::HALF_CLOSE_PERFORMED;
+		assert(req->session != NULL);
+		::shutdown(req->session->fd(), SHUT_WR);
+	}
+}
+
 bool
 Controller::shouldDisconnectClientOnShutdown(Client *client) {
 	return ParentClass::shouldDisconnectClientOnShutdown(client) || !gracefulExit;

data/src/agent/Core/Controller/InitRequest.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -188,6 +188,9 @@ Controller::fillPoolOptionsFromAgentsOptions(Options &options) {
 	if (agentsOptions->has("meteor_app_settings")) {
 		options.meteorAppSettings = agentsOptions->get("meteor_app_settings");
 	}
+	if (agentsOptions->has("app_file_descriptor_ulimit")) {
+		options.fileDescriptorUlimit = agentsOptions->getUint("app_file_descriptor_ulimit");
+	}
 
 	options.logLevel = getLogLevel();
 	options.integrationMode = agentsOptions->get("integration_mode",
@@ -378,6 +381,7 @@ Controller::createNewPoolOptions(Client *client, Request *req,
 	fillPoolOption(req, options.restartDir, "!~PASSENGER_RESTART_DIR");
 	fillPoolOption(req, options.startupFile, "!~PASSENGER_STARTUP_FILE");
 	fillPoolOption(req, options.loadShellEnvvars, "!~PASSENGER_LOAD_SHELL_ENVVARS");
+	fillPoolOption(req, options.fileDescriptorUlimit, "!~PASSENGER_APP_FILE_DESCRIPTOR_ULIMIT");
 	fillPoolOption(req, options.raiseInternalError, "!~PASSENGER_RAISE_INTERNAL_ERROR");
 	/******************/
 

data/src/agent/Core/Controller/InitializationAndShutdown.cpp

@@ -143,6 +143,7 @@ Controller::Controller(ServerKit::Context *context, const VariantMap *_agentsOpt
 }
 
 Controller::~Controller() {
+	ev_check_stop(getLoop(), &checkWatcher);
 	psg_destroy_pool(stringPool);
 }
 

data/src/agent/Core/Controller/Request.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -59,6 +59,13 @@ public:
 		WAITING_FOR_APP_OUTPUT
 	};
 
+	enum HalfClosePolicy {
+		HALF_CLOSE_POLICY_UNINITIALIZED,
+		HALF_CLOSE_UPON_REACHING_REQUEST_BODY_END,
+		HALF_CLOSE_UPON_NEXT_REQUEST_EARLY_READ_ERROR,
+		HALF_CLOSE_PERFORMED
+	};
+
 	ev_tstamp startedAt;
 
 	State state: 3;
@@ -66,16 +73,16 @@ public:
 	bool requestBodyBuffering: 1;
 	bool https: 1;
 	bool stickySession: 1;
-	bool halfCloseAppConnection: 1;
 
 	// Range: 0..MAX_SESSION_CHECKOUT_TRY
-	boost::uint8_t sessionCheckoutTry;
+	boost::uint8_t sessionCheckoutTry: 4;
+	HalfClosePolicy halfClosePolicy: 2;
 	bool appResponseInitialized: 1;
 	bool strip100ContinueHeader: 1;
 	bool hasPragmaHeader: 1;
 
 	Options options;
-	SessionPtr session;
+	AbstractSessionPtr session;
 	const LString *host;
 
 	ServerKit::FdSinkChannel appSink;

data/src/agent/Core/Controller/SendRequest.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -89,19 +89,38 @@ Controller::sendHeaderToApp(Client *client, Request *req) {
 	SKC_TRACE(client, 2, "Sending headers to application with " <<
 		req->session->getProtocol() << " protocol");
 	req->state = Request::SENDING_HEADER_TO_APP;
+	P_ASSERT_EQ(req->halfClosePolicy, Request::HALF_CLOSE_POLICY_UNINITIALIZED);
 
-	/**
-	 * HTTP does not formally support half-closing, and Node.js treats a
-	 * half-close as a full close, so we only half-close session sockets, not
-	 * HTTP sockets.
-	 */
 	if (req->session->getProtocol() == "session") {
 		UPDATE_TRACE_POINT();
-		req->halfCloseAppConnection = req->bodyType != Request::RBT_NO_BODY;
+		if (req->bodyType == Request::RBT_NO_BODY) {
+			// When there is no request body we will try to keep-alive the
+			// application connection, so half-close the application
+			// connection upon encountering the next request's early error
+			// in order not to break the keep-alive.
+			req->halfClosePolicy = Request::HALF_CLOSE_UPON_NEXT_REQUEST_EARLY_READ_ERROR;
+		} else {
+			// When there is a request body we won't try to keep-alive
+			// the application connection, so it's safe to half-close immediately
+			// upon reaching the end of the request body.
+			req->halfClosePolicy = Request::HALF_CLOSE_UPON_REACHING_REQUEST_BODY_END;
+		}
 		sendHeaderToAppWithSessionProtocol(client, req);
 	} else {
 		UPDATE_TRACE_POINT();
-		req->halfCloseAppConnection = false;
+		if (req->bodyType == Request::RBT_UPGRADE) {
+			req->halfClosePolicy = Request::HALF_CLOSE_UPON_REACHING_REQUEST_BODY_END;
+		} else {
+			// HTTP does not formally support half-closing. Some apps support
+			// HTTP with half-closing, others (such as Node.js http.Server with
+			// default settings) treat a half-close as a full close. Furthermore,
+			// we always try to keep-alive the application connection.
+			//
+			// So we can't half-close immediately upon reaching the end of the
+			// request body. The app might not have yet sent a response by then.
+			// We only half-close upon the next request's early error.
+			req->halfClosePolicy = Request::HALF_CLOSE_UPON_NEXT_REQUEST_EARLY_READ_ERROR;
+		}
 		sendHeaderToAppWithHttpProtocol(client, req);
 	}
 
@@ -944,15 +963,17 @@ Controller::sendBodyToApp(Client *client, Request *req) {
 		// data is forwarded.
 		SKC_TRACE(client, 2, "No body to send to application");
 		req->state = Request::WAITING_FOR_APP_OUTPUT;
-		halfCloseAppSink(client, req);
+		maybeHalfCloseAppSinkBecauseRequestBodyEndReached(client, req);
 	}
 }
 
 void
-Controller::halfCloseAppSink(Client *client, Request *req) {
+Controller::maybeHalfCloseAppSinkBecauseRequestBodyEndReached(Client *client, Request *req) {
 	P_ASSERT_EQ(req->state, Request::WAITING_FOR_APP_OUTPUT);
-	if (req->halfCloseAppConnection) {
-		SKC_TRACE(client, 3, "Half-closing application socket with SHUT_WR");
+	if (req->halfClosePolicy == Request::HALF_CLOSE_UPON_REACHING_REQUEST_BODY_END) {
+		SKC_TRACE(client, 3, "Half-closing application socket with SHUT_WR"
+			" because end of request body reached");
+		req->halfClosePolicy = Request::HALF_CLOSE_PERFORMED;
 		::shutdown(req->session->fd(), SHUT_WR);
 	}
 }
@@ -1007,7 +1028,7 @@ Controller::whenSendingRequest_onRequestBody(Client *client, Request *req,
 		// care of ending the request, once all response
 		// data is forwarded.
 		req->state = Request::WAITING_FOR_APP_OUTPUT;
-		halfCloseAppSink(client, req);
+		maybeHalfCloseAppSinkBecauseRequestBodyEndReached(client, req);
 		return Channel::Result(0, true);
 	} else {
 		const unsigned int BUFSIZE = 1024;

data/src/agent/Core/Controller/StateInspectionAndConfiguration.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2014-2015 Phusion Holding B.V.
+ *  Copyright (c) 2014-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -117,7 +117,7 @@ Controller::inspectRequestStateAsJson(const Request *req) const {
 
 	if (req->session != NULL) {
 		Json::Value &sessionDoc = doc["session"] = Json::Value(Json::objectValue);
-		const Session *session = req->session.get();
+		const AbstractSession *session = req->session.get();
 
 		if (req->session->isClosed()) {
 			sessionDoc["closed"] = true;

data/src/agent/Core/CoreMain.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2015 Phusion Holding B.V.
+ *  Copyright (c) 2010-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -40,6 +40,7 @@
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/un.h>
+#include <sys/resource.h>
 #include <cstring>
 #include <cassert>
 #include <cerrno>
@@ -259,6 +260,30 @@ initializeSingleAppMode() {
 	P_NOTICE("App startup file: " << options.get("startup_file"));
 }
 
+static void
+setUlimits() {
+	TRACE_POINT();
+	VariantMap &options = *agentsOptions;
+
+	if (options.has("core_file_descriptor_ulimit")) {
+		unsigned int number = options.getUint("core_file_descriptor_ulimit");
+		struct rlimit limit;
+		int ret;
+
+		limit.rlim_cur = number;
+		limit.rlim_max = number;
+		do {
+			ret = setrlimit(RLIMIT_NOFILE, &limit);
+		} while (ret == -1 && errno == EINTR);
+
+		if (ret == -1) {
+			int e = errno;
+			P_ERROR("Unable to set file descriptor ulimit to " << number
+				<< ": " << strerror(e) << " (errno=" << e << ")");
+		}
+	}
+}
+
 static void
 makeFileWorldReadableAndWritable(const string &path) {
 	int ret;
@@ -960,6 +985,7 @@ runCore() {
 		UPDATE_TRACE_POINT();
 		initializePrivilegedWorkingObjects();
 		initializeSingleAppMode();
+		setUlimits();
 		startListening();
 		createPidFile();
 		lowerPrivilege();

data/src/agent/Core/OptionParser.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2015 Phusion Holding B.V.
+ *  Copyright (c) 2010-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -118,6 +118,8 @@ coreUsage() {
 	printf("      --meteor-app-settings PATH\n");
 	printf("                            File with settings for a Meteor (non-bundled) app.\n");
 	printf("                            (passed to Meteor using --settings)\n");
+	printf("      --app-file-descriptor-ulimit NUMBER\n");
+	printf("                            Set custom file descriptor ulimit for the app\n");
 	printf("      --debugger            Enable Ruby debugger support (Enterprise only)\n");
 	printf("\n");
 	printf("      --rolling-restarts    Enable rolling restarts (Enterprise only)\n");
@@ -185,6 +187,8 @@ coreUsage() {
 	printf("                            Default: number of CPU cores (%d)\n",
 		boost::thread::hardware_concurrency());
 	printf("      --cpu-affine          Enable per-thread CPU affinity (Linux only)\n");
+	printf("      --core-file-descriptor-ulimit NUMBER\n");
+	printf("                            Set custom file descriptor ulimit for the core\n");
 	printf("  -h, --help                Show this help\n");
 	printf("\n");
 	printf("API account privilege levels (ordered from most to least privileges):\n");
@@ -345,6 +349,9 @@ parseCoreOption(int argc, const char *argv[], int &i, VariantMap &options) {
 	} else if (p.isValueFlag(argc, i, argv[i], '\0', "--meteor-app-settings")) {
 		options.set("meteor_app_settings", argv[i + 1]);
 		i += 2;
+	} else if (p.isValueFlag(argc, i, argv[i], '\0', "--app-file-descriptor-ulimit")) {
+		options.setUint("app_file_descriptor_ulimit", atoi(argv[i + 1]));
+		i += 2;
 	} else if (p.isFlag(argv[i], '\0', "--debugger")) {
 		options.setBool("debugger", true);
 		i++;
@@ -393,6 +400,9 @@ parseCoreOption(int argc, const char *argv[], int &i, VariantMap &options) {
 	} else if (p.isFlag(argv[i], '\0', "--cpu-affine")) {
 		options.setBool("core_cpu_affine", true);
 		i++;
+	} else if (p.isValueFlag(argc, i, argv[i], '\0', "--core-file-descriptor-ulimit")) {
+		options.setUint("core_file_descriptor_ulimit", atoi(argv[i + 1]));
+		i += 2;
 	} else if (!startsWith(argv[i], "-")) {
 		if (!options.has("app_root")) {
 			options.set("app_root", argv[i]);