passenger 5.0.29 → 5.0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: de34a89bcd92faaf4c870691894fdc98545e933f
-  data.tar.gz: de6ab1650b829f01146d3851f85cc38f347b02f5
+  metadata.gz: 722c1e1bd4ec567159a9176ac6ccb0e207f605af
+  data.tar.gz: 51d23530f41934e66d5c8a8c13d68386400765fb
 SHA512:
-  metadata.gz: fec73a5f401c95111c03dde4e99f2d393d2b340e82ced8e051c11862f8810ca8f9d6b5164701622d6e1e80655f58ac6859297dbff1089f0ae4a7c4a939bcfc35
-  data.tar.gz: 8aa0e543b0ef031f56ef902f67e6ac6b6e032cf057cac7c9562d21b095d70ad8c14194e278ff692acd5e62a3ae0904f65b1f050946ed0993854e14af1cf8d762
+  metadata.gz: 746361cde0146a6ad7ff161553d61eee22889cbf682e45306e1510337d0826957ea6f61b434ca659b450c82b58e8af2ff578ddbdf2297dc937b5adfa14817e8c
+  data.tar.gz: afd17292ab8b5764edd39574914817462268fd85293ab49c8ffc7b3a117239726c9817b332bcf9675f0649f88994938e8e5fc9f5dea53ad724f97a621b83cf92

data/CHANGELOG

@@ -1,3 +1,12 @@
+Release 5.0.30
+--------------
+
+ * Changes mbuf block size from 512 to 4096 bytes to better fit modern requests and significantly speed up disk buffering.
+ * [Nginx] Fixes PCRE checksum after the preferred version update in 5.0.29 (contributed by: clemensg).
+ * [Apache] Fixes buffer limit crash on large file upload (when core disk buffer can't keep up with client for some time), and limits per-client buffer memory usage to 130 KB. Closes GH-1620.
+ * Fixes potential hang when an UnseekableSocket gets serialized to json. Closes GH-1838.
+
+
 Release 5.0.29
 --------------
 

data/src/agent/Core/Controller.h

@@ -215,6 +215,7 @@ private:
 	void beginBufferingBody(Client *client, Request *req);
 	Channel::Result whenBufferingBody_onRequestBody(Client *client, Request *req,
 		const MemoryKit::mbuf &buffer, int errcode);
+	static void _bodyBufferFlushed(FileBufferedChannel *_channel);
 
 
 	/****** Stage: checkout session ******/

data/src/agent/Core/Controller/BufferBody.cpp

@@ -56,6 +56,23 @@ Controller::beginBufferingBody(Client *client, Request *req) {
 	req->beginStopwatchLog(&req->stopwatchLogs.bufferingRequestBody, "buffering request body");
 }
 
+/**
+ * Relevant when our body data source (bodyChannel) was throttled (by whenBufferingBody_onRequestBody).
+ * Called when our data sink (bodyBuffer) in-memory part is drained and ready for more data.
+ */
+void
+Controller::_bodyBufferFlushed(FileBufferedChannel *channel) {
+	Request *req = static_cast<Request *>(static_cast<
+		ServerKit::BaseHttpRequest *>(channel->getHooks()->userData));
+
+	req->bodyBuffer.clearBuffersFlushedCallback();
+	req->bodyChannel.start();
+}
+
+/**
+ * Receives data (buffer) originating from the bodyChannel, to be passed on to the bodyBuffer.
+ * Backpressure is applied when the bodyBuffer in-memory part exceeds a threshold.
+ */
 ServerKit::Channel::Result
 Controller::whenBufferingBody_onRequestBody(Client *client, Request *req,
 	const MemoryKit::mbuf &buffer, int errcode)
@@ -70,6 +87,15 @@ Controller::whenBufferingBody_onRequestBody(Client *client, Request *req,
 			cEscapeString(StaticString(buffer.start, buffer.size())) <<
 			"\"; " << req->bodyBytesBuffered << " bytes buffered so far");
 		req->bodyBuffer.feed(buffer);
+
+		if (req->bodyBuffer.passedThreshold()) {
+			// Apply backpressure..
+			req->bodyChannel.stop();
+			// ..until the in-memory part of our bodyBuffer is drained.
+			assert(req->bodyBuffer.getBuffersFlushedCallback() == NULL);
+			req->bodyBuffer.setBuffersFlushedCallback(_bodyBufferFlushed);
+		}
+
 		return Channel::Result(buffer.size(), false);
 	} else if (errcode == 0 || errcode == ECONNRESET) {
 		// EOF

data/src/agent/Core/Controller/ForwardResponse.cpp

@@ -1049,7 +1049,7 @@ Controller::outputBuffersFlushed(Client *client, Request *req) {
 	if (!req->ended()) {
 		assert(!req->appSource.isStarted());
 		SKC_TRACE(client, 2, "Buffered response data has been written to disk. Resuming application socket");
-		client->output.setBuffersFlushedCallback(NULL);
+		client->output.clearBuffersFlushedCallback();
 		req->appSource.start();
 	}
 }

data/src/agent/Core/Controller/Hooks.cpp

@@ -112,7 +112,7 @@ Controller::onRequestObjectCreated(Client *client, Request *req) {
 void
 Controller::deinitializeClient(Client *client) {
 	ParentClass::deinitializeClient(client);
-	client->output.setBuffersFlushedCallback(NULL);
+	client->output.clearBuffersFlushedCallback();
 	client->output.setDataFlushedCallback(getClientOutputDataFlushedCallback());
 }
 
@@ -165,6 +165,7 @@ Controller::deinitializeRequest(Client *client, Request *req) {
 	req->appSink.setConsumedCallback(NULL);
 	req->appSink.deinitialize();
 	req->appSource.deinitialize();
+	req->bodyBuffer.clearBuffersFlushedCallback();
 	req->bodyBuffer.deinitialize();
 
 	/***************/

data/src/cxx_supportlib/Constants.h

@@ -55,7 +55,7 @@
 #define DEFAULT_MAX_POOL_SIZE 6
 #define DEFAULT_MAX_PRELOADER_IDLE_TIME 300
 #define DEFAULT_MAX_REQUEST_QUEUE_SIZE 100
-#define DEFAULT_MBUF_CHUNK_SIZE 512
+#define DEFAULT_MBUF_CHUNK_SIZE 4096
 #define DEFAULT_NODEJS "node"
 #define DEFAULT_POOL_IDLE_TIME 300
 #define DEFAULT_PYTHON "python"
@@ -80,7 +80,7 @@
 #define PASSENGER_API_VERSION_MAJOR 0
 #define PASSENGER_API_VERSION_MINOR 3
 #define PASSENGER_DEFAULT_USER "nobody"
-#define PASSENGER_VERSION "5.0.29"
+#define PASSENGER_VERSION "5.0.30"
 #define POOL_HELPER_THREAD_STACK_SIZE 262144
 #define PROCESS_SHUTDOWN_TIMEOUT 60
 #define PROCESS_SHUTDOWN_TIMEOUT_DISPLAY "1 minute"

data/src/cxx_supportlib/ServerKit/FileBufferedChannel.h

@@ -443,7 +443,7 @@ private:
 	 * mode, the reader is responsible for popping buffers. In the in-file mode, the writer
 	 * is responsible for popping buffers (and writing them to the file).
 	 */
-	boost::uint32_t bytesBuffered;
+	boost::uint32_t bytesBuffered; // number of bytes buffered in memory
 	MemoryKit::mbuf firstBuffer;
 	deque<MemoryKit::mbuf> moreBuffers;
 
@@ -1339,12 +1339,18 @@ public:
 	 * buffers. For example, in case of FileBufferedFdSinkChannel, this event does
 	 * not imply that all the in-memory buffers have been written to the sink FD.
 	 * That's what `dataFlushedCallback` is for.
+	 *
+	 * N.B. this class intentionally doesn't manage buffersFlushedCallback in any way,
+	 * the user is responsible for ensuring correctness of the variable's content.
 	 */
 	Callback buffersFlushedCallback;
 	/**
 	 * Called when all buffered data (whether in-memory or on-disk) has been consumed
 	 * by the data callback. In case of FileBufferedFdSinkChannel, this means that all
 	 * buffered data has been written out to the sink FD.
+	 *
+	 * N.B. this class intentionally doesn't manage dataFlushedCallback in any way,
+	 * the user is responsible for ensuring correctness of the variable's content.
 	 */
 	Callback dataFlushedCallback;
 
@@ -1546,6 +1552,11 @@ public:
 		return buffersFlushedCallback;
 	}
 
+	OXT_FORCE_INLINE
+	void clearBuffersFlushedCallback() {
+		buffersFlushedCallback = NULL;
+	}
+
 	OXT_FORCE_INLINE
 	void setBuffersFlushedCallback(Callback callback) {
 		buffersFlushedCallback = callback;

data/src/cxx_supportlib/ServerKit/FileBufferedFdSinkChannel.h

@@ -239,6 +239,11 @@ public:
 		return FileBufferedChannel::getBuffersFlushedCallback();
 	}
 
+	OXT_FORCE_INLINE
+	void clearBuffersFlushedCallback() {
+		FileBufferedChannel::clearBuffersFlushedCallback();
+	}
+
 	OXT_FORCE_INLINE
 	void setBuffersFlushedCallback(Callback callback) {
 		FileBufferedChannel::setBuffersFlushedCallback(callback);

data/src/cxx_supportlib/ServerKit/HttpRequest.h

@@ -122,6 +122,7 @@ public:
 	// headers is variable, but the number of secure headers is more or less
 	// constant.
 	HeaderTable secureHeaders;
+	// HttpServer feeds all body data received via client->input to bodyChannel
 	Channel bodyChannel;
 
 	union {

data/src/ruby_supportlib/phusion_passenger.rb

@@ -31,13 +31,13 @@ module PhusionPassenger
 
   PACKAGE_NAME = 'passenger'
   # Run 'rake src/cxx_supportlib/Constants.h' after changing this number.
-  VERSION_STRING = '5.0.29'
+  VERSION_STRING = '5.0.30'
 
   PREFERRED_NGINX_VERSION = '1.10.1'
   NGINX_SHA256_CHECKSUM = '1fd35846566485e03c0e318989561c135c598323ff349c503a6c14826487a801'
 
   PREFERRED_PCRE_VERSION  = '8.39'
-  PCRE_SHA256_CHECKSUM = '1dd78994c81e44ac41cf30b2a21d4b4cc6d76ccde7fc6e77713ed51d7bddca47'
+  PCRE_SHA256_CHECKSUM = 'ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7'
 
   STANDALONE_INTERFACE_VERSION  = 1
 

data/src/ruby_supportlib/phusion_passenger/constants.rb

@@ -76,7 +76,14 @@ module PhusionPassenger
     MESSAGE_SERVER_MAX_USERNAME_SIZE = 100
     MESSAGE_SERVER_MAX_PASSWORD_SIZE = 100
     POOL_HELPER_THREAD_STACK_SIZE = 1024 * 256
-    DEFAULT_MBUF_CHUNK_SIZE = 16 * 32
+    # Small mbuf sizes avoid memory overhead (up to 1 blocksize per request), but
+    # also introduce context switching and smaller transfer writes. The size is picked 
+    # to balance this out.
+    DEFAULT_MBUF_CHUNK_SIZE = 1024 * 4
+    # Affects input and output buffering (between app and client). Threshold is picked
+    # such that it fits most output (i.e. html page size, not assets), and allows for
+    # high concurrency with low mem overhead. On the upload side there is a penalty 
+    # but there's no real average upload size anyway so we choose mem safety instead. 
     DEFAULT_FILE_BUFFERED_CHANNEL_THRESHOLD = 1024 * 128
     SERVER_KIT_MAX_SERVER_ENDPOINTS = 4
 

data/src/ruby_supportlib/phusion_passenger/utils/unseekable_socket.rb

@@ -275,6 +275,10 @@ module PhusionPassenger
         return exception.instance_variable_get(:"@from_unseekable_socket") == @socket.object_id
       end
 
+      def to_hash
+        { :socket => "Not JSON Encodable", :eof => @simulate_eof }
+      end
+
     private
       def annotate(exception)
         exception.instance_variable_set(:"@from_unseekable_socket", @socket.object_id)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: passenger
 version: !ruby/object:Gem::Version
-  version: 5.0.29
+  version: 5.0.30
 platform: ruby
 authors:
 - Phusion - http://www.phusion.nl/
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-22 00:00:00.000000000 Z
+date: 2016-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake