passenger 5.0.25 → 5.0.26

data/src/agent/Core/SpawningKit/DirectSpawner.h

@@ -181,6 +181,7 @@ public:
 			dup2(errorPipeCopy, 2);
 			closeAllFileDescriptors(2);
 			setChroot(preparation);
+			setUlimits(options);
 			switchUser(preparation);
 			setWorkingDirectory(preparation);
 			execvp(args[0], (char * const *) args.get());

data/src/agent/Core/SpawningKit/SmartSpawner.h

@@ -228,6 +228,7 @@ private:
 			dup2(errorPipeCopy, 2);
 			closeAllFileDescriptors(2);
 			setChroot(preparation);
+			setUlimits(options);
 			switchUser(preparation);
 			setWorkingDirectory(preparation);
 			execvp(command[0].c_str(), (char * const *) args.get());

data/src/agent/Core/SpawningKit/Spawner.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2015 Phusion Holding B.V.
+ *  Copyright (c) 2011-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -971,6 +971,26 @@ protected:
 		}
 	}
 
+	void setUlimits(const Options &options) {
+		if (options.fileDescriptorUlimit != 0) {
+			struct rlimit limit;
+			int ret;
+
+			limit.rlim_cur = options.fileDescriptorUlimit;
+			limit.rlim_max = options.fileDescriptorUlimit;
+			do {
+				ret = setrlimit(RLIMIT_NOFILE, &limit);
+			} while (ret == -1 && errno == EINTR);
+
+			if (ret == -1) {
+				int e = errno;
+				fprintf(stderr, "Unable to set file descriptor ulimit to %u: %s (errno=%d)",
+					options.fileDescriptorUlimit, strerror(e), e);
+				fflush(stderr);
+			}
+		}
+	}
+
 	void setWorkingDirectory(const SpawnPreparationInfo &info) {
 		vector<string>::const_iterator it, end = info.appRootPathsInsideChroot.end();
 		int ret;

data/src/agent/SpawnPreparer/SpawnPreparerMain.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2012-2015 Phusion Holding B.V.
+ *  Copyright (c) 2012-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.

data/src/agent/UstRouter/OptionParser.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2015 Phusion Holding B.V.
+ *  Copyright (c) 2010-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -80,6 +80,9 @@ ustRouterUsage() {
 	printf("      --log-file PATH         Log to the given file.\n");
 	printf("      --log-level LEVEL       Logging level. Default: %d\n", DEFAULT_LOG_LEVEL);
 	printf("\n");
+	printf("      --core-file-descriptor-ulimit NUMBER\n");
+	printf("                              Set custom file descriptor ulimit for the core\n");
+	printf("\n");
 	printf("  -h, --help                  Show this help\n");
 	printf("\n");
 	printf("API account privilege levels (ordered from most to least privileges):\n");
@@ -162,6 +165,9 @@ parseUstRouterOption(int argc, const char *argv[], int &i, VariantMap &options)
 		// the Watchdog, we don't want to affect the Watchdog's own log file.
 		options.set("ust_router_log_file", argv[i + 1]);
 		i += 2;
+	} else if (p.isValueFlag(argc, i, argv[i], '\0', "--core-file-descriptor-ulimit")) {
+		options.setUint("core_file_descriptor_ulimit", atoi(argv[i + 1]));
+		i += 2;
 	} else {
 		return false;
 	}

data/src/agent/UstRouter/UstRouterMain.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2015 Phusion Holding B.V.
+ *  Copyright (c) 2010-2016 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -31,6 +31,7 @@
 #include <oxt/thread.hpp>
 
 #include <sys/types.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <pwd.h>
 #include <grp.h>
@@ -188,6 +189,30 @@ initializePrivilegedWorkingObjects() {
 	wo->resourceLocator = new ResourceLocator(options.get("passenger_root"));
 }
 
+static void
+setUlimits() {
+	TRACE_POINT();
+	VariantMap &options = *agentsOptions;
+
+	if (options.has("core_file_descriptor_ulimit")) {
+		unsigned int number = options.getUint("core_file_descriptor_ulimit");
+		struct rlimit limit;
+		int ret;
+
+		limit.rlim_cur = number;
+		limit.rlim_max = number;
+		do {
+			ret = setrlimit(RLIMIT_NOFILE, &limit);
+		} while (ret == -1 && errno == EINTR);
+
+		if (ret == -1) {
+			int e = errno;
+			P_ERROR("Unable to set file descriptor ulimit to " << number
+				<< ": " << strerror(e) << " (errno=" << e << ")");
+		}
+	}
+}
+
 static void
 startListening() {
 	TRACE_POINT();
@@ -506,6 +531,7 @@ runUstRouter() {
 	try {
 		UPDATE_TRACE_POINT();
 		initializePrivilegedWorkingObjects();
+		setUlimits();
 		startListening();
 		lowerPrivilege();
 		initializeUnprivilegedWorkingObjects();

data/src/cxx_supportlib/Algorithms/MovingAverage.h

@@ -0,0 +1,223 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2016 Phusion Holding B.V.
+ *
+ *  "Passenger", "Phusion Passenger" and "Union Station" are registered
+ *  trademarks of Phusion Holding B.V.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#ifndef _PASSENGER_ALGORITHMS_EXP_MOVING_AVERAGE_H_
+#define _PASSENGER_ALGORITHMS_EXP_MOVING_AVERAGE_H_
+
+#include <oxt/macros.hpp>
+#include <boost/config.hpp>
+#include <algorithm>
+#include <utility>
+#include <cmath>
+
+namespace Passenger {
+
+using namespace std;
+
+
+/**
+ * Implements discontiguous exponential moving averaging, as described by John C. Gunther
+ * 1998. Can be used to compute moving exponentially decaying averages and standard
+ * deviations. Unlike normal exponential moving average, this algorithm also works when
+ * the data has gaps, and it also avoids initial value bias and postgap bias. See
+ * http://www.drdobbs.com/tools/discontiguous-exponential-averaging/184410671
+ *
+ * ## Template parameters
+ *
+ * ### alpha
+ *
+ * Specifies by what factor data should decay. Its range is [0, 1000]. Higher values
+ * cause the current value to have more weight (and thus the previous average
+ * to decay more quickly), lower values have the opposite effect.
+ *
+ * ### alphaTimeUnit
+ *
+ * Specifies the time, in microseconds, after which the data should decay
+ * by a factor of exactly `alpha`. For example, if `alpha = 0.5` and `alphaTimeUnit = 2000000`,
+ * then data decays by 0.5 per 2 seconds.
+ *
+ * The default value is 1 second.
+ *
+ * ### maxAge
+ *
+ * Represents an educational guess as to how long (in microseconds) it takes
+ * for the sampled data sequence to change significantly. If you don't expect large random
+ * variations then you should set this to a large value. For a data sequence dominated by
+ * large random variations, setting this to 1000000 (1 second) might be appropriate.
+ *
+ * If the time interval between updates is `dt`, using a `maxAge` of `N * dt` will cause
+ * each update to fill in up to `N - 1` of any preceeding skipped updates with the current
+ * data value.
+ */
+template<
+	unsigned int alpha,
+	unsigned long long alphaTimeUnit = 1000000,
+	unsigned long long maxAge = 1000000
+>
+class DiscExpMovingAverage {
+private:
+	template<unsigned int, unsigned long long, unsigned long long>
+	friend class DiscExpMovingAverageWithStddev;
+
+	double sumOfWeights, sumOfData;
+	unsigned long long prevTime;
+
+	static BOOST_CONSTEXPR double floatingAlpha() {
+		return alpha / 1000.0;
+	}
+
+	static BOOST_CONSTEXPR double newDataWeightUpperBound() {
+		return pow(floatingAlpha(), maxAge / (double) alphaTimeUnit);
+	}
+
+	pair<double, double> internalUpdate(double value, unsigned long long now) {
+		double weightReductionFactor = pow(1 - floatingAlpha(),
+			(now - prevTime) / (double) alphaTimeUnit);
+		double newDataWeight = std::min(1 - weightReductionFactor,
+			newDataWeightUpperBound());
+		sumOfWeights = weightReductionFactor * sumOfWeights + newDataWeight;
+		sumOfData = weightReductionFactor * sumOfData + newDataWeight * value;
+		prevTime = now;
+		return make_pair(weightReductionFactor, newDataWeight);
+	}
+
+public:
+	DiscExpMovingAverage(unsigned long long _prevTime = 0)
+		: sumOfWeights(0),
+		  sumOfData(0),
+		  prevTime(_prevTime)
+		{ }
+
+	void update(double value, unsigned long long now) {
+		if (OXT_LIKELY(now > prevTime)) {
+			internalUpdate(value, now);
+		}
+	}
+
+	bool available() const {
+		return sumOfWeights > 0;
+	}
+
+	double completeness(unsigned long long now) const {
+		return pow(floatingAlpha(), now - prevTime) * sumOfWeights;
+	}
+
+	double average() const {
+		return sumOfData / sumOfWeights;
+	}
+
+	double average(unsigned long long now) const {
+		DiscExpMovingAverage<alpha, alphaTimeUnit, maxAge> copy(*this);
+		copy.update(0, now);
+		return copy.average();
+	}
+};
+
+
+/**
+ * Like DescExpMovingAverage, but also keeps track of the standard deviation.
+ */
+template<
+	unsigned int alpha,
+	unsigned long long alphaTimeUnit = 1000000,
+	unsigned long long maxAge = 1
+>
+class DiscExpMovingAverageWithStddev {
+private:
+	DiscExpMovingAverage<alpha, alphaTimeUnit, maxAge> dema;
+	double sumOfSquaredData;
+
+public:
+	DiscExpMovingAverageWithStddev(unsigned long long prevTime = 0)
+		: dema(prevTime),
+		  sumOfSquaredData(0)
+		{ }
+
+	void update(double value, unsigned long long now) {
+		if (OXT_UNLIKELY(now <= dema.prevTime)) {
+			return;
+		}
+
+		pair<double, double> p = dema.internalUpdate(value, now);
+		double weightReductionFactor = p.first;
+		double newDataWeight = p.second;
+		sumOfSquaredData = weightReductionFactor * sumOfSquaredData
+			+ newDataWeight * pow(value, 2.0);
+	}
+
+	bool available() const {
+		return dema.available();
+	}
+
+	double completeness(unsigned long long now) const {
+		return dema.completeness(now);
+	}
+
+	double average() const {
+		return dema.average();
+	}
+
+	double average(unsigned long long now) const {
+		return dema.average(now);
+	}
+
+	double stddev() const {
+		return sqrt(sumOfSquaredData / dema.sumOfWeights - pow(average(), 2));
+	}
+
+	double stddev(unsigned long long now) const {
+		DiscExpMovingAverageWithStddev<alpha, alphaTimeUnit, maxAge> copy(*this);
+		copy.update(0, now);
+		return sqrt(copy.sumOfSquaredData / copy.sumOfWeights - pow(copy.average(), 2));
+	}
+};
+
+
+/**
+ * Calculates an exponential moving average. `alpha` determines how much weight the
+ * current value has compared to the previous average. Higher values of `alpha`
+ * cause the current value to have more weight (and thus the previous average
+ * to decay more quickly), lower values have the opposite effect.
+ *
+ * This algorithm is not timing sensitive: it doesn't take into account gaps in the
+ * data over time, and treats all values equally regardless of when the value was
+ * collected. See also DiscExpMovingAverage.
+ *
+ * You should initialize the the average value with a value equal to `nullValue`.
+ * If `prevAverage` equals `nullValue` then this function simply returns `currentValue`.
+ */
+inline double
+expMovingAverage(double prevAverage, double currentValue, double alpha, double nullValue = -1) {
+	if (OXT_UNLIKELY(prevAverage == nullValue)) {
+		return currentValue;
+	} else {
+		return alpha * currentValue + (1 - alpha) * prevAverage;
+	}
+}
+
+
+} // namespace Passenger
+
+#endif /* _PASSENGER_ALGORITHMS_EXP_MOVING_AVERAGE_H_ */

data/src/cxx_supportlib/Constants.h

@@ -83,7 +83,7 @@
 
 	#define DEFAULT_RUBY "ruby"
 
-	#define DEFAULT_SOCKET_BACKLOG 1024
+	#define DEFAULT_SOCKET_BACKLOG 2048
 
 	#define DEFAULT_SPAWN_METHOD "smart"
 
@@ -121,7 +121,7 @@
 
 	#define PASSENGER_DEFAULT_USER "nobody"
 
-	#define PASSENGER_VERSION "5.0.25"
+	#define PASSENGER_VERSION "5.0.26"
 
 	#define POOL_HELPER_THREAD_STACK_SIZE 262144
 

data/src/cxx_supportlib/DataStructures/StringKeyTable.h

@@ -26,6 +26,7 @@
 #ifndef _PASSENGER_DATA_STRUCTURES_STRING_KEY_TABLE_H_
 #define _PASSENGER_DATA_STRUCTURES_STRING_KEY_TABLE_H_
 
+#include <boost/move/move.hpp>
 #include <boost/cstdint.hpp>
 #include <limits>
 #include <cstring>
@@ -39,6 +40,11 @@ namespace Passenger {
 
 using namespace std;
 
+
+struct SKT_EnableMoveSupport { };
+struct SKT_DisableMoveSupport { };
+
+
 /**
  * An optimized hash table that accepts string keys, optimized for the following workload:
  *
@@ -63,7 +69,7 @@ using namespace std;
  * This implementation is based on https://github.com/preshing/CompareIntegerMaps.
  * See also http://preshing.com/20130107/this-hash-table-is-faster-than-a-judy-array
  */
-template<typename T>
+template<typename T, typename MoveSupport = SKT_DisableMoveSupport>
 class StringKeyTable {
 public:
 	#define SKT_FIRST_CELL(hash) (m_cells + ((hash) & (m_arraySize - 1)))
@@ -88,6 +94,16 @@ public:
 		Cell()
 			: keyOffset(EMPTY_CELL_KEY_OFFSET)
 			{ }
+
+		void move(Cell &target) {
+			target.keyOffset = keyOffset;
+			target.keyLength = keyLength;
+			target.hash = hash;
+			target.value = boost::move(value);
+			keyOffset = 0;
+			keyLength = 0;
+			hash = 0;
+		}
 	};
 
 private:
@@ -181,7 +197,7 @@ private:
 				while (true) {
 					if (cellIsEmpty(newCell)) {
 						// Insert here
-						*newCell = *oldCell;
+						copyOrMoveCell(*oldCell, *newCell, MoveSupport());
 						break;
 					} else {
 						newCell = SKT_CIRCULAR_NEXT(newCell);
@@ -194,7 +210,23 @@ private:
 		delete[] oldCells;
 	}
 
-	void copyFrom(const StringKeyTable &other) {
+	void copyOrMoveCell(Cell &source, Cell &target, const SKT_EnableMoveSupport &t) {
+		source.move(target);
+	}
+
+	void copyOrMoveCell(Cell &source, Cell &target, const SKT_DisableMoveSupport &t) {
+		target = source;
+	}
+
+	void copyOrMoveValue(T &source, T &target, const SKT_EnableMoveSupport &t) {
+		target = boost::move(source);
+	}
+
+	void copyOrMoveValue(const T &source, T &target, const SKT_DisableMoveSupport &t) {
+		target = source;
+	}
+
+	void copyTableFrom(const StringKeyTable &other) {
 		m_arraySize  = other.m_arraySize;
 		m_population = other.m_population;
 		m_cells      = new Cell[other.m_arraySize];
@@ -212,13 +244,54 @@ private:
 		}
 	}
 
+	template<typename ValueType, typename LocalMoveSupport>
+	void realInsert(const HashedStaticString &key, ValueType val, bool overwrite) {
+		assert(!key.empty());
+		assert(key.size() <= MAX_KEY_LENGTH);
+		assert(m_population < MAX_ITEMS);
+
+		if (OXT_UNLIKELY(m_cells == NULL)) {
+			init(DEFAULT_SIZE, DEFAULT_STORAGE_SIZE);
+		}
+
+		while (true) {
+			Cell *cell = SKT_FIRST_CELL(key.hash());
+			while (true) {
+				const char *cellKey = lookupCellKey(cell);
+				if (cellKey == NULL) {
+					// Cell is empty. Insert here.
+					if (shouldRepopulateOnInsert()) {
+						// Time to resize
+						repopulate(m_arraySize * 2);
+						break;
+					}
+					m_population++;
+					cell->keyOffset = appendToStorage(key);
+					cell->keyLength = key.size();
+					cell->hash = key.hash();
+					copyOrMoveValue(val, cell->value, LocalMoveSupport());
+					nonEmptyIndex = cell - &m_cells[0];
+					return;
+				} else if (compareKeys(cellKey, cell->keyLength, key)) {
+					// Cell matches.
+					if (overwrite) {
+						copyOrMoveValue(val, cell->value, LocalMoveSupport());
+					}
+					return;
+				} else {
+					cell = SKT_CIRCULAR_NEXT(cell);
+				}
+			}
+		}
+	}
+
 public:
 	StringKeyTable(unsigned int initialSize = DEFAULT_SIZE, unsigned int initialStorageSize = DEFAULT_STORAGE_SIZE) {
 		init(initialSize, initialStorageSize);
 	}
 
 	StringKeyTable(const StringKeyTable &other) {
-		copyFrom(other);
+		copyTableFrom(other);
 	}
 
 	~StringKeyTable() {
@@ -227,14 +300,17 @@ public:
 	}
 
 	StringKeyTable &operator=(const StringKeyTable &other) {
-		delete[] m_cells;
-		free(m_storage);
-		copyFrom(other);
+		if (this != &other) {
+			delete[] m_cells;
+			free(m_storage);
+			copyTableFrom(other);
+		}
 		return *this;
 	}
 
 	void init(unsigned int initialSize, unsigned int initialStorageSize) {
 		assert((initialSize & (initialSize - 1)) == 0);   // Must be a power of 2
+		assert((initialSize == 0) == (initialStorageSize == 0));
 
 		nonEmptyIndex = NON_EMPTY_INDEX_NONE;
 
@@ -314,7 +390,7 @@ public:
 
 	OXT_FORCE_INLINE
 	bool lookup(const HashedStaticString &key, T **result) {
-		return static_cast<const StringKeyTable<T> *>(this)->lookup(key,
+		return static_cast<const StringKeyTable<T, MoveSupport> *>(this)->lookup(key,
 			const_cast<const T **>(result));
 	}
 
@@ -355,45 +431,21 @@ public:
 	}
 
 	void insert(const HashedStaticString &key, const T &val, bool overwrite = true) {
-		assert(!key.empty());
-		assert(key.size() <= MAX_KEY_LENGTH);
-		assert(m_population < MAX_ITEMS);
+		realInsert<const T &, SKT_DisableMoveSupport>(key, val, overwrite);
+	}
 
-		while (true) {
-			Cell *cell = SKT_FIRST_CELL(key.hash());
-			while (true) {
-				const char *cellKey = lookupCellKey(cell);
-				if (cellKey == NULL) {
-					// Cell is empty. Insert here.
-					if (shouldRepopulateOnInsert()) {
-						// Time to resize
-						repopulate(m_arraySize * 2);
-						break;
-					}
-					m_population++;
-					cell->keyOffset = appendToStorage(key);
-					cell->keyLength = key.size();
-					cell->hash = key.hash();
-					cell->value = val;
-					nonEmptyIndex = cell - &m_cells[0];
-					return;
-				} else if (compareKeys(cellKey, cell->keyLength, key)) {
-					// Cell matches.
-					if (overwrite) {
-						cell->value = val;
-					}
-					return;
-				} else {
-					cell = SKT_CIRCULAR_NEXT(cell);
-				}
-			}
-		}
+	void insertByMoving(const HashedStaticString &key, BOOST_RV_REF(T) val, bool overwrite = true) {
+		realInsert<BOOST_RV_REF(T), SKT_EnableMoveSupport>(key, boost::move(val), overwrite);
 	}
 
 	void erase(Cell *cell) {
 		assert(cell >= m_cells && cell - m_cells < m_arraySize);
 		assert(!cellIsEmpty(cell));
 
+		if (OXT_UNLIKELY(m_cells == NULL)) {
+			return;
+		}
+
 		// Remove this cell by shuffling neighboring cells so there are no gaps in anyone's probe chain
 		Cell *neighbor = SKT_CIRCULAR_NEXT(cell);
 		while (true) {
@@ -433,6 +485,10 @@ public:
 
 	/** Does not resize the array. */
 	void clear() {
+		if (OXT_UNLIKELY(m_cells == NULL)) {
+			return;
+		}
+
 		for (unsigned int i = 0; i < m_arraySize; i++) {
 			m_cells[i].keyOffset = EMPTY_CELL_KEY_OFFSET;
 			m_cells[i].value = T();