passenger 4.0.42 → 4.0.43

data/ext/common/Utils.h

@@ -201,7 +201,7 @@ string extractBaseName(const StaticString &path);
  * @throws std::bad_alloc Something went wrong.
  * @ingroup Support
  */
-string escapeForXml(const string &input);
+string escapeForXml(const StaticString &input);
 
 /**
  * Returns the username of the user that the current process is running as.
@@ -385,6 +385,14 @@ void disableMallocDebugging();
  */
 int runShellCommand(const StaticString &command);
 
+/**
+ * Run a command and capture its stdout output.
+ *
+ * @param command The argument to pass to execvp();
+ * @throws SystemException.
+ */
+string runCommandAndCaptureOutput(const char **command);
+
 /**
  * Async-signal safe way to fork().
  *
@@ -395,15 +403,28 @@ int runShellCommand(const StaticString &command);
  * OS X apparently does something similar, except they use a
  * spinlock so it results in 100% CPU. See _cthread_fork_prepare()
  * at http://www.opensource.apple.com/source/Libc/Libc-166/threads.subproj/cthreads.c
+ * However, since POSIX in OS X is implemented on top of a Mach layer,
+ * calling asyncFork() can mess up the state of the Mach layer, causing
+ * some POSIX functions to mysteriously fail. See
+ * https://code.google.com/p/phusion-passenger/issues/detail?id=1094
+ * You should therefore not use asyncFork() unless you're in a signal
+ * handler.
  */
 pid_t asyncFork();
 
 /**
  * Close all file descriptors that are higher than <em>lastToKeepOpen</em>.
- * This function is async-signal safe. But make sure there are no other
- * threads running that might open file descriptors!
+ *
+ * If you set `asyncSignalSafe` to true, then this function becomes fully async-signal,
+ * through the use of asyncFork() instead of fork(). However, read the documentation
+ * for asyncFork() to learn about its caveats.
+ *
+ * Also, regardless of whether `asyncSignalSafe` is true or not, this function is not
+ * *thread* safe. Make sure there are no other threads running that might open file
+ * descriptors, otherwise some file descriptors might not be closed even though they
+ * should be.
  */
-void closeAllFileDescriptors(int lastToKeepOpen);
+void closeAllFileDescriptors(int lastToKeepOpen, bool asyncSignalSafe = false);
 
 /**
  * A no-op, but usually set as a breakpoint in gdb. See CONTRIBUTING.md.

data/ext/common/Utils/AnsiColorConstants.h

@@ -26,6 +26,7 @@
 
 #define ANSI_COLOR_RESET    "\e[0m"
 #define ANSI_COLOR_BOLD     "\e[1m"
+#define ANSI_COLOR_DGRAY    "\e[90m"
 #define ANSI_COLOR_RED      "\e[31m"
 #define ANSI_COLOR_GREEN    "\e[32m"
 #define ANSI_COLOR_YELLOW   "\e[33m"

data/ext/common/Utils/ProcessMetricsCollector.h

@@ -62,6 +62,7 @@
 #include <Utils.h>
 #include <Utils/ScopeGuard.h>
 #include <Utils/IOUtils.h>
+#include <Utils/StringScanning.h>
 
 namespace Passenger {
 
@@ -185,183 +186,18 @@ public:
  * command name, etc.
  */
 class ProcessMetricsCollector {
-public:
-	struct ParseException {};
-	
 private:
 	bool canMeasureRealMemory;
 	string psOutput;
 	
-	/**
-	 * Scan the given data for the first word that appears on the first line.
-	 * Leading whitespaces (but not newlines) are ignored. If a word is found
-	 * then the word is returned and the data pointer is moved to the end of
-	 * the word. Otherwise, a ParseException is thrown.
-	 *
-	 * @post result.size() > 0
-	 */
-	static StaticString readNextWord(const char **data) {
-		// Skip leading whitespaces.
-		while (**data == ' ') {
-			(*data)++;
-		}
-		if (**data == '\n' || **data == '\0') {
-			throw ParseException();
-		}
-		
-		// Find end of word and extract the word.
-		const char *endOfWord = *data;
-		while (*endOfWord != ' ' && *endOfWord != '\n' && *endOfWord != '\0') {
-			endOfWord++;
-		}
-		StaticString result(*data, endOfWord - *data);
-		
-		// Move data pointer to the end of this word.
-		*data = endOfWord;
-		return result;
-	}
-
-	static long long processNextWordAsLongLong(const StaticString &word, char *nullTerminatedWord) {
-		memcpy(nullTerminatedWord, word.c_str(), word.size());
-		nullTerminatedWord[word.size()] = '\0';
-		if (*nullTerminatedWord == '\0') {
-			throw ParseException();
-		} else {
-			return atoll(nullTerminatedWord);
-		}
-	}
-	
-	static long long readNextWordAsLongLong(const char **data) {
-		StaticString word = readNextWord(data);
-		if (word.size() < 50) {
-			char nullTerminatedWord[50];
-			return processNextWordAsLongLong(word, nullTerminatedWord);
-		} else {
-			string nullTerminatedWord(word.size() + 1, '\0');
-			return processNextWordAsLongLong(word, &nullTerminatedWord[0]);
-		}
-	}
-
-	static int processNextWordAsInt(const StaticString &word, char *nullTerminatedWord) {
-		memcpy(nullTerminatedWord, word.c_str(), word.size());
-		nullTerminatedWord[word.size()] = '\0';
-		if (*nullTerminatedWord == '\0') {
-			throw ParseException();
-		} else {
-			return atoi(nullTerminatedWord);
-		}
-	}
-	
-	static int readNextWordAsInt(const char **data) {
-		StaticString word = readNextWord(data);
-		if (word.size() < 50) {
-			char nullTerminatedWord[50];
-			return processNextWordAsInt(word, nullTerminatedWord);
-		} else {
-			string nullTerminatedWord(word.size() + 1, '\0');
-			return processNextWordAsInt(word, &nullTerminatedWord[0]);
-		}
-	}
-	
-	string runCommandAndCaptureOutput(const char **command) const {
-		pid_t pid;
-		int e;
-		Pipe p;
-		
-		p = createPipe();
-		
-		this_thread::disable_syscall_interruption dsi;
-		pid = syscalls::fork();
-		if (pid == 0) {
-			// Make ps nicer, we want to have as little impact on the rest
-			// of the system as possible while collecting the metrics.
-			int prio = getpriority(PRIO_PROCESS, getpid());
-			prio++;
-			if (prio > 20) {
-				prio = 20;
-			}
-			setpriority(PRIO_PROCESS, getpid(), prio);
-			
-			dup2(p[1], 1);
-			close(p[0]);
-			close(p[1]);
-			closeAllFileDescriptors(2);
-			execvp(command[0], (char * const *) command);
-			_exit(1);
-		} else if (pid == -1) {
-			e = errno;
-			throw SystemException("Cannot fork() a new process", e);
-		} else {
-			bool done = false;
-			string result;
-			
-			p[1].close();
-			while (!done) {
-				char buf[1024 * 4];
-				ssize_t ret;
-				
-				try {
-					this_thread::restore_syscall_interruption rsi(dsi);
-					ret = syscalls::read(p[0], buf, sizeof(buf));
-				} catch (const thread_interrupted &) {
-					syscalls::kill(SIGKILL, pid);
-					syscalls::waitpid(pid, NULL, 0);
-					throw;
-				}
-				if (ret == -1) {
-					e = errno;
-					syscalls::kill(SIGKILL, pid);
-					syscalls::waitpid(pid, NULL, 0);
-					throw SystemException("Cannot read output from the 'ps' command", e);
-				}
-				done = ret == 0;
-				result.append(buf, ret);
-			}
-			p[0].close();
-			syscalls::waitpid(pid, NULL, 0);
-			
-			if (result.empty()) {
-				throw RuntimeException("The 'ps' command failed");
-			} else {
-				return result;
-			}
-		}
-	}
-	
-	string readRestOfLine(const char *data) const {
-		// Skip leading whitespaces.
-		while (*data == ' ') {
-			data++;
-		}
-		// Rest of line is allowed to be empty.
-		if (*data == '\n' || *data == '\0') {
-			return "";
-		}
-		
-		// Look for newline character. From there, scan back until we've
-		// found a non-whitespace character.
-		const char *endOfLine = strchr(data, '\n');
-		if (endOfLine == NULL) {
-			throw ParseException();
-		}
-		while (*(endOfLine - 1) == ' ') {
-			endOfLine--;
-		}
-		
-		return string(data, endOfLine - data);
-	}
-	
 	template<typename Collection, typename ConstIterator>
 	ProcessMetricMap parsePsOutput(const string &output, const Collection &allowedPids) const {
 		ProcessMetricMap result;
+		const char *start = output.c_str();
+
 		// Ignore first line, it contains the column names.
-		const char *start = strchr(output.c_str(), '\n');
-		if (start != NULL) {
-			// Skip to beginning of next line.
-			start++;
-			if (*start == '\0') {
-				start = NULL;
-			}
+		if (!skipToNextLine(&start) || *start == '\0') {
+			start = NULL;
 		}
 
 		#ifndef PS_SUPPORTS_MULTIPLE_PIDS
@@ -428,7 +264,7 @@ public:
 	 *
 	 * Returns a map which maps a given PID to its collected metrics.
 	 *
-	 * @throws ProcessMetricsCollector::ParseException
+	 * @throws ParseException The ps output cannot be parsed.
 	 * @throws SystemException
 	 * @throws RuntimeException
 	 */

data/ext/common/Utils/SpeedMeter.h

@@ -0,0 +1,258 @@
+/*
+ *  Phusion Passenger - https://www.phusionpassenger.com/
+ *  Copyright (c) 2014 Phusion
+ *
+ *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to deal
+ *  in the Software without restriction, including without limitation the rights
+ *  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ *  copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in
+ *  all copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ *  THE SOFTWARE.
+ */
+#ifndef _PASSENGER_SPEED_METER_H_
+#define _PASSENGER_SPEED_METER_H_
+
+#include <cstdio>
+#include <cstdlib>
+#include <cassert>
+#include <limits>
+#include <Utils/SystemTime.h>
+
+namespace Passenger {
+
+/**
+ * Utility class which, when periodically fed with quantities, measures the
+ * rate of change in the most recent time period. It does this by storing
+ * samples of the most recent quantities, calculating a weighted average of
+ * the differences between each sample, and extrapolating that over a time
+ * period.
+ *
+ * For example,
+ *
+ *  * When periodically given the number of miles a car has driven so far,
+ *    the average speed of the most recent 10 minutes can be calculated.
+ *  * When periodically given the number of processes the OS has created
+ *    so far, the average fork rate (per minute) of the most recent minute
+ *    can be calculated.
+ *
+ * SpeedMeter is designed to use as little memory as possible, and designed
+ * so that samples can be taken at erratic intervals. This is why it limits
+ * the number of samples that can be stored in memory, and why it puts time
+ * limits on the samples.
+ *
+ *
+ * ## Template parameters
+ *
+ * ### `maxSamples`
+ *
+ * `maxSamples` dictates the sample buffer size: the maximum number of samples
+ * to hold in memory. Defaults to 6.
+ *
+ * The more samples you have, the more historical data influences the results.
+ * If you have few samples then recent data have the most impact on the results.
+ *
+ * ### `minAge`
+ * 
+ * A new sample is only accepted if at least `minAge` microseconds
+ * have passed since the last sample. This is to ensure that the sample
+ * buffer contains enough historical data so that results are not skewed by
+ * adding samples too quickly. A full sample buffer is guaranteed to cover
+ * the last `minAge * maxSamples` microseconds.
+ *
+ * Defaults to 1 second (1 000 000 usec).
+ *
+ * To see why this is useful, consider when `maxSamples` is 5 and `minAge` is 0.
+ * If you measure the number of processes 5 times with 10 ms in between,
+ * then it's unlikely that the OS has spawned any processes in that 50 ms time
+ * interval. However you've now filled the sample buffer with only data from
+ * those 5 recent measurements, so SpeedMeter thinks the rate of change is 0.
+ * If you set `minAge` to, say, 5 seconds, then the sample buffer will be updated
+ * at most once every 5 seconds, guaranteeing that it can retain historical data
+ * for the last 25 seconds.
+ *
+ * `minAge * maxSamples` should be larger than `window`.
+ *
+ * ### `maxAge`
+ *
+ * When calculating the result, samples older than `maxAge` usec ago are ignored.
+ * This value should be larger than `window`. Defaults to 1 minute (60 000 000
+ * usec).
+ *
+ * ### `window`
+ *
+ * When calculating the result, this value dictates the time interval over which
+ * the rate of change should be calculated. Defaults to 1 second (1 000 000 usec),
+ * meaning that by default the result describes the speed per second.
+ *
+ *
+ * ## Interpreting the parameters
+ *
+ * Given parameters, the behavior of SpeedMeter can be interpreted as follows:
+ *
+ * All samples that have been collected in the last `minAge * maxSamples` usecs
+ * (or the last `maxAge` usecs, if samples have been collected slowly), are used for
+ * calculating the speed per `window` usecs.
+ *
+ * Thus, given the default parameter values, we can say:
+ *
+ * All samples that have been collected in the last 8 seconds (the last 1 minute,
+ * if samples have been collected slowly), are used for calculating the speed
+ * per second.
+ */
+template<
+	typename ValueType = double,
+	unsigned int maxSamples = 8,
+	unsigned int minAge = 1000000,
+	unsigned int maxAge = 60 * 1000000,
+	unsigned int window = 1000000
+>
+class SpeedMeter {
+private:
+	struct Sample {
+		unsigned long long timestamp;
+		ValueType val;
+
+		Sample()
+			: timestamp(0),
+			  val(0)
+			{ }
+	};
+
+	unsigned short start, count;
+	Sample samples[maxSamples];
+
+	static int mod(int a, int b) {
+		int r = a % b;
+		return r < 0 ? r + b : r;
+	}
+
+	const Sample &getSample(int index) const {
+		return samples[mod(start + index, maxSamples)];
+	}
+
+	const Sample &getLastSample() const {
+		return getSample(count - 1);
+	}
+
+	unsigned long long getTimeThreshold() const {
+		const unsigned long long currentTime = SystemTime::getUsec();
+		if (currentTime > maxAge) {
+			return currentTime - maxAge;
+		} else {
+			return 0;
+		}
+	}
+
+public:
+	SpeedMeter()
+		: start(0),
+		  count(0)
+		{ }
+
+	bool addSample(ValueType val, unsigned long long timestamp = 0) {
+		if (timestamp == 0) {
+			timestamp = SystemTime::getUsec();
+		}
+
+		const Sample &lastSample = getLastSample();
+		// Timestamp must be larger than that of previous sample.
+		assert(lastSample.timestamp <= timestamp);
+
+		if (lastSample.timestamp <= timestamp - minAge) {
+			Sample &nextSample = samples[(start + count) % maxSamples];
+			nextSample.timestamp = timestamp;
+			nextSample.val = val;
+			if (count == maxSamples) {
+				start = (start + 1) % maxSamples;
+			} else {
+				count++;
+			}
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	/** Number of items in the sample buffer. */
+	unsigned int size() const {
+		return count;
+	}
+
+	/** Current speed over the configured window. Returns NaN if less than 2
+	 * samples have been collected so far.
+	 */
+	double currentSpeed() const {
+		const unsigned long long timeThreshold = getTimeThreshold();
+		int begin = 0, i;
+		unsigned long long interval;
+		double avgWeight = 0, sum = 0;
+
+		// Ignore samples that are too old.
+		while (begin < (int) count - 1
+		    && getSample(begin).timestamp <= timeThreshold)
+		{
+			begin++;
+		}
+
+		// Given the deltas of each sample and the next sample,
+		// calculate the weighted average of all those deltas,
+		// with time interval of each delta as the weight,
+		// and calculate the average weight.
+		for (i = begin; i < (int) count - 1; i++) {
+			const Sample &cur  = getSample(i);
+			const Sample &next = getSample(i + 1);
+			ValueType delta = next.val - cur.val;
+			unsigned long long weight = next.timestamp - cur.timestamp;
+			sum += (double) delta * weight;
+			avgWeight += weight;
+		}
+		avgWeight /= std::max(1, (int) count - 1 - begin);
+
+		interval = getSample(count - 1).timestamp - getSample(begin).timestamp;
+		if (interval > 0) {
+			// sum / interval is the speed per average delta interval,
+			// so we extrapolate that over the entire window interval.
+			return (sum / interval) * (window / avgWeight);
+		} else {
+			return numeric_limits<double>::quiet_NaN();
+		}
+	}
+
+	#if 1
+		void debug() const {
+			const unsigned long long timeThreshold = getTimeThreshold();
+
+			printf("---- begin debug ----\n");
+			printf("timeThreshold: %llu\n", timeThreshold);
+			for (int i = 0; i < (int) maxSamples; i++) {
+				const Sample &sample = samples[i];
+				printf("elem %u, sample %u: timestamp = %llu, val = %.0f%s%s%s\n",
+					i,
+					mod(i - start, maxSamples),
+					sample.timestamp,
+					(double) sample.val,
+					(sample.timestamp <= timeThreshold) ? "  X" : "",
+					(i == start) ? "  <-- start" : "",
+					(count > 0 && i == mod((int) start + count - 1, maxSamples)) ? "  <-- last"   : "");
+			}
+			printf("---- end debug ----\n");
+		}
+	#endif
+};
+
+} // namespace Passenger
+
+#endif /* _PASSENGER_SPEED_METER_H_ */