passenger 5.2.1 → 5.2.2

data/src/cxx_supportlib/BackgroundEventLoop.cpp

@@ -167,7 +167,7 @@ pollLibuv(BackgroundEventLoop *bg) {
 
 	fd = uv_backend_fd(&bg->priv->libuv_loop);
 
-	while (!this_thread::interruption_requested()) {
+	while (!boost::this_thread::interruption_requested()) {
 		timeout = uv_backend_timeout(&bg->priv->libuv_loop);
 
 		ctx->syscall_interruption_lock.unlock();
@@ -203,13 +203,13 @@ pollLibuv(BackgroundEventLoop *bg) {
 		} while (ret == -1
 			&& lastErrno == EINTR
 			&& (!boost::this_thread::syscalls_interruptable()
-				|| !(intrRequested = this_thread::interruption_requested())));
+				|| !(intrRequested = boost::this_thread::interruption_requested())));
 
 		ctx->syscall_interruption_lock.lock();
 
 		if (ret == -1
 			&& lastErrno == EINTR
-			&& this_thread::syscalls_interruptable()
+			&& boost::this_thread::syscalls_interruptable()
 			&& intrRequested)
 		{
 			throw boost::thread_interrupted();

data/src/cxx_supportlib/ConfigKit/Schema.h

@@ -74,44 +74,65 @@ public:
 			  inspectFilter(_inspectFilter)
 			{ }
 
-		Json::Value typecastValue(const Json::Value &val) const {
+		bool tryTypecastValue(const Json::Value &val, Json::Value &result) const {
 			if (val.isNull()) {
-				return Json::nullValue;
+				result = Json::nullValue;
+				return true;
 			}
 
 			switch (type) {
 			case STRING_TYPE:
-				if (val.isString()) {
-					return val;
+				if (val.isConvertibleTo(Json::stringValue)) {
+					result = val.asString();
+					return true;
 				} else {
-					return val.asString();
+					return false;
 				}
 			case INT_TYPE:
-				if (val.isInt()) {
-					return val;
+				if (val.isConvertibleTo(Json::intValue)) {
+					result = val.asInt();
+					return true;
 				} else {
-					return val.asInt();
+					return false;
 				}
 			case UINT_TYPE:
-				if (val.isUInt()) {
-					return val;
+				if (val.isConvertibleTo(Json::uintValue)) {
+					result = val.asUInt();
+					return true;
 				} else {
-					return val.asUInt();
+					return false;
 				}
 			case FLOAT_TYPE:
-				if (val.isDouble()) {
-					return val;
+				if (val.isConvertibleTo(Json::realValue)) {
+					result = val.asDouble();
+					return true;
 				} else {
-					return val.asDouble();
+					return false;
 				}
 			case BOOL_TYPE:
-				if (val.isBool()) {
-					return val;
+				if (val.isConvertibleTo(Json::booleanValue)) {
+					result = val.asBool();
+					return true;
 				} else {
-					return val.asBool();
+					return false;
+				}
+			case ARRAY_TYPE:
+				if (val.isConvertibleTo(Json::arrayValue)) {
+					result = val;
+					return true;
+				} else {
+					return false;
+				}
+			case OBJECT_TYPE:
+				if (val.isConvertibleTo(Json::objectValue)) {
+					result = val;
+					return true;
+				} else {
+					return false;
 				}
 			default:
-				return val;
+				result = val;
+				return true;
 			}
 		}
 
@@ -321,10 +342,11 @@ public:
 	 * configuration store -- to the given configuration key and value.
 	 * Validators added with `addValidator()` won't be applied.
 	 *
-	 * Returns whether validation passed. If not, then `error` is set.
+	 * Returns whether validation passed. If not, then an Error is appended
+	 * to `errors`.
 	 */
 	bool validateValue(const HashedStaticString &key, const Json::Value &value,
-		Error &error) const
+		vector<Error> &errors) const
 	{
 		const Entry *entry;
 
@@ -335,7 +357,7 @@ public:
 
 		if (value.isNull()) {
 			if (entry->flags & REQUIRED) {
-				error = Error("'{{" + key + "}}' is required");
+				errors.push_back(Error("'{{" + key + "}}' is required"));
 				return false;
 			} else {
 				return true;
@@ -347,14 +369,14 @@ public:
 			if (value.isConvertibleTo(Json::stringValue)) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be a string");
+				errors.push_back(Error("'{{" + key + "}}' must be a string"));
 				return false;
 			}
 		case INT_TYPE:
 			if (value.isConvertibleTo(Json::intValue)) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be an integer");
+				errors.push_back(Error("'{{" + key + "}}' must be an integer"));
 				return false;
 			}
 		case UINT_TYPE:
@@ -362,32 +384,32 @@ public:
 				if (value.isConvertibleTo(Json::uintValue)) {
 					return true;
 				} else {
-					error = Error("'{{" + key + "}}' must be greater than 0");
+					errors.push_back(Error("'{{" + key + "}}' must be greater than 0"));
 					return false;
 				}
 			} else {
-				error = Error("'{{" + key + "}}' must be an integer");
+				errors.push_back(Error("'{{" + key + "}}' must be an integer"));
 				return false;
 			}
 		case FLOAT_TYPE:
 			if (value.isConvertibleTo(Json::realValue)) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be a number");
+				errors.push_back(Error("'{{" + key + "}}' must be a number"));
 				return false;
 			}
 		case BOOL_TYPE:
 			if (value.isConvertibleTo(Json::booleanValue)) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be a boolean");
+				errors.push_back(Error("'{{" + key + "}}' must be a boolean"));
 				return false;
 			}
 		case ARRAY_TYPE:
 			if (value.isConvertibleTo(Json::arrayValue)) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be an array");
+				errors.push_back(Error("'{{" + key + "}}' must be an array"));
 				return false;
 			}
 		case STRING_ARRAY_TYPE:
@@ -395,20 +417,20 @@ public:
 				Json::Value::const_iterator it, end = value.end();
 				for (it = value.begin(); it != end; it++) {
 					if (it->type() != Json::stringValue) {
-						error = Error("'{{" + key + "}}' may only contain strings");
+						errors.push_back(Error("'{{" + key + "}}' may only contain strings"));
 						return false;
 					}
 				}
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be an array");
+				errors.push_back(Error("'{{" + key + "}}' must be an array"));
 				return false;
 			}
 		case OBJECT_TYPE:
 			if (value.isObject()) {
 				return true;
 			} else {
-				error = Error("'{{" + key + "}}' must be a JSON object");
+				errors.push_back(Error("'{{" + key + "}}' must be a JSON object"));
 				return false;
 			}
 		case ANY_TYPE:

data/src/cxx_supportlib/ConfigKit/Store.h

@@ -385,14 +385,17 @@ public:
 		StringKeyTable<Entry>::Iterator p_it(storeWithPreviewData.entries);
 		StringKeyTable<Entry>::ConstIterator it(entries);
 		vector<Error> tmpErrors;
-		Error error;
 
 		while (*p_it != NULL) {
 			const HashedStaticString &key = p_it.getKey();
 			Entry &entry = p_it.getValue();
 
 			if (isWritable(entry) && updates.isMember(key)) {
-				entry.userValue = updates[key];
+				bool ok = entry.schemaEntry->tryTypecastValue(
+					updates[key], entry.userValue);
+				if (!ok) {
+					entry.userValue = updates[key];
+				}
 			}
 
 			p_it.next();
@@ -406,7 +409,11 @@ public:
 			entry.schemaEntry->inspect(subdoc);
 
 			if (isWritable(entry) && updates.isMember(key)) {
-				subdoc["user_value"] = updates[key];
+				bool ok = entry.schemaEntry->tryTypecastValue(updates[key],
+					subdoc["user_value"]);
+				if (!ok) {
+					subdoc["user_value"] = updates[key];
+				}
 			} else {
 				subdoc["user_value"] = entry.userValue;
 			}
@@ -419,9 +426,7 @@ public:
 				subdoc["effective_value"] =
 					getEffectiveValue(subdoc["user_value"],
 						subdoc["default_value"]);
-			if (!schema->validateValue(it.getKey(), effectiveValue, error)) {
-				tmpErrors.push_back(error);
-			}
+			schema->validateValue(it.getKey(), effectiveValue, tmpErrors);
 
 			result[it.getKey()] = subdoc;
 			it.next();
@@ -469,8 +474,7 @@ public:
 				if (isWritable(entry)) {
 					const Json::Value &subdoc =
 						const_cast<const Json::Value &>(preview)[it.getKey()];
-					entry.userValue = entry.schemaEntry->typecastValue(
-						subdoc["user_value"]);
+					entry.userValue = subdoc["user_value"];
 				}
 				it.next();
 			}

data/src/cxx_supportlib/Constants.h

@@ -74,13 +74,14 @@
 #define FEEDBACK_FD 3
 #define FLYING_PASSENGER_NAME "Flying Passenger"
 #define GLOBAL_NAMESPACE_DIRNAME "passenger"
+#define LOG_MONITORING_MAX_LINES 200
 #define MESSAGE_SERVER_MAX_PASSWORD_SIZE 100
 #define MESSAGE_SERVER_MAX_USERNAME_SIZE 100
 #define PASSENGER_API_VERSION "0.3"
 #define PASSENGER_API_VERSION_MAJOR 0
 #define PASSENGER_API_VERSION_MINOR 3
 #define PASSENGER_DEFAULT_USER "nobody"
-#define PASSENGER_VERSION "5.2.1"
+#define PASSENGER_VERSION "5.2.2"
 #define POOL_HELPER_THREAD_STACK_SIZE 262144
 #define PROCESS_SHUTDOWN_TIMEOUT 60
 #define PROCESS_SHUTDOWN_TIMEOUT_DISPLAY "1 minute"

data/src/cxx_supportlib/DataStructures/StringKeyTable.h

@@ -386,6 +386,10 @@ public:
 		}
 	}
 
+	bool contains(const HashedStaticString &key) const {
+		return (lookupCell(key) != NULL);
+	}
+
 	bool lookup(const HashedStaticString &key, const T **result) const {
 		const Cell * const cell = lookupCell(key);
 		if (cell != NULL) {

data/src/cxx_supportlib/FileTools/PathManipCBindings.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2017 Phusion Holding B.V.
+ *  Copyright (c) 2010-2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -26,7 +26,9 @@
 
 #include <FileTools/PathManipCBindings.h>
 #include <FileTools/PathManip.h>
+#include <Exceptions.h>
 #include <cstring>
+#include <cerrno>
 
 using namespace std;
 using namespace Passenger;
@@ -47,6 +49,25 @@ psg_absolutize_path(const char *path, size_t path_len,
 	return strdup(result.c_str());
 }
 
+char *
+psg_resolve_symlink(const char *path, size_t path_len,
+	size_t *result_len)
+{
+	try {
+		string result = resolveSymlink(StaticString(path, path_len));
+		if (result_len != NULL) {
+			*result_len = result.size();
+		}
+		return strdup(result.c_str());
+	} catch (const SystemException &e) {
+		errno = e.code();
+		return NULL;
+	} catch (const std::bad_alloc &) {
+		errno = ENOMEM;
+		return NULL;
+	}
+}
+
 const char *
 psg_extract_dir_name_static(const char *path,
 	size_t path_len, size_t *result_len)

data/src/cxx_supportlib/FileTools/PathManipCBindings.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2017 Phusion Holding B.V.
+ *  Copyright (c) 2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -36,6 +36,8 @@ extern "C" {
 char *psg_absolutize_path(const char *path, size_t path_len,
 	const char *working_dir, size_t working_dir_len,
 	size_t *result_len);
+char *psg_resolve_symlink(const char *path, size_t path_len,
+	size_t *result_len);
 const char *psg_extract_dir_name_static(const char *path,
 	size_t path_len, size_t *result_len);
 

data/src/cxx_supportlib/LoggingKit/Config.h

@@ -50,6 +50,7 @@ using namespace std;
  * by 'rake configkit_schemas_inline_comments')
  *
  *   app_output_log_level         string    -   default("notice")
+ *   buffer_logs                  boolean   -   default(false)
  *   file_descriptor_log_target   any       -   -
  *   level                        string    -   default("notice")
  *   redirect_stderr              boolean   -   default(true)
@@ -82,6 +83,7 @@ struct ConfigRealization {
 	TargetType targetType;
 	TargetType fileDescriptorLogTargetType;
 	int targetFd;
+	bool saveLog;
 	int fileDescriptorLogTargetFd;
 	FdClosePolicy targetFdClosePolicy;
 	FdClosePolicy fileDescriptorLogTargetFdClosePolicy;

data/src/cxx_supportlib/LoggingKit/Context.h

@@ -28,14 +28,17 @@
 
 #include <queue>
 
+#include <jsoncpp/json.h>
 #include <oxt/macros.hpp>
 #include <oxt/thread.hpp>
+#include <boost/circular_buffer.hpp>
 #include <boost/thread.hpp>
 #include <boost/atomic.hpp>
 #include <ConfigKit/ConfigKit.h>
 #include <LoggingKit/Forward.h>
 #include <LoggingKit/Config.h>
 #include <Utils/SystemTime.h>
+#include <DataStructures/StringKeyTable.h>
 
 namespace Passenger {
 namespace LoggingKit {
@@ -78,12 +81,37 @@ private:
 	queue< pair<ConfigRealization *, MonotonicTimeUsec> > oldConfigs;
 	bool shuttingDown;
 
+	struct TimestampedLog {
+		// time at which time the log entered the core, which is unfortunately somewhat
+		// arbitrarily later than that it was logged in the user program
+		unsigned long long timestamp;
+		string sourceId;
+		string lineText;
+	};
+	typedef boost::circular_buffer<TimestampedLog> TimestampedLogBuffer;
+
+	typedef boost::circular_buffer<string> SimpleLogBuffer;
+	typedef StringKeyTable<SimpleLogBuffer> SimpleLogMap;
+	struct AppGroupLog {
+		TimestampedLogBuffer pidLog; // combined logs from PIDs
+		SimpleLogMap watchFileLog; // a separate log buffer per (watched file name)
+	};
+	typedef StringKeyTable<AppGroupLog> LogStore;
+	LogStore logStore;
+
 public:
 	Context(const Json::Value &initialConfig = Json::Value(),
 		const ConfigKit::Translator &translator = ConfigKit::DummyTranslator());
 	~Context();
 	ConfigKit::Store getConfig() const;
 
+	// specifically for logging output from application processes
+	void saveNewLog(const HashedStaticString &groupName, const char *sourceStr, unsigned int sourceStrLen, const char *message, unsigned int messageLen);
+	// the message might already have been logged (e.g. because the source is a `tail -n`)
+	void updateLog(const HashedStaticString &groupName, const char *sourceStr, unsigned int sourceStrLen, const char *message, unsigned int messageLen);
+	// snapshot logStore to a JSON structure for external relay
+	Json::Value convertLog();
+
 	bool prepareConfigChange(const Json::Value &updates,
 		vector<ConfigKit::Error> &errors,
 		LoggingKit::ConfigChangeRequest &req);

data/src/cxx_supportlib/LoggingKit/Forward.h

@@ -73,7 +73,6 @@ enum TargetType {
 
 extern Context *context;
 
-
 void shutdown();
 
 const char *_strdupFastStringStream(const FastStringStream<> &stream);

data/src/cxx_supportlib/LoggingKit/Implementation.cpp

@@ -41,7 +41,8 @@
 #include <pthread.h>
 
 #include <boost/cstdint.hpp>
-#include <oxt/system_calls.hpp>
+#include <boost/circular_buffer.hpp>
+#include <boost/foreach.hpp>
 #include <oxt/thread.hpp>
 #include <oxt/detail/context.hpp>
 
@@ -69,7 +70,6 @@ using namespace std;
 Context *context = NULL;
 AssertionFailureInfo lastAssertionFailure;
 
-
 void
 initialize(const Json::Value &initialConfig, const ConfigKit::Translator &translator) {
 	context = new Context(initialConfig, translator);
@@ -309,11 +309,94 @@ _writeFileDescriptorLogEntry(const ConfigRealization *configRealization,
 	writeExactWithoutOXT(configRealization->fileDescriptorLogTargetFd, str, size);
 }
 
+void
+Context::saveNewLog(const HashedStaticString &groupName, const char *sourceStr, unsigned int sourceStrLen, const char *message, unsigned int messageLen) {
+	boost::lock_guard<boost::mutex> l(syncher); //lock
+
+	unsigned long long timestamp = SystemTime::getUsec();
+
+	LogStore::Cell *c = logStore.lookupCell(groupName);
+	if (c == NULL) {
+		AppGroupLog appGroupLog;
+		appGroupLog.pidLog = TimestampedLogBuffer(LOG_MONITORING_MAX_LINES * 5);
+		c = logStore.insert(groupName, appGroupLog);
+	}
+	AppGroupLog &rec = c->value;
+
+	TimestampedLog ll;
+	ll.timestamp = timestamp;
+	ll.sourceId = string(sourceStr, sourceStrLen);
+	ll.lineText = string(message, messageLen);
+	rec.pidLog.push_back(ll);
+	//unlock
+}
+
+void
+Context::updateLog(const HashedStaticString &groupName, const char *sourceStr, unsigned int sourceStrLen, const char *message, unsigned int messageLen) {
+	boost::lock_guard<boost::mutex> l(syncher); //lock
+
+	LogStore::Cell *c = logStore.lookupCell(groupName);
+	if (c == NULL) {
+		AppGroupLog appGroupLog;
+		appGroupLog.pidLog = TimestampedLogBuffer(LOG_MONITORING_MAX_LINES * 5);
+		c = logStore.insert(groupName, appGroupLog);
+	}
+	AppGroupLog &rec = c->value;
+
+	HashedStaticString source(sourceStr, sourceStrLen);
+	if (!rec.watchFileLog.contains(source)) {
+		SimpleLogBuffer logBuffer(LOG_MONITORING_MAX_LINES);
+		rec.watchFileLog.insert(source, logBuffer);
+	}
+	rec.watchFileLog.lookupCell(source)->value.push_back(string(message, messageLen));
+	//unlock
+}
+
+Json::Value
+Context::convertLog(){
+	boost::lock_guard<boost::mutex> l(syncher); //lock
+	Json::Value reply = Json::objectValue;
+
+	if (!logStore.empty()) {
+		Context::LogStore::ConstIterator appGroupIter(logStore);
+		while (*appGroupIter != NULL) {
+			reply[appGroupIter.getKey()] = Json::objectValue;
+
+			Json::Value &processLog = reply[appGroupIter.getKey()]["Application process log (combined)"];
+			foreach (TimestampedLog logLine, appGroupIter->value.pidLog) {
+				Json::Value logLineJson = Json::objectValue;
+				logLineJson["source_id"] = logLine.sourceId;
+				logLineJson["timestamp"] = (Json::UInt64) logLine.timestamp;
+				logLineJson["line"] = logLine.lineText;
+				processLog.append(logLineJson);
+			}
+
+			Context::SimpleLogMap::ConstIterator watchFileLogIter(appGroupIter->value.watchFileLog);
+			while (*watchFileLogIter != NULL) {
+				if (!reply[appGroupIter.getKey()].isMember(watchFileLogIter.getKey())){
+					reply[appGroupIter.getKey()][watchFileLogIter.getKey()] = Json::arrayValue;
+				}
+				foreach (string line, watchFileLogIter->value) {
+					reply[appGroupIter.getKey()][watchFileLogIter.getKey()].append(line);
+				}
+				watchFileLogIter.next();
+			}
+
+			appGroupIter.next();
+		}
+	}
+
+	return reply;
+	//unlock
+}
+
 static void
-realLogAppOutput(int targetFd, char *buf, unsigned int bufSize,
+realLogAppOutput(const HashedStaticString &groupName, int targetFd,
+    char *buf, unsigned int bufSize,
 	const char *pidStr, unsigned int pidStrLen,
 	const char *channelName, unsigned int channelNameLen,
-	const char *message, unsigned int messageLen)
+	const char *message, unsigned int messageLen, int appLogFile,
+	bool saveLog)
 {
 	char *pos = buf;
 	char *end = buf + bufSize;
@@ -325,12 +408,20 @@ realLogAppOutput(int targetFd, char *buf, unsigned int bufSize,
 	pos = appendData(pos, end, ": ");
 	pos = appendData(pos, end, message, messageLen);
 	pos = appendData(pos, end, "\n");
+
+	if (OXT_UNLIKELY(context != NULL && saveLog)) {
+		context->saveNewLog(groupName, pidStr, pidStrLen, message, messageLen);
+	}
+	if (appLogFile > -1) {
+		writeExactWithoutOXT(appLogFile, buf, pos - buf);
+	}
 	writeExactWithoutOXT(targetFd, buf, pos - buf);
 }
 
 void
-logAppOutput(pid_t pid, const char *channelName, const char *message, unsigned int size) {
+logAppOutput(const HashedStaticString &groupName, pid_t pid, const char *channelName, const char *message, unsigned int size, const StaticString &appLogFile) {
 	int targetFd;
+	bool saveLog = false;
 
 	if (OXT_LIKELY(context != NULL)) {
 		const ConfigRealization *configRealization = context->getConfigRealization();
@@ -339,10 +430,19 @@ logAppOutput(pid_t pid, const char *channelName, const char *message, unsigned i
 		}
 
 		targetFd = configRealization->targetFd;
+		saveLog = configRealization->saveLog;
 	} else {
 		targetFd = STDERR_FILENO;
 	}
 
+	int fd = -1;
+	if (!appLogFile.empty()) {
+		fd = open(appLogFile.data(), O_WRONLY | O_APPEND | O_CREAT, 0640);
+		if (fd == -1) {
+			int e = errno;
+			P_ERROR("opening file: " << appLogFile << " for logging " << groupName << " failed. Error: " << strerror(e));
+		}
+	}
 	char pidStr[sizeof("4294967295")];
 	unsigned int pidStrLen, channelNameLen, totalLen;
 
@@ -358,19 +458,20 @@ logAppOutput(pid_t pid, const char *channelName, const char *message, unsigned i
 	totalLen = (sizeof("App X Y: \n") - 2) + pidStrLen + channelNameLen + size;
 	if (totalLen < 1024) {
 		char buf[1024];
-		realLogAppOutput(targetFd,
+		realLogAppOutput(groupName, targetFd,
 			buf, sizeof(buf),
 			pidStr, pidStrLen,
 			channelName, channelNameLen,
-			message, size);
+			message, size, fd, saveLog);
 	} else {
 		DynamicBuffer buf(totalLen);
-		realLogAppOutput(targetFd,
+		realLogAppOutput(groupName, targetFd,
 			buf.data, totalLen,
 			pidStr, pidStrLen,
 			channelName, channelNameLen,
-			message, size);
+			message, size, fd, saveLog);
 	}
+	if(fd > -1){close(fd);}
 }
 
 
@@ -662,6 +763,7 @@ Schema::Schema() {
 		.setInspectFilter(filterTargetFd);
 	add("redirect_stderr", BOOL_TYPE, OPTIONAL, true);
 	add("app_output_log_level", STRING_TYPE, OPTIONAL, DEFAULT_APP_OUTPUT_LOG_LEVEL_NAME);
+	add("buffer_logs", BOOL_TYPE, OPTIONAL, false);
 
 	addValidator(boost::bind(validateLogLevel, "level",
 		boost::placeholders::_1, boost::placeholders::_2));
@@ -681,6 +783,7 @@ Schema::Schema() {
 ConfigRealization::ConfigRealization(const ConfigKit::Store &store)
 	: level(parseLevel(store["level"].asString())),
 	  appOutputLogLevel(parseLevel(store["app_output_log_level"].asString())),
+	  saveLog(store["buffer_logs"].asBool()),
 	  finalized(false)
 {
 	if (store["target"].isMember("stderr")) {