passenger 5.1.12 → 5.2.0

data/src/agent/Shared/Fundamentals/Initialization.cpp

@@ -413,6 +413,48 @@ extraArgumentsPassed(int argc, char *argv[], int argStartIndex) {
 		|| (argc == argStartIndex + 1 && !isBlank(argv[argStartIndex]));
 }
 
+static void
+parseAndCommitConfig(ConfigKit::Store &config, const StaticString &jsonData) {
+	Json::Reader reader;
+	Json::Value doc;
+
+	if (getEnvBool("PASSENGER_DEBUG_INITIAL_CONFIG", false)) {
+		P_NOTICE("Initial raw configuration: " << jsonData);
+	}
+
+	if (reader.parse(jsonData, doc)) {
+		vector<ConfigKit::Error> errors;
+		if (!config.update(doc, errors)) {
+			vector<ConfigKit::Error>::const_iterator it, end = errors.end();
+			fprintf(stderr, "ERROR: invalid configuration:\n");
+			for (it = errors.begin(); it != end; it++) {
+				fprintf(stderr, "- %s\n", it->getMessage().c_str());
+			}
+			fprintf(stderr, "Raw configuration: %s\n", doc.toStyledString().c_str());
+			exit(1);
+		}
+	} else {
+		fprintf(stderr, "ERROR: JSON configuration parse error: %s\n",
+			reader.getFormattedErrorMessages().c_str());
+		fprintf(stderr, "Raw JSON data: %s\n", jsonData.toString().c_str());
+		exit(1);
+	}
+}
+
+static void
+readConfigFromFd(int fd, ConfigKit::Store &config) {
+	parseAndCommitConfig(config, readScalarMessage(fd));
+}
+
+static void
+readConfigFromJsonPassedToArgs(int argc, char **argv[], int argStartIndex, ConfigKit::Store &config) {
+	if (argc <= argStartIndex) {
+		return;
+	}
+
+	parseAndCommitConfig(config, (*argv)[argStartIndex]);
+}
+
 static void
 maybeInitializeAbortHandler() {
 	if (!getEnvBool("PASSENGER_ABORT_HANDLER", true)) {
@@ -438,29 +480,13 @@ maybeInitializeSyscallFailureSimulation(const char *processName) {
 }
 
 static void
-initializeLoggingKit(const char *processName, VariantMap &options) {
-	Json::Value config(Json::objectValue);
-
-	options.setDefaultInt("log_level", DEFAULT_LOG_LEVEL);
-	config["level"] = options.get("log_level");
-
-	if (options.has("log_file")) {
-		config["target"]["path"] = options.get("log_file");
-	} else if (options.has("debug_log_file")) {
-		config["target"]["path"] = options.get("debug_log_file");
-	}
-	if (options.getBool("log_file_is_stderr", false, false)) {
-		config["target"]["stderr"] = true;
-	}
-
-	if (options.has("file_descriptor_log_file")) {
-		config["file_descriptor_log_target"] =
-			options.get("file_descriptor_log_file");
-	}
-
-	LoggingKit::initialize(config);
+initializeLoggingKit(const char *processName, ConfigKit::Store &config,
+	const ConfigKit::Translator &loggingKitTranslator)
+{
+	LoggingKit::initialize(config.inspectEffectiveValues(), loggingKitTranslator);
+	Json::Value dump = LoggingKit::context->inspectConfig();
 
-	if (options.has("file_descriptor_log_file")) {
+	if (!dump["file_descriptor_log_target"]["effective_value"].isNull()) {
 		// This information helps ./dev/parse_file_descriptor_log.
 		FastStringStream<> stream;
 		LoggingKit::_prepareLogEntry(stream, LoggingKit::CRIT, __FILE__, __LINE__);
@@ -468,12 +494,11 @@ initializeLoggingKit(const char *processName, VariantMap &options) {
 		_writeFileDescriptorLogEntry(LoggingKit::context->getConfigRealization(),
 			stream.data(), stream.size());
 
-		config = LoggingKit::context->getConfig().inspect();
 		P_LOG_FILE_DESCRIPTOR_OPEN4(
 			LoggingKit::context->getConfigRealization()->fileDescriptorLogTargetFd,
 			__FILE__, __LINE__,
 			"file descriptor log file "
-			<< config["file_descriptor_log_target"]["effective_value"]["path"].asString());
+			<< dump["file_descriptor_log_target"]["effective_value"]["path"].asString());
 	} else {
 		// This information helps ./dev/parse_file_descriptor_log.
 		P_DEBUG("Starting agent: " << processName);
@@ -509,12 +534,22 @@ changeProcessTitle(int argc, char **argv[], const char *processName) {
 	*argv = context->origArgv;
 }
 
-VariantMap
+static string
+dumpConfigForDebugging(const ConfigKit::Store &config) {
+	Json::Value result = config.inspectEffectiveValues();
+	if (!result["config_manifest"].isNull()) {
+		// The config manifest is too large so we omit it from the debug output.
+		result["config_manifest"] = "[OMITTED]";
+	}
+	return result.toStyledString();
+}
+
+void
 initializeAgent(int argc, char **argv[], const char *processName,
-	OptionParserFunc optionParser, PreinitializationFunc preinit,
-	int argStartIndex)
+	ConfigKit::Store &config, const ConfigKit::Translator &loggingKitTranslator,
+	OptionParserFunc optionParser,
+	PreinitializationFunc preinit, int argStartIndex)
 {
-	VariantMap options;
 	const char *seedStr;
 
 	context = new Context();
@@ -546,28 +581,31 @@ initializeAgent(int argc, char **argv[], const char *processName,
 				exit(1);
 			}
 			context->feedbackFdAvailable = true;
-			options.readFrom(FEEDBACK_FD);
+			readConfigFromFd(FEEDBACK_FD, config);
 		} else if (optionParser != NULL) {
-			optionParser(argc, (const char **) *argv, options);
+			optionParser(argc, (const char **) *argv, config);
 		} else {
-			options.readFrom((const char **) *argv + argStartIndex,
-				argc - argStartIndex);
+			readConfigFromJsonPassedToArgs(argc, argv, argStartIndex, config);
 		}
 
-		if (options.has("passenger_root")) {
-			context->resourceLocator = new ResourceLocator(options.get("passenger_root", true));
+		if (!config["passenger_root"].isNull()) {
+			context->resourceLocator = new ResourceLocator(config["passenger_root"].asString());
 			if (abortHandlerInstalled()) {
-				context->abortHandlerConfig.ruby = strdup(options.get("default_ruby", false, DEFAULT_RUBY).c_str());
+				string defaultRuby = config["default_ruby"].asString();
+				if (defaultRuby.empty()) {
+					defaultRuby = DEFAULT_RUBY;
+				}
+				context->abortHandlerConfig.ruby = strdup(defaultRuby.c_str());
 				context->abortHandlerConfig.resourceLocator = context->resourceLocator;
 				abortHandlerConfigChanged();
 			}
 		}
 
 		if (preinit != NULL) {
-			preinit(options);
+			preinit(config);
 		}
 
-		initializeLoggingKit(processName, options);
+		initializeLoggingKit(processName, config, loggingKitTranslator);
 	} catch (const tracable_exception &e) {
 		P_ERROR("*** ERROR: " << e.what() << "\n" << e.backtrace());
 		exit(1);
@@ -576,14 +614,12 @@ initializeAgent(int argc, char **argv[], const char *processName,
 	storeArgvCopy(argc, *argv);
 	changeProcessTitle(argc, argv, processName);
 
-	P_DEBUG("Random seed: " << context->randomSeed);
-
-	return options;
+	P_DEBUG(processName << " config: " << dumpConfigForDebugging(config));
+	P_DEBUG(processName << " random seed: " << context->randomSeed);
 }
 
 void
-shutdownAgent(VariantMap *agentOptions) {
-	delete agentOptions;
+shutdownAgent(ConfigKit::Schema *schema, ConfigKit::Store *config) {
 	LoggingKit::shutdown();
 	oxt::shutdown();
 	if (abortHandlerInstalled()) {
@@ -597,11 +633,12 @@ shutdownAgent(VariantMap *agentOptions) {
 	delete context->resourceLocator;
 	delete context;
 	context = NULL;
+	delete config;
+	delete schema;
 }
 
 void
-restoreOomScore(VariantMap *agentOptions) {
-	string score = agentOptions->get("original_oom_score", false);
+restoreOomScore(const string &score) {
 	bool isLegacy;
 	int ret = tryRestoreOomScore(score, isLegacy);
 	if (ret != 0) {

data/src/agent/Shared/Fundamentals/Initialization.h

@@ -29,13 +29,16 @@
 /** Common initialization code for all agents. */
 
 #include <cstddef>
-#include <Utils/VariantMap.h>
+#include <string>
+#include <ConfigKit/Store.h>
 #include <Shared/Fundamentals/AbortHandler.h>
 
 namespace Passenger {
 namespace Agent {
 namespace Fundamentals {
 
+using namespace std;
+
 
 struct Context {
 	ResourceLocator *resourceLocator;
@@ -46,19 +49,21 @@ struct Context {
 	AbortHandlerConfig abortHandlerConfig;
 };
 
-typedef void (*OptionParserFunc)(int argc, const char **argv, VariantMap &options);
-typedef void (*PreinitializationFunc)(VariantMap &options);
+typedef void (*OptionParserFunc)(int argc, const char **argv, ConfigKit::Store &config);
+typedef void (*PreinitializationFunc)(ConfigKit::Store &config);
 
 extern Context *context;
 
 
-VariantMap initializeAgent(int argc, char **argv[], const char *processName,
-	OptionParserFunc optionParser = NULL, PreinitializationFunc preinit = NULL,
+void initializeAgent(int argc, char **argv[], const char *processName,
+	ConfigKit::Store &config, const ConfigKit::Translator &loggingKitTranslator,
+	OptionParserFunc optionParser = NULL,
+	PreinitializationFunc preinit = NULL,
 	int argStartIndex = 1);
-void shutdownAgent(VariantMap *agentOptions);
+void shutdownAgent(ConfigKit::Schema *schema, ConfigKit::Store *config);
 
 bool feedbackFdAvailable();
-void restoreOomScore(VariantMap *agentOptions);
+void restoreOomScore(const string &score);
 
 
 } // namespace Fundamentals

data/src/agent/TempDirToucher/TempDirToucherMain.cpp

@@ -59,6 +59,7 @@ static bool verbose = false;
 static const char *pidFile = NULL;
 static const char *logFile = NULL;
 static uid_t uid = 0;
+static pid_t nginxPid = 0;
 static int sleepInterval = 1800;
 static int terminationPipe[2];
 static sig_atomic_t shouldIgnoreNextTermSignal = 0;
@@ -79,6 +80,7 @@ usage() {
 	printf("\n");
 	printf("Options:\n");
 	printf("  --cleanup           Remove directory on exit\n");
+	printf("  --nginx-pid PID     Optional PID to wait for before cleanup\n");
 	printf("  --daemonize         Daemonize into background\n");
 	printf("  --interval SECONDS  Customize interval\n");
 	printf("  --pid-file PATH     Save PID into the given file\n");
@@ -108,6 +110,13 @@ parseArguments(int argc, char *argv[], int offset) {
 			}
 			sleepInterval = atoi(argv[i + 1]);
 			i++;
+		} else if (strcmp(argv[i], "--nginx-pid") == 0) {
+			if (i == argc - 1) {
+				fprintf(stderr, ERROR_PREFIX ": --nginx-pid requires the nginx master process pid as an argument\n");
+				exit(1);
+			}
+			nginxPid = atoi(argv[i + 1]);
+			i++;
 		} else if (strcmp(argv[i], "--pid-file") == 0) {
 			pidFile = argv[i + 1];
 			i++;
@@ -450,6 +459,17 @@ performCleanup(const char *dir) {
 	}
 }
 
+void
+maybeWaitForNginxToExit() {
+	// If a PID was specified, wait for kill to tell us it's gone.
+	if (nginxPid == 0) {
+		return;
+	}
+	while (0 == kill(nginxPid, 0)) {
+		sleep(1); // regular sleep; doSleep() can't sleep while terminating
+	}
+}
+
 int
 tempDirToucherMain(int argc, char *argv[]) {
 	initialize(argc, argv, 2);
@@ -474,6 +494,7 @@ tempDirToucherMain(int argc, char *argv[]) {
 
 	maybeDeletePidFile();
 	if (shouldCleanup) {
+		maybeWaitForNginxToExit();
 		DEBUG("Cleaning up directory");
 		performCleanup(dir);
 	}

data/src/agent/Watchdog/AgentWatcher.cpp

@@ -246,7 +246,7 @@ public:
 	 *
 	 * @pre start() has been called and succeeded.
 	 */
-	virtual void reportAgentsInformation(VariantMap &report) = 0;
+	virtual void reportAgentStartupResult(Json::Value &report) = 0;
 
 	/** Returns the name of the agent that this class is watching. */
 	virtual const char *name() const = 0;

data/src/agent/Watchdog/ApiServer.h

@@ -23,8 +23,8 @@
  *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  *  THE SOFTWARE.
  */
-#ifndef _PASSENGER_WATCHDOG_AGENT_API_SERVER_H_
-#define _PASSENGER_WATCHDOG_AGENT_API_SERVER_H_
+#ifndef _PASSENGER_WATCHDOG_API_SERVER_H_
+#define _PASSENGER_WATCHDOG_API_SERVER_H_
 
 #include <string>
 #include <exception>
@@ -33,6 +33,7 @@
 #include <modp_b64.h>
 
 #include <Shared/ApiServerUtils.h>
+#include <Shared/ApiAccountUtils.h>
 #include <ServerKit/HttpServer.h>
 #include <DataStructures/LString.h>
 #include <Exceptions.h>
@@ -43,11 +44,59 @@
 #include <Utils/MessageIO.h>
 
 namespace Passenger {
-namespace WatchdogAgent {
+namespace Watchdog {
+namespace ApiServer {
 
 using namespace std;
 
 
+/*
+ * BEGIN ConfigKit schema: Passenger::Watchdog::ApiServer::Schema
+ * (do not edit: following text is automatically generated
+ * by 'rake configkit_schemas_inline_comments')
+ *
+ *   accept_burst_count           unsigned integer   -          default(32)
+ *   authorizations               array              -          default("[FILTERED]"),secret
+ *   client_freelist_limit        unsigned integer   -          default(0)
+ *   fd_passing_password          string             required   secret
+ *   min_spare_clients            unsigned integer   -          default(0)
+ *   request_freelist_limit       unsigned integer   -          default(1024)
+ *   start_reading_after_accept   boolean            -          default(true)
+ *
+ * END
+ */
+class Schema: public ServerKit::HttpServerSchema {
+private:
+	static Json::Value normalizeAuthorizations(const Json::Value &effectiveValues) {
+		Json::Value updates;
+		updates["authorizations"] = ApiAccountUtils::normalizeApiAccountsJson(
+			effectiveValues["authorizations"]);
+		return updates;
+	}
+
+public:
+	Schema()
+		: ServerKit::HttpServerSchema(false)
+	{
+		using namespace ConfigKit;
+
+		add("fd_passing_password", STRING_TYPE, REQUIRED | SECRET);
+		add("authorizations", ARRAY_TYPE, OPTIONAL | SECRET, Json::arrayValue);
+
+		addValidator(boost::bind(ApiAccountUtils::validateAuthorizationsField,
+			"authorizations", boost::placeholders::_1, boost::placeholders::_2));
+
+		addNormalizer(normalizeAuthorizations);
+
+		finalize();
+	}
+};
+
+struct ConfigChangeRequest {
+	ServerKit::HttpServerConfigChangeRequest forParent;
+	boost::scoped_ptr<ApiAccountUtils::ApiAccountDatabase> apiAccountDatabase;
+};
+
 class Request: public ServerKit::BaseHttpRequest {
 public:
 	string body;
@@ -57,11 +106,16 @@ public:
 };
 
 class ApiServer: public ServerKit::HttpServer<ApiServer, ServerKit::HttpClient<Request> > {
-private:
+public:
 	typedef ServerKit::HttpServer<ApiServer, ServerKit::HttpClient<Request> > ParentClass;
 	typedef ServerKit::HttpClient<Request> Client;
 	typedef ServerKit::HeaderTable HeaderTable;
 
+	typedef Passenger::Watchdog::ApiServer::ConfigChangeRequest ConfigChangeRequest;
+
+private:
+	ApiAccountUtils::ApiAccountDatabase apiAccountDatabase;
+
 	void route(Client *client, Request *req, const StaticString &path) {
 		if (path == P_STATIC_STRING("/status.txt")) {
 			processStatusTxt(client, req);
@@ -180,7 +234,7 @@ private:
 
 		password = psg_lstr_make_contiguous(password, req->pool);
 		return constantTimeCompare(StaticString(password->start->data, password->size),
-			fdPassingPassword);
+			config["fd_passing_password"].asString());
 	}
 
 	void processConfigLogFileFd(Client *client, Request *req) {
@@ -273,16 +327,27 @@ protected:
 	}
 
 public:
-	ApiAccountDatabase *apiAccountDatabase;
+	typedef ApiAccountUtils::ApiAccount ApiAccount;
+
+	// Dependencies
 	EventFd *exitEvent;
-	string fdPassingPassword;
 
-	ApiServer(ServerKit::Context *context, const ServerKit::HttpServerSchema &schema,
-		const Json::Value &initialConfig = Json::Value())
-		: ParentClass(context, schema, initialConfig),
-		  apiAccountDatabase(NULL),
+	ApiServer(ServerKit::Context *context, const Schema &schema,
+		const Json::Value &initialConfig,
+		const ConfigKit::Translator &translator = ConfigKit::DummyTranslator())
+		: ParentClass(context, schema, initialConfig, translator),
 		  exitEvent(NULL)
-		{ }
+	{
+		apiAccountDatabase = ApiAccountUtils::ApiAccountDatabase(
+			config["authorizations"]);
+	}
+
+	virtual void initialize() {
+		if (exitEvent == NULL) {
+			throw RuntimeException("exitEvent must be non-NULL");
+		}
+		ParentClass::initialize();
+	}
 
 	virtual StaticString getServerName() const {
 		return P_STATIC_STRING("WatchdogApiServer");
@@ -292,6 +357,10 @@ public:
 		return ParentClass::getClientName(client, buf, size);
 	}
 
+	const ApiAccountUtils::ApiAccountDatabase &getApiAccountDatabase() const {
+		return apiAccountDatabase;
+	}
+
 	bool authorizeByUid(uid_t uid) const {
 		return uid == 0 || uid == geteuid();
 	}
@@ -299,10 +368,27 @@ public:
 	bool authorizeByApiKey(const ApplicationPool2::ApiKey &apiKey) const {
 		return apiKey.isSuper();
 	}
+
+
+	bool prepareConfigChange(const Json::Value &updates,
+		vector<ConfigKit::Error> &errors, ConfigChangeRequest &req)
+	{
+		if (ParentClass::prepareConfigChange(updates, errors, req.forParent)) {
+			req.apiAccountDatabase.reset(new ApiAccountUtils::ApiAccountDatabase(
+				req.forParent.forParent.config->get("authorizations")));
+		}
+		return errors.empty();
+	}
+
+	void commitConfigChange(ConfigChangeRequest &req) BOOST_NOEXCEPT_OR_NOTHROW {
+		ParentClass::commitConfigChange(req.forParent);
+		apiAccountDatabase.swap(*req.apiAccountDatabase);
+	}
 };
 
 
-} // namespace WatchdogAgent
+} // namespace ApiServer
+} // namespace Watchdog
 } // namespace Passenger
 
-#endif /* _PASSENGER_WATCHDOG_AGENT_API_SERVER_H_ */
+#endif /* _PASSENGER_WATCHDOG_API_SERVER_H_ */