passenger 4.0.5 → 4.0.6

data/ext/common/agents/LoggingAgent/LoggingServer.h

@@ -56,6 +56,7 @@
 #include <Utils/MD5.h>
 #include <Utils/IOUtils.h>
 #include <Utils/MessageIO.h>
+#include <Utils/VariantMap.h>
 #include <Utils/StrIntUtils.h>
 #include <Utils/StringMap.h>
 
@@ -119,10 +120,20 @@ private:
 		virtual bool isRemote() const {
 			return false;
 		}
+
+		// Default interval at which this sink should be flushed.
+		virtual unsigned int defaultFlushInterval() const {
+			return 15;
+		}
 		
 		virtual void append(const DataStoreId &dataStoreId,
 			const StaticString &data) = 0;
-		virtual bool flush() { return true; }
+		
+		virtual bool flush() {
+			lastFlushed = ev_now(server->getLoop());
+			return true;
+		}
+		
 		virtual void dump(ostream &stream) const { }
 	};
 	
@@ -154,12 +165,7 @@ private:
 		virtual void append(const DataStoreId &dataStoreId, const StaticString &data) {
 			syscalls::write(fd, data.data(), data.size());
 		}
-		
-		virtual bool flush() {
-			lastFlushed = ev_now(server->getLoop());
-			return true;
-		}
-		
+
 		virtual void dump(ostream &stream) const {
 			stream << "   * Log file: " << filename << "\n";
 			stream << "     Opened     : " << opened << "\n";
@@ -211,6 +217,10 @@ private:
 		virtual bool isRemote() const {
 			return true;
 		}
+
+		virtual unsigned int defaultFlushInterval() const {
+			return 60;
+		}
 		
 		virtual void append(const DataStoreId &dataStoreId, const StaticString &data) {
 			if (bufferSize + data.size() > BUFFER_CAPACITY) {
@@ -235,11 +245,17 @@ private:
 				server->remoteSender.schedule(unionStationKey, nodeName,
 					category, &data, 1);
 				bufferSize = 0;
+				P_DEBUG("Flushed remote sink " << inspect() << ": " << bufferSize << " bytes");
 				return true;
 			} else {
+				P_DEBUG("Flushed remote sink " << inspect() << ": 0 bytes");
 				return false;
 			}
 		}
+
+		string inspect() const {
+			return "(key=" + unionStationKey + ", node=" + nodeName + ", category=" + category + ")";
+		}
 		
 		virtual void dump(ostream &stream) const {
 			stream << "   * Remote sink\n";
@@ -369,7 +385,6 @@ private:
 	
 	typedef shared_ptr<FilterSupport::Filter> FilterPtr;
 	
-	string dumpFile;
 	RemoteSender remoteSender;
 	ev::timer garbageCollectionTimer;
 	ev::timer sinkFlushingTimer;
@@ -390,6 +405,8 @@ private:
 	bool refuseNewConnections;
 	bool exitRequested;
 	unsigned long long exitBeginTime;
+	int sinkFlushInterval;
+	string dumpFile;
 	
 	void sendErrorToClient(Client *client, const string &message) {
 		client->writeArrayMessage("error", message.c_str(), NULL);
@@ -703,23 +720,24 @@ private:
 		P_DEBUG("Garbage collection time");
 		releaseInactiveLogSinks(ev_now(getLoop()));
 	}
+
+	ev_tstamp getFlushInterval(const LogSink *sink) const {
+		if (sinkFlushInterval == 0) {
+			return sink->defaultFlushInterval();
+		} else {
+			return sinkFlushInterval;
+		}
+	}
 	
 	void sinkFlushTimeout(ev::timer &timer, int revents) {
-		P_TRACE(2, "Flushing all sinks (periodic action)");
+		P_DEBUG("Flushing all sinks");
 		LogSinkCache::iterator it;
 		LogSinkCache::iterator end = logSinkCache.end();
 		ev_tstamp now = ev_now(getLoop());
 		
 		for (it = logSinkCache.begin(); it != end; it++) {
 			LogSink *sink = it->second.get();
-			
-			// Flush log file sinks every 15 seconds,
-			// remote sinks every 60 seconds.
-			if (sink->isRemote()) {
-				if (now - sink->lastFlushed >= 60) {
-					sink->flush();
-				}
-			} else {
+			if (now - sink->lastFlushed >= getFlushInterval(sink)) {
 				sink->flush();
 			}
 		}
@@ -1080,25 +1098,24 @@ public:
 	LoggingServer(struct ev_loop *loop,
 		FileDescriptor fd,
 		const AccountsDatabasePtr &accountsDatabase,
-		const string &dumpFile,
-		const string &unionStationGatewayAddress = DEFAULT_UNION_STATION_GATEWAY_ADDRESS,
-		unsigned short unionStationGatewayPort = DEFAULT_UNION_STATION_GATEWAY_PORT,
-		const string &unionStationGatewayCert = "",
-		const string &unionStationProxyAddress = "")
+		const VariantMap &options = VariantMap()/**/)
 		: EventedMessageServer(loop, fd, accountsDatabase),
-		  remoteSender(unionStationGatewayAddress,
-		               unionStationGatewayPort,
-		               unionStationGatewayCert,
-		               unionStationProxyAddress),
+		  remoteSender(
+		      options.get("union_station_gateway_address", false, DEFAULT_UNION_STATION_GATEWAY_ADDRESS),
+		      options.getInt("union_station_gateway_port", false, DEFAULT_UNION_STATION_GATEWAY_PORT),
+		      options.get("union_station_gateway_cert", false, ""),
+		      options.get("union_station_proxy_address", false)),
 		  garbageCollectionTimer(loop),
 		  sinkFlushingTimer(loop),
-		  exitTimer(loop)
+		  exitTimer(loop),
+		  dumpFile(options.get("analytics_dump_file", false, "/dev/null"))
 	{
-		this->dumpFile = dumpFile;
+		int sinkFlushTimerInterval = options.getInt("analytics_sink_flush_timer_interval", false, 15);
+		sinkFlushInterval = options.getInt("analytics_sink_flush_interval", false, 0);
 		garbageCollectionTimer.set<LoggingServer, &LoggingServer::garbageCollect>(this);
 		garbageCollectionTimer.start(GARBAGE_COLLECTION_TIMEOUT, GARBAGE_COLLECTION_TIMEOUT);
 		sinkFlushingTimer.set<LoggingServer, &LoggingServer::sinkFlushTimeout>(this);
-		sinkFlushingTimer.start(15, 15);
+		sinkFlushingTimer.start(sinkFlushTimerInterval, sinkFlushTimerInterval);
 		exitTimer.set<LoggingServer, &LoggingServer::exitTimerTimeout>(this);
 		exitTimer.set(0.05, 0.05);
 		refuseNewConnections = false;

data/ext/common/agents/LoggingAgent/Main.cpp

@@ -38,10 +38,13 @@
 
 #include <agents/Base.h>
 #include <agents/LoggingAgent/LoggingServer.h>
+#include <agents/LoggingAgent/AdminController.h>
 
 #include <AccountsDatabase.h>
 #include <Account.h>
 #include <Exceptions.h>
+#include <ResourceLocator.h>
+#include <MessageServer.h>
 #include <Utils.h>
 #include <Utils/IOUtils.h>
 #include <Utils/MessageIO.h>
@@ -56,6 +59,9 @@ static struct ev_loop *eventLoop;
 static LoggingServer *loggingServer;
 static int exitCode = 0;
 
+static const int MESSAGE_SERVER_THREAD_STACK_SIZE = 128 * 1024;
+
+
 static struct ev_loop *
 createEventLoop() {
 	struct ev_loop *loop;
@@ -150,21 +156,29 @@ myself() {
 	}
 }
 
+static string
+findUnionStationGatewayCert(const ResourceLocator &locator,
+	const string &cert)
+{
+	if (cert.empty()) {
+		return locator.getResourcesDir() + "/union_station_gateway.crt";
+	} else if (cert != "-") {
+		return cert;
+	} else {
+		return "";
+	}
+}
+
 int
 main(int argc, char *argv[]) {
 	VariantMap options        = initializeAgent(argc, argv, "PassengerLoggingAgent");
+	string passengerRoot      = options.get("passenger_root");
 	string socketAddress      = options.get("logging_agent_address");
-	string dumpFile           = options.get("analytics_dump_file", false, "/dev/null");
+	string adminSocketAddress = options.get("logging_agent_admin_address");
 	string password           = options.get("logging_agent_password");
-	string username           = options.get("analytics_log_user",
-		false, myself());
+	string username           = options.get("analytics_log_user", false, myself());
 	string groupname          = options.get("analytics_log_group", false);
-	string unionStationGatewayAddress = options.get("union_station_gateway_address",
-		false, DEFAULT_UNION_STATION_GATEWAY_ADDRESS);
-	int    unionStationGatewayPort = options.getInt("union_station_gateway_port",
-		false, DEFAULT_UNION_STATION_GATEWAY_PORT);
-	string unionStationGatewayCert  = options.get("union_station_gateway_cert", false);
-	string unionStationProxyAddress = options.get("union_station_proxy_address", false);
+	string adminToolStatusPassword = options.get("admin_tool_status_password");
 	
 	curl_global_init(CURL_GLOBAL_ALL);
 	
@@ -172,14 +186,19 @@ main(int argc, char *argv[]) {
 		/********** Now begins the real initialization **********/
 		
 		/* Create all the necessary objects and sockets... */
-		AccountsDatabasePtr  accountsDatabase;
+		ResourceLocator      resourceLocator(passengerRoot);
+		AccountsDatabasePtr  accountsDatabase, adminAccountsDatabase;
 		FileDescriptor       serverSocketFd;
 		struct passwd       *user;
 		struct group        *group;
 		int                  ret;
+
+		options.set("union_station_gateway_cert", findUnionStationGatewayCert(
+			resourceLocator, options.get("union_station_gateway_cert", false)));
 		
 		eventLoop = createEventLoop();
-		accountsDatabase = ptr(new AccountsDatabase());
+		accountsDatabase = make_shared<AccountsDatabase>();
+		adminAccountsDatabase = make_shared<AccountsDatabase>();
 		serverSocketFd = createServer(socketAddress.c_str());
 		if (getSocketAddressType(socketAddress) == SAT_UNIX) {
 			do {
@@ -240,12 +259,19 @@ main(int argc, char *argv[]) {
 		/* Now setup the actual logging server. */
 		accountsDatabase->add("logging", password, false);
 		LoggingServer server(eventLoop, serverSocketFd,
-			accountsDatabase, dumpFile,
-			unionStationGatewayAddress,
-			unionStationGatewayPort,
-			unionStationGatewayCert,
-			unionStationProxyAddress);
+			accountsDatabase, options);
 		loggingServer = &server;
+
+		/* Setup the admin server. */
+		adminAccountsDatabase->add("_passenger-status", adminToolStatusPassword, false);
+		MessageServer adminServer(parseUnixSocketAddress(adminSocketAddress),
+			adminAccountsDatabase);
+		adminServer.addHandler(make_shared<AdminController>(&server));
+		function<void ()> adminServerFunc = boost::bind(&MessageServer::mainLoop, &adminServer);
+		oxt::thread adminServerThread(
+			boost::bind(runAndPrintExceptions, adminServerFunc, true),
+			"AdminServer thread", MESSAGE_SERVER_THREAD_STACK_SIZE
+		);
 		
 		
 		ev::io feedbackFdWatcher(eventLoop);
@@ -270,6 +296,7 @@ main(int argc, char *argv[]) {
 		
 		P_WARN("PassengerLoggingAgent online, listening at " << socketAddress);
 		ev_run(eventLoop, 0);
+		adminServerThread.interrupt_and_join();
 		P_DEBUG("Logging agent exiting with code " << exitCode << ".");
 		return exitCode;
 	} catch (const tracable_exception &e) {

data/ext/common/agents/LoggingAgent/RemoteSender.h

@@ -299,7 +299,7 @@ private:
 	}
 	
 	bool firstStarted() const {
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		return nextCheckupTime == 0;
 	}
 	
@@ -334,12 +334,12 @@ private:
 			scheduleNextCheckup(3 * 60 * 60);
 		}
 
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		this->servers = servers;
 	}
 	
 	void freeThreadData() {
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		servers.clear(); // Invoke destructors inside this thread.
 	}
 	
@@ -357,7 +357,7 @@ private:
 	}
 	
 	unsigned int msecUntilNextCheckup() const {
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		time_t now = SystemTime::get();
 		if (now >= nextCheckupTime) {
 			return 0;
@@ -367,7 +367,7 @@ private:
 	}
 	
 	bool timeForCheckup() const {
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		return SystemTime::get() >= nextCheckupTime;
 	}
 	
@@ -525,7 +525,7 @@ public:
 
 		if (!queue.tryAdd(item)) {
 			P_WARN("The Union Station gateway isn't responding quickly enough; dropping packet.");
-			lock_guard<boost::mutex> l(syncher);
+			boost::lock_guard<boost::mutex> l(syncher);
 			packetsDropped++;
 		}
 	}
@@ -536,7 +536,7 @@ public:
 
 	template<typename Stream>
 	void inspect(Stream &stream) const {
-		lock_guard<boost::mutex> l(syncher);
+		boost::lock_guard<boost::mutex> l(syncher);
 		stream << "  Available servers (" << servers.size() << "): ";
 		foreach (const ServerPtr server, servers) {
 			stream << server->name() << " ";

data/ext/common/agents/Watchdog/AgentWatcher.cpp

@@ -38,7 +38,7 @@ private:
 			
 			while (!this_thread::interruption_requested()) {
 				{
-					lock_guard<boost::mutex> l(lock);
+					boost::lock_guard<boost::mutex> l(lock);
 					pid = this->pid;
 				}
 				
@@ -65,7 +65,7 @@ private:
 				}
 				
 				{
-					lock_guard<boost::mutex> l(lock);
+					boost::lock_guard<boost::mutex> l(lock);
 					this->pid = 0;
 				}
 				
@@ -103,16 +103,16 @@ private:
 			}
 		} catch (const boost::thread_interrupted &) {
 		} catch (const tracable_exception &e) {
-			lock_guard<boost::mutex> l(lock);
+			boost::lock_guard<boost::mutex> l(lock);
 			threadExceptionMessage = e.what();
 			threadExceptionBacktrace = e.backtrace();
 			errorEvent->notify();
 		} catch (const std::exception &e) {
-			lock_guard<boost::mutex> l(lock);
+			boost::lock_guard<boost::mutex> l(lock);
 			threadExceptionMessage = e.what();
 			errorEvent->notify();
 		} catch (...) {
-			lock_guard<boost::mutex> l(lock);
+			boost::lock_guard<boost::mutex> l(lock);
 			threadExceptionMessage = "Unknown error";
 			errorEvent->notify();
 		}
@@ -415,7 +415,7 @@ public:
 					args[0] + "'");
 			}
 			
-			lock_guard<boost::mutex> l(lock);
+			boost::lock_guard<boost::mutex> l(lock);
 			this->feedbackFd = feedbackFd;
 			this->pid = pid;
 			failGuard.clear();
@@ -433,7 +433,7 @@ public:
 	 * @throws thread_resource_error
 	 */
 	virtual void startWatching() {
-		lock_guard<boost::mutex> l(lock);
+		boost::lock_guard<boost::mutex> l(lock);
 		if (pid == 0) {
 			throw RuntimeException("start() hasn't been called yet");
 		}
@@ -466,7 +466,7 @@ public:
 	 * or false if it wasn't started.
 	 */
 	virtual bool forceShutdown() {
-		lock_guard<boost::mutex> l(lock);
+		boost::lock_guard<boost::mutex> l(lock);
 		if (pid == 0) {
 			return false;
 		} else {
@@ -482,7 +482,7 @@ public:
 	 * everything is still OK.
 	 */
 	string getErrorMessage() const {
-		lock_guard<boost::mutex> l(lock);
+		boost::lock_guard<boost::mutex> l(lock);
 		return threadExceptionMessage;
 	}
 	
@@ -490,7 +490,7 @@ public:
 	 * The error backtrace, if applicable.
 	 */
 	string getErrorBacktrace() const {
-		lock_guard<boost::mutex> l(lock);
+		boost::lock_guard<boost::mutex> l(lock);
 		return threadExceptionBacktrace;
 	}
 	
@@ -500,7 +500,7 @@ public:
 	 * has exited without using waitpid().
 	 */
 	const FileDescriptor getFeedbackFd() const {
-		lock_guard<boost::mutex> l(lock);
+		boost::lock_guard<boost::mutex> l(lock);
 		return feedbackFd;
 	}
 };

data/ext/common/agents/Watchdog/LoggingAgentWatcher.cpp

@@ -44,6 +44,7 @@ protected:
 		VariantMap options = agentsOptions;
 		options.set("logging_agent_address", loggingAgentAddress);
 		options.set("logging_agent_password", loggingAgentPassword);
+		options.set("logging_agent_admin_address", loggingAgentAdminAddress);
 		options.writeToFd(fd);
 	}
 	
@@ -63,6 +64,7 @@ public:
 	virtual void reportAgentsInformation(VariantMap &report) {
 		report
 			.set("logging_socket_address", loggingAgentAddress)
-			.set("logging_socket_password", loggingAgentPassword);
+			.set("logging_socket_password", loggingAgentPassword)
+			.set("logging_socket_admin_address", loggingAgentAdminAddress);
 	}
 };

data/ext/common/agents/Watchdog/Main.cpp

@@ -94,8 +94,13 @@ static ResourceLocator *resourceLocator;
 static ServerInstanceDirPtr serverInstanceDir;
 static ServerInstanceDir::GenerationPtr generation;
 static ServerInstanceDirToucher *serverInstanceDirToucher;
+static uid_t defaultUid;
+static gid_t defaultGid;
 static string loggingAgentAddress;
 static string loggingAgentPassword;
+static string loggingAgentAdminAddress;
+static string adminToolStatusPassword;
+static string adminToolManipulationPassword;
 
 #include "AgentWatcher.cpp"
 #include "HelperAgentWatcher.cpp"
@@ -440,6 +445,41 @@ initializeOptions() {
 	P_INFO("Options: " << agentsOptions.inspect());
 }
 
+static void
+maybeSetsid() {
+	/* Become the session leader so that Apache can't kill the
+	 * watchdog with killpg() during shutdown, so that a
+	 * Ctrl-C only affects the web server, and so that
+	 * we can kill all of our subprocesses in a single killpg().
+	 *
+	 * AgentsStarter.h already calls setsid() before exec()ing
+	 * the Watchdog, but FlyingPassenger does not.
+	 */
+	if (agentsOptions.getBool("setsid", false)) {
+		setsid();
+	}
+}
+
+static void
+lookupDefaultUidGid(uid_t &uid, gid_t &gid) {
+	struct passwd *userEntry;
+	struct group  *groupEntry;
+	
+	userEntry = getpwnam(defaultUser.c_str());
+	if (userEntry == NULL) {
+		throw NonExistentUserException("Default user '" + defaultUser +
+			"' does not exist.");
+	}
+	uid = userEntry->pw_uid;
+
+	groupEntry = getgrnam(defaultGroup.c_str());
+	if (groupEntry == NULL) {
+		throw NonExistentGroupException("Default group '" + defaultGroup +
+			"' does not exist.");
+	}
+	gid = groupEntry->gr_gid;
+}
+
 static void
 initializeWorkingObjects() {
 	TRACE_POINT();
@@ -474,9 +514,30 @@ initializeWorkingObjects() {
 	UPDATE_TRACE_POINT();
 	serverInstanceDirToucher = new ServerInstanceDirToucher();
 
+	UPDATE_TRACE_POINT();
+	lookupDefaultUidGid(defaultUid, defaultGid);
+
 	UPDATE_TRACE_POINT();
 	loggingAgentAddress  = "unix:" + generation->getPath() + "/logging";
 	loggingAgentPassword = randomGenerator->generateAsciiString(64);
+	loggingAgentAdminAddress  = "unix:" + generation->getPath() + "/logging_admin";
+
+	UPDATE_TRACE_POINT();
+	adminToolStatusPassword = randomGenerator->generateAsciiString(MESSAGE_SERVER_MAX_PASSWORD_SIZE);
+	adminToolManipulationPassword = randomGenerator->generateAsciiString(MESSAGE_SERVER_MAX_PASSWORD_SIZE);
+	agentsOptions.set("admin_tool_status_password", adminToolStatusPassword);
+	agentsOptions.set("admin_tool_manipulation_password", adminToolManipulationPassword);
+	if (geteuid() == 0 && !userSwitching) {
+		createFile(generation->getPath() + "/passenger-status-password.txt",
+			adminToolStatusPassword, S_IRUSR, defaultUid, defaultGid);
+		createFile(generation->getPath() + "/admin-manipulation-password.txt",
+			adminToolManipulationPassword, S_IRUSR, defaultUid, defaultGid);
+	} else {
+		createFile(generation->getPath() + "/passenger-status-password.txt",
+			adminToolStatusPassword, S_IRUSR | S_IWUSR);
+		createFile(generation->getPath() + "/admin-manipulation-password.txt",
+			adminToolManipulationPassword, S_IRUSR | S_IWUSR);
+	}
 }
 
 static void
@@ -545,6 +606,7 @@ main(int argc, char *argv[]) {
 	try {
 		TRACE_POINT();
 		initializeOptions();
+		maybeSetsid();
 		initializeWorkingObjects();
 		initializeAgentWatchers();
 	} catch (const std::exception &e) {