passenger 5.1.4 → 5.1.5

data/dev/ci/run-tests-natively

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../.." && pwd)
+# shellcheck source=lib/functions.sh
+source "$SELFDIR/lib/functions.sh"
+cd "$PASSENGER_ROOT"
+
+autodetect_environment
+sanity_check_environment
+echo
+
+if [[ "$OS" != macos ]]; then
+	echo "ERROR: this script can only be run on macOS." >&2
+	exit 1
+fi
+
+security unlock-keychain -p `cat ~/.password` ~/Library/Keychains/login.keychain
+
+export CI_COMMAND="./dev/ci/run-tests-natively $*"
+exec "$PASSENGER_ROOT/dev/ci/scripts/debug-console-wrapper.sh" \
+	"$PASSENGER_ROOT/dev/ci/scripts/run-tests-natively-stage2.sh" "$@"

data/dev/ci/run-tests-with-docker

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../.." && pwd)
+# shellcheck source=lib/functions.sh
+source "$SELFDIR/lib/functions.sh"
+cd "$PASSENGER_ROOT"
+
+autodetect_environment
+sanity_check_environment
+echo
+
+if [[ "$OS" != linux ]]; then
+	echo "ERROR: this script can only be run on Linux." >&2
+	exit 1
+fi
+
+if tty -s; then
+	TTY_ARGS='-t -i'
+else
+	TTY_ARGS=
+fi
+
+run_exec docker run --rm $TTY_ARGS \
+	-v "$PASSENGER_ROOT:/passenger" \
+	-v "$CACHE_DIR:/passenger/.ci_cache" \
+	-e "APP_UID=$(id -u)" \
+	-e "APP_GID=$(id -g)" \
+	-e "TRAP_DEBUG_CONSOLE=$TRAP_DEBUG_CONSOLE" \
+	-e "TEST_RUBY_VERSION=$TEST_RUBY_VERSION" \
+	-e "COMPILE_CONCURRENCY=$COMPILE_CONCURRENCY" \
+	-e "SUDO=$SUDO" \
+	phusion/passenger-ci-docker-env:1.0.0 \
+	/sbin/my_init --skip-runit --skip-startup-files --quiet -- \
+	/passenger/dev/ci/scripts/inituidgid \
+	/sbin/setuser app \
+	/bin/bash -lc 'cd /passenger && exec "$@"' \
+	./dev/ci/scripts/docker-entrypoint.sh \
+	./dev/ci/scripts/docker-entrypoint.sh \
+	"$@"

data/dev/ci/scripts/debug-console-wrapper.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../../.." && pwd)
+# shellcheck source=lib/functions.sh
+source "$SELFDIR/../lib/functions.sh"
+
+if ! "$@"; then
+	echo
+	echo "-----------------------------"
+	echo
+	echo "*** An error occurred ***"
+	if [[ "$DEBUG_CONSOLE" == 1 ]]; then
+		echo "DEBUG_CONSOLE set to 1, so launching a debugging console..."
+		echo
+		# shellcheck source=../lib/set-container-envvars.sh
+		set +e
+		source "$PASSENGER_ROOT/dev/ci/lib/set-container-envvars.sh"
+		header2 "Launching bash"
+		bash -l
+	else
+		echo "If you want to debug this, run '$CI_COMMAND' with the environment variable DEBUG_CONSOLE=1."
+	fi
+	exit 1
+fi

data/dev/ci/scripts/docker-entrypoint-stage2.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=/passenger/dev/ci/scripts
+PASSENGER_ROOT=/passenger
+# shellcheck source=../lib/functions.sh
+source "$SELFDIR/../lib/functions.sh"
+
+# shellcheck source=../lib/setup-container.sh
+source "$PASSENGER_ROOT/dev/ci/lib/setup-container.sh"
+add_bundler_path_to_gem_path
+echo
+echo
+
+header "Running test suite: $1"
+# shellcheck source=/dev/null
+source "$PASSENGER_ROOT/dev/ci/tests/$1/run"

data/dev/ci/scripts/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=/passenger/dev/ci/scripts
+PASSENGER_ROOT=/passenger
+# shellcheck source=../lib/functions.sh
+source "$SELFDIR/../lib/functions.sh"
+cd "$PASSENGER_ROOT"
+
+header "Inside Docker container"
+
+autodetect_environment
+echo
+
+export CI_COMMAND="./dev/ci/run-tests-with-docker $*"
+exec "$PASSENGER_ROOT/dev/ci/scripts/debug-console-wrapper.sh" \
+	"$PASSENGER_ROOT/dev/ci/scripts/docker-entrypoint-stage2.sh" "$@"

data/dev/ci/scripts/inituidgid

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+chown -R "$APP_UID:$APP_GID" /home/app
+groupmod -g "$APP_GID" app
+usermod -u "$APP_UID" -g "$APP_GID" app
+
+# There's something strange with either Docker or the kernel, so that
+# the 'app' user cannot access its home directory even after a proper
+# chown/chmod. We work around it like this.
+mv /home/app /home/app2
+cp -dpR /home/app2 /home/app
+rm -rf /home/app2
+
+if [[ $# -gt 0 ]]; then
+	exec "$@"
+fi

data/dev/ci/scripts/run-tests-natively-stage2.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../../.." && pwd)
+# shellcheck source=lib/functions.sh
+source "$SELFDIR/../lib/functions.sh"
+
+# shellcheck source=lib/set-container-envvars.sh
+source "$SELFDIR/../lib/set-container-envvars.sh"
+add_bundler_path_to_gem_path
+echo
+
+header "Running test suite: $1"
+# shellcheck source=/dev/null
+source "$PASSENGER_ROOT/dev/ci/tests/$1/run"

data/dev/ci/scripts/setup-host-natively.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../../.." && pwd)
+# shellcheck source=../lib/functions.sh
+source "$SELFDIR/../lib/functions.sh"
+
+# shellcheck source=../lib/setup-container.sh
+source "$PASSENGER_ROOT/dev/ci/lib/setup-container.sh"

data/dev/ci/setup-host

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -eo pipefail
+
+SELFDIR=$(dirname "$0")
+SELFDIR=$(cd "$SELFDIR" && pwd)
+PASSENGER_ROOT=$(cd "$SELFDIR/../.." && pwd)
+# shellcheck source=lib/functions.sh
+source "$SELFDIR/lib/functions.sh"
+cd "$PASSENGER_ROOT"
+
+header "Setting up stuff on the host"
+
+autodetect_environment
+sanity_check_environment
+echo
+
+if [[ "$OS" = macos && "$1" = "" ]]; then
+	echo "Usage: ./dev/ci/setup-host <TEST NAME>" >&2
+	exit 1
+fi
+
+if $IN_JENKINS; then
+	# Relax permissions. Necessary for unit tests which test permissions.
+	header2 "Relaxing file permissions"
+	echo "+ Setting umask"
+	umask u=rwx,g=rx,o=rx
+	run chmod g+rx,o+rx .
+	echo "+ chmodding files"
+	find ./* -type f -print0 | xargs -0 -n 512 chmod g+r,o+r
+	echo "+ chmodding directories"
+	find ./* -type d -print0 | xargs -0 -n 512 chmod g+rx,o+rx
+
+	# Create this file now because otherwise it would be owned by root,
+	# which Jenkins cannot remove.
+	run touch test/test.log
+
+	echo
+fi
+
+header2 "Creating cache directories"
+run mkdir -p "$CACHE_DIR"
+run mkdir -p "$CACHE_DIR/bundle"
+run mkdir -p "$CACHE_DIR/ccache"
+echo
+
+if [[ "$OS" = macos ]]; then
+	export CI_COMMAND="./dev/ci/setup-host $*"
+	exec "$PASSENGER_ROOT/dev/ci/scripts/debug-console-wrapper.sh" \
+		"$PASSENGER_ROOT/dev/ci/scripts/setup-host-natively.sh" "$@"
+fi

data/dev/ci/tests/apache2/run

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+run ./bin/passenger-install-apache2-module --auto
+run rvmsudo ./bin/passenger-install-apache2-module --auto --no-compile
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:integration:apache2

data/dev/ci/tests/apache2/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes NODE_MODULES=yes

data/dev/ci/tests/cxx/run

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+header2 "Running C++ tests"
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:cxx
+echo
+
+header2 "Running OXT tests"
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:oxt

data/dev/ci/tests/cxx/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes

data/dev/ci/tests/nginx-dynamic/run

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+TEST_DYNAMIC_WITH_NGINX_VERSION=1.9.15
+NGINX_ADDON_DIR=$(./bin/passenger-config --nginx-addon-dir)
+echo "+ Nginx addon dir: $NGINX_ADDON_DIR"
+echo
+
+header2 "Compliing Passenger"
+run drake "-j$COMPILE_CONCURRENCY" nginx:as_dynamic_module
+echo
+
+header2 "Downloading and compiling Nginx"
+cd buildout
+run curl -sSLO https://www.nginx.org/download/nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION.tar.gz
+run tar -xzf nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION.tar.gz
+echo "+ Entering nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION"
+cd "nginx-$TEST_DYNAMIC_WITH_NGINX_VERSION"
+run ./configure --with-cc='ccache cc' --add-dynamic-module="$NGINX_ADDON_DIR"
+run make "-j$COMPILE_CONCURRENCY"

data/dev/ci/tests/nginx-dynamic/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes

data/dev/ci/tests/nginx/run

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+run ./bin/passenger-install-nginx-module --auto --prefix=/tmp/nginx --auto-download
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:integration:nginx

data/dev/ci/tests/nginx/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes NODE_MODULES=yes

data/dev/ci/tests/nodejs/run

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:node

data/dev/ci/tests/nodejs/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes NODE_MODULES=yes

data/dev/ci/tests/ruby/run

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:ruby

data/dev/ci/tests/ruby/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes

data/dev/ci/tests/source-packaging/run

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+run bundle exec rspec -f s -c test/integration_tests/source_packaging_test.rb

data/dev/ci/tests/source-packaging/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes DOCTOOLS=yes

data/dev/ci/tests/standalone/run

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+run bundle exec drake "-j$COMPILE_CONCURRENCY" test:integration:standalone

data/dev/ci/tests/standalone/setup

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+retry_run 3 rake test:install_deps BASE_DEPS=yes

data/dev/copy_boost_headers

@@ -83,6 +83,7 @@ PROGRAM_SOURCE = %q{
   #include <boost/make_shared.hpp>
   #include <boost/enable_shared_from_this.hpp>
   #include <boost/intrusive_ptr.hpp>
+  #include <boost/smart_ptr/intrusive_ref_counter.hpp>
   #include <boost/thread.hpp>
   #include <boost/noncopyable.hpp>
   #include <boost/function.hpp>
@@ -203,8 +204,13 @@ def copy_dependencies(cflags)
 end
 
 def start
-  if BOOST_DIR.nil? || BOOST_DIR.empty?
-    puts "Usage: copy_boost_headers.rb <boost source directory>"
+  if BOOST_DIR.nil? || BOOST_DIR.empty? || BOOST_DIR == "--help"
+    puts "Usage:"
+    puts "Remove the src/cxx_supportlib/vendor-modified/boost directory,"
+    puts "then call: copy_boost_headers <boost source directory>"
+    puts "then reapply patches to boost source files,"
+    puts "they are kept in dev/boost-patches and can be applied by:"
+    puts "patch -p1 < dev/boost-patches/000*-Patch-Name.patch"
     exit 1
   end
   begin

data/src/agent/Core/ApiServer.h

@@ -460,7 +460,7 @@ private:
 	void processConfig_getControllerConfig(Client *client, Request *req,
 		Controller *controller)
 	{
-		Json::Value config = controller->getConfigAsJson();
+		Json::Value config = controller->inspectConfig();
 		getContext()->libev->runLater(boost::bind(
 			&ApiServer::processConfig_controllerConfigGathered, this,
 			client, req, controller, config));
@@ -497,8 +497,13 @@ private:
 		unrefRequest(req, __FILE__, __LINE__);
 	}
 
-	static void configureController(Controller *controller, Json::Value json) {
-		controller->configure(json);
+	static void configureController(Controller *controller, Json::Value updates) {
+		vector<ConfigKit::Error> errors;
+		if (!controller->configure(updates, errors)) {
+			P_ERROR("Unable to apply configuration change to Core controller.\n"
+				"Configuration: " << updates.toStyledString() << "\n"
+				"Errors: " << toString(errors));
+		}
 	}
 
 	void processConfigBody(Client *client, Request *req) {
@@ -646,8 +651,9 @@ public:
 	EventFd *exitEvent;
 	vector<Authorization> authorizations;
 
-	ApiServer(ServerKit::Context *context)
-		: ParentClass(context),
+	ApiServer(ServerKit::Context *context, const ServerKit::HttpServerSchema &schema,
+		const Json::Value &initialConfig = Json::Value())
+		: ParentClass(context, schema, initialConfig),
 		  serverConnectionPath("^/server/(.+)\\.json$"),
 		  apiAccountDatabase(NULL),
 		  exitEvent(NULL)

data/src/agent/Core/Controller.h

@@ -32,7 +32,7 @@
 
 #define CC_BENCHMARK_POINT(client, req, value) \
 	do { \
-		if (OXT_UNLIKELY(benchmarkMode == value)) { \
+		if (OXT_UNLIKELY(mainConfigCache.benchmarkMode == value)) { \
 			writeBenchmarkResponse(&client, &req); \
 			return; \
 		} \
@@ -64,6 +64,7 @@
 #include <Logging.h>
 #include <MessageReadersWriters.h>
 #include <Constants.h>
+#include <ConfigKit/ConfigKit.h>
 #include <ServerKit/Errors.h>
 #include <ServerKit/HttpServer.h>
 #include <ServerKit/HttpHeaderParser.h>
@@ -76,9 +77,9 @@
 #include <Utils/IOUtils.h>
 #include <Utils/JsonUtils.h>
 #include <Utils/HttpConstants.h>
-#include <Utils/VariantMap.h>
 #include <Utils/Timer.h>
 #include <Core/ApplicationPool/ErrorRenderer.h>
+#include <Core/Controller/Config.h>
 #include <Core/Controller/Client.h>
 #include <Core/Controller/AppResponse.h>
 #include <Core/Controller/TurboCaching.h>
@@ -101,16 +102,6 @@ namespace Core {
 
 
 class Controller: public ServerKit::HttpServer<Controller, Client> {
-public:
-	enum BenchmarkMode {
-		BM_NONE,
-		BM_AFTER_ACCEPT,
-		BM_BEFORE_CHECKOUT,
-		BM_AFTER_CHECKOUT,
-		BM_RESPONSE_BEGIN,
-		BM_UNKNOWN
-	};
-
 private:
 	typedef ServerKit::HttpServer<Controller, Client> ParentClass;
 	typedef ServerKit::Channel Channel;
@@ -123,29 +114,10 @@ private:
 	// has enough bits.
 	static const unsigned int MAX_SESSION_CHECKOUT_TRY = 10;
 
-	unsigned int statThrottleRate;
-	unsigned int responseBufferHighWatermark;
-	BenchmarkMode benchmarkMode: 3;
-	bool singleAppMode: 1;
-	bool showVersionInHeader: 1;
-	bool stickySessions: 1;
-	bool gracefulExit: 1;
-
-	const VariantMap *agentsOptions;
-	psg_pool_t *stringPool;
+	ControllerMainConfigCache mainConfigCache;
+	ControllerRequestConfigCachePtr requestConfigCache;
 	StringKeyTable< boost::shared_ptr<Options> > poolOptionsCache;
 
-	StaticString defaultRuby;
-	StaticString ustRouterAddress;
-	StaticString ustRouterPassword;
-	StaticString defaultUser;
-	StaticString defaultGroup;
-	StaticString defaultServerName;
-	StaticString defaultServerPort;
-	StaticString serverSoftware;
-	StaticString defaultStickySessionsCookieName;
-	StaticString defaultVaryTurbocacheByCookie;
-
 	HashedStaticString PASSENGER_APP_GROUP_NAME;
 	HashedStaticString PASSENGER_ENV_VARS;
 	HashedStaticString PASSENGER_MAX_REQUESTS;
@@ -168,9 +140,6 @@ private:
 	HashedStaticString HTTP_STATUS;
 	HashedStaticString HTTP_TRANSFER_ENCODING;
 
-	unsigned int threadNumber;
-	StaticString serverLogName;
-
 	friend class TurboCaching<Request>;
 	friend class ResponseCache<Request>;
 	struct ev_check checkWatcher;
@@ -189,7 +158,8 @@ private:
 	void initializeFlags(Client *client, Request *req, RequestAnalysis &analysis);
 	bool respondFromTurboCache(Client *client, Request *req);
 	void initializePoolOptions(Client *client, Request *req, RequestAnalysis &analysis);
-	void fillPoolOptionsFromAgentsOptions(Options &options);
+	void fillPoolOptionsFromConfigCaches(Options &options,
+		const ControllerRequestConfigCachePtr &requestConfigCache);
 	static void fillPoolOption(Request *req, StaticString &field,
 		const HashedStaticString &name);
 	static void fillPoolOption(Request *req, int &field,
@@ -332,9 +302,6 @@ private:
 
 	/****** Internal utility functions ******/
 
-	static TurboCaching<Request>::State getTurboCachingInitialState(
-		const VariantMap *agentsOptions);
-	void generateServerLogName(unsigned int number);
 	void disconnectWithClientSocketWriteError(Client **client, int e);
 	void disconnectWithAppSocketIncompleteResponseError(Client **client);
 	void disconnectWithAppSocketReadError(Client **client, int e);
@@ -383,6 +350,7 @@ protected:
 	virtual void onNextRequestEarlyReadError(Client *client, Request *req, int errcode);
 	virtual bool shouldDisconnectClientOnShutdown(Client *client);
 	virtual bool supportsUpgrade(Client *client, Request *req);
+	virtual void onConfigChange(const ConfigKit::Store *oldConfig);
 
 
 	/****** Marked virtual so that unit tests can mock these ******/
@@ -392,6 +360,7 @@ protected:
 
 
 public:
+	// Dependencies
 	ResourceLocator *resourceLocator;
 	PoolPtr appPool;
 	UnionStation::ContextPtr unionStationContext;
@@ -399,10 +368,10 @@ public:
 
 	/****** Initialization and shutdown ******/
 
-	Controller(ServerKit::Context *context, const VariantMap *_agentsOptions,
-		unsigned int _threadNumber = 1);
+	Controller(ServerKit::Context *context, const ControllerSchema &schema,
+		const Json::Value &initialConfig);
 	virtual ~Controller();
-	void initialize();
+	virtual void initialize();
 
 
 	/****** Hooks ******/
@@ -413,8 +382,6 @@ public:
 
 	/****** State inspection and configuration ******/
 
-	virtual Json::Value getConfigAsJson() const;
-	virtual void configure(const Json::Value &doc);
 	virtual Json::Value inspectStateAsJson() const;
 	virtual Json::Value inspectClientStateAsJson(const Client *client) const;
 	virtual Json::Value inspectRequestStateAsJson(const Request *req) const;
@@ -422,7 +389,6 @@ public:
 
 	/****** Miscellaneous *******/
 
-	static BenchmarkMode parseBenchmarkMode(const StaticString mode);
 	void disconnectLongRunningConnections(const StaticString &gupid);
 };