passenger 5.0.0.beta3 → 5.0.0.rc1

data/dev/runner

@@ -9,25 +9,25 @@ require File.expand_path(File.dirname(__FILE__) + "/../lib/phusion_passenger")
 PhusionPassenger.locate_directories
 
 while true
-	if ARGV[0] =~ /^-r(.*)/
-		if $1.empty?
-			lib = ARGV[1]
-			ARGV.shift
-			ARGV.shift
-		else
-			lib = $1
-			ARGV.shift
-		end
-		begin
-			PhusionPassenger.require_passenger_lib(lib)
-		rescue LoadError
-			require(lib)
-		end
-	else
-		break
-	end
+  if ARGV[0] =~ /^-r(.*)/
+    if $1.empty?
+      lib = ARGV[1]
+      ARGV.shift
+      ARGV.shift
+    else
+      lib = $1
+      ARGV.shift
+    end
+    begin
+      PhusionPassenger.require_passenger_lib(lib)
+    rescue LoadError
+      require(lib)
+    end
+  else
+    break
+  end
 end
 
 module PhusionPassenger
-	p eval(ARGV.join(" "))
+  p eval(ARGV.join(" "))
 end

data/doc/ServerOptimizationGuide.txt.md

@@ -307,14 +307,67 @@ If you cannot use the event MPM, consider putting Apache behind an Nginx reverse
 
 ### Turbocaching
 
-Phusion Passenger supports turbocaching since version 4. Turbocaching is an HTTP cache built inside Phusion Passenger. When used correctly, the cache can accelerate your app tremendously.
+Phusion Passenger supports turbocaching since version 4. Turbocaching is an HTTP cache built inside Phusion Passenger. When used correctly, the cache can accelerate your app tremendously. To utilize turbocaching, you only need to set HTTP caching headers.
 
-To utilize turbocaching, you only need to set HTTP caching headers. Please refer to [Google's HTTP caching tutorial](https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching). Phusion Passenger takes advantage of the HTTP headers automatically.
+#### Learning about HTTP caching headers
+
+The first thing you should do is to [learn how to use HTTP caching headers](https://developers.google.com/web/fundamentals/performance/optimizing-content-efficiency/http-caching). It's pretty simple and straightforward. Since the turbocache is just a normal HTTP shared cache, it respects all the HTTP caching rules.
+
+#### Set an Expires or Cache-Control header
+
+To activate the turbocache, the response must contain either an "Expires" header or a "Cache-Control" header.
+
+The "Expires" header tells the turbocache how long to cache a response. Its value is an HTTP timestamp, e.g. "Thu, 01 Dec 1994 16:00:00 GMT".
+
+The Cache-Control header is a more advanced header that not only allows you to set the caching time, but also how the cache should behave. The easiest way to use it is to set the **max-age** flag, which has the same effect as setting "Expires". For example, this tells the turbocache that the response is cacheable for at most 60 seconds:
+
+    Cache-Control: max-age=60
+
+As you can see, a "Cache-Control" header is much easier to generate than an "Expires" header. Furthermore, "Expires" doesn't work if the visitor's computer's clock is wrongly configured, while "Cache-Control" does. This is why we recommend using "Cache-Control".
+
+Another flag to be aware of is the **private** flag. This flag tells any shared caches -- caches which are meant to store responses for many users -- not to cache the response. The turbocache is a shared cache. However, the browser's cache is not, so the browser can still cache the response. You should set the "private" flag on responses which are meant for a single user, as you will learn later in this article.
+
+And finally, there is the **no-store** flag, which tells *all* caches -- even the browser's -- not to cache the response.
+
+Here is an example of a response which is cacheable for 60 seconds by the browser's cache, but not by the turbocache:
+
+    Cache-Control: max-age=60,private
+
+The HTTP specification specifies a bunch of other flags, but they're not relevant for the turbocache. 
+
+#### Only GET requests are cacheable
+
+The turbocache currently only caches GET requests. POST, PUT, DELETE and other requests are never cached. If you want your response to be cacheable by the turbocache, be sure to use GET requests, but also be sure that your request is idempotent.
+
+#### Avoid using the "Vary" header
+
+The "Vary" header is used to tell caches that the response depends on one or more request headers. But the turbocache does not implement support for the "Vary" header, so if you output a "Vary" header then the turbocache will not cache your response at all. Avoid using the "Vary" header where possible.
 
 ### Out-of-band garbage collection
 
 Phusion Passenger supports out-of-band garbage collection for Ruby apps. With this feature enabled, Phusion Passenger can run the garbage collector in between requests, so that the garbage collector doesn't delay the app as much. Please refer to the Users Guide for more information about this feature.
 
+### Using the builtin HTTP engine
+
+In certain situations, using the builtin HTTP engine in Passenger Standalone may yield some performance benefits because it skips a layer of processing.
+
+Passenger normally works by integrating into Nginx or Apache. As described in the [Design & Architecture](Design%20and%20architecture.html) document, requests are first handled by Nginx or Apache, and then forwarded to the Passenger core process (the HelperAgent) and the application process. This architecture provides various benefits, such as security benefits (Nginx and Apache's HTTP connection handling routines are thoroughly battle-tested and secure) and feature benefits (e.g. Gzip compression, superb static file handling).
+
+This is even true if you use the Standalone mode. Although it acts standalone, it is implemented under the hood by running Passenger in a builtin Nginx engine.
+
+However, the fact that all requests go through Nginx or Apache means that there is a slight overhead, which can be avoided. This overhead is small (much smaller than typical application and network overhead), and using Nginx or Apache is very useful, but in certain special situations it may be beneficial to skip this layer.
+
+ * In **microbenchmarks**, the overhead of Nginx and Apache are very noticeable. Removing Nginx and Apache from the setup, and benchmarking against the Passenger HelperAgent directly, will yield much better results.
+ * In some **multi-server setups**, Nginx and Apache may be redundant. Recall that in typical multi-server setups there is a load balancer which forwards requests to one of the many web servers. Each web server in this setup runs Passenger. But the load balancer is sometimes already responsible for many of the tasks that Nginx and Apache perform, e.g. the secure handling of HTTP connections, buffering, slow client protection or even static file serving. In these cases, removing Nginx and Apache from the web servers and load balancing to the Passenger HelperAgent directly may have a minor improvement on performance.
+
+Nginx and Apache can be removed by using Passenger's builtin HTTP engine. By using this engine, Passenger will listen directly on a socket for HTTP requests, without using Nginx or Apache.
+
+This builtin HTTP engine can be accessed by starting Passenger Standalone using the `--engine=builtin` parameter, like this:
+
+    passenger start --engine=builtin
+
+It should be noted that the builtin HTTP engine has fewer features than the Nginx engine, by design. For example the builtin HTTP engine does not support serving static files, nor does it support gzip compression. Thus, we recommend using the Nginx engine in most situations, unless you have special needs such as documented above.
+
 ## Benchmarking recommendations
 
 ### Tooling recommendations

data/doc/Users guide Nginx.txt

@@ -430,32 +430,6 @@ Please see <<deploying_a_rack_app,Deploying a Rack-based Ruby application>>
 and <<deploying_a_wsgi_app,Deploying a WSGI (Python) application>>
 for examples.
 
-.This configuration option is NOT inherited across contexts
-[WARNING]
-===============================================
-In each new context (e.g. in each new 'location' block), you must re-specify `passenger_enabled`. Values set in parent contexts have no effect on subcontexts. For example:
-
-------------------------------
-server {
-    ...
-    passenger_enabled on;
-
-    location /users {
-        passenger_set_header X-Foo bar;
-        # !!!THIS IS WRONG!!! We did not re-specify 'passenger_enabled on'
-        # here, so Phusion Passenger will not handle requests that begin with
-        # /users.
-    }
-
-    location /apps {
-        passenger_enabled on;
-        passenger_set_header X-Foo bar;
-        # This is correct. Here we re-specify 'passenger_enabled'.
-    }
-}
-------------------------------
-===============================================
-
 [[PassengerBaseURI]]
 ==== passenger_base_uri <uri>
 Used to specify that the given URI is an distinct application that should

data/doc/Users guide Standalone.txt

@@ -100,8 +100,12 @@ The following configuration options are supported:
 `ssl_port`::
 	Equivalent to the `--ssl-port` command line option.
 +
-When in mass deployment mode, you will probably want to set a different `port` too. If you don't, and you end up in a situation in which a port is used for both HTTP and HTTPS traffic, then the builtin Nginx core will abort with an error.
+When in <<mass_deployment,mass deployment mode>>, you will probably want to set a different `port` too. If you don't, and you end up in a situation in which a port is used for both HTTP and HTTPS traffic, then the builtin Nginx core will abort with an error.
 
+`server_names` (Enterprise only)::
+	Only relevant in <<mass_deployment,mass deployment mode>>. This option specifies which server names (host names) should be routed to this application. This option must be an array of names, e.g. `"server_names": ["foo.com", "bar.com"]`. By default, the name of the application directory is used as the server name. For example, for an application located in /webapps/foobar.com, "foobar.com" is used as the server name.
+`envvars` (since 5.0.0 RC 1)::
+	Environment variables to pass to the application. This option must be a key-value map, e.g. `"envvars": { "ADMIN_EMAIL": "root@phusion.nl", "SECRET_KEY": "12345" }`.
 `nginx_config_template`::
 	Equivalent to the `--nginx-config-template` command line option.
 

data/doc/users_guide_snippets/tips.txt

@@ -487,6 +487,10 @@ Because hooks are inherently tied to the implementation of Phusion Passenger, th
 	Called at the very beginning of Phusion Passenger's life cycle, during the start of the Watchdog process. The first hook is called before initialization is performed (before the HelperAgent is started). Errors in the hook script cause Phusion Passenger to abort.
 `after_watchdog_initialization`::
 	Like `before_watchdog_initialization`, but called after initialization of all Phusion Passenger agent processes. Errors in the hook script cause Phusion Passenger to abort.
+`before_watchdog_shutdown`::	
+	Called after an exit signal has been noticed (e.g. webserver exit), before the Watchdog starts terminating agents 
+`after_watchdog_shutdown`::
+	Called after the Watchdog is done and about to exit
 `attached_process`::
 	Called when Phusion Passenger has successfully spawned an application processes and added it to the process pool. Extra environment variables: `PASSENGER_PROCESS_PID`, `PASSENGER_APP_ROOT`. Errors in the hook script are ignored.
 `detached_process`::
@@ -507,6 +511,11 @@ Extra environment variables: `PASSENGER_PROCESS_PID`, `PASSENGER_APP_ROOT`. Erro
 - Operating system-level problems, such as running out of memory.
 - The application taking too long to start, and hitting Phusion Passenger's timeout.
 
++
+`queue_full_error` (since 5.0.0 RC 1)::
+	The server rejects new requests (default: HTTP 503) while the request queue is full (https://www.phusionpassenger.com/documentation/Users%20guide%20Nginx.html#passenger_max_request_queue_size). 
+	This hook gets called for each rejection.
+	
 +
 Extra environment variables:
 +

data/ext/common/ApplicationPool2/Group.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2014 Phusion
+ *  Copyright (c) 2011-2015 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -206,9 +206,9 @@ public:
 		unsigned int restartsInitiated);
 	void spawnThreadRealMain(const SpawnerPtr &spawner, const Options &options,
 		unsigned int restartsInitiated);
-	void finalizeRestart(GroupPtr self, Options options, RestartMethod method,
-		SpawnerFactoryPtr spawnerFactory, unsigned int restartsInitiated,
-		boost::container::vector<Callback> postLockActions);
+	void finalizeRestart(GroupPtr self, Options oldOptions, Options newOptions,
+		RestartMethod method, SpawnerFactoryPtr spawnerFactory,
+		unsigned int restartsInitiated, boost::container::vector<Callback> postLockActions);
 	void startCheckingDetachedProcesses(bool immediately);
 	void detachedProcessesCheckerMain(GroupPtr self);
 	void wakeUpGarbageCollector();
@@ -221,6 +221,7 @@ public:
 	void callAbortLongRunningConnectionsCallback(const ProcessPtr &process);
 	psg_pool_t *getPallocPool() const;
 	const ResourceLocator &getResourceLocator() const;
+	bool prepareHookScriptOptions(HookScriptOptions &hsOptions, const char *name);
 	void runAttachHooks(const ProcessPtr process) const;
 	void runDetachHooks(const ProcessPtr process) const;
 	void setupAttachOrDetachHook(const ProcessPtr process, HookScriptOptions &options) const;
@@ -328,14 +329,18 @@ public:
 	#endif
 
 	/**
-	 * Sets options for this Group. Called at creation time and at restart time.
+	 * Persists options into this Group. Called at creation time and at restart time.
+	 * Values will be persisted into `destination`. Or if it's NULL, into `this->options`.
 	 */
-	void resetOptions(const Options &newOptions) {
-		options = newOptions;
-		options.persist(newOptions);
-		options.clearPerRequestFields();
-		options.groupSecret = StaticString(secret, SECRET_SIZE);
-		options.groupUuid   = uuid;
+	void resetOptions(const Options &newOptions, Options *destination = NULL) {
+		if (destination == NULL) {
+			destination = &this->options;
+		}
+		*destination = newOptions;
+		destination->persist(newOptions);
+		destination->clearPerRequestFields();
+		destination->groupSecret = StaticString(secret, SECRET_SIZE);
+		destination->groupUuid   = uuid;
 	}
 
 	/**
@@ -442,6 +447,13 @@ public:
 			P_WARN("Request queue is full. Returning an error");
 			postLockActions.push_back(boost::bind(GetCallback::call,
 				callback, SessionPtr(), boost::make_shared<RequestQueueFullException>()));
+
+			HookScriptOptions hsOptions;
+			if (prepareHookScriptOptions(hsOptions, "queue_full_error")) {
+				// TODO <Feb 17, 2015] DK> should probably rate limit this, since we are already at heavy load
+				postLockActions.push_back(boost::bind(runHookScripts, hsOptions));
+			}
+
 			return false;
 		}
 	}

data/ext/common/ApplicationPool2/Implementation.cpp

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2014 Phusion
+ *  Copyright (c) 2011-2015 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -526,13 +526,14 @@ SuperGroup::realDoRestart(const Options &options, unsigned int generation) {
 }
 
 
-Group::Group(SuperGroup *_superGroup, const Options &options, const ComponentInfo &info)
+Group::Group(SuperGroup *_superGroup, const Options &_options, const ComponentInfo &info)
 	: superGroup(_superGroup),
 	  name(_superGroup->name + "#" + info.name),
 	  uuid(generateUuid(_superGroup)),
 	  componentInfo(info)
 {
 	generateSecret(_superGroup, secret);
+	resetOptions(_options);
 	enabledCount   = 0;
 	disablingCount = 0;
 	disabledCount  = 0;
@@ -556,7 +557,6 @@ Group::Group(SuperGroup *_superGroup, const Options &options, const ComponentInf
 		restartFile = options.appRoot + "/" + options.restartDir + "/restart.txt";
 		alwaysRestartFile = options.appRoot + "/" + options.restartDir + "/always_restart.txt";
 	}
-	resetOptions(options);
 
 	detachedProcessesCheckerActive = false;
 }
@@ -1181,6 +1181,7 @@ Group::restart(const Options &options, RestartMethod method) {
 	detachAll(actions);
 	getPool()->interruptableThreads.create_thread(
 		boost::bind(&Group::finalizeRestart, this, shared_from_this(),
+			this->options.copyAndPersist().clearPerRequestFields(),
 			options.copyAndPersist().clearPerRequestFields(),
 			method, getPool()->spawnerFactory, restartsInitiated, actions),
 		"Group restarter: " + name,
@@ -1190,8 +1191,11 @@ Group::restart(const Options &options, RestartMethod method) {
 
 // The 'self' parameter is for keeping the current Group object alive while this thread is running.
 void
-Group::finalizeRestart(GroupPtr self, Options options, RestartMethod method,
-	SpawnerFactoryPtr spawnerFactory, unsigned int restartsInitiated,
+Group::finalizeRestart(GroupPtr self,
+	Options oldOptions,
+	Options newOptions, RestartMethod method,
+	SpawnerFactoryPtr spawnerFactory,
+	unsigned int restartsInitiated,
 	boost::container::vector<Callback> postLockActions)
 {
 	TRACE_POINT();
@@ -1203,7 +1207,9 @@ Group::finalizeRestart(GroupPtr self, Options options, RestartMethod method,
 	this_thread::disable_syscall_interruption dsi;
 
 	// Create a new spawner.
-	SpawnerPtr newSpawner = spawnerFactory->create(options);
+	Options spawnerOptions = oldOptions;
+	resetOptions(newOptions, &spawnerOptions);
+	SpawnerPtr newSpawner = spawnerFactory->create(spawnerOptions);
 	SpawnerPtr oldSpawner;
 
 	UPDATE_TRACE_POINT();
@@ -1239,7 +1245,7 @@ Group::finalizeRestart(GroupPtr self, Options options, RestartMethod method,
 	UPDATE_TRACE_POINT();
 
 	// Atomically swap the new spawner with the old one.
-	resetOptions(options);
+	resetOptions(newOptions);
 	oldSpawner = spawner;
 	spawner    = newSpawner;
 
@@ -1464,6 +1470,25 @@ Group::getResourceLocator() const {
 	return *getPool()->getSpawnerConfig()->resourceLocator;
 }
 
+/* Given a hook name like "queue_full_error", we return HookScriptOptions filled in with this name and a spec
+ * (user settings that can be queried from agentsOptions using the external hook name that is prefixed with "hook_")
+
+ * @return false if the user parameters (agentsOptions) are not available (e.g. during ApplicationPool2_PoolTest)
+ */
+bool
+Group::prepareHookScriptOptions(HookScriptOptions &hsOptions, const char *name) {
+	SpawnerConfigPtr config = getPool()->getSpawnerConfig();
+	if (config->agentsOptions == NULL) {
+		return false;
+	}
+
+	hsOptions.name = name;
+	string hookName = string("hook_") + name;
+	hsOptions.spec = config->agentsOptions->get(hookName, false);
+
+	return true;
+}
+
 // 'process' is not a reference so that bind(runAttachHooks, ...) causes the shared
 // pointer reference to increment.
 void

data/ext/common/ApplicationPool2/Pool.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2014 Phusion
+ *  Copyright (c) 2011-2015 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -827,8 +827,24 @@ public:
 		while (!this_thread::interruption_requested()) {
 			try {
 				UPDATE_TRACE_POINT();
-				unsigned long long sleepTime = self->realCollectAnalytics();
-				syscalls::usleep(sleepTime);
+				self->realCollectAnalytics();
+			} catch (const thread_interrupted &) {
+				break;
+			} catch (const tracable_exception &e) {
+				P_WARN("ERROR: " << e.what() << "\n  Backtrace:\n" << e.backtrace());
+			}
+
+			// Sleep for about 4 seconds, aligned to seconds boundary
+			// for saving power on laptops.
+			UPDATE_TRACE_POINT();
+			unsigned long long currentTime = SystemTime::getUsec();
+			unsigned long long deadline =
+				roundUp<unsigned long long>(currentTime, 1000000) + 4000000;
+			P_DEBUG("Analytics collection done; next analytics collection in " <<
+				std::fixed << std::setprecision(3) << ((deadline - currentTime) / 1000000.0) <<
+				" sec");
+			try {
+				syscalls::usleep(deadline - currentTime);
 			} catch (const thread_interrupted &) {
 				break;
 			} catch (const tracable_exception &e) {
@@ -902,7 +918,7 @@ public:
 		}
 	}
 
-	unsigned long long realCollectAnalytics() {
+	void realCollectAnalytics() {
 		TRACE_POINT();
 		this_thread::disable_interruption di;
 		this_thread::disable_syscall_interruption dsi;
@@ -944,14 +960,14 @@ public:
 			processMetrics = ProcessMetricsCollector().collect(pids);
 		} catch (const ParseException &) {
 			P_WARN("Unable to collect process metrics: cannot parse 'ps' output.");
-			goto end;
+			return;
 		}
 		try {
 			UPDATE_TRACE_POINT();
 			systemMetricsCollector.collect(systemMetrics);
 		} catch (const RuntimeException &e) {
 			P_WARN("Unable to collect system metrics: " << e.what());
-			goto end;
+			return;
 		}
 
 		{
@@ -1008,17 +1024,6 @@ public:
 			// Run destructors with updated trace point.
 			actions.clear();
 		}
-
-		end:
-		// Sleep for about 4 seconds, aligned to seconds boundary
-		// for saving power on laptops.
-		unsigned long long currentTime = SystemTime::getUsec();
-		unsigned long long deadline =
-			roundUp<unsigned long long>(currentTime, 1000000) + 4000000;
-		P_DEBUG("Analytics collection done; next analytics collection in " <<
-			std::fixed << std::setprecision(3) << ((deadline - currentTime) / 1000000.0) <<
-			" sec");
-		return deadline - currentTime;
 	}
 
 	SuperGroupPtr createSuperGroup(const Options &options) {

data/ext/common/ApplicationPool2/SmartSpawner.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2011-2014 Phusion
+ *  Copyright (c) 2011-2015 Phusion
  *
  *  "Phusion Passenger" is a trademark of Hongli Lai & Ninh Bui.
  *
@@ -339,6 +339,9 @@ private:
 				"ruby_libdir: " + config->resourceLocator->getRubyLibDir() + "\n"
 				"passenger_version: " PASSENGER_VERSION "\n"
 				"UNIX_PATH_MAX: " + toString(UNIX_PATH_MAX) + "\n";
+			if (!details.options->groupSecret.empty()) {
+				data.append("connect_password: " + details.options->groupSecret + "\n");
+			}
 			if (!config->instanceDir.empty()) {
 				data.append("socket_dir: " + config->instanceDir + "/apps.s\n");
 			}