passenger 5.3.4 → 6.0.1

data/src/cxx_supportlib/ConfigKit/README.md

@@ -20,6 +20,7 @@ ConfigKit is a configuration management system that lets you define configuratio
     - [Flags](#flags)
   - [Defining default values](#defining-default-values)
   - [Defining custom validators](#defining-custom-validators)
+  - [Nested schemas](#nested-schemas)
   - [Inspecting the schema](#inspecting-the-schema)
 - [Using the store](#using-the-store)
   - [Putting data in the store](#putting-data-in-the-store)
@@ -171,9 +172,9 @@ The following types are available:
  * `UINT_TYPE` -- an unsigned integer.
  * `FLOAT_TYPE` -- a floating point number.
  * `BOOL_TYPE` -- a boolean.
- * `ARRAY_TYPE` -- an generic array. May contain any values.
+ * `ARRAY_TYPE` -- usually a generic array (which may contain any values). But with a special `add()` invocation this can indicate an array of [nested ConfigKit schemas](#nested-schemas).
  * `STRING_ARRAY_TYPE` -- an array of strings.
- * `OBJECT_TYPE` -- a generic JSON object. May contain any values.
+ * `OBJECT_TYPE` -- usually a generic JSON object (which may contain any values). But with a special `add()` invocation this can indicate a map of [nested ConfigKit schemas](#nested-schemas).
  * `ANY_TYPE` -- any JSON value.
 
 #### Flags
@@ -227,6 +228,81 @@ Miscellaneous notes about custom validators:
  - They are always run, even if the normal type validation fails. For example if the caller tries to set the "foo" key to an array value (which is incompatible with the string type), then `myValidator` will still be called.
  - All registered validators are called. A validator cannot prevent other validators from running.
 
+### Nested schemas
+
+> **Warning:** do not confuse nested schemas with **subschemas**. The latter is a mechanism for composing multiple schemas together, not a way to define the type of a field.
+
+It is possible to define a config field that is an array of objects, or a string map of objects, that should conform to a certain schema. We call such a schema a **nested schema**. For example:
+
+~~~c++
+ConfigKit::Schema personSchema;
+personSchema.add("name", STRING_TYPE, REQUIRED);
+personSchema.add("age", INT_TYPE, OPTIONAL);
+personSchema.finalize();
+
+ConfigKit::Schema mainSchema;
+mainSchema.add("people", ARRAY_TYPE, personSchema, REQUIRED);
+mainSchema.add("frobnicate", BOOL_TYPE, OPTIONAL, false);
+mainSchema.finalize();
+~~~
+
+A field with a nested schema is defined by calling `add(<field name>, ARRAY_TYPE or OBJECT_TYPE, <nested schema object>, <flags>)`.
+
+The above example mainSchema says the following format is valid:
+
+~~~javascript
+{
+  "people": [
+    {
+      "name": "John",
+      "age": 12 // optional
+    },
+    {
+      "name": "Jane"
+    },
+    // ...
+  }
+  ],
+  "frobinate": boolean-value
+}
+~~~
+
+Nested schema fields simply accept corresponding JSON values, like this:
+
+~~~c++
+Json::Value initialValue;
+initialValue["people"][0]["name"] = "John";
+initialValue["people"][0]["age"] = 12;
+initialValue["people"][1]["name"] = "Jane";
+ConfigKit::Store store(mainSchema, initialValue);
+~~~
+
+Note that it is not possible to define a nested schema field with a default value. This is because the default values are taken from the nested schema. For example, take this code which inserts an empty object into the "people" array:
+
+~~~c++
+ConfigKit::Schema personSchema;
+personSchema.add("name", STRING_TYPE, OPTIONAL, "anonymous");
+personSchema.finalize();
+
+ConfigKit::Schema mainSchema;
+mainSchema.add("people", ARRAY_TYPE, personSchema, REQUIRED);
+mainSchema.finalize();
+
+Json::Value initialValue;
+initialValue["people"][0] = Json::objectValue;
+ConfigKit::Store store(mainSchema, initialValue);
+~~~
+
+Because personSchema defines a default value of "anonymous" on its "name" field, the above code would result in the following effective values:
+
+~~~json
+{
+  "people": [
+    { "name": "anonymous" }
+  ]
+}
+~~~
+
 ### Inspecting the schema
 
 You can inspect the schema using the `inspect()` method. It returns a Json::Value in the following format:
@@ -248,6 +324,17 @@ You can inspect the schema using the `inspect()` method. It returns a Json::Valu
   "password": {
     "type": "string",
     "secret": true
+  },
+  "people": {
+    "type": "array",
+    "nested_schema": {
+      "name": {
+        "type": "string"
+      },
+      "age": {
+        "type": "integer"
+      }
+    }
   }
 }
 ~~~
@@ -258,6 +345,7 @@ Description of the members:
  - `required`: whether this key is required.
  - `has_default_value`: "static" if a static a default value is defined, "dynamic" if a dynamic default value is defined.
  - `default_value`: the static default value. This field is absent when there is no default value, or if the default value is dynamic.
+ - `nested_schema`: if this field has a [nested schema](#nested-schemas), then a description of that nested schema is stored here.
 
 ## Using the store
 

data/src/cxx_supportlib/ConfigKit/Schema.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2017 Phusion Holding B.V.
+ *  Copyright (c) 2017-2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -40,7 +40,7 @@
 #include <ConfigKit/DummyTranslator.h>
 #include <ConfigKit/Utils.h>
 #include <DataStructures/StringKeyTable.h>
-#include <Utils/StrIntUtils.h>
+#include <StrIntTools/StrIntUtils.h>
 
 namespace Passenger {
 namespace ConfigKit {
@@ -60,18 +60,22 @@ public:
 		Flags flags;
 		ValueGetter defaultValueGetter;
 		ValueFilter inspectFilter;
+		// Can only be non-NULL when type == ARRAY_TYPE or OBJECT_TYPE.
+		const Schema *nestedSchema;
 
 		Entry()
 			: type(UNKNOWN_TYPE),
-			  flags(OPTIONAL)
+			  flags(OPTIONAL),
+			  nestedSchema(NULL)
 			{ }
 
 		Entry(Type _type, Flags _flags, const ValueGetter &_defaultValueGetter,
-			const ValueFilter &_inspectFilter)
+			const ValueFilter &_inspectFilter, const Schema *_nestedSchema = NULL)
 			: type(_type),
 			  flags(_flags),
 			  defaultValueGetter(_defaultValueGetter),
-			  inspectFilter(_inspectFilter)
+			  inspectFilter(_inspectFilter),
+			  nestedSchema(_nestedSchema)
 			{ }
 
 		bool tryTypecastValue(const Json::Value &val, Json::Value &result) const {
@@ -117,25 +121,34 @@ public:
 					return false;
 				}
 			case ARRAY_TYPE:
-				if (val.isConvertibleTo(Json::arrayValue)) {
-					result = val;
-					return true;
+			case OBJECT_TYPE: {
+				Json::ValueType targetType;
+				if (type == ARRAY_TYPE) {
+					targetType = Json::arrayValue;
 				} else {
-					return false;
+					targetType = Json::objectValue;
 				}
-			case OBJECT_TYPE:
-				if (val.isConvertibleTo(Json::objectValue)) {
-					result = val;
-					return true;
+				if (val.isConvertibleTo(targetType)) {
+					if (nestedSchema != NULL) {
+						return tryTypecastArrayOrObjectValueWithNestedSchema(val,
+							result, "user_value");
+					} else {
+						result = val;
+						return true;
+					}
 				} else {
 					return false;
 				}
+			}
 			default:
 				result = val;
 				return true;
 			}
 		}
 
+		bool tryTypecastArrayOrObjectValueWithNestedSchema(const Json::Value &val,
+			Json::Value &result, const char *userOrEffectiveValue) const;
+
 		Json::Value inspect() const {
 			Json::Value result(Json::objectValue);
 			inspect(result);
@@ -161,6 +174,9 @@ public:
 					doc["default_value"] = Schema::getStaticDefaultValue(*this);
 				}
 			}
+			if (nestedSchema != NULL) {
+				doc["nested_schema"] = nestedSchema->inspect();
+			}
 		}
 	};
 
@@ -208,6 +224,11 @@ private:
 
 	static Json::Value getStaticDefaultValue(const Schema::Entry &entry);
 
+	static bool validateNestedSchemaArrayValue(const HashedStaticString &key,
+		const Entry &entry, const Json::Value &value, vector<Error> &errors);
+	static bool validateNestedSchemaObjectValue(const HashedStaticString &key,
+		const Entry &entry, const Json::Value &value, vector<Error> &errors);
+
 public:
 	Schema()
 		: finalized(false)
@@ -237,6 +258,20 @@ public:
 		}
 	}
 
+	/**
+	 * Register a new schema entry whose value corresponds to a nested schema.
+	 */
+	EntryBuilder add(const HashedStaticString &key, Type type,
+		const Schema &nestedSchema, unsigned int flags)
+	{
+		assert(!finalized);
+		assert(nestedSchema.finalized);
+		assert(type == ARRAY_TYPE || type == OBJECT_TYPE);
+		Entry entry(type, (Flags) flags, ValueGetter(), ValueFilter(),
+			&nestedSchema);
+		return EntryBuilder(entries.insert(key, entry)->value);
+	}
+
 	/**
 	 * Register a new schema entry with a dynamic default value.
 	 */
@@ -407,7 +442,12 @@ public:
 			}
 		case ARRAY_TYPE:
 			if (value.isConvertibleTo(Json::arrayValue)) {
-				return true;
+				if (entry->nestedSchema == NULL) {
+					return true;
+				} else {
+					return validateNestedSchemaArrayValue(key, *entry,
+						value, errors);
+				}
 			} else {
 				errors.push_back(Error("'{{" + key + "}}' must be an array"));
 				return false;
@@ -428,7 +468,12 @@ public:
 			}
 		case OBJECT_TYPE:
 			if (value.isObject()) {
-				return true;
+				if (entry->nestedSchema == NULL) {
+					return true;
+				} else {
+					return validateNestedSchemaObjectValue(key, *entry,
+						value, errors);
+				}
 			} else {
 				errors.push_back(Error("'{{" + key + "}}' must be a JSON object"));
 				return false;

data/src/cxx_supportlib/ConfigKit/Store.h

@@ -46,7 +46,7 @@
 #include <LoggingKit/Assert.h>
 #include <Exceptions.h>
 #include <DataStructures/StringKeyTable.h>
-#include <Utils/StrIntUtils.h>
+#include <StrIntTools/StrIntUtils.h>
 
 namespace Passenger {
 namespace ConfigKit {
@@ -94,8 +94,16 @@ private:
 		Json::Value getEffectiveValue(const Store &store) const {
 			if (userValue.isNull()) {
 				return getDefaultValue(store);
-			} else {
+			} else if (schemaEntry->nestedSchema == NULL) {
 				return userValue;
+			} else {
+				// The user value may contain nulls that should
+				// be populated with the default value from the
+				// corresponding nested schema.
+				Json::Value result;
+				schemaEntry->tryTypecastArrayOrObjectValueWithNestedSchema(
+					userValue, result, "effective_value");
+				return result;
 			}
 		}
 	};
@@ -105,12 +113,20 @@ private:
 	bool updatedOnce;
 
 	static Json::Value getEffectiveValue(const Json::Value &userValue,
-		const Json::Value &defaultValue)
+		const Json::Value &defaultValue, const Schema::Entry &schemaEntry)
 	{
 		if (userValue.isNull()) {
 			return defaultValue;
-		} else {
+		} else if (schemaEntry.nestedSchema == NULL) {
 			return userValue;
+		} else {
+			// The user value may contain nulls that should
+			// be populated with the default value from the
+			// corresponding nested schema.
+			Json::Value result;
+			schemaEntry.tryTypecastArrayOrObjectValueWithNestedSchema(
+				userValue, result, "effective_value");
+			return result;
 		}
 	}
 
@@ -425,7 +441,8 @@ public:
 			const Json::Value &effectiveValue =
 				subdoc["effective_value"] =
 					getEffectiveValue(subdoc["user_value"],
-						subdoc["default_value"]);
+						subdoc["default_value"],
+						*entry.schemaEntry);
 			schema->validateValue(it.getKey(), effectiveValue, tmpErrors);
 
 			result[it.getKey()] = subdoc;
@@ -585,6 +602,39 @@ public:
 };
 
 
+inline bool
+Schema::Entry::tryTypecastArrayOrObjectValueWithNestedSchema(const Json::Value &val,
+	Json::Value &result, const char *userOrEffectiveValue) const
+{
+	assert(type == ARRAY_TYPE || type == OBJECT_TYPE);
+	assert(nestedSchema != NULL);
+	assert(!val.isNull());
+	assert(val.isConvertibleTo(Json::arrayValue)
+		|| val.isConvertibleTo(Json::objectValue));
+
+	bool ok = true;
+	result = val;
+
+	Json::Value::iterator it, end = result.end();
+	for (it = result.begin(); it != end; it++) {
+		Json::Value &userSubdoc = *it;
+		if (!userSubdoc.isConvertibleTo(Json::objectValue)) {
+			ok = false;
+			continue;
+		}
+
+		vector<Error> errors;
+		Json::Value preview = Store(*nestedSchema).previewUpdate(
+			userSubdoc, errors);
+		Json::Value::const_iterator p_it, p_end = preview.end();
+		for (p_it = preview.begin(); p_it != p_end; p_it++) {
+			const Json::Value &previewSubdoc = *p_it;
+			userSubdoc[p_it.name()] = previewSubdoc[userOrEffectiveValue];
+		}
+	}
+	return ok;
+}
+
 inline Json::Value
 Schema::getValueFromSubSchema(
 	const Store &store,
@@ -648,6 +698,80 @@ Schema::getStaticDefaultValue(const Schema::Entry &entry) {
 	return Store::maybeFilterSecret(storeEntry, storeEntry.getDefaultValue(Store()));
 }
 
+inline bool
+Schema::validateNestedSchemaArrayValue(const HashedStaticString &key,
+	const Schema::Entry &entry, const Json::Value &value, vector<Error> &errors)
+{
+	bool warnedAboutNonObjectValue = false;
+	bool result = true;
+
+	Json::Value::const_iterator it, end = value.end();
+	for (it = value.begin(); it != end; it++) {
+		if (!it->isConvertibleTo(Json::objectValue)) {
+			if (!warnedAboutNonObjectValue) {
+				warnedAboutNonObjectValue = true;
+				result = false;
+				errors.push_back(Error(
+					"'{{" + key + "}}' may only contain JSON objects"));
+			}
+			continue;
+		}
+
+		Store store(*entry.nestedSchema);
+		vector<Error> nestedSchemaErrors;
+		if (store.update(*it, nestedSchemaErrors)) {
+			continue;
+		}
+
+		vector<Error>::const_iterator e_it, e_end = nestedSchemaErrors.end();
+		for (e_it = nestedSchemaErrors.begin(); e_it != e_end; e_it++) {
+			errors.push_back(Error("'{{" + key + "}}' element "
+				+ Passenger::toString(it.index() + 1) + " is invalid: "
+				+ e_it->getMessage()));
+		}
+		result = false;
+	}
+
+	return result;
+}
+
+inline bool
+Schema::validateNestedSchemaObjectValue(const HashedStaticString &key,
+	const Schema::Entry &entry, const Json::Value &value, vector<Error> &errors)
+{
+	bool warnedAboutNonObjectValue = false;
+	bool result = true;
+
+	Json::Value::const_iterator it, end = value.end();
+	for (it = value.begin(); it != end; it++) {
+		if (!it->isConvertibleTo(Json::objectValue)) {
+			if (!warnedAboutNonObjectValue) {
+				warnedAboutNonObjectValue = true;
+				result = false;
+				errors.push_back(Error(
+					"'{{" + key + "}}' may only contain JSON objects"));
+			}
+			continue;
+		}
+
+		Store store(*entry.nestedSchema);
+		vector<Error> nestedSchemaErrors;
+		if (store.update(*it, nestedSchemaErrors)) {
+			continue;
+		}
+
+		vector<Error>::const_iterator e_it, e_end = nestedSchemaErrors.end();
+		for (e_it = nestedSchemaErrors.begin(); e_it != e_end; e_it++) {
+			errors.push_back(Error("'{{" + key + "}}' key '"
+				+ it.name() + "' is invalid: "
+				+ e_it->getMessage()));
+		}
+		result = false;
+	}
+
+	return result;
+}
+
 
 } // namespace ConfigKit
 } // namespace Passenger

data/src/cxx_supportlib/Constants.h

@@ -81,7 +81,7 @@
 #define PASSENGER_API_VERSION_MAJOR 0
 #define PASSENGER_API_VERSION_MINOR 3
 #define PASSENGER_DEFAULT_USER "nobody"
-#define PASSENGER_VERSION "5.3.4"
+#define PASSENGER_VERSION "6.0.1"
 #define POOL_HELPER_THREAD_STACK_SIZE 262144
 #define PROCESS_SHUTDOWN_TIMEOUT 60
 #define PROCESS_SHUTDOWN_TIMEOUT_DISPLAY "1 minute"

data/src/cxx_supportlib/{Utils → DataStructures}/HashMap.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2010-2017 Phusion Holding B.V.
+ *  Copyright (c) 2010-2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -24,8 +24,8 @@
  *  THE SOFTWARE.
  */
 
-#ifndef _PASSENGER_HASH_MAP_H_
-#define _PASSENGER_HASH_MAP_H_
+#ifndef _PASSENGER_DATA_STRUCTURES_HASH_MAP_H_
+#define _PASSENGER_DATA_STRUCTURES_HASH_MAP_H_
 
 /*
  * There are too many ways to include hash_map/unordered_map!
@@ -57,4 +57,4 @@
 	#define HashMap boost::unordered_map
 #endif
 
-#endif /* _PASSENGER_HASH_MAP_H_ */
+#endif /* _PASSENGER_DATA_STRUCTURES_HASH_MAP_H_ */

data/src/cxx_supportlib/DataStructures/HashedStaticString.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2014-2017 Phusion Holding B.V.
+ *  Copyright (c) 2014-2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -23,14 +23,14 @@
  *  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  *  THE SOFTWARE.
  */
-#ifndef _PASSENGER_HASHED_STATIC_STRING_H_
-#define _PASSENGER_HASHED_STATIC_STRING_H_
+#ifndef _PASSENGER_DATA_STRUCTURES_HASHED_STATIC_STRING_H_
+#define _PASSENGER_DATA_STRUCTURES_HASHED_STATIC_STRING_H_
 
 #include <boost/cstdint.hpp>
 #include <oxt/macros.hpp>
 #include <string>
 #include <StaticString.h>
-#include <Utils/Hasher.h>
+#include <Algorithms/Hasher.h>
 
 namespace Passenger {
 
@@ -101,4 +101,4 @@ public:
 
 } // namespace Passenger
 
-#endif /* _PASSENGER_HASHED_STATIC_STRING_H_ */
+#endif /* _PASSENGER_DATA_STRUCTURES_HASHED_STATIC_STRING_H_ */

data/src/cxx_supportlib/DataStructures/LString.h

@@ -1,6 +1,6 @@
 /*
  *  Phusion Passenger - https://www.phusionpassenger.com/
- *  Copyright (c) 2014-2017 Phusion Holding B.V.
+ *  Copyright (c) 2014-2018 Phusion Holding B.V.
  *
  *  "Passenger", "Phusion Passenger" and "Union Station" are registered
  *  trademarks of Phusion Holding B.V.
@@ -36,8 +36,8 @@
 #include <MemoryKit/palloc.h>
 #include <MemoryKit/mbuf.h>
 #include <StaticString.h>
-#include <Utils/StrIntUtils.h>
-#include <Utils/Hasher.h>
+#include <StrIntTools/StrIntUtils.h>
+#include <Algorithms/Hasher.h>
 
 namespace Passenger {