alchemymvc 1.4.2 → 1.4.4

package/lib/app/datasource/mongo_datasource.js

@@ -722,64 +722,87 @@ Mongo.setMethod(function _remove(context) {
  */
 Mongo.setMethod(function _ensureIndex(model, index, callback) {
 
-	this.collection(model.table, async function gotCollection(err, collection) {
+	// `collection()` returns a promise since 1.4 (its callback form was removed),
+	// so this must await it - the old callback form silently never fired, leaving
+	// indexes added via `addIndex` uncreated.
+	let pledge = Swift.waterfall(
+		this.collection(model.table),
+		async (collection) => {
 
-		if (err != null) {
-			return callback(err);
-		}
+			let options = {
+				name   : index.options.name,
+				unique : index.options.unique ? true : false,
+				sparse : index.options.sparse ? true : false,
+			};
 
-		let options = {
-			name   : index.options.name,
-			unique : index.options.unique ? true : false,
-			sparse : index.options.sparse ? true : false,
-		};
+			let index_specs;
 
-		let index_specs;
+			// Hack in the text indexes
+			if (options.name == 'text') {
+				index_specs = {};
 
-		// Hack in the text indexes
-		if (options.name == 'text') {
-			let key;
+				for (let key in index.fields) {
+					index_specs[key] = 'text';
+				}
+			} else {
+				index_specs = index.fields;
+			}
 
-			index_specs = {};
+			// Reconcile any conflicting index to the schema's definition. A single
+			// drop can expose a second conflict (the wanted key may already exist
+			// under a different name), and concurrent boot-time ensures can drop an
+			// index from under us - so loop, dropping the conflicting index each
+			// round, until the create succeeds. Bounded, so a genuine problem still
+			// surfaces instead of spinning.
+			let attempts = 0;
 
-			for (key in index.fields) {
-				index_specs[key] = 'text';
-			}
-		} else {
-			index_specs = index.fields;
-		}
+			while (true) {
 
-		try {
-			await collection.createIndex(index_specs, options);
-		} catch (err) {
+				try {
+					await collection.createIndex(index_specs, options);
+					return;
+				} catch (err) {
 
-			// Check for IndexOptionsConflict
-			if (err.code === 85) {
-				let index_to_drop;
+					// 85 = IndexOptionsConflict (our key held under a different name),
+					// 86 = IndexKeySpecsConflict (our name held by a different key).
+					if ((err.code !== 85 && err.code !== 86) || attempts >= 5) {
+						throw err;
+					}
 
-				if (err.message.includes('already exists with a different name:')) {
-					index_to_drop = err.message.after('different name:').trim();
-				}
+					attempts++;
 
-				if (!index_to_drop) {
-					index_to_drop = options.name;
-				}
+					// Find the conflicting index(es) by inspecting the collection -
+					// the one holding our name, or one holding our exact key - rather
+					// than parsing the version-specific error message.
+					let specs_json = JSON.stringify(index_specs),
+					    dropped = false;
 
-				try {
+					for (let existing of await collection.indexes()) {
 
-					// Index already exists, drop it
-					await collection.dropIndex(index_to_drop);
+						if (existing.name === '_id_') {
+							continue;
+						}
 
-					// Try again
-					await collection.createIndex(index_specs, options);
-				} catch (second_err) {
-					return callback(second_err);
+						if (existing.name === options.name || JSON.stringify(existing.key) === specs_json) {
+							try {
+								await collection.dropIndex(existing.name);
+								dropped = true;
+							} catch (drop_err) {
+								// Already gone (a concurrent ensure dropped it).
+							}
+						}
+					}
+
+					// Nothing identifiable to drop: surface the error, don't spin.
+					if (!dropped) {
+						throw err;
+					}
 				}
-			} else {
-				return callback(err);
 			}
 		}
+	);
 
-		callback();
-	});
+	pledge.done(callback);
+
+	return pledge;
 });

package/lib/app/helper_component/paginate_component.js

@@ -107,12 +107,7 @@ Paginate.setMethod(function find(model, criteria) {
 			return;
 		}
 
-		let PageInfo = conduit.internal('PageInfo');
-
-		if (PageInfo == null) {
-			PageInfo = {};
-			conduit.internal('PageInfo', PageInfo);
-		}
+		let PageInfo = conduit.internal('PageInfo') || {};
 
 		let entry = {
 			page     : page,
@@ -127,6 +122,11 @@ Paginate.setMethod(function find(model, criteria) {
 		}
 
 		PageInfo[name] = entry;
+
+		// Store the populated object, not an empty one we mutate afterwards: the
+		// renderer's internal store snapshots/transforms the value on set, so a
+		// later mutation of a previously-stored object no longer propagates.
+		conduit.internal('PageInfo', PageInfo);
 	});
 
 	return pledge;

package/lib/app/helper_datasource/05-fallback_datasource.js

@@ -130,6 +130,13 @@ Fallback.setMethod(function storeInUpperDatasource(model, data, options) {
 		// Allow extraneous values (the ones we just added in this function)
 		options.extraneous = true;
 
+		// Store the *converted* data (with the temp values added above), not the
+		// original app-side record. Without this the upper datasource caches the
+		// raw working values: e.g. a class-typed field is structured-cloned by
+		// IndexedDB into a plain, prototype-less object that can't be revived on
+		// read. The normal create()/update() flow does this same setConvertedData.
+		context.setConvertedData(data);
+
 		let promise;
 
 		if (options.create === false) {

package/lib/app/helper_datasource/remote_datasource.js

@@ -81,7 +81,7 @@ Remote.setMethod(async function doServerCommand(action, model, data, callback) {
 	let fetch_options = {
 		post        : data,
 		headers     : {'content-type': 'application/json-dry'},
-		max_timeout : this.options.max_timeout ?? 3500 // Configurable timeout, default 3.5s
+		max_timeout : this.options.max_timeout ?? 15000 // Configurable timeout, default 15s
 	};
 
 	alchemy.fetch(route_name, fetch_options, function gotResult(err, result) {

package/lib/app/helper_model/document.js

@@ -1478,12 +1478,45 @@ Document.setMethod(function hasFieldValue(name) {
 	return Object.hasProperty(this.$main, name);
 });
 
+/**
+ * Compare two field values, treating them as equal when their datasource
+ * (stored) form matches.
+ *
+ * `hasChanged()` compares the live, normalized field value (a cast object such
+ * as a TagTree, an ObjectId instance, ...) against the original record, which
+ * holds the raw stored form. Without this, a normalized value always looked
+ * different from its own unchanged stored counterpart, so such fields reported
+ * a change on every load. If both sides serialize to the same datasource value,
+ * saving either would write identical data, so the field has not changed.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.4
+ * @version  1.4.4
+ *
+ * @param    {Mixed}   a
+ * @param    {Mixed}   b
+ *
+ * @return   {boolean}
+ */
+Document.setMethod(function alikeWhenStored(a, b) {
+
+	if (Object.alike(a, b)) {
+		return true;
+	}
+
+	try {
+		return Object.alike(JSON.clone(a, 'toDatasource'), JSON.clone(b, 'toDatasource'));
+	} catch (err) {
+		return false;
+	}
+});
+
 /**
  * Has this document changed since it was created?
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.0.4
- * @version  1.3.0
+ * @version  1.4.4
  *
  * @param    {string}   name   The optional field name
  *
@@ -1505,27 +1538,47 @@ Document.setMethod(function hasChanged(name) {
 
 	let result;
 
+	// Fields excluded from change-detection (computed fields, or `track_changes:
+	// false`) never count as changed and are never compared. The `$model` getter
+	// throws for an unresolvable model (a detached document, or a client document
+	// whose model is not registered), so guard against that and compare every
+	// field when no schema is available.
+	let untracked;
+	try {
+		let schema = this.$model && this.$model.schema;
+		untracked = schema ? schema.getUntrackedFieldNames() : null;
+	} catch (err) {
+		untracked = null;
+	}
+
 	// If we only want to check a single field
 	if (name) {
 		let current_value,
 		    old_value;
-		
-		if (name.includes('.')) {
-			current_value = Object.path(this, name);
-			old_value = Object.path(this.$attributes.original_record, name);
+
+		if (untracked && untracked.has(name)) {
+			result = false;
 		} else {
-			current_value = this[name];
-			old_value = this.$attributes.original_record[name];
-		}
 
-		result = !Object.alike(old_value, current_value);
+			if (name.includes('.')) {
+				current_value = Object.path(this, name);
+				old_value = Object.path(this.$attributes.original_record, name);
+			} else {
+				current_value = this[name];
+				old_value = this.$attributes.original_record[name];
+			}
+
+			result = !this.alikeWhenStored(old_value, current_value);
+		}
 	} else {
 
 		let key;
 
 		for (key in this.$attributes.original_record) {
-			if (!Object.alike(this.$attributes.original_record[key], this[key])) {
-				// @TODO: some special fields always end up being different
+			if (untracked && untracked.has(key)) {
+				continue;
+			}
+			if (!this.alikeWhenStored(this.$attributes.original_record[key], this[key])) {
 				result = true;
 				break;
 			}
@@ -1533,7 +1586,10 @@ Document.setMethod(function hasChanged(name) {
 
 		if (!result) {
 			for (key in this.$main) {
-				if (!Object.alike(this.$main[key], this.$attributes.original_record[key])) {
+				if (untracked && untracked.has(key)) {
+					continue;
+				}
+				if (!this.alikeWhenStored(this.$main[key], this.$attributes.original_record[key])) {
 					result = true;
 					break;
 				}

package/lib/class/plugin.js

@@ -116,7 +116,7 @@ Plugin.setMethod(async function startPlugin() {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.4.0
- * @version  1.4.0
+ * @version  1.4.3
  *
  * @return   {boolean}
  */
@@ -138,31 +138,16 @@ Plugin.setMethod(function loadSettingDefinitions() {
 	// Get/create this plugin's group definition
 	let group = this.getSettingsGroup();
 
-	// Get any already-existing settings
-	let parent = alchemy.system_settings.get('plugins');
-	let existing = parent.get(this.name);
-
-	if (existing) {
-		// Remove the existing settings.
-		// They are not linked properly anyway.
-		parent.remove(this.name);
-	}
-
-	// Require the settings.js file now
+	// Register the plugin's setting definitions
 	this.useFile(settings_path, {client: false});
 
-	// Create the new group settings with default values
-	let default_group = group.generateValue();
-
-	if (this.default_settings) {
-		default_group.setDefaultValue(this.default_settings);
-	}
-
-	if (existing) {
-		default_group.setValueSilently(existing);
-	}
-
-	parent.injectSubGroupValue(this.name, default_group);
+	// Rebuild this plugin's sub-group from its real definitions, preserving any
+	// values already applied to it (config overrides / an orphan created before
+	// the plugin loaded) plus the plugin's hard-coded default_settings. This is
+	// the same orphan-fix the general reconciler uses
+	// (Setting.GroupValue#rebuildSubGroup) - a single source of truth.
+	let parent = alchemy.system_settings.get('plugins');
+	parent.rebuildSubGroup(this.name, group, this.default_settings);
 
 	return true;
 });

package/lib/class/schema_client.js

@@ -862,6 +862,10 @@ Schema.setMethod(function addField(name, type, options) {
 
 	this.set(name, field);
 
+	// A new field can change which fields are excluded from change-detection, so
+	// drop the cached set (rebuilt lazily by getUntrackedFieldNames).
+	this.untracked_field_names = null;
+
 	if (options.rules) {
 		let rules = Array.cast(options.rules),
 		    i;
@@ -1074,6 +1078,50 @@ Schema.setMethod(function getFieldNames() {
 	return Object.keys(this.dict);
 });
 
+/**
+ * The set of field names excluded from change-detection, computed once and
+ * cached. A field is excluded when `options.track_changes === false`, or - when
+ * `track_changes` is unset - when it is a computed field (`is_computed`), since
+ * its stored value is regenerated from its inputs and so is never independent
+ * state worth (deep-)comparing in `Document#hasChanged()`. Lazily computed and
+ * cached; `addField` clears the cache, so a later field addition is reflected.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.4
+ * @version  1.4.4
+ *
+ * @return   {Set<string>}
+ */
+Schema.setMethod(function getUntrackedFieldNames() {
+
+	if (this.untracked_field_names) {
+		return this.untracked_field_names;
+	}
+
+	let names = new Set();
+
+	for (let name of this.getFieldNames()) {
+
+		let field = this.getField(name);
+
+		if (!field || !field.options) {
+			continue;
+		}
+
+		let skip = (field.options.track_changes != null)
+			? (field.options.track_changes === false)
+			: !!field.options.is_computed;
+
+		if (skip) {
+			names.add(name);
+		}
+	}
+
+	this.untracked_field_names = names;
+
+	return names;
+});
+
 /**
  * Add an index
  *

package/lib/core/alchemy.js

@@ -2439,6 +2439,36 @@ Alchemy.setMethod(function start(options, callback) {
 	// Indicate the server is starting
 	starting = true;
 
+	// `--migrate`: apply pending migrations (app/migrations/) and exit, without
+	// starting the HTTP server - so it can run alongside a live instance.
+	if (this.argv.migrate) {
+
+		STAGES.afterStages('settings', async () => {
+
+			let code = 0;
+
+			try {
+				await Classes.Alchemy.Migration.start();
+			} catch (err) {
+				log.error('Migration run failed:', err);
+				code = 1;
+			}
+
+			process.exit(code);
+		});
+
+		STAGES.launch([
+			'load_app',
+			'datasource',
+			'tasks',
+			'settings',
+		]);
+
+		Blast.doLoaded();
+
+		return this.ready(callback);
+	}
+
 	// Start the stages
 	STAGES.launch([
 		'load_app',

package/lib/core/setting.js

@@ -1255,6 +1255,118 @@ GroupValue.setMethod(function injectSubGroupValue(name, value) {
 	this[VALUE][name] = value;
 });
 
+/**
+ * Rebuild a single sub-group value from its real definition, preserving any
+ * values that were already applied to the current (possibly orphaned) node.
+ *
+ * A config-file override that lands before a group's definition is registered
+ * leaves an "orphan" value backed by an ad-hoc, action-less definition (see
+ * `Group#assign`). This rebuilds the node from the real definition so it
+ * regains its full sub-structure and its leaves' actions, then silently
+ * replays the captured values. It fires NO actions - those run once, later,
+ * in the `settings.perform_actions` stage.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.3
+ * @version  1.4.3
+ *
+ * @param    {string}                  name                 Child to rebuild
+ * @param    {Alchemy.Setting.Group}   group_definition     The real definition
+ * @param    {Object}                  [hardcoded_defaults] Defaults to re-apply
+ *
+ * @return   {Alchemy.Setting.GroupValue}
+ */
+GroupValue.setMethod(function rebuildSubGroup(name, group_definition, hardcoded_defaults) {
+
+	// Capture whatever value currently sits under `name` (orphan or not).
+	let existing = this.get(name);
+
+	if (existing) {
+		// Detach it so the slot is rebuilt cleanly.
+		this.remove(name);
+	}
+
+	// Materialise a fresh value tree from the REAL definition: recursively
+	// creates every real child with its default and the real `.definition`
+	// (which carries the action / global_variable an orphan stub lacks).
+	let fresh = group_definition.generateValue();
+
+	// Re-apply any hard-coded defaults (e.g. a plugin's `default_settings`).
+	if (hardcoded_defaults) {
+		fresh.setDefaultValue(hardcoded_defaults);
+	}
+
+	// Replay the previously-applied values WITHOUT firing actions. Only a
+	// group value can be replayed onto a group; a scalar sitting where a group
+	// belongs is a malformed override - drop it (and warn) rather than crash.
+	if (existing) {
+		if (existing.is_group) {
+			fresh.setValueSilently(existing);
+		} else {
+			alchemy.distinctProblem(
+				'setting-scalar-over-group-' + group_definition.setting_id,
+				'Setting "' + group_definition.setting_id + '" is a group but received a non-group override value; ignoring it'
+			);
+		}
+	}
+
+	// Slot the rebuilt node back into the live tree, in place.
+	this.injectSubGroupValue(name, fresh);
+
+	return fresh;
+});
+
+/**
+ * Walk this group's REAL definition tree and rebuild any sub-group whose live
+ * value was fabricated ad-hoc (by a config-file override that arrived before
+ * the definition was registered). Healthy nodes - identified by definition
+ * identity - are left untouched (recursed only, to catch nested orphans), so
+ * this is a strict no-op when there are no orphans. Fires NO actions.
+ *
+ * Intended to run after all setting definitions are registered and before the
+ * database values are applied / actions are fired (the `settings.reconcile`
+ * stage).
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.3
+ * @version  1.4.3
+ */
+GroupValue.setMethod(function reconcileOrphanGroups() {
+
+	let definition = this.definition;
+
+	if (!definition || !definition.children) {
+		return;
+	}
+
+	for (let [child_name, child_definition] of definition.children) {
+
+		// Only sub-groups can be orphaned. A leaf either already carries the
+		// real definition, or is an unknown key with nothing to rebuild.
+		if (!child_definition.is_group) {
+			continue;
+		}
+
+		let live = this.get(child_name);
+
+		// Nothing applied here yet: nothing to preserve, nothing orphaned.
+		if (!live) {
+			continue;
+		}
+
+		// Healthy node: its value is backed by the canonical definition.
+		// Leave it byte-for-byte, but recurse to catch nested orphans.
+		if (live.is_group && live.definition === child_definition) {
+			live.reconcileOrphanGroups();
+			continue;
+		}
+
+		// Orphan: the live value carries an ad-hoc definition. Rebuild it from
+		// the real definition, replaying the captured values silently.
+		this.rebuildSubGroup(child_name, child_definition);
+	}
+});
+
 /**
  * Get all the setting values with executable actions in order.
  *
@@ -1429,7 +1541,7 @@ GroupValue.setMethod(function _setPath(silent, path, raw_value) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.4.0
- * @version  1.4.0
+ * @version  1.4.3
  *
  * @param    {string|Array}   path
  * @param    {Value}
@@ -1450,7 +1562,31 @@ GroupValue.setMethod(function forceValueInstanceAtPath(path, value) {
 
 	while (path.length && current) {
 		let next = path.shift();
-		current = current.get(next);
+
+		let child = current.get(next);
+
+		// Create missing intermediate group values instead of crashing.
+		// (A partial config-file override of a group can drop sibling
+		// sub-group value instances; recreate them on demand here.)
+		if (!child) {
+
+			// Can only create a child under a group value; a leaf has no
+			// child definitions to generate from.
+			if (!current.is_group) {
+				return;
+			}
+
+			let child_definition = current.definition.get(next);
+
+			if (!child_definition) {
+				return;
+			}
+
+			child = child_definition.generateValue();
+			current.injectSubGroupValue(next, child);
+		}
+
+		current = child;
 	}
 
 	current[VALUE][last] = value;

package/lib/stages/20-settings.js

@@ -15,13 +15,32 @@ const settings = STAGES.createStage('settings');
 // Do not start this stage before the datasources are connected
 settings.dependsOn('datasource.connect');
 
+/**
+ * "settings.reconcile"
+ * Rebuild any setting groups that were orphaned by a config-file override
+ * landing before the group's definition was registered (app definitions load
+ * in `app_bootstrap`, plugins in `load_app.plugins` - both after the config
+ * merge in the Alchemy constructor). Runs before `load` so the database values
+ * land on the real-definition value nodes, and before `perform_actions` so
+ * actions fire exactly once over the reconciled tree. Silent: fires no actions.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.3
+ * @version  1.4.3
+ *
+ * @type     {Alchemy.Stages.Stage}
+ */
+const reconcile = settings.createStage('reconcile', () => {
+	alchemy.system_settings.reconcileOrphanGroups();
+});
+
 /**
  * "settings.load"
  * Load the settings from the database
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.4.0
- * @version  1.4.0
+ * @version  1.4.3
  *
  * @type     {Alchemy.Stages.Stage}
  */
@@ -38,6 +57,11 @@ const load = settings.createStage('load', async () => {
 	}
 });
 
+// Orphan reconciliation must finish before DB values are applied, so the
+// values land on the real-definition nodes. Creation order already runs
+// `reconcile` first; this makes the contract explicit and reorder-proof.
+load.dependsOn('settings.reconcile');
+
 /**
  * "settings.perform_actions"
  * Do all the setting-associated actions.

package/lib/stages/50-routes.js

@@ -204,7 +204,7 @@ const middleware = routes.createStage('middleware', () => {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.4.0
- * @version  1.4.0
+ * @version  1.4.3
  *
  * @type     {Alchemy.Stages.Stage}
  */
@@ -212,6 +212,16 @@ const app_routes = routes.createStage('app_routes', () => {
 	try {
 		alchemy.useOnce(libpath.resolve(PATH_APP, 'config', 'routes.js'));
 	} catch (err) {
+
+		// A missing routes.js is fine (the file is optional). Any other error
+		// is a real problem inside the file and must NOT be swallowed - that
+		// would leave the app running with no routes and only a warning.
+		let is_not_found = err.code === 'ENOENT' || err.message.indexOf('Cannot find') > -1;
+
+		if (!is_not_found) {
+			throw err;
+		}
+
 		// Only output warning when not in client mode
 		if (!alchemy.getSetting('client_mode')) {
 			log.warn('No app routes were found:', err);

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "alchemymvc",
 	"description": "MVC framework for Node.js",
-	"version": "1.4.2",
+	"version": "1.4.4",
 	"author": "Jelle De Loecker <jelle@elevenways.be>",
 	"keywords": [
 		"alchemy",