alchemymvc 1.4.3 → 1.4.5

package/lib/app/behaviour/revision_behaviour.js

@@ -285,6 +285,16 @@ Revision.setMethod(function afterSave(record, options, created) {
 	// Find the complete saved item
 	Model.get(this.model.model_name).findByPk(doc.$pk, async function gotRecord(err, result) {
 
+		try {
+			await createRevision(result);
+			pledge.resolve();
+		} catch (err) {
+			pledge.reject(err);
+		}
+	});
+
+	async function createRevision(result) {
+
 		if (result) {
 
 			// Get the new data
@@ -320,13 +330,13 @@ Revision.setMethod(function afterSave(record, options, created) {
 					[that.revision_model.model_name] : revision_data
 				};
 
-				// Save the data (but do not wait for it)
-				that.revision_model.save(revision_data, {allowFields: true});
+				// Save the revision. This IS awaited: a `revert()` right after
+				// a save has to see this revision, and a fire-and-forget save
+				// would also swallow any insert error.
+				await that.revision_model.save(revision_data, {allowFields: true});
 			}
 		}
-
-		pledge.resolve();
-	});
+	}
 
 	return pledge;
 });

package/lib/app/datasource/mongo_datasource.js

@@ -722,64 +722,87 @@ Mongo.setMethod(function _remove(context) {
  */
 Mongo.setMethod(function _ensureIndex(model, index, callback) {
 
-	this.collection(model.table, async function gotCollection(err, collection) {
+	// `collection()` returns a promise since 1.4 (its callback form was removed),
+	// so this must await it - the old callback form silently never fired, leaving
+	// indexes added via `addIndex` uncreated.
+	let pledge = Swift.waterfall(
+		this.collection(model.table),
+		async (collection) => {
 
-		if (err != null) {
-			return callback(err);
-		}
+			let options = {
+				name   : index.options.name,
+				unique : index.options.unique ? true : false,
+				sparse : index.options.sparse ? true : false,
+			};
 
-		let options = {
-			name   : index.options.name,
-			unique : index.options.unique ? true : false,
-			sparse : index.options.sparse ? true : false,
-		};
+			let index_specs;
 
-		let index_specs;
+			// Hack in the text indexes
+			if (options.name == 'text') {
+				index_specs = {};
 
-		// Hack in the text indexes
-		if (options.name == 'text') {
-			let key;
+				for (let key in index.fields) {
+					index_specs[key] = 'text';
+				}
+			} else {
+				index_specs = index.fields;
+			}
 
-			index_specs = {};
+			// Reconcile any conflicting index to the schema's definition. A single
+			// drop can expose a second conflict (the wanted key may already exist
+			// under a different name), and concurrent boot-time ensures can drop an
+			// index from under us - so loop, dropping the conflicting index each
+			// round, until the create succeeds. Bounded, so a genuine problem still
+			// surfaces instead of spinning.
+			let attempts = 0;
 
-			for (key in index.fields) {
-				index_specs[key] = 'text';
-			}
-		} else {
-			index_specs = index.fields;
-		}
+			while (true) {
 
-		try {
-			await collection.createIndex(index_specs, options);
-		} catch (err) {
+				try {
+					await collection.createIndex(index_specs, options);
+					return;
+				} catch (err) {
 
-			// Check for IndexOptionsConflict
-			if (err.code === 85) {
-				let index_to_drop;
+					// 85 = IndexOptionsConflict (our key held under a different name),
+					// 86 = IndexKeySpecsConflict (our name held by a different key).
+					if ((err.code !== 85 && err.code !== 86) || attempts >= 5) {
+						throw err;
+					}
 
-				if (err.message.includes('already exists with a different name:')) {
-					index_to_drop = err.message.after('different name:').trim();
-				}
+					attempts++;
 
-				if (!index_to_drop) {
-					index_to_drop = options.name;
-				}
+					// Find the conflicting index(es) by inspecting the collection -
+					// the one holding our name, or one holding our exact key - rather
+					// than parsing the version-specific error message.
+					let specs_json = JSON.stringify(index_specs),
+					    dropped = false;
 
-				try {
+					for (let existing of await collection.indexes()) {
 
-					// Index already exists, drop it
-					await collection.dropIndex(index_to_drop);
+						if (existing.name === '_id_') {
+							continue;
+						}
 
-					// Try again
-					await collection.createIndex(index_specs, options);
-				} catch (second_err) {
-					return callback(second_err);
+						if (existing.name === options.name || JSON.stringify(existing.key) === specs_json) {
+							try {
+								await collection.dropIndex(existing.name);
+								dropped = true;
+							} catch (drop_err) {
+								// Already gone (a concurrent ensure dropped it).
+							}
+						}
+					}
+
+					// Nothing identifiable to drop: surface the error, don't spin.
+					if (!dropped) {
+						throw err;
+					}
 				}
-			} else {
-				return callback(err);
 			}
 		}
+	);
 
-		callback();
-	});
+	pledge.done(callback);
+
+	return pledge;
 });

package/lib/app/helper_datasource/05-fallback_datasource.js

@@ -165,7 +165,7 @@ Fallback.setMethod(function storeInUpperDatasource(model, data, options) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.1.0
- * @version  1.4.2
+ * @version  1.4.5
  *
  * @param    {Alchemy.OperationalContext.ReadDocumentFromDatasource}   context
  *
@@ -183,7 +183,28 @@ Fallback.setMethod(function read(context) {
 		let lower_context = context.createChild();
 		lower_context.setCriteria(lower_criteria);
 
-		tasks.push(() => this.lower.read(lower_context));
+		// A rejected lower (remote) read must not reject the whole query:
+		// resolve to null instead, so the waterfall falls through to the upper
+		// (local cache) read. Without this, being offline made every find fail
+		// even though the cache had the data.
+		tasks.push(() => {
+			let attempt = new Swift();
+
+			Swift.done(this.lower.read(lower_context), (err, result) => {
+
+				if (err) {
+					if (Blast.isBrowser && typeof alchemy != 'undefined' && alchemy.distinctProblem) {
+						alchemy.distinctProblem('fallback-lower-read', 'Remote read failed, using local cache', {repeat_after: 60000});
+					}
+
+					return attempt.resolve(null);
+				}
+
+				attempt.resolve(result);
+			});
+
+			return attempt;
+		});
 	}
 
 	let upper_criteria = criteria.clone();
@@ -202,7 +223,7 @@ Fallback.setMethod(function read(context) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.1.0
- * @version  1.4.2
+ * @version  1.4.5
  *
  * @param    {Model}   model
  *
@@ -220,7 +241,13 @@ Fallback.setMethod(function getRecordsToSync(model) {
 	context.setModel(model);
 	context.setCriteria(criteria);
 
-	return this.upper.read(context);
+	// Datasource#read resolves with `{items, available}` since 1.4.2, but the
+	// consumers of this method (getRecordsToBeSavedRemotely) iterate the result
+	// directly - so unwrap to the items array, or offline saves never sync.
+	return Swift.waterfall(
+		this.upper.read(context),
+		result => (result && result.items) ? result.items : (result || [])
+	);
 });
 
 /**

package/lib/app/helper_datasource/remote_datasource.js

@@ -81,7 +81,7 @@ Remote.setMethod(async function doServerCommand(action, model, data, callback) {
 	let fetch_options = {
 		post        : data,
 		headers     : {'content-type': 'application/json-dry'},
-		max_timeout : this.options.max_timeout ?? 3500 // Configurable timeout, default 3.5s
+		max_timeout : this.options.max_timeout ?? 15000 // Configurable timeout, default 15s
 	};
 
 	alchemy.fetch(route_name, fetch_options, function gotResult(err, result) {

package/lib/app/helper_model/document.js

@@ -1170,6 +1170,31 @@ Document.setMethod(function save(data, options, callback) {
 
 		save_result = save_result[0];
 
+		// Fields edited while the save was in flight would be silently
+		// reverted by adopting the response wholesale (the response only
+		// reflects the record as it was sent), so detect and keep them.
+		let kept_fields;
+
+		if (in_flight_snapshot && that.$main) {
+			let key;
+
+			for (key in that.$main) {
+
+				if (key == pk_name) {
+					continue;
+				}
+
+				if (!that.alikeWhenStored(that.$main[key], in_flight_snapshot[key])) {
+
+					if (!kept_fields) {
+						kept_fields = {};
+					}
+
+					kept_fields[key] = that.$main[key];
+				}
+			}
+		}
+
 		// Use the saved data from now on
 		that.$main = save_result.$main;
 
@@ -1177,10 +1202,21 @@ Document.setMethod(function save(data, options, callback) {
 		that.$attributes.original_record = undefined;
 		that.markUnchanged();
 
-		if (that.hasObjectFields()) {
+		if (kept_fields || that.hasObjectFields()) {
 			that.storeCurrentDataAsOriginalRecord();
 		}
 
+		if (kept_fields) {
+			let key;
+
+			// Put the newer, in-flight edits back on top of the saved state
+			// and mark them as changed, so a follow-up save persists them.
+			for (key in kept_fields) {
+				that.markChangedField(key, kept_fields[key]);
+				that.$main[key] = kept_fields[key];
+			}
+		}
+
 		pledge.resolve(that);
 	}
 
@@ -1218,6 +1254,16 @@ Document.setMethod(function save(data, options, callback) {
 		}
 	}
 
+	// Snapshot the record as it is being sent, so `updateDoc` can tell which
+	// fields were edited while the save was in flight
+	let in_flight_snapshot = null;
+
+	try {
+		in_flight_snapshot = JSON.clone(main);
+	} catch (err) {
+		// Without a snapshot the response is simply adopted as-is
+	}
+
 	sub_pledge = this.$model.save(this, options, updateDoc);
 
 	return pledge;
@@ -1478,12 +1524,45 @@ Document.setMethod(function hasFieldValue(name) {
 	return Object.hasProperty(this.$main, name);
 });
 
+/**
+ * Compare two field values, treating them as equal when their datasource
+ * (stored) form matches.
+ *
+ * `hasChanged()` compares the live, normalized field value (a cast object such
+ * as a TagTree, an ObjectId instance, ...) against the original record, which
+ * holds the raw stored form. Without this, a normalized value always looked
+ * different from its own unchanged stored counterpart, so such fields reported
+ * a change on every load. If both sides serialize to the same datasource value,
+ * saving either would write identical data, so the field has not changed.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.4
+ * @version  1.4.4
+ *
+ * @param    {Mixed}   a
+ * @param    {Mixed}   b
+ *
+ * @return   {boolean}
+ */
+Document.setMethod(function alikeWhenStored(a, b) {
+
+	if (Object.alike(a, b)) {
+		return true;
+	}
+
+	try {
+		return Object.alike(JSON.clone(a, 'toDatasource'), JSON.clone(b, 'toDatasource'));
+	} catch (err) {
+		return false;
+	}
+});
+
 /**
  * Has this document changed since it was created?
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.0.4
- * @version  1.3.0
+ * @version  1.4.4
  *
  * @param    {string}   name   The optional field name
  *
@@ -1505,27 +1584,47 @@ Document.setMethod(function hasChanged(name) {
 
 	let result;
 
+	// Fields excluded from change-detection (computed fields, or `track_changes:
+	// false`) never count as changed and are never compared. The `$model` getter
+	// throws for an unresolvable model (a detached document, or a client document
+	// whose model is not registered), so guard against that and compare every
+	// field when no schema is available.
+	let untracked;
+	try {
+		let schema = this.$model && this.$model.schema;
+		untracked = schema ? schema.getUntrackedFieldNames() : null;
+	} catch (err) {
+		untracked = null;
+	}
+
 	// If we only want to check a single field
 	if (name) {
 		let current_value,
 		    old_value;
-		
-		if (name.includes('.')) {
-			current_value = Object.path(this, name);
-			old_value = Object.path(this.$attributes.original_record, name);
+
+		if (untracked && untracked.has(name)) {
+			result = false;
 		} else {
-			current_value = this[name];
-			old_value = this.$attributes.original_record[name];
-		}
 
-		result = !Object.alike(old_value, current_value);
+			if (name.includes('.')) {
+				current_value = Object.path(this, name);
+				old_value = Object.path(this.$attributes.original_record, name);
+			} else {
+				current_value = this[name];
+				old_value = this.$attributes.original_record[name];
+			}
+
+			result = !this.alikeWhenStored(old_value, current_value);
+		}
 	} else {
 
 		let key;
 
 		for (key in this.$attributes.original_record) {
-			if (!Object.alike(this.$attributes.original_record[key], this[key])) {
-				// @TODO: some special fields always end up being different
+			if (untracked && untracked.has(key)) {
+				continue;
+			}
+			if (!this.alikeWhenStored(this.$attributes.original_record[key], this[key])) {
 				result = true;
 				break;
 			}
@@ -1533,7 +1632,10 @@ Document.setMethod(function hasChanged(name) {
 
 		if (!result) {
 			for (key in this.$main) {
-				if (!Object.alike(this.$main[key], this.$attributes.original_record[key])) {
+				if (untracked && untracked.has(key)) {
+					continue;
+				}
+				if (!this.alikeWhenStored(this.$main[key], this.$attributes.original_record[key])) {
 					result = true;
 					break;
 				}

package/lib/app/helper_model/model.js

@@ -1892,20 +1892,6 @@ Model.setMethod(function saveRecord(document, options, callback) {
 
 		creating = options.create || document.$pk == null;
 		next();
-
-		return;
-
-		// Look through unique indexes if no _id is present
-		that.auditRecord(document, options, function afterAudit(err, doc) {
-
-			if (err) {
-				return next(err);
-			}
-
-			// Is a new record being created?
-			creating = options.create || doc.$pk == null;
-			next();
-		});
 	}, function doBeforeNormalize(next) {
 		// @TODO: make "beforeSave" only use promises
 		that.issueDataEvent('beforeNormalize', [document, options], next);

package/lib/class/model.js

@@ -1402,7 +1402,7 @@ Model.setMethod(function remove(id, callback) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    0.5.0
- * @version  1.2.0
+ * @version  1.4.5
  *
  * @param    {Object}    options    Find options
  * @param    {Function}  task       Task to perform on each record
@@ -1425,7 +1425,12 @@ Model.setMethod(function eachRecord(options, task, callback) {
 		options = {};
 	}
 
-	if (!callback) {
+	if (callback) {
+		// Errors are also delivered through the callback, and callers that
+		// pass one routinely discard the returned pledge - its rejection
+		// must not be reported as an uncaught error on top of the callback
+		pledge.warn_uncaught_errors = false;
+	} else {
 		callback = Function.thrower;
 	}
 

package/lib/class/schema_client.js

@@ -862,6 +862,10 @@ Schema.setMethod(function addField(name, type, options) {
 
 	this.set(name, field);
 
+	// A new field can change which fields are excluded from change-detection, so
+	// drop the cached set (rebuilt lazily by getUntrackedFieldNames).
+	this.untracked_field_names = null;
+
 	if (options.rules) {
 		let rules = Array.cast(options.rules),
 		    i;
@@ -1074,6 +1078,50 @@ Schema.setMethod(function getFieldNames() {
 	return Object.keys(this.dict);
 });
 
+/**
+ * The set of field names excluded from change-detection, computed once and
+ * cached. A field is excluded when `options.track_changes === false`, or - when
+ * `track_changes` is unset - when it is a computed field (`is_computed`), since
+ * its stored value is regenerated from its inputs and so is never independent
+ * state worth (deep-)comparing in `Document#hasChanged()`. Lazily computed and
+ * cached; `addField` clears the cache, so a later field addition is reflected.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.4
+ * @version  1.4.4
+ *
+ * @return   {Set<string>}
+ */
+Schema.setMethod(function getUntrackedFieldNames() {
+
+	if (this.untracked_field_names) {
+		return this.untracked_field_names;
+	}
+
+	let names = new Set();
+
+	for (let name of this.getFieldNames()) {
+
+		let field = this.getField(name);
+
+		if (!field || !field.options) {
+			continue;
+		}
+
+		let skip = (field.options.track_changes != null)
+			? (field.options.track_changes === false)
+			: !!field.options.is_computed;
+
+		if (skip) {
+			names.add(name);
+		}
+	}
+
+	this.untracked_field_names = names;
+
+	return names;
+});
+
 /**
  * Add an index
  *

package/lib/core/alchemy.js

@@ -2439,6 +2439,36 @@ Alchemy.setMethod(function start(options, callback) {
 	// Indicate the server is starting
 	starting = true;
 
+	// `--migrate`: apply pending migrations (app/migrations/) and exit, without
+	// starting the HTTP server - so it can run alongside a live instance.
+	if (this.argv.migrate) {
+
+		STAGES.afterStages('settings', async () => {
+
+			let code = 0;
+
+			try {
+				await Classes.Alchemy.Migration.start();
+			} catch (err) {
+				log.error('Migration run failed:', err);
+				code = 1;
+			}
+
+			process.exit(code);
+		});
+
+		STAGES.launch([
+			'load_app',
+			'datasource',
+			'tasks',
+			'settings',
+		]);
+
+		Blast.doLoaded();
+
+		return this.ready(callback);
+	}
+
 	// Start the stages
 	STAGES.launch([
 		'load_app',

package/lib/core/client_base.js

@@ -291,6 +291,13 @@ ClientBase.setMethod(function issueEvent(name, args, next) {
 		args = [];
 	}
 
+	if (next) {
+		// Errors are also delivered through `next`, and callers that pass a
+		// callback routinely discard the returned pledge - its rejection
+		// must not be reported as an uncaught error on top of the callback.
+		pledge.warn_uncaught_errors = false;
+	}
+
 	if (this.constructor.event_to_method_map) {
 		method_name = this.constructor.event_to_method_map.get(name);
 	}

package/lib/stages/50-routes.js

@@ -118,7 +118,17 @@ const hawkejs = routes.createStage('hawkejs', () => {
 				return req.conduit.notFound('Could not find any of the given templates');
 			}
 
-			req.conduit.setHeader('cache-control', 'public, max-age=3600, must-revalidate');
+			// Only allow long-lived caching when the request is versioned
+			// with the CURRENT app version: unversioned (or stale-versioned)
+			// urls would keep serving an old template for up to an hour
+			// after a deploy.
+			let requested_version = req.conduit.param('v');
+
+			if (requested_version && String(requested_version) === String(alchemy.hawkejs.app_version)) {
+				req.conduit.setHeader('cache-control', 'public, max-age=3600, must-revalidate');
+			} else {
+				req.conduit.setHeader('cache-control', 'no-cache');
+			}
 
 			// Don't use json dry, hawkejs expects regular json
 			req.conduit.json_dry = false;

package/lib/testing/browser.js

@@ -24,6 +24,78 @@
  */
 'use strict';
 
+const fs = require('fs'),
+      child_process = require('child_process');
+
+/**
+ * Check if a browser executable actually works in headless mode.
+ * Puppeteer's own bundled-Chrome download can be missing/corrupt in some
+ * sandboxes (e.g. a pinned old revision missing its ICU data file). Such a
+ * binary still exits 0 for `--version`, but hangs and never emits a CDP
+ * handshake when actually launched headless - which is what makes
+ * `puppeteer.launch()` hang until its connect-timeout instead of failing
+ * fast. So a real headless run is checked synchronously up front instead.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.5
+ * @version  1.4.5
+ *
+ * @param    {string}   executable_path
+ *
+ * @return   {boolean}
+ */
+function isUsableBrowser(executable_path) {
+
+	if (!executable_path || !fs.existsSync(executable_path)) {
+		return false;
+	}
+
+	let result = child_process.spawnSync(executable_path, [
+		'--headless',
+		'--disable-gpu',
+		'--no-sandbox',
+		'--dump-dom',
+		'about:blank',
+	], {timeout: 8000});
+
+	return result.status === 0 && !result.error;
+}
+
+/**
+ * Resolve which Chrome/Chromium executable to launch.
+ * Prefers `PUPPETEER_EXECUTABLE_PATH` (puppeteer already honors it, but we
+ * validate it here too) and puppeteer's own bundled download; falls back to
+ * common system-installed locations when neither actually works.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.5
+ * @version  1.4.5
+ *
+ * @param    {Object}   puppeteer
+ *
+ * @return   {string|undefined}
+ */
+function resolveExecutablePath(puppeteer) {
+
+	let candidates = [
+		process.env.PUPPETEER_EXECUTABLE_PATH,
+		puppeteer.executablePath(),
+		'/usr/bin/chromium',
+		'/usr/bin/chromium-browser',
+		'/usr/bin/google-chrome',
+	];
+
+	for (let candidate of candidates) {
+		if (isUsableBrowser(candidate)) {
+			return candidate;
+		}
+	}
+
+	// Nothing usable found: let puppeteer try its own default and produce
+	// its normal error/timeout, rather than silently launching with none
+	return undefined;
+}
+
 /**
  * The BrowserHelper class
  *
@@ -133,7 +205,8 @@ BrowserHelper.setMethod(async function load() {
 	} else {
 		// Launch new browser
 		this.browser = await puppeteer.launch({
-			headless: this.options.headless,
+			headless       : this.options.headless,
+			executablePath : resolveExecutablePath(puppeteer),
 		});
 	}
 

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "alchemymvc",
 	"description": "MVC framework for Node.js",
-	"version": "1.4.3",
+	"version": "1.4.5",
 	"author": "Jelle De Loecker <jelle@elevenways.be>",
 	"keywords": [
 		"alchemy",
@@ -22,7 +22,7 @@
 		"chokidar"         : "~4.0.3",
 		"formidable"       : "~3.5.4",
 		"graceful-fs"      : "~4.2.11",
-		"hawkejs"          : "~2.4.2",
+		"hawkejs"          : "~2.4.3",
 		"jsondiffpatch"    : "~0.5.0",
 		"mime-types"       : "~3.0.2",
 		"minimist"         : "~1.2.8",
@@ -32,7 +32,7 @@
 		"ncp"              : "~2.0.0",
 		"postcss"          : "~8.5.6",
 		"postcss-prune-var": "~1.1.2",
-		"protoblast"       : "~0.9.8",
+		"protoblast"       : "~0.9.9",
 		"semver"           : "~7.7.2",
 		"socket.io"        : "~4.7.5",
 		"@11ways/socket.io-stream" : "~0.9.2",
@@ -66,7 +66,7 @@
 		"mocha"                   : "^11.7.5",
 		"mongo-unit"              : "^3.4.0",
 		"nyc"                     : "^15.1.0",
-		"puppeteer"               : "~21.3.6",
+		"puppeteer"               : "^24.43.1",
 		"source-map"              : "~0.7.3"
 	},
 	"scripts": {