alchemymvc 1.4.0 → 1.4.2

package/lib/app/model/system_task_history_model.js

@@ -116,4 +116,138 @@ SystemTaskHistory.constitute(function chimeraConfig() {
 		route        : 'Chimera.Editor#taskMonitor',
 		route_params : {task_history_id: '$pk'},
 	});
+});
+
+/**
+ * Derive a single-word status from the recorded fields.
+ *
+ *   - 'running': is_running is still true
+ *   - 'failed':  had_error is true
+ *   - 'done':    started AND ended cleanly
+ *   - 'aborted': started but never ended (pre-1.4.2 zombies, or a process
+ *                that died mid-task without unwinding the try/finally)
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.2
+ *
+ * @return   {string}
+ */
+SystemTaskHistory.setDocumentMethod(function getStatus() {
+
+	if (this.had_error) {
+		return 'failed';
+	}
+
+	if (this.is_running) {
+		return 'running';
+	}
+
+	if (this.ended_at) {
+		return 'done';
+	}
+
+	if (this.started_at) {
+		return 'aborted';
+	}
+
+	return 'scheduled';
+});
+
+/**
+ * Return how long this run took (or has been running so far), in
+ * milliseconds. Null when nothing has started yet.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.2
+ *
+ * @return   {number|null}
+ */
+SystemTaskHistory.setDocumentMethod(function getDuration() {
+
+	if (!this.started_at) {
+		return null;
+	}
+
+	let end = this.ended_at || (this.is_running ? new Date() : null);
+
+	if (!end) {
+		return null;
+	}
+
+	return end.getTime() - this.started_at.getTime();
+});
+
+/**
+ * Query the most recent task runs with optional filters. Centralized
+ * here so that both Chimera-side dashboards and MCP / API consumers can
+ * use the same paginated query - avoids forking the criteria.
+ *
+ * Options:
+ *   type    {string}  Filter by task type_path (e.g. 'arcana.task.sync_harvest_clients')
+ *   status  {string}  'running' | 'done' | 'failed' | 'aborted'
+ *   since   {Date}    Earliest started_at to include
+ *   limit   {number}  Page size (1-100, default 25)
+ *   offset  {number}  Skip count
+ *
+ * Returns `{rows, total}` where `total` is the unpaginated count.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.2
+ *
+ * @param    {Object}   [options]
+ *
+ * @return   {Promise<{rows: DocumentList, total: number}>}
+ */
+SystemTaskHistory.setMethod(async function findRecent(options) {
+
+	options = options || {};
+
+	let crit = this.find();
+
+	if (options.type) {
+		crit.where('type').equals(options.type);
+	}
+
+	if (options.status) {
+		switch (options.status) {
+			case 'running':
+				crit.where('is_running').equals(true);
+				break;
+
+			case 'failed':
+				crit.where('had_error').equals(true);
+				break;
+
+			case 'done':
+				crit.where('is_running').equals(false);
+				crit.where('had_error').not().equals(true);
+				crit.where('ended_at').exists(true);
+				break;
+
+			case 'aborted':
+				crit.where('is_running').equals(false);
+				crit.where('had_error').not().equals(true);
+				crit.where('ended_at').isEmpty();
+				crit.where('started_at').exists(true);
+				break;
+
+			default:
+				throw new Error('Unknown task-run status filter: ' + options.status);
+		}
+	}
+
+	if (options.since) {
+		crit.where('started_at').gte(options.since);
+	}
+
+	let limit = Math.min(Math.max(options.limit || 25, 1), 100);
+	let offset = Math.max(options.offset || 0, 0);
+
+	crit.sort({started_at: -1});
+	crit.setOption('available', true);
+	crit.limit(limit);
+	if (offset > 0) crit.skip(offset);
+
+	let rows = await this.find('all', crit);
+	return {rows, total: rows.available || 0};
 });

package/lib/class/conduit.js

@@ -1825,7 +1825,7 @@ Conduit.setMethod(function end(message) {
 		}
 
 		// Use regular JSON if DRY has been disabled in settings
-		if (alchemy.settings.network.use_json_dry_response === false || this.json_dry === false) {
+		if (alchemy.settings.network.use_json_dry_responses === false || this.json_dry === false) {
 			json_type = 'json';
 			json_fnc = JSON.stringify;
 		} else {
@@ -2153,7 +2153,7 @@ Conduit.setTypedMethod([Types.String, Types.Object.optional()], function serveFi
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    0.2.0
- * @version  1.4.0
+ * @version  1.4.1
  *
  * @param    {Alchemy.Inode.File}   file      The file to serve
  * @param    {Object}               options   Options, including headers
@@ -2171,6 +2171,9 @@ Conduit.setTypedMethod([Types.Alchemy.Inode.File, Types.Object.optional()], asyn
 	} catch (err) {
 
 		if (err.code == 'ENOENT') {
+			if (options.onError) {
+				return options.onError(err);
+			}
 			return this.notFound(err);
 		}
 

package/lib/class/controller.js

@@ -432,6 +432,7 @@ Controller.setAction(async function readDatasource(conduit) {
 		user_id = this.getUserId();
 
 		// @TODO: non-users should also be able to send criteria :/
+		// We allow everyone to do so for now, fix later.
 		if (true || user_id) {
 			conduit.body = JSON.undry(conduit.body);
 		}

package/lib/class/datasource.js

@@ -508,7 +508,7 @@ Datasource.setMethod(function read(context) {
 
 	model.emit('reading_datasource', criteria);
 
-	return Pledge.Swift.waterfall(
+	let read_pledge = Pledge.Swift.waterfall(
 		that._read(context),
 		_result => {
 
@@ -548,6 +548,18 @@ Datasource.setMethod(function read(context) {
 			}
 		);
 	});
+
+	// If a cache_pledge was created, make sure it gets rejected on error
+	// so that subsequent queries with the same hash don't hang forever
+	if (cache_pledge) {
+		Pledge.Swift.done(read_pledge, (err) => {
+			if (err) {
+				cache_pledge.reject(err);
+			}
+		});
+	}
+
+	return read_pledge;
 });
 
 /**
@@ -603,7 +615,7 @@ Datasource.setMethod(function update(context) {
 
 	let result = Swift.waterfall(
 		// Convert the data into something the datasource will understand
-		this.toDatasource(context),
+		() => this.toDatasource(context),
 
 		// Actually create the data
 		converted_data => this._update(context.setConvertedData(converted_data)),

package/lib/class/document.js

@@ -427,7 +427,7 @@ Document.setMethod(function toHawkejs(wm) {
 
 		result.setDataRecord(record, options);
 	} else {
-		record.$record = record;
+		result.$record = record;
 	}
 
 	// Clone $hold values if they are available
@@ -908,35 +908,63 @@ Document.setMethod(function exportToStream(output) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.0.5
- * @version  1.0.5
+ * @version  1.4.1
  *
  * @param    {Buffer}   buffer
  *
  * @return   {Pledge}
  */
-Document.setMethod(function importFromBuffer(buffer) {
+Document.setMethod(function importFromBuffer(buffer, options) {
 
 	var that = this,
 	    pledge = new Pledge();
 
+	if (!options) {
+		options = {};
+	}
+
 	zlib.gunzip(buffer, async function unzipped(err, data) {
 
 		if (err) {
 			return pledge.reject(err);
 		}
 
-		data = JSON.undry(data.toString());
+		try {
+			data = JSON.undry(data.toString());
 
-		that.$main = data;
+			that.$main = data;
 
-		await that.save(null, {
-			validate         : false,
-			override_created : true,
-			set_updated      : false,
-			importing        : true
-		});
+			let save_options = {
+				validate         : false,
+				override_created : true,
+				set_updated      : false,
+				importing        : true,
+			};
 
-		pledge.resolve();
+			// If replace_existing is true, don't force create (allow update)
+			// Otherwise, force INSERT
+			if (!options.replace_existing) {
+				save_options.create = true;
+			}
+
+			await that.save(null, save_options);
+
+			pledge.resolve();
+		} catch (save_err) {
+			// Ensure we have a proper Error object with a message
+			let error = save_err;
+			
+			if (!error) {
+				error = new Error('Unknown error during document import');
+			} else if (!(error instanceof Error)) {
+				// If it's not an Error object, wrap it
+				let message = error.message || error.errmsg || String(error);
+				error = new Error('Import save failed: ' + message);
+				error.originalError = save_err;
+			}
+			
+			pledge.reject(error);
+		}
 	});
 
 	return pledge;

package/lib/class/import_stream_parser.js

@@ -0,0 +1,299 @@
+const libstream = require('stream');
+
+/**
+ * The ImportStreamParser class:
+ * Handles parsing of Alchemy's binary import stream format.
+ * 
+ * Stream format:
+ * - 0x01 [1-byte size] [model name]: Model header
+ * - 0x02 [4-byte size BE] [document data]: Document data
+ * - 0xFF [4-byte size BE] [extra data]: Extra import data for current document
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.1
+ * @version  1.4.1
+ */
+const ImportStreamParser = Function.inherits('Alchemy.Base', function ImportStreamParser(input, options) {
+
+	// The input stream to parse
+	this.input = input;
+
+	// Options
+	this.options = options || {};
+
+	// The model resolver function - receives model name, returns Model instance
+	// Can throw an error to abort parsing
+	this.model_resolver = null;
+
+	// State machine variables
+	this.current_type = null;
+	this.extra_stream = null;
+	this.stopped = false;
+	this.paused = false;
+	this.buffer = null;
+	this.model = null;
+	this.value = null;
+	this.seen = 0;
+	this.left = 0;
+	this.size = 0;
+	this.doc = null;
+
+	// Track stream end and pending imports
+	this.stream_ended = false;
+	this.pending_import = false;
+
+	// The pledge that resolves when parsing is complete
+	this.pledge = new Pledge();
+});
+
+/**
+ * Set the model resolver function.
+ * Called when a 0x01 model header is encountered.
+ * 
+ * The resolver receives (model_name, current_model) and should:
+ * - Return a Model instance to use for subsequent documents
+ * - Throw an error to abort parsing
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.1
+ * @version  1.4.1
+ *
+ * @param    {Function}   resolver   (model_name, current_model) => Model
+ */
+ImportStreamParser.setMethod(function setModelResolver(resolver) {
+	this.model_resolver = resolver;
+});
+
+/**
+ * Start parsing the input stream.
+ * Returns a Pledge that resolves when parsing is complete.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.1
+ * @version  1.4.1
+ *
+ * @return   {Pledge}
+ */
+ImportStreamParser.setMethod(function parse() {
+
+	let that = this;
+
+	if (!this.model_resolver) {
+		return Pledge.reject(new Error('No model resolver has been set'));
+	}
+
+	this.input.on('data', function onData(data) {
+
+		if (that.stopped) {
+			return;
+		}
+
+		if (that.buffer) {
+			that.buffer = Buffer.concat([that.buffer, data]);
+		} else {
+			that.buffer = data;
+		}
+
+		that.handleBuffer();
+	});
+
+	this.input.on('end', function onEnd() {
+		that.stream_ended = true;
+
+		// Only resolve if we're not in the middle of importing a document
+		if (!that.stopped && !that.pending_import) {
+			that.pledge.resolve();
+		}
+	});
+
+	this.input.on('error', function onError(err) {
+		that.stopped = true;
+		that.pledge.reject(err);
+	});
+
+	return this.pledge;
+});
+
+/**
+ * Handle the current buffer data.
+ * Implements the state machine for parsing packet headers.
+ *
+ * State machine:
+ * - current_type = null: waiting for a new packet header
+ * - current_type = 0x01/0x02/0xFF: header parsed, processing payload
+ *
+ * We must NOT set current_type until we have the FULL header,
+ * otherwise a TCP chunk boundary could leave us in an invalid state.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.1
+ * @version  1.4.1
+ */
+ImportStreamParser.setMethod(function handleBuffer() {
+
+	if (this.paused) {
+		return;
+	}
+
+	if (!this.current_type) {
+		// Need at least 1 byte to peek at the marker type
+		if (this.buffer.length < 1) {
+			return;
+		}
+
+		let marker = this.buffer.readUInt8(0);
+
+		if (marker == 0x01) {
+			// Type 0x01: 1-byte marker + 1-byte size = 2 bytes header
+			if (this.buffer.length < 2) {
+				return; // Wait for more data (don't consume the marker yet)
+			}
+			this.current_type = marker;
+			this.size = this.buffer.readUInt8(1);
+			this.buffer = this.buffer.slice(2);
+		} else if (marker == 0x02) {
+			// Type 0x02: 1-byte marker + 4-byte size = 5 bytes header
+			if (this.buffer.length < 5) {
+				return; // Wait for more data (don't consume the marker yet)
+			}
+			this.current_type = marker;
+			this.size = this.buffer.readUInt32BE(1);
+			this.buffer = this.buffer.slice(5);
+		} else if (marker == 0xFF) {
+			// Type 0xFF: 1-byte marker + 4-byte size = 5 bytes header
+			if (this.buffer.length < 5) {
+				return; // Wait for more data (don't consume the marker yet)
+			}
+			this.current_type = marker;
+			this.size = this.buffer.readUInt32BE(1);
+			this.buffer = this.buffer.slice(5);
+			this.seen = 0;
+
+			if (!this.doc) {
+				this.stopped = true;
+				this.pledge.reject(new Error('Found extra import data, but no active document'));
+			} else {
+				this.extra_stream = new libstream.PassThrough();
+				this.doc.extraImportFromStream(this.extra_stream);
+			}
+		} else {
+			// Unknown marker - this shouldn't happen in valid data
+			this.stopped = true;
+			this.pledge.reject(new Error('Unknown marker byte: 0x' + marker.toString(16)));
+			return;
+		}
+	}
+
+	this.handlePayload();
+});
+
+/**
+ * Handle the payload data after a header has been parsed.
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.1
+ * @version  1.4.1
+ */
+ImportStreamParser.setMethod(function handlePayload() {
+
+	let that = this;
+
+	// Handle extra data streaming (0xFF)
+	if (this.current_type == 0xFF) {
+		this.left = this.size - this.seen;
+		this.value = this.buffer.slice(0, this.left);
+
+		this.seen += this.value.length;
+
+		if (this.value.length == this.buffer.length) {
+			this.buffer = null;
+		} else if (this.value.length < this.buffer.length) {
+			this.buffer = this.buffer.slice(this.left);
+		}
+
+		this.extra_stream.write(this.value);
+
+		if (this.value.length == this.left) {
+			this.extra_stream.end();
+			this.current_type = null;
+
+			if (this.buffer) {
+				this.handleBuffer();
+			}
+		}
+
+		return;
+	}
+
+	// Wait for full payload
+	if (this.buffer.length >= this.size) {
+		this.value = this.buffer.slice(0, this.size);
+		this.buffer = this.buffer.slice(this.size);
+	} else {
+		// Wait for next call
+		return;
+	}
+
+	// Handle model header (0x01)
+	if (this.current_type == 0x01) {
+		let model_name = this.value.toString();
+
+		try {
+			this.model = this.model_resolver(model_name, this.model);
+			this.doc = null;
+		} catch (err) {
+			this.stopped = true;
+			return this.pledge.reject(err);
+		}
+
+		if (!this.model) {
+			this.stopped = true;
+			return this.pledge.reject(new Error('Model resolver returned no model for "' + model_name + '"'));
+		}
+
+		this.current_type = null;
+		this.size = 0;
+	}
+	// Handle document data (0x02)
+	else if (this.current_type == 0x02) {
+		this.doc = this.model.createDocument();
+		this.input.pause();
+		this.paused = true;
+		this.pending_import = true;
+
+		this.doc.importFromBuffer(this.value, this.options).done(function done(err, result) {
+
+			that.pending_import = false;
+
+			if (err) {
+				that.stopped = true;
+				return that.pledge.reject(err);
+			}
+
+			that.current_type = null;
+			that.paused = false;
+
+			// Check if there's more data to process
+			if (that.buffer && that.buffer.length > 0) {
+				that.input.resume();
+				that.handleBuffer();
+				return;
+			}
+
+			// No more data in buffer
+			that.input.resume();
+
+			// If the stream has ended and there's no more data, resolve
+			if (that.stream_ended && !that.stopped) {
+				that.pledge.resolve();
+			}
+		});
+
+		return;
+	}
+
+	// Continue processing remaining buffer
+	if (this.buffer && this.buffer.length) {
+		this.handleBuffer();
+	}
+});

package/lib/class/inode_file.js

@@ -74,6 +74,8 @@ File.setStatic(function getMimetype(path) {
 
 			pledge.resolve(result);
 		});
+	} else {
+		pledge.resolve(File.guessMimetypeFromPath(path));
 	}
 
 	return pledge;

package/lib/class/migration.js

@@ -20,7 +20,7 @@ const Migration = Function.inherits('Alchemy.Base', function Migration(document)
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.2.0
- * @version  1.2.0
+ * @version  1.4.1
  */
 Migration.setStatic(async function start() {
 
@@ -32,7 +32,10 @@ Migration.setStatic(async function start() {
 
 	await dir.loadContents();
 
-	for (let entry of dir) {
+	// Sort entries alphabetically so numbered prefixes (001_, 002_) run in order
+	let entries = [...dir].sort((a, b) => a.name.localeCompare(b.name));
+
+	for (let entry of entries) {
 
 		let name = entry.name.beforeLast('.js');