alchemymvc 1.4.0-alpha.5 → 1.4.0-alpha.7

package/lib/app/datasource/mongo_datasource.js

@@ -131,7 +131,11 @@ Mongo.setMethod(function castToBigInt(value) {
 	}
 
 	if (typeof value == 'object') {
-		return value.toBigInt();
+		if (value.toBigInt) {
+			return value.toBigInt();
+		}
+
+		return null;
 	}
 
 	return BigInt(value);
@@ -487,9 +491,21 @@ Mongo.setMethod(function _create(context) {
 		this.collection(model.table),
 		collection => {
 
-			const data = context.getConvertedData();
+			const converted_data = context.getConvertedData();
 			const pledge = new Swift();
 
+			let data = {};
+
+			for (let key in converted_data) {
+				let val = converted_data[key];
+
+				if (val == null) {
+					continue;
+				}
+
+				data[key] = val;
+			}
+
 			Pledge.done(collection.insertOne(data, {w: 1, fullResult: true}), function afterInsert(err, result) {
 
 				// Clear the cache
@@ -541,110 +557,125 @@ Mongo.setMethod(function _create(context) {
  */
 Mongo.setMethod(function _update(context) {
 
-	const model = context.getModel(),
-	      options = context.getSaveOptions();
+	const model = context.getModel();
 
 	return Swift.waterfall(
 		this.collection(model.table),
 		collection => {
+			return performUpdate(collection, model, context);
+		}
+	);
+});
 
-			let key;
+/**
+ * Perform the update
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ *
+ * @return   {Pledge}
+ */
+const performUpdate = (collection, model, context) => {
 
-			// Get the converted data
-			const data = context.getConvertedData();
+	const options = context.getSaveOptions();
 
-			// Get the id to update, it should always be inside the given data
-			let id = data._id;
+	let key;
 
-			// Clone the data object
-			let doc = {...data};
+	// Get the converted data
+	const data = context.getConvertedData();
 
-			// Values that will get flattened
-			let to_flatten = {};
+	// Get the id to update, it should always be inside the given data
+	let id = data._id;
 
-			// Field names that won't get flattened
-			let no_flatten = {};
+	// Clone the data object
+	let doc = {...data};
 
-			// Remove the id
-			delete doc._id;
+	// Values that will get flattened
+	let to_flatten = {};
 
-			if (!options.override_created) {
-				delete doc.created;
-			}
+	// Field names that won't get flattened
+	let no_flatten = {};
 
-			// Iterate over the fields
-			for (key in doc) {
-				let field = model.getField(key);
+	// Remove the id
+	delete doc._id;
 
-				if (field && (field.is_self_contained || field.is_translatable)) {
-					no_flatten[key] = doc[key];
-				} else {
-					to_flatten[key] = doc[key];
-				}
-			}
+	if (!options.override_created) {
+		delete doc.created;
+	}
 
-			// Flatten the object, using periods & NOT flattening arrays
-			let flat = Object.flatten(to_flatten, '.', false);
+	// Iterate over the fields
+	for (key in doc) {
+		let field = model.getField(key);
 
-			// Assign the no-flatten values, too
-			Object.assign(flat, no_flatten);
+		if (field && (field.is_self_contained || field.is_translatable || typeof doc[key] == 'object')) {
+			no_flatten[key] = doc[key];
+		} else {
+			to_flatten[key] = doc[key];
+		}
+	}
 
-			let unset = {};
+	// Flatten the object, using periods & NOT flattening arrays
+	let flat = Object.flatten(to_flatten, '.', false);
 
-			for (key in flat) {
-				// Undefined or null means we want to delete the value
-				// We can't set null, because that could interfere with dot notation updates
-				if (flat[key] == null) {
+	// Assign the no-flatten values, too
+	Object.assign(flat, no_flatten);
 
-					// Add the key to the unset object
-					unset[key] = '';
+	let unset = {};
 
-					// Remove it from the flat object
-					delete flat[key];
-				}
-			}
+	for (key in flat) {
+		// Undefined or null means we want to delete the value
+		// We can't set null, because that could interfere with dot notation updates
+		if (flat[key] == null) {
 
-			let update_object = {
-				$set: flat
-			};
+			// Add the key to the unset object
+			unset[key] = '';
 
-			if (!Object.isEmpty(unset)) {
-				update_object.$unset = unset;
-			}
+			// Remove it from the flat object
+			delete flat[key];
+		}
+	}
 
-			if (options.debug) {
-				console.log('Updating with obj', id, update_object);
-			}
+	let update_object = {
+		$set: flat
+	};
 
-			let promise;
+	if (!Object.isEmpty(unset)) {
+		update_object.$unset = unset;
+	}
 
-			if (collection.findOneAndUpdate) {
-				promise = collection.findOneAndUpdate({_id: id}, update_object, {upsert: true});
-			} else if (collection.findAndModify) {
-				promise = collection.findAndModify({_id: id}, [['_id', 1]], update_object, {upsert: true});
-			} else {
-				// If it's not available (like nedb)
-				promise = collection.update({_id: ''+id}, update_object, {upsert: true});
-			}
+	if (options.debug) {
+		console.log('Updating with obj', id, update_object);
+	}
 
-			let pledge = new Swift();
+	let promise;
 
-			Pledge.done(promise, function afterUpdate(err, result) {
+	if (collection.findOneAndUpdate) {
+		promise = collection.findOneAndUpdate({_id: id}, update_object, {upsert: true});
+	} else if (collection.findAndModify) {
+		promise = collection.findAndModify({_id: id}, [['_id', 1]], update_object, {upsert: true});
+	} else {
+		// If it's not available (like nedb)
+		promise = collection.update({_id: ''+id}, update_object, {upsert: true});
+	}
 
-				// Clear the cache
-				model.nukeCache();
+	let pledge = new Swift();
 
-				if (err != null) {
-					return pledge.reject(err);
-				}
+	Pledge.done(promise, function afterUpdate(err, result) {
 
-				pledge.resolve(Object.assign({}, data));
-			});
+		// Clear the cache
+		model.nukeCache();
 
-			return pledge;
+		if (err != null) {
+			return pledge.reject(err);
 		}
-	);
-});
+
+		pledge.resolve(Object.assign({}, data));
+	});
+
+	return pledge;
+
+};
 
 /**
  * Remove a record from the database

package/lib/core/alchemy.js

@@ -1745,12 +1745,43 @@ Alchemy.setMethod(function addAppcacheEntry(entry) {
 	ac_entries[entry.type].push(entry);
 });
 
+/**
+ * Get a size limit for the given route
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ *
+ * @param    {Route?}   route
+ * @param    {string}   name
+ */
+function getRouteSizeLimit(route, name) {
+
+	let global_size = alchemy.settings.network[name];
+
+	if (!route) {
+		return global_size;
+	}
+
+	let route_value = route.options[name];
+
+	if (route_value == null || typeof route_value != 'number') {
+		return global_size;
+	}
+
+	if (route_value <= 0) {
+		return Infinity;
+	}
+
+	return route_value;
+}
+
 /**
  * Get the body of an IncomingMessage
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    1.1.0
- * @version  1.3.18
+ * @version  1.4.0
  *
  * @param    {IncomingMessage}   req
  * @param    {OutgoingMessage}   res       Optional
@@ -1777,24 +1808,44 @@ Alchemy.setMethod(function parseRequestBody(req, res, callback) {
 
 	let content_type = req.headers['content-type'];
 
+	let request_body_size_limit = getRouteSizeLimit(conduit?.route, 'request_body_size_limit');
+
 	// Multipart data is handled by "formidable"
 	if (content_type && content_type.startsWith('multipart/form-data')) {
 
+		let request_individual_file_size_limit = getRouteSizeLimit(conduit?.route, 'request_individual_file_size_limit'),
+		    request_total_file_size_limit = getRouteSizeLimit(conduit?.route, 'request_total_file_size_limit');
+
 		let form = new this.formidable.IncomingForm({
 			multiples     : true,
 			hashAlgorithm : this.settings.data_management.file_hash_algorithm || 'sha1',
+			minFileSize      : 0,
+			allowEmptyFiles  : true,
+			maxFileSize      : request_individual_file_size_limit,
+			maxFieldsSize    : request_body_size_limit,
+			maxTotalFileSize : request_total_file_size_limit,
 		});
 
 		form.parse(req, function parsedMultipart(err, form_fields, form_files) {
 
-			var fields = {},
-			    files = {},
-			    key;
+			if (err) {
+
+				// Ignore the error if the request was already aborted
+				if (conduit?.aborted) {
+					return callback(null);
+				}
 
-			if (err && req.conduit && req.conduit.aborted) {
-				return callback(null);
+				if (conduit) {
+					return conduit.error(err);
+				}
+
+				return callback(err);
 			}
 
+			let fields = {},
+			    files = {},
+			    key;
+
 			// Since formidable v3, all the fields are now arrays.
 			// We already had a lot of logic to deal with this,
 			// so we just have to un-array everything
@@ -1811,18 +1862,12 @@ Alchemy.setMethod(function parseRequestBody(req, res, callback) {
 				Object.setFormPath(files, key, form_files[key]);
 			}
 
-			if (err) {
-				log.error('Error parsing multipart POST', {err: err});
-				req.body = {};
-				req.files = {};
-			} else {
-				req.body = fields;
-				req.files = files;
+			req.body = fields;
+			req.files = files;
 
-				if (conduit) {
-					conduit.setRequestBody(fields);
-					conduit.setRequestFiles(files);
-				}
+			if (conduit) {
+				conduit.setRequestBody(fields);
+				conduit.setRequestFiles(files);
 			}
 
 			callback(null, fields);
@@ -1834,24 +1879,29 @@ Alchemy.setMethod(function parseRequestBody(req, res, callback) {
 	// Regular form-encoded data
 	if (content_type && content_type.indexOf('form-urlencoded') > -1) {
 
-		this.url_form_body(req, res, function parsedBody(err) {
+		this.url_form_body(req, res, {limit: request_body_size_limit}, function parsedBody(err) {
+
+			if (err) {
+
+				// Ignore the error if the request was already aborted
+				if (conduit?.aborted) {
+					return callback(null);
+				}
+
+				if (conduit) {
+					return conduit.error(err);
+				}
 
-			if (err && req.conduit && req.conduit.aborted) {
-				return callback(null);
+				return callback(err);
 			}
 
 			// You can't send files using a regular post
 			req.files = {};
 
-			if (err) {
-				log.error('Error parsing x-www-form-urlencoded body data', {err: err});
-				req.body = {};
-			} else {
-				req.body = req.body;
+			req.body = req.body;
 
-				if (conduit) {
-					conduit.setRequestBody(req.body);
-				}
+			if (conduit) {
+				conduit.setRequestBody(req.body);
 			}
 
 			callback(null, req.body);
@@ -1861,25 +1911,30 @@ Alchemy.setMethod(function parseRequestBody(req, res, callback) {
 	}
 
 	// Any other encoded data (like JSON)
-	this.any_body(req, function parsedBody(err, body) {
+	this.any_body(req, res, {limit: request_body_size_limit}, function parsedBody(err, body) {
 
 		function handleResponse(err, body) {
-			if (err && req.conduit && req.conduit.aborted) {
-				return callback(null);
+
+			if (err) {
+
+				// Ignore the error if the request was already aborted
+				if (conduit?.aborted) {
+					return callback(null);
+				}
+
+				if (conduit) {
+					return conduit.error(err);
+				}
+
+				return callback(err);
 			}
-	
+
 			// You can't send files using a regular post
 			req.files = {};
+			req.body = body;
 	
-			if (err) {
-				log.error('Error parsing body data', {err: err});
-				req.body = {};
-			} else {
-				req.body = body;
-	
-				if (conduit) {
-					conduit.setRequestBody(body);
-				}
+			if (conduit) {
+				conduit.setRequestBody(body);
 			}
 	
 			callback(null, req.body);

package/lib/core/alchemy_functions.js

@@ -1,6 +1,7 @@
 let mkdirp       = alchemy.use('mkdirp')?.mkdirp,
     ncp          = alchemy.use('ncp').ncp,
     fs           = alchemy.use('fs'),
+    fsp          = fs.promises,
     libpath      = alchemy.use('path'),
     child        = alchemy.use('child_process'),
     crypto       = alchemy.use('crypto'),
@@ -1106,6 +1107,53 @@ Alchemy.setMethod(function request(url, options, callback) {
 	return Blast.fetch(options, callback);
 });
 
+/**
+ * Turn a `data:` uri into a file
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ *
+ * @param    {string}   data_uri
+ *
+ * @return   {Pledge<File>}
+ */
+function convertDataUriToFile(data_uri) {
+
+	// Split the data uri into its 2 parts
+	let pieces = data_uri.slice(5).split(',');
+
+	// Split the info bit
+	let info = pieces[0].split(';');
+
+	// Get the expected mime type
+	let mime_type = info[0];
+
+	// And is it base64?
+	let is_b64 = info[1] && info[1].toLowerCase() == 'base64';
+
+	let buffer;
+
+	if (is_b64) {
+		buffer = Buffer.from(pieces[1], 'base64');
+	} else {
+		buffer = Buffer.from(String.decodeURI(pieces[1]));
+	}
+
+	return Pledge.Swift.waterfall(
+		() => Blast.createTempDir({prefix: 'aldl'}),
+		async (temp_dir) => {
+			let full_path = libpath.resolve(temp_dir, 'buffer_' + alchemy.ObjectId());
+			
+			await fsp.writeFile(full_path, buffer);
+
+			return full_path;
+		},
+		(full_path) => Classes.Alchemy.Inode.Inode.from(full_path)
+	);
+
+}
+
 /**
  * Download a file
  *
@@ -1119,6 +1167,14 @@ Alchemy.setMethod(function request(url, options, callback) {
  */
 Alchemy.setMethod(function download(url, options) {
 
+	if (url.startsWith('data:')) {
+		try {
+			return convertDataUriToFile(url, options);
+		} catch (err) {
+			return Pledge.reject(err);
+		}
+	}
+
 	const pledge = new Pledge();
 
 	// Get the file

package/lib/core/setting.js

@@ -477,6 +477,11 @@ Group.setStatic(function unDry(obj, cloned) {
 	    child;
 
 	for (child of children) {
+
+		if (!child) {
+			continue;
+		}
+
 		result.children.set(child.name, child);
 		child.group = result;
 	}

package/lib/scripts/create_settings.js

@@ -73,6 +73,24 @@ network.addSetting('use_compression', {
 	description : 'Compress responses using gzip/deflate',
 });
 
+network.addSetting('request_body_size_limit', {
+	type        : 'number',
+	default     : 20 * 1024 * 1024,
+	description : 'Maximum allowed size in bytes for the request body, excluding files',
+});
+
+network.addSetting('request_individual_file_size_limit', {
+	type        : 'number',
+	default     : 200 * 1024 * 1024,
+	description : 'Maximum allowed size in bytes for individual uploaded files',
+});
+
+network.addSetting('request_total_file_size_limit', {
+	type        : 'number',
+	default     : 200 * 1024 * 1024,
+	description : 'Maximum allowed size in bytes for all uploaded files',
+});
+
 network.addSetting('use_json_dry_responses', {
 	type        : 'boolean',
 	default     : true,

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "alchemymvc",
 	"description": "MVC framework for Node.js",
-	"version": "1.4.0-alpha.5",
+	"version": "1.4.0-alpha.7",
 	"author": "Jelle De Loecker <jelle@elevenways.be>",
 	"keywords": [
 		"alchemy",