jxp 2.14.6 → 2.15.1

package/libs/jxp.js

@@ -72,7 +72,7 @@ const outputCSV = (req, res, next) => {
 	try {
 		const data = res.result.data.map(row => row._doc);
 		if (!data.length) {
-			throw("")
+			throw ("")
 		}
 		res.writeHead(200, {
 			'Content-Type': 'text/csv',
@@ -91,7 +91,7 @@ const outputCSV = (req, res, next) => {
 const actionGet = async (req, res) => {
 	const opname = `get ${req.modelname} ${ops++}`;
 	console.time(opname);
-	const parseSearch = function(search) {
+	const parseSearch = function (search) {
 		let result = {};
 		for (let i in search) {
 			result[i] = new RegExp(search[i], "i");
@@ -103,6 +103,10 @@ const actionGet = async (req, res) => {
 		filters = parseFilter(req.query.filter);
 	} catch (err) {
 		console.trace(new Date(), err);
+		// Preserve BadRequestError, convert others to InternalServerError
+		if (err instanceof errors.BadRequestError) {
+			throw err;
+		}
 		throw new errors.InternalServerError(err.toString());
 	}
 	let search = parseSearch(req.query.search);
@@ -120,9 +124,9 @@ const actionGet = async (req, res) => {
 	}
 	if (req.query.search) {
 		// console.log({ search: req.query.search });
-		q = req.Model.find({ $text: { $search: req.query.search }}, { score : { $meta: "textScore" } }).sort( { score: { $meta : "textScore" } } );
-		countquery = Object.assign({ $text: { $search: req.query.search }}, countquery);
-		qcount = req.Model.find({ $text: { $search: req.query.search }});
+		q = req.Model.find({ $text: { $search: req.query.search } }, { score: { $meta: "textScore" } }).sort({ score: { $meta: "textScore" } });
+		countquery = Object.assign({ $text: { $search: req.query.search } }, countquery);
+		qcount = req.Model.find({ $text: { $search: req.query.search } });
 	}
 	if (res.user) {
 		q.options = ({ user: res.user });
@@ -164,7 +168,7 @@ const actionGet = async (req, res) => {
 			} else {
 				q.populate(req.query.populate);
 			}
-			result.populate = req.query.populate;	
+			result.populate = req.query.populate;
 		}
 		if (req.query.autopopulate) {
 			for (let key in req.Model.schema.paths) {
@@ -189,9 +193,13 @@ const actionGet = async (req, res) => {
 		result.data = await q.exec();
 		res.result = result;
 		if (debug) console.timeEnd(opname);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (debug) console.timeEnd(opname);
+		// Preserve BadRequestError, convert others to InternalServerError
+		if (err instanceof errors.BadRequestError) {
+			throw err;
+		}
 		if (err.code) throw err;
 		throw new errors.InternalServerError(err.toString());
 	}
@@ -204,7 +212,7 @@ const actionGetOne = async (req, res) => {
 		const data = await getOne(req.Model, req.params.item_id, req.query, { user: res.user });
 		res.result = { data };
 		if (debug) console.timeEnd(opname);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (debug) console.timeEnd(opname);
 		if (err.code) throw err;
@@ -262,7 +270,7 @@ const actionPut = async (req, res) => {
 		let silence = req.params._silence;
 		if (req.body && req.body._silence) silence = true;
 		if (!silence) {
-			req.config.callbacks.put.call(null, req.modelname, item, res.user );
+			req.config.callbacks.put.call(null, req.modelname, item, res.user);
 			ws.putHook.call(null, req.modelname, item, res.user);
 		}
 		res.json({
@@ -283,12 +291,12 @@ const actionUpdate = async (req, res) => {
 	const opname = `update ${req.modelname}/${req.params.item_id} ${ops++}`;
 	console.time(opname);
 	try {
-		let body_data =  datamunging.deserialize(req.body);
+		let body_data = datamunging.deserialize(req.body);
 		const data = await req.Model.update({ _id: req.params.item_id }, body_data);
 		let silence = req.params._silence;
 		if (req.body && req.body._silence) silence = true;
 		if (!silence) {
-			req.config.callbacks.put.call(null, req.modelname, data, res.user );
+			req.config.callbacks.put.call(null, req.modelname, data, res.user);
 			ws.putHook.call(null, req.modelname, data, res.user);
 		}
 		res.json({
@@ -370,10 +378,10 @@ const actionDelete = async (req, res) => {
 		}
 		res.json({
 			status: "ok",
-			message: `${req.modelname}/${ req.params.item_id } deleted`
+			message: `${req.modelname}/${req.params.item_id} deleted`
 		});
 		if (debug) console.timeEnd(opname);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (debug) console.timeEnd(opname);
 		if (err.code) throw err;
@@ -384,7 +392,7 @@ const actionDelete = async (req, res) => {
 const actionCount = async (req, res) => {
 	const opname = `count ${req.modelname} ${ops++}`;
 	console.time(opname);
-	const parseSearch = function(search) {
+	const parseSearch = function (search) {
 		let result = {};
 		for (let i in search) {
 			result[i] = new RegExp(search[i], "i");
@@ -409,7 +417,7 @@ const actionCount = async (req, res) => {
 		const count = await req.Model.countDocuments(filters).exec();
 		res.result = { count };
 		if (debug) console.timeEnd(opname);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (debug) console.timeEnd(opname);
 		if (err.code) throw err;
@@ -424,7 +432,7 @@ const actionCall = async (req, res) => {
 	try {
 		const result = await req.Model[req.params.method_name](req.body);
 		res.json(result);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (err.code) throw err;
 		throw new errors.InternalServerError(err.toString());
@@ -440,7 +448,7 @@ const actionCallItem = async (req, res) => {
 		req.params.__user = res.user || null;
 		const result = await req.Model[req.params.method_name](item);
 		res.json(result);
-	} catch(err) {
+	} catch (err) {
 		console.trace(err);
 		if (err.code) throw err;
 		throw new errors.InternalServerError(err.toString());
@@ -493,7 +501,7 @@ const actionQuery = async (req, res) => {
 			} else {
 				q.populate(req.query.populate);
 			}
-			result.populate = req.query.populate;	
+			result.populate = req.query.populate;
 		}
 		if (req.query.autopopulate) {
 			for (let key in req.Model.schema.paths) {
@@ -516,7 +524,7 @@ const actionQuery = async (req, res) => {
 		res.result = result;
 		if (debug) console.timeEnd(opname);
 		res.json(result);
-	} catch(err) {
+	} catch (err) {
 		console.error(new Date(), err);
 		if (debug) console.timeEnd(opname);
 		if (err.code) throw err;
@@ -611,7 +619,7 @@ const actionBulkWrite = async (req, res) => {
 const getOne = async (Model, item_id, params, options) => {
 	const query = Model.findById(item_id, {}, options);
 	if (params.populate) {
-		if ((typeof params.populate === "object")  && !Array.isArray(params.populate)) {
+		if ((typeof params.populate === "object") && !Array.isArray(params.populate)) {
 			for (let i in params.populate) {
 				query.populate(i, params.populate[i].replace(/,/g, " "));
 			}
@@ -641,56 +649,143 @@ const getOne = async (Model, item_id, params, options) => {
 		//Don't ever return passwords
 		delete item.password;
 		return item;
-	} catch(err) {
+	} catch (err) {
 		console.error(err);
 		if (err.code) throw err;
 		throw new errors.InternalServerError(err.toString());
 	}
 };
 
-const parseFilter = (filter) => {
-	if (!filter)
-		return {};
-	if (typeof filter == "object") {
-		Object.keys(filter).forEach(function(key) {
-			var val = filter[key];
-			if (filter[key] === "false") filter[key] = false;
-			if (filter[key] === "true") filter[key] = true;
-			if (val.indexOf) {
-				if (val.indexOf(":") !== -1) {
-					let tmp = val.split(":");
-					filter[key] = {};
-					let tmpkey = tmp.shift();
-					let tmpval = tmp.join(":");
-					if ((tmpval[0] === "[") && (tmpval[tmpval.length - 1] === "]")) { // Could be an array for a $in or similar
-						let arr = tmpval.slice(1, tmpval.length - 1).split(",");
-						tmpval = arr;
-					}
-					filter[key][tmpkey] = tmpval;
-					if (tmpkey === "$regex" && tmpval[0] === "/") {
-						let match = tmpval.match(new RegExp('^/(.*?)/([gimy]*)$'));
-						let regex = new RegExp(match[1], match[2]);
-						filter[key][tmpkey] = regex;
+// Helper function to check if a string is an ISO date string
+function isISODateString(str) {
+	if (typeof str !== 'string') return false;
+	const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/;
+	if (!isoDateRegex.test(str)) return false;
+	const date = new Date(str);
+	if (isNaN(date.getTime())) {
+		throw new errors.BadRequestError("Invalid date format");
+	}
+	return true;
+}
+
+const parseFilter = (filter, depth = 0) => {
+	const MAX_DEPTH = 10;
+
+	if (!filter) return {};
+	if (depth > MAX_DEPTH) {
+		console.warn('Maximum filter depth exceeded');
+		return filter;
+	}
+
+	if (typeof filter !== "object" || filter === null) return filter;
+
+	// Handle arrays by merging their operators
+	if (Array.isArray(filter)) {
+		const result = {};
+		filter.forEach(item => {
+			if (typeof item === "string" && item.includes(":")) {
+				const parts = item.split(":");
+				const key = parts[0];
+				const value = parts.slice(1).join(":");
+				if (key.startsWith("$")) {
+					try {
+						if (isISODateString(value)) {
+							result[key] = new Date(value);
+						} else {
+							result[key] = value;
+						}
+					} catch (err) {
+						if (err instanceof errors.BadRequestError) {
+							throw err;
+						}
+						throw new errors.BadRequestError("Invalid date format");
 					}
 				}
-				if (typeof val == "object") {
-					let result = parseFilter(val);
-					filter[key] = {};
-					for (let x = 0; x < result.length; x++) {
-						filter[key][Object.keys(result[x])[0]] =
-							result[x][Object.keys(result[x])[0]];
+			}
+		});
+		return result;
+	}
+
+	// Create a new object to avoid modifying the input
+	const parsedFilter = {};
+
+	for (let i in filter) {
+		if (filter[i] === "false") {
+			parsedFilter[i] = false;
+			continue;
+		}
+		if (filter[i] === "true") {
+			parsedFilter[i] = true;
+			continue;
+		}
+		if (typeof filter[i] === "string") {
+			try {
+				if (isISODateString(filter[i])) {
+					parsedFilter[i] = new Date(filter[i]);
+					continue;
+				}
+			} catch (err) {
+				if (err instanceof errors.BadRequestError) {
+					throw err;
+				}
+				throw new errors.BadRequestError("Invalid date format");
+			}
+			if (filter[i].includes(":")) {
+				const parts = filter[i].split(":");
+				const key = parts[0];
+				const value = parts.slice(1).join(":");
+				if (key.startsWith("$")) {
+					try {
+						if (isISODateString(value)) {
+							if (!parsedFilter[i]) parsedFilter[i] = {};
+							parsedFilter[i][key] = new Date(value);
+						} else if (value.startsWith("[") && value.endsWith("]")) {
+							if (!parsedFilter[i]) parsedFilter[i] = {};
+							parsedFilter[i][key] = value.slice(1, -1).split(",");
+						} else if (key === "$regex" && value.startsWith("/")) {
+							if (!parsedFilter[i]) parsedFilter[i] = {};
+							const match = value.match(/^\/(.+?)\/([gimy]*)$/);
+							if (match) {
+								parsedFilter[i][key] = new RegExp(match[1], match[2]);
+							}
+						} else {
+							if (!parsedFilter[i]) parsedFilter[i] = {};
+							parsedFilter[i][key] = value;
+						}
+					} catch (err) {
+						if (err instanceof errors.BadRequestError) {
+							throw err;
+						}
+						throw new errors.BadRequestError("Invalid date format");
 					}
+				} else {
+					parsedFilter[i] = filter[i];
 				}
+			} else {
+				parsedFilter[i] = filter[i];
 			}
-		});
+		} else if (Array.isArray(filter[i])) {
+			parsedFilter[i] = parseFilter(filter[i], depth + 1);
+		} else if (typeof filter[i] === "object") {
+			parsedFilter[i] = parseFilter(filter[i], depth + 1);
+		} else {
+			parsedFilter[i] = filter[i];
+		}
 	}
-	return filter;
-}
+
+	return parsedFilter;
+};
 
 const _deSerialize = (data) => {
 	function assign(obj, keyPath, value) {
-		// http://stackoverflow.com/questions/5484673/javascript-how-to-dynamically-create-nested-objects-using-object-names-given-by
+		const MAX_DEPTH = 20; // Prevent excessive nesting
 		const lastKeyIndex = keyPath.length - 1;
+
+		if (lastKeyIndex >= MAX_DEPTH) {
+			console.warn('Maximum nesting depth exceeded in _deSerialize');
+			return;
+		}
+
 		for (let i = 0; i < lastKeyIndex; ++i) {
 			let key = keyPath[i];
 			if (!(key in obj)) obj[key] = {};
@@ -698,10 +793,13 @@ const _deSerialize = (data) => {
 		}
 		obj[keyPath[lastKeyIndex]] = value;
 	}
+
+	if (!data || typeof data !== 'object') return;
+
 	for (let datum in data) {
 		const matches = datum.match(/\[(.+?)\]/g);
 		if (matches) {
-			const params = matches.map(function(match) {
+			const params = matches.map(function (match) {
 				return match.replace(/[[\]]/g, "");
 			});
 			if (isNaN(params[0])) {
@@ -765,7 +863,7 @@ const changeUrlParams = (req, key, val) => {
 	return req.config.url + req.path() + "?" + querystring.stringify(q);
 };
 
-const JXP = function(options) {
+const JXP = function (options) {
 	const server = restify.createServer();
 	const model_dir = options.model_dir || modeldir.findModelDir(path.dirname(process.argv[1]));
 	//Set up config with default
@@ -773,12 +871,12 @@ const JXP = function(options) {
 		model_dir: path.join(model_dir),
 		mongo: options.mongo,
 		callbacks: {
-			put: function() {},
-			post: function() {},
-			delete: function() {},
-			get: function() {},
-			getOne: function() {},
-			update: function() {},
+			put: function () { },
+			post: function () { },
+			delete: function () { },
+			get: function () { },
+			getOne: function () { },
+			update: function () { },
 		},
 		log: "access.log",
 		pre_hooks: {
@@ -841,13 +939,13 @@ const JXP = function(options) {
 	global.apikey = config.apikey;
 	global.server = config.server;
 	global.model_dir = model_dir;
-	
+
 	// Pre-load models
 	var files = fs.readdirSync(config.model_dir);
-	let modelnames = files.filter(function(fname) {
+	let modelnames = files.filter(function (fname) {
 		return fname.indexOf("_model.js") !== -1;
 	});
-	modelnames.forEach(function(fname) {
+	modelnames.forEach(function (fname) {
 		var modelname = fname.replace("_model.js", "");
 		models[modelname] = require(path.join(config.model_dir, fname));
 	});
@@ -856,9 +954,9 @@ const JXP = function(options) {
 	security.init(config);
 	login.init(config);
 	groups.init(config);
-	ws.init({models});
+	ws.init({ models });
 	cache.init(config);
-	const docs = new Docs({config, models});
+	const docs = new Docs({ config, models });
 
 	// Set up our API server
 
@@ -877,7 +975,7 @@ const JXP = function(options) {
 	const cors = corsMiddleware({
 		preflightMaxAge: 5, //Optional
 		origins: ['*'],
-		allowHeaders: ['X-Requested-With','Authorization'],
+		allowHeaders: ['X-Requested-With', 'Authorization'],
 		exposeHeaders: ['Authorization']
 	});
 

package/libs/schema_description.js

@@ -1,40 +1,87 @@
 const fs = require("fs");
 const path = require("path");
+const mongoose = require("mongoose");
 
 const getModelFileContents = (model_dir) => {
     return new Promise((resolve, reject) => {
         try {
-            fs.readdir(model_dir, function(err, files) {
+            fs.readdir(model_dir, function (err, files) {
                 if (err) {
                     return reject(err);
                 }
+
+                // Filter for only model files first
+                const modelFiles = files.filter(file =>
+                    file.endsWith('_model.js') &&
+                    fs.statSync(path.join(model_dir, file)).isFile()
+                );
+
+                // Limit number of files processed at once
+                const MAX_FILES = 1000;
+                if (modelFiles.length > MAX_FILES) {
+                    return reject(new Error(`Too many model files (${modelFiles.length}). Maximum allowed is ${MAX_FILES}`));
+                }
+
                 let models = [];
-                for (const file of files) {
+                let errors = [];
+
+                for (const file of modelFiles) {
                     const modelname = path.basename(file, ".js").replace("_model", "");
                     try {
-                        const modelobj = require(path.join(model_dir, file));
-                        if (
-                            modelobj.schema &&
-                            modelobj.schema.get("_perms") &&
-                            (modelobj.schema.get("_perms").admin ||
-                                modelobj.schema.get("_perms").user ||
-                                modelobj.schema.get("_perms").owner ||
-                                modelobj.schema.get("_perms").all)
-                        ) {
-                            let model = {
-                                model: modelname,
-                                file: file,
-                                perms: modelobj.schema.get("_perms")
-                            };
-                            models.push(model);
+                        // Try to get the model from mongoose first
+                        let modelobj;
+                        try {
+                            // Convert modelname to proper case for mongoose (first letter uppercase)
+                            const modelKey = modelname.charAt(0).toUpperCase() + modelname.slice(1);
+                            modelobj = mongoose.models[modelKey];
+
+                            // If model doesn't exist in mongoose, try to load it from file
+                            if (!modelobj) {
+                                const filePath = path.join(model_dir, file);
+                                delete require.cache[require.resolve(filePath)];
+                                modelobj = require(filePath);
+                            }
+                        } catch (e) {
+                            // If getting from mongoose fails, try loading from file
+                            const filePath = path.join(model_dir, file);
+                            delete require.cache[require.resolve(filePath)];
+                            modelobj = require(filePath);
+                        }
+
+                        // Check if we have a valid model with schema and permissions
+                        if (modelobj && modelobj.schema) {
+                            const perms = modelobj.schema.get("_perms");
+                            if (perms && (perms.admin || perms.user || perms.owner || perms.all)) {
+                                models.push({
+                                    model: modelname,
+                                    file: file,
+                                    perms: perms
+                                });
+                                continue; // Skip error handling if successful
+                            }
                         }
+
+                        // If we get here, the model was loaded but didn't have proper schema/perms
+                        errors.push(`Invalid model structure for ${modelname}`);
+
                     } catch (error) {
-                        console.error("Error with model " + modelname, error);
+                        // Only add to errors if it's not an OverwriteModelError
+                        if (!error.message.includes('Cannot overwrite')) {
+                            errors.push(`Error with model ${modelname}: ${error.message}`);
+                            console.error(`Error processing model ${modelname}:`, error);
+                        }
                     }
                 }
+
+                // If we have errors but also some valid models, just log the errors
+                if (errors.length > 0 && models.length > 0) {
+                    console.warn('Some models failed to load:', errors);
+                }
+
+                // Return empty array if no models found, but don't treat it as an error
                 return resolve(models);
             });
-        } catch(err) {
+        } catch (err) {
             return reject(err);
         }
     });

package/models/test_model.js

@@ -11,30 +11,31 @@ const TestSchema = new JXPSchema({
     fulltext: { type: String, index: { text: true } },
     link_id: { type: ObjectId, link: "Link", }, // We can populate these links during a query
     other_link_id: { type: ObjectId, link: "Link", map_to: "other_link" },
-    array_link_id: [{ type: ObjectId, link: "Link", map_to: "array_link", justOne: false } ],
-    composite_array: [ 
+    array_link_id: [{ type: ObjectId, link: "Link", map_to: "array_link", justOne: false }],
+    composite_array: [
         { afoo: String, abar: Number }
-    ]
+    ],
+    date_field: { type: Date, index: true } // Added date field for testing date filtering
 },
-{
-    perms: {
-        admin: "crud", // CRUD = Create, Retrieve, Update and Delete
-        owner: "crud",
-        user: "cr",
-        all: "r" // Unauthenticated users will be able to read from test, but that is all
+    {
+        perms: {
+            admin: "crud", // CRUD = Create, Retrieve, Update and Delete
+            owner: "crud",
+            user: "cr",
+            all: "r" // Unauthenticated users will be able to read from test, but that is all
+        }
     }
-}
 );
 
 // Full text index
 // TestSchema.index( { "$**": "text" } );
 
 // We can define useful functions that we can call through the API using our /call endpoint
-TestSchema.statics.test = function() {
+TestSchema.statics.test = function () {
     return "Testing OKAY!";
 };
 
-TestSchema.pre("save", function(next) {
+TestSchema.pre("save", function (next) {
     // If we have the setting "error" set to true, we will throw an error
     if (this.bar == "Throw an error") {
         throw new errors.ImATeapotError("I'm a teapot");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "jxp",
   "description:": "An opinionated RESTful API library based on Mongoose and Restify. Make an API by just writing Mongoose models.",
-  "version": "2.14.6",
+  "version": "2.15.1",
   "private": false,
   "main": "libs/jxp.js",
   "scripts": {
@@ -29,12 +29,12 @@
     "url": "https://github.com/WorkSpaceMan/jxp/issues"
   },
   "dependencies": {
-    "axios": "^1.6.5",
-    "bcryptjs": "^2.4.3",
-    "commander": "^11.1.0",
-    "config": "^3.3.10",
-    "dotenv": "^16.4.0",
-    "glob": "10.3.10",
+    "axios": "^1.8.4",
+    "bcryptjs": "^3.0.2",
+    "commander": "^13.1.0",
+    "config": "^3.3.12",
+    "dotenv": "^16.4.7",
+    "glob": "11.0.1",
     "js-yaml": "4.1.0",
     "json2csv": "^5.0.7",
     "jsonwebtoken": "^9.0.2",
@@ -46,25 +46,25 @@
     "mongoose-friendly": "^0.1.4",
     "morgan": "^1.10.0",
     "node-cache": "^5.1.2",
-    "nodemailer": "^6.9.8",
+    "nodemailer": "^6.10.0",
     "nodemailer-smtp-transport": "^2.7.4",
     "path": "^0.12.7",
-    "pug": "^3.0.2",
+    "pug": "^3.0.3",
     "querystring": "^0.2.1",
     "rand-token": "^1.0.1",
     "readline-sync": "^1.4.10",
     "restify": "^11.1.0",
     "restify-cors-middleware2": "^2.2.1",
     "restify-errors": "^8.0.2",
-    "traverse": "^0.6.8",
-    "underscore": "^1.13.6",
-    "ws": "^8.16.0"
+    "traverse": "^0.6.11",
+    "underscore": "^1.13.7",
+    "ws": "^8.18.1"
   },
   "devDependencies": {
-    "chai": "^4.3.7",
+    "chai": "^4.3.10",
     "chai-http": "^4.4.0",
     "mocha": "^10.2.0",
     "moment": "^2.29.4",
     "should": "^13.2.3"
   }
-}
+}