monastery 3.0.22 → 3.0.23

package/changelog.md

@@ -2,6 +2,8 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [3.0.23](https://github.com/boycce/monastery/compare/3.0.22...3.0.23) (2024-05-25)
+
 ### [3.0.22](https://github.com/boycce/monastery/compare/3.0.21...3.0.22) (2024-05-08)
 
 ### [3.0.21](https://github.com/boycce/monastery/compare/3.0.20...3.0.21) (2024-05-07)

package/lib/model-crud.js

@@ -83,6 +83,8 @@ Model.prototype.find = async function (opts, _one) {
     // Get projection
     if (opts.project) opts.projection = this._getProjectionFromProject(opts.project)
     else opts.projection = this._getProjectionFromBlacklist(opts.type, opts.blacklist)
+    opts.projectionKeys = Object.keys(opts.projection || {})
+    opts.projectionInclusion = (opts.projection || {})[opts.projectionKeys[0]] ? true : false // default false
 
     // Has text search?
     // if (opts.query.$text) {
@@ -95,17 +97,19 @@ Model.prototype.find = async function (opts, _one) {
       var response = await this[`_find${opts._one? 'One' : ''}`](opts.query, util.omit(opts, this._queryOptions))
     } else {
       loop: for (let item of opts.populate) {
-        let path = util.isObject(item)? item.as : item
+        let path = util.isObject(item) ? item.as : item
         // Blacklisted?
-        if (this._pathBlacklisted(path, opts.projection)) continue loop
+        if (this._pathBlacklisted(path, opts.projectionInclusion, opts.projectionKeys)) continue loop
         // Custom $lookup definition
         // https://thecodebarbarian.com/a-nodejs-perspective-on-mongodb-36-lookup-expr
         if (util.isObject(item)) {
           lookups.push({ $lookup: item })
+        // String populate, e.g. 'comments.author'
         } else {
           let arrayTarget
           let arrayCount = 0
-          let schema = path.split('.').reduce((o, i) => {
+          // Find the field type for the path
+          let field = path.split('.').reduce((o, i) => {
             if (util.isArray(o[i])) {
               arrayCount++
               arrayTarget = true
@@ -115,7 +119,8 @@ Model.prototype.find = async function (opts, _one) {
               return o[i]
             }
           }, this.fields)
-          let modelName = (schema||{}).model
+
+          let modelName = ((field||{}).schema||{}).model
           if (!modelName) {
             this.error(
               `The field "${path}" passed to populate is not of type model. You would ` +
@@ -302,7 +307,7 @@ Model.prototype.update = async function (opts, type='update') {
     }
 
     // Update
-    let update = await this['_' + type](opts.query, operators, util.omit(opts, this._queryOptions))
+    let update = await this['_' + type](opts.query, operators, util.omit(opts, this._queryOptions)) // 50ms for 100k
     if (type == 'findOneAndUpdate') {
       response = update
     } else if (util.isDefined(update.upsertedId)) {
@@ -393,6 +398,9 @@ Model.prototype._getProjectionFromBlacklist = function (type, customBlacklist) {
   if (type == 'find') {
     util.forEach(this.fieldsFlattened, (schema, path) => {
       if (!schema.model) return
+      if (!manager.models[schema.model]) {
+        throw new Error(`The model "${schema.model}" referenced from the model "${this.name}" doesn't exist.`)
+      }
       let deepBL = manager.models[schema.model][`${type}BL`] || []
       let pathWithoutArrays = path.replace(/\.0(\.|$)/, '$1')
       list = list.concat(deepBL.map(o => {
@@ -532,34 +540,37 @@ Model.prototype._queryObject = async function (opts, type, _one) {
   return opts
 }
 
-Model.prototype._pathBlacklisted = function (path, projection, matchDeepWhitelistedKeys=true) {
+Model.prototype._pathBlacklisted = function (path, projectionInclusion, projectionKeys, matchDeepWhitelistedKeys=true) {
   /**
    * Checks if the path is blacklisted within a inclusion/exclusion projection
    * @param {string} path - path without array brackets e.g. '.[]'
-   * @param {object} projection - inclusion/exclusion projection, not mixed
+   * @param {boolean} projectionInclusion - is a inclusion or exclusion projection (default is exclusion)
+   * @param {array} projectionKeys - inclusion/exclusion projection keys, not mixed
    * @param {boolean} matchDeepWhitelistedKeys - match deep whitelisted keys containing path
    *     E.g. pets.color == pets.color.age
    * @return {boolean}
    */
-  for (let key in projection) {
-    if (projection[key]) {
+  if (projectionInclusion) {
+    for (let key of projectionKeys) {
       // Inclusion (whitelisted)
       // E.g. pets.color.age == pets.color.age  (exact match)
       // E.g. pets.color.age == pets.color      (path contains key)
-      var inclusion = true
-      if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return false
+      if (path.indexOf(key + '.') == 0 || path == key) return false
       if (matchDeepWhitelistedKeys) {
         // E.g. pets.color   == pets.color.age  (key contains path)
-        if (key.match(new RegExp('^' + path.replace(/\./g, '\\.') + '\\.'))) return false
+        if (key.indexOf(path + '.') == 0) return false
       }
-    } else {
+    }
+    return true
+  } else {
+    for (let key of projectionKeys) {
       // Exclusion (blacklisted)
       // E.g. pets.color.age == pets.color.age  (exact match)
       // E.g. pets.color.age == pets.color      (path contains key)
-      if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
+      if (path.indexOf(key + '.') == 0 || path == key) return true
     }
+    return false
   }
-  return inclusion? true : false
 }
 
 Model.prototype._processAfterFind = async function (data, projection={}, afterFindContext={}) {
@@ -580,6 +591,8 @@ Model.prototype._processAfterFind = async function (data, projection={}, afterFi
   let seriesGroups = []
   let models = this.manager.models
   let isArray = util.isArray(data)
+  const projectionKeys = Object.keys(projection)
+  const projectionInclusion = projection[projectionKeys[0]] ? true : false // default false
   if (!isArray) data = [data]
   let modelFields = this
     ._recurseAndFindModels(data)
@@ -608,7 +621,7 @@ Model.prototype._processAfterFind = async function (data, projection={}, afterFi
           .replace(/\.[0-9]+(\.|$)/g, '$1')
 
         // Ignore default fields that are blacklisted
-        if (this._pathBlacklisted(fullPathWithoutArrays, projection)) continue
+        if (this._pathBlacklisted(fullPathWithoutArrays, projectionInclusion, projectionKeys)) continue
 
         // Set default value
         const value = util.isFunction(schema.default) ? schema.default(this.manager) : schema.default
@@ -634,7 +647,7 @@ Model.prototype._processAfterFind = async function (data, projection={}, afterFi
   return isArray ? data : data[0]
 }
 
-Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', modelPaths) {
+Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='') {
   /**
    * Returns a flattened list of models fields, sorted by depth
    * @param {object|array} dataArr
@@ -650,16 +663,6 @@ Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', mo
    */
   let out = []
   let dataParentPath2 = dataParentPath.replace(/\.[0-9]+(\.|$)/g, '$1')
-  
-  // Get all model fields
-  if (!modelPaths) {
-    modelPaths = Object.keys(this.fieldsFlattened).reduce((acc, k) => {
-      if (this.fieldsFlattened[k].model) {
-        acc.push({ k: k, k2: k.replace(/\.[0-9]+(\.|$)/g, '$1') }) // e.g. [{k: 'pets.0.dog', 'k2': 'pets.dog'}, ...]
-      }
-      return acc
-    }, [])
-  }
 
   for (let data of util.toArray(dataArr)) {
     for (let key in data) {
@@ -668,23 +671,23 @@ Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', mo
       const dataPath2 = dataParentPath2 ? `${dataParentPath2}.${key}` : key
 
       // modelPath has to contain data path
-      const modelPath = modelPaths.find(o => o.k2.match(new RegExp('^' + dataPath2 + '(\\.|$)')))
-      if (!modelPath) continue
+      const modelField = this.modelFieldsArray.find(o => o.path2.match(new RegExp('^' + dataPath2 + '(\\.|$)')))
+      if (!modelField) continue
       
       // Exact path match
-      const pathMatch = modelPath.k2 == dataPath2
+      const pathMatch = modelField.path2 == dataPath2
 
       // Ignore id values
       if (util.isId(data[key])) continue
 
       // Recurse through sub-document fields
-      if (!pathMatch && util.isSubdocument(data[key])) {
-        out = [...out, ...this._recurseAndFindModels(data[key], dataPath, modelPaths)]
+      if (!pathMatch && util.isSubdocument(data[key])) { /// subdocument? wouldn't isObject be better?
+        out = [...out, ...this._recurseAndFindModels(data[key], dataPath)]
       
       // Recurse through array of sub-documents
       } else if (!pathMatch && util.isSubdocument((data[key]||{})[0])) {
         for (let i=0, l=data[key].length; i<l; i++) {
-          out = [...out, ...this._recurseAndFindModels(data[key][i], dataPath + '.' + i, modelPaths)]
+          out = [...out, ...this._recurseAndFindModels(data[key][i], dataPath + '.' + i)]
         }
 
       // Array of data models (schema field can be either a single or array of models, due to custom $lookup's)
@@ -694,7 +697,7 @@ Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', mo
           dataRefKey: key,
           dataPath: dataPath,
           dataFieldName: key,
-          modelName: this.fieldsFlattened[modelPath.k].model,
+          modelName: this.fieldsFlattened[modelField.path].model,
         })
 
       // Array of data models (schema field can be either a single or array of models, due to custom $lookup's)
@@ -705,7 +708,7 @@ Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', mo
             dataRefKey: i,
             dataPath: dataPath + '.' + i,
             dataFieldName: key,
-            modelName: this.fieldsFlattened[modelPath.k].model,
+            modelName: this.fieldsFlattened[modelField.path].model,
           })
         }
       }
@@ -721,6 +724,6 @@ Model.prototype._recurseAndFindModels = function (dataArr, dataParentPath='', mo
 Model.prototype._queryOptions = [
   // todo: remove type properties
   'blacklist', 'data', 'find', 'findOneAndUpdate', 'insert', 'model', '_one', 'populate', 'project',
-  'projectionValidate', 'query', 'remove', 'req', 'respond', 'skipValidation', 'type', 'update',
+  'projectionInclusion', 'projectionKeys', 'query', 'remove', 'req', 'respond', 'skipValidation', 'type', 'update',
   'validateUndefined',
 ]

package/lib/model-validate.js

@@ -18,25 +18,31 @@ Model.prototype.validate = async function (data, opts) {
    * @return promise(errors[] || pruned data{})
    * @this model
    */
-  data = util.deepCopy(data)
+  // console.time('1')
+  data = util.deepCopy(data) // 30ms for 100k fields
+  // console.timeEnd('1')
   opts = opts || {}
   opts.update = opts.update || opts.findOneAndUpdate
   opts.insert = !opts.update
   opts.skipValidation = opts.skipValidation === true ? true : util.toArray(opts.skipValidation||[])
 
   // Get projection
-  if (opts.project) opts.projectionValidate = this._getProjectionFromProject(opts.project)
-  else opts.projectionValidate = this._getProjectionFromBlacklist(opts.update ? 'update' : 'insert', opts.blacklist)
+  if (opts.project) var projectionValidate = this._getProjectionFromProject(opts.project)
+  else projectionValidate = this._getProjectionFromBlacklist(opts.update ? 'update' : 'insert', opts.blacklist)
+  opts.projectionKeys = Object.keys(projectionValidate || {})
+  opts.projectionInclusion = (projectionValidate || {})[opts.projectionKeys[0]] ? true : false // default false
 
   // Hook: beforeValidate
   data = await util.runSeries.call(this, this.beforeValidate.map(f => f.bind(opts)), 'beforeValidate', data)
 
   // Recurse and validate fields
+  // console.time('_validateFields')
   let response = util.toArray(data).map(item => {
-    let validated = this._validateFields(item, this.fields, item, opts, '')
+    let validated = this._validateFields(item, this.fields, item, opts, '', '')
     if (validated[0].length) throw validated[0]
     else return validated[1]
   })
+  // console.timeEnd('_validateFields')
 
   // Single document?
   response = util.isArray(data)? response : response[0]
@@ -62,14 +68,17 @@ Model.prototype._getMostSpecificKeyMatchingPath = function (object, path) {
   return key
 }
 
-Model.prototype._validateFields = function (dataRoot, fields, data, opts, path) {
+Model.prototype._validateFields = function (dataRoot, fields, data, opts, parentPath, parentPath2) {
   /**
-   * Recurse through and retrieve any errors and valid data
+   * Recurse through and retrieve any errors and valid data (this needs to be perfomant)
+   * Note: This is now super fast, it can validate 100k possible fields in 235ms
+   * 
    * @param {any} dataRoot
    * @param {object|array} fields
    * @param {any} data
    * @param {object} opts
-   * @param {string} path
+   * @param {string} parentPath - data localised parent, e.g. pets.1.name
+   * @param {string} parentPath2 - no numerical keys, e.g. pets.name
    * @return [errors, valid-data]
    * @this model
    *
@@ -77,19 +86,31 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, path)
    *   Fields second recursion = [0]: { name: {}, color: {} }
    */
   let errors = []
-  let data2 = util.isArray(fields)? [] : {}
+  let fieldsIsArray = util.isArray(fields)
+  let fieldsArray = fieldsIsArray ? fields : Object.keys(fields)
   let timestamps = util.isDefined(opts.timestamps) ? opts.timestamps : this.manager.opts.timestamps
+  let dataArray = util.forceArray(data)
+  let data2 = fieldsIsArray ? [] : {}
 
-  util.forEach(util.forceArray(data), function(data, i) {
-    util.forEach(fields, function(field, fieldName) {
+  for (let i=0, l=dataArray.length; i<l; i++) {
+    const item = dataArray[i]
+
+    for (let m=0, n=fieldsArray.length; m<n; m++) {
+      // iterations++
+      const fieldName = fieldsIsArray ? m : fieldsArray[m] // array|object key
+      const field = fields[fieldName]
+      if (fieldName == 'schema') continue
+      // if (!parentPath && fieldName == 'categories') console.time(fieldName)
+      // if (!parentPath && fieldName == 'categories') console.time(fieldName + 1)
       let verrors = []
-      let schema = field.schema || field
-      let value = util.isArray(fields)? data : (data||{})[fieldName]
-      let indexOrFieldName = util.isArray(fields)? i : fieldName
-      let path2 = `${path}.${indexOrFieldName}`.replace(/^\./, '')
-      let path3 = path2.replace(/(^|\.)[0-9]+(\.|$)/, '$2') // no numerical keys, e.g. pets.1.name = pets.name
-      let isType = 'is' + util.ucFirst(schema.type)
-      let isTypeRule = this.rules[isType] || rules[isType]
+      let schema = field.schema
+      let value = fieldsIsArray ? item : (item||{})[fieldName]
+      let indexOrFieldName = fieldsIsArray ? i : fieldName
+      let path = `${parentPath}.${indexOrFieldName}`
+      let path2 = fieldsIsArray ? parentPath2 : `${parentPath2}.${fieldName}`
+      if (path[0] == '.') path = path.slice(1) // remove leading dot, e.g. .pets.1.name
+      if (path2[0] == '.') path2 = path2.slice(1) // remove leading dot, e.g. .pets.1.name
+      let isTypeRule = this.rules[schema.isType] || rules[schema.isType]
 
       // Timestamp overrides
       if (schema.timestampField) {
@@ -103,68 +124,70 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, path)
         }
       }
 
-      // Ignore blacklisted
-      if (this._pathBlacklisted(path3, opts.projectionValidate) && !schema.defaultOverride) return
       // Ignore insert only
-      if (opts.update && schema.insertOnly) return
+      if (opts.update && schema.insertOnly) continue
       // Ignore virtual fields
-      if (schema.virtual) return
+      if (schema.virtual) continue
+      // Ignore blacklisted
+      if (!schema.defaultOverride && this._pathBlacklisted(path2, opts.projectionInclusion, opts.projectionKeys)) continue
       // Type cast the value if tryParse is available, .e.g. isInteger.tryParse
-      if (isTypeRule && util.isFunction(isTypeRule.tryParse)) {
-        value = isTypeRule.tryParse.call(dataRoot, value, fieldName, this)
+      if (isTypeRule && typeof isTypeRule.tryParse == 'function') {
+        value = isTypeRule.tryParse.call(dataRoot, value, fieldName, this) // 80ms // DISABLE
       }
-
-      // Schema field (ignore object/array schemas)
-      if (util.isSchema(field) && fieldName !== 'schema') {
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path2)))
-        if (util.isDefined(value) && !verrors.length) data2[indexOrFieldName] = value
-
-      // Fields can be a subdocument
-      } else if (util.isSubdocument(field)) {
+      
+      // Field is a subdocument
+      if (schema.isObject) {
         // Object schema errors
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path2)))
+        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
         // Recurse if inserting, value is a subdocument, or a deep property (todo: not dot-notation)
         if (
           opts.insert ||
           util.isObject(value) ||
-          (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : (path2||'').match(/\./))
+          (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : (path||'').indexOf('.') !== -1)
         ) {
-          var res = this._validateFields(dataRoot, field, value, opts, path2)
+          var res = this._validateFields(dataRoot, field, value, opts, path, path2)
           errors.push(...res[0])
         }
         if (util.isDefined(value) && !verrors.length) {
           data2[indexOrFieldName] = res? res[1] : value
         }
 
-      // Fields can be an array
-      } else if (util.isArray(field)) {
+      // Field is an array
+      } else if (schema.isArray) {
         // Array schema errors
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path2)))
+        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
         // Data value is array too
         if (util.isArray(value)) {
-          var res2 = this._validateFields(dataRoot, field, value, opts, path2)
+          var res2 = this._validateFields(dataRoot, field, value, opts, path, path2)
           errors.push(...res2[0])
         }
         if (util.isDefined(value) && !verrors.length) {
           data2[indexOrFieldName] = res2? res2[1] : value
         }
+
+      // Field is a field-type/field-schema
+      } else {
+        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
+        if (util.isDefined(value) && !verrors.length) data2[indexOrFieldName] = value
       }
-    }, this)
-  }, this)
+      // if (!parentPath && fieldName == 'categories') console.timeEnd(fieldName)
+    }
+  }
 
   // Normalise array indexes and return
-  if (util.isArray(fields)) data2 = data2.filter(() => true)
+  if (fieldsIsArray) data2 = data2.filter(() => true) //todo: remove???
   if (data === null) data2 = null
   return [errors, data2]
 }
 
-Model.prototype._validateRules = function (dataRoot, field, value, opts, path) {
+Model.prototype._validateRules = function (dataRoot, fieldSchema, value, opts, path) {
   /**
    * Validate all the field's rules
    * @param {object} dataRoot - data
-   * @param {object} field - field schema
-   * @param {string} path - full field path
+   * @param {object} fieldSchema - field schema
+   * @param {any} value - data value
    * @param {object} opts - original validate() options
+   * @param {string} path - full field path, e.g. pets.1.name
    * @return {array} errors
    * @this model
    */
@@ -192,47 +215,49 @@ Model.prototype._validateRules = function (dataRoot, field, value, opts, path) {
     }
   }
 
-  for (let ruleName in field) {
+  for (let ruleName in fieldSchema) {
     if (this._ignoredRules.indexOf(ruleName) > -1) continue
-    let error = this._validateRule(dataRoot, ruleName, field, field[ruleName], value, opts, path)
+    let error = this._validateRule(dataRoot, ruleName, fieldSchema, fieldSchema[ruleName], value, opts, path)
     if (error && ruleName == 'required') return [error] // only show the required error
     if (error) errors.push(error)
   }
   return errors
 }
 
-Model.prototype._validateRule = function (dataRoot, ruleName, field, ruleArg, value, opts, path) {
-  // this.debug(path, field, ruleName, ruleArg, value)
+Model.prototype._validateRule = function (dataRoot, ruleName, fieldSchema, ruleArg, value, opts, path) {
+  // this.debug(path, fieldSchema, ruleName, ruleArg, value)
   // Remove [] from the message path, and simply ignore non-numeric children to test for all array items
-  ruleArg = ruleArg === true? undefined : ruleArg
+  ruleArg = ruleArg === true ? undefined : ruleArg
   let rule = this.rules[ruleName] || rules[ruleName]
-  let fieldName = path.match(/[^.]+$/)[0]
-  let isDeepProp = path.match(/\./) // todo: not dot-notation
-  let ruleMessageKey = this.messagesLen && this._getMostSpecificKeyMatchingPath(this.messages, path)
-  let ruleMessage = ruleMessageKey && this.messages[ruleMessageKey][ruleName]
-  let validateUndefined = util.isDefined(opts.validateUndefined) ? opts.validateUndefined : opts.insert || isDeepProp
-  if (!ruleMessage) ruleMessage = rule.message
+  let validateUndefined = typeof opts.validateUndefined != 'undefined' 
+    ? opts.validateUndefined 
+    : opts.insert || path.includes('.') // is a deep property
 
   // Undefined value
   if (typeof value === 'undefined' && (!validateUndefined  || !rule.validateUndefined)) return
 
   // Ignore null (if nullObject is set on objects or arrays)
-  if (value === null && (field.isObject || field.isArray) && field.nullObject && !rule.validateNull) return
+  if (value === null && (fieldSchema.isObject || fieldSchema.isArray) && fieldSchema.nullObject && !rule.validateNull) return
 
   // Ignore null
-  if (value === null && !(field.isObject || field.isArray) && !rule.validateNull) return
+  if (value === null && !(fieldSchema.isObject || fieldSchema.isArray) && !rule.validateNull) return
 
   // Ignore empty strings
   if (value === '' && !rule.validateEmptyString) return
 
   // Rule failed
-  if (!rule.fn.call(dataRoot, value, ruleArg, path, this)) return {
-    detail: util.isFunction(ruleMessage)
-      ? ruleMessage.call(dataRoot, value, ruleArg, path, this)
-      : ruleMessage,
-    meta: { rule: ruleName, model: this.name, field: fieldName, detailLong: rule.messageLong },
-    status: '400',
-    title: path,
+  if (!rule.fn.call(dataRoot, value, ruleArg, path, this)) {
+    let ruleMessageKey = this.messagesLen && this._getMostSpecificKeyMatchingPath(this.messages, path)
+    let ruleMessage = ruleMessageKey && this.messages[ruleMessageKey][ruleName]
+    if (!ruleMessage) ruleMessage = rule.message
+    return {
+      detail: util.isFunction(ruleMessage)
+        ? ruleMessage.call(dataRoot, value, ruleArg, path, this)
+        : ruleMessage,
+      meta: { rule: ruleName, model: this.name, field: path.match(/[^.]+$/)[0], detailLong: rule.messageLong },
+      status: '400',
+      title: path,
+    }
   }
 }
 
@@ -241,5 +266,5 @@ Model.prototype._ignoredRules = [
   // todo: need to remove filesize and formats..
   'awsAcl', 'awsBucket', 'default', 'defaultOverride', 'filename', 'filesize', 'fileSize', 'formats',
   'image', 'index', 'insertOnly', 'model', 'nullObject', 'params', 'path', 'getSignedUrl', 'timestampField',
-  'type', 'virtual',
+  'type', 'isType', 'isSchema', 'virtual',
 ]