monastery 3.3.0 → 3.4.0

package/lib/model-validate.js

@@ -9,8 +9,8 @@ Model.prototype.validate = async function (data, opts) {
    * @param {object} <opts>
    *     @param {array|string|false} <opts.blacklist> - augment insertBL/updateBL, `false` will remove blacklisting
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
-   *     @param {array|string|true} <opts.skipValidation> - skip validation for these fields, or pass `true` to skip
-   *         all fields and hooks
+   *     @param {boolran} <opts.skipHooks> - skip hooks
+   *     @param {array|string} <opts.skipValidation> - skip validation for these fields
    *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is
    *         updated, depending on the `options.update` value
    *     @param {boolean(false)} <opts.update> - are we validating for insert or update? todo: change to `type`
@@ -25,8 +25,8 @@ Model.prototype.validate = async function (data, opts) {
   opts = opts || {}
   opts.update = opts.update || opts.findOneAndUpdate
   opts.insert = !opts.update
-  opts.skipValidation = opts.skipValidation === true ? true : util.toArray(opts.skipValidation||[])
-  if (opts.skipValidation === true) return data
+  opts.skipValidation = opts.skipValidation === true ? true : util.toArray(opts.skipValidation || [])
+  opts.timestamps = util.isDefined(opts.timestamps) ? opts.timestamps : this.manager.opts.timestamps
 
   // Get projection
   if (opts.project) var projectionValidate = this._getProjectionFromProject(opts.project)
@@ -35,14 +35,14 @@ Model.prototype.validate = async function (data, opts) {
   opts.projectionInclusion = (projectionValidate || {})[opts.projectionKeys[0]] ? true : false // default false
 
   // Hook: beforeValidate
-  data = await util.runSeries.call(this, this.beforeValidate.map(f => f.bind(opts)), 'beforeValidate', data)
+  data = await this._callHooks('beforeValidate', data, opts)
 
   // Recurse and validate fields
   // console.time('_validateFields')
   let response = util.toArray(data).map(item => {
-    let validated = this._validateFields(item, this.fields, item, opts, '', '')
-    if (validated[0].length) throw validated[0]
-    else return validated[1]
+    const [errors, validated] = this._validateFields(item, this.fields, item, opts, '', '')
+    if (errors.length) throw errors // todo: maybe add trace to this object?
+    else return validated
   })
   // console.timeEnd('_validateFields')
 
@@ -76,11 +76,11 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
    * Note: This is now super fast, it can validate 100k possible fields in 235ms
    * 
    * @param {any} dataRoot
-   * @param {object|array} fields
+   * @param {object|array} fields (from definition)
    * @param {any} data
    * @param {object} opts
-   * @param {string} parentPath - data localised parent, e.g. pets.1.name
-   * @param {string} parentPath2 - no numerical keys, e.g. pets.name
+   * @param {string} parentPath - parent data path, e.g. pets.1.name
+   * @param {string} parentPath2 - parent field path, no numerical keys, e.g. pets.name
    * @return [errors, valid-data]
    * @this model
    *
@@ -90,112 +90,146 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
   let errors = []
   let fieldsIsArray = util.isArray(fields)
   let fieldsArray = fieldsIsArray ? fields : Object.keys(fields)
-  let timestamps = util.isDefined(opts.timestamps) ? opts.timestamps : this.manager.opts.timestamps
   let dataArray = util.forceArray(data)
   let data2 = fieldsIsArray ? [] : {}
   let notStrict = fields.schema.strict === false
 
   for (let i=0, l=dataArray.length; i<l; i++) {
-    const item = dataArray[i]
+    const dataItem = dataArray[i]
+    const dataKeys = Object.keys(dataItem || {}) // may be false when inserting, e.g. mode.insert({ data: false })
 
-    for (let m=0, n=fieldsArray.length; m<n; m++) {
-      // iterations++
-      const fieldName = fieldsIsArray ? m : fieldsArray[m] // array|object key
-      const field = fields[fieldName]
-      if (fieldName == 'schema') continue
-      // if (!parentPath && fieldName == 'categories') console.time(fieldName)
-      // if (!parentPath && fieldName == 'categories') console.time(fieldName + 1)
-      let schema = field.schema
-      let value = fieldsIsArray ? item : (item||{})[fieldName]
-      let indexOrFieldName = fieldsIsArray ? i : fieldName
-      let path = `${parentPath}.${indexOrFieldName}`
-      let path2 = fieldsIsArray ? parentPath2 : `${parentPath2}.${fieldName}`
-      if (path[0] == '.') path = path.slice(1) // remove leading dot, e.g. .pets.1.name
-      if (path2[0] == '.') path2 = path2.slice(1) // remove leading dot, e.g. .pets.1.name
-      let isTypeRule = this.rules[schema.isType] || rules[schema.isType]
-
-      // Timestamp overrides
-      if (schema.timestampField) {
-        if (timestamps && ((fieldName == 'createdAt' && opts.insert) || fieldName == 'updatedAt')) {
-          value = schema.default.call(dataRoot, fieldName, this)
-        }
-      // Use the default if available
-      } else if (util.isDefined(schema.default)) {
-        if ((!util.isDefined(value) && opts.insert) || schema.defaultOverride) {
-          value = util.isFunction(schema.default)? schema.default.call(dataRoot, fieldName, this) : schema.default
-        }
+    // Add any non-schema properties, excluding array properties
+    if (notStrict && !fieldsIsArray) {
+      for (let m=0, n=dataKeys.length; m<n; m++) {
+        if (!fieldsArray.includes(dataKeys[m])) data2[dataKeys[m]] = dataItem[dataKeys[m]]
       }
+    }
 
-      // Ignore insert only
-      if (opts.update && schema.insertOnly) continue
-      // Ignore virtual fields
-      if (schema.virtual) continue
-      // Ignore blacklisted
-      if (!schema.defaultOverride && this._pathBlacklisted(path2, opts.projectionInclusion, opts.projectionKeys)) continue
-      // Type cast the value if tryParse is available, .e.g. isInteger.tryParse
-      if (isTypeRule && typeof isTypeRule.tryParse == 'function') {
-        value = isTypeRule.tryParse.call(dataRoot, value, fieldName, this) // 80ms // DISABLE
-      }
-      
-      // Field is a subdocument
-      if (schema.isObject) {
-        // Object schema errors
-        let res
-        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
-        if (verrors.length) errors.push(...verrors)
-        // Recurse if inserting, value is a subdocument, or a deep property (todo: not dot-notation)
-        if (
-          opts.insert ||
-          util.isObject(value) ||
-          (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : (path||'').indexOf('.') !== -1)
-        ) {
-          res = this._validateFields(dataRoot, field, value, opts, path, path2)
-          if (res[0].length) errors.push(...res[0])
-        }
-        if (util.isDefined(value) && !verrors.length) {
-          data2[indexOrFieldName] = res ? res[1] : value
-        }
+    // Loop through each schema field
+    for (let m=0, n=fieldsArray.length; m<n; m++) {
+      const fieldName = fieldsIsArray ? m : fieldsArray[m] // array|object key
+      const dataFieldName = fieldsIsArray ? i : fieldName
+      const value = fieldsIsArray ? dataItem : (dataItem||{})[fieldName]
+      const field = fields[fieldName] // schema field
 
-      // Field is an array
-      } else if (schema.isArray) {
-        // Array schema errors
-        let res2
-        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
-        if (verrors.length) errors.push(...verrors)
-        // Data value is array too
-        if (util.isArray(value)) {
-          res2 = this._validateFields(dataRoot, field, value, opts, path, path2)
-          if (res2[0].length) errors.push(...res2[0])
-        }
-        if (util.isDefined(value) && !verrors.length) {
-          data2[indexOrFieldName] = res2? res2[1] : value
-        }
+      // Field paths
+      const path = `${parentPath ? parentPath + '.' : ''}${dataFieldName}` // e.g. pets.1.name
+      const path2 = fieldsIsArray ? parentPath2 : (`${parentPath2 ? parentPath2 + '.' : ''}${fieldName}`) // e.g. pets.name
 
-      // Field is a field-type/field-schema
-      } else {
-        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
-        if (verrors.length) errors.push(...verrors)
-        if (util.isDefined(value) && !verrors.length) data2[indexOrFieldName] = value
-      }
-      // if (!parentPath && fieldName == 'categories') console.timeEnd(fieldName)
+      const [errors2, value2] = this._validateField(dataRoot, field, fieldName, value, opts, path, path2)
+      if (errors2.length) errors = errors.concat(errors2)
+      else if (typeof value2 !== 'undefined') data2[dataFieldName] = value2
     }
 
-    // Add any extra fields that are not in the schema. Item maybe false when inserting (from recursing above)
-    if (notStrict && !fieldsIsArray && item) {
-      const allDataKeys = Object.keys(item)
-      for (let m=0, n=allDataKeys.length; m<n; m++) {
-        const key = allDataKeys[m]
-        if (!fieldsArray.includes(key)) data2[key] = item[key]
+    // Validate dot-notation fields on the dataRoot, e.g. pets.1.name
+    if (!parentPath) {
+      for (let j=0, k=dataKeys.length; j<k; j++) {
+        if (dataKeys[j].includes('.')) {
+          const path = dataKeys[j]
+          const path2 = path.replace(/\.(\d|\$)+/g, '')
+          const pathWithZeroIndexes = path.replace(/\.(\d|\$)+/g, '.0')
+          const fieldName = pathWithZeroIndexes.split('.')[0]
+          const field = util.deepFind(fields, pathWithZeroIndexes)
+          if (!field) continue
+
+          const [errors2, value2] = this._validateField(dataRoot, field, fieldName, dataItem[path], opts, path, path2)
+          if (errors2.length) errors = errors.concat(errors2)
+          else if (typeof value2 !== 'undefined') data2[path] = value2
+        }
       }
     }
   }
 
   // Normalise array indexes and return
-  if (fieldsIsArray) data2 = data2.filter(() => true) //todo: remove???
   if (data === null) data2 = null
   return [errors, data2]
 }
 
+Model.prototype._validateField = function (dataRoot, field, fieldName, value, opts, path, path2) {
+  /**
+   * Validate a field
+   * 
+   * @param {object} dataRoot - data
+   * @param {object} field - field (from definition)
+   * @param {string} fieldName
+   * @param {any} value
+   * @param {object} opts - original validate() options
+   * @param {string} path - full data path, e.g. pets.1.name
+   * @param {string} path2 - full field path, without numerical keys, e.g. pets.name
+   * @return [errors[], valid-value]
+   * @this model
+   */
+  // iterations++
+  const schema = field.schema
+  if (fieldName == 'schema') return [[]]
+  // if (!parentPath && fieldName == 'categories') console.time(fieldName)
+  // if (!parentPath && fieldName == 'categories') console.time(fieldName + 1)
+
+  const isTypeRule = this.rules[schema.isType] || rules[schema.isType]
+
+  // Timestamp overrides
+  if (schema.timestampField) {
+    if (opts.timestamps && ((fieldName == 'createdAt' && opts.insert) || fieldName == 'updatedAt')) {
+      value = schema.default.call(dataRoot, fieldName, this)
+    }
+  // Use the default if available
+  } else if (util.isDefined(schema.default)) {
+    if ((!util.isDefined(value) && opts.insert) || schema.defaultOverride) {
+      value = util.isFunction(schema.default)? schema.default.call(dataRoot, fieldName, this) : schema.default
+    }
+  }
+
+  // Ignore insert only
+  if (opts.update && schema.insertOnly) return [[]]
+  // Ignore virtual fields
+  if (schema.virtual) return [[]]
+  // Ignore blacklisted
+  if (!schema.defaultOverride && this._pathBlacklisted(path2, opts.projectionInclusion, opts.projectionKeys)) return [[]]
+  // Type cast the value if tryParse is available, .e.g. isInteger.tryParse
+  if (isTypeRule && typeof isTypeRule.tryParse == 'function') {
+    value = isTypeRule.tryParse.call(dataRoot, value, fieldName, this) // 80ms // DISABLE
+  }
+  
+  // Field is a subdocument
+  if (schema.isObject) {
+    // Object schema errors
+    let verrors2, value2
+    let verrors = this._validateRules(dataRoot, schema, value, opts, path)
+    // Recurse if inserting, value is a subdocument, or we're within a subdocument (todo: not dot-notation)
+    const parentIsSubdocument = (path||'').indexOf('.') !== -1
+    if (
+      opts.insert ||
+      util.isObject(value) ||
+      (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : parentIsSubdocument)
+    ) {
+      [verrors2, value2] = this._validateFields(dataRoot, field, value, opts, path, path2)
+      if (verrors2.length) verrors = verrors.concat(verrors2)
+    }
+    if (verrors.length) return [verrors]
+    else return [[], typeof value2 !== 'undefined' && typeof value !== 'undefined' ? value2 : value]
+
+  // Field is an array
+  } else if (schema.isArray) {
+    // Array schema errors
+    let verrors2, value2
+    let verrors = this._validateRules(dataRoot, schema, value, opts, path)
+    // Data value is array too
+    if (util.isArray(value)) {
+      [verrors2, value2] = this._validateFields(dataRoot, field, value, opts, path, path2)
+      if (verrors2.length) verrors = verrors.concat(verrors2)
+    }
+    if (verrors.length) return [verrors]
+    else return [[], typeof value2 !== 'undefined' && typeof value !== 'undefined' ? value2 : value]
+
+  // Field is a field-type/field-schema
+  } else {
+    const verrors = this._validateRules(dataRoot, schema, value, opts, path)
+    if (verrors.length) return [verrors]
+    else return [[], value]
+  }
+  // if (!parentPath && fieldName == 'categories') console.timeEnd(fieldName)
+}
+
 Model.prototype._validateRules = function (dataRoot, fieldSchema, value, opts, path) {
   /**
    * Validate all the field's rules
@@ -203,7 +237,7 @@ Model.prototype._validateRules = function (dataRoot, fieldSchema, value, opts, p
    * @param {object} fieldSchema - field schema
    * @param {any} value - data value
    * @param {object} opts - original validate() options
-   * @param {string} path - full field path, e.g. pets.1.name
+   * @param {string} path - full data path, e.g. pets.1.name
    * @return {array} errors
    * @this model
    */

package/lib/model.js

@@ -405,6 +405,21 @@ Model.prototype._setupIndexes = async function(fields, opts={}) {
   }
 }
 
+Model.prototype._callHooks = async function(hookName, data, opts) {
+  /**
+   * Calls hooks in series
+   * 
+   * @param {string} hookName - e.g. 'beforeValidate'
+   * @param {object} opts - operation options, e.g. { data, skipValidation, ... }
+   * @param {any} arg - data to pass to the first function
+   * 
+   * @return {any} - the result of the last function
+   * @this model
+   */
+  if (opts.skipHooks) return data
+  return await util.runSeries.call(this, this[hookName].map(f => f.bind(opts)), hookName, data)
+}
+
 Model.prototype._defaultFields = {
   _id: {
     insertOnly: true,

package/lib/util.js

@@ -40,6 +40,20 @@ module.exports = {
     return obj2
   },
 
+  deepFind: (obj, path) => {
+    // Returns a nested value from a path URI e.g. user.books.1.title
+    if (!obj) return undefined
+    let last
+    let chunks = (path || '').split('.')
+    let target = obj
+    for (let i = 0, l = chunks.length; i < l; i++) {
+      last = l === i + 1
+      if (!last && !target[chunks[i]]) break
+      else target = target[chunks[i]]
+    }
+    return last ? target : undefined
+  },
+
   forEach: function(obj, iteratee, context) {
     if (this.isArrayLike(obj)) {
       for (let i=0, l=obj.length; i<l; i++) {
@@ -205,97 +219,111 @@ module.exports = {
     return shallowCopy
   },
 
-  parseData: function(obj) {
+  parseData: function(obj, parseBracketToDotNotation, parseDotNotation) {
     /**
-     * Mutates dot notation objects, and then FormData (bracket notation) objects into a deep object
-     * @param {object}
-     * @return promise(data)
+     * Coverts dot notation objects, and then bracket notation objects (form data) into deep objects
+     * @param {object} 
+     * @param {boolean} parseBracketToDotNotation - converts bracket notation to dot notation, instead of deep objects
+     * @param {boolean} parseDotNotation - converts dot notation to deep objects
+     * @return data
      */
-    let data = this.parseDotNotation(obj)
-    return this.parseFormData(data)
+    obj = parseBracketToDotNotation ? this.parseBracketToDotNotation(obj) : this.parseBracketNotation(obj)
+    if (parseDotNotation) obj = this.parseDotNotation(obj)
+    return obj
   },
 
   parseDotNotation: function(obj) {
     /**
-     * Mutates dot notation objects into a deep object
-     * @param {object}
+     * Converts dot notation field paths into deep objects
+     * @param {object} obj - e.g. { 'deep.companyLogos2.1.logo': '' } (not mutated)
+     * @return {object} - e.g. { deep: { companyLogos2: [{ logo: '' }] }}
      */
-    if (!Object.keys(obj).find(o => o.indexOf('.') !== -1)) return obj
-    let objCopy = this.deepCopy(obj) // maybe convert to JSON.parse(JSON.stringify(obj))
-
-    for (let key in obj) {
-      if (key.indexOf('.') !== -1) {
-        setup(key, obj[key], obj)
+    if (!Object.keys(obj).some(key => key.includes('.'))) return obj
+  
+    const result = {}
+  
+    for (const key in obj) {
+      if (key.includes('.')) {
+        setValue(result, key.split('.'), obj[key])
       } else {
-        // Ordinary values may of been updated by the bracket notation values, we are
-        // reassigning, trying to preserve the order of keys (not always guaranteed in for loops)
-        obj[key] = objCopy[key]
+        result[key] = obj[key] // keep non-dot notation values
       }
     }
-    return obj
-    function setup(str, val, obj) {
-      let parentObj = obj
-      let grandparentObj = obj
-      let keys = str.split(/\./)
-
-      for (var i=0, l=Math.max(1, keys.length-1); i<l; ++i) {
-        let key = keys[i]
-        // If denoting an array, make sure parent is an array
-        if (key.match(/^[0-9]+$/) && !Array.isArray(parentObj)) {
-          parentObj = grandparentObj[keys[i-1]] = []
+  
+    function setValue(target, keys, value) {
+      let current = target
+      for (let i = 0; i < keys.length - 1; i++) {
+        const key = keys[i]
+        if (!current[key]) {
+          current[key] = /^\d+$/.test(keys[i + 1]) ? [] : {}
         }
-        grandparentObj = parentObj
-        parentObj = parentObj[key] = parentObj[key] || {}
+        current = current[key]
       }
-
-      parentObj[keys[i]] = val
-      delete obj[str]
+      current[keys[keys.length - 1]] = value
     }
+  
+    return result
   },
 
-  parseFormData: function(obj) {
+  parseBracketNotation: function(obj) {
     /**
-     * Mutates FormData (bracket notation) objects into a deep object
-     * @param {object}
-     * @return promise(data)
-     * E.g. ['user']['name']
-     * E.g. ['user']['petnames'][0]
-     * E.g. ['users'][0]['name']
+     * Converts bracket notation field paths (form data) into deep objects
+     * @param {object} obj - e.g. { 'users[0][first]': 'Martin' } (not mutated)
+     * @return {object} - e.g. { users: [{ first: 'Martin' }] }
      */
-    if (!Object.keys(obj).find(o => o.indexOf('[') !== -1)) return obj
-    let objCopy = this.deepCopy(obj) // maybe convert to JSON.parse(JSON.stringify(obj))
-
-    for (let key in obj) {
-      if (key.match(/\[\]\[/i)) {
+    if (!Object.keys(obj).some(key => key.includes('['))) return obj
+  
+    const result = {}
+  
+    for (const key in obj) {
+      if (key.includes('[][')) {
         throw new Error(`Monastery: Array items in bracket notation need array indexes "${key}", e.g. users[0][name]`)
       }
-      if (key.indexOf('[') !== -1) {
-        setup(key)
+      if (key.includes('[')) {
+        // console.log(key, key.split(/[[\]]/).filter(Boolean), key.split(/[[\]]/))
+        setValue(result, key.split(/[[\]]/).filter(Boolean), obj[key])
       } else {
-        // Ordinary values may of been updated by the bracket notation values, we are
-        // reassigning, trying to preserve the order of keys (not always guaranteed in for loops)
-        obj[key] = objCopy[key]
+        result[key] = obj[key] // keep non-dot notation values
       }
     }
-    return obj
-    function setup(path) {
-      let parent = obj
-      let grandparent = obj
-      let chunks = path.replace(/]/g, '').split('[')
-      for (var i=0, l=chunks.length; i<l; ++i) {
-        // If denoting an array, make sure parent is an array
-        if (chunks[i].match(/^$|^[0-9]+$/) && !Array.isArray(parent)) {
-          parent = grandparent[chunks[i-1]] = []
-        }
-        if (i !== l-1) {
-          grandparent = parent
-          parent = parent[chunks[i]] = parent[chunks[i]] || {}
-        } else {
-          parent[chunks[i]] = obj[path]
+
+    function setValue(target, keys, value) {
+      let current = target
+      for (let i = 0; i < keys.length - 1; i++) {
+        const key = keys[i]
+        if (!current[key]) {
+          current[key] = /^\d+$/.test(keys[i + 1]) ? [] : {}
         }
+        current = current[key]
+      }
+      current[keys[keys.length - 1]] = value
+    }
+  
+    return result
+  },
+
+  parseBracketToDotNotation: function(obj) {
+    /**
+     * Converts bracket notation field paths (form data) into dot notation paths
+     * @param {object} obj - e.g. { 'users[0][first]': 'Martin' } (not mutated)
+     * @return {object} - e.g. { 'users.0.first': 'Martin' }
+     */
+    const result = {}
+  
+    for (const key in obj) {
+      if (key.includes('[][')) {
+        throw new Error(`Monastery: Array items in bracket notation need array indexes "${key}", e.g. users[0][name]`)
+      }
+      if (key.includes('[')) {
+        const keys = key.split(/[[\]]/).filter(Boolean)
+        const newKey = keys.shift() + keys.map(k => `.${k}`).join('')
+        result[newKey] = obj[key]
+      } else {
+        result[key] = obj[key] // keep non-bracket notation values
       }
-      delete obj[path]
     }
+  
+    return result
   },
 
   pick: function(obj, keys) {
@@ -330,16 +358,17 @@ module.exports = {
   },
 
   runSeries: function(tasks, hookName, data) {
-    /*
+    /**
      * Runs functions in series
-     * @param {function(err, result)[]} tasks - array of functions
-     * @param {string} <hookName> - e.g. 'afterFind'
-     * @param {any} data - data to pass to the first function
-     * @param {function(err, results[])} <cb>
+     * 
+     * @param {function(err, result)[]}  tasks - array of functions
+     * @param {string}  hookName - e.g. 'afterFind'
+     * @param {any}  data - data to pass to the first function
+     * 
      * @return promise
      * @this Model
      * @source https://github.com/feross/run-series
-     */
+     **/
     let current = 0
     let isSync = true
     let caller = (this.afterFindName || this.name) + '.' + hookName

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A simple, straightforward MongoDB ODM",
   "author": "Ricky Boyce",
-  "version": "3.3.0",
+  "version": "3.4.0",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",