monastery 1.29.0 → 1.30.3

package/lib/model-crud.js

@@ -9,7 +9,7 @@ module.exports = {
      *     @param {object|array} <opts.data> - documents to insert
      *     @param {array|string|false} <opts.blacklist> - augment schema.insertBL, `false` will remove all blacklisting
      *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-     *     @param {array|string|true} ignoreUndefined - ignore all required fields during insert, or
+     *     @param {array|string|false} validateUndefined - ignore all required fields during insert, or
      *         undefined subdocument required fields that have a defined parent/grandparent during update
      *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
      *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are automatically inserted
@@ -23,7 +23,7 @@ module.exports = {
     opts.model = this
     let data = opts.data = opts.data || (opts.req? opts.req.body : {})
     let options = util.omit(opts, [
-      'data', 'insert', 'model', 'respond', 'ignoreUndefined', 'skipValidation', 'blacklist'
+      'data', 'insert', 'model', 'respond', 'validateUndefined', 'skipValidation', 'blacklist'
     ])
     if (cb && !util.isFunction(cb)) {
       throw new Error(`The callback passed to ${this.name}.insert() is not a function`)
@@ -194,7 +194,7 @@ module.exports = {
      *     @param {object} <opts.query> - mongodb query object
      *     @param {object|array} <opts.data> - mongodb document update object(s)
      *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-     *     @param {array|string|true} ignoreUndefined - ignore all required fields during insert, or
+     *     @param {array|string|false} validateUndefined - ignore all required fields during insert, or
      *         undefined subdocument required fields that have a defined parent/grandparent during update
      *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
      *     @param {boolean} <opts.timestamps> - whether `updatedAt` is automatically updated
@@ -215,7 +215,7 @@ module.exports = {
       data = opts.data = opts.data || (opts.req? opts.req.body : null)
       operators = util.pluck(opts, [/^\$/])
       // Operation options
-      options = util.omit(opts, ['data', 'query', 'respond', 'ignoreUndefined', 'skipValidation', 'blacklist'])
+      options = util.omit(opts, ['data', 'query', 'respond', 'validateUndefined', 'skipValidation', 'blacklist'])
       options.sort = options.sort || { 'createdAt': -1 }
       options.limit = parseInt(options.limit || 0)
       // Sort string passed

package/lib/model-validate.js

@@ -12,7 +12,7 @@ module.exports = {
      *     @param {boolean(false)} update - are we validating for insert or update?
      *     @param {array|string|false} blacklist - augment schema blacklist, `false` will remove all blacklisting
      *     @param {array|string} projection - only return these fields, ignores blacklist
-     *     @param {array|string|true} ignoreUndefined - ignore all required fields during insert, or undefined
+     *     @param {array|string|false} validateUndefined - ignore all required fields during insert, or undefined
      *         subdocument required fields that have a defined parent/grandparent during update
      *     @param {array|string|true} skipValidation - skip validation on these fields
      *     @param {boolean} timestamps - whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is
@@ -177,7 +177,11 @@ module.exports = {
           // Object schema errors
           errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path2)))
           // Recurse if data value is a subdocument, or when inserting, or when updating deep properties (non-root)
-          if (util.isObject(value) || opts.insert || ((path2||'').match(/\./) && !opts.ignoreUndefined)) {
+          if (
+            util.isObject(value) ||
+            opts.insert ||
+            ((path2||'').match(/\./) && (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : true))
+          ) {
             var res = this._validateFields(dataRoot, field, value, opts, path2)
             errors.push(...res[0])
           }
@@ -258,17 +262,17 @@ module.exports = {
     let fieldName = path.match(/[^.]+$/)[0]
     let ruleMessageKey = this._getMostSpecificKeyMatchingPath(this.messages, path)
     let ruleMessage = ruleMessageKey && this.messages[ruleMessageKey][ruleName]
-    let ignoreUndefined = util.isDefined(opts.ignoreUndefined) ? opts.ignoreUndefined : rule.ignoreUndefined
+    let validateUndefined = util.isDefined(opts.validateUndefined) ? opts.validateUndefined : rule.validateUndefined
     if (!ruleMessage) ruleMessage = rule.message
 
     // Ignore undefined (if updated root property, or ignoring)
-    if ((ignoreUndefined || (opts.update && !path.match(/\./))) && typeof value === 'undefined') return
+    if ((!validateUndefined || (opts.update && !path.match(/\./))) && typeof value === 'undefined') return
 
     // Ignore null (if nullObject is set on objects or arrays) (todo: change to ignoreNull)
     if (field.nullObject && (field.isObject || field.isArray) && value === null) return
 
     // Ignore empty strings
-    if (rule.ignoreEmptyString && value === '') return
+    if (!rule.validateEmptyString && value === '') return
 
     // Rule failed
     if (!rule.fn.call(dataRoot, value, ruleArg, path, this)) return {

package/lib/model.js

@@ -61,7 +61,7 @@ let Model = module.exports = function(name, opts, manager) {
       // Update with formatted rule
       let formattedRule = util.isObject(rule)? rule : { fn: rule }
       if (!formattedRule.message) formattedRule.message = `Invalid data property for rule "${ruleName}".`
-      if (typeof formattedRule.ignoreUndefined == 'undefined') formattedRule.ignoreUndefined = true
+      if (typeof formattedRule.validateEmptyString == 'undefined') formattedRule.validateEmptyString = true
       this.rules[ruleName] = formattedRule
     }
   }, this)
@@ -242,11 +242,10 @@ Model.prototype._setupFieldsAndWhitelists = function(fields, path) {
   this.findBLProject = this.findBL.reduce((o, v) => { (o[v] = 0); return o }, {})
 },
 
-Model.prototype._setupIndexes = function(fields) {
+Model.prototype._setupIndexes = function(fields, opts={}) {
   /**
-   * Creates indexes for the model
+   * Creates indexes for the model (multikey, and sub-document supported)
    * Note: only one text index per model(collection) is allowed due to mongodb limitations
-   * Note: we and currently don't support indexes on sub-collections, but sub-documents yes!
    * @link https://docs.mongodb.com/manual/reference/command/createIndexes/
    * @link https://mongodb.github.io/node-mongodb-native/2.1/api/Collection.html#createIndexes
    * @param {object} <fields>
@@ -279,6 +278,7 @@ Model.prototype._setupIndexes = function(fields) {
   recurseFields(fields || model.fields, '')
   // console.log(2, indexes, fields)
   if (hasTextIndex) indexes.push(textIndex)
+  if (opts.dryRun) return Promise.resolve(indexes || [])
   if (!indexes.length) return Promise.resolve([]) // No indexes defined
 
   // Create indexes
@@ -331,22 +331,24 @@ Model.prototype._setupIndexes = function(fields) {
     util.forEach(fields, (field, name) => {
       let index = field.index
       if (index) {
-        let path = name == 'schema'? parentPath.slice(0, -1) : parentPath + name
         let options = util.isObject(index)? util.omit(index, ['type']) : {}
         let type = util.isObject(index)? index.type : index
+        let path = name == 'schema'? parentPath.slice(0, -1) : parentPath + name
+        let path2 = path.replace(/(^|\.)[0-9]+(\.|$)/g, '$2') // no numirical keys, e.g. pets.1.name
         if (type === true) type = 1
-
         if (type == 'text') {
-          hasTextIndex = textIndex.key[path] = 'text'
+          hasTextIndex = textIndex.key[path2] = 'text'
           Object.assign(textIndex, options)
         } else if (type == '1' || type == '-1' || type == '2dsphere') {
-          indexes.push({ name: `${path}_${type}`, key: { [path]: type }, ...options })
+          indexes.push({ name: `${path2}_${type}`, key: { [path2]: type }, ...options })
         } else if (type == 'unique') {
-          indexes.push({ name: `${path}_1`, key: { [path]: 1 }, unique: true, ...options  })
+          indexes.push({ name: `${path2}_1`, key: { [path2]: 1 }, unique: true, ...options  })
         }
       }
       if (util.isSubdocument(field)) {
         recurseFields(field, parentPath + name + '.')
+      } else if (util.isArray(field)) {
+        recurseFields(field, parentPath + name + '.')
       }
     })
   }

package/lib/rules.js

@@ -5,7 +5,8 @@ let validator = require('validator')
 module.exports = {
 
   required: {
-    ignoreUndefined: false,
+    validateUndefined: true,
+    validateEmptyString: true,
     message: 'This field is required.',
     fn: function(x) {
       if (util.isArray(x) && !x.length) return false
@@ -16,7 +17,7 @@ module.exports = {
   // Type rules below ignore undefined (default for custom model rules)
 
   'isBoolean': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not a boolean.',
     tryParse: function(x) {
       if (typeof x === 'string' && x === 'true') return true
@@ -28,7 +29,7 @@ module.exports = {
     }
   },
   'isArray': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not an array.',
     tryParse: function(x) {
       if (x === '') return null
@@ -41,7 +42,7 @@ module.exports = {
     }
   },
   'isDate': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not a unix timestamp.',
     tryParse: function(x) {
       if (util.isString(x) && x.match(/^[+-]?[0-9]+$/)) return x // keep string nums intact
@@ -53,7 +54,7 @@ module.exports = {
     }
   },
   'isImageObject': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Invalid image value',
     messageLong: 'Image fields need to either be null, undefined, file, or an object containing the following '
       + 'fields \'{ bucket, date, filename, filesize, path, uid }\'',
@@ -69,7 +70,7 @@ module.exports = {
     }
   },
   'isInteger': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not an integer.',
     tryParse: function(x) {
       if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return x // keep string nums intact
@@ -81,7 +82,7 @@ module.exports = {
     }
   },
   'isNumber': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not a number.',
     tryParse: function(x) {
       if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return x // keep string nums intact
@@ -93,7 +94,7 @@ module.exports = {
     }
   },
   'isObject': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not an object.',
     tryParse: function(x) {
       if (x === '') return null
@@ -106,21 +107,21 @@ module.exports = {
     }
   },
   'isString': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not a string.',
     fn: function(x) {
       return typeof x === 'string'
     }
   },
   'isAny': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: '',
     fn: function(x) {
       return true
     }
   },
   'isId': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was not a valid ObjectId.',
     tryParse: function(x) {
       // Try and parse value to a mongodb ObjectId
@@ -135,7 +136,7 @@ module.exports = {
     }
   },
   'max': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: (x, arg) => 'Value was greater than the configured maximum (' + arg + ')',
     fn: function(x, arg) {
       if (typeof x !== 'number') { throw new Error ('Value was not a number.') }
@@ -143,7 +144,7 @@ module.exports = {
     }
   },
   'min': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: (x, arg) => 'Value was less than the configured minimum (' + arg + ')',
     fn: function(x, arg) {
       if (typeof x !== 'number') { throw new Error ('Value was not a number.') }
@@ -151,19 +152,17 @@ module.exports = {
     }
   },
   'isNotEmptyString': {
-    ignoreUndefined: true,
+    validateEmptyString: true,
     message: 'Value was an empty string.',
     fn: function(x) {
       return x !== ''
     }
   },
 
-  // Rules below ignore undefined & empty strings
+  // Rules below ignore undefined, & empty strings
   // (e.g. an empty email field can be saved that isn't required)
 
   'enum': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Invalid enum value',
     fn: function(x, arg) {
       for (let item of arg) {
@@ -176,68 +175,46 @@ module.exports = {
   //   fn: function(x, arg) { return !x }
   // },
   'isAfter': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Value was before the configured time (' + arg + ')',
     fn: function(x, arg) { return validator.isAfter(x, arg) }
   },
   'isBefore': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Value was after the configured time (' + arg + ')',
     fn: function(x, arg) { return validator.isBefore(x, arg) }
   },
   'isCreditCard': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Value was not a valid credit card.',
     fn: function(x, arg) { return validator.isCreditCard(x, arg) }
   },
   'isEmail': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Please enter a valid email address.',
     fn: function(x, arg) { return validator.isEmail(x, arg) }
   },
   'isHexColor': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Value was not a valid hex color.',
     fn: function(x, arg) { return validator.isHexColor(x, arg) }
   },
   'isIn': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Value was not in the configured whitelist (' + arg.join(', ') + ')',
     fn: function(x, arg) { return validator.isIn(x, arg) }
   },
   'isIP': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Value was not a valid IP address.',
     fn: function(x, arg) { return validator.isIP(x, arg) }
   },
   'isNotIn': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Value was in the configured blacklist (' + arg.join(', ') + ')',
     fn: function(x, arg) { return !validator.isIn(x, arg) }
   },
   'isURL': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Value was not a valid URL.',
     fn: function(x, arg) { return validator.isURL(x, arg === true? undefined : arg) }
   },
   'isUUID': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: 'Value was not a valid UUID.',
     fn: function(x, arg) { return validator.isUUID(x) }
   },
   'minLength': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: function(x, arg) {
       if (typeof x === 'string') return 'Value needs to be at least ' + arg + ' characters long.'
       else return 'Value needs to contain a minimum of ' + arg + ' items.'
@@ -249,8 +226,6 @@ module.exports = {
     }
   },
   'maxLength': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: function(x, arg) {
       if (typeof x === 'string') return 'Value was longer than the configured maximum length (' + arg + ')'
       else return 'Value cannot contain more than ' + arg + ' items.'
@@ -262,8 +237,6 @@ module.exports = {
     }
   },
   'regex': {
-    ignoreEmptyString: true,
-    ignoreUndefined: true,
     message: (x, arg) => 'Value did not match the configured regular expression (' + arg + ')',
     fn: function(x, arg) {
       if (util.isRegex(arg)) return validator.matches(x, arg)

package/lib/util.js

@@ -300,7 +300,7 @@ module.exports = {
      * @param {boolean} ignoreEmptyArrays -  ignore empty arrays
      * @return obj
      */
-    let chunks = path.split('.') //  ['pets', '$', 'dog']
+    let chunks = path.split('.') //  ['pets', '$|0-9', 'dog']
     let target = obj
     for (let i=0, l=chunks.length; i<l; i++) {
       if (l === i+1) { // Last

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A straight forward MongoDB ODM built around Monk",
   "author": "Ricky Boyce",
-  "version": "1.29.0",
+  "version": "1.30.3",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",

package/plugins/images/index.js

@@ -54,8 +54,11 @@ let plugin = module.exports = {
     model.imageFields = plugin._findAndTransformImageFields(model.fields, '')
 
     if (model.imageFields.length) {
-      // Update image fields and whitelists with the new object schema
-      // model._setupFieldsAndWhitelists(model.fields)
+      // Todo?: Update image fields and whitelists with the new object schema
+      //   model._setupFieldsAndWhitelists(model.fields)
+      model.beforeValidate.push(function(data, n) {
+        plugin.keepImagePlacement(this, data).then(() => n(null, data)).catch(e => n(e))
+      })
       model.beforeUpdate.push(function(data, n) {
         plugin.removeImages(this, data).then(() => n(null, data)).catch(e => n(e))
       })
@@ -91,7 +94,9 @@ let plugin = module.exports = {
      * @param {object} options - monastery operation options {model, query, files, ..}
      * @param {object} data -
      * @param {boolean} test -
-     * @return promise
+     * @return promise(
+     *   {object} data - data object containing new S3 image-object
+     * ])
      * @this plugin
      */
     let { model, query, files } = options
@@ -167,7 +172,33 @@ let plugin = module.exports = {
     })
   },
 
-  removeImages: function(options, data, test) {
+  keepImagePlacement: async function(options, data) {
+    /**
+     * Hook before update/remove
+     * Since monastery removes undefined array items on validate, we need to convert any
+     * undefined array items to null where files are located to maintain image ordering
+     * Todo: maybe dont remove undefined array items in general
+     *
+     * E.g.
+     * req.body  = 'photos[0]' : undefined || non existing (set to null)
+     * req.files = 'photos[0]' : { ...binary }
+     *
+     * @param {object} options - monastery operation options {query, model, files, multi, ..}
+     * @return promise
+     * @this plugin
+     */
+    if (typeof options.files == 'undefined') return
+    // Check upload errors and find valid uploaded images
+    let files = await plugin._findValidImages(options.files || {}, options.model)
+    // Set undefined primative-array items to null where files are located
+    for (let filesArray of files) {
+      if (filesArray.inputPath.match(/\.[0-9]+$/)) {
+        util.setDeepValue(data, filesArray.inputPath, null, true, false, true)
+      }
+    }
+  },
+
+  removeImages: async function(options, data, test) {
     /**
      * Hook before update/remove
      * Removes images not found in data, this means you will need to pass the image objects to every update operation
@@ -179,108 +210,113 @@ let plugin = module.exports = {
      *
      * @ref https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#deleteObjects-property
      * @param {object} options - monastery operation options {query, model, files, multi, ..}
-     * @return promise
+     * @return promise([
+     *   {object} useCount - images that wont be removed, e.g. { lion1: 1 }
+     *   {array} unused - S3 image uris to be removed,  e.g. [{ Key: 'small/lion1.jpg' }, ..]
+     * ])
      * @this plugin
      */
     let pre
     let preExistingImages = []
     let useCount = {}
-    if (typeof options.files == 'undefined') return new Promise(res => res())
+    if (typeof options.files == 'undefined') return
 
     // Find all documents from the same query
-    return options.model._find(options.query, options)
-      .then(docs => {
-        // Find all pre-existing image objects in documents
-        for (let doc of util.toArray(docs)) { //x2
-          for (let imageField of options.model.imageFields) { //x5
-            let images = plugin._findImagesInData(doc, imageField, 0, '').filter(o => o.image)
-            for (let image of images) {
-              preExistingImages.push(image)
-              useCount[image.image.uid] = (useCount[image.image.uid] || 0) + 1
-            }
-          }
+    let docs = await options.model._find(options.query, options)
+
+    // Find all pre-existing image objects in documents
+    for (let doc of util.toArray(docs)) { //x2
+      for (let imageField of options.model.imageFields) { //x5
+        let images = plugin._findImagesInData(doc, imageField, 0, '').filter(o => o.image)
+        for (let image of images) {
+          preExistingImages.push(image)
+          useCount[image.image.uid] = (useCount[image.image.uid] || 0) + 1
         }
+      }
+    }
 
-        // console.log(1, useCount, preExistingImages)
+    // console.log(1, useCount, preExistingImages)
 
-        // Assign pre-existing images within undefined deep objects and missing array items to null
-        // ignore undefined root images
-        let dataFilled = util.deepCopy(data)
-        for (let key in dataFilled) {
-          for (let pre of preExistingImages) {
-            if (!pre.dataPath.match(new RegExp('^' + key + '(\\.|$)'))) continue
-            util.setDeepValue(dataFilled, pre.dataPath, null, true)
-          }
-        }
+    // Assign pre-existing images within undefined deep objects and missing array items to null,
+    // ignore undefined root images
+    let dataFilled = util.deepCopy(data)
+    for (let key in dataFilled) {
+      for (let pre of preExistingImages) {
+        if (!pre.dataPath.match(new RegExp('^' + key + '(\\.|$)'))) continue
+        util.setDeepValue(dataFilled, pre.dataPath, null, true)
+      }
+    }
+    // console.log(dataFilled)
 
-        // Loop all schema image fields
-        for (let imageField of options.model.imageFields) { //x5
-          let images = plugin._findImagesInData(dataFilled, imageField, 0, '')
-          if (!images.length) continue
-          // console.log(images)
+    // Check upload errors and find valid uploaded images
+    let files = await plugin._findValidImages(options.files || {}, options.model)
 
-          // Data contains null images that once had a pre-existing image
-          for (let image of images) {
-            if (image.image == null && (pre = preExistingImages.find(o => o.dataPath == image.dataPath))) {
-              useCount[pre.image.uid]--
-            }
-          }
+    // Loop all schema image fields
+    for (let imageField of options.model.imageFields) { //x5
+      let images = plugin._findImagesInData(dataFilled, imageField, 0, '')
+      if (!images.length) continue
+      // console.log(images)
 
-          // Data contains valid (pre-existing) image-objects? And are we overriding a pre-existing image?
-          for (let image of images) {
-            if (image.image != null) {
-              let pre = preExistingImages.find(o => o.dataPath == image.dataPath)
-              if (typeof useCount[image.image.uid] == 'undefined') {
-                throw `The passed image object for '${image.dataPath}' does not match any pre-existing
-                  images saved on this document.`
-              } else if (pre && pre.image.uid != image.image.uid) {
-                useCount[pre.image.uid]--
-                useCount[image.image.uid]++
-              } else if (!pre) {
-                useCount[image.image.uid]++
-              }
-            }
-          }
+      // Data contains null images that once had a pre-existing image
+      for (let image of images) {
+        if (image.image == null && (pre = preExistingImages.find(o => o.dataPath == image.dataPath))) {
+          useCount[pre.image.uid]--
         }
-        // Check upload errors and find valid uploaded images. If any file is overriding a
-        // pre-existing image, push to unused
-        return plugin._findValidImages(options.files || {}, options.model).then(files => {
+      }
+
+      // Loop images found in the data
+      for (let image of images) {
+        if (image.image != null) {
+          let preExistingImage = preExistingImages.find(o => o.dataPath == image.dataPath)
+          // valid image-object?
+          if (typeof useCount[image.image.uid] == 'undefined') {
+            throw `The passed image object for '${image.dataPath}' does not match any pre-existing
+              images saved on this document.`
+          // Different image from prexisting image
+          } else if (preExistingImage && preExistingImage.image.uid != image.image.uid) {
+            useCount[preExistingImage.image.uid]--
+            useCount[image.image.uid]++
+          // No pre-existing image found
+          } else if (!preExistingImage) {
+            useCount[image.image.uid]++
+          }
+          // Any file overriding this image?
           for (let filesArray of files) {
-            if ((pre = preExistingImages.find(o => o.dataPath == filesArray.inputPath))) {
-              useCount[pre.image.uid]--
+            if (image.dataPath == filesArray.inputPath) {
+              useCount[image.image.uid]--
             }
           }
-        })
-
-      }).then(() => {
-        // Retrieve all the unused files
-        let unused = []
-        // console.log(3, useCount)
-        for (let key in useCount) {
-          if (useCount[key] > 0) continue
-          let pre = preExistingImages.find(o => o.image.uid == key)
-          unused.push(
-            // original key can have a different extension
-            { Key: pre.image.path },
-            { Key: `small/${key}.jpg` },
-            { Key: `medium/${key}.jpg` },
-            { Key: `large/${key}.jpg` }
-          )
-          this.manager.info(
-            `Removing '${pre.image.filename}' from '${pre.image.bucket}/${pre.image.path}'`
-          )
         }
-        if (test) return Promise.resolve([useCount, unused])
-        // Delete any unused images from s3. If the image is on a different bucket
-        // the file doesnt get deleted, we only delete from plugin.awsBucket.
-        if (!unused.length) return
-        return new Promise((resolve, reject) => {
-          plugin.s3.deleteObjects({ Bucket: plugin.awsBucket, Delete: { Objects: unused }}, (err, data) => {
-            if (err) reject(err)
-            resolve()
-          })
-        })
+      }
+    }
+
+    // Retrieve all the unused files
+    // console.log(3, useCount)
+    let unused = []
+    for (let key in useCount) {
+      if (useCount[key] > 0) continue
+      let pre = preExistingImages.find(o => o.image.uid == key)
+      unused.push(
+        // original key can have a different extension
+        { Key: pre.image.path },
+        { Key: `small/${key}.jpg` },
+        { Key: `medium/${key}.jpg` },
+        { Key: `large/${key}.jpg` }
+      )
+      this.manager.info(
+        `Removing '${pre.image.filename}' from '${pre.image.bucket}/${pre.image.path}'`
+      )
+    }
+    if (test) return [useCount, unused]
+    // Delete any unused images from s3. If the image is on a different bucket
+    // the file doesnt get deleted, we only delete from plugin.awsBucket.
+    if (!unused.length) return
+    await new Promise((resolve, reject) => {
+      plugin.s3.deleteObjects({ Bucket: plugin.awsBucket, Delete: { Objects: unused }}, (err, data) => {
+        if (err) reject(err)
+        resolve()
       })
+    })
   },
 
   _addImageObjectsToData: function(path, data, image) {

package/test/model.js

@@ -116,10 +116,9 @@ module.exports = function(monastery, opendb) {
   })
 
   test('Model indexes', async () => {
-    // Need to test different types of indexes
+    // Setup: Need to test different types of indexes
     let db = (await opendb(null)).db
-
-    // Drop previously tested collections
+    // Setup: Drop previously tested collections
     if ((await db._db.listCollections().toArray()).find(o => o.name == 'userIndexRaw')) {
       await db._db.collection('userIndexRaw').drop()
     }
@@ -190,6 +189,96 @@ module.exports = function(monastery, opendb) {
     db.close()
   })
 
+  test('Model subdocument indexes', async () => {
+    // Setup: Need to test different types of indexes
+    let db = (await opendb(null)).db
+    // Setup: Drop previously tested collections
+    if ((await db._db.listCollections().toArray()).find(o => o.name == 'userIndexSubdoc')) {
+      await db._db.collection('userIndexSubdoc').drop()
+    }
+    // Run
+    let userModel = await db.model('userIndexSubdoc', {
+      fields: {}
+    })
+    await expect(userModel._setupIndexes(
+      {
+        animals: {
+          name: { type: 'string', index: 'unique' },
+        },
+        animals2: {
+          names: {
+            name: { type: 'string', index: 'unique' },
+          },
+        },
+        animals3: {
+          names: {
+            name: { type: 'string', index: 'text' },
+          },
+        },
+      }, {
+        dryRun: true
+      }
+    )).resolves.toEqual([{
+      'key': { 'animals.name': 1 },
+      'name': 'animals.name_1',
+      'unique': true,
+    }, {
+      'key': { 'animals2.names.name': 1 },
+      'name': 'animals2.names.name_1',
+      'unique': true,
+    }, {
+      'key': { 'animals3.names.name': 'text' },
+      'name': 'text',
+    }])
+
+    db.close()
+  })
+
+  test('Model array indexes', async () => {
+    // Setup: Need to test different types of indexes
+    let db = (await opendb(null)).db
+    // Setup: Drop previously tested collections
+    if ((await db._db.listCollections().toArray()).find(o => o.name == 'userIndexArray')) {
+      await db._db.collection('userIndexArray').drop()
+    }
+    // Run
+    let userModel = await db.model('userIndexArray', {
+      fields: {}
+    })
+    await expect(userModel._setupIndexes(
+      {
+        animals: [{
+          name: { type: 'string', index: 'unique' },
+        }],
+        animals2: [{ type: 'string', index: true }],
+        animals3: [[{ type: 'string', index: true }]],
+        animals4: [{
+          names: [{
+            name: { type: 'string', index: 'unique' },
+          }],
+        }],
+      }, {
+        dryRun: true
+      }
+    )).resolves.toEqual([{
+      'key': { 'animals.name': 1 },
+      'name': 'animals.name_1',
+      'unique': true,
+    }, {
+      'key': { 'animals2': 1 },
+      'name': 'animals2_1',
+    }, {
+      'key': { 'animals3.0': 1 },
+      'name': 'animals3.0_1',
+    }, {
+      'key': { 'animals4.names.name': 1 },
+      'name': 'animals4.names.name_1',
+      'unique': true,
+    }])
+
+    db.close()
+  })
+
   test('Model 2dsphere indexes', async () => {
     // Setup. The tested model needs to be unique as race condition issue arises when the same model
     // with text indexes are setup at the same time

package/test/plugin-images.js

@@ -1,3 +1,5 @@
+let util = require('../lib/util')
+
 module.exports = function(monastery, opendb) {
 
   // Data no images doesn't throw error
@@ -299,13 +301,8 @@ module.exports = function(monastery, opendb) {
 
         plugin.removeImages(options, req.body, true)
           .then(res => {
-            expect(res[0]).toEqual({ test1: 0, test2: -1, test3: 1, test4: 0 })
+            expect(res[0]).toEqual({ test1: 1, test2: 0, test3: 1, test4: 0 })
             expect(res[1]).toEqual([
-              { Key: 'dir/test1.png' },
-              { Key: 'small/test1.jpg' },
-              { Key: 'medium/test1.jpg' },
-              { Key: 'large/test1.jpg' },
-
               { Key: 'dir/test2.png' },
               { Key: 'small/test2.jpg' },
               { Key: 'medium/test2.jpg' },
@@ -560,4 +557,154 @@ module.exports = function(monastery, opendb) {
     })()
   })
 
+  test('images reorder', async () => {
+    let db = (await opendb(null, {
+      timestamps: false,
+      serverSelectionTimeoutMS: 2000,
+      imagePlugin: { awsBucket: 'fake', awsAccessKeyId: 'fake', awsSecretAccessKey: 'fake' }
+    })).db
+
+    let user = db.model('user', { fields: {
+      logos: [{ type: 'image' }],
+    }})
+
+    let image = {
+      bucket: 'test',
+      date: 1234,
+      filename: 'lion1.png',
+      filesize: 1234,
+      path: 'test/lion1.png',
+      uid: 'lion1'
+    }
+
+    let user1 = await db.user._insert({
+      logos: [image],
+    })
+
+    let plugin = db.imagePluginFile
+    let supertest = require('supertest')
+    let express = require('express')
+    let upload = require('express-fileupload')
+    let app = express()
+    app.use(upload())
+
+    // Reorder
+    app.post('/', async function(req, res) {
+      try {
+        req.body.logos = JSON.parse(req.body.logos)
+        let options = { files: req.files, model: user, query: { _id: user1._id } }
+        let response = await plugin.removeImages(options, req.body, true)
+        expect(response[0]).toEqual({ lion1: 1 })
+        expect(response[1]).toEqual([])
+        res.send()
+      } catch (e) {
+        console.log(e.message || e)
+        res.status(500).send()
+      }
+    })
+    await supertest(app)
+      .post('/')
+      .field('logos', JSON.stringify([ null, image ]))
+      .expect(200)
+
+    db.close()
+  })
+
+  test('images reorder and added image', async () => {
+    // latest (2022.02)
+    let db = (await opendb(null, {
+      timestamps: false,
+      serverSelectionTimeoutMS: 2000,
+      imagePlugin: { awsBucket: 'fake', awsAccessKeyId: 'fake', awsSecretAccessKey: 'fake' }
+    })).db
+
+    let user = db.model('user', { fields: {
+      photos: [{ type: 'image' }],
+    }})
+
+    let image = {
+      bucket: 'test',
+      date: 1234,
+      filename: 'lion1.png',
+      filesize: 1234,
+      path: 'test/lion1.png',
+      uid: 'lion1'
+    }
+
+    let user1 = await db.user._insert({
+      photos: [image],
+    })
+
+    let plugin = db.imagePluginFile
+    let supertest = require('supertest')
+    let express = require('express')
+    let upload = require('express-fileupload')
+    let app = express()
+    app.use(upload())
+
+    app.post('/', async function(req, res) {
+      try {
+        // Parse and validate data which is used before in update/insert
+        let options = { files: req.files, model: user, query: { _id: user1._id } }
+        let data = await util.parseData(req.body)
+        data = await user.validate(data, { ...options, update: true })
+
+        // Empty photo placeholder not removed in validate?
+        expect(data.photos[0]).toEqual(null)
+        expect(data.photos[1]).toEqual(image)
+
+        // Remove images
+        let response = await plugin.removeImages(options, data, true)
+        expect(response[0]).toEqual({ lion1: 1 }) // useCount
+        expect(response[1]).toEqual([]) // unused
+
+        // New file exists
+        let validFiles = await plugin._findValidImages(req.files, user)
+        expect(((validFiles||[])[0]||{}).inputPath).toEqual('photos.0') // Valid inputPath
+
+        // Add images
+        response = await plugin.addImages(options, data, true)
+        expect(response[0]).toEqual({
+          photos: [{
+            bucket: 'fake',
+            date: expect.any(Number),
+            filename: 'lion2.jpg',
+            filesize: expect.any(Number),
+            path: expect.any(String),
+            uid: expect.any(String)
+          }, {
+            bucket: 'test', // still the same image-object reference (nothing new)
+            date: expect.any(Number),
+            filename: 'lion1.png',
+            filesize: expect.any(Number),
+            path: expect.any(String),
+            uid: expect.any(String)
+          },],
+        })
+
+        res.send()
+      } catch (e) {
+        console.log(e.message || e)
+        res.status(500).send()
+      }
+    })
+
+    await supertest(app)
+      .post('/')
+      // Mock multipart/form-data syntax which is not supported by supertest (formdata sent with axios)
+      //  E.g.
+      //    req.body  = 'photos[1][bucket]' : '...'
+      //    req.files = 'photos[0]' : { ...binary }
+      .field('photos[1][bucket]', image.bucket)
+      .field('photos[1][date]', image.date)
+      .field('photos[1][filename]', image.filename)
+      .field('photos[1][filesize]', image.filesize)
+      .field('photos[1][path]', image.path)
+      .field('photos[1][uid]', image.uid)
+      .attach('photos[0]', `${__dirname}/assets/lion2.jpg`)
+      .expect(200)
+
+    db.close()
+  })
+
 }

package/test/validate.js

@@ -19,7 +19,7 @@ module.exports = function(monastery, opendb) {
       detail: 'This field is required.',
       meta: { rule: 'required', model: 'user', field: 'name' }
     })
-    await expect(user.validate({ name : '' }, { ignoreUndefined: true })).rejects.toContainEqual({
+    await expect(user.validate({ name : '' }, { validateUndefined: false })).rejects.toContainEqual({
       status: '400',
       title: 'name',
       detail: 'This field is required.',
@@ -27,8 +27,8 @@ module.exports = function(monastery, opendb) {
     })
 
     // Required error (insert, and with ignoreRequired)
-    await expect(user.validate({}, { ignoreUndefined: true })).resolves.toEqual({})
-    await expect(user.validate({}, { ignoreUndefined: true, update: true })).resolves.toEqual({})
+    await expect(user.validate({}, { validateUndefined: false })).resolves.toEqual({})
+    await expect(user.validate({}, { validateUndefined: false, update: true })).resolves.toEqual({})
 
     // No required error (update)
     await expect(user.validate({}, { update: true })).resolves.toEqual({})
@@ -87,77 +87,120 @@ module.exports = function(monastery, opendb) {
     let db = (await opendb(false)).db
     let user = db.model('user', { fields: {
       animals: {
-        cat: { type: 'string', required: true },
+        cat: { type: 'string', required: true }, // {} = required on insert
         dog: {
           name:  { type: 'string' },
-          color: { type: 'string', required: true }
+          color: { type: 'string', required: true } // {} = required on insert
         }
       }
     }})
 
-    // Invalid subdocument type
-    await expect(user.validate({ animals: { dog: 1 }})).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.dog',
-      detail: 'Value was not an object.',
-      meta: { rule: 'isObject', model: 'user', field: 'dog' }
-    })
+    // Insert: Required subdocument properties
+    await expect(user.validate({})).rejects.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.cat',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'cat' }
+        }),
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.dog.color',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'color' }
+        }),
+      ])
+    )
 
-    // Required subdocument property (required on insert)
-    await expect(user.validate({})).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.cat',
-      detail: 'This field is required.',
-      meta: { rule: 'required', model: 'user', field: 'cat' }
-    })
+    // Insert: Required subdocument properties
+    await expect(user.validate({ animals: {} })).rejects.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.cat',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'cat' }
+        }),
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.dog.color',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'color' }
+        }),
+      ])
+    )
 
-    // Required subdocument property (required on insert)
-    await expect(user.validate({})).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.dog.color',
-      detail: 'This field is required.',
-      meta: { rule: 'required', model: 'user', field: 'color' }
-    })
+    // Insert: Invalid subdocument type
+    await expect(user.validate({ animals: { dog: 1 }})).rejects.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.cat',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'cat' }
+        }),
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.dog',
+          detail: 'Value was not an object.',
+          meta: { rule: 'isObject', model: 'user', field: 'dog' }
+        }),
+      ])
+    )
 
-    // Required subdocument property (required on update when a parent is specified)
-    await expect(user.validate({ animals: {} }, { update: true })).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.cat',
-      detail: 'This field is required.',
-      meta: { rule: 'required', model: 'user', field: 'cat' }
+    // Insert: Ignore required subdocument property with a defined parent
+    await expect(user.validate({ animals: {} }, { validateUndefined: false })).resolves.toEqual({
+      animals: {}
     })
 
-    // Required subdocument property (required on update when a parent is specified)
-    await expect(user.validate({ animals: { dog: {}} }, { update: true })).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.dog.color',
-      detail: 'This field is required.',
-      meta: { rule: 'required', model: 'user', field: 'color' }
-    })
+    // Update: Required subdocument property when a parent/grandparent is specified
+    await expect(user.validate({ animals: {} }, { update: true })).rejects.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.cat',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'cat' }
+        }),
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.dog.color',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'color' }
+        }),
+      ])
+    )
 
-    // Required subdocument property (required on update when a grand-parent is specified)
-    await expect(user.validate({ animals: {} }, { update: true })).rejects.toContainEqual({
-      status: '400',
-      title: 'animals.dog.color',
-      detail: 'This field is required.',
-      meta: { rule: 'required', model: 'user', field: 'color' }
-    })
+    // Update: Required subdocument property when a parent is specified
+    await expect(user.validate({ animals: { dog: {}} }, { update: true })).rejects.toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.cat',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'cat' }
+        }),
+        expect.objectContaining({
+          status: '400',
+          title: 'animals.dog.color',
+          detail: 'This field is required.',
+          meta: { rule: 'required', model: 'user', field: 'color' }
+        }),
+      ])
+    )
 
-    // Ignore required subdocument property (not required on update)
+    // Update: Ignore required subdocument property when root parent is undefined
     await expect(user.validate({}, { update: true })).resolves.toEqual({})
 
-    // Ignore required subdocument property with a defined parent (update) (not required if ignoreUndefined set)
-    await expect(user.validate({ animals: {} }, { ignoreUndefined: true })).resolves.toEqual({
-      animals: {}
-    })
 
-    // Ignore required subdocument property with a defined parent (not required if ignoreUndefined set)
-    await expect(user.validate({ animals: {} }, { update: true, ignoreUndefined: true })).resolves.toEqual({
+    // Update: Ignore required subdocument property with a defined parent when validateUndefined = false
+    await expect(user.validate({ animals: {} }, { update: true, validateUndefined: false })).resolves.toEqual({
       animals: {}
     })
 
-    // Required subdocument property (defined with ignoreUndefined)
-    await expect(user.validate({ animals: { cat: '' }}, { update: true, ignoreUndefined: true }))
+    // Update: Required defined subdocument property when validateUndefined = false
+    await expect(user.validate({ animals: { cat: '' }}, { update: true, validateUndefined: false }))
     .rejects.toContainEqual({
       status: '400',
       title: 'animals.cat',
@@ -215,17 +258,17 @@ module.exports = function(monastery, opendb) {
     await expect(user.validate({ animals: { dogs: [] }}))
       .resolves.toEqual({ animals: { dogs: [] }})
 
-    // No undefined item errors with ignoreUndefined
+    // No undefined item errors with validateUndefined=false
     await expect(user.validate(
       { animals: { dogs: [{ name: 'sparky' }] }},
-      { update: true, ignoreUndefined: true }
+      { update: true, validateUndefined: false }
     ))
       .resolves.toEqual({ animals: { dogs: [{ name: 'sparky' }] }})
 
-    // Requried error within an array subdocument (even during update when parent defined && ignoreUndefined = true)
+    // Requried error within an array subdocument (even during update when parent defined && validateUndefined = false)
     await expect(user.validate(
       { animals: { dogs: [{ name: 'sparky', color: '' }] }},
-      { update: true, ignoreUndefined: true }
+      { update: true, validateUndefined: false }
     ))
       .rejects.toContainEqual(error)
   })
@@ -497,7 +540,7 @@ module.exports = function(monastery, opendb) {
       },
       rules: {
         requiredIfNoName: {
-          ignoreUndefined: false,
+          validateUndefined: true,
           fn: function(value, ruleArg) {
             return value || this.name
           }
@@ -547,7 +590,7 @@ module.exports = function(monastery, opendb) {
         'title': 'age'
       }
     ])
-    await expect(user2.validate({  }, { ignoreUndefined: true })).resolves.toEqual({})
+    await expect(user2.validate({  }, { validateUndefined: false })).resolves.toEqual({})
 
     // Required rule based off another field (update)
     await expect(user2.validate({ }, { update: true })).resolves.toEqual({})