monastery 2.1.1 → 2.2.1

package/changelog.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [2.2.1](https://github.com/boycce/monastery/compare/2.2.0...2.2.1) (2024-04-07)
+
+## [2.2.0](https://github.com/boycce/monastery/compare/2.1.1...2.2.0) (2024-04-07)
+
 ### [2.1.1](https://github.com/boycce/monastery/compare/2.1.0...2.1.1) (2024-04-05)
 
 ## [2.1.0](https://github.com/boycce/monastery/compare/1.42.2...2.1.0) (2024-04-05)

package/docs/readme.md

@@ -89,7 +89,6 @@ Coming soon...
 - ~~Add FindOneAndUpdate~~
 - ~~Add beforeInsertUpdate / afterInsertUpdate~~
 - Bug: Setting an object literal on an ID field ('model') saves successfully
-- Population within array items
 - ~~Blacklist false removes all blacklisting~~
 - ~~Add project to insert/update/validate~~
 - ~~Whitelisting a parent will remove any previously blacklisted children~~
@@ -97,6 +96,8 @@ Coming soon...
 - Automatic embedded document ids/createdAt/updatedAt fields
 - ~~Ability to change ACL default on the manager~~
 - ~~Public db.arrayWithSchema method~~
+- ~~Added support for array population~~
+- Change population warnings into errors
 - Global after/before hooks
 - before hooks can receive a data array, remove this
 - docs: Make the implicit ID query conversion more apparent
@@ -106,6 +107,9 @@ Coming soon...
 - double check await db.model.remove({ query: idfromparam }) doesnt cause issues for null, undefined or '', but continue to allow {}
 - ~~can't insert/update model id (maybe we can allow this and add _id to default insert/update blacklists)~~
 - timstamps are blacklisted by default (instead of the `timestamps` opt), and can be switched off via blacklisting
+- Allow rules on image types, e.g. `required`
+- test importing of models
+- Docs: model.methods
 
 ## Versions
 

package/lib/model-crud.js

@@ -89,13 +89,37 @@ module.exports = {
           if (util.isObject(item)) {
             lookups.push({ $lookup: item })
           } else {
-            let modelName = (path.split('.').reduce((o,i) => o[i], this.fields) ||{}).model
+            let arrayTarget
+            let arrayCount = 0
+            let schema = path.split('.').reduce((o, i) => {
+              if (util.isArray(o[i])) {
+                arrayCount++
+                arrayTarget = true
+                return o[i][0]
+              } else {
+                arrayTarget = false
+                return o[i]
+              }
+            }, this.fields)
+            let modelName = (schema||{}).model
             if (!modelName) {
               this.error(
                 `The field "${path}" passed to populate is not of type model. You would ` +
                 'need to add the field option e.g. { model: \'comment\' } in your schema.'
               )
               continue
+            } else if (arrayCount > 1) {
+              this.error(
+                `You cannot populate on array's nested in array's: ${path}: ` +
+                `{ model: "${modelName}" }`
+              )
+              continue
+            } else if (arrayCount == 1 && !arrayTarget) {
+              this.error(
+                `You cannot populate within an array of sub-documents: ${path}: ` +
+                `{ model: "${modelName}" }`
+              )
+              continue
             } else if (!this.manager.model[modelName]) {
               this.error(
                 `The field's model defined in your schema does not exist: ${path}: ` +
@@ -103,8 +127,11 @@ module.exports = {
               )
               continue
             }
-            // Populate model (convert array into document & create lookup)
-            (opts.addFields = opts.addFields || {})[path] = { '$arrayElemAt': [ '$' + path, 0 ] }
+            // Convert array into a document for non-array targets
+            if (!arrayTarget) {
+              (opts.addFields = opts.addFields || {})[path] = { '$arrayElemAt': [ '$' + path, 0 ] }
+            }
+            // Create lookup
             lookups.push({ $lookup: {
               from: modelName,
               localField: path,
@@ -482,6 +509,36 @@ module.exports = {
     return opts
   },
 
+  _pathBlacklisted: function(path, projection, matchDeepWhitelistedKeys=true) {
+    /**
+     * Checks if the path is blacklisted within a inclusion/exclusion projection
+     * @param {string} path - path without array brackets e.g. '.[]'
+     * @param {object} projection - inclusion/exclusion projection, not mixed
+     * @param {boolean} matchDeepWhitelistedKeys - match deep whitelisted keys containing path
+     *     E.g. pets.color == pets.color.age
+     * @return {boolean}
+     */
+    for (let key in projection) {
+      if (projection[key]) {
+        // Inclusion (whitelisted)
+        // E.g. pets.color.age == pets.color.age  (exact match)
+        // E.g. pets.color.age == pets.color      (path contains key)
+        var inclusion = true
+        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return false
+        if (matchDeepWhitelistedKeys) {
+          // E.g. pets.color   == pets.color.age  (key contains path)
+          if (key.match(new RegExp('^' + path.replace(/\./g, '\\.') + '\\.'))) return false
+        }
+      } else {
+        // Exclusion (blacklisted)
+        // E.g. pets.color.age == pets.color.age  (exact match)
+        // E.g. pets.color.age == pets.color      (path contains key)
+        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
+      }
+    }
+    return inclusion? true : false
+  },
+
   _processAfterFind: function(data, projection={}, afterFindContext={}) {
     /**
      * Todo: Maybe make this method public?
@@ -490,7 +547,7 @@ module.exports = {
      * e.g. "nurses": [{ model: 'user' }]
      *
      * @param {object|array|null} data
-     * @param {object} projection - $project object
+     * @param {object} projection - opts.projection (== opts.blacklist is merged with all found deep model blacklists)
      * @param {object} afterFindContext - handy context object given to schema.afterFind
      * @return Promise(data)
      * @this model
@@ -499,25 +556,31 @@ module.exports = {
     // want to manipulate any populated models
     let callbackSeries = []
     let model = this.manager.model
-    let parentModelData = util.toArray(data).map(o => ({ modelName: this.name, dataRef: o }))
-    let modelData = this._recurseAndFindModels(this.fields, data).concat(parentModelData)
+    let parent = util.toArray(data).map(o => ({ dataRef: o, dataPath: '', dataFieldName: '', modelName: this.name }))
+    let modelFields = this._recurseAndFindModels('', this.fields, data).concat(parent)
 
     // Loop found model/deep-model data objects, and populate missing default-fields and call afterFind on each
-    for (let item of modelData) {
+    for (let item of modelFields) {
       // Populate missing default fields if data !== null
       // NOTE: maybe only call functions if default is being set.. fine for now
       if (item.dataRef) {
-        util.forEach(model[item.modelName].fieldsFlattened, (schema, path) => {
-          if (!util.isDefined(schema.default) || path.match(/^\.?(createdAt|updatedAt)$/)) return
-          let parentPath = item.fieldName? item.fieldName + '.' : ''
-          let pathWithoutArrays = (parentPath + path).replace(/\.0(\.|$)/, '$1')
+        for (const localSchemaFieldPath in model[item.modelName].fieldsFlattened) {
+          const schema = model[item.modelName].fieldsFlattened[localSchemaFieldPath]
+          if (!util.isDefined(schema.default) || localSchemaFieldPath.match(/^\.?(createdAt|updatedAt)$/)) continue
+
+          // const parentPath = item.dataFieldName ? item.dataFieldName + '.' : ''
+          const fullPathWithoutArrays = [item.dataPath, localSchemaFieldPath]
+            .filter(o => o)
+            .join('.')
+            .replace(/\.[0-9]+(\.|$)/g, '$1')
+
           // Ignore default fields that are blacklisted
-          if (this._pathBlacklisted(pathWithoutArrays, projection)) return
-          // console.log(pathWithoutArrays, path, projection)
-          // Set value
-          let value = util.isFunction(schema.default)? schema.default(this.manager) : schema.default
-          util.setDeepValue(item.dataRef, path.replace(/\.0(\.|$)/g, '.$$$1'), value, true, false, true)
-        })
+          if (this._pathBlacklisted(fullPathWithoutArrays, projection)) continue
+
+          // Set default value
+          const value = util.isFunction(schema.default) ? schema.default(this.manager) : schema.default
+          util.setDeepValue(item.dataRef, localSchemaFieldPath.replace(/\.0(\.|$)/g, '.$$$1'), value, true, false, true)
+        }
       }
       // Collect all of the model's afterFind hooks
       for (let fn of model[item.modelName].afterFind) {
@@ -527,94 +590,56 @@ module.exports = {
     return util.runSeries(callbackSeries).then(() => data)
   },
 
-  _pathBlacklisted: function(path, projection, matchDeepWhitelistedKeys=true) {
+  _recurseAndFindModels: function(parentPath, schemaFields, dataArr) {
     /**
-     * Checks if the path is blacklisted within a inclusion/exclusion projection
-     * @param {string} path - path without array brackets e.g. '.[]'
-     * @param {object} projection - inclusion/exclusion projection, not mixed
-     * @param {boolean} matchDeepWhitelistedKeys - match deep whitelisted keys containing path
-     *     E.g. pets.color == pets.color.age
-     * @return {boolean}
-     */
-    for (let key in projection) {
-      if (projection[key]) {
-        // Inclusion (whitelisted)
-        // E.g. pets.color.age == pets.color.age  (exact match)
-        // E.g. pets.color.age == pets.color      (path contains key)
-        var inclusion = true
-        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return false
-        if (matchDeepWhitelistedKeys) {
-          // E.g. pets.color   == pets.color.age  (key contains path)
-          if (key.match(new RegExp('^' + path.replace(/\./g, '\\.') + '\\.'))) return false
-        }
-      } else {
-        // Exclusion (blacklisted)
-        // E.g. pets.color.age == pets.color.age  (exact match)
-        // E.g. pets.color.age == pets.color      (path contains key)
-        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
-      }
-    }
-    return inclusion? true : false
-  },
-
-  _recurseAndFindModels: function(fields, dataArr) {
-    /**
-     * Returns a flattened list of data-objects that are models
-     * @param {object} fields
+     * Returns a flattened list of models fields
+     * @param {string} parentPath
+     * @param {object} schemaFields - schema fields object
      * @param {object|array} dataArr
      * @return [{
      *   dataRef: { *fields here* },
-     *   fieldName: usersNewCompany,
+     *   dataPath: 'usersNewCompany',
+     *   dataFieldName: usersNewCompany,
      *   modelName: company
      * },..]
      */
     let out = []
     for (let data of util.toArray(dataArr)) {
-      util.forEach(fields, (field, fieldName) => {
-        if (!data) return
-
-        // Valid model object field.
-        if (
-            ((util.isArray(field) && field[0].model) || field.model) &&
-            data[fieldName] &&
-            util.isObjectAndNotID(data[fieldName])
-          ) {
-          // Note that sometimes a single model is passed instead of an array of models via a custom populate $lookup
+      util.forEach(schemaFields, (field, fieldName) => {
+        if (!data || !data[fieldName]) return
+        const newParentPath = parentPath ? `${parentPath}.${fieldName}` : fieldName 
+        // console.log(11, newParentPath, fieldName, field)
+
+        // Recurse through sub-document fields
+        if (util.isObjectAndNotID(data[fieldName]) && util.isSubdocument(field)) {
+          out = [...out, ...this._recurseAndFindModels(newParentPath, field, data[fieldName])]
+
+        // Recurse through array of sub-documents
+        } else if (util.isArray(data[fieldName]) && util.isSubdocument((field||[])[0])) {
+          for (let i=0, l=data[fieldName].length; i<l; i++) {
+            out = [...out, ...this._recurseAndFindModels(newParentPath + '.' + i, field[0], data[fieldName][i])]
+          }
+
+        // Single data model (schema field can be either a single or array of models, due to custom $lookup's)
+        } else if (util.isObjectAndNotID(data[fieldName]) && (field.model || (field[0]||{}).model)) {
           out.push({
             dataRef: data[fieldName],
-            fieldName: fieldName,
+            dataPath: newParentPath,
+            dataFieldName: fieldName,
             modelName: field.model || field[0].model
           })
-
-        // Recurse through fields that are sub-documents
-        } else if (
-            util.isSubdocument(field) &&
-            util.isObjectAndNotID(data[fieldName])
-          ) {
-          out = [...out, ...this._recurseAndFindModels(field, data[fieldName])]
-
-        // Array of sub-documents or models
-        } else if (
-            (util.isArray(field) || field.model) &&
-            data[fieldName] &&
-            util.isObjectAndNotID(data[fieldName][0])
-          ) {
-          // Valid model object found in array
-          // Note that sometimes an array of models are passed instead of single object via a custom populate $lookup
-          if (field.model || field[0].model) {
-            for (let item of data[fieldName]) {
-              out.push({
-                dataRef: item,
-                fieldName: fieldName,
-                modelName: field.model || field[0].model
-              })
-            }
-          // Objects ares sub-documents, recurse through fields
-          } else if (!field.model && util.isSubdocument(field[0])) {
-            // console.log('101', fieldName, field[0])
-            out = [...out, ...this._recurseAndFindModels(field[0], data[fieldName])]
+        
+        // Array of data models (schema field can be either a single or array of models, due to custom $lookup's)
+        } else if (util.isObjectAndNotID(data[fieldName][0]) && (field.model || (field[0]||{}).model)) {
+          for (let i=0, l=data[fieldName].length; i<l; i++) {
+            out.push({
+              dataRef: data[fieldName][i],
+              dataPath: newParentPath + '.' + i,
+              dataFieldName: fieldName,
+              modelName: field.model || field[0].model
+            })
           }
-        }
+        } 
       }, this)
     }
     return out

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A straight forward MongoDB ODM built around Monk",
   "author": "Ricky Boyce",
-  "version": "2.1.1",
+  "version": "2.2.1",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",
@@ -34,6 +34,7 @@
     "debug": "4.3.4",
     "file-type": "^16.5.4",
     "monk": "7.3.4",
+    "mongodb": "^3.2.3",
     "nanoid": "3.2.0",
     "validator": "13.7.0"
   },

package/test/crud.js

@@ -460,6 +460,7 @@ module.exports = function(monastery, opendb) {
         addresses: [{ city: { type: 'string' }, country: { type: 'string', default: 'Germany' } }],
         address: { country:  { type: 'string', default: 'Germany' }},
         pet:  { dog: { model: 'dog' }},
+        pets:  { dog: [{ model: 'dog' }]},
         dogs: [{ model: 'dog' }], // virtual association
       }
     })
@@ -489,18 +490,20 @@ module.exports = function(monastery, opendb) {
     // Note that addresses.1.country shouldn't be overridden
     // Insert documents (without defaults)
     let dog1 = await db.dog._insert({})
+    let dog2 = await db.dog._insert({})
     let user1 = await db.user._insert({
       addresses: [
         { city: 'Frankfurt' },
         { city: 'Christchurch', country: 'New Zealand' }
       ],
-      pet: { dog: dog1._id }
+      pet: { dog: dog1._id },
+      pets: { dog: [dog1._id, dog2._id]},
     })
     await db.dog._update(dog1._id, { $set: { user: user1._id }})
 
     let find1 = await db.user.findOne({
       query: user1._id,
-      populate: ['pet.dog', {
+      populate: ['pet.dog', 'pets.dog', {
         from: 'dog',
         localField: '_id',
         foreignField: 'user',
@@ -516,19 +519,64 @@ module.exports = function(monastery, opendb) {
       ],
       address: { country: 'Germany' },
       pet: { dog: { _id: dog1._id, name: 'Scruff', user: user1._id }},
+      pets: {
+        dog: [
+          { _id: dog1._id, name: 'Scruff', user: user1._id },
+          { _id: dog2._id, name: 'Scruff' },
+        ]
+      },
       dogs: [{ _id: dog1._id, name: 'Scruff', user: user1._id }]
     })
 
-    // Blacklisted default field population test
+    db.close()
+  })
+
+  test('find default field blacklisted', async () => {
+    let db = (await opendb(null)).db
+    db.model('user', {
+      fields: {
+        name: { type: 'string', default: 'Martin Luther' },
+        addresses: [{ city: { type: 'string' }, country: { type: 'string', default: 'Germany' } }],
+        address: { country:  { type: 'string', default: 'Germany' }},
+        pet:  { dog: { model: 'dog' }},
+        pets:  { dog: [{ model: 'dog' }]},
+        dogs: [{ model: 'dog' }], // virtual association
+      }
+    })
+    db.model('dog', {
+      fields: {
+        age: { type: 'number', default: 12 },
+        name: { type: 'string', default: 'Scruff' },
+        user: { model: 'user' }
+      },
+      findBL: ['age']
+    })
+    let dog1 = await db.dog._insert({})
+    let dog2 = await db.dog._insert({})
+    let user1 = await db.user._insert({
+      addresses: [
+        { city: 'Frankfurt' },
+        { city: 'Christchurch', country: 'New Zealand' }
+      ],
+      pet: { dog: dog1._id },
+      pets: { dog: [dog1._id, dog2._id]},
+    })
+    await db.dog._update(dog1._id, { $set: { user: user1._id }})
+
+    // Blacklisted direct/populated default fields (should be removed)
     let find2 = await db.user.findOne({
       query: user1._id,
-      populate: ['pet.dog', {
-        from: 'dog',
-        localField: '_id',
-        foreignField: 'user',
-        as: 'dogs'
-      }],
-      blacklist: ['address', 'addresses.country', 'dogs.name']
+      populate: [
+        'pet.dog', 
+        'pets.dog', 
+        {
+          from: 'dog',
+          localField: '_id',
+          foreignField: 'user',
+          as: 'dogs',
+        }
+      ],
+      blacklist: ['address', 'addresses.country', 'pets.dog.name', 'dogs.name'],
       // ^ great test (address should cancel addresses if not stopping at the .)
     })
     expect(find2).toEqual({
@@ -536,14 +584,20 @@ module.exports = function(monastery, opendb) {
       name: 'Martin Luther',
       addresses: [{ city: 'Frankfurt' }, { city: 'Christchurch' }],
       pet: { dog: { _id: dog1._id, name: 'Scruff', user: user1._id }},
-      dogs: [{ _id: dog1._id, user: user1._id }]
+      dogs: [{ _id: dog1._id, user: user1._id }],
+      pets: {
+        dog: [
+          { _id: dog1._id, user: user1._id, /*age, name*/ },
+          { _id: dog2._id, /*age, name*/ }, 
+        ]
+      },
     })
 
     db.close()
   })
 
   test('findOneAndUpdate general', async () => {
-    // todo: test all findOneAndUpdate options
+    // todo: test all findOneAndUpdate options (e.g. array population)
     // todo: test find & update hooks
     let db = (await opendb(null)).db
     let dog = db.model('dog', {

package/test/populate.js

@@ -82,6 +82,81 @@ module.exports = function(monastery, opendb) {
     db.close()
   })
 
+  test('model populate array', async () => {
+    // Setup
+    let db = (await opendb(null)).db
+    let bird = db.model('bird', { fields: {
+      name: { type: 'string' }
+    }})
+    let user = db.model('user', { fields: {
+      birds: [{ model: 'bird' }],
+      animal: { birds: [{ model: 'bird' }] },
+      animals: [{ bird: { model: 'bird' }, num: { type: 'number' } }],
+    }})
+    let bird1 = await bird.insert({ data: { name: 'ponyo' }})
+    let bird2 = await bird.insert({ data: { name: 'jack' }})
+    let bird3 = await bird.insert({ data: { name: 'sophie' }})
+    let user1 = await user.insert({ data: {
+      birds: [bird1._id, bird2._id],
+      animal: { birds: [bird1._id, bird2._id] },
+      animals: [{ bird: bird1._id, num: 1 }, { bird: bird3._id, num: 2 }],
+    }})
+
+    // Array
+    let find1 = await user.findOne({ query: user1._id, populate: ['birds'] })
+    expect(find1).toEqual({
+      _id: user1._id,
+      birds: [
+        { _id: bird1._id, name: 'ponyo' },
+        { _id: bird2._id, name: 'jack' },
+      ],
+      animal: {
+        birds: [bird1._id, bird2._id],
+      },
+      animals: [
+        { bird: bird1._id, num: 1 },
+        { bird: bird3._id, num: 2 },
+      ],
+    })
+
+    // Nested array
+    let find2 = await user.findOne({ query: user1._id, populate: ['animal.birds'] })
+    expect(find2).toEqual({
+      _id: user1._id,
+      birds: [bird1._id, bird2._id],
+      animal: {
+        birds: [
+          { _id: bird1._id, name: 'ponyo' },
+          { _id: bird2._id, name: 'jack' },
+        ],
+      },
+      animals: [
+        { bird: bird1._id, num: 1 },
+        { bird: bird3._id, num: 2 },
+      ],
+    })
+
+    // modelId within an array of subdocuments (won't populate, but show a debug error)
+    user.error = (e) => {/*console.log(e)*/} // hide debug error
+    let find3 = await user.findOne({ query: user1._id, populate: ['animals.bird'] })
+    expect(find3).toEqual({
+      _id: user1._id,
+      birds: [bird1._id, bird2._id],
+      animal: {
+        birds: [
+          bird1._id, 
+          bird2._id
+        ],
+      },
+      animals: [
+        { bird: bird1._id, num: 1 },
+        { bird: bird3._id, num: 2 },
+      ],
+    })
+
+    db.close()
+  })
+
   test('model populate type=any', async () => {
     let db = (await opendb(null)).db
     db.model('company', { fields: {