monastery 1.34.0 → 1.35.0

package/.eslintrc.json

@@ -21,7 +21,7 @@
   "plugins": [],
   "rules": {
     "brace-style": ["error", "1tbs", { "allowSingleLine": true }],
-    "max-len": ["error", { "code": 120, "ignorePattern": "^\\s*<(rect|path|line)\\s" }],
+    "max-len": ["error", { "code": 125, "ignorePattern": "^\\s*<(rect|path|line)\\s" }],
     "no-prototype-builtins": "off",
     "no-unused-vars": ["error", { "args": "none" }],
     "object-shorthand": ["error", "consistent"],

package/changelog.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.35.0](https://github.com/boycce/monastery/compare/1.34.0...1.35.0) (2022-04-08)
+
+
+### Features
+
+* `false` removes blacklist, added tests for blacklisting/project stirng ([5999859](https://github.com/boycce/monastery/commit/599985972cc14b980148db26c03108feabf23756))
+
 ## [1.34.0](https://github.com/boycce/monastery/compare/1.33.0...1.34.0) (2022-04-05)
 
 

package/docs/readme.md

@@ -87,11 +87,13 @@ Coming soon...
 - Add before/afterInsertUpdate
 - Bug: Setting an object literal on an ID field ('model') saves successfully
 - Population within array items
-- ~~Blackislisitng which aggregates from the order of appearance~~
+- ~~Blacklist `false` removes all blacklisting~~
+- ~~Add project to insert/update/validate~~
 - ~~Whitelisting a parent will remove any previously blacklisted children~~
+- ~~Blacklist/project works the same across find/insert/update/validate~~
 - Automatic subdocument ids
 - Remove ACL default 'public read'
-- Public db.arrayWithSchema method
+- ~~Public db.arrayWithSchema method~~
 - Global after/before hooks
 - Split away from Monk (unless updated)
 - docs: Make the implicit ID query conversion more apparent

package/docs/schema/index.md

@@ -248,9 +248,9 @@ schema.messages = {
   }
   // You can also target any rules set on the array or sub arrays
   // e.g.
-  // let arrayWithSchema = (array, schema) => { array.schema = schema; return array }
-  // petGroups = arrayWithSchema(
-  //   [arrayWithSchema(
+  // // let arrayWithSchema = (array, schema) => { array.schema = schema; return array }, OR you can use db.arrayWithSchema
+  // petGroups = db.arrayWithSchema(
+  //   [db.arrayWithSchema(
   //     [{ name: { type: 'string' }}],
   //     { minLength: 1 }
   //   )],

package/lib/model-crud.js

@@ -6,33 +6,32 @@ module.exports = {
     /**
      * Inserts document(s) with monk after validating data & before hooks.
      * @param {object} opts
-     *     @param {object|array} <opts.data> - documents to insert
-     *     @param {array|string|false} <opts.blacklist> - augment schema.insertBL, `false` will remove all blacklisting
+     *     @param {object|array} opts.data - documents to insert
+     *     @param {array|string|false} <opts.blacklist> - augment schema.insertBL, `false` will remove blacklisting
+     *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
      *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-     *     @param {array|string|false} validateUndefined - validates all 'required' undefined fields, true by
-     *         default, but false on update
      *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
      *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are automatically inserted
-     *     @param {any} <opts.any> - any mongodb option
+     *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
+     *         default, but false on update
+     *     @param {any} <any mongodb option>
      * @param {function} <cb> - execute cb(err, data) instead of responding
-     * @this model
      * @return promise
+     * @this model
      */
     if (cb && !util.isFunction(cb)) {
       throw new Error(`The callback passed to ${this.name}.insert() is not a function`)
     }
     try {
       opts = await this._queryObject(opts, 'insert')
-      let options = util.omit(opts, [
-        'data', 'insert', 'model', 'respond', 'validateUndefined', 'skipValidation', 'blacklist'
-      ])
+      let custom = ['blacklist', 'data', 'insert', 'model', 'respond', 'skipValidation',  'validateUndefined']
 
       // Validate
-      let data = await this.validate(opts.data||{}, { ...opts })
+      let data = await this.validate(opts.data || {}, { ...opts })
 
       // Insert
       await util.runSeries(this.beforeInsert.map(f => f.bind(opts, data)))
-      let response = await this._insert(data, options)
+      let response = await this._insert(data, util.omit(opts, custom))
       await util.runSeries(this.afterInsert.map(f => f.bind(opts, response)))
 
       // Success/error
@@ -51,53 +50,42 @@ module.exports = {
     /**
      * Finds document(s) with monk, also auto populates
      * @param {object} opts
-     *     @param {object} <opts.query> - mongodb query object
-     *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-     *     @param {array} <opts.populate> - population, see docs
      *     @param {array|string|false} <opts.blacklist> - augment schema.findBL, `false` will remove all blacklisting
+     *     @param {array} <opts.populate> - population, see docs
      *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
-     *     @param {any} <opts.any> - any mongodb option
+     *     @param {object} <opts.query> - mongodb query object
+     *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
+     *     @param {any} <any mongodb option>
      * @param {function} <cb> - execute cb(err, data) instead of responding
-     * @this model
      * @return promise
+     * @this model
      */
     if (cb && !util.isFunction(cb)) {
       throw new Error(`The callback passed to ${this.name}.find() is not a function`)
     }
     try {
       opts = await this._queryObject(opts, 'find', one)
+      let custom = ['blacklist', 'one', 'populate', 'project', 'query', 'respond']
       let lookups = []
-      let options = util.omit(opts, ['blacklist', 'one', 'populate', 'project', 'query', 'respond'])
-      options.addFields = options.addFields || {}
 
-      // Using project (can be an inclusion/exclusion)
-      if (opts.project) {
-        if (util.isString(opts.project)) {
-          opts.project = opts.project.trim().split(/\s+/)
-        }
-        if (util.isArray(opts.project)) {
-          options.projection = opts.project.reduce((o, v) => {
-            o[v.replace(/^-/, '')] = v.match(/^-/)? 0 : 1
-            return o
-          }, {})
-        }
-      // Or blacklisting
-      } else {
-        options.projection = this._getBlacklistProjection('find', opts.blacklist)
-      }
+      // Get projection
+      if (opts.project) opts.projection = this._getProjectionFromProject(opts.project)
+      else opts.projection = this._getProjectionFromBlacklist(opts.type, opts.blacklist)
+
       // Has text search?
       // if (opts.query.$text) {
-      //   options.projection.score = { $meta: 'textScore' }
-      //   options.sort = { score: { $meta: 'textScore' }}
+      //   opts.projection.score = { $meta: 'textScore' }
+      //   opts.sort = { score: { $meta: 'textScore' }}
       // }
+
       // Wanting to populate?
       if (!opts.populate) {
-        var response = await this[`_find${opts.one? 'One' : ''}`](opts.query, options)
+        var response = await this[`_find${opts.one? 'One' : ''}`](opts.query, util.omit(opts, custom))
       } else {
         loop: for (let item of opts.populate) {
           let path = util.isObject(item)? item.as : item
           // Blacklisted?
-          if (!this._pathInProjection(path, options.projection, true)) continue loop
+          if (this._pathBlacklisted(path, opts.projection)) continue loop
           // Custom $lookup definition
           // https://thecodebarbarian.com/a-nodejs-perspective-on-mongodb-36-lookup-expr
           if (util.isObject(item)) {
@@ -118,7 +106,7 @@ module.exports = {
               continue
             }
             // Populate model (convert array into document & create lookup)
-            options.addFields[path] = { '$arrayElemAt': [ '$' + path, 0 ] }
+            (opts.addFields = opts.addFields || {})[path] = { '$arrayElemAt': [ '$' + path, 0 ] }
             lookups.push({ $lookup: {
               from: modelName,
               localField: path,
@@ -127,23 +115,26 @@ module.exports = {
             }})
           }
         }
-        // console.log(1, options.projection)
+        // console.log(1, opts.projection)
         // console.log(2, lookups)
         let aggregate = [
           { $match: opts.query },
-          { $sort: options.sort },
-          { $skip: options.skip },
-          ...(options.limit? [{ $limit: options.limit }] : []),
+          { $sort: opts.sort },
+          { $skip: opts.skip },
+          ...(opts.limit? [{ $limit: opts.limit }] : []),
           ...lookups,
-          ...util.isEmpty(options.addFields)? [] : [{ $addFields: options.addFields }],
-          { $project: options.projection }
+          ...(opts.addFields? [{ $addFields: opts.addFields }] : []),
+          ...(opts.projection? [{ $project: opts.projection }] : []),
         ]
         response = await this._aggregate(aggregate)
         this.info('aggregate', JSON.stringify(aggregate))
       }
 
+      // Returning one?
       if (opts.one && util.isArray(response)) response = response[0]
-      response = await this._processAfterFind(response, options.projection, opts)
+
+      // Process afterFind hooks
+      response = await this._processAfterFind(response, opts.projection, opts)
 
       // Success
       if (cb) cb(null, response)
@@ -161,26 +152,27 @@ module.exports = {
     return this.find(opts, cb, true)
   },
 
-  // findOneAndUpdate: function(opts, cb) {
-  //   return this._findOneAndUpdate(opts, cb)
-  // },
+  findOneAndUpdate: function(opts, cb) {
+    return this._findOneAndUpdate(opts, cb)
+  },
 
   update: async function(opts, cb) {
     /**
      * Updates document(s) with monk after validating data & before hooks.
      * @param {object} opts
+     *     @param {object|array} opts.data - mongodb document update object(s)
+     *     @param {array|string|false} <opts.blacklist> - augment schema.updateBL, `false` will remove blacklisting
+     *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
      *     @param {object} <opts.query> - mongodb query object
-     *     @param {object|array} <opts.data> - mongodb document update object(s)
      *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-     *     @param {array|string|false} validateUndefined - validates all 'required' undefined fields, true by
-     *         default, but false on update
      *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
      *     @param {boolean} <opts.timestamps> - whether `updatedAt` is automatically updated
-     *     @param {array|string|false} <opts.blacklist> - augment schema.updateBL, `false` will remove all blacklisting
-     *     @param {any} <opts.any> - any mongodb option
+     *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
+     *         default, but false on update
+     *     @param {any} <any mongodb option>
      * @param {function} <cb> - execute cb(err, data) instead of responding
-     * @this model
      * @return promise(data)
+     * @this model
      */
     if (cb && !util.isFunction(cb)) {
       throw new Error(`The callback passed to ${this.name}.update() is not a function`)
@@ -190,7 +182,7 @@ module.exports = {
       let data = opts.data
       let response = null
       let operators = util.pluck(opts, [/^\$/])
-      let options = util.omit(opts, ['data', 'query', 'respond', 'validateUndefined', 'skipValidation', 'blacklist'])
+      let custom = ['blacklist', 'data', 'query', 'respond', 'skipValidation', 'validateUndefined']
 
       // Validate
       if (util.isDefined(data)) data = await this.validate(opts.data, { ...opts })
@@ -209,7 +201,7 @@ module.exports = {
         operators['$set'] = { ...data, ...(operators['$set'] || {}) }
       }
       // Update
-      let update = await this._update(opts.query, operators, options)
+      let update = await this._update(opts.query, operators, util.omit(opts, custom))
       if (update.n) response = Object.assign(Object.create({ _output: update }), operators['$set']||{})
 
       // Hook: afterUpdate (doesn't have access to validated data)
@@ -234,22 +226,22 @@ module.exports = {
      *     @param {object} <opts.query> - mongodb query object
      *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
      *     @param {boolean=true} <opts.multi> - set to false to limit the deletion to just one document
-     *     @param {any} <opts.any> - any mongodb option
+     *     @param {any} <any mongodb option>
      * @param {function} <cb> - execute cb(err, data) instead of responding
-     * @this model
      * @return promise
+     * @this model
      */
     if (cb && !util.isFunction(cb)) {
       throw new Error(`The callback passed to ${this.name}.remove() is not a function`)
     }
     try {
       opts = await this._queryObject(opts, 'remove')
-      let options = util.omit(opts, ['query', 'respond'])
+      let custom =  ['query', 'respond']
       if (util.isEmpty(opts.query)) throw new Error('Please specify opts.query')
 
       // Remove
       await util.runSeries(this.beforeRemove.map(f => f.bind(opts)))
-      let response = await this._remove(opts.query, options)
+      let response = await this._remove(opts.query, util.omit(opts, custom))
       await util.runSeries(this.afterRemove.map(f => f.bind(response)))
 
       // Success
@@ -264,13 +256,16 @@ module.exports = {
     }
   },
 
-  _getBlacklistProjection: function(type, customBlacklist) {
+  _getProjectionFromBlacklist: function(type, customBlacklist) {
     /**
      * Returns an exclusion projection
      *
+     * Path collisions are removed
+     * E.g. ['pets.dogs', 'pets.dogs.name', '-cat', 'pets.dogs.age'] = { 'pets.dog': 0 }
+     *
      * @param {string} type - find, insert, or update
-     * @param {array} customBlacklist - normally passed through options
-     * @return {array} exclusion projection {'pets.name': 0}
+     * @param {array|string|false} customBlacklist - normally passed through options
+     * @return {array|undefined} exclusion $project {'pets.name': 0}
      * @this model
      *
      * 1. collate deep-blacklists
@@ -281,16 +276,24 @@ module.exports = {
     let list = []
     let manager = this.manager
     let projection = {}
+    if (customBlacklist === false) return
+
+    // String?
+    if (typeof customBlacklist === 'string') {
+      customBlacklist = customBlacklist.trim().split(/\s+/)
+    }
 
     // Concat deep blacklists
-    util.forEach(this.fieldsFlattened, (schema, path) => {
-      if (!schema.model) return
-      let deepBL = manager.model[schema.model][`${type}BL`] || []
-      let pathWithoutArrays = path.replace(/\.0(\.|$)/, '$1')
-      list = list.concat(deepBL.map(o => {
-        return `${o.charAt(0) == '-'? '-' : ''}${pathWithoutArrays}.${o.replace(/^-/, '')}`
-      }))
-    })
+    if (type == 'find') {
+      util.forEach(this.fieldsFlattened, (schema, path) => {
+        if (!schema.model) return
+        let deepBL = manager.model[schema.model][`${type}BL`] || []
+        let pathWithoutArrays = path.replace(/\.0(\.|$)/, '$1')
+        list = list.concat(deepBL.map(o => {
+          return `${o.charAt(0) == '-'? '-' : ''}${pathWithoutArrays}.${o.replace(/^-/, '')}`
+        }))
+      })
+    }
 
     // Concat model, and custom blacklists
     list = list.concat([...this[`${type}BL`]]).concat(customBlacklist || [])
@@ -302,7 +305,8 @@ module.exports = {
 
       // Remove any child fields. E.g remove { user.token: 0 } = key2 if iterating { user: 0 } = key
       for (let key2 in projection) {
-        if (key2.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) { // test that scoped to the \.
+        // todo: need to write a test, testing that this is scoped to \.
+        if (key2.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) {
           delete projection[key2]
         }
       }
@@ -335,6 +339,28 @@ module.exports = {
       if (projection[key]) delete projection[key]
     }
 
+    return util.isEmpty(projection) ? undefined : projection
+  },
+
+  _getProjectionFromProject: function(customProject) {
+    /**
+     * Returns an in/exclusion projection
+     * todo: tests
+     *
+     * @param {object|array|string} customProject - normally passed through options
+     * @return {array|undefined} in/exclusion projection {'pets.name': 0}
+     * @this model
+     */
+    let projection
+    if (util.isString(customProject)) {
+      customProject = customProject.trim().split(/\s+/)
+    }
+    if (util.isArray(customProject)) {
+      projection = customProject.reduce((o, v) => {
+        o[v.replace(/^-/, '')] = v.match(/^-/)? 0 : 1
+        return o
+      }, {})
+    }
     return projection
   },
 
@@ -344,9 +370,10 @@ module.exports = {
      * @param {MongoId|string|object} opts
      * @param {string} type - operation type
      * @param {boolean} one - return one document
-     * @this model
      * @return {Promise} opts
+     * @this model
      *
+     * Query parsing logic:
      * opts == string|MongodId - treated as an id
      * opts == undefined|null|false - throw error
      * opts.query == string|MongodID - treated as an id
@@ -389,6 +416,7 @@ module.exports = {
     if (!util.isDefined(opts.data) && util.isDefined((opts.req||{}).body)) opts.data = opts.req.body
     if (util.isDefined(opts.data)) opts.data = await util.parseData(opts.data)
 
+    opts.type = type
     opts[type] = true
     opts.model = this
     return opts
@@ -400,11 +428,12 @@ module.exports = {
      * Recurses through fields that are models and populates missing default-fields and calls model.afterFind([fn,..])
      * Be sure to add any virtual fields to the schema that your populating on,
      * e.g. "nurses": [{ model: 'user' }]
+     *
      * @param {object|array|null} data
      * @param {object} projection - $project object
      * @param {object} afterFindContext - handy context object given to schema.afterFind
-     * @this model
      * @return Promise(data)
+     * @this model
      */
     // Recurse down from the parent model, ending with the parent model as the parent afterFind hook may
     // want to manipulate any populated models
@@ -422,8 +451,8 @@ module.exports = {
           if (!util.isDefined(schema.default) || path.match(/^\.?(createdAt|updatedAt)$/)) return
           let parentPath = item.fieldName? item.fieldName + '.' : ''
           let pathWithoutArrays = (parentPath + path).replace(/\.0(\.|$)/, '$1')
-          // Ignore default fields that are excluded in a blacklist/parent-blacklist
-          if (!this._pathInProjection(pathWithoutArrays, projection, true)) return
+          // Ignore default fields that are blacklisted
+          if (this._pathBlacklisted(pathWithoutArrays, projection)) return
           // console.log(pathWithoutArrays, path, projection)
           // Set value
           let value = util.isFunction(schema.default)? schema.default(this.manager) : schema.default
@@ -438,39 +467,34 @@ module.exports = {
     return util.runSeries(callbackSeries).then(() => data)
   },
 
-  _pathInProjection: function(path, projection, matchParentPaths) {
+  _pathBlacklisted: function(path, projection, matchDeepWhitelistedKeys=true) {
     /**
-     * Checks if the path is valid within a inclusion/exclusion projection
+     * Checks if the path is blacklisted within a inclusion/exclusion projection
      * @param {string} path - path without array brackets e.g. '.[]'
-     * @param {object} projection
-     * @param {boolean} matchParentPaths - match paths included in a key (inclusive only)
+     * @param {object} projection - inclusion/exclusion projection, not mixed
+     * @param {boolean} matchDeepWhitelistedKeys - match deep whitelisted keys containing path
+     *     E.g. pets.color == pets.color.age
      * @return {boolean}
      */
-    let inc
-    // console.log(path, projection)
     for (let key in projection) {
-      if (projection[key] && matchParentPaths) {
-        // Inclusion
-        // E.g. 'pets.color.age'.match(/^pets.color.age(.|$)/) = match
-        // E.g. 'pets.color.age'.match(/^pets.color(.|$)/) = match
-        // E.g. 'pets.color'.match(/^pets.color.age(.|$)/) = match
-        inc = true
-        if (key.match(new RegExp('^' + path.replace(/\./g, '\\.') + '(\\.|$)'))) return true
-        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
-      } else if (projection[key]) {
-        // Inclusion (equal to key, or key included in path)
-        // E.g. 'pets.color.age'.match(/^pets.color.age(.|$)/) = match
-        // E.g. 'pets.color.age'.match(/^pets.color(.|$)/) = match
-        // E.g. 'pets.color'.match(/^pets.color.age(.|$)/) = no match
-        inc = true
-        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
-      } else {
-        // Exclusion
-        // E.g. 'pets.color.age'.match(/^pets.color(.|$)/) = match
+      if (projection[key]) {
+        // Inclusion (whitelisted)
+        // E.g. pets.color.age == pets.color.age  (exact match)
+        // E.g. pets.color.age == pets.color      (path contains key)
+        var inclusion = true
         if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return false
+        if (matchDeepWhitelistedKeys) {
+          // E.g. pets.color   == pets.color.age  (key contains path)
+          if (key.match(new RegExp('^' + path.replace(/\./g, '\\.') + '\\.'))) return false
+        }
+      } else {
+        // Exclusion (blacklisted)
+        // E.g. pets.color.age == pets.color.age  (exact match)
+        // E.g. pets.color.age == pets.color      (path contains key)
+        if (path.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) return true
       }
     }
-    return inc? false : true
+    return inclusion? true : false
   },
 
   _recurseAndFindModels: function(fields, dataArr) {

package/lib/model-validate.js

@@ -6,21 +6,19 @@ module.exports = {
   validate: function(data, opts, cb) {
     /**
      * Validates a model
-     * @param {instance} model
      * @param {object} data
      * @param {object} <opts>
-     *     @param {boolean(false)} update - are we validating for insert or update?
-     *     @param {array|string|false} blacklist - augment schema blacklist, `false` will remove all blacklisting
-     *     @param {array|string} projection - only return these fields, ignores blacklist
-     *     @param {array|string|false} validateUndefined - validates all 'required' undefined fields, true by
-     *         default, but false on update
-     *     @param {array|string|true} skipValidation - skip validation on these fields
-     *     @param {boolean} timestamps - whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is
+     *     @param {array|string|false} <opts.blacklist> - augment insertBL/updateBL, `false` will remove blacklisting
+     *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
+     *     @param {array|string|true} <opts.skipValidation> - skip validation on these fields
+     *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is
      *         updated, depending on the `options.update` value
+     *     @param {boolean(false)} <opts.update> - are we validating for insert or update? todo: change to `type`
+     *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
+     *         default, but false on update
      * @param {function} <cb> - instead of returning a promise
-     * @this model
-
      * @return promise(errors[] || pruned data{})
+     * @this model
      */
 
     // Optional cb and opts
@@ -31,41 +29,12 @@ module.exports = {
     data = util.deepCopy(data)
     opts = opts || {}
     opts.insert = !opts.update
-    opts.action = opts.update? 'update' : 'insert'
-    opts.skipValidation = opts.skipValidation === true? true : util.toArray(opts.skipValidation||[])
+    opts.action = opts.update ? 'update' : 'insert'
+    opts.skipValidation = opts.skipValidation === true ? true : util.toArray(opts.skipValidation||[])
 
-    // Blacklist
-    if (opts.blacklist) {
-      let whitelist = []
-      let blacklist = [ ...this[`${opts.action}BL`] ]
-      if (typeof opts.blacklist === 'string') {
-        opts.blacklist = opts.blacklist.trim().split(/\s+/)
-      }
-      // Auguemnt the schema blacklist
-      for (let _path of opts.blacklist) {
-        let path = _path.replace(/^-/, '')
-        if (_path.match(/^-/)) whitelist.push(path)
-        else blacklist.push(path)
-      }
-      // Remove whitelisted/negated fields
-      blacklist = blacklist.filter(o => !whitelist.includes(o))
-      // Remove any deep blacklisted fields that have a whitelisted parent specified.
-      // E.g remove ['deep.deep2.deep3'] if ['deep'] exists in the whitelist
-      for (let i=blacklist.length; i--;) {
-        let split = blacklist[i].split('.')
-        for (let j=split.length; j--;) {
-          if (split.length > 1) split.pop()
-          else continue
-          if (whitelist.includes(split.join())) {
-            blacklist.splice(i, 1)
-            break
-          }
-        }
-      }
-      opts.blacklist = blacklist
-    } else {
-      opts.blacklist = [ ...this[`${opts.action}BL`] ]
-    }
+    // Get projection
+    if (opts.project) opts.projection = this._getProjectionFromProject(opts.project)
+    else opts.projection = this._getProjectionFromBlacklist(opts.action, opts.blacklist)
 
     // Run before hook, then recurse through the model's fields
     return util.runSeries(this.beforeValidate.map(f => f.bind(opts, data))).then(() => {
@@ -140,7 +109,7 @@ module.exports = {
         let value = util.isArray(fields)? data : (data||{})[fieldName]
         let indexOrFieldName = util.isArray(fields)? i : fieldName
         let path2 = `${path}.${indexOrFieldName}`.replace(/^\./, '')
-        let path3 = path2.replace(/(^|\.)[0-9]+(\.|$)/, '$2') // no numerical keys, e.g. pets.1.name
+        let path3 = path2.replace(/(^|\.)[0-9]+(\.|$)/, '$2') // no numerical keys, e.g. pets.1.name = pets.name
         let isType = 'is' + util.ucFirst(schema.type)
         let isTypeRule = this.rules[isType] || rules[isType]
 
@@ -157,7 +126,7 @@ module.exports = {
         }
 
         // Ignore blacklisted
-        if (opts.blacklist.indexOf(path3) >= 0 && !schema.defaultOverride) return
+        if (this._pathBlacklisted(path3, opts.projection) && !schema.defaultOverride) return
         // Ignore insert only
         if (opts.update && schema.insertOnly) return
         // Ignore virtual fields
@@ -218,8 +187,8 @@ module.exports = {
      * @param {object} field - field schema
      * @param {string} path - full field path
      * @param {object} opts - original validate() options
-     * @this model
      * @return {array} errors
+     * @this model
      */
     let errors = []
     if (opts.skipValidation === true) return []

package/lib/model.js

@@ -9,8 +9,8 @@ let Model = module.exports = function(name, opts, manager) {
    * @param {string} name
    * @param {object} opts - see mongodb colleciton documentation
    * @param {boolean} opts.waitForIndexes
-   * @this model
    * @return Promise(model) | this
+   * @this model
    */
   if (!(this instanceof Model)) {
     return new Model(name, opts, this)

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A straight forward MongoDB ODM built around Monk",
   "author": "Ricky Boyce",
-  "version": "1.34.0",
+  "version": "1.35.0",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",

package/test/blacklisting.js

@@ -1,6 +1,6 @@
 module.exports = function(monastery, opendb) {
 
-  test('find blacklisting', async () => {
+  test('find blacklisting basic', async () => {
     // Setup
     let db = (await opendb(null)).db
     let bird = db.model('bird', {
@@ -93,7 +93,7 @@ module.exports = function(monastery, opendb) {
       }
     }})
 
-    // Test initial blacklist
+    // initial blacklist
     let find1 = await user.findOne({
       query: user1._id
     })
@@ -107,18 +107,30 @@ module.exports = function(monastery, opendb) {
       deepModel: { myBird: bird1._id }
     })
 
-    // Test augmented blacklist
+    // augmented blacklist
     let find2 = await user.findOne({
       query: user1._id,
       blacklist: ['pet', 'pet', 'deep', 'deepModel', '-dog', '-animals.cat']
     })
-    expect(find2).toEqual({
+    let customBlacklist
+    expect(find2).toEqual((customBlacklist = {
       _id: user1._id,
       dog: 'Bruce',
       list: [44, 54],
       pets: [{ name: 'Pluto' }, { name: 'Milo' }],
       animals: { dog: 'Max', cat: 'Ginger' }
+    }))
+
+    // blacklist string
+    let find3 = await user.findOne({
+      query: user1._id,
+      blacklist: 'pet pet deep deepModel -dog -animals.cat'
     })
+    expect(find3).toEqual(customBlacklist)
+
+    // blacklist removal
+    let find4 = await user.findOne({ query: user1._id, blacklist: false })
+    expect(find4).toEqual(user1)
 
     db.close()
   })
@@ -219,6 +231,11 @@ module.exports = function(monastery, opendb) {
       _id: user1._id,
       bird5: { ...bird1Base, name: 'ponyo', height: 40 },
     })
+    // blacklist removal
+    expect(await user.findOne({ query: user1._id, blacklist: false, populate: ['bird1'] })).toEqual({
+      ...user1,
+      bird1: { ...bird1Base, height: 40, name: 'ponyo', wing: { size: 1, sizes: { one: 1, two: 1 }} },
+    })
 
     db.close()
   })
@@ -243,13 +260,13 @@ module.exports = function(monastery, opendb) {
       },
     })
     // default
-    expect(db.user._getBlacklistProjection('find')).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find')).toEqual({
       'bird1.wing': 0,
       'bird1.age': 0,
       'password': 0,
     })
     // blacklist /w invalid field (which goes through)
-    expect(db.user._getBlacklistProjection('find', ['name', 'invalidfield'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['name', 'invalidfield'])).toEqual({
       'bird1.wing': 0,
       'bird1.age': 0,
       'invalidfield': 0,
@@ -257,38 +274,38 @@ module.exports = function(monastery, opendb) {
       'password': 0,
     })
     // whitelist
-    expect(db.user._getBlacklistProjection('find', ['-password', '-bird1.age'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['-password', '-bird1.age'])).toEqual({
       'bird1.wing': 0,
     })
     // whitelist parent
-    expect(db.user._getBlacklistProjection('find', ['-bird1'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['-bird1'])).toEqual({
       'password': 0,
     })
     // whitelist parent, then blacklist child
-    expect(db.user._getBlacklistProjection('find', ['-bird1', 'bird1.name'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['-bird1', 'bird1.name'])).toEqual({
       'password': 0,
       'bird1.name': 0,
     })
     // the model's blacklists are applied after deep model's
     db.user.findBL = ['-bird1.age']
-    expect(db.user._getBlacklistProjection('find')).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find')).toEqual({
       'bird1.wing': 0,
     })
     // custom blacklists are applied after the model's, which are after deep model's
     db.user.findBL = ['-bird1.age']
-    expect(db.user._getBlacklistProjection('find', ['bird1'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['bird1'])).toEqual({
       'bird1': 0,
     })
     // blacklisted parent with a blacklisted child
-    expect(db.user._getBlacklistProjection('find', ['bird1', 'bird1.wing'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['bird1', 'bird1.wing'])).toEqual({
       'bird1': 0,
     })
     // A mess of things
-    expect(db.user._getBlacklistProjection('find', ['-bird1', 'bird1.wing', '-bird1.wing','bird1.wing.size'])).toEqual({
+    expect(db.user._getProjectionFromBlacklist('find', ['-bird1', 'bird1.wing', '-bird1.wing','bird1.wing.size'])).toEqual({
       'bird1.wing.size': 0,
     })
     // blacklisted parent with a whitelisted child (expect blacklist expansion in future version?)
-    // expect(db.user._getBlacklistProjection('find', ['bird1', '-bird1.wing'])).toEqual({
+    // expect(db.user._getProjectionFromBlacklist('find', ['bird1', '-bird1.wing'])).toEqual({
     //   'bird1.age': 0,
     //   'bird1.name': 0,
     // })
@@ -296,7 +313,7 @@ module.exports = function(monastery, opendb) {
     db.close()
   })
 
-  test('find project', async () => {
+  test('find project basic', async () => {
     // Test mongodb native project option
     // Setup
     let db = (await opendb(null)).db
@@ -370,14 +387,14 @@ module.exports = function(monastery, opendb) {
           color: { type: 'string', default: 'red' },
         }
       },
-      findBL: ['age']
+      findBL: ['age'],
     })
     let user = db.model('user', {
       fields: {
         dog: { type: 'string' },
         bird: { model: 'bird' },
         bird2: { model: 'bird' },
-        bird3: { model: 'bird' }
+        bird3: { model: 'bird' },
       },
       findBL: [
         // allll these should be ignored.....?/////
@@ -390,38 +407,47 @@ module.exports = function(monastery, opendb) {
       name: 'ponyo',
       age: 3,
       height: 40,
-      sub: {}
+      sub: {},
     }})
     let user1 = await user.insert({ data: {
       dog: 'Bruce',
       bird: bird1._id,
       bird2: bird1._id,
-      bird3: bird1._id
+      bird3: bird1._id,
     }})
 
-    // Test project
+    // project
     let find1 = await user.findOne({
       query: user1._id,
       populate: ['bird', 'bird2'],
-      project: ['bird.age', 'bird2']
+      project: ['bird.age', 'bird2'],
     })
     expect(find1).toEqual({
       _id: user1._id,
       bird: { age: 3 },
-      bird2: { _id: bird1._id, age: 3, name: 'ponyo', height: 40, color: 'red', sub: { color: 'red' }}
+      bird2: { _id: bird1._id, age: 3, name: 'ponyo', height: 40, color: 'red', sub: { color: 'red' }},
     })
 
-    // Test project (different project details)
+    // project (different project details)
     let find2 = await user.findOne({
       query: user1._id,
       populate: ['bird', 'bird2'],
-      project: ['bird', 'bird2.height']
+      project: ['bird', 'bird2.height'],
     })
-    expect(find2).toEqual({
+    let customProject
+    expect(find2).toEqual((customProject={
       _id: user1._id,
       bird: { _id: bird1._id, age: 3, name: 'ponyo', height: 40, color: 'red', sub: { color: 'red' }},
       bird2: { height: 40 },
+    }))
+
+    // project string
+    let find3 = await user.findOne({
+      query: user1._id,
+      populate: ['bird', 'bird2'],
+      project: 'bird bird2.height',
     })
+    expect(find3).toEqual(customProject)
 
     db.close()
   })
@@ -506,7 +532,8 @@ module.exports = function(monastery, opendb) {
         '-deep' // blacklist a parent
       ],
     })
-    expect(user2).toEqual({
+    let customBlacklist
+    expect(user2).toEqual((customBlacklist = {
       list: [44, 54],
       dog: 'Bruce',
       pet: 'Freddy',
@@ -522,6 +549,36 @@ module.exports = function(monastery, opendb) {
           }
         }
       }
+    }))
+
+    // Blacklist string
+    let user3 = await user.validate(doc1, {
+      blacklist: '-dog -animals.dog pets.name -hiddenList -deep'
+    })
+    expect(user3).toEqual(customBlacklist)
+
+    // Blacklist removal
+    let user4 = await user.validate(doc1, { blacklist: false })
+    expect(user4).toEqual(doc1)
+
+    // Project whitelist
+    let user5 = await user.validate(doc1, {
+      project: [
+        'dog',
+        'pets.name',
+        'deep'
+      ],
+    })
+    expect(user5).toEqual({
+      dog: 'Bruce',
+      pets: [ {name: 'Pluto'}, {name: 'Milo'} ],
+      deep: {
+        deep2: {
+          deep3: {
+            deep4: 'hideme'
+          }
+        }
+      }
     })
     db.close()
   })