monastery 1.32.4 → 1.34.0

package/changelog.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.34.0](https://github.com/boycce/monastery/compare/1.33.0...1.34.0) (2022-04-05)
+
+
+### Features
+
+* added db.arraySchema / db.arrayWithSchema ([4210dc3](https://github.com/boycce/monastery/commit/4210dc33486de757a22d0973e661522e19230158))
+
+## [1.33.0](https://github.com/boycce/monastery/compare/1.32.5...1.33.0) (2022-04-05)
+
+
+### Features
+
+* whitelisting a parent will remove any previously blacklisted children ([d335c2e](https://github.com/boycce/monastery/commit/d335c2e2e9e691b2c0b406f06cb9c2ed14f8bb25))
+
+### [1.32.5](https://github.com/boycce/monastery/compare/1.32.4...1.32.5) (2022-03-21)
+
+
+### Bug Fixes
+
+* dates not enforcing numbers ([d5a2a53](https://github.com/boycce/monastery/commit/d5a2a533f497b3c9e50cbeb66407cf9bc08dd2e4))
+
 ### [1.32.4](https://github.com/boycce/monastery/compare/1.32.3...1.32.4) (2022-03-07)
 
 ### [1.32.3](https://github.com/boycce/monastery/compare/1.32.2...1.32.3) (2022-03-07)

package/docs/readme.md

@@ -81,6 +81,21 @@ npm run dev -- -t 'Model indexes'
 
 Coming soon...
 
+## Roadmap
+
+- Add FindOneAndUpdate
+- Add before/afterInsertUpdate
+- Bug: Setting an object literal on an ID field ('model') saves successfully
+- Population within array items
+- ~~Blackislisitng which aggregates from the order of appearance~~
+- ~~Whitelisting a parent will remove any previously blacklisted children~~
+- Automatic subdocument ids
+- Remove ACL default 'public read'
+- Public db.arrayWithSchema method
+- Global after/before hooks
+- Split away from Monk (unless updated)
+- docs: Make the implicit ID query conversion more apparent
+
 ## Special Thanks
 
 [Jerome Gravel-Niquet](https://github.com/jeromegn)

package/lib/index.js

@@ -53,6 +53,9 @@ module.exports = function(uri, opts, fn) {
   manager.timestamps = timestamps
   manager.useMilliseconds = useMilliseconds
   manager.beforeModel = []
+  manager.arrayWithSchema = manager.arraySchema = (array, schema) => {
+    array.schema = schema; return array
+  }
 
   // Depreciation warnings
   if (depreciationWarningDefaultField) {

package/lib/model-crud.js

@@ -70,9 +70,8 @@ module.exports = {
       let options = util.omit(opts, ['blacklist', 'one', 'populate', 'project', 'query', 'respond'])
       options.addFields = options.addFields || {}
 
-      // Project, or use blacklisting
+      // Using project (can be an inclusion/exclusion)
       if (opts.project) {
-        // Can be an inclusion or exclusion projection
         if (util.isString(opts.project)) {
           opts.project = opts.project.trim().split(/\s+/)
         }
@@ -82,11 +81,9 @@ module.exports = {
             return o
           }, {})
         }
+      // Or blacklisting
       } else {
-        // Calculate the exclusion-projection
-        let blacklistProjection = { ...this.findBLProject }
-        blacklistProjection = this._addDeepBlacklists(blacklistProjection, opts.populate)
-        options.projection = this._addBlacklist(blacklistProjection, opts.blacklist)
+        options.projection = this._getBlacklistProjection('find', opts.blacklist)
       }
       // Has text search?
       // if (opts.query.$text) {
@@ -146,9 +143,6 @@ module.exports = {
       }
 
       if (opts.one && util.isArray(response)) response = response[0]
-      // (Not using) Project works with lookup.
-      // Remove blacklisted properties from joined models, because subpiplines with 'project' are slower
-      // if (opts.populate) this._depreciated_removeBlacklisted(data, opts.populate, options.projection)
       response = await this._processAfterFind(response, options.projection, opts)
 
       // Success
@@ -270,6 +264,80 @@ module.exports = {
     }
   },
 
+  _getBlacklistProjection: function(type, customBlacklist) {
+    /**
+     * Returns an exclusion projection
+     *
+     * @param {string} type - find, insert, or update
+     * @param {array} customBlacklist - normally passed through options
+     * @return {array} exclusion projection {'pets.name': 0}
+     * @this model
+     *
+     * 1. collate deep-blacklists
+     * 2. concatenate the model's blacklist and any custom blacklist
+     * 3. create an exclusion projection object from the blacklist, overriding from left to right
+     */
+
+    let list = []
+    let manager = this.manager
+    let projection = {}
+
+    // Concat deep blacklists
+    util.forEach(this.fieldsFlattened, (schema, path) => {
+      if (!schema.model) return
+      let deepBL = manager.model[schema.model][`${type}BL`] || []
+      let pathWithoutArrays = path.replace(/\.0(\.|$)/, '$1')
+      list = list.concat(deepBL.map(o => {
+        return `${o.charAt(0) == '-'? '-' : ''}${pathWithoutArrays}.${o.replace(/^-/, '')}`
+      }))
+    })
+
+    // Concat model, and custom blacklists
+    list = list.concat([...this[`${type}BL`]]).concat(customBlacklist || [])
+
+    // Loop blacklists
+    for (let _key of list) {
+      let key = _key.replace(/^-/, '')
+      let whitelisted = _key.match(/^-/)
+
+      // Remove any child fields. E.g remove { user.token: 0 } = key2 if iterating { user: 0 } = key
+      for (let key2 in projection) {
+        if (key2.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) { // test that scoped to the \.
+          delete projection[key2]
+        }
+      }
+
+      // Whitelist
+      if (whitelisted) {
+        projection[key] = 1
+        // Whitelisting a child of a blacklisted field (blacklist expansion)
+        // let parent = '' // highest blacklisted parent
+        // for (let key2 in projection) {
+        //   if (key2.length > parent.length && key.match(new RegExp('^' + key2.replace(/\./g, '\\.')))) {
+        //     parent = key2
+        //   }
+        // }
+
+      // Blacklist (only if there isn't a parent blacklisted)
+      } else {
+        let parent
+        for (let key2 in projection) { // E.g. [address = key2, addresses.country = key]
+          if (projection[key2] == 0 && key.match(new RegExp('^' + key2.replace(/\./g, '\\.') + '\\.'))) {
+            parent = key2
+          }
+        }
+        if (!parent) projection[key] = 0
+      }
+    }
+
+    // Remove whitelist projections
+    for (let key in projection) {
+      if (projection[key]) delete projection[key]
+    }
+
+    return projection
+  },
+
   _queryObject: async function(opts, type, one) {
     /**
      * Normalise options
@@ -347,17 +415,18 @@ module.exports = {
 
     // Loop found model/deep-model data objects, and populate missing default-fields and call afterFind on each
     for (let item of modelData) {
-      // Populuate missing default fields if data !== null
+      // Populate missing default fields if data !== null
       // NOTE: maybe only call functions if default is being set.. fine for now
       if (item.dataRef) {
-        util.forEach(model[item.modelName].defaultFieldsFlattened, (schema, path) => {
+        util.forEach(model[item.modelName].fieldsFlattened, (schema, path) => {
+          if (!util.isDefined(schema.default) || path.match(/^\.?(createdAt|updatedAt)$/)) return
           let parentPath = item.fieldName? item.fieldName + '.' : ''
           let pathWithoutArrays = (parentPath + path).replace(/\.0(\.|$)/, '$1')
           // Ignore default fields that are excluded in a blacklist/parent-blacklist
           if (!this._pathInProjection(pathWithoutArrays, projection, true)) return
-          // Ignore default
+          // console.log(pathWithoutArrays, path, projection)
+          // Set value
           let value = util.isFunction(schema.default)? schema.default(this.manager) : schema.default
-          // console.log(item.dataRef)
           util.setDeepValue(item.dataRef, path.replace(/\.0(\.|$)/g, '.$$$1'), value, true, false, true)
         })
       }
@@ -369,157 +438,6 @@ module.exports = {
     return util.runSeries(callbackSeries).then(() => data)
   },
 
-  _addDeepBlacklists: function(blacklistProjection, populate) {
-    /**
-     * Include deep-model blacklists into the projection
-     * @param {object} blacklistProject
-     * @param {array} populate - find populate array
-     * @return {object} exclusion blacklist
-     * @this model
-     */
-    let model = this
-    let manager = this.manager
-    let paths = (populate||[]).map(o => o && o.as? o.as : o)
-
-    if (!paths.length) return blacklistProjection
-    this._recurseFields(model.fields, '', function(path, field) {
-      // Remove array indexes from the path e.g. '0.'
-      path = path.replace(/(\.[0-9]+)(\.|$)/, '$2')
-      if (!field.model || !paths.includes(path)) return
-      loop: for (let prop of manager.model[field.model].findBL) {
-        // Don't include any deep model projection keys that already have a parent specified. E.g if
-        // both { users.secrets: 1 } and { users.secrets.token: 1 } exist, remove the later
-        for (let key in blacklistProjection) {
-          if ((path + '.' + prop).match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) {
-            continue loop
-          }
-        }
-        // Add model property to blacklist
-        blacklistProjection[path + '.' + prop] = 0
-      }
-    })
-    return blacklistProjection
-  },
-
-  _addBlacklist: function(blacklistProjection, blacklist) {
-    /**
-     * Merge blacklist in
-     * @param {object} blacklistProjection
-     * @param {array} paths - e.g. ['password', '-email'] - email will be whitelisted / removed from
-     *   exlcusion projection
-     * @return {object} exclusion blacklist
-     * @this model
-     */
-    if (blacklist === false) {
-      return {}
-    } else if (!blacklist) {
-      return blacklistProjection
-    } else {
-      // Loop blacklist
-      for (let _key of blacklist) {
-        let key = _key.replace(/^-/, '')
-        let negated = _key.match(/^-/)
-        // Remove any deep projection keys that already have a parent specified. E.g if
-        // both { users.secrets: 0 } and { users.secrets.token: 0 } exist, remove the later
-        for (let key2 in blacklistProjection) {
-          if (key2.match(new RegExp('^' + key.replace(/\./g, '\\.') + '(\\.|$)'))) {
-            delete blacklistProjection[key2]
-          }
-        }
-        if (negated) {
-          if (blacklistProjection.hasOwnProperty(key)) delete blacklistProjection[key]
-        } else {
-          blacklistProjection[key] = 0
-        }
-      }
-      return blacklistProjection
-    }
-  },
-
-  _depreciated_removeBlacklisted: function(data, populate, whitelist) {
-    /**
-     * Remove blacklisted fields, takes foreign model blacklists into account
-     * @param {object|array} data
-     * @param {array} <populate> find populate list
-     * @param {array} <whitelist>
-     */
-    let findBL = this.findBL
-    let findWL = [ '_id', ...this.findWL ]
-    let model = this.manager.model
-
-    this._recurseFields(this.fields, '', function(path, field) {
-      // Remove array indexes from the path e.g. '0.'
-      path = path.replace(/(\.[0-9]+)(\.|$)/, '$2')
-      //if (field.type == 'any') findWL.push(path) //exclude.push(path)
-
-      // Below: Merge in model whitelists (we should cache the results)
-      if (!field.model) return
-      else if (!model[field.model]) return
-
-      // Has this path been blacklisted already?
-      for (let blacklisted of findBL) {
-        if (blacklisted === path || path.includes(blacklisted + '.', 0)) {
-          return
-        }
-      }
-
-      // Populating?
-      if ((populate||[]).includes(path)) {
-        // Remove the model-field from the whitelist if populating
-        let parentIndex = findWL.indexOf(path)
-        if (parentIndex !== -1) findWL.splice(parentIndex, 1)
-
-        // Okay, merge in the model's whitelist
-        findWL.push(path + '.' + '_id')
-        for (let prop of model[field.model].findWL) {
-          // Dont add model properties that are already blacklisted
-          if (findBL.includes(path + '.' + prop)) continue
-          // Add model prop to whitelist
-          findWL.push(path + '.' + prop)
-        }
-      }
-    })
-
-    // Merge in the passed in whitelist
-    findWL = findWL.concat(whitelist || [])
-
-    // Fill in missing parents e.g. pets.dog.name, pets.dog doesn't exist
-    // Doesn't matter what order these are added in
-    // for (let whitelisted of findWL) {
-    //   let parents = whitelisted.split('.')
-    //   let target = ''
-    //   for (let parent of parents) {
-    //     if (!findWL.includes(target + parent)) findWL.push(target + parent)
-    //     target = target + parent + '.'
-    //   }
-    // }
-
-    console.log(1, findWL)
-    console.log(2, data)
-
-    // "data.cat.colour" needs to add "data.cat"
-    // "data.cat" needs to everything
-
-    function recurseAndDeleteData(data, path) {
-      // Remove array indexes from the path e.g. '0.'
-      let newpath = path.replace(/(\.[0-9]+)(\.|$)/, '$2')
-      util.forEach(data, function(field, fieldName) {
-        if ((util.isArray(field) || util.isObjectAndNotID(field))/* && !exclude.includes(newpath + fieldName)*/) {
-          if (findWL.includes(newpath + fieldName)) return
-          recurseAndDeleteData(field, newpath + fieldName + '.')
-        } else if (!findWL.includes(newpath + fieldName) && !util.isNumber(fieldName)) {
-          console.log(3, fieldName)
-          delete data[fieldName]
-        }
-      })
-    }
-
-    for (let doc of util.toArray(data)) {
-      recurseAndDeleteData(doc, '')
-    }
-    return data
-  },
-
   _pathInProjection: function(path, projection, matchParentPaths) {
     /**
      * Checks if the path is valid within a inclusion/exclusion projection
@@ -630,6 +548,6 @@ module.exports = {
         cb(path + fieldName, field)
       }
     }, this)
-  }
+  },
 
 }

package/lib/model.js

@@ -69,12 +69,8 @@ let Model = module.exports = function(name, opts, manager) {
   }, this)
 
   // Extend default fields with passed in fields and check for invalid fields
-  this.fields = Object.assign({}, this._timestampFields, this.fields)
-  this._setupFieldsAndWhitelists(this.fields)
-
-  //   console.log(0, this.fieldlist)
-  //   console.log(0, this.findBL)
-  //   console.log(0, this.findBLProject)
+  this._setupFields(this.fields = Object.assign({}, this._timestampFields, this.fields))
+  this.fieldsFlattened = this._getFieldsFlattened(this.fields, '') // test output?
 
   // Extend model with monk collection actions
   this._collection = manager.get? manager.get(name, { castIds: false }) : null
@@ -107,52 +103,24 @@ let Model = module.exports = function(name, opts, manager) {
   else this._setupIndexes().catch(errHandler) // returns this
 }
 
-Model.prototype._getFieldlist = function(fields, path) {
-  /**
-   * Get all field paths (without array indices, handy for blacklisting)
-   * @param {object|array} fields - subdocument or array
-   * @param {string} path
-   * @return {array} e.g. ['name', 'pets.dog']
-   */
-  let list = []
-  util.forEach(fields, function(field, fieldName) {
-    // Don't append array indexes to the new path e.g. '0.'
-    let newPath = util.isArray(fields)? path : path + fieldName + '.'
-    if (fieldName == 'schema') {
-      return
-    /*} else if (this.findBL.includes(newPath.replace(/\.$/, ''))) {
-      return*/
-    } else if (util.isArray(field)) {
-      list = list.concat(this._getFieldlist(field, newPath))
-    } else if (util.isSubdocument(field)) {
-      list = list.concat(this._getFieldlist(field, newPath))
-    } else {
-      list.push(newPath.replace(/\.$/, ''))
-    }
-  }, this)
-  return list
-}
-
 Model.prototype._getFieldsFlattened = function(fields, path) {
   /**
    * Flatten fields
-   * @param {object|array} fields - subdocument or array
+   * @param {object|array} fields - can be a nested subdocument or array
    * @param {string} path
-   * @return {object} e.g. ['name', 'pets.dog']
+   * @return {object} e.g. {'name': Schema, 'pets.dog': Schema}
    */
   let obj = {}
   util.forEach(fields, function(field, fieldName) {
-    let newPath = path + fieldName + '.'
+    let newPath = /*util.isArray(fields)? path : */path + fieldName + '.'
     if (fieldName == 'schema') {
       return
     } else if (util.isArray(field)) {
       Object.assign(obj, this._getFieldsFlattened(field, newPath))
     } else if (util.isSubdocument(field)) {
       Object.assign(obj, this._getFieldsFlattened(field, newPath))
-    } else if (fieldName != 'createdAt' && fieldName != 'updatedAt') {
-      if (util.isDefined(field.default)) {
-        obj[newPath.replace(/\.$/, '')] = field
-      }
+    } else {
+      obj[newPath.replace(/\.$/, '')] = field
     }
   }, this)
   return obj
@@ -234,19 +202,6 @@ Model.prototype._setupFields = function(fields) {
   }, this)
 },
 
-Model.prototype._setupFieldsAndWhitelists = function(fields, path) {
-  /**
-   * Setup fields and retrieve a handy schema field list. This can be called mulitple times.
-   * Note: the project fields are only required when finding with projections.
-   * @param {object} fields - can be a nested subdocument or array
-   * @param {string} <path>
-   */
-  this._setupFields(fields)
-  this.fieldlist = this._getFieldlist(fields, path || '')
-  this.defaultFieldsFlattened = this._getFieldsFlattened(fields, path || '') // test output?
-  this.findBLProject = this.findBL.reduce((o, v) => { (o[v] = 0); return o }, {})
-},
-
 Model.prototype._setupIndexes = function(fields, opts={}) {
   /**
    * Creates indexes for the model (multikey, and sub-document supported)

package/lib/rules.js

@@ -1,3 +1,4 @@
+// Todo: remove stringnums in date/number/integer rules
 let ObjectId = require('mongodb').ObjectId
 let util = require('./util')
 let validator = require('validator')
@@ -48,12 +49,12 @@ module.exports = {
     message: 'Value was not a unix timestamp.',
     tryParse: function(x) {
       if (x === '') return null
-      if (util.isString(x) && x.match(/^[+-]?[0-9]+$/)) return x // keep string nums intact
-      return isNaN(parseInt(x))? x : parseInt(x)
+      if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return x // keep string nums intact
+      return isNaN(Number(x)) || (!x && x!==0) || x === true? x : Number(x)
     },
     fn: function(x) {
-      if (util.isString(x) && x.match(/^[+-]?[0-9]+$/)) return true
-      return typeof x === 'number' && (parseInt(x) === x)
+      if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return true
+      return typeof x === 'number'
     }
   },
   isImageObject: {
@@ -78,10 +79,10 @@ module.exports = {
     tryParse: function(x) {
       if (x === '') return null
       if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return x // keep string nums intact
-      return isNaN(parseInt(x)) || (!x && x!==0) || x===true? x : parseInt(x)
+      return isNaN(parseInt(x)) || (!x && x!==0) || x === true? x : parseInt(x)
     },
     fn: function(x) {
-      if (util.isString(x) && x.match(/^[+-]?[0-9]+$/)) return true
+      if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return true
       return typeof x === 'number' && (parseInt(x) === x)
     }
   },
@@ -91,7 +92,7 @@ module.exports = {
     tryParse: function(x) {
       if (x === '') return null
       if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return x // keep string nums intact
-      return isNaN(Number(x)) || (!x && x!==0) || x===true? x : Number(x)
+      return isNaN(Number(x)) || (!x && x!==0) || x === true? x : Number(x)
     },
     fn: function(x) {
       if (util.isString(x) && x.match(/^[+-][0-9]+$/)) return true

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A straight forward MongoDB ODM built around Monk",
   "author": "Ricky Boyce",
-  "version": "1.32.4",
+  "version": "1.34.0",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",

package/plugins/images/index.js

@@ -56,8 +56,8 @@ let plugin = module.exports = {
     model.imageFields = plugin._findAndTransformImageFields(model.fields, '')
 
     if (model.imageFields.length) {
-      // Todo?: Update image fields and whitelists with the new object schema
-      //   model._setupFieldsAndWhitelists(model.fields)
+      // Todo?: Update image fields / blacklists with the new object schema
+      //   model._setupFields(model.fields)/model._getFieldsFlattened(model.fields)
       model.beforeValidate.push(function(data, n) {
         plugin.keepImagePlacement(this, data).then(() => n(null, data)).catch(e => n(e))
       })

package/test/blacklisting.js

@@ -93,8 +93,6 @@ module.exports = function(monastery, opendb) {
       }
     }})
 
-    // Note: testing mongodb projections
-
     // Test initial blacklist
     let find1 = await user.findOne({
       query: user1._id
@@ -122,37 +120,184 @@ module.exports = function(monastery, opendb) {
       animals: { dog: 'Max', cat: 'Ginger' }
     })
 
-    // Test positive projection
-    let find3 = await user.findOne({
-      query: user1._id,
-      project: ['dog', 'list', 'pets.age']
+    db.close()
+  })
+
+  test('find blacklisting population', async () => {
+    // inprogresss
+    // Setup
+    let db = monastery('localhost/monastery', {
+      timestamps: false,
+      serverSelectionTimeoutMS: 2000,
     })
-    expect(find3).toEqual({
-      _id: user1._id,
-      dog: 'Bruce',
-      list: [44, 54],
-      pets: [{ age: 5 }, { age: 4 }]
+    let bird = db.model('bird', {
+      fields: {
+        color: { type: 'string', default: 'red' },
+        height: { type: 'number' },
+        name: { type: 'string' },
+        sub: {
+          color: { type: 'string', default: 'red' },
+        },
+        subs: [{
+          color: { type: 'string', default: 'red'},
+        }],
+        wing: {
+          size: { type: 'number' },
+          sizes: {
+            one: { type: 'number' },
+            two: { type: 'number' },
+          }
+        },
+      },
+      findBL: ['wing']
     })
-
-    // Test negative projection
-    let find5 = await user.findOne({
-      query: user1._id,
-      project: [
-        '-list', '-hiddenDeepModel', '-pet', '-pet', '-pets', '-deep', '-deeper', '-deepModel',
-        '-dog', '-animals.cat'
+    let user = db.model('user', {
+      fields: {
+        dog: { type: 'string' },
+        bird1: { model: 'bird' },
+        bird2: { model: 'bird' },
+        bird3: { model: 'bird' },
+        bird4: { model: 'bird' },
+        bird5: { model: 'bird' },
+      },
+      findBL: [
+        'bird1.name',                                 // bird1.name & bird1.wing blacklisted
+        '-bird2', 'bird2.name',                       // bird2.name blacklisted
+        'bird3.name', '-bird3', 'bird3.height',       // bird3.height blacklisted
+        '-bird4.wing.sizes.one', '-bird4.wing.size',  // ignored
+        // bird4.wing.sizes.two blacklisted (expand in future verion)
+        '-bird5.wing.sizes.one',                      // bird5.wing.sizes.one ignored, wing blacklisted
+        // bird5.wing.sizes.two, wing.size blacklisted (expand in future verion)
       ]
     })
-    expect(find5).toEqual({
+    let bird1 = await bird.insert({
+      data: {
+        name: 'ponyo',
+        height: 40,
+        sub: {},
+        wing: { size: 1, sizes: { one: 1, two: 1 }}
+      }
+    })
+    let userData = {
+      dog: 'Bruce',
+      bird1: bird1._id,
+      bird2: bird1._id,
+      bird3: bird1._id,
+      bird4: bird1._id,
+      bird5: bird1._id
+    }
+    let user1 = await user.insert({ data: userData })
+    let bird1Base = { _id: bird1._id, color: 'red', sub: { color: 'red' }}
+
+    // Test bird1
+    expect(await user.findOne({ query: user1._id, populate: ['bird1'] })).toEqual({
+      ...userData,
       _id: user1._id,
-      animals: { dog: 'Max' },
-      hiddenList: [12, 23],
-      hiddenPets: [{ name: 'secretPet' }]
+      bird1: { ...bird1Base, height: 40 },
+    })
+    // Test bird2
+    expect(await user.findOne({ query: user1._id, populate: ['bird2'] })).toEqual({
+      ...userData,
+      _id: user1._id,
+      bird2: { ...bird1Base, height: 40, wing: { size: 1, sizes: { one: 1, two: 1 }} },
+    })
+    // Test bird3
+    expect(await user.findOne({ query: user1._id, populate: ['bird3'] })).toEqual({
+      ...userData,
+      _id: user1._id,
+      bird3: { ...bird1Base, name: 'ponyo', wing: { size: 1, sizes: { one: 1, two: 1 }} },
+    })
+    // Test bird4
+    expect(await user.findOne({ query: user1._id, populate: ['bird4'] })).toEqual({
+      ...userData,
+      _id: user1._id,
+      bird4: { ...bird1Base, name: 'ponyo', height: 40 },
+    })
+    // Test bird5
+    expect(await user.findOne({ query: user1._id, populate: ['bird5'] })).toEqual({
+      ...userData,
+      _id: user1._id,
+      bird5: { ...bird1Base, name: 'ponyo', height: 40 },
     })
 
     db.close()
   })
 
-  test('find blacklisting (default fields)', async () => {
+  test('find blacklisting getProjection', async () => {
+    let db = (await opendb(null)).db
+    // Setup
+    db.model('bird', {
+      fields: {
+        age: { type: 'number' },
+        name: { type: 'string' },
+        wing: {
+          size: { type: 'number' },
+        },
+      },
+      findBL: ['age', 'wing']
+    })
+    db.model('user', {
+      fields: {
+        name: { type: 'string' },
+        bird1: { model: 'bird' },
+      },
+    })
+    // default
+    expect(db.user._getBlacklistProjection('find')).toEqual({
+      'bird1.wing': 0,
+      'bird1.age': 0,
+      'password': 0,
+    })
+    // blacklist /w invalid field (which goes through)
+    expect(db.user._getBlacklistProjection('find', ['name', 'invalidfield'])).toEqual({
+      'bird1.wing': 0,
+      'bird1.age': 0,
+      'invalidfield': 0,
+      'name': 0,
+      'password': 0,
+    })
+    // whitelist
+    expect(db.user._getBlacklistProjection('find', ['-password', '-bird1.age'])).toEqual({
+      'bird1.wing': 0,
+    })
+    // whitelist parent
+    expect(db.user._getBlacklistProjection('find', ['-bird1'])).toEqual({
+      'password': 0,
+    })
+    // whitelist parent, then blacklist child
+    expect(db.user._getBlacklistProjection('find', ['-bird1', 'bird1.name'])).toEqual({
+      'password': 0,
+      'bird1.name': 0,
+    })
+    // the model's blacklists are applied after deep model's
+    db.user.findBL = ['-bird1.age']
+    expect(db.user._getBlacklistProjection('find')).toEqual({
+      'bird1.wing': 0,
+    })
+    // custom blacklists are applied after the model's, which are after deep model's
+    db.user.findBL = ['-bird1.age']
+    expect(db.user._getBlacklistProjection('find', ['bird1'])).toEqual({
+      'bird1': 0,
+    })
+    // blacklisted parent with a blacklisted child
+    expect(db.user._getBlacklistProjection('find', ['bird1', 'bird1.wing'])).toEqual({
+      'bird1': 0,
+    })
+    // A mess of things
+    expect(db.user._getBlacklistProjection('find', ['-bird1', 'bird1.wing', '-bird1.wing','bird1.wing.size'])).toEqual({
+      'bird1.wing.size': 0,
+    })
+    // blacklisted parent with a whitelisted child (expect blacklist expansion in future version?)
+    // expect(db.user._getBlacklistProjection('find', ['bird1', '-bird1.wing'])).toEqual({
+    //   'bird1.age': 0,
+    //   'bird1.name': 0,
+    // })
+
+    db.close()
+  })
+
+  test('find project', async () => {
+    // Test mongodb native project option
     // Setup
     let db = (await opendb(null)).db
     let user = db.model('user', {
@@ -184,7 +329,6 @@ module.exports = function(monastery, opendb) {
     let find1 = await user.findOne({
       query: user1._id,
       project: ['animal.name', 'animals.name']
-      //blacklist: ['animal.color', 'animals']
     })
     expect(find1).toEqual({
       _id: user1._id,
@@ -209,97 +353,74 @@ module.exports = function(monastery, opendb) {
       ]
     })
 
-    // Test exclusion blacklist
-    let find3 = await user.findOne({
-      query: user1._id,
-      blacklist: ['animal.color', 'animals', 'color']
-    })
-    expect(find3).toEqual({
-      _id: user1._id,
-      name: 'Bruce',
-      animal: { name: 'max' }
-    })
-
     db.close()
   })
 
-  test('find blacklisting (populate)', async () => {
+  test('find project population', async () => {
+    // Test mongodb native project option
     // Setup
     let db = (await opendb(null)).db
     let bird = db.model('bird', {
       fields: {
         name: { type: 'string' },
         age: { type: 'number' },
+        height: { type: 'number' },
         color: { type: 'string', default: 'red' },
         sub: {
           color: { type: 'string', default: 'red' },
         }
-      }
+      },
+      findBL: ['age']
     })
     let user = db.model('user', {
       fields: {
         dog: { type: 'string' },
-        myBird: { model: 'bird' },
-        myBird2: { model: 'bird' }
+        bird: { model: 'bird' },
+        bird2: { model: 'bird' },
+        bird3: { model: 'bird' }
       },
+      findBL: [
+        // allll these should be ignored.....?/////
+        'bird.name',                             // bird.name & bird.age blacklisted
+        '-bird2', 'bird2.name',                  // bird2.name blacklisted
+        'bird3.name', '-bird3', 'bird3.height',  // bird3.height blacklisted
+      ]
     })
     let bird1 = await bird.insert({ data: {
       name: 'ponyo',
       age: 3,
+      height: 40,
       sub: {}
     }})
     let user1 = await user.insert({ data: {
       dog: 'Bruce',
-      myBird: bird1._id,
-      myBird2: bird1._id
+      bird: bird1._id,
+      bird2: bird1._id,
+      bird3: bird1._id
     }})
 
     // Test project
     let find1 = await user.findOne({
       query: user1._id,
-      populate: ['myBird', 'myBird2'],
-      project: ['myBird.age', 'myBird2']
+      populate: ['bird', 'bird2'],
+      project: ['bird.age', 'bird2']
     })
     expect(find1).toEqual({
       _id: user1._id,
-      myBird: { age: 3 },
-      myBird2: { _id: bird1._id, age: 3, name: 'ponyo', color: 'red', sub: { color: 'red' }},
+      bird: { age: 3 },
+      bird2: { _id: bird1._id, age: 3, name: 'ponyo', height: 40, color: 'red', sub: { color: 'red' }}
     })
 
     // Test project (different project details)
     let find2 = await user.findOne({
       query: user1._id,
-      populate: ['myBird', 'myBird2'],
-      project: ['myBird', 'myBird2.age']
+      populate: ['bird', 'bird2'],
+      project: ['bird', 'bird2.height']
     })
     expect(find2).toEqual({
       _id: user1._id,
-      myBird: { _id: bird1._id, age: 3, name: 'ponyo', color: 'red', sub: { color: 'red' }},
-      myBird2: { age: 3 },
-    })
-
-    // Test blacklisting
-    let find22 = await user.findOne({
-      query: user1._id,
-      populate: ['myBird', 'myBird2'],
-      blacklist: ['dog', 'myBird2.name', 'myBird2._id']
-    })
-    expect(find22).toEqual({
-      _id: user1._id,
-      myBird: { _id: bird1._id, age: 3, name: 'ponyo', color: 'red', sub: { color: 'red' }},
-      myBird2: { age: 3, color: 'red', sub: { color: 'red' }}
-    })
-
-    // Test blacklisting overrides
-    let find3 = await user.findOne({
-      query: user1._id,
-      populate: ['myBird', 'myBird2'],
-      blacklist: ['dog', 'myBird2.name', 'myBird2._id', '-myBird2']
-    })
-    expect(find3).toEqual({
-      _id: user1._id,
-      myBird: { _id: bird1._id, age: 3, name: 'ponyo', color: 'red', sub: { color: 'red' }},
-      myBird2: { _id: bird1._id, age: 3, name: 'ponyo', color: 'red', sub: { color: 'red' }}
+      bird: { _id: bird1._id, age: 3, name: 'ponyo', height: 40, color: 'red', sub: { color: 'red' }},
+      bird2: { height: 40 },
     })
 
     db.close()

package/test/crud.js

@@ -391,7 +391,10 @@ module.exports = function(monastery, opendb) {
     expect(find1).toEqual({
       _id: inserted2._id,
       name: 'Martin Luther',
-      addresses: [{ city: 'Frankfurt', country: 'Germany' }, { city: 'Christchurch', country: 'New Zealand' }],
+      addresses: [
+        { city: 'Frankfurt', country: 'Germany' },
+        { city: 'Christchurch', country: 'New Zealand' }
+      ],
       address: { country: 'Germany' },
       pet: { dog: { _id: inserted._id, name: 'Scruff', user: inserted2._id }},
       dogs: [{ _id: inserted._id, name: 'Scruff', user: inserted2._id }]
@@ -407,6 +410,7 @@ module.exports = function(monastery, opendb) {
         as: 'dogs'
       }],
       blacklist: ['address', 'addresses.country', 'dogs.name']
+      // ^ great test (address should cancel addresses if not stopping at the .)
     })
     expect(find2).toEqual({
       _id: inserted2._id,

package/test/model.js

@@ -411,169 +411,4 @@ module.exports = function(monastery, opendb) {
     db.close()
   })
 
-  test('model findBL findBLProject', async () => {
-    let db = (await opendb(null)).db
-    db.model('bird', { fields: {
-      name: { type: 'string' }
-    }})
-    let user = db.model('user', {
-      findBL: [
-        'pets.hiddenAge',
-        'animals.hiddenCat',
-        'hiddenDog',
-        'hiddenPets',
-        'hiddenList',
-        'deep.deep2.hiddenDeep3',
-        'hiddenDeeper',
-        'hiddenDeepModel',
-        'hiddenDeepModels'
-      ],
-      fields: {
-        list: [{ type: 'number' }],
-        pet: { type: 'string' },
-        anyPet: { type: 'any' },
-        pets: [{ name: { type: 'string'}, hiddenAge: { type: 'number'} }],
-        animals: {
-          dog: { type: 'string' },
-          hiddenCat: { type: 'string' }
-        },
-        deep: {
-          deep2: {
-            hiddenDeep3: {
-              deep4: { type: 'string' }
-            }
-          }
-        },
-        deepModel: {
-          myBird: { model: 'bird' }
-        },
-        deepModel2: {
-          myBird: { model: 'bird' }
-        },
-        hiddenDog: { type: 'string' },
-        hiddenPets: [{
-          name: { type: 'string'}
-        }],
-        hiddenList: [{ type: 'number'}],
-        hiddenDeeper: {
-          deeper2: {
-            deeper3: {
-              deeper4: { type: 'string' }
-            }
-          }
-        },
-        hiddenDeepModel: {
-          myBird: { model: 'bird' }
-        },
-        hiddenDeepModels: [{ model: 'bird' }]
-      }
-    })
-
-    // Test find whitelist
-    // expect(user.fieldlist.filter(o => !user.findBL.includes(o))).toEqual([
-    //   'list',
-    //   'pet',
-    //   'anyPet',
-    //   'pets.name',
-    //   'animals.dog',
-    //   'deepModel.myBird',
-    //   'deepModel2.myBird'
-    // ])
-
-    // Test findBLProject
-    expect(user.findBLProject).toEqual({
-      'pets.hiddenAge': 0,
-      'animals.hiddenCat': 0,
-      'hiddenDog': 0,
-      'hiddenPets': 0,
-      'hiddenList': 0,
-      'deep.deep2.hiddenDeep3': 0,
-      'hiddenDeeper': 0,
-      'hiddenDeepModel': 0,
-      'hiddenDeepModels': 0
-    })
-
-    // Test inclusion of deep model blacklists
-    var findBLProject = user._addDeepBlacklists(user.findBLProject, [
-      'deepModel.myBird',
-      'hiddenDeepModel.myBird',
-      'hiddenDeepModels',
-      {
-        // raw $lookup object
-        as: 'deepModel2.myBird'
-      }
-    ])
-    expect(findBLProject).toEqual({
-      'pets.hiddenAge': 0,
-      'animals.hiddenCat': 0,
-      'hiddenDog': 0,
-      'hiddenPets': 0,
-      'hiddenList': 0,
-      'deep.deep2.hiddenDeep3': 0,
-      'hiddenDeeper': 0,
-      'hiddenDeepModel': 0,
-      'hiddenDeepModels': 0,
-      // Deep model blacklists
-      'deepModel.myBird.password': 0,
-      'deepModel2.myBird.password': 0
-    })
-
-    // Test whitelisting
-    expect(user._addBlacklist(findBLProject, ['-pets', '-deep.deep2.hiddenDeep3'])).toEqual({
-      // 'pets.hiddenAge': 0,
-      // 'deep.deep2.hiddenDeep3': 0,
-      'animals.hiddenCat': 0,
-      'hiddenDog': 0,
-      'hiddenPets': 0,
-      'hiddenList': 0,
-      'hiddenDeeper': 0,
-      'hiddenDeepModel': 0,
-      'hiddenDeepModels': 0,
-      // Deep model blacklists
-      'deepModel.myBird.password': 0,
-      'deepModel2.myBird.password': 0
-    })
-
-    // Test aggregate with projection exclusions. This is mainly to test how $lookup reacts
-    let bird1 = await db.bird.insert({ data: { name: 'bird1' } })
-    let bird2 = await db.bird.insert({ data: { name: 'bird2' } })
-    let user1 = await db.user.insert({ data: {
-      pet: 'carla',
-      pets: [{ name: 'carla', hiddenAge: 12 }, { name: 'sparky', hiddenAge: 14 }],
-      anyPet: { hi: 1234 },
-      hiddenDeepModel: { myBird: bird1._id },
-      hiddenDeepModels: [bird1._id, bird2._id]
-    }})
-    let res = await db.user._aggregate([
-      { $match: { _id: user1._id } },
-      { $lookup: {
-        from: 'bird',
-        localField: 'hiddenDeepModel.myBird',
-        foreignField: '_id',
-        as: 'hiddenDeepModel.myBird'
-      }},
-      { $lookup: {
-        from: 'bird',
-        localField: 'hiddenDeepModels',
-        foreignField: '_id',
-        as: 'hiddenDeepModels'
-      }},
-      { $project: {
-        'anyPet': 0,
-        'pets.hiddenAge': 0,
-        'hiddenDeepModel.myBird': 0,
-        'hiddenDeepModels.name': 0,
-      }}
-    ])
-    expect(res[0]).toEqual({
-      _id: user1._id,
-      pet: 'carla',
-      pets: [ { name: 'carla' }, { name: 'sparky' } ],
-      hiddenDeepModel: {},
-      hiddenDeepModels: [ { _id: bird1._id }, { _id: bird2._id } ]
-    })
-
-    db.close()
-  })
-
 }

package/test/util.js

@@ -39,4 +39,11 @@ module.exports = function(monastery, opendb) {
     expect(db.isId('5ff50fe955da2c00170de734')).toEqual(true)
   })
 
+  test('utilities arrayWithSchema', async () => {
+    let db = (await opendb(false)).db
+    let res = db.arrayWithSchema([{ name: { type: 'string' }}], { minLength: 1 })
+    expect(res).toContainEqual({ name: { type: 'string' }})
+    expect(res.schema).toEqual({ minLength: 1 })
+  })
+
 }

package/test/validate.js

@@ -1,3 +1,5 @@
+// Todo: split out basic 'type' tests
+
 let validate = require('../lib/model-validate')
 
 module.exports = function(monastery, opendb) {
@@ -51,21 +53,55 @@ module.exports = function(monastery, opendb) {
       meta: { rule: 'isString', model: 'user', field: 'name' }
     })
 
+
     // Type error (date)
-    await expect(user.validate({ name: 'a', date: null })).resolves.toEqual({ name: 'a', date: null })
-    await expect(user.validate({ name: 'a', date: 'fe' })).rejects.toContainEqual({
+    let userdate = db.model('userdate', { fields: { amount: { type: 'date', required: true }}})
+    let userdate2 = db.model('userdate2', { fields: { amount: { type: 'date' }}})
+    await expect(userdate.validate({ amount: 0 })).resolves.toEqual({ amount: 0 })
+    await expect(userdate.validate({ amount: '0' })).resolves.toEqual({ amount: 0 })
+    await expect(userdate.validate({ amount: '1646778655000' })).resolves.toEqual({ amount: 1646778655000 })
+    await expect(userdate2.validate({ amount: '' })).resolves.toEqual({ amount: null })
+    await expect(userdate2.validate({ amount: null })).resolves.toEqual({ amount: null })
+    await expect(userdate.validate({ amount: 'badnum' })).rejects.toEqual([{
       status: '400',
-      title: 'date',
+      title: 'amount',
       detail: 'Value was not a unix timestamp.',
-      meta: { rule: 'isDate', model: 'user', field: 'date' }
-    })
+      meta: { rule: 'isDate', model: 'userdate', field: 'amount' }
+    }])
+    await expect(userdate.validate({ amount: false })).rejects.toEqual([{
+      status: '400',
+      title: 'amount',
+      detail: 'Value was not a unix timestamp.',
+      meta: { rule: 'isDate', model: 'userdate', field: 'amount' }
+    }])
+    await expect(userdate.validate({ amount: undefined })).rejects.toEqual([{
+      status: '400',
+      title: 'amount',
+      detail: 'This field is required.',
+      meta: { rule: 'required', model: 'userdate', field: 'amount' },
+    }])
+    await expect(userdate.validate({ amount: null })).rejects.toEqual([{
+      status: '400',
+      title: 'amount',
+      detail: 'This field is required.',
+      meta: { rule: 'required', model: 'userdate', field: 'amount' },
+    }])
+
 
     // Type error (number)
     let usernum = db.model('usernum', { fields: { amount: { type: 'number', required: true }}})
     let usernum2 = db.model('usernum2', { fields: { amount: { type: 'number' }}})
     await expect(usernum.validate({ amount: 0 })).resolves.toEqual({ amount: 0 })
     await expect(usernum.validate({ amount: '0' })).resolves.toEqual({ amount: 0 })
+    await expect(usernum.validate({ amount: '1646778655000' })).resolves.toEqual({ amount: 1646778655000 })
     await expect(usernum2.validate({ amount: '' })).resolves.toEqual({ amount: null })
+    await expect(usernum2.validate({ amount: null })).resolves.toEqual({ amount: null })
+    await expect(usernum.validate({ amount: 'badnum' })).rejects.toEqual([{
+      status: '400',
+      title: 'amount',
+      detail: 'Value was not a number.',
+      meta: { rule: 'isNumber', model: 'usernum', field: 'amount' }
+    }])
     await expect(usernum.validate({ amount: false })).rejects.toEqual([{
       status: '400',
       title: 'amount',