monastery 3.1.0 → 3.2.0

package/changelog.md

@@ -2,6 +2,10 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [3.2.0](https://github.com/boycce/monastery/compare/3.1.0...3.2.0) (2024-07-17)
+
+### [3.1.1](https://github.com/boycce/monastery/compare/3.1.0...3.1.1) (2024-05-27)
+
 ## [3.1.0](https://github.com/boycce/monastery/compare/3.0.23...3.1.0) (2024-05-27)
 
 ### [3.0.23](https://github.com/boycce/monastery/compare/3.0.22...3.0.23) (2024-05-25)

package/docs/definition/index.md

@@ -48,7 +48,7 @@ Model definition object.
     address: {
       name: { type: 'string' },
       type: { type: 'string' },
-      schema: { minLength: 1 },
+      schema: { strict: false }, // allows non-defined fields to save
     },
     pets: db.arrayWithSchema(
       [{
@@ -135,6 +135,9 @@ fieldType: {
   // below for more information
   index: true|1|-1|'text'|'unique'|Object,
 
+  // Allows non-defined fields to save
+  strict: false,
+
   // The field  won't stored, handy for fields that get populated with documents, see ./find for more details
   virtual: true
 

package/docs/manager/index.md

@@ -50,7 +50,7 @@ db.onError((err) => {
 ### Properties
 
 - `manager.db`: Raw Mongo db instance
-- `manager.client`: Raw Mongo client instance
+- `manager.client`: Raw Mongo client instance, which you can use to create [transactions](#transactions).
 
 ### Methods
 

package/docs/model/find.md

@@ -14,6 +14,7 @@ Find document(s) in a collection, and call the model hook: `afterFind`
 - `query` *(object\|id)*: [`MongoDB query document`](https://www.mongodb.com/docs/v5.0/tutorial/query-documents/), or id
 - [[`blacklist`](#blacklisting)] *(array\|string\|false)*: augment `definition.findBL`. `false` will remove all blacklisting
 - [`getSignedUrls`] *(boolean)*: get signed urls for all image objects
+- [`noDefaults`] *(boolean\|string\|array)*: dont add defaults for any matching paths, e.g. ['pet.name']]
 - [[`populate`](#populate)] *(array)*
 - [`project`] *(string\|array\|object)*: return only these fields, ignores blacklisting
 - [`sort`] *(string\|array\|object)*: same as the mongodb option, but allows string parsing e.g. 'name', 'name:1'

package/docs/model/insert.md

@@ -14,7 +14,7 @@ Validate and insert document(s) in a collection and calls model hooks: `beforeIn
 - `data` *(object\|array)* - Data that is validated against the model fields. Key names can be in dot or bracket notation which is handy for HTML FormData.
 - [[`blacklist`](#blacklisting)] *(array\|string\|false)*: augment `definition.insertBL`. `false` will remove all blacklisting
 - [`project`] *(string\|array\|object)*: project these fields, ignores blacklisting
-- [`skipValidation`] (string\|array\|boolean): skip validation for these field name(s), or `true` for all fields
+- [`skipValidation`] *(string\|array\|boolean)*: skip validation for these fields, or pass `true` to skip all fields and validation hooks
 - [`timestamps`] *(boolean)*: whether `createdAt` and `updatedAt` are automatically inserted, defaults to `manager.timestamps`
 - [[`any mongodb option`](https://mongodb.github.io/node-mongodb-native/5.9/classes/Collection.html#insertMany)] *(any)*
 

package/docs/model/update.md

@@ -15,7 +15,7 @@ Update document(s) in a collection and calls model hooks: `beforeUpdate`,  `afte
 - [`data`](#data) *(object)* - data that's validated against the model fields (always wrapped in `{ $set: .. }`)
 - [[`blacklist`](#blacklisting)]*(array\|string\|false)*: augment `definition.updateBL`. `false` will remove all blacklisting
 - [`project`] *(string\|array\|object)*: project these fields, ignores blacklisting
-- [`skipValidation`] (string\|array\|boolean): skip validation for these field name(s), or `true` for all fields
+- [`skipValidation`] *(string\|array\|boolean)*:  skip validation for these fields, or pass `true` to skip all fields and validation hooks
 - [`sort`] *(string\|object\|array)*: same as the mongodb option, but  allows for string parsing e.g. 'name', 'name:1'
 - [`timestamps`] *(boolean)*: whether `updatedAt` is automatically updated, defaults to the `manager.timestamps` value
 - [[`any mongodb option`](https://mongodb.github.io/node-mongodb-native/5.9/classes/Collection.html#updateMany)] *(any)*

package/docs/model/validate.md

@@ -14,9 +14,9 @@ Validate a model and calls the model hook: `beforeValidate`
 
 [`options`] *(object)*
 
-- [`skipValidation`] (string\|array\/boolean): skip validation for thse field name(s), or `true` for all fields
 - [[`blacklist`](#blacklisting)] *(array\|string\|false)*: augment the model's blacklist. `false` will remove all blacklisting
 - [`project`] *(string\|array\|object)*: project these fields, ignores blacklisting
+- [`skipValidation`] *(string\|array\/boolean)*:  skip validation for these fields, or pass `true` to skip all fields and validation hooks
 - [`timestamps`] *(boolean)*: whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is updated, depending on the `update` value. Defaults to the `manager.timestamps` value
 - [`update`] *(boolean)*: If true, required rules will be skipped, defaults to false
 

package/docs/readme.md

@@ -14,6 +14,7 @@
 * Blacklist sensitive fields once in your model definition, or per operation
 * Model methods can accept data in bracket (multipart/form) and dot notation, you can also mix these together
 * Automatic Mongo index creation
+* Documents validate, insert and update 7-10x faster than Mongoose
 
 #### Why Monastery over Mongoose?
 
@@ -94,6 +95,7 @@ You can view MongoDB's [compatibility table here](https://www.mongodb.com/docs/d
   - db._db moved to db.db
   - db.catch/then() moved to db.onError/db.onOpen()
   - next() is now redundant when returning promises from hooks, e.g. `afterFind: [async (data) => {...}]`
+  - option `skipValidation: true` now skips validation hooks 
 
 ## v2 Breaking Changes
 
@@ -130,6 +132,8 @@ You can view MongoDB's [compatibility table here](https://www.mongodb.com/docs/d
 - ~~Docs: model.methods~~
 - ~~Convert hooks to promises~~
 - ~~added `model.count()`~~
+- Typescript support
+- Add soft remove plugin
 
 ## Debugging
 

package/docs/transactions.md

@@ -0,0 +1,26 @@
+---
+title: Transactions
+nav_order: 8
+---
+
+# Transactions
+
+You can create a Mongo transaction using the `manager.client` instance:
+
+```js
+try {
+  // Create a new session
+  var session = db.client.startSession()
+  // Start the transaction, any thrown errors rollback all operations within the callback. The callback must return a promise.
+  await session.withTransaction(async () => {
+    // Important:: You must pass the session to the operations
+    await db.person.insert({ data, session })
+  })
+} catch (err) {
+  console.error(err)
+} finally {
+  if (session) {
+    await session.endSession()
+  }
+}
+```

package/lib/model-crud.js

@@ -33,7 +33,8 @@ Model.prototype.insert = async function (opts) {
    *     @param {array|string|false} <opts.blacklist> - augment schema.insertBL, `false` will remove blacklisting
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
    *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-   *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
+   *     @param {array|string|true} <opts.skipValidation> - skip validation for these fields, or pass `true` to skip
+   *         all fields and hooks
    *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are automatically inserted
    *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
    *         default, but false on update
@@ -65,8 +66,9 @@ Model.prototype.insert = async function (opts) {
 Model.prototype.find = async function (opts, _one) {
   /**
    * Finds document(s), with auto population
-   * @param {object} opts
+   * @param {object} opts (todo doc getSignedUrls like in the doc)
    *     @param {array|string|false} <opts.blacklist> - augment schema.findBL, `false` will remove all blacklisting
+   *     @param {boolean|string|array} <opts.noDefaults> - dont add defaults for any matching paths, e.g. ['pet.name']
    *     @param {array} <opts.populate> - population, see docs
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
    *     @param {object} <opts.query> - mongodb query object
@@ -148,7 +150,7 @@ Model.prototype.find = async function (opts, _one) {
           }
           if (arrayTarget) {
             // Create lookup
-            lookups.push({ 
+            lookups.push({
               $lookup: {
                 as: path,
                 from: modelName,
@@ -164,7 +166,7 @@ Model.prototype.find = async function (opts, _one) {
             // Convert array into a document for non-array targets
             (opts.addFields = opts.addFields || {})[path] = { '$arrayElemAt': [ '$' + path, 0 ] }
             // Create basic lookup
-            lookups.push({ 
+            lookups.push({
               $lookup: {
                 as: path,
                 from: modelName,
@@ -215,6 +217,7 @@ Model.prototype.findOneAndUpdate = async function (opts) {
    * Find and update document(s) with auto population
    * @param {object} opts
    *     @param {array|string|false} <opts.blacklist> - augment findBL/updateBL, `false` will remove all blacklisting
+   *     @param {boolean|string|array} <opts.noDefaults> - dont add defaults for any matching paths, e.g. ['pet.name']
    *     @param {array} <opts.populate> - find population, see docs
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
    *     @param {object} <opts.query> - mongodb query object
@@ -223,7 +226,8 @@ Model.prototype.findOneAndUpdate = async function (opts) {
    *
    *     Update options:
    *     @param {object|array} opts.data - mongodb document update object(s)
-   *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
+   *     @param {array|string|true} <opts.skipValidation>- skip validation for these fields, or pass `true` to skip
+   *         all fields and hooks
    *     @param {boolean} <opts.timestamps> - whether `updatedAt` is automatically updated
    *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
    *         default, but false on update
@@ -261,7 +265,8 @@ Model.prototype.update = async function (opts, type='update') {
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
    *     @param {object} <opts.query> - mongodb query object
    *     @param {boolean} <opts.respond> - automatically call res.json/error (requires opts.req)
-   *     @param {array|string|true} <opts.skipValidation> - skip validation for this field name(s)
+   *     @param {array|string|true} <opts.skipValidation> - skip validation for these fields, or pass `true` to skip
+   *         all fields and hooks
    *     @param {boolean} <opts.timestamps> - whether `updatedAt` is automatically updated
    *     @param {array|string|false} <opts.validateUndefined> - validates all 'required' undefined fields, true by
    *         default, but false on update
@@ -526,6 +531,9 @@ Model.prototype._queryObject = async function (opts, type, _one) {
       let order = (opts.sort.match(/:(-?[0-9])/) || [])[1]
       opts.sort = { [name]: parseInt(order || 1) }
     }
+    if (util.isString(opts.noDefaults)) {
+      opts.noDefaults = [opts.noDefaults]
+    }   
   }
 
   // Data
@@ -591,6 +599,7 @@ Model.prototype._processAfterFind = async function (data, projection={}, afterFi
   let seriesGroups = []
   let models = this.manager.models
   let isArray = util.isArray(data)
+  const noDefaults = afterFindContext.noDefaults
   const projectionKeys = Object.keys(projection)
   const projectionInclusion = projection[projectionKeys[0]] ? true : false // default false
   if (!isArray) data = [data]
@@ -623,6 +632,9 @@ Model.prototype._processAfterFind = async function (data, projection={}, afterFi
         // Ignore default fields that are blacklisted
         if (this._pathBlacklisted(fullPathWithoutArrays, projectionInclusion, projectionKeys)) continue
 
+        // Ignore default fields if they are included in noDefaults
+        if (noDefaults && (noDefaults === true || this._pathBlacklisted(fullPathWithoutArrays, false, noDefaults))) continue
+
         // Set default value
         const value = util.isFunction(schema.default) ? schema.default(this.manager) : schema.default
         util.setDeepValue(dataRef, localSchemaFieldPath.replace(/\.0(\.|$)/g, '.$$$1'), value, true, false, true)

package/lib/model-validate.js

@@ -9,7 +9,8 @@ Model.prototype.validate = async function (data, opts) {
    * @param {object} <opts>
    *     @param {array|string|false} <opts.blacklist> - augment insertBL/updateBL, `false` will remove blacklisting
    *     @param {array|string} <opts.project> - return only these fields, ignores blacklisting
-   *     @param {array|string|true} <opts.skipValidation> - skip validation on these fields
+   *     @param {array|string|true} <opts.skipValidation> - skip validation for these fields, or pass `true` to skip
+   *         all fields and hooks
    *     @param {boolean} <opts.timestamps> - whether `createdAt` and `updatedAt` are inserted, or `updatedAt` is
    *         updated, depending on the `options.update` value
    *     @param {boolean(false)} <opts.update> - are we validating for insert or update? todo: change to `type`
@@ -25,6 +26,7 @@ Model.prototype.validate = async function (data, opts) {
   opts.update = opts.update || opts.findOneAndUpdate
   opts.insert = !opts.update
   opts.skipValidation = opts.skipValidation === true ? true : util.toArray(opts.skipValidation||[])
+  if (opts.skipValidation === true) return data
 
   // Get projection
   if (opts.project) var projectionValidate = this._getProjectionFromProject(opts.project)
@@ -103,7 +105,6 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
       if (fieldName == 'schema') continue
       // if (!parentPath && fieldName == 'categories') console.time(fieldName)
       // if (!parentPath && fieldName == 'categories') console.time(fieldName + 1)
-      let verrors = []
       let schema = field.schema
       let value = fieldsIsArray ? item : (item||{})[fieldName]
       let indexOrFieldName = fieldsIsArray ? i : fieldName
@@ -139,7 +140,8 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
       // Field is a subdocument
       if (schema.isObject) {
         // Object schema errors
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
+        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
+        if (verrors.length) errors.push(...verrors)
         // Recurse if inserting, value is a subdocument, or a deep property (todo: not dot-notation)
         if (
           opts.insert ||
@@ -147,7 +149,7 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
           (util.isDefined(opts.validateUndefined) ? opts.validateUndefined : (path||'').indexOf('.') !== -1)
         ) {
           var res = this._validateFields(dataRoot, field, value, opts, path, path2)
-          errors.push(...res[0])
+          if (res[0].length) errors.push(...res[0])
         }
         if (util.isDefined(value) && !verrors.length) {
           data2[indexOrFieldName] = res? res[1] : value
@@ -156,11 +158,12 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
       // Field is an array
       } else if (schema.isArray) {
         // Array schema errors
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
+        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
+        if (verrors.length) errors.push(...verrors)
         // Data value is array too
         if (util.isArray(value)) {
           var res2 = this._validateFields(dataRoot, field, value, opts, path, path2)
-          errors.push(...res2[0])
+          if (res2[0].length) errors.push(...res2[0])
         }
         if (util.isDefined(value) && !verrors.length) {
           data2[indexOrFieldName] = res2? res2[1] : value
@@ -168,7 +171,8 @@ Model.prototype._validateFields = function (dataRoot, fields, data, opts, parent
 
       // Field is a field-type/field-schema
       } else {
-        errors.push(...(verrors = this._validateRules(dataRoot, schema, value, opts, path)))
+        const verrors = this._validateRules(dataRoot, schema, value, opts, path)
+        if (verrors.length) errors.push(...verrors)
         if (util.isDefined(value) && !verrors.length) data2[indexOrFieldName] = value
       }
       // if (!parentPath && fieldName == 'categories') console.timeEnd(fieldName)
@@ -212,9 +216,11 @@ Model.prototype._validateRules = function (dataRoot, fieldSchema, value, opts, p
       // Make sure there is numerical character representing arrays
       let skippedFieldChunks = skippedField.split('.')
       for (let i=0, l=pathChunks.length; i<l; i++) {
-        if (pathChunks[i].match(/^[0-9]+$/)
-            && skippedFieldChunks[i]
-            && !skippedFieldChunks[i].match(/^\$$|^[0-9]+$/)) {
+        if (
+          pathChunks[i].match(/^[0-9]+$/)
+          && skippedFieldChunks[i]
+          && !skippedFieldChunks[i].match(/^\$$|^[0-9]+$/)
+        ) {
           skippedFieldChunks.splice(i, 0, '$')
         }
       }

package/lib/util.js

@@ -35,7 +35,7 @@ module.exports = {
     let obj2 = Array.isArray(obj)? [] : {}
     for (let key in obj) {
       let v = obj[key]
-      obj2[key] = (typeof v === 'object' && !this.isIdFast(v))? this.deepCopy(v) : v
+      obj2[key] = (typeof v === 'object' && !this.isHex24(v))? this.deepCopy(v) : v //isHex24
     }
     return obj2
   },
@@ -113,15 +113,23 @@ module.exports = {
     else return false
   },
 
-  isIdFast: function(value) {
-    // Check quickly if the value is an ObjectId. We can use db.isId() but this may be slower
-    // console.log(isId('66333b1b3343d7e3b200005b')) = true
-    // console.log(isId(db.id())) = true
-    // console.log(isId(null)) = undefined
-    // console.log(isId('qwefqwefqwef')) = undefined
-    // console.log(isId({})) = undefined
-    // console.log(isId(['66333b1b3343d7e3b200005b'])) = undefined
-    if ((value||'').toString()?.match(/^[0-9a-fA-F]{24}$/) && !this.isArray(value)) return true
+  isHex24: (value) => {
+    // Fast function to check if the length is exactly 24 and all characters are valid hexadecimal digits
+    const str = (value||'').toString()
+    if (str.length !== 24) return false
+    else if (Array.isArray(value)) return false
+    
+    // Check if all characters are valid hexadecimal digits
+    for (let i=24; i--;) {
+      const charCode = str.charCodeAt(i)
+      const isDigit = charCode >= 48 && charCode <= 57     // '0' to '9'
+      const isLowerHex = charCode >= 97 && charCode <= 102 // 'a' to 'f'
+      const isUpperHex = charCode >= 65 && charCode <= 70  // 'A' to 'F'
+      if (!isDigit && !isLowerHex && !isUpperHex) {
+        return false
+      }
+    }
+    return true
   },
 
   isNumber: function(value) {

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A simple, straightforward MongoDB ODM",
   "author": "Ricky Boyce",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",
@@ -20,6 +20,7 @@
   ],
   "scripts": {
     "dev": "npm run lint & jest --watchAll --runInBand --verbose=false",
+    "dev-profile": "node --inspect-brk ./node_modules/jest/bin/jest.js --runInBand",
     "docs": "cd docs && bundle exec jekyll serve --livereload --livereload-port 4001",
     "lint": "eslint ./lib ./plugins ./test",
     "mona": "nodemon resources/mona.js",

package/test/comparison.js

@@ -0,0 +1,122 @@
+const monastery = require('../lib/index.js')
+const mongoose = require('mongoose') // very slow initialisation
+
+test('comparison insert', async () => {
+  const db = monastery('127.0.0.1/monastery', { timestamps: false }) 
+  const Test1 = db.model('Test1', { fields: {
+    raw: {
+      words: [{
+        id: { type: 'number' },
+        isVertical: { type: 'boolean' },
+        text: { type: 'string' },
+        confidence: { type: 'number' },
+      }],
+    },
+  }})
+  function getData() {
+    return {
+      words: Array(500).fill(0).map((_, i) => ({
+        id: i,
+        isVertical: i % 2 === 0,
+        text: 'Test ' + i,
+        confidence: Math.random(),
+        sub: {
+          greeting: 'Hello ' + i,
+        },
+      })),
+    }
+  }
+  console.time('Monastery')
+  for (let i = 0; i < 100; ++i) {
+    const res = await Test1.insert({ data: { raw: getData() } })
+    if (i == 50) console.log(res.raw.words[0])
+  }
+  console.timeEnd('Monastery') // 320ms =50,  935ms =500
+  db.close()
+
+  
+  await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_test')
+  const schema = new mongoose.Schema({
+    raw: {
+      words: [{
+        id: Number,
+        isVertical: Boolean,
+        text: String,
+        confidence: Number,
+      }],
+    },
+  })
+  const Test2 = mongoose.model('Test2', schema)
+  console.time('Mongoose')
+  for (let i = 0; i < 100; ++i) {
+    const res = await Test2.create({ raw: getData() })
+    if (i == 50) console.log(res.raw.words[0])
+  }
+  console.timeEnd('Mongoose') // 1202ms =50, 9203ms =500
+  //result: monastery is 4.5-10x faster than mongoose
+}, 20000)
+
+test('comparison validate', async () => {
+  function getData() {
+    return {
+      words: Array(5000).fill(0).map((_, i) => ({
+        id: i,
+        isVertical: i % 2 === 0,
+        // text: 'Test ' + i,
+        confidence: Math.random(),
+        sub: {
+          greeting: 'Hello ' + i,
+        },
+      })),
+    }
+  }
+  const db = monastery('127.0.0.1/monastery', { timestamps: false }) 
+  const Test1 = db.model('Test1', { fields: {
+    raw: {
+      words: [{
+        id: { type: 'number' },
+        isVertical: { type: 'boolean' },
+        text: { type: 'string', default: 'hi'},
+        confidence: { type: 'number' },
+        sub: {
+          greeting: { type: 'string' },
+        },
+      }],
+    },
+  }})
+  const data1 = getData()
+  console.time('Monastery')
+  const test1 = await Test1.validate({
+    raw: data1,
+  })
+  console.log(test1.raw.words[0])
+  console.timeEnd('Monastery') // 59ms =5000, 495ms =50000  
+
+  await mongoose.connect('mongodb://127.0.0.1:27017/mongoose_test')
+  const schema = new mongoose.Schema({
+    raw: {
+      words: [{
+        id: Number,
+        isVertical: Boolean,
+        text: { type: String, default: 'hi' },
+        confidence: Number,
+        sub: {
+          greeting: String,
+        },
+      }],
+    },
+  })
+  const Test2 = mongoose.model('Test2', schema)
+
+  const data2 = getData()
+  console.time('Mongoose 1')
+  const test2 = new Test2({ raw: data2 }) // calls validate too
+  console.timeEnd('Mongoose 1') // 429ms =5000, 3722ms =50000
+  console.time('Mongoose 2')
+  test2.validateSync()
+  console.timeEnd('Mongoose 2') // 284ms =5000, 2486ms =50000
+  console.log(test2.raw.words[0])
+
+  //result: monastery is 7.3x faster than mongoose
+}, 20000)
+

package/test/crud.js

@@ -337,9 +337,100 @@ test('find default field blacklisted', async () => {
   })
 })
 
+test('find default field population with noDefaults', async () => {
+  // similar to "find default field population"
+  db.model('user', {
+    fields: {
+      name: { type: 'string', default: 'Martin Luther' },
+      addresses: [{ city: { type: 'string' }, country: { type: 'string', default: 'Germany' } }],
+      address: { country:  { type: 'string', default: 'Germany' }},
+      pet:  { dog: { model: 'dog' }},
+      pets:  { dog: [{ model: 'dog' }]},
+      dogs: [{ model: 'dog' }], // virtual association
+    },
+  })
+  db.model('dog', {
+    fields: {
+      name: { type: 'string', default: 'Scruff' },
+      user: { model: 'user' },
+    },
+  })
+
+  // Default field population test
+  // Insert documents (without defaults)
+  let dog1 = await db.dog._insert({})
+  let dog2 = await db.dog._insert({})
+  let user1 = await db.user._insert({
+    addresses: [
+      { city: 'Frankfurt' },
+      { city: 'Christchurch', country: 'New Zealand' },
+    ],
+    pet: { dog: dog1._id },
+    pets: { dog: [dog1._id, dog2._id]},
+  })
+  await db.dog._update(dog1._id, { $set: { user: user1._id }})
+
+  let find1 = await db.user.findOne({
+    query: user1._id,
+    populate: ['pet.dog', 'pets.dog', {
+      from: 'dog',
+      localField: '_id',
+      foreignField: 'user',
+      as: 'dogs',
+    }],
+    noDefaults: true,
+  })
+
+  expect(find1).toEqual({
+    _id: user1._id,
+    addresses: [
+      { city: 'Frankfurt' },
+      { city: 'Christchurch', country: 'New Zealand' },
+    ],
+    pet: { dog: { _id: dog1._id, user: user1._id }},
+    pets: {
+      dog: [
+        { _id: dog1._id, user: user1._id },
+        { _id: dog2._id },
+      ],
+    },
+    dogs: [{ _id: dog1._id, user: user1._id }],
+  })
+
+  let find2 = await db.user.findOne({
+    query: user1._id,
+    populate: ['pet.dog', 'pets.dog', {
+      from: 'dog',
+      localField: '_id',
+      foreignField: 'user',
+      as: 'dogs',
+    }],
+    noDefaults: ['dogs', 'pet.dog'],
+  })
+
+  expect(find2).toEqual({
+    _id: user1._id,
+    name: 'Martin Luther',
+    addresses: [
+      { city: 'Frankfurt', country: 'Germany' },
+      { city: 'Christchurch', country: 'New Zealand' },
+    ],
+    address: { country: 'Germany' },
+    pet: { dog: { _id: dog1._id, user: user1._id }},
+    pets: {
+      dog: [
+        { _id: dog1._id, name: 'Scruff', user: user1._id },
+        { _id: dog2._id, name: 'Scruff' },
+      ],
+    },
+    dogs: [{ _id: dog1._id, user: user1._id }],
+  })
+})
+
 test('update general', async () => {
   let user = db.model('user', {
     fields: {
+      age: { type: 'number' },
       name: { type: 'string' },
     },
   })
@@ -364,8 +455,8 @@ test('update general', async () => {
   ])
 
   // Update
-  await expect(user.update({ query: inserted._id, data: { name: 'Martin Luther2' }}))
-    .resolves.toEqual({ name: 'Martin Luther2' })
+  await expect(user.update({ query: inserted._id, data: { name: 'Martin Luther2', age: Infinity }}))
+    .resolves.toEqual({ name: 'Martin Luther2', age: Infinity })
 
   // Update (no/empty data object)
   await expect(user.update({ query: inserted._id, data: {}}))
@@ -590,20 +681,20 @@ test('update large document', async () => {
   // Insert
   let inserted = await large._insert({})
   // Update
-  // console.time('update large document')
+  console.time('update large document')
   let update = await large.update({
     query: inserted._id,
     data: largePayload,
   })
-  // console.timeEnd('update large document')
+  console.timeEnd('update large document')
   // Check
   await expect(update).toEqual(removePrunedProperties(largePayload))
   // Find
-  // console.time('find large document')
-  // await large.findOne({
-  //   query: inserted._id,
-  // })
-  // console.timeEnd('find large document')
+  console.time('find large document')
+  await large.findOne({
+    query: inserted._id,
+  })
+  console.timeEnd('find large document')
 
   function removePrunedProperties(entity) {
     for (let entitiesKey of [
@@ -633,6 +724,71 @@ test('update large document', async () => {
   }
 })
 
+// test('update larger document', async () => {
+//   // todo: sereach util.deepCopy
+//   // todo: check castIds and any other recursive functions
+//   // todo: move default fields to before validate
+//   db.model('a', { fields: {} })
+//   db.model('b', { fields: {} })
+//   db.model('c', { fields: {} })
+//   db.model('d', { fields: {} })
+//   db.model('e', { fields: {} })
+//   try {
+//     var larger = db.model('larger', require('../resources/fixtures/larger-definition.js'))
+//     var largerPayload = require('../resources/fixtures/larger-payload.json').version
+//   } catch (e) {
+//     // ignore publicly for now
+//     return
+//   }
+//   // Validate
+//   console.time('validate larger document')
+//   let validate = await larger.validate(largerPayload, { update: true })
+//   console.timeEnd('validate larger document')
+//   // // Insert
+//   // let inserted = await larger._insert({})
+//   // // Update
+//   // console.time('update larger document')
+//   // let update = await larger.update({
+//   //   query: inserted._id,
+//   //   data: largerPayload,
+//   // })
+//   // console.timeEnd('update larger document')
+//   // // Check
+//   // await expect(update).toEqual(convertIdsToObjectIdsAndRemoveActuallyValues(largerPayload))
+//   // // Find
+//   // console.time('find larger document')
+//   // await larger.findOne({
+//   //   query: inserted._id,
+//   // })
+//   // console.timeEnd('find larger document')
+
+//   // function convertIdsToObjectIdsAndRemoveActuallyValues(entity) {
+//   //   for (let entitiesKey of [
+//   //     'components', 'connections', 'bridges', 'openings', 'spaces', 'elements', 'elementTypes', 'categories', 
+//   //     'typologies',
+//   //   ]) {
+//   //     if (entity[entitiesKey]) {
+//   //       for (let i=0, l=entity[entitiesKey].length; i<l; i++) {
+//   //         entity[entitiesKey][i] = convertIdsToObjectIdsAndRemoveActuallyValues(entity[entitiesKey][i])
+//   //       }
+//   //     }
+//   //   }
+//   //   // remove index and actually keys
+//   //   if (entity.metrics) {
+//   //     delete entity.index
+//   //     for (let key in entity.metrics) {
+//   //       if (entity.metrics[key].S_ == 'A_') delete entity.metrics[key].S_
+//   //       delete entity.metrics[key].A_
+//   //     }
+//   //   }
+//   //   // convert _id to ObjectId
+//   //   if (entity._id) {
+//   //     entity._id = db.id(entity._id)
+//   //   }
+//   //   return entity
+//   // }
+// })
+
 test('findOneAndUpdate general', async () => {
   // todo: test all findOneAndUpdate options (e.g. array population)
   // todo: test find & update hooks

package/test/index.test.js

@@ -25,3 +25,4 @@ require('./populate.js')
 require('./validate.js')
 require('./plugin-images.js')
 require('./virtuals.js')
+// require('./comparison.js')

package/test/util.js

@@ -35,9 +35,25 @@ test('util > isId', async () => {
   expect(util.isId('')).toEqual(false)
   expect(util.isId(1234)).toEqual(false)
   expect(util.isId('1234')).toEqual(false)
+  expect(util.isId(null)).toEqual(false)
+  expect(util.isId({})).toEqual(false)
+  expect(util.isId(['5ff50fe955da2c00170de734'])).toEqual(false)
   expect(util.isId('5ff50fe955da2c00170de734')).toEqual(true)
+  expect(util.isId(monastery.prototype.id())).toEqual(true)
 })
 
+test('util > isHex24', async () => {
+  expect(util.isHex24('')).toEqual(false)
+  expect(util.isHex24(1234)).toEqual(false)
+  expect(util.isHex24('1234')).toEqual(false)
+  expect(util.isHex24(null)).toEqual(false)
+  expect(util.isHex24({})).toEqual(false)
+  expect(util.isHex24(['5ff50fe955da2c00170de734'])).toEqual(false)
+  expect(util.isHex24('5ff50fe955da2c00170de734')).toEqual(true)
+  expect(util.isHex24(monastery.prototype.id())).toEqual(true)
+})
+
+
 test('util > arrayWithSchema', async () => {
   let res = monastery.prototype.arrayWithSchema([{ name: { type: 'string' }}], { minLength: 1 })
   expect(res).toContainEqual({ name: { type: 'string' }})

package/test/validate.js

@@ -787,6 +787,7 @@ test('validation custom rules', async () => {
 
 test('validated data', async () => {
   let fields = {
+    age: { type: 'number' },
     name: { type: 'string' },
     names: [{ type: 'string' }],
     animals: {
@@ -808,6 +809,12 @@ test('validated data', async () => {
   await expect(user.validate({ name: 'Martin Luther' })).resolves.toEqual({ name: 'Martin Luther' })
   await expect(user.validate({ name: null })).resolves.toEqual({ name: null })
 
+  // Number data
+  await expect(user.validate({ age: 1 })).resolves.toEqual({ age: 1 })
+  await expect(user.validate({ age: Infinity })).resolves.toEqual({ age: Infinity })
+  await expect(user.validate({ age: NaN })).resolves.toEqual({ age: NaN })
+  await expect(user.validate({ age: null })).resolves.toEqual({ age: null })
+
   // Array data
   await expect(user.validate({ names: ['blue'] })).resolves.toEqual({ names: ['blue'] })