monastery 3.0.22 → 3.1.0

package/lib/model.js

@@ -21,6 +21,11 @@ function Model(name, opts, manager) {
   } else if (!opts.fields) {
     throw `We couldn't find ${name}.fields in the model definition, the model maybe setup ` 
       + `or exported incorrectly:\n${JSON.stringify(opts, null, 2)}`
+  } else if (!util.isSubdocument(opts.fields) && opts.fields.type == 'any') {
+    throw `Instead of using { type: 'any' } for ${name}.fields, please use the new 'strict' definition rule` +
+      ', e.g. { schema: { strict: false }}'
+  } else if (!util.isSubdocument(opts.fields) && !util.isEmpty(opts.fields)) {
+    throw `The ${name}.fields object should be a valid document, e.g. { name: { type: 'string' }}`
   }
 
   // Add schema options
@@ -107,8 +112,11 @@ function Model(name, opts, manager) {
   }, this)
 
   // Extend default fields with passed in fields and check for invalid fields
-  this._setupFields(this.fields = Object.assign({}, this._defaultFields, this.fields))
+  this._setupFields(
+    this.fields = util.isSchema(this.fields) ? this.fields : Object.assign({}, this._defaultFields, this.fields)
+  )
   this.fieldsFlattened = this._getFieldsFlattened(this.fields, '') // test output?
+  this.modelFieldsArray = this._getModelFieldsArray()
 
   // Get collection, and extend model with collection methods
   this.collection = this.manager.get(name, { castIds: false })
@@ -138,115 +146,158 @@ function Model(name, opts, manager) {
 
 Model.prototype._getFieldsFlattened = function(fields, path) {
   /**
-   * Flatten fields
+   * Get flattened fields
    * @param {object|array} fields - can be a nested subdocument or array
    * @param {string} path
    * @return {object} e.g. {'name': Schema, 'pets.dog': Schema}
    */
   let obj = {}
   util.forEach(fields, function(field, fieldName) {
-    let newPath = /*util.isArray(fields)? path : */path + fieldName + '.'
-    if (fieldName == 'schema') {
-      return
-    } else if (util.isArray(field)) {
+    const schema = field.schema
+    const newPath = /*util.isArray(fields)? path : */path + fieldName + '.'
+    if (fieldName == 'schema') return
+    if (schema.isArray) {
       Object.assign(obj, this._getFieldsFlattened(field, newPath))
-    } else if (util.isSubdocument(field)) {
+    } else if (schema.isObject) {
       Object.assign(obj, this._getFieldsFlattened(field, newPath))
     } else {
-      obj[newPath.replace(/\.$/, '')] = field
+      obj[newPath.replace(/\.$/, '')] = schema
     }
   }, this)
   return obj
 }
 
-Model.prototype._setupFields = function(fields) {
+Model.prototype._getModelFieldsArray = function() {
+  /**
+   * Get all the model fields in an array
+   * @return {array} e.g. [{ path: 'pets.0.dog', 'path2': 'pets.dog'}, ...]
+   */
+  return Object.keys(this.fieldsFlattened).reduce((acc, path) => {
+    if (this.fieldsFlattened[path].model) {
+      acc.push({ path: path, path2: path.replace(/\.[0-9]+(\.|$)/g, '$1') })
+    }
+    return acc
+  }, [])
+},
+
+Model.prototype._setupFields = function(fields, isSub) {
   /**
    * Check for invalid rules on a field object, and set field.isType
    * @param {object|array} fields - subsdocument or array
    */
+  // We need to allow the processing of the root schema object
+  if (!isSub) fields = { fields }
+  
   util.forEach(fields, function(field, fieldName) {
     // Schema field
+    if (fieldName == 'schema') return
     if (util.isSchema(field)) {
-      // No image schema pre-processing done yet by a plugin
-      if (field.type == 'image' && !field.image) field.image = true, field.type = 'any'
-      if (field.model) field.type = 'id'
-      let isType = 'is' + util.ucFirst(field.type)
+      const schema = field
+      fields[fieldName] = field = { schema }
 
-      // No type defined
-      if (!field.type) {
+      // Type 'model'
+      if (schema.model) {
+        schema.type = 'id'
+
+      // Type 'image', but no image plugin schema processing done, e.g. image plugin not setup
+      } else if (schema.type == 'image' && !schema.image) {
+        schema.image = true
+        schema.type = 'any'
+
+      // No type
+      } else if (!schema.type) {
         this.error(`No type defined on "${this.name}" for field "${fieldName}". Defaulting to string.`)
-        field.type = 'string'
+        schema.type = 'string'
+      }
 
-      // Type doesn't exist
-      } else if (!this.rules[isType] && !rules[isType]) {
-        this.error(`Not a valid type "${field.type}" defined on "${this.name}" for field "${fieldName}".
+      // Type isn't a rule
+      const isType = schema.isType = 'is' + util.ucFirst(schema.type)
+      if (!this.rules[isType] && !rules[isType]) {
+        this.error(`Not a valid type "${schema.type}" defined on "${this.name}" for field "${fieldName}".
           Defaulting to string.`)
-        field.type = 'string'
+         schema.type = 'string'
       }
 
-      // Convert type into a is{type} rule
-      field[isType] = true
+      field.schema = {
+        ...field.schema,
+        [isType]: true, // e.g. isString rule
+        isSchema: true,
+      }
 
-      // Rule doesn't exist
-      util.forEach(field, (rule, ruleName) => {
-        if ((this.rules[ruleName] || rules[ruleName]) && this._ignoredRules.indexOf(ruleName) != -1) {
-          this.error(`The rule name "${ruleName}" for the model "${this.name}" is a reserved keyword, ignoring rule.`)
-        }
-        if (!this.rules[ruleName] && !rules[ruleName] && this._ignoredRules.indexOf(ruleName) == -1) {
-          // console.log(field)
-          this.error(`No rule "${ruleName}" exists for model "${this.name}", ignoring rule.`)
-          delete field[ruleName]
-        }
-      }, this)
+      // Remove invalid rules
+      this._removeInvalidRules(field)
 
     // Misused schema property
-    } else if (fieldName == 'schema') {
-      this.error(`Invalid schema on model "${this.name}", remember 'schema' is a reserverd property, ignoring field.`)
+    } else if (fieldName == 'schema' || fieldName == 'isSchema') {
+      this.error(`Invalid field '${fieldName}' on model '${this.name}', this is a reserved property, ignoring field.`)
       delete fields[fieldName]
 
     // Fields be an array
     } else if (util.isArray(field)) {
-      let arrayDefault = this.manager.opts.defaultObjects? () => [] : undefined
-      let nullObject = this.manager.opts.nullObjects
-      let virtual = field.length == 1 && (field[0]||{}).virtual ? true : undefined
-      field.schema = {
+      this._removeInvalidRules(field)
+      field.schema = util.removeUndefined({
         type: 'array',
         isArray: true,
-        default: arrayDefault,
-        nullObject: nullObject,
-        virtual: virtual,
+        isSchema: true,
+        isType: 'isArray',
+        default: this.manager.opts.defaultObjects? () => [] : undefined,
+        nullObject: this.manager.opts.nullObjects,
+        virtual: field.length == 1 && (field[0] || {}).virtual ? true : undefined,
         ...(field.schema || {}),
-      }
-      this._setupFields(field)
+      })
+      this._setupFields(field, true)
 
     // Fields can be a subdocument, e.g. user.pet = { name: {}, ..}
     } else if (util.isSubdocument(field)) {
-      let objectDefault = this.manager.opts.defaultObjects? () => ({}) : undefined
-      let nullObject = this.manager.opts.nullObjects
       let index2dsphere = util.isSubdocument2dsphere(field)
       field.schema = field.schema || {}
       if (index2dsphere) {
         field.schema.index = index2dsphere
         delete field.index
       }
-      field.schema = { 
+      this._removeInvalidRules(field)
+      field.schema = util.removeUndefined({ 
         type: 'object', 
         isObject: true, 
-        default: objectDefault, 
-        nullObject: nullObject, 
+        isSchema: true,
+        isType: 'isObject',
+        default: this.manager.opts.defaultObjects? () => ({}) : undefined, 
+        nullObject: this.manager.opts.nullObjects, 
         ...field.schema, 
-      }
-      this._setupFields(field)
+      })
+      this._setupFields(field, true)
     }
   }, this)
 },
 
+Model.prototype._removeInvalidRules = function(field) {
+  /**
+   * Remove invalid rules on a field object
+   * @param {object} field
+   * @return {object} field
+   **/
+  for (let ruleName in (field||{}).schema) {
+    const ruleFn = this.rules[ruleName] || rules[ruleName]
+    // Rule doesn't exist
+    if (!ruleFn && this._ignoredRules.indexOf(ruleName) == -1) {
+      // console.log(field.schema)
+      this.error(`No rule "${ruleName}" exists for model "${this.name}", ignoring rule.`)
+      delete field.schema[ruleName]
+    }
+    // Reserved rule
+    if (this.rules[ruleName] && this._ignoredRules.indexOf(ruleName) != -1) {
+      this.error(`The rule "${ruleName}" for the model "${this.name}" is a reserved keyword, ignoring custom rule function.`)
+    }
+  }
+  return field
+},
+
 Model.prototype._setupIndexes = async function(fields, opts={}) {
   /**
    * Creates indexes for the model (multikey, and sub-document supported)
    * Note: the collection be created beforehand???
    * Note: only one text index per model(collection) is allowed due to mongodb limitations
-   * @param {object} <fields>
+   * @param {object} <fields> - processed or unprocessed fields, e.g. {schema: {name: {index}}} or {name: {index}}
    * @return Promise( {array} indexes ensured ) || error
    *
    * MongoDB index structures = [
@@ -267,8 +318,16 @@ Model.prototype._setupIndexes = async function(fields, opts={}) {
     throw new Error(`Skipping createIndex on the '${model.name||''}' model, no mongodb connection found.`)
   }
 
+  // Process custom 'unprocessed' fields
+  if (fields && !(fields[Object.keys(fields)[0]].schema||{}).isSchema) {
+    fields = util.deepCopy(fields)
+    this._setupFields(fields)
+    // console.dir(fields, { depth: null })
+  }
+
   // Find all indexes
   recurseFields(fields || model.fields, '')
+
   // console.log(2, indexes, fields)
   if (hasTextIndex) indexes.push(textIndex)
   if (opts.dryRun) return indexes || []
@@ -313,14 +372,19 @@ Model.prototype._setupIndexes = async function(fields, opts={}) {
   model.info('db index(s) created for ' + model.name)
   return indexes
 
-  function recurseFields(fields, parentPath) {
-    util.forEach(fields, (field, name) => {
-      let index = field.index
+  function recurseFields(schemaFields, parentPath) {
+    /**
+     * Recursively find fields with an index property
+     * @param {object} schemaFields
+     */
+    util.forEach(schemaFields, (field, fieldName) => {
+      const index = (field.schema||{}).index
+      if (fieldName == 'schema') return
       if (index) {
         let options = util.isObject(index)? util.omit(index, ['type']) : {}
         let type = util.isObject(index)? index.type : index
-        let path = name == 'schema'? parentPath.slice(0, -1) : parentPath + name
-        let path2 = path.replace(/(^|\.)[0-9]+(\.|$)/g, '$2') // no numirical keys, e.g. pets.1.name
+        let path = parentPath + fieldName
+        let path2 = path.replace(/(^|\.)[0-9]+(\.|$)/g, '$2') // no numirical keys, e.g. pets.1.fieldName
         if (type === true) type = 1
         if (type == 'text') {
           hasTextIndex = textIndex.key[path2] = 'text'
@@ -331,10 +395,11 @@ Model.prototype._setupIndexes = async function(fields, opts={}) {
           indexes.push({ name: `${path2}_1`, key: { [path2]: 1 }, unique: true, ...options  })
         }
       }
-      if (util.isSubdocument(field)) {
-        recurseFields(field, parentPath + name + '.')
-      } else if (util.isArray(field)) {
-        recurseFields(field, parentPath + name + '.')
+      if (field.schema.isObject) {
+        recurseFields(field, parentPath + fieldName + '.')
+
+      } else if (field.schema.isArray) {
+        recurseFields(field, parentPath + fieldName + '.')
       }
     })
   }

package/lib/util.js

@@ -35,8 +35,7 @@ module.exports = {
     let obj2 = Array.isArray(obj)? [] : {}
     for (let key in obj) {
       let v = obj[key]
-      if (this.isId(v)) obj2[key] = v//.toString()
-      else obj2[key] = (typeof v === 'object')? this.deepCopy(v) : v
+      obj2[key] = (typeof v === 'object' && !this.isIdFast(v))? this.deepCopy(v) : v
     }
     return obj2
   },
@@ -114,6 +113,17 @@ module.exports = {
     else return false
   },
 
+  isIdFast: function(value) {
+    // Check quickly if the value is an ObjectId. We can use db.isId() but this may be slower
+    // console.log(isId('66333b1b3343d7e3b200005b')) = true
+    // console.log(isId(db.id())) = true
+    // console.log(isId(null)) = undefined
+    // console.log(isId('qwefqwefqwef')) = undefined
+    // console.log(isId({})) = undefined
+    // console.log(isId(['66333b1b3343d7e3b200005b'])) = undefined
+    if ((value||'').toString()?.match(/^[0-9a-fA-F]{24}$/) && !this.isArray(value)) return true
+  },
+
   isNumber: function(value) {
     return !isNaN(parseFloat(value)) && isFinite(value)
   },

package/package.json

@@ -2,7 +2,7 @@
   "name": "monastery",
   "description": "⛪ A simple, straightforward MongoDB ODM",
   "author": "Ricky Boyce",
-  "version": "3.0.22",
+  "version": "3.1.0",
   "license": "MIT",
   "repository": "github:boycce/monastery",
   "homepage": "https://boycce.github.io/monastery/",

package/plugins/images/index.js

@@ -482,27 +482,27 @@ let plugin = module.exports = {
     })).then(() => validFiles)
   },
 
-  _findAndTransformImageFields: function(fields, path) {
+  _findAndTransformImageFields: function(unprocessedFields, path) {
     /**
-     * Returns a list of valid image fields
-     * @param {object|array} fields
+     * Returns a list of valid image field schemas
+     * @param {object|array} unprocessedFields - fields not yet setup
      * @param {string} path
      * @return [{}, ...]
      * @this plugin
      */
     let list = []
     let that = this
-    util.forEach(fields, (field, fieldName) => {
+    util.forEach(unprocessedFields, (field, fieldName) => {
       let path2 = `${path}.${fieldName}`.replace(/^\./, '')
-      // let schema = field.schema || {}
+      if (fieldName == 'schema') return
 
       // Subdocument field
-      if (util.isSubdocument(field)) {//schema.isObject
+      if (util.isSubdocument(field)) {
         // log(`Recurse 1: ${path2}`)
         list = list.concat(plugin._findAndTransformImageFields(field, path2))
 
       // Array field
-      } else if (util.isArray(field)) {//schema.isArray
+      } else if (util.isArray(field)) {
         // log(`Recurse 2: ${path2}`)
         list = list.concat(plugin._findAndTransformImageFields(field, path2))
 
@@ -532,15 +532,15 @@ let plugin = module.exports = {
           params: field.params ? util.deepCopy(field.params) : undefined,
         })
         // Convert image field to subdocument
-        fields[fieldName] = {
+        unprocessedFields[fieldName] = {
           bucket: { type: 'string' },
           date: { type: 'number' },
           filename: { type: 'string' },
           filesize: { type: 'number' },
           metadata: { type: 'any' },
           path: { type: 'string' },
-          schema: { image: true, nullObject: true, isImageObject: true },
           uid: { type: 'string' },
+          schema: { image: true, isImageObject: true, nullObject: true },
         }
       }
     })

package/test/blacklisting.js

@@ -328,7 +328,8 @@ test('find project population', async () => {
   expect(find3).toEqual(customProject)
 })
 
-test('insert blacklisting (validate)', async () => {
+test('insert blacklisting validate', async () => {
+  // todo: isolated model._pathBlacklisted test
   let user = db.model('user', {
     fields: {
       list: [{ type: 'number' }],

package/test/crud.js

@@ -571,6 +571,68 @@ test('update mixing formData', async() => {
   })
 })
 
+test('update large document', async () => {
+  // todo: sereach util.deepCopy
+  // todo: check castIds and any other recursive functions
+  // todo: move default fields to before validate
+  db.model('a', { fields: {} })
+  db.model('b', { fields: {} })
+  db.model('c', { fields: {} })
+  db.model('d', { fields: {} })
+  db.model('e', { fields: {} })
+  try {
+    var large = db.model('large', require('../resources/fixtures/large-definition.js'))
+    var largePayload = require('../resources/fixtures/large-payload.json')
+  } catch (e) {
+    // ignore publicly for now
+    return
+  }
+  // Insert
+  let inserted = await large._insert({})
+  // Update
+  // console.time('update large document')
+  let update = await large.update({
+    query: inserted._id,
+    data: largePayload,
+  })
+  // console.timeEnd('update large document')
+  // Check
+  await expect(update).toEqual(removePrunedProperties(largePayload))
+  // Find
+  // console.time('find large document')
+  // await large.findOne({
+  //   query: inserted._id,
+  // })
+  // console.timeEnd('find large document')
+
+  function removePrunedProperties(entity) {
+    for (let entitiesKey of [
+      'components', 'connections', 'bridges', 'openings', 'spaces', 'elements', 'elementTypes', 
+      'categories', 'typologies',
+    ]) {
+      if (entity[entitiesKey]) {
+        for (let i=0, l=entity[entitiesKey].length; i<l; i++) {
+          entity[entitiesKey][i] = removePrunedProperties(entity[entitiesKey][i])
+        }
+      }
+    }
+    // remove actually keys
+    if (entity.metrics) {
+      for (let key in entity.metrics) {
+        delete entity.metrics[key].actually
+      }
+    }
+    if (entity.code?.match(/^(ELEM|CAT)/)) {
+      delete entity.name
+    }
+    // // convert _id to ObjectId
+    // if (entity._id) {
+    //   entity._id = db.id(entity._id)
+    // }
+    return entity
+  }
+})
+
 test('findOneAndUpdate general', async () => {
   // todo: test all findOneAndUpdate options (e.g. array population)
   // todo: test find & update hooks

package/test/manager.js

@@ -8,7 +8,7 @@ test('manager > basics', async () => {
   // Basic find command
   expect(await manager.db.collection('non-collection').findOne({})).toEqual(null)
   // Raw MongoDB ping command
-  expect(await manager.command({ ping: 1 })).toEqual({ ok: 1 })
+  expect(await manager.command({ ping: 1 })).toMatchObject({ ok: 1 }) // cluster connections return extra fields
   manager.close()
 })