@bedrockio/model 0.7.5 → 0.8.0

package/CHANGELOG.md

@@ -1,3 +1,18 @@
+## 0.8.0
+
+- Moved "cache" field in "search" to definition root.
+- Changed "lazy" on cached fields to instead use "sync".
+- Removed "sync" field in "search". Syncing now happens automatically on synced
+  cache fields.
+- Removed "force" option from "syncCacheFields".
+- Better handling of deep nested and array cache fields.
+- Small fix for field messages on errors.
+
+## 0.7.4
+
+- NANP phone number validation support
+- Better handling of empty string fields for unsetting fields
+
 ## 0.7.0
 
 - Handling null fields in search queries.

package/README.md

@@ -15,6 +15,7 @@ Bedrock utilities for model creation.
   - [Soft Delete](#soft-delete)
   - [Validation](#validation)
   - [Search](#search)
+  - [Cache](#cache)
   - [Includes](#includes)
   - [Delete Hooks](#delete-hooks)
   - [Access Control](#access-control)
@@ -124,7 +125,7 @@ Links:
 
 ### Schema Extensions
 
-This package provides a number of extensions to assist schema creation outside
+This module provides a number of extensions to assist schema creation outside
 the scope of Mongoose.
 
 #### Attributes
@@ -302,7 +303,7 @@ with `minLength` and `maxLength` on strings.
 
 ### Gotchas
 
-#### The `type` field is a special:
+#### The `type` field is special:
 
 ```js
 {
@@ -421,7 +422,7 @@ Note that although monogoose allows a `unique` option on fields, this will add a
 unique index to the mongo collection itself which is incompatible with soft
 deletion.
 
-This package will intercept `unique: true` to create a soft delete compatible
+This module will intercept `unique: true` to create a soft delete compatible
 validation which will:
 
 - Throw an error if other non-deleted documents with the same fields exist when
@@ -479,7 +480,7 @@ There are 4 main methods to generate schemas:
 - `getCreateValidation`: Validates all fields while disallowing reserved fields
   like `id`, `createdAt`, and `updatedAt`.
 - `getUpdateValidation`: Validates all fields as optional (ie. they will not be
-  validated if they don't exist on the object). Additionally will strip out
+  validated if they don't exist on the input). Additionally will strip out
   reserved fields to allow created objects to be passed in. Unknown fields will
   also be stripped out rather than error to allow virtuals to be passed in.
 - `getSearchValidation`: Validates fields for use with [search](#search). The
@@ -659,11 +660,16 @@ a text index applied, then a Mongo text query will be attempted:
 }
 ```
 
-#### Keyword Field Caching
+#### Search Validation
+
+The [validation](#validation) generated for search using `getSearchValidation`
+is inherently looser and allows more fields to be passed to allow complex
+searches compatible with the above.
+
+### Cache
 
-A common problem with search is filtering on fields belonging to foreign models.
-The search module helps to alleviate this issue by allowing a simple way to
-cache foreign fields on the model to allow filtering on them.
+The cache module allows a simple way to cache foreign fields on a document and
+optionally keep them in sync.
 
 ```json
 {
@@ -673,20 +679,17 @@ cache foreign fields on the model to allow filtering on them.
       "ref": "User"
     }
   },
-  "search": {
-    "cache": {
-      "userName": {
-        "type": "String",
-        "path": "user.name"
-      }
-    },
-    "fields": ["userName"]
+  "cache": {
+    "userName": {
+      "type": "String",
+      "path": "user.name"
+    }
   }
 }
 ```
 
 The above example is equivalent to creating a field called `userName` and
-updating it when a document is saved:
+setting it when a document is saved:
 
 ```js
 schema.add({
@@ -700,54 +703,44 @@ schema.pre('save', function () {
 
 #### Syncing Cached Fields
 
-When a foreign document is updated the cached fields will be out of sync, for
-example:
+By default cached fields are only updated when the reference changes. This is
+fine when the field on the foreign document will not change or to keep a
+snapshot of the value. However in some cases the local cached field should be
+kept in sync when the foreign field changes:
 
-```js
-await shop.save();
-console.log(shop.userName);
-// The current user name
-
-user.name = 'New Name';
-await user.save();
-
-shop = await Shop.findById(shop.id);
-console.log(shop.userName);
-// Cached userName is out of sync as the user has been updated
-```
-
-A simple mechanism is provided via the `sync` key to keep the documents in sync:
-
-```jsonc
-// In user.json
+```json
 {
   "attributes": {
-    "name": "String"
+    "user": {
+      "type": "ObjectId",
+      "ref": "User"
+    }
   },
-  "search": {
-    "sync": [
-      {
-        "ref": "Shop",
-        "path": "user"
-      }
-    ]
+  "cache": {
+    "userName": {
+      "type": "String",
+      "path": "user.name",
+      "sync": true
+    }
   }
 }
 ```
 
-This is the equivalent of running the following in a post save hook:
+The "sync" field is the equivelent of running a post save hook on the foreign
+model to keep the cached field in sync:
 
 ```js
-const shops = await Shop.find({
-  user: user.id,
+userSchema.post('save', function () {
+  await Shop.updateMany({
+    user: this.id,
+  }, {
+    $set: {
+      userName: this.name
+    }
+  })
 });
-for (let shop of shops) {
-  await shop.save();
-}
 ```
 
-This will run the hooks on each shop, synchronizing the cached fields.
-
 ##### Initial Sync
 
 When first applying or making changes to defined cached search fields, existing
@@ -756,61 +749,11 @@ to synchronize them:
 
 ```js
 // Find and update any documents that do not have
-// existing cached fields. Generally called when
-// adding a cached field.
+// existing cached fields. Generally called after
+// adding or modifying a cached field.
 await Model.syncCacheFields();
-
-// Force an update on ALL documents to resync their
-// cached fields. Generally called to force a cache
-// refresh.
-await Model.syncCacheFields({
-  force: true,
-});
 ```
 
-##### Lazy Cached Fields
-
-Cached fields can be made lazy:
-
-```json
-{
-  "attributes": {
-    "user": {
-      "type": "ObjectId",
-      "ref": "User"
-    }
-  },
-  "search": {
-    "cache": {
-      "userName": {
-        "type": "String",
-        "path": "user.name",
-        "lazy": true
-      }
-    },
-    "fields": ["userName"]
-  }
-}
-```
-
-Lazy cached fields will not update themselves once set. They can only be updated
-by forcing a sync:
-
-```js
-await Model.syncCacheFields({
-  force: true,
-});
-```
-
-Making fields lazy alleviates performance impact on writes and allows caches to
-be updated at another time (such as a background job).
-
-#### Search Validation
-
-The [validation](#validation) generated for search using `getSearchValidation`
-is inherently looser and allows more fields to be passed to allow complex
-searches compatible with the above.
-
 ### Includes
 
 Populating foreign documents with
@@ -1135,7 +1078,7 @@ await shop.include('owner', {
 
 ### Access Control
 
-This package applies two forms of access control:
+This module applies two forms of access control:
 
 - [Field Access](#field-access)
 - [Document Access](#document-access)
@@ -1589,16 +1532,58 @@ string, both of which would be stored in the database if naively assigned with
 
 This module adds a single `findOrCreate` convenience method that is easy to
 understand and avoids some of the gotchas that come with upserting documents in
-Mongoose.
+Mongoose:
+
+```js
+const shop = await Shop.findOrCreate({
+  name: 'My Shop',
+});
+
+// This is equivalent to running:
+let shop = await Shop.findOne({
+  name: 'My Shop',
+});
+
+if (!shop) {
+  shop = await Shop.create({
+    name: 'My Shop',
+  });
+}
+```
+
+In most cases not all of the fields should be queried on to determine if an
+existing document exists. In this case two arguments should be passed the first
+of which is the query:
+
+```js
+const shop = await Shop.findOrCreate(
+  {
+    slug: 'my-shop',
+  },
+  {
+    name: 'My Shop',
+    slug: 'my-shop',
+  }
+);
+
+// This is equivalent to running:
+let shop = await Shop.findOne({
+  slug: 'my-shop',
+});
+
+if (!shop) {
+  shop = await Shop.create({
+    name: 'My Shop',
+    slug: 'my-shop',
+  });
+}
+```
 
 ### Slugs
 
 A common requirement is to allow slugs on documents to serve as ids for human
-readable URLs. To load a single document this way the naive approach would be to
-run a search on all documents matching the `slug` then pull the first one off.
-
-This module simplifies this by assuming a `slug` field on a model and adding a
-`findByIdOrSlug` method that allows searching on both:
+readable URLs. This module simplifies this by assuming a `slug` field on a model
+and adding a `findByIdOrSlug` method that allows searching on either:
 
 ```js
 const post = await Post.findByIdOrSlug(str);

package/dist/cjs/access.js

@@ -7,7 +7,7 @@ exports.hasAccess = hasAccess;
 var _errors = require("./errors");
 var _utils = require("./utils");
 var _warn = _interopRequireDefault(require("./warn"));
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * @param {string|string[]} allowed
  */

package/dist/cjs/assign.js

@@ -7,7 +7,7 @@ exports.applyAssign = applyAssign;
 var _lodash = require("lodash");
 var _mongoose = _interopRequireDefault(require("mongoose"));
 var _utils = require("./utils");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 function applyAssign(schema) {
   schema.method('assign', function assign(fields) {
     unsetReferenceFields(fields, schema.obj);

package/dist/cjs/cache.js

@@ -0,0 +1,240 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.applyCache = applyCache;
+var _mongoose = _interopRequireDefault(require("mongoose"));
+var _lodash = require("lodash");
+var _utils = require("./utils");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+const definitionMap = new Map();
+_mongoose.default.plugin(cacheSyncPlugin);
+function applyCache(schema, definition) {
+  definitionMap.set(schema, definition);
+  if (!definition.cache) {
+    return;
+  }
+  createCacheFields(schema, definition);
+  applyStaticMethods(schema, definition);
+  applyCacheHook(schema, definition);
+}
+function createCacheFields(schema, definition) {
+  for (let [cachedField, def] of Object.entries(definition.cache)) {
+    const {
+      type,
+      path,
+      ...rest
+    } = def;
+    schema.add({
+      [cachedField]: type
+    });
+    schema.obj[cachedField] = {
+      ...rest,
+      type,
+      writeAccess: 'none'
+    };
+  }
+}
+function applyStaticMethods(schema, definition) {
+  schema.static('syncCacheFields', async function syncCacheFields() {
+    assertIncludeModule(this);
+    const fields = resolveCachedFields(schema, definition);
+    const hasSynced = fields.some(entry => {
+      return entry.sync;
+    });
+    const query = {};
+    if (!hasSynced) {
+      const $or = fields.map(field => {
+        return {
+          [field.name]: null
+        };
+      });
+      query.$or = $or;
+    }
+    const includes = getIncludes(fields);
+    const docs = await this.find(query).include(includes);
+    const ops = docs.flatMap(doc => {
+      return fields.map(field => {
+        const {
+          name,
+          sync
+        } = field;
+        const updates = getUpdates(doc, [field]);
+        const filter = {
+          _id: doc._id
+        };
+        if (!sync) {
+          filter[name] = null;
+        }
+        return {
+          updateOne: {
+            filter,
+            update: {
+              $set: updates
+            }
+          }
+        };
+      });
+    });
+    return await this.bulkWrite(ops);
+  });
+}
+function applyCacheHook(schema, definition) {
+  const fields = resolveCachedFields(schema, definition);
+  schema.pre('save', async function () {
+    assertIncludeModule(this.constructor);
+    assertAssignModule(this.constructor);
+    const doc = this;
+    const changes = fields.filter(field => {
+      const {
+        sync,
+        local,
+        name
+      } = field;
+      if (sync || doc.isModified(local)) {
+        // Always update if we are actively syncing
+        // or if the field has been changed.
+        return true;
+      } else {
+        // Otherwise only update if the value does
+        // not exist yet.
+        const value = (0, _lodash.get)(doc, name);
+        return Array.isArray(value) ? !value.length : !value;
+      }
+    });
+    await this.include(getIncludes(changes));
+    this.assign(getUpdates(doc, changes));
+  });
+}
+
+// Syncing
+
+const syncOperations = {};
+const compiledModels = new Set();
+function cacheSyncPlugin(schema) {
+  // Compile sync fields each time a new schema
+  // is registered but only do it one time for.
+  const initialize = (0, _lodash.once)(compileSyncOperations);
+  schema.pre('save', async function () {
+    this.$locals.modifiedPaths = this.modifiedPaths();
+  });
+  schema.post('save', async function () {
+    initialize();
+
+    // @ts-ignore
+    const {
+      modelName
+    } = this.constructor;
+    const ops = syncOperations[modelName] || [];
+    for (let op of ops) {
+      await op(this);
+    }
+  });
+}
+function compileSyncOperations() {
+  for (let Model of Object.values(_mongoose.default.models)) {
+    const {
+      schema
+    } = Model;
+    if (compiledModels.has(Model)) {
+      // Model has already been compiled so skip.
+      continue;
+    }
+    const definition = definitionMap.get(schema);
+    const fields = resolveCachedFields(schema, definition);
+    for (let [ref, group] of Object.entries((0, _lodash.groupBy)(fields, 'ref'))) {
+      const hasSynced = group.some(entry => {
+        return entry.sync;
+      });
+      if (!hasSynced) {
+        continue;
+      }
+      const fn = async doc => {
+        const {
+          modifiedPaths
+        } = doc.$locals;
+        const changes = group.filter(entry => {
+          return entry.sync && modifiedPaths.includes(entry.foreign);
+        });
+        if (changes.length) {
+          const $or = changes.map(change => {
+            const {
+              local
+            } = change;
+            return {
+              [local]: doc.id
+            };
+          });
+          const docs = await Model.find({
+            $or
+          });
+          await Promise.all(docs.map(doc => doc.save()));
+        }
+      };
+      syncOperations[ref] ||= [];
+      syncOperations[ref].push(fn);
+    }
+    compiledModels.add(Model);
+  }
+}
+
+// Utils
+
+function resolveCachedFields(schema, definition) {
+  const {
+    cache = {}
+  } = definition;
+  return Object.entries(cache).map(([name, def]) => {
+    const {
+      path,
+      sync = false
+    } = def;
+    const resolved = (0, _utils.resolveRefPath)(schema, path);
+    if (!resolved) {
+      throw new Error(`Could not resolve path ${path}.`);
+    }
+    return {
+      ...resolved,
+      name,
+      path,
+      sync
+    };
+  });
+}
+function getUpdates(doc, fields) {
+  const updates = {};
+  for (let field of fields) {
+    const {
+      name,
+      path
+    } = field;
+
+    // doc.get will not return virtuals (even with specified options),
+    // so fall back to lodash to ensure they are included here.
+    // https://mongoosejs.com/docs/api/document.html#Document.prototype.get()
+    const value = doc.get(path) ?? (0, _lodash.get)(doc, path);
+    updates[name] = value;
+  }
+  return updates;
+}
+function getIncludes(fields) {
+  const includes = new Set();
+  for (let field of fields) {
+    includes.add(field.local);
+  }
+  return includes;
+}
+
+// Assertions
+
+function assertIncludeModule(Model) {
+  if (!Model.schema.methods.include) {
+    throw new Error('Include module is required for cached fields.');
+  }
+}
+function assertAssignModule(Model) {
+  if (!Model.schema.methods.assign) {
+    throw new Error('Assign module is required for cached fields.');
+  }
+}

package/dist/cjs/delete-hooks.js

@@ -8,7 +8,7 @@ var _mongoose = _interopRequireDefault(require("mongoose"));
 var _lodash = require("lodash");
 var _errors = require("./errors");
 var _utils = require("./utils");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 const {
   ObjectId: SchemaObjectId
 } = _mongoose.default.Schema.Types;

package/dist/cjs/disallowed.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.applyDisallowed = applyDisallowed;
 var _warn = _interopRequireDefault(require("./warn"));
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 function applyDisallowed(schema) {
   schema.method('deleteOne', function () {
     (0, _warn.default)('The "deleteOne" method on documents is disallowed due to ambiguity', 'Use either "delete" or "deleteOne" on the model.');

package/dist/cjs/include.js

@@ -13,7 +13,7 @@ var _lodash = require("lodash");
 var _yada = _interopRequireDefault(require("@bedrockio/yada"));
 var _utils = require("./utils");
 var _const = require("./const");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 // @ts-ignore
 // Overloading mongoose Query prototype to
 // allow an "include" method for queries.
@@ -149,10 +149,19 @@ function checkSelects(doc, ret) {
 
 // Exported for testing.
 function getParams(modelName, arg) {
-  const paths = Array.isArray(arg) ? arg : [arg];
+  const paths = resolvePathsArg(arg);
   const node = pathsToNode(paths, modelName);
   return nodeToPopulates(node);
 }
+function resolvePathsArg(arg) {
+  if (Array.isArray(arg)) {
+    return arg;
+  } else if (arg instanceof Set) {
+    return Array.from(arg);
+  } else {
+    return [arg];
+  }
+}
 
 // Exported for testing.
 function getDocumentParams(doc, arg, options = {}) {

package/dist/cjs/load.js

@@ -10,7 +10,7 @@ var _path = _interopRequireDefault(require("path"));
 var _mongoose = _interopRequireDefault(require("mongoose"));
 var _lodash = require("lodash");
 var _schema = require("./schema");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * Loads a single model by definition and name.
  * @param {object} definition

package/dist/cjs/schema.js

@@ -10,6 +10,7 @@ var _lodash = require("lodash");
 var _utils = require("./utils");
 var _serialization = require("./serialization");
 var _slug = require("./slug");
+var _cache = require("./cache");
 var _search = require("./search");
 var _assign = require("./assign");
 var _upsert = require("./upsert");
@@ -19,7 +20,7 @@ var _softDelete = require("./soft-delete");
 var _deleteHooks = require("./delete-hooks");
 var _disallowed = require("./disallowed");
 var _validation = require("./validation");
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
 /**
  * Creates a new Mongoose schema with Bedrock extensions
  * applied. For more about syntax and functionality see
@@ -54,6 +55,7 @@ function createSchema(definition, options = {}) {
   (0, _validation.applyValidation)(schema, definition);
   (0, _deleteHooks.applyDeleteHooks)(schema, definition);
   (0, _search.applySearch)(schema, definition);
+  (0, _cache.applyCache)(schema, definition);
   (0, _disallowed.applyDisallowed)(schema);
   (0, _include.applyInclude)(schema);
   (0, _hydrate.applyHydrate)(schema);