@bedrockio/model 0.2.18 → 0.2.20

package/README.md

@@ -11,7 +11,7 @@ Bedrock utilities for model creation.
   - [Scopes](#scopes)
   - [Tuples](#tuples)
   - [Array Extensions](#array-extensions)
-- [Features](#features)
+- [Modules](#modules)
   - [Soft Delete](#soft-delete)
   - [Validation](#validation)
   - [Search](#search)
@@ -352,7 +352,7 @@ unfortunately cannot be disambiguated in this case.
 
 This will manually create a new nested subschema.
 
-## Features
+## Modules
 
 ### Soft Delete
 
@@ -531,8 +531,6 @@ The method takes the following options:
 - `include` - Allows [include](#includes) based population.
 - `keyword` - A keyword to perform a [keyword search](#keyword-search).
 - `ids` - An array of document ids to search on.
-- `fields` - Used by [keyword search](#keyword-search). Generally for internal
-  use.
 
 Any other fields passed in will be forwarded to `find`. The return value
 contains the found documents in `data` and `meta` which contains metadata about
@@ -603,8 +601,8 @@ this feature a `fields` key must be present on the model definition:
 }
 ```
 
-This will use the `$or` operator to search on multiple fields. If `fields` is
-not defined then a Mongo text query will be attempted:
+This will use the `$or` operator to search on multiple fields. If the model has
+a text index applied, then a Mongo text query will be attempted:
 
 ```
 {
@@ -614,7 +612,118 @@ not defined then a Mongo text query will be attempted:
 }
 ```
 
-Note that this will fail unless a text index is defined on the model.
+#### Keyword Field Caching
+
+A common problem with search is filtering on fields belonging to foreign models.
+The search module helps to alleviate this issue by allowing a simple way to
+cache foreign fields on the model to allow filtering on them.
+
+```json
+{
+  "attributes": {
+    "user": {
+      "type": "ObjectId",
+      "ref": "User"
+    }
+  },
+  "search": {
+    "cache": {
+      "cachedUserName": {
+        "type": "String",
+        "path": "user.name"
+      }
+    },
+    "fields": ["cachedUserName"]
+  }
+}
+```
+
+The above example is equivalent to creating a field called `cachedUserName` and
+updating it when a document is saved:
+
+```js
+schema.add({
+  cachedUserName: 'String',
+});
+schema.pre('save', function () {
+  await this.populate('user');
+  this.cachedUserName = this.user.name;
+});
+```
+
+Specifying a foreign path in `fields` serves as a shortcut to manually defining
+the cached fields:
+
+```json
+// Equivalent to the above example.
+{
+  "attributes": {
+    "user": {
+      "type": "ObjectId",
+      "ref": "User"
+    }
+  },
+  "search": {
+    "fields": ["user.name"]
+  }
+}
+```
+
+##### Syncing Search Fields
+
+When first applying or making changes to defined cached search fields, existing
+documents will be out of sync. The static method `syncSearchFields` is provided
+to synchronize them:
+
+```js
+// Find and update any documents that do not have
+// existing cached fields. Generally called when
+// adding a cached field.
+await Model.syncSearchFields();
+
+// Force an update on ALL documents to resync their
+// cached fields. Generally called to force a cache
+// refresh.
+await Model.syncSearchFields({
+  force: true,
+});
+```
+
+##### Lazy Cached Fields
+
+Cached fields can be made lazy:
+
+```json
+{
+  "attributes": {
+    "user": {
+      "type": "ObjectId",
+      "ref": "User"
+    }
+  },
+  "search": {
+    "cache": {
+      "cachedUserName": {
+        "lazy": true,
+        "path": "user.name"
+      }
+    },
+    "fields": ["user.name"]
+  }
+}
+```
+
+Lazy cached fields will not update themselves once set. They can only be updated
+by forcing a sync:
+
+```js
+await Model.syncSearchFields({
+  force: true,
+});
+```
+
+Making fields lazy alleviates performance impact on writes and allows caches to
+be updated at another time (such as a background job).
 
 #### Search Validation
 

package/dist/cjs/include.js

@@ -242,15 +242,19 @@ function setNodePath(node, options) {
           node[key] = null;
         }
       } else if (type === 'virtual') {
-        node[key] ||= {};
         const virtual = schema.virtual(key);
-        setNodePath(node[key], {
-          // @ts-ignore
-          modelName: virtual.options.ref,
-          path: path.slice(parts.length),
-          depth: depth + 1,
-          exclude
-        });
+        // @ts-ignore
+        const ref = virtual.options.ref;
+        if (ref) {
+          node[key] ||= {};
+          setNodePath(node[key], {
+            // @ts-ignore
+            modelName: ref,
+            path: path.slice(parts.length),
+            depth: depth + 1,
+            exclude
+          });
+        }
         halt = true;
       } else if (type !== 'nested') {
         throw new Error(`Unknown path on ${modelName}: ${key}.`);

package/dist/cjs/schema.js

@@ -227,13 +227,13 @@ function applyScopeExtension(typedef, definition) {
         ...options
       };
     } else {
-      val = attributesToMongoose({
+      val = {
         type: 'Object',
         attributes: val,
         ...options
-      });
+      };
     }
-    definition[key] = val;
+    definition[key] = attributesToMongoose(val);
   }
 }
 

package/dist/cjs/search.js

@@ -16,15 +16,19 @@ var _env = require("./env");
 var _query = require("./query");
 var _warn = _interopRequireDefault(require("./warn"));
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+const {
+  SchemaTypes
+} = _mongoose.default;
 const {
   ObjectId
 } = _mongoose.default.Types;
 function applySearch(schema, definition) {
   validateDefinition(definition);
+  applySearchCache(schema, definition);
   schema.static('search', function search(body = {}) {
     const options = {
       ..._const.SEARCH_DEFAULTS,
-      ...definition.search,
+      ...definition.search?.query,
       ...body
     };
     const {
@@ -33,7 +37,6 @@ function applySearch(schema, definition) {
       skip = 0,
       limit,
       sort,
-      fields,
       ...rest
     } = options;
     const query = {};
@@ -43,7 +46,7 @@ function applySearch(schema, definition) {
       };
     }
     if (keyword) {
-      Object.assign(query, buildKeywordQuery(schema, keyword, fields));
+      Object.assign(query, buildKeywordQuery(schema, keyword, definition.search?.fields));
     }
     Object.assign(query, normalizeQuery(rest, schema.obj));
     if (_env.debug) {
@@ -162,7 +165,7 @@ function buildKeywordQuery(schema, keyword, fields) {
   } else if (hasTextIndex(schema)) {
     queries = [getTextQuery(keyword)];
   } else {
-    queries = [];
+    throw new Error('No keyword fields defined.');
   }
   if (ObjectId.isValid(keyword)) {
     queries.push({
@@ -284,4 +287,200 @@ function parseRegexQuery(str) {
     $regex,
     $options
   };
+}
+
+// Search field caching
+
+function applySearchCache(schema, definition) {
+  normalizeCacheFields(schema, definition);
+  if (!definition.search?.cache) {
+    return;
+  }
+  createCacheFields(schema, definition);
+  applyCacheHook(schema, definition);
+  schema.static('syncSearchFields', async function syncSearchFields(options = {}) {
+    assertIncludeModule(this);
+    const {
+      force
+    } = options;
+    const {
+      cache = {}
+    } = definition.search || {};
+    const paths = getCachePaths(definition);
+    const cachedFields = Object.keys(cache);
+    if (!cachedFields.length) {
+      throw new Error('No search fields to sync.');
+    }
+    const query = {};
+    if (!force) {
+      const $or = Object.entries(cache).map(entry => {
+        const [cachedField, def] = entry;
+        const {
+          base
+        } = def;
+        return {
+          [base]: {
+            $exists: true
+          },
+          [cachedField]: {
+            $exists: false
+          }
+        };
+      });
+      query.$or = $or;
+    }
+    const docs = await this.find(query).include(paths);
+    const ops = docs.map(doc => {
+      return {
+        updateOne: {
+          filter: {
+            _id: doc._id
+          },
+          update: {
+            $set: getUpdates(doc, paths, definition)
+          }
+        }
+      };
+    });
+    return await this.bulkWrite(ops);
+  });
+}
+function normalizeCacheFields(schema, definition) {
+  const {
+    fields,
+    cache = {}
+  } = definition.search || {};
+  if (!fields) {
+    return;
+  }
+  const normalized = [];
+  for (let path of fields) {
+    if (isForeignField(schema, path)) {
+      const cacheName = generateCacheFieldName(path);
+      const type = resolveSchemaType(schema, path);
+      const base = getRefBase(schema, path);
+      cache[cacheName] = {
+        type,
+        base,
+        path: path
+      };
+      normalized.push(cacheName);
+    } else {
+      normalized.push(path);
+    }
+  }
+  definition.search.cache = cache;
+  definition.search.fields = normalized;
+}
+function createCacheFields(schema, definition) {
+  for (let [cachedField, def] of Object.entries(definition.search.cache)) {
+    // Fall back to string type for virtuals or not defined.
+    const {
+      type = 'String'
+    } = def;
+    schema.add({
+      [cachedField]: type
+    });
+    schema.obj[cachedField] = {
+      type,
+      readAccess: 'none'
+    };
+  }
+}
+function applyCacheHook(schema, definition) {
+  schema.pre('save', async function () {
+    assertIncludeModule(this.constructor);
+    assertAssignModule(this.constructor);
+    const doc = this;
+    const paths = getCachePaths(definition, (cachedField, def) => {
+      if (def.lazy) {
+        return !(0, _lodash.get)(doc, cachedField);
+      } else {
+        return true;
+      }
+    });
+    await this.include(paths);
+    this.assign(getUpdates(this, paths, definition));
+  });
+}
+function resolveSchemaType(schema, path) {
+  if (!path.includes('.')) {
+    return (0, _lodash.get)(schema.obj, path)?.type;
+  }
+  const field = getRefField(schema, path);
+  if (field) {
+    const {
+      type,
+      rest
+    } = field;
+    const Model = _mongoose.default.models[type.options.ref];
+    return resolveSchemaType(Model.schema, rest.join('.'));
+  }
+}
+function isForeignField(schema, path) {
+  if (!path.includes('.')) {
+    return false;
+  }
+  return !!getRefField(schema, path);
+}
+function getRefBase(schema, path) {
+  const field = getRefField(schema, path);
+  if (field) {
+    return field.base.join('.');
+  }
+}
+function getRefField(schema, path) {
+  const split = path.split('.');
+  for (let i = 1; i < split.length; i++) {
+    const base = split.slice(0, i);
+    const rest = split.slice(i);
+    const type = schema.path(base);
+    if (type instanceof SchemaTypes.ObjectId) {
+      return {
+        type,
+        base,
+        rest
+      };
+    }
+  }
+}
+function getUpdates(doc, paths, definition) {
+  const updates = {};
+  const entries = Object.entries(definition.search.cache).filter(entry => {
+    return paths.includes(entry[1].path);
+  });
+  for (let [cachedField, def] of entries) {
+    // doc.get will not return virtuals (even with specified options),
+    // so use lodash to ensure they are included here.
+    // https://mongoosejs.com/docs/api/document.html#Document.prototype.get()
+    updates[cachedField] = (0, _lodash.get)(doc, def.path);
+  }
+  return updates;
+}
+function getCachePaths(definition, filter) {
+  filter ||= () => true;
+  const {
+    cache
+  } = definition.search || {};
+  return Object.entries(cache).filter(entry => {
+    return filter(...entry);
+  }).map(entry => {
+    return entry[1].path;
+  });
+}
+function generateCacheFieldName(field) {
+  return `cached${(0, _lodash.upperFirst)((0, _lodash.camelCase)(field))}`;
+}
+
+// Assertions
+
+function assertIncludeModule(Model) {
+  if (!Model.schema.methods.include) {
+    throw new Error('Include module is required for cached search fields.');
+  }
+}
+function assertAssignModule(Model) {
+  if (!Model.schema.methods.assign) {
+    throw new Error('Assign module is required for cached search fields.');
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bedrockio/model",
-  "version": "0.2.18",
+  "version": "0.2.20",
   "description": "Bedrock utilities for model creation.",
   "type": "module",
   "scripts": {
@@ -31,7 +31,7 @@
   },
   "peerDependencies": {
     "@bedrockio/yada": "^1.0.39",
-    "mongoose": "^6.9.0 || ^7.6.4"
+    "mongoose": "^7.6.4"
   },
   "devDependencies": {
     "@babel/cli": "^7.20.7",

package/src/include.js

@@ -228,15 +228,20 @@ function setNodePath(node, options) {
           node[key] = null;
         }
       } else if (type === 'virtual') {
-        node[key] ||= {};
         const virtual = schema.virtual(key);
-        setNodePath(node[key], {
-          // @ts-ignore
-          modelName: virtual.options.ref,
-          path: path.slice(parts.length),
-          depth: depth + 1,
-          exclude,
-        });
+        // @ts-ignore
+        const ref = virtual.options.ref;
+
+        if (ref) {
+          node[key] ||= {};
+          setNodePath(node[key], {
+            // @ts-ignore
+            modelName: ref,
+            path: path.slice(parts.length),
+            depth: depth + 1,
+            exclude,
+          });
+        }
         halt = true;
       } else if (type !== 'nested') {
         throw new Error(`Unknown path on ${modelName}: ${key}.`);

package/src/schema.js

@@ -236,13 +236,13 @@ function applyScopeExtension(typedef, definition) {
         ...options,
       };
     } else {
-      val = attributesToMongoose({
+      val = {
         type: 'Object',
         attributes: val,
         ...options,
-      });
+      };
     }
-    definition[key] = val;
+    definition[key] = attributesToMongoose(val);
   }
 }
 

package/src/search.js

@@ -1,7 +1,15 @@
 import yd from '@bedrockio/yada';
 import logger from '@bedrockio/logger';
 import mongoose from 'mongoose';
-import { pick, isEmpty, escapeRegExp, isPlainObject } from 'lodash';
+import {
+  get,
+  pick,
+  isEmpty,
+  camelCase,
+  upperFirst,
+  escapeRegExp,
+  isPlainObject,
+} from 'lodash';
 
 import { isDateField, isNumberField, getField } from './utils';
 import { SEARCH_DEFAULTS } from './const';
@@ -11,19 +19,21 @@ import { wrapQuery } from './query';
 
 import warn from './warn';
 
+const { SchemaTypes } = mongoose;
 const { ObjectId } = mongoose.Types;
 
 export function applySearch(schema, definition) {
   validateDefinition(definition);
+  applySearchCache(schema, definition);
 
   schema.static('search', function search(body = {}) {
     const options = {
       ...SEARCH_DEFAULTS,
-      ...definition.search,
+      ...definition.search?.query,
       ...body,
     };
 
-    const { ids, keyword, skip = 0, limit, sort, fields, ...rest } = options;
+    const { ids, keyword, skip = 0, limit, sort, ...rest } = options;
 
     const query = {};
 
@@ -32,7 +42,10 @@ export function applySearch(schema, definition) {
     }
 
     if (keyword) {
-      Object.assign(query, buildKeywordQuery(schema, keyword, fields));
+      Object.assign(
+        query,
+        buildKeywordQuery(schema, keyword, definition.search?.fields)
+      );
     }
 
     Object.assign(query, normalizeQuery(rest, schema.obj));
@@ -172,7 +185,7 @@ function buildKeywordQuery(schema, keyword, fields) {
   } else if (hasTextIndex(schema)) {
     queries = [getTextQuery(keyword)];
   } else {
-    queries = [];
+    throw new Error('No keyword fields defined.');
   }
 
   if (ObjectId.isValid(keyword)) {
@@ -300,3 +313,217 @@ function parseRegexQuery(str) {
     $options,
   };
 }
+
+// Search field caching
+
+function applySearchCache(schema, definition) {
+  normalizeCacheFields(schema, definition);
+
+  if (!definition.search?.cache) {
+    return;
+  }
+
+  createCacheFields(schema, definition);
+  applyCacheHook(schema, definition);
+
+  schema.static(
+    'syncSearchFields',
+    async function syncSearchFields(options = {}) {
+      assertIncludeModule(this);
+
+      const { force } = options;
+      const { cache = {} } = definition.search || {};
+
+      const paths = getCachePaths(definition);
+
+      const cachedFields = Object.keys(cache);
+
+      if (!cachedFields.length) {
+        throw new Error('No search fields to sync.');
+      }
+
+      const query = {};
+
+      if (!force) {
+        const $or = Object.entries(cache).map((entry) => {
+          const [cachedField, def] = entry;
+          const { base } = def;
+          return {
+            [base]: {
+              $exists: true,
+            },
+            [cachedField]: {
+              $exists: false,
+            },
+          };
+        });
+        query.$or = $or;
+      }
+
+      const docs = await this.find(query).include(paths);
+
+      const ops = docs.map((doc) => {
+        return {
+          updateOne: {
+            filter: {
+              _id: doc._id,
+            },
+            update: {
+              $set: getUpdates(doc, paths, definition),
+            },
+          },
+        };
+      });
+
+      return await this.bulkWrite(ops);
+    }
+  );
+}
+
+function normalizeCacheFields(schema, definition) {
+  const { fields, cache = {} } = definition.search || {};
+  if (!fields) {
+    return;
+  }
+
+  const normalized = [];
+
+  for (let path of fields) {
+    if (isForeignField(schema, path)) {
+      const cacheName = generateCacheFieldName(path);
+      const type = resolveSchemaType(schema, path);
+      const base = getRefBase(schema, path);
+      cache[cacheName] = {
+        type,
+        base,
+        path: path,
+      };
+      normalized.push(cacheName);
+    } else {
+      normalized.push(path);
+    }
+  }
+
+  definition.search.cache = cache;
+  definition.search.fields = normalized;
+}
+
+function createCacheFields(schema, definition) {
+  for (let [cachedField, def] of Object.entries(definition.search.cache)) {
+    // Fall back to string type for virtuals or not defined.
+    const { type = 'String' } = def;
+    schema.add({
+      [cachedField]: type,
+    });
+    schema.obj[cachedField] = {
+      type,
+      readAccess: 'none',
+    };
+  }
+}
+
+function applyCacheHook(schema, definition) {
+  schema.pre('save', async function () {
+    assertIncludeModule(this.constructor);
+    assertAssignModule(this.constructor);
+
+    const doc = this;
+    const paths = getCachePaths(definition, (cachedField, def) => {
+      if (def.lazy) {
+        return !get(doc, cachedField);
+      } else {
+        return true;
+      }
+    });
+
+    await this.include(paths);
+    this.assign(getUpdates(this, paths, definition));
+  });
+}
+
+function resolveSchemaType(schema, path) {
+  if (!path.includes('.')) {
+    return get(schema.obj, path)?.type;
+  }
+  const field = getRefField(schema, path);
+  if (field) {
+    const { type, rest } = field;
+    const Model = mongoose.models[type.options.ref];
+    return resolveSchemaType(Model.schema, rest.join('.'));
+  }
+}
+
+function isForeignField(schema, path) {
+  if (!path.includes('.')) {
+    return false;
+  }
+  return !!getRefField(schema, path);
+}
+
+function getRefBase(schema, path) {
+  const field = getRefField(schema, path);
+  if (field) {
+    return field.base.join('.');
+  }
+}
+
+function getRefField(schema, path) {
+  const split = path.split('.');
+  for (let i = 1; i < split.length; i++) {
+    const base = split.slice(0, i);
+    const rest = split.slice(i);
+    const type = schema.path(base);
+    if (type instanceof SchemaTypes.ObjectId) {
+      return {
+        type,
+        base,
+        rest,
+      };
+    }
+  }
+}
+
+function getUpdates(doc, paths, definition) {
+  const updates = {};
+
+  const entries = Object.entries(definition.search.cache).filter((entry) => {
+    return paths.includes(entry[1].path);
+  });
+  for (let [cachedField, def] of entries) {
+    // doc.get will not return virtuals (even with specified options),
+    // so use lodash to ensure they are included here.
+    // https://mongoosejs.com/docs/api/document.html#Document.prototype.get()
+    updates[cachedField] = get(doc, def.path);
+  }
+  return updates;
+}
+
+function getCachePaths(definition, filter) {
+  filter ||= () => true;
+  const { cache } = definition.search || {};
+  return Object.entries(cache)
+    .filter((entry) => {
+      return filter(...entry);
+    })
+    .map((entry) => {
+      return entry[1].path;
+    });
+}
+
+function generateCacheFieldName(field) {
+  return `cached${upperFirst(camelCase(field))}`;
+}
+
+// Assertions
+
+function assertIncludeModule(Model) {
+  if (!Model.schema.methods.include) {
+    throw new Error('Include module is required for cached search fields.');
+  }
+}
+
+function assertAssignModule(Model) {
+  if (!Model.schema.methods.assign) {
+    throw new Error('Assign module is required for cached search fields.');
+  }
+}

package/types/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.js"],"names":[],"mappings":"AAeA,gEAmDC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBC"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.js"],"names":[],"mappings":"AAwBA,gEAuDC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAyBC"}