@bedrockio/model 0.8.4 → 0.9.0

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.0
+
+- Added keyword search decomposition.
+
 ## 0.8.4
 
 - Fixed issues with external models colliding with cache module.

package/README.md

@@ -679,6 +679,62 @@ a text index applied, then a Mongo text query will be attempted:
 }
 ```
 
+#### Keyword Search Decomposition
+
+Mongo text indexes don't allow partial matches by default which can be limiting.
+Field based keyword search can do this but with limitations:
+
+```json
+{
+  "attributes": {
+    "firstName": "String",
+    "lastName": "String"
+  },
+  "search": {
+    "fields": ["firstName", "lastName"]
+  }
+}
+```
+
+Although this will perform partial matches for each field, a full name keyword
+like`Frank Reynolds` will not match:
+
+```json
+{
+  "$or": [
+    {
+      "firstName": {
+        "$regex": "Frank Reynolds"
+      }
+    },
+    {
+      "lastName": {
+        "$regex": "Frank Reynolds"
+      }
+    }
+  ]
+}
+```
+
+Field decomposition provides a hint to decompose the keyword to provide matches:
+
+```json
+{
+  "attributes": {
+    "firstName": "String",
+    "lastName": "String"
+  },
+  "search": {
+    "decompose": "{firstName} {lastName...}",
+    "fields": ["firstName", "lastName"]
+  }
+}
+```
+
+This tells the keyword query builder that the first token should be taken as the
+`firstName` and any tokens after that (`...`) should be taken as the last. Note
+the `decompose` field may also be an array.
+
 #### Search Validation
 
 The [validation](#validation) generated for search using `getSearchValidation`

package/dist/cjs/search.js

@@ -23,13 +23,12 @@ function applySearch(schema, definition) {
   validateDefinition(definition);
   validateSearchFields(schema, definition);
   const {
-    query: searchQuery,
-    fields: searchFields
-  } = definition.search || {};
+    search: config = {}
+  } = definition;
   schema.static('search', function search(body = {}) {
     const options = {
       ..._const.SEARCH_DEFAULTS,
-      ...searchQuery,
+      ...config.query,
       ...body
     };
     const {
@@ -50,7 +49,7 @@ function applySearch(schema, definition) {
       };
     }
     if (keyword) {
-      const keywordQuery = buildKeywordQuery(schema, keyword, searchFields);
+      const keywordQuery = buildKeywordQuery(schema, keyword, config);
       query = (0, _query.mergeQuery)(query, keywordQuery);
     }
     if (_env.debug) {
@@ -159,39 +158,97 @@ function resolveSort(sort, schema) {
 // https://stackoverflow.com/questions/44833817/mongodb-full-and-partial-text-search
 // https://jira.mongodb.org/browse/SERVER-15090
 
-function buildKeywordQuery(schema, keyword, fields) {
-  let queries;
+function buildKeywordQuery(schema, keyword, config) {
+  if (hasTextIndex(schema)) {
+    _logger.default.debug('Using text index for keyword search.');
+    return getTextIndexQuery(keyword);
+  }
+  keyword = (0, _lodash.escapeRegExp)(keyword);
+  const queries = [...getDecomposedQueries(keyword, config), ...getFieldQueries(keyword, config)];
 
-  // Prefer defined search fields over
-  // text indexes to perform keyword search.
-  if (fields) {
-    queries = buildRegexQuery(keyword, fields);
-  } else if (hasTextIndex(schema)) {
-    queries = [getTextQuery(keyword)];
+  // Note: Mongo will error on empty $or/$and array.
+  if (queries.length > 1) {
+    return {
+      $or: queries
+    };
+  } else if (queries.length) {
+    return queries[0];
   } else {
-    throw new Error('No keyword fields defined.');
+    _logger.default.debug('Could not find search fields on the model.');
+    throw new Error('Could not compose keyword query.');
   }
-  if (ObjectId.isValid(keyword)) {
-    queries.push({
-      _id: keyword
-    });
+}
+function getTextIndexQuery(keyword) {
+  return {
+    $text: {
+      $search: keyword
+    }
+  };
+}
+function getDecomposedQueries(keyword, config) {
+  const {
+    decompose
+  } = config;
+  if (!decompose) {
+    return [];
   }
-
-  // Note: Mongo will error on empty $or/$and array.
-  return queries.length ? {
-    $or: queries
-  } : {};
+  const decomposers = compileDecomposers(decompose);
+  return decomposers.map(decomposer => {
+    return decomposer(keyword);
+  }).filter(Boolean);
+}
+function compileDecomposers(arg) {
+  const arr = Array.isArray(arg) ? arg : [arg];
+  return arr.map(compileDecomposer);
 }
-function buildRegexQuery(keyword, fields) {
-  return fields.map(field => {
-    const regexKeyword = (0, _lodash.escapeRegExp)(keyword);
+const DECOMPOSE_TEMPLATE_REG = /{(\w+)(\.\.\.)?}/g;
+const compileDecomposer = (0, _lodash.memoize)(template => {
+  if (!template.match(DECOMPOSE_TEMPLATE_REG)) {
+    throw new Error(`Could not compile decompose template ${template}.`);
+  }
+  const fields = [];
+  let src = template;
+  src = src.replace(DECOMPOSE_TEMPLATE_REG, (_, field, rest) => {
+    fields.push(field);
+    return rest ? '(.+)' : '(\\S+)';
+  });
+  src = src.replace(/\s+/, '\\s+');
+  const reg = RegExp(src);
+  return keyword => {
+    const match = keyword.match(reg);
+    if (match) {
+      const query = {};
+      fields.forEach((field, i) => {
+        query[field] = {
+          $regex: match[i + 1],
+          $options: 'i'
+        };
+      });
+      return query;
+    }
+  };
+});
+function getFieldQueries(keyword, config) {
+  const {
+    fields
+  } = config;
+  if (!fields) {
+    return [];
+  }
+  const queries = fields.map(field => {
     return {
       [field]: {
-        $regex: regexKeyword,
+        $regex: keyword,
         $options: 'i'
       }
     };
   });
+  if (ObjectId.isValid(keyword)) {
+    queries.push({
+      _id: keyword
+    });
+  }
+  return queries;
 }
 function hasTextIndex(schema) {
   return schema.indexes().some(([spec]) => {
@@ -200,13 +257,6 @@ function hasTextIndex(schema) {
     });
   });
 }
-function getTextQuery(keyword) {
-  return {
-    $text: {
-      $search: keyword
-    }
-  };
-}
 
 // Normalizes mongo queries. Flattens plain nested paths
 // to dot syntax while preserving mongo operators and

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bedrockio/model",
-  "version": "0.8.4",
+  "version": "0.9.0",
   "description": "Bedrock utilities for model creation.",
   "type": "module",
   "scripts": {

package/src/search.js

@@ -1,7 +1,7 @@
 import yd from '@bedrockio/yada';
 import logger from '@bedrockio/logger';
 import mongoose from 'mongoose';
-import { pick, isEmpty, escapeRegExp, isPlainObject } from 'lodash';
+import { pick, isEmpty, memoize, escapeRegExp, isPlainObject } from 'lodash';
 
 import {
   getField,
@@ -24,12 +24,12 @@ export function applySearch(schema, definition) {
   validateDefinition(definition);
   validateSearchFields(schema, definition);
 
-  const { query: searchQuery, fields: searchFields } = definition.search || {};
+  const { search: config = {} } = definition;
 
   schema.static('search', function search(body = {}) {
     const options = {
       ...SEARCH_DEFAULTS,
-      ...searchQuery,
+      ...config.query,
       ...body,
     };
 
@@ -45,14 +45,14 @@ export function applySearch(schema, definition) {
     }
 
     if (keyword) {
-      const keywordQuery = buildKeywordQuery(schema, keyword, searchFields);
+      const keywordQuery = buildKeywordQuery(schema, keyword, config);
       query = mergeQuery(query, keywordQuery);
     }
 
     if (debug) {
       logger.info(
         `Search query for ${this.modelName}:\n`,
-        JSON.stringify(query, null, 2)
+        JSON.stringify(query, null, 2),
       );
     }
 
@@ -121,7 +121,7 @@ function validateDefinition(definition) {
       [
         '"search" field on model definition must not be an array.',
         'Use "search.fields" to define fields for keyword queries.',
-      ].join('\n')
+      ].join('\n'),
     );
     throw new Error('Invalid model definition.');
   }
@@ -174,37 +174,113 @@ function resolveSort(sort, schema) {
 // https://stackoverflow.com/questions/44833817/mongodb-full-and-partial-text-search
 // https://jira.mongodb.org/browse/SERVER-15090
 
-function buildKeywordQuery(schema, keyword, fields) {
-  let queries;
+function buildKeywordQuery(schema, keyword, config) {
+  if (hasTextIndex(schema)) {
+    logger.debug('Using text index for keyword search.');
+    return getTextIndexQuery(keyword);
+  }
+
+  keyword = escapeRegExp(keyword);
+
+  const queries = [
+    ...getDecomposedQueries(keyword, config),
+    ...getFieldQueries(keyword, config),
+  ];
 
-  // Prefer defined search fields over
-  // text indexes to perform keyword search.
-  if (fields) {
-    queries = buildRegexQuery(keyword, fields);
-  } else if (hasTextIndex(schema)) {
-    queries = [getTextQuery(keyword)];
+  // Note: Mongo will error on empty $or/$and array.
+  if (queries.length > 1) {
+    return {
+      $or: queries,
+    };
+  } else if (queries.length) {
+    return queries[0];
   } else {
-    throw new Error('No keyword fields defined.');
+    logger.debug('Could not find search fields on the model.');
+    throw new Error('Could not compose keyword query.');
   }
+}
 
-  if (ObjectId.isValid(keyword)) {
-    queries.push({ _id: keyword });
+function getTextIndexQuery(keyword) {
+  return {
+    $text: {
+      $search: keyword,
+    },
+  };
+}
+
+function getDecomposedQueries(keyword, config) {
+  const { decompose } = config;
+  if (!decompose) {
+    return [];
   }
 
-  // Note: Mongo will error on empty $or/$and array.
-  return queries.length ? { $or: queries } : {};
+  const decomposers = compileDecomposers(decompose);
+
+  return decomposers
+    .map((decomposer) => {
+      return decomposer(keyword);
+    })
+    .filter(Boolean);
+}
+
+function compileDecomposers(arg) {
+  const arr = Array.isArray(arg) ? arg : [arg];
+  return arr.map(compileDecomposer);
 }
 
-function buildRegexQuery(keyword, fields) {
-  return fields.map((field) => {
-    const regexKeyword = escapeRegExp(keyword);
+const DECOMPOSE_TEMPLATE_REG = /{(\w+)(\.\.\.)?}/g;
+
+const compileDecomposer = memoize((template) => {
+  if (!template.match(DECOMPOSE_TEMPLATE_REG)) {
+    throw new Error(`Could not compile decompose template ${template}.`);
+  }
+
+  const fields = [];
+
+  let src = template;
+  src = src.replace(DECOMPOSE_TEMPLATE_REG, (_, field, rest) => {
+    fields.push(field);
+    return rest ? '(.+)' : '(\\S+)';
+  });
+  src = src.replace(/\s+/, '\\s+');
+  const reg = RegExp(src);
+
+  return (keyword) => {
+    const match = keyword.match(reg);
+    if (match) {
+      const query = {};
+      fields.forEach((field, i) => {
+        query[field] = {
+          $regex: match[i + 1],
+          $options: 'i',
+        };
+      });
+      return query;
+    }
+  };
+});
+
+function getFieldQueries(keyword, config) {
+  const { fields } = config;
+
+  if (!fields) {
+    return [];
+  }
+
+  const queries = fields.map((field) => {
     return {
       [field]: {
-        $regex: regexKeyword,
+        $regex: keyword,
         $options: 'i',
       },
     };
   });
+
+  if (ObjectId.isValid(keyword)) {
+    queries.push({ _id: keyword });
+  }
+
+  return queries;
 }
 
 function hasTextIndex(schema) {
@@ -215,14 +291,6 @@ function hasTextIndex(schema) {
   });
 }
 
-function getTextQuery(keyword) {
-  return {
-    $text: {
-      $search: keyword,
-    },
-  };
-}
-
 // Normalizes mongo queries. Flattens plain nested paths
 // to dot syntax while preserving mongo operators and
 // handling specialed query syntax:

package/types/include.d.ts

@@ -15,6 +15,7 @@ export const INCLUDE_FIELD_SCHEMA: {
     required(): /*elided*/ any;
     default(arg: any): /*elided*/ any;
     custom(fn: Function): /*elided*/ any;
+    missing(fn: Function): /*elided*/ any;
     strip(strip: any): /*elided*/ any;
     allow(...set: any[]): /*elided*/ any;
     reject(...set: any[]): /*elided*/ any;
@@ -42,9 +43,10 @@ export const INCLUDE_FIELD_SCHEMA: {
     assertEnum(set: any, allow: any): /*elided*/ any;
     assert(type: any, fn: any): /*elided*/ any;
     pushAssertion(assertion: any): void;
+    canSkipAssertion(value: any, assertion: any, options: any): any;
     transform(fn: any): /*elided*/ any;
     getSortIndex(type: any): number;
-    runAssertion(assertion: any, value: any, options?: {}): Promise<any>;
+    runAssertion(value: any, assertion: any, options?: {}): Promise<any>;
     enumToOpenApi(): any;
 };
 //# sourceMappingURL=include.d.ts.map

package/types/include.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"include.d.ts","sourceRoot":"","sources":["../src/include.js"],"names":[],"mappings":"AA2BA,gDAuEC;AAMD,uDA4BC;AAGD,yDAIC;AAaD,yEAUC;AA9ID;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA+CK,CAAC;;;;;;;;;;;;;;;;EA1CH"}
+{"version":3,"file":"include.d.ts","sourceRoot":"","sources":["../src/include.js"],"names":[],"mappings":"AA2BA,gDAuEC;AAMD,uDA4BC;AAGD,yDAIC;AAaD,yEAUC;AA9ID;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAsDoB,CAAC;;;;;;;;;;;;;;;;;EAjDlB"}

package/types/search.d.ts

@@ -12,6 +12,7 @@ export function searchValidation(options?: {}): {
     required(): /*elided*/ any;
     default(arg: any): /*elided*/ any;
     custom(fn: Function): /*elided*/ any;
+    missing(fn: Function): /*elided*/ any;
     strip(strip: any): /*elided*/ any;
     allow(...set: any[]): /*elided*/ any;
     reject(...set: any[]): /*elided*/ any;
@@ -39,9 +40,10 @@ export function searchValidation(options?: {}): {
     assertEnum(set: any, allow: any): /*elided*/ any;
     assert(type: any, fn: any): /*elided*/ any;
     pushAssertion(assertion: any): void;
+    canSkipAssertion(value: any, assertion: any, options: any): any;
     transform(fn: any): /*elided*/ any;
     getSortIndex(type: any): number;
-    runAssertion(assertion: any, value: any, options?: {}): Promise<any>;
+    runAssertion(value: any, assertion: any, options?: {}): Promise<any>;
     enumToOpenApi(): any;
 };
 //# sourceMappingURL=search.d.ts.map

package/types/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.js"],"names":[],"mappings":"AAsBA,gEAwDC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAA2C,CAAA;;;;;;;;;;;;;;;;EAyB1C"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.js"],"names":[],"mappings":"AAsBA,gEAwDC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAWS,CAAA;;;;;;;;;;;;;;;;;EAcR"}

package/types/validation.d.ts

@@ -73,6 +73,7 @@ export const OBJECT_ID_SCHEMA: {
     meta: {};
     default(arg: any): /*elided*/ any;
     custom(fn: Function): /*elided*/ any;
+    missing(fn: Function): /*elided*/ any;
     strip(strip: any): /*elided*/ any;
     allow(...set: any[]): /*elided*/ any;
     reject(...set: any[]): /*elided*/ any;
@@ -101,9 +102,10 @@ export const OBJECT_ID_SCHEMA: {
     assertEnum(set: any, allow: any): /*elided*/ any;
     assert(type: any, fn: any): /*elided*/ any;
     pushAssertion(assertion: any): void;
+    canSkipAssertion(value: any, assertion: any, options: any): any;
     transform(fn: any): /*elided*/ any;
     getSortIndex(type: any): number;
-    runAssertion(assertion: any, value: any, options?: {}): Promise<any>;
+    runAssertion(value: any, assertion: any, options?: {}): Promise<any>;
     enumToOpenApi(): any;
 };
 //# sourceMappingURL=validation.d.ts.map

package/types/validation.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../src/validation.js"],"names":[],"mappings":"AAoFA,kDAEC;AAED,oEAgGC;AAsBD,wEA2BC;AAkVD;;;EAEC;AAED;;;EAOC;AApjBD;;;;;;;;;;;;;;;;eAwBE,CAAF;;;;;;;;;;;;iBAcmC,CAAC;kBAC3B,CAAC;kBAEH,CAAC;oBACA,CAAC;oBACF,CAAC;;;wBAkBkB,CAAC;8BAEf,CAAC;oBAGD,CAAC;oBACV,CAAC;oCAGG,CAAC;uBAAkC,CAAC;8BACb,CAAC;uBAEjB,CAAC;iBAEX,CAAJ;;;mBAa6B,CAAC;yBAEjB,CAAC;0BAEN,CAAF;yBAKE,CAAC;sBACgB,CAAC;yBACS,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAvDzB,CAAC;;;;;;;;;;;;;;;;EApCP"}
+{"version":3,"file":"validation.d.ts","sourceRoot":"","sources":["../src/validation.js"],"names":[],"mappings":"AAoFA,kDAEC;AAED,oEAgGC;AAsBD,wEA2BC;AAkVD;;;EAEC;AAED;;;EAOC;AApjBD;;;;;;;;;;;;;;;;eAwBE,CAAF;;;;;;;;;;;;iBAcmC,CAAC;kBAC3B,CAAC;kBAEH,CAAC;oBACA,CAAC;oBACF,CAAC;;;wBAkBkB,CAAC;8BAEf,CAAC;oBAGD,CAAC;oBACV,CAAC;oCAGG,CAAC;uBAAkC,CAAC;8BACb,CAAC;uBAEjB,CAAC;iBAEX,CAAJ;;;mBAa6B,CAAC;yBAEjB,CAAC;0BAEN,CAAF;yBAKE,CAAC;sBACgB,CAAC;yBACS,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAhDxB,CAAC;;;;;;;;;;;;;;;;;EA3CR"}