@bedrockio/model 0.1.0

package/src/schema.js

@@ -0,0 +1,286 @@
+import mongoose from 'mongoose';
+import { pick, isPlainObject, capitalize, camelCase } from 'lodash';
+
+import { isSchemaTypedef } from './utils';
+
+import { serializeOptions } from './serialization';
+import { applySlug } from './slug';
+import { applySearch } from './search';
+import { applyAssign } from './assign';
+import { applyInclude } from './include';
+import { applyReferences } from './references';
+import { applySoftDelete } from './soft-delete';
+import {
+  applyValidation,
+  getNamedValidator,
+  getTupleValidator,
+} from './validation';
+
+export const RESERVED_FIELDS = [
+  'createdAt',
+  'updatedAt',
+  'deletedAt',
+  'deleted',
+];
+
+export function createSchema(definition, options = {}) {
+  const schema = new mongoose.Schema(
+    attributesToMongoose(
+      normalizeAttributes({
+        ...definition.attributes,
+
+        // Although timestamps are being set below, we still need to add
+        // them to the schema so that validation can be generated for them,
+        // namely in getSearchValidation.
+        createdAt: 'Date',
+        updatedAt: 'Date',
+        deletedAt: 'Date',
+        deleted: { type: 'Boolean', default: false },
+      })
+    ),
+    {
+      timestamps: true,
+      toJSON: serializeOptions,
+      toObject: serializeOptions,
+      ...options,
+    }
+  );
+
+  applySoftDelete(schema, definition);
+  applyValidation(schema, definition);
+  applyReferences(schema, definition);
+  applyInclude(schema, definition);
+  applySearch(schema, definition);
+  applyAssign(schema, definition);
+  applySlug(schema, definition);
+
+  return schema;
+}
+
+export function normalizeAttributes(arg, path = []) {
+  if (arg instanceof mongoose.Schema) {
+    return arg;
+  } else if (typeof arg === 'function') {
+    throw new Error('Native functions are not allowed as types.');
+  } else if (typeof arg === 'string') {
+    return normalizeSchemaTypedef({ type: arg }, path);
+  } else if (Array.isArray(arg)) {
+    const type = normalizeArrayAttributes(arg, path);
+    return normalizeSchemaTypedef({ type }, path);
+  } else if (typeof arg === 'object') {
+    assertRefs(arg, path);
+
+    if (isSchemaTypedef(arg)) {
+      return normalizeSchemaTypedef(arg, path);
+    }
+
+    const attributes = {};
+    for (let [key, val] of Object.entries(arg)) {
+      attributes[key] = normalizeAttributes(val, [...path, key]);
+    }
+    return attributes;
+  }
+}
+
+function normalizeSchemaTypedef(typedef, path) {
+  const { type } = typedef;
+
+  if (Array.isArray(type)) {
+    typedef.type = normalizeArrayAttributes(type, path);
+  } else if (typeof type === 'object') {
+    typedef.type = normalizeAttributes(type, path);
+  } else {
+    assertSchemaType(type, path);
+  }
+
+  return typedef;
+}
+
+function normalizeArrayAttributes(arr, path) {
+  return arr.map((el, i) => {
+    return normalizeAttributes(el, [...path, i]);
+  });
+}
+
+function attributesToMongoose(attributes) {
+  if (typeof attributes === 'string') {
+    return attributes;
+  } else if (Array.isArray(attributes)) {
+    return attributes.map(attributesToMongoose);
+  }
+
+  let definition = {};
+
+  const isTypedef = isSchemaTypedef(attributes);
+
+  for (let [key, val] of Object.entries(attributes)) {
+    const type = typeof val;
+    if (isTypedef) {
+      if (key === 'type' && type !== 'function') {
+        val = attributesToMongoose(val);
+      } else if (key === 'match' && type === 'string') {
+        // Convert match field to RegExp that cannot be expressed in JSON.
+        val = parseRegExp(val);
+      } else if (key === 'validate' && type === 'string') {
+        // Allow custom mongoose validation function that derives from the schema.
+        val = getNamedValidator(val);
+      }
+    } else if (isPlainObject(val)) {
+      if (isScopeExtension(val)) {
+        applyScopeExtension(val, definition);
+        continue;
+      } else {
+        val = attributesToMongoose(val);
+      }
+    }
+    definition[key] = val;
+  }
+
+  if (isTypedef) {
+    applyExtensions(definition);
+  }
+
+  return definition;
+}
+
+function assertSchemaType(type, path) {
+  if (type === 'Mixed') {
+    throw new Error('Type "Mixed" is not allowed. Use "Object" instead.');
+  }
+
+  if (typeof type === 'string') {
+    if (!isMongooseType(type)) {
+      const p = path.join('.');
+      const upper = camelUpper(type);
+      if (isMongooseType(upper)) {
+        throw new Error(`Type "${type}" in "${p}" should be "${upper}".`);
+      } else if (type !== 'Scope') {
+        throw new Error(`Invalid type "${type}" for "${p}".`);
+      }
+    }
+  }
+}
+
+function assertRefs(field, path) {
+  const { type, ref, refPath } = field;
+  const p = path.join('.');
+  if (isObjectIdType(type) && !ref && !refPath) {
+    throw new Error(`Ref must be passed for "${p}".`);
+  } else if (ref && !isMongooseType(ref) && !isObjectIdType(type)) {
+    throw new Error(`Ref field "${p}" must be type "ObjectId".`);
+  }
+}
+
+function camelUpper(str) {
+  return capitalize(camelCase(str));
+}
+
+function isObjectIdType(type) {
+  return type === 'ObjectId' || type === mongoose.Schema.Types.ObjectId;
+}
+
+function isMongooseType(type) {
+  return !!mongoose.Schema.Types[type];
+}
+
+function applyExtensions(typedef) {
+  applySyntaxExtensions(typedef);
+  applyTupleExtension(typedef);
+}
+
+function applySyntaxExtensions(typedef) {
+  const { type, attributes } = typedef;
+  if (isExtendedSyntax(typedef)) {
+    typedef.type = new mongoose.Schema(attributes);
+    if (type === 'Array') {
+      typedef.type = [typedef.type];
+    }
+  }
+  if (Array.isArray(typedef.type)) {
+    applyArrayValidators(typedef);
+    applyOptionHoisting(typedef);
+  }
+}
+
+// Hoist read/write scopes from a nested element.
+// See the readme for more.
+function applyOptionHoisting(typedef) {
+  Object.assign(typedef, pick(typedef.type[0], 'readAccess', 'writeAccess'));
+}
+
+function isExtendedSyntax(typedef) {
+  const { type, attributes } = typedef;
+  return attributes && (type === 'Object' || type === 'Array');
+}
+
+function isScopeExtension(obj) {
+  return isSchemaTypedef(obj) && obj.type === 'Scope';
+}
+
+function applyScopeExtension(scope, definition) {
+  const { type, attributes, ...options } = scope;
+  for (let [key, val] of Object.entries(normalizeAttributes(attributes))) {
+    definition[key] = {
+      ...val,
+      ...options,
+    };
+  }
+}
+
+// Extended tuple syntax. Return mixed type and set validator.
+function applyTupleExtension(typedef) {
+  const { type } = typedef;
+  if (Array.isArray(type) && type.length > 1) {
+    typedef.type = ['Mixed'];
+    typedef.validate = getTupleValidator(type);
+  }
+}
+
+function applyArrayValidators(typedef) {
+  let { minLength, maxLength, validate } = typedef;
+  if (minLength) {
+    validate = chain(validate, validateMinLength(minLength));
+  }
+  if (maxLength) {
+    validate = chain(validate, validateMaxLength(maxLength));
+  }
+  if (validate) {
+    typedef.validate = validate;
+  }
+}
+
+function validateMinLength(min) {
+  const s = min === 1 ? '' : 's';
+  return (arr) => {
+    if (arr.length < min) {
+      throw new Error(`Must have at least ${min} element${s}.`);
+    }
+  };
+}
+
+function validateMaxLength(max) {
+  const s = max === 1 ? '' : 's';
+  return (arr) => {
+    if (arr.length > max) {
+      throw new Error(`Cannot have more than ${max} element${s}.`);
+    }
+  };
+}
+
+function chain(fn1, fn2) {
+  return (...args) => {
+    fn1?.(...args);
+    fn2?.(...args);
+  };
+}
+
+const REG_MATCH = /^\/(.+)\/(\w+)$/;
+
+function parseRegExp(str) {
+  const match = str.match(REG_MATCH);
+  if (!match) {
+    throw new Error('Could not parse regex.');
+  }
+  const [, source, flags] = match;
+  return RegExp(source, flags);
+}

package/src/search.js

@@ -0,0 +1,263 @@
+import yd from '@bedrockio/yada';
+import mongoose from 'mongoose';
+import { pick, isEmpty, isPlainObject } from 'lodash';
+
+import { isDateField, isNumberField, resolveField } from './utils';
+import { SEARCH_DEFAULTS } from './const';
+
+import warn from './warn';
+
+const { ObjectId } = mongoose.Types;
+
+const SORT_SCHEMA = yd.object({
+  field: yd.string().required(),
+  order: yd.string().allow('desc', 'asc').required(),
+});
+
+export function applySearch(schema, definition) {
+  validateDefinition(definition);
+
+  schema.static('search', function search(body = {}) {
+    const options = {
+      ...SEARCH_DEFAULTS,
+      ...definition.search,
+      ...body,
+    };
+
+    const { ids, keyword, skip = 0, limit, sort, fields, ...rest } = options;
+
+    const query = {};
+
+    if (ids?.length) {
+      query._id = { $in: ids };
+    }
+
+    if (keyword) {
+      Object.assign(query, buildKeywordQuery(keyword, fields));
+    }
+
+    Object.assign(query, normalizeQuery(rest, schema.obj));
+
+    const mQuery = this.find(query)
+      .sort(resolveSort(sort))
+      .skip(skip)
+      .limit(limit);
+
+    // The following construct is awkward but it allows the mongoose query
+    // object to be returned while still ultimately resolving with metadata
+    // so that this method can behave like other find methods and importantly
+    // allow custom population with the same API.
+
+    const runQuery = mQuery.then.bind(mQuery);
+
+    mQuery.then = async (resolve, reject) => {
+      try {
+        const [data, total] = await Promise.all([
+          runQuery(),
+          this.countDocuments(query),
+        ]);
+        resolve({
+          data,
+          meta: {
+            total,
+            skip,
+            limit,
+          },
+        });
+      } catch (err) {
+        reject(err);
+      }
+    };
+
+    return mQuery;
+  });
+}
+
+export function searchValidation(definition, options = {}) {
+  const { limit, sort, ...rest } = {
+    ...SEARCH_DEFAULTS,
+    ...pick(definition.search, 'limit', 'sort'),
+    ...options,
+  };
+
+  return {
+    ids: yd.array(yd.string().mongo()),
+    keyword: yd.string(),
+    include: yd.string(),
+    skip: yd.number().default(0),
+    sort: yd.allow(SORT_SCHEMA, yd.array(SORT_SCHEMA)).default(sort),
+    limit: yd.number().positive().default(limit),
+    ...rest,
+  };
+}
+
+function validateDefinition(definition) {
+  if (Array.isArray(definition.search)) {
+    warn(
+      [
+        '"search" field on model definition must not be an array.',
+        'Use "search.fields" to define fields for keyword queries.',
+      ].join('\n')
+    );
+    throw new Error('Invalid model definition.');
+  }
+}
+
+function resolveSort(sort) {
+  if (!Array.isArray(sort)) {
+    sort = [sort];
+  }
+  return sort.map(({ field, order }) => {
+    return [field, order === 'desc' ? -1 : 1];
+  });
+}
+
+// Keyword queries
+//
+// Mongo supports text indexes, however search operations do not support partial
+// word matches except for stemming rules (eg: "taste", "tastes", and "tasteful").
+//
+// Text indexes are preferred for performance, diacritic handling and more, however
+// for smaller collections partial matches can be manually enabled by specifying an
+// array of "search" fields on the definition:
+//
+// {
+//   "attributes": {
+//     "name": {
+//       "type": "String",
+//       "required": true,
+//       "trim": true
+//     },
+//   },
+//   "search": [
+//     "name",
+//     "description"
+//   ]
+// },
+//
+// Be aware that this may impact performance in which case moving to a text index
+// may be preferable, however partial word matches will stop working. Support for
+// ngram based text search appears to be coming but has no landing date yet.
+//
+// References:
+// https://stackoverflow.com/questions/44833817/mongodb-full-and-partial-text-search
+// https://jira.mongodb.org/browse/SERVER-15090
+
+function buildKeywordQuery(keyword, fields) {
+  if (fields) {
+    return buildRegexQuery(keyword, fields);
+  } else {
+    return buildTextIndexQuery(keyword);
+  }
+}
+
+function buildRegexQuery(keyword, fields) {
+  const queries = fields.map((field) => {
+    const regexKeyword = keyword.replace(/\+/g, '\\+');
+    return {
+      [field]: {
+        $regex: `${regexKeyword}`,
+        $options: 'i',
+      },
+    };
+  });
+  if (ObjectId.isValid(keyword)) {
+    queries.push({ _id: keyword });
+  }
+  return { $or: queries };
+}
+
+function buildTextIndexQuery(keyword) {
+  if (ObjectId.isValid(keyword)) {
+    return {
+      $or: [{ $text: { $search: keyword } }, { _id: keyword }],
+    };
+  } else {
+    return {
+      $text: {
+        $search: keyword,
+      },
+    };
+  }
+}
+
+// Normalizes mongo queries. Flattens plain nested paths
+// to dot syntax while preserving mongo operators and
+// handling specialed query syntax:
+// ranges:
+//  path: { min: n, max n }
+// regex:
+//  path: "/reg/"
+// array:
+//  path; [1,2,3]
+function normalizeQuery(query, schema, root = {}, rootPath = []) {
+  for (let [key, value] of Object.entries(query)) {
+    const path = [...rootPath, key];
+    if (isRangeQuery(schema, key, value)) {
+      if (!isEmpty(value)) {
+        root[path.join('.')] = mapOperatorQuery(value);
+      }
+    } else if (isNestedQuery(key, value)) {
+      normalizeQuery(value, resolveField(schema, key), root, path);
+    } else if (isRegexQuery(key, value)) {
+      root[path.join('.')] = parseRegexQuery(value);
+    } else if (isArrayQuery(key, value)) {
+      root[path.join('.')] = { $in: value };
+    } else {
+      root[path.join('.')] = value;
+    }
+  }
+  return root;
+}
+
+function isNestedQuery(key, value) {
+  if (isMongoOperator(key) || !isPlainObject(value)) {
+    return false;
+  }
+  return Object.keys(value).every((key) => {
+    return !isMongoOperator(key);
+  });
+}
+
+function isArrayQuery(key, value) {
+  return !isMongoOperator(key) && Array.isArray(value);
+}
+
+function isRangeQuery(schema, key, value) {
+  // Range queries only allowed on Date and Number fields.
+  if (!isDateField(schema, key) && !isNumberField(schema, key)) {
+    return false;
+  }
+  return typeof value === 'object';
+}
+
+function mapOperatorQuery(obj) {
+  const query = {};
+  for (let [key, val] of Object.entries(obj)) {
+    query[`$${key}`] = val;
+  }
+  return query;
+}
+
+function isMongoOperator(str) {
+  return str.startsWith('$');
+}
+
+// Regex queries
+
+const REGEX_QUERY = /^\/(.+)\/(\w*)$/;
+
+function isRegexQuery(key, value) {
+  return REGEX_QUERY.test(value);
+}
+
+function parseRegexQuery(str) {
+  // Note that using the $options syntax allows for PCRE features
+  // that aren't supported in Javascript as compared to RegExp(...):
+  // https://docs.mongodb.com/manual/reference/operator/query/regex/#pcre-vs-javascript
+  const [, $regex, $options] = str.match(REGEX_QUERY);
+  return {
+    $regex,
+    $options,
+  };
+}

package/src/serialization.js

@@ -0,0 +1,49 @@
+import { isPlainObject } from 'lodash';
+
+import { checkSelects } from './include';
+import { hasReadAccess } from './access';
+import { resolveField } from './utils';
+
+export const serializeOptions = {
+  getters: true,
+  versionKey: false,
+  transform: (doc, ret, options) => {
+    options.document = doc;
+    checkSelects(doc, ret);
+    transformField(ret, doc.schema.obj, options);
+  },
+};
+
+function transformField(obj, field, options) {
+  if (Array.isArray(obj)) {
+    for (let el of obj) {
+      transformField(el, field, options);
+    }
+  } else if (isPlainObject(obj)) {
+    for (let [key, val] of Object.entries(obj)) {
+      if (!isAllowedField(key, field, options)) {
+        delete obj[key];
+      } else {
+        transformField(val, resolveField(field, key), options);
+      }
+    }
+  }
+}
+
+function isAllowedField(key, field, options) {
+  if (key[0] === '_') {
+    // Strip internal keys like _id and __v
+    return false;
+  } else if (key === 'deleted') {
+    // Strip "deleted" field which defaults
+    // to false and should not be exposed.
+    return false;
+  } else {
+    const { readAccess } = resolveField(field, key);
+    try {
+      return hasReadAccess(readAccess, options);
+    } catch {
+      return false;
+    }
+  }
+}

package/src/slug.js

@@ -0,0 +1,45 @@
+import mongoose from 'mongoose';
+
+const { ObjectId } = mongoose.Types;
+
+export function applySlug(schema) {
+  schema.static('findByIdOrSlug', function findByIdOrSlug(str, ...args) {
+    return find(this, str, args);
+  });
+
+  schema.static(
+    'findByIdOrSlugDeleted',
+    function findByIdOrSlugDeleted(str, ...args) {
+      return find(this, str, args, {
+        deleted: true,
+      });
+    }
+  );
+
+  schema.static(
+    'findByIdOrSlugWithDeleted',
+    function findByIdOrSlugWithDeleted(str, ...args) {
+      return find(this, str, args, {
+        deleted: { $in: [true, false] },
+      });
+    }
+  );
+}
+
+function find(Model, str, args, query) {
+  const isObjectId = str.length === 24 && ObjectId.isValid(str);
+  // There is a non-zero chance of a slug colliding with an ObjectId but
+  // is exceedingly rare (run of exactly 24 [a-f0-9] chars together
+  // without a hyphen) so this should be acceptable.
+  if (!query && isObjectId) {
+    return Model.findById(str, ...args);
+  } else {
+    query = { ...query };
+    if (isObjectId) {
+      query._id = str;
+    } else {
+      query.slug = str;
+    }
+    return Model.findOne(query, ...args);
+  }
+}