objection 3.7.0 → 3.9.0

package/lib/model/AjvValidator.js

@@ -4,8 +4,14 @@ const Ajv = require('ajv');
 const addFormats = require('ajv-formats');
 const { Validator } = require('./Validator');
 const { ValidationErrorType } = require('../model/ValidationError');
-const { isObject, once, cloneDeep: lodashCloneDeep, omit } = require('../utils/objectUtils');
-const { parseFieldExpression } = require('../utils/parseFieldExpression');
+const {
+  isObject,
+  hasOwn,
+  once,
+  cloneDeep: lodashCloneDeep,
+  omit,
+} = require('../utils/objectUtils');
+const { parseFieldExpression, isFieldExpressionKey } = require('../utils/parseFieldExpression');
 const { getQueryPropKeys } = require('./modelQueryProps');
 
 class AjvValidator extends Validator {
@@ -28,7 +34,7 @@ class AjvValidator extends Validator {
 
     // Create an instance that doesn't set default values. We need this one
     // to validate `patch` objects (objects that have a subset of properties).
-    self.ajvNoDefaults = new Ajv(
+    self.ajvNoDefaults = new PatchAjv(
       Object.assign({}, self.ajvOptions, {
         useDefaults: false,
       }),
@@ -79,27 +85,60 @@ class AjvValidator extends Validator {
       return json;
     }
 
-    const modelClass = model.constructor;
-    const validator = this.getValidator(
-      modelClass,
-      ctx.jsonSchema,
-      !!options.patch,
-      options.patch ? null : getMissingQueryPropKeys(model, json, ctx.jsonSchema),
-    );
-
     // We need to clone the input json if we are about to set default values.
     if (!options.mutable && !options.patch && setsDefaultValues(ctx.jsonSchema)) {
       json = cloneDeep(json);
     }
 
-    // Patch objects can contain field expression keys like `'meta:b'` that update
-    // a single property inside a json column. Validate them against the nested
-    // schema by converting them into nested objects like `{ meta: { b: value } }`.
-    const fieldExpressions = options.patch ? nestFieldExpressions(json) : null;
-    const validationJson = fieldExpressions ? fieldExpressions.json : json;
+    // Patch and update objects can contain field expression keys like `'meta:b'`
+    // that update a single property inside a json column. Validate them against
+    // the nested schema by converting them into nested objects like
+    // `{ meta: { b: value } }`.
+    const queryPropKeys = getQueryPropKeys(model);
+    const fieldExpressions = nestFieldExpressions(json, queryPropKeys, ctx.jsonSchema);
+    let validationJson = fieldExpressions ? fieldExpressions.json : json;
+
+    const modelClass = model.constructor;
+    const validate = (json, isPatchObject, givenKeys = []) => {
+      const validator = this.getValidator(
+        modelClass,
+        ctx.jsonSchema,
+        isPatchObject,
+        isPatchObject ? null : getRequiredKeysGivenOutside(json, ctx.jsonSchema, givenKeys),
+      );
+      validator.call(model, json);
+      return validator.errors || [];
+    };
+
+    let errors;
+
+    if (options.patch) {
+      errors = validate(validationJson, true);
+    } else if (fieldExpressions && fieldExpressions.columns.length > 0) {
+      // Field expressions only write parts of their json columns. Validate these
+      // parts like patches, and the rest normally, with the columns counted as
+      // present like the query properties.
+      const { columns } = fieldExpressions;
+      const columnsJson = Object.fromEntries(
+        columns
+          .filter((column) => hasOwn(validationJson, column))
+          .map((column) => [column, validationJson[column]]),
+      );
+      const restJson = omit(validationJson, columns);
 
-    validator.call(model, validationJson);
-    const error = parseValidationError(validator.errors, modelClass, options, this.ajvOptions);
+      errors = [
+        ...validate(columnsJson, true),
+        ...validate(restJson, false, [...queryPropKeys, ...columns]),
+      ];
+
+      // Combine the parts again, without the defaults that Ajv set for the
+      // columns in `restJson`.
+      validationJson = Object.assign(restJson, columnsJson);
+    } else {
+      errors = validate(validationJson, false, queryPropKeys);
+    }
+
+    const error = parseValidationError(errors, modelClass, options, this.ajvOptions);
 
     if (error) {
       throw error;
@@ -107,7 +146,7 @@ class AjvValidator extends Validator {
 
     if (fieldExpressions) {
       // Copy back the changes made by Ajv options like `coerceTypes` and `removeAdditional`.
-      unnestFieldExpressions(json, fieldExpressions);
+      unnestFieldExpressions(json, validationJson, fieldExpressions);
     }
 
     return json;
@@ -148,7 +187,8 @@ class AjvValidator extends Validator {
         normalValidator: null,
 
         // Validators created for the schema object without the `required`
-        // properties that are given as query properties, keyed by their names.
+        // properties that are given outside the json, as query properties or
+        // json columns written by field expressions, keyed by their names.
         queryPropValidators: null,
       };
 
@@ -234,6 +274,20 @@ class AjvValidator extends Validator {
   }
 }
 
+// Patches don't need to contain the `required` properties, also not those of
+// the schemas that model schemas reference by `$id`, like the ones added in
+// `onCreateAjv()`. Strip them from all schemas added to the patch instance:
+class PatchAjv extends Ajv {
+  addSchema(schema, key, meta, ...args) {
+    return super.addSchema(
+      meta || Array.isArray(schema) ? schema : jsonSchemaWithoutRequired(schema),
+      key,
+      meta,
+      ...args,
+    );
+  }
+}
+
 function parseValidationError(errors, modelClass, options, ajvOptions) {
   if (!errors) {
     return null;
@@ -298,10 +352,11 @@ function parseValidationError(errors, modelClass, options, ajvOptions) {
   });
 }
 
-function nestFieldExpressions(json) {
+function nestFieldExpressions(json, queryPropKeys, jsonSchema) {
+  const isFieldExpression = (key) => isFieldExpressionKey(key, jsonSchema.properties);
   const keys = Object.keys(json);
 
-  if (!keys.some(isFieldExpression)) {
+  if (!keys.some(isFieldExpression) && !queryPropKeys.some(isFieldExpression)) {
     return null;
   }
 
@@ -310,6 +365,15 @@ function nestFieldExpressions(json) {
   // The objects created here for the nested paths. Other objects are values
   // of the input json and must not be modified.
   const createdObjects = new Set();
+  // The json columns that are only written partially, by field expressions in
+  // the json or the query properties.
+  const columns = new Set();
+
+  const addColumn = (expression) => {
+    if (expression && !hasOwn(json, expression.column)) {
+      columns.add(expression.column);
+    }
+  };
 
   for (const key of keys) {
     if (!isFieldExpression(key)) {
@@ -320,7 +384,10 @@ function nestFieldExpressions(json) {
 
   for (const key of keys) {
     if (isFieldExpression(key)) {
-      const path = fieldExpressionPath(key);
+      const expression = parseFieldExpressionKey(key);
+      // Not a valid field expression, validate it as a top-level property:
+      const path = expression ? expression.path : [key];
+      addColumn(expression);
 
       // Field expressions with array access, and those that conflict with other
       // keys of the patch, can't be validated without the current value of the
@@ -331,10 +398,16 @@ function nestFieldExpressions(json) {
     }
   }
 
-  return { json: nested, paths };
+  for (const key of queryPropKeys) {
+    if (isFieldExpression(key)) {
+      addColumn(parseFieldExpressionKey(key));
+    }
+  }
+
+  return { json: nested, paths, columns: [...columns] };
 }
 
-function unnestFieldExpressions(json, { json: nested, paths }) {
+function unnestFieldExpressions(json, nested, { paths, columns }) {
   for (const [key, path] of paths) {
     let value = nested;
     let found = true;
@@ -356,29 +429,35 @@ function unnestFieldExpressions(json, { json: nested, paths }) {
       json[key] = value;
     }
   }
-}
 
-function isFieldExpression(key) {
-  return key.indexOf(':') !== -1;
+  // Copy over the default values that Ajv set for missing properties, but not
+  // the json columns written by the field expressions.
+  for (const key of Object.keys(nested)) {
+    if (!paths.has(key) && !columns.includes(key)) {
+      json[key] = nested[key];
+    }
+  }
 }
 
-function fieldExpressionPath(key) {
+// Parse a field expression key into its json column and nested path, with the
+// path set to `null` if the key accesses array elements.
+function parseFieldExpressionKey(key) {
   let parsed;
 
   try {
     parsed = parseFieldExpression(key);
   } catch (err) {
-    // Not a valid field expression, validate it as a top-level property.
-    return [key];
+    return null;
   }
 
+  const { column, access } = parsed;
   // Numeric keys like the `0` in `tags:0` also access array elements when the
   // json value is an array, so treat them like `[0]`.
-  if (parsed.access.some((access) => access.type !== 'object' || /^-?\d+$/.test(access.ref))) {
-    return null;
-  }
-
-  return [parsed.column, ...parsed.access.map((access) => String(access.ref))];
+  const isArrayAccess = access.some(({ type, ref }) => type !== 'object' || /^-?\d+$/.test(ref));
+  return {
+    column,
+    path: isArrayAccess ? null : [column, ...access.map(({ ref }) => String(ref))],
+  };
 }
 
 function setPath(obj, path, value, createdObjects) {
@@ -406,10 +485,6 @@ function setPath(obj, path, value, createdObjects) {
   return true;
 }
 
-function hasOwn(obj, prop) {
-  return Object.prototype.hasOwnProperty.call(obj, prop);
-}
-
 function cloneDeep(obj) {
   if (isObject(obj) && obj.$isObjectionModel) {
     return obj.$clone();
@@ -462,22 +537,20 @@ function objectHasDefaults(obj) {
   return false;
 }
 
-// Returns the keys of the query properties of `model` (`raw()`, `val()`,
-// `ref()`, subqueries...) that are required by the schema but missing from
-// `json`, sorted, or `null` if there are none. Query properties are split from
-// the json before validation, but they do provide a value for the property.
-function getMissingQueryPropKeys(model, json, jsonSchema) {
+// Returns the `keys` that are required by the schema but given outside of
+// `json`, sorted, or `null` if there are none. Query properties (`raw()`,
+// `val()`, `ref()`, subqueries...) are split from the json before validation,
+// but they do provide a value for the property, and so do field expressions
+// for their json columns.
+function getRequiredKeysGivenOutside(json, jsonSchema, keys) {
   const required = jsonSchema.required;
 
   if (!Array.isArray(required) || required.length === 0) {
     return null;
   }
 
-  const keys = getQueryPropKeys(model).filter(
-    (key) => required.includes(key) && !Object.prototype.hasOwnProperty.call(json, key),
-  );
-
-  return keys.length > 0 ? keys.sort() : null;
+  const givenKeys = required.filter((key) => keys.includes(key) && !hasOwn(json, key));
+  return givenKeys.length > 0 ? givenKeys.sort() : null;
 }
 
 function jsonSchemaWithoutRequiredKeys(jsonSchema, keys) {

package/lib/model/Model.js

@@ -11,6 +11,7 @@ const { toJson, toDatabaseJson } = require('./modelToJson');
 const { values, propKey, hasProps } = require('./modelValues');
 const { defineNonEnumerableProperty } = require('./modelUtils');
 const { parseRelationsIntoModelInstances } = require('./modelParseRelations');
+const { removeOverriddenQueryProps } = require('./modelQueryProps');
 const { fetchTableMetadata, tableMetadata } = require('./modelTableMetadata');
 const { asArray, isFunction, isString, asSingle } = require('../utils/objectUtils');
 const { setJson, setFast, setRelated, appendRelated, setDatabaseJson } = require('./modelSet');
@@ -576,6 +577,7 @@ class Model {
     }
 
     if (model instanceof modelClass) {
+      removeOverriddenQueryProps(model);
       return parseRelationsIntoModelInstances(model, model, options);
     } else {
       return modelClass.fromJson(model, options);

package/lib/model/graph/ModelGraphNode.js

@@ -240,7 +240,8 @@ class ModelGraphNode {
     const idPath = this.idPath;
 
     if (idPath) {
-      return this.idPath.join('.');
+      // JSON keeps the ids and relation names apart, whatever they contain.
+      return JSON.stringify(idPath);
     } else {
       return null;
     }

package/lib/model/modelJsonAttributes.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { asArray, isObject, flatten, isString } = require('../utils/objectUtils');
+const { isQueryProp } = require('./modelQueryProps');
 
 function parseJsonAttributes(json, modelClass) {
   const jsonAttr = modelClass.getJsonAttributes();
@@ -31,11 +32,12 @@ function formatJsonAttributes(json, modelClass) {
 
   if (jsonAttr.length) {
     // All database clients want JSON columns as strings. Do the conversion here.
+    // Leave query properties set in hooks like `$beforeUpdate()` as they are.
     for (let i = 0, l = jsonAttr.length; i < l; ++i) {
       const attr = jsonAttr[i];
       const value = json[attr];
 
-      if (value != null) {
+      if (value != null && !isQueryProp(value)) {
         json[attr] = JSON.stringify(value);
       }
     }
@@ -57,16 +59,7 @@ function getJsonAttributes(modelClass) {
     const props = modelClass.getJsonSchema().properties || {};
 
     for (const propName of Object.keys(props)) {
-      const prop = props[propName];
-      let types = asArray(prop.type).filter((it) => !!it);
-
-      if (types.length === 0 && Array.isArray(prop.anyOf)) {
-        types = flatten(prop.anyOf.map((it) => it.type));
-      }
-
-      if (types.length === 0 && Array.isArray(prop.oneOf)) {
-        types = flatten(prop.oneOf.map((it) => it.type));
-      }
+      const types = getPropertyTypes(props[propName]);
 
       if (types.indexOf('object') !== -1 || types.indexOf('array') !== -1) {
         jsonAttributes.push(propName);
@@ -77,6 +70,21 @@ function getJsonAttributes(modelClass) {
   return jsonAttributes;
 }
 
+// The types of a property schema, from `type` or else from `anyOf` / `oneOf`.
+function getPropertyTypes(prop) {
+  let types = asArray(prop.type).filter((it) => !!it);
+
+  if (types.length === 0 && Array.isArray(prop.anyOf)) {
+    types = flatten(prop.anyOf.map((it) => it.type));
+  }
+
+  if (types.length === 0 && Array.isArray(prop.oneOf)) {
+    types = flatten(prop.oneOf.map((it) => it.type));
+  }
+
+  return types;
+}
+
 function tryParseJson(maybeJsonStr) {
   try {
     return JSON.parse(maybeJsonStr);
@@ -89,4 +97,5 @@ module.exports = {
   parseJsonAttributes,
   formatJsonAttributes,
   getJsonAttributes,
+  getPropertyTypes,
 };

package/lib/model/modelQueryProps.js

@@ -1,18 +1,22 @@
 'use strict';
 
-const { isObject, isFunction } = require('../utils/objectUtils');
+const { isObject, isFunction, omit } = require('../utils/objectUtils');
 const { defineNonEnumerableProperty } = require('./modelUtils');
 const { isKnexRaw, isKnexQueryBuilder } = require('../utils/knexUtils');
+const { parseFieldExpression, isFieldExpressionKey } = require('../utils/parseFieldExpression');
 
 const QUERY_PROPS_PROPERTY = '$$queryProps';
 
-// Removes query properties from `json` and stores them into a hidden property
-// inside `model` so that they can be later merged back to `json`.
+// Moves the query properties of `json` into a hidden property of `model` so
+// that they can be later merged back to its database json. A key holds either
+// a value or a query property, so the keys given in `json` replace the ones
+// that the model holds as the other kind.
 function splitQueryProps(model, json) {
   const keys = Object.keys(json);
+  const oldQueryProps = model[QUERY_PROPS_PROPERTY];
 
-  if (hasQueryProps(json, keys)) {
-    const queryProps = {};
+  if (!json.$isObjectionModel && hasQueryProps(json, keys)) {
+    const queryProps = Object.assign({}, oldQueryProps);
     const modelProps = {};
 
     for (let i = 0, l = keys.length; i < l; ++i) {
@@ -21,8 +25,10 @@ function splitQueryProps(model, json) {
 
       if (isQueryProp(value)) {
         queryProps[key] = value;
+        delete model[key];
       } else {
         modelProps[key] = value;
+        delete queryProps[key];
       }
     }
 
@@ -30,10 +36,45 @@ function splitQueryProps(model, json) {
 
     return modelProps;
   } else {
+    if (oldQueryProps) {
+      defineNonEnumerableProperty(model, QUERY_PROPS_PROPERTY, omit(oldQueryProps, keys));
+    }
+
     return json;
   }
 }
 
+// Omits the values that parsing and validation add to `json` for the keys that
+// `model` holds query properties for, like Ajv defaults.
+function omitQueryProps(model, json) {
+  const keys = getQueryPropKeys(model);
+  return keys.length > 0 ? omit(json, keys) : json;
+}
+
+// Removes the query properties of the keys that were given a value since, by an
+// assignment, `$set()` or a write that fetched the value from the database. We
+// do this when the model is passed to a write, and not when its database json
+// is built, so that the values set by its hooks don't override them.
+function removeOverriddenQueryProps(model) {
+  const queryProps = model[QUERY_PROPS_PROPERTY];
+
+  if (queryProps) {
+    const jsonSchema = model.constructor.getJsonSchema();
+    const properties = jsonSchema && jsonSchema.properties;
+    // The json column of a field expression key like `data:a` overrides it too:
+    const keys = Object.keys(queryProps).filter(
+      (key) =>
+        model[key] !== undefined ||
+        (isFieldExpressionKey(key, properties) &&
+          model[parseFieldExpression(key).column] !== undefined),
+    );
+
+    if (keys.length > 0) {
+      defineNonEnumerableProperty(model, QUERY_PROPS_PROPERTY, omit(queryProps, keys));
+    }
+  }
+}
+
 function hasQueryProps(json, keys) {
   for (let i = 0, l = keys.length; i < l; ++i) {
     if (isQueryProp(json[keys[i]])) {
@@ -149,6 +190,9 @@ function isKnexRawConvertable(queryProp) {
 
 module.exports = {
   splitQueryProps,
+  omitQueryProps,
+  removeOverriddenQueryProps,
   mergeQueryProps,
   getQueryPropKeys,
+  isQueryProp,
 };

package/lib/model/modelSet.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { isInternalProp } = require('../utils/internalPropUtils');
-const { splitQueryProps } = require('./modelQueryProps');
+const { splitQueryProps, omitQueryProps } = require('./modelQueryProps');
 const { isFunction, isString, isPlainObject } = require('../utils/objectUtils');
 const { parseRelationsIntoModelInstances } = require('./modelParseRelations');
 
@@ -17,15 +17,13 @@ function setJson(model, json, options) {
     );
   }
 
-  if (!json.$isObjectionModel) {
-    // json can contain "query properties" like `raw` instances, query builders etc.
-    // We take them out of `json` and store them to a hidden property $$queryProps
-    // in the model instance for later use.
-    json = splitQueryProps(model, json);
-  }
-
+  // json can contain "query properties" like `raw` instances, query builders etc.
+  // We take them out of `json` and store them to a hidden property $$queryProps
+  // in the model instance for later use.
+  json = splitQueryProps(model, json);
   json = model.$parseJson(json, options);
   json = model.$validate(json, options);
+  json = omitQueryProps(model, json);
 
   model.$set(json);
 

package/lib/model/modelToJson.js

@@ -2,7 +2,8 @@
 
 const { isInternalProp } = require('../utils/internalPropUtils');
 const { mergeQueryProps } = require('./modelQueryProps');
-const { isObject, cloneDeep, isFunction } = require('../utils/objectUtils');
+const { isObject, hasOwn, cloneDeep, isFunction } = require('../utils/objectUtils');
+const { parseFieldExpression, isFieldExpressionKey } = require('../utils/parseFieldExpression');
 const EMPTY_ARRAY = [];
 
 function toJson(model, optIn) {
@@ -134,7 +135,21 @@ function shouldOmit(opt, key) {
 }
 
 function shouldPick(opt, key) {
-  return opt.pick === null || key in opt.pick;
+  return opt.pick === null || hasOwn(opt.pick, key) || isPickedFieldExpression(opt.pick, key);
+}
+
+// Pick field expressions like `'meta:a'` by their json column.
+function isPickedFieldExpression(pick, key) {
+  if (isFieldExpressionKey(key, pick)) {
+    try {
+      return hasOwn(pick, parseFieldExpression(key).column);
+    } catch (err) {
+      // Not a valid field expression.
+      return false;
+    }
+  } else {
+    return false;
+  }
 }
 
 function assignVirtualAttributes(json, model, vAttr, opt) {

package/lib/model/modelValidate.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { clone } = require('./modelClone');
+const { omitQueryProps } = require('./modelQueryProps');
 
 function validate(model, json, options = {}) {
   json = json || model;
@@ -37,7 +38,7 @@ function validate(model, json, options = {}) {
 
   if (validatingModelInstance) {
     // If we cloned `json`, we need to copy the possible default values.
-    return inputJson.$set(json);
+    return inputJson.$set(omitQueryProps(inputJson, json));
   } else {
     return json;
   }

package/lib/model/modelValues.js

@@ -19,16 +19,33 @@ function values(model, args) {
   }
 }
 
+// Turn the props of a model into a key to look up models by.
 function propKey(model, props) {
-  switch (props.length) {
-    case 1:
-      return propKey1(model, props);
-    case 2:
-      return propKey2(model, props);
-    case 3:
-      return propKey3(model, props);
-    default:
-      return propKeyN(model, props);
+  if (props.length === 1) {
+    return PROP_KEY_PREFIX + propToStr(model[props[0]]);
+  } else {
+    let key = PROP_KEY_PREFIX + compositePropToStr(model[props[0]]);
+
+    for (let i = 1, l = props.length; i < l; ++i) {
+      key += ',' + compositePropToStr(model[props[i]]);
+    }
+
+    return key;
+  }
+}
+
+// Same as `propKey()`, for values that were already collected.
+function valuesKey(values) {
+  if (values.length === 1) {
+    return PROP_KEY_PREFIX + propToStr(values[0]);
+  } else {
+    let key = PROP_KEY_PREFIX + compositePropToStr(values[0]);
+
+    for (let i = 1, l = values.length; i < l; ++i) {
+      key += ',' + compositePropToStr(values[i]);
+    }
+
+    return key;
   }
 }
 
@@ -66,50 +83,27 @@ function valuesN(model, args) {
   return ret;
 }
 
-function propKey1(model, props) {
-  return PROP_KEY_PREFIX + propToStr(model[props[0]]);
-}
-
-function propKey2(model, props) {
-  return PROP_KEY_PREFIX + propToStr(model[props[0]]) + ',' + propToStr(model[props[1]]);
-}
-
-function propKey3(model, props) {
-  return (
-    PROP_KEY_PREFIX +
-    propToStr(model[props[0]]) +
-    ',' +
-    propToStr(model[props[1]]) +
-    ',' +
-    propToStr(model[props[2]])
-  );
-}
-
-function propKeyN(model, props) {
-  let key = PROP_KEY_PREFIX;
-
-  for (let i = 0, l = props.length; i < l; ++i) {
-    key += propToStr(model[props[i]]);
-
-    if (i < l - 1) {
-      key += ',';
-    }
-  }
-
-  return key;
-}
-
 function propToStr(value) {
-  if (value === null) {
-    return 'null';
-  } else if (value === undefined) {
-    return 'undefined';
+  if (!isObject(value)) {
+    return `${value}`;
+  } else if (value instanceof Date) {
+    // Key dates by their time, as some drivers return them as numbers.
+    return `${value.getTime()}`;
   } else if (Buffer.isBuffer(value)) {
     return value.toString('hex');
-  } else if (isObject(value)) {
-    return JSON.stringify(value);
   } else {
+    return JSON.stringify(value);
+  }
+}
+
+// Escape the separators in composite keys, since strings can contain them.
+// Numbers can't, so we don't need to check them.
+function compositePropToStr(value) {
+  if (typeof value === 'number') {
     return `${value}`;
+  } else {
+    const str = propToStr(value);
+    return str.includes(',') || str.includes('\\') ? str.replace(/[,\\]/g, '\\$&') : str;
   }
 }
 
@@ -118,9 +112,8 @@ function isNullOrUndefined(val) {
 }
 
 module.exports = {
-  PROP_KEY_PREFIX,
-  propToStr,
   values,
   hasProps,
   propKey,
+  valuesKey,
 };