objection 3.8.0 → 3.9.1

package/lib/model/AjvValidator.js

@@ -4,7 +4,13 @@ const Ajv = require('ajv');
 const addFormats = require('ajv-formats');
 const { Validator } = require('./Validator');
 const { ValidationErrorType } = require('../model/ValidationError');
-const { isObject, once, cloneDeep: lodashCloneDeep, omit } = require('../utils/objectUtils');
+const {
+  isObject,
+  hasOwn,
+  once,
+  cloneDeep: lodashCloneDeep,
+  omit,
+} = require('../utils/objectUtils');
 const { parseFieldExpression, isFieldExpressionKey } = require('../utils/parseFieldExpression');
 const { getQueryPropKeys } = require('./modelQueryProps');
 
@@ -28,7 +34,7 @@ class AjvValidator extends Validator {
 
     // Create an instance that doesn't set default values. We need this one
     // to validate `patch` objects (objects that have a subset of properties).
-    self.ajvNoDefaults = new Ajv(
+    self.ajvNoDefaults = new PatchAjv(
       Object.assign({}, self.ajvOptions, {
         useDefaults: false,
       }),
@@ -268,6 +274,20 @@ class AjvValidator extends Validator {
   }
 }
 
+// Patches don't need to contain the `required` properties, also not those of
+// the schemas that model schemas reference by `$id`, like the ones added in
+// `onCreateAjv()`. Strip them from all schemas added to the patch instance:
+class PatchAjv extends Ajv {
+  addSchema(schema, key, meta, ...args) {
+    return super.addSchema(
+      meta || Array.isArray(schema) ? schema : jsonSchemaWithoutRequired(schema),
+      key,
+      meta,
+      ...args,
+    );
+  }
+}
+
 function parseValidationError(errors, modelClass, options, ajvOptions) {
   if (!errors) {
     return null;
@@ -465,10 +485,6 @@ function setPath(obj, path, value, createdObjects) {
   return true;
 }
 
-function hasOwn(obj, prop) {
-  return Object.prototype.hasOwnProperty.call(obj, prop);
-}
-
 function cloneDeep(obj) {
   if (isObject(obj) && obj.$isObjectionModel) {
     return obj.$clone();

package/lib/model/Model.js

@@ -11,6 +11,7 @@ const { toJson, toDatabaseJson } = require('./modelToJson');
 const { values, propKey, hasProps } = require('./modelValues');
 const { defineNonEnumerableProperty } = require('./modelUtils');
 const { parseRelationsIntoModelInstances } = require('./modelParseRelations');
+const { removeOverriddenQueryProps } = require('./modelQueryProps');
 const { fetchTableMetadata, tableMetadata } = require('./modelTableMetadata');
 const { asArray, isFunction, isString, asSingle } = require('../utils/objectUtils');
 const { setJson, setFast, setRelated, appendRelated, setDatabaseJson } = require('./modelSet');
@@ -576,6 +577,7 @@ class Model {
     }
 
     if (model instanceof modelClass) {
+      removeOverriddenQueryProps(model);
       return parseRelationsIntoModelInstances(model, model, options);
     } else {
       return modelClass.fromJson(model, options);

package/lib/model/graph/ModelGraphNode.js

@@ -143,21 +143,6 @@ class ModelGraphNode {
     return null;
   }
 
-  /**
-   * Returns all nodes that are references to this node.
-   */
-  get referencingNodes() {
-    const nodes = [];
-
-    for (const edge of this.refEdges) {
-      if (edge.refType === ModelGraphEdge.ReferenceType.Object && edge.isRelatedNode(this)) {
-        nodes.push(edge.ownerNode);
-      }
-    }
-
-    return nodes;
-  }
-
   get descendantRelationNodes() {
     return this._collectDescendantRelationNodes([]);
   }
@@ -240,7 +225,8 @@ class ModelGraphNode {
     const idPath = this.idPath;
 
     if (idPath) {
-      return this.idPath.join('.');
+      // JSON keeps the ids and relation names apart, whatever they contain.
+      return JSON.stringify(idPath);
     } else {
       return null;
     }

package/lib/model/modelJsonAttributes.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { asArray, isObject, flatten, isString } = require('../utils/objectUtils');
+const { isQueryProp } = require('./modelQueryProps');
 
 function parseJsonAttributes(json, modelClass) {
   const jsonAttr = modelClass.getJsonAttributes();
@@ -31,11 +32,12 @@ function formatJsonAttributes(json, modelClass) {
 
   if (jsonAttr.length) {
     // All database clients want JSON columns as strings. Do the conversion here.
+    // Leave query properties set in hooks like `$beforeUpdate()` as they are.
     for (let i = 0, l = jsonAttr.length; i < l; ++i) {
       const attr = jsonAttr[i];
       const value = json[attr];
 
-      if (value != null) {
+      if (value != null && !isQueryProp(value)) {
         json[attr] = JSON.stringify(value);
       }
     }
@@ -57,16 +59,7 @@ function getJsonAttributes(modelClass) {
     const props = modelClass.getJsonSchema().properties || {};
 
     for (const propName of Object.keys(props)) {
-      const prop = props[propName];
-      let types = asArray(prop.type).filter((it) => !!it);
-
-      if (types.length === 0 && Array.isArray(prop.anyOf)) {
-        types = flatten(prop.anyOf.map((it) => it.type));
-      }
-
-      if (types.length === 0 && Array.isArray(prop.oneOf)) {
-        types = flatten(prop.oneOf.map((it) => it.type));
-      }
+      const types = getPropertyTypes(props[propName]);
 
       if (types.indexOf('object') !== -1 || types.indexOf('array') !== -1) {
         jsonAttributes.push(propName);
@@ -77,6 +70,21 @@ function getJsonAttributes(modelClass) {
   return jsonAttributes;
 }
 
+// The types of a property schema, from `type` or else from `anyOf` / `oneOf`.
+function getPropertyTypes(prop) {
+  let types = asArray(prop.type).filter((it) => !!it);
+
+  if (types.length === 0 && Array.isArray(prop.anyOf)) {
+    types = flatten(prop.anyOf.map((it) => it.type));
+  }
+
+  if (types.length === 0 && Array.isArray(prop.oneOf)) {
+    types = flatten(prop.oneOf.map((it) => it.type));
+  }
+
+  return types;
+}
+
 function tryParseJson(maybeJsonStr) {
   try {
     return JSON.parse(maybeJsonStr);
@@ -89,4 +97,5 @@ module.exports = {
   parseJsonAttributes,
   formatJsonAttributes,
   getJsonAttributes,
+  getPropertyTypes,
 };

package/lib/model/modelQueryProps.js

@@ -1,39 +1,107 @@
 'use strict';
 
-const { isObject, isFunction } = require('../utils/objectUtils');
+const { isObject, isFunction, omit } = require('../utils/objectUtils');
 const { defineNonEnumerableProperty } = require('./modelUtils');
 const { isKnexRaw, isKnexQueryBuilder } = require('../utils/knexUtils');
+const { getFieldExpressionColumn } = require('../utils/parseFieldExpression');
 
 const QUERY_PROPS_PROPERTY = '$$queryProps';
-
-// Removes query properties from `json` and stores them into a hidden property
-// inside `model` so that they can be later merged back to `json`.
+// The keys of the field expression query properties like `'data:a'` that were
+// given while their json column `data` had a value:
+const COMBINED_QUERY_PROPS_PROPERTY = '$$combinedQueryProps';
+
+// Moves the query properties of `json` into a hidden property of `model` so
+// that they can be later merged back to its database json. A key holds either
+// a value or a query property, so the keys given in `json` replace the ones
+// that the model holds as the other kind.
 function splitQueryProps(model, json) {
   const keys = Object.keys(json);
+  const oldQueryProps = model[QUERY_PROPS_PROPERTY];
 
-  if (hasQueryProps(json, keys)) {
-    const queryProps = {};
+  if (!json.$isObjectionModel && hasQueryProps(json, keys)) {
+    const queryProps = Object.assign({}, oldQueryProps);
+    const combinedKeys = new Set(model[COMBINED_QUERY_PROPS_PROPERTY]);
     const modelProps = {};
+    const jsonSchema = model.constructor.getJsonSchema();
+    const properties = jsonSchema && jsonSchema.properties;
 
     for (let i = 0, l = keys.length; i < l; ++i) {
       const key = keys[i];
       const value = json[key];
 
       if (isQueryProp(value)) {
+        const column = getFieldExpressionColumn(key, properties);
+
+        if (column !== null && (json[column] !== undefined || model[column] !== undefined)) {
+          combinedKeys.add(key);
+        } else {
+          combinedKeys.delete(key);
+        }
+
         queryProps[key] = value;
+        delete model[key];
       } else {
         modelProps[key] = value;
+        delete queryProps[key];
       }
     }
 
     defineNonEnumerableProperty(model, QUERY_PROPS_PROPERTY, queryProps);
+    defineNonEnumerableProperty(model, COMBINED_QUERY_PROPS_PROPERTY, [...combinedKeys]);
 
     return modelProps;
   } else {
+    if (oldQueryProps) {
+      defineNonEnumerableProperty(model, QUERY_PROPS_PROPERTY, omit(oldQueryProps, keys));
+    }
+
     return json;
   }
 }
 
+// Omits the values that parsing and validation add to `json` for the keys that
+// `model` holds query properties for, like Ajv defaults.
+function omitQueryProps(model, json) {
+  const keys = getQueryPropKeys(model);
+  return keys.length > 0 ? omit(json, keys) : json;
+}
+
+// Removes the query properties of the keys that were given a value since, by an
+// assignment, `$set()` or a write that fetched the value from the database. We
+// do this when the model is passed to a write, and not when its database json
+// is built, so that the values set by its hooks don't override them.
+function removeOverriddenQueryProps(model) {
+  const queryProps = model[QUERY_PROPS_PROPERTY];
+
+  if (queryProps) {
+    const jsonSchema = model.constructor.getJsonSchema();
+    const properties = jsonSchema && jsonSchema.properties;
+    const combinedKeys = model[COMBINED_QUERY_PROPS_PROPERTY] || [];
+    const keys = Object.keys(queryProps).filter((key) => {
+      const column = getFieldExpressionColumn(key, properties);
+      return (
+        model[key] !== undefined ||
+        // The json column of a field expression key like `data:a` overrides it
+        // too, unless it was given while the column had a value (#206):
+        (column !== null && !combinedKeys.includes(key) && model[column] !== undefined)
+      );
+    });
+
+    if (keys.length > 0) {
+      defineNonEnumerableProperty(model, QUERY_PROPS_PROPERTY, omit(queryProps, keys));
+    }
+  }
+}
+
+// Removes the values that the hooks of a write set for the keys that `model`
+// holds query properties for. The query properties are written instead, so the
+// values must not count as given in later writes.
+function removeOverriddenValues(model) {
+  for (const key of getQueryPropKeys(model)) {
+    delete model[key];
+  }
+}
+
 function hasQueryProps(json, keys) {
   for (let i = 0, l = keys.length; i < l; ++i) {
     if (isQueryProp(json[keys[i]])) {
@@ -149,6 +217,10 @@ function isKnexRawConvertable(queryProp) {
 
 module.exports = {
   splitQueryProps,
+  omitQueryProps,
+  removeOverriddenQueryProps,
+  removeOverriddenValues,
   mergeQueryProps,
   getQueryPropKeys,
+  isQueryProp,
 };

package/lib/model/modelSet.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const { isInternalProp } = require('../utils/internalPropUtils');
-const { splitQueryProps } = require('./modelQueryProps');
+const { splitQueryProps, omitQueryProps } = require('./modelQueryProps');
 const { isFunction, isString, isPlainObject } = require('../utils/objectUtils');
 const { parseRelationsIntoModelInstances } = require('./modelParseRelations');
 
@@ -17,15 +17,13 @@ function setJson(model, json, options) {
     );
   }
 
-  if (!json.$isObjectionModel) {
-    // json can contain "query properties" like `raw` instances, query builders etc.
-    // We take them out of `json` and store them to a hidden property $$queryProps
-    // in the model instance for later use.
-    json = splitQueryProps(model, json);
-  }
-
+  // json can contain "query properties" like `raw` instances, query builders etc.
+  // We take them out of `json` and store them to a hidden property $$queryProps
+  // in the model instance for later use.
+  json = splitQueryProps(model, json);
   json = model.$parseJson(json, options);
   json = model.$validate(json, options);
+  json = omitQueryProps(model, json);
 
   model.$set(json);
 

package/lib/model/modelToJson.js

@@ -2,7 +2,8 @@
 
 const { isInternalProp } = require('../utils/internalPropUtils');
 const { mergeQueryProps } = require('./modelQueryProps');
-const { isObject, cloneDeep, isFunction } = require('../utils/objectUtils');
+const { isObject, hasOwn, cloneDeep, isFunction } = require('../utils/objectUtils');
+const { getFieldExpressionColumn } = require('../utils/parseFieldExpression');
 const EMPTY_ARRAY = [];
 
 function toJson(model, optIn) {
@@ -134,7 +135,13 @@ function shouldOmit(opt, key) {
 }
 
 function shouldPick(opt, key) {
-  return opt.pick === null || key in opt.pick;
+  return opt.pick === null || hasOwn(opt.pick, key) || isPickedFieldExpression(opt.pick, key);
+}
+
+// Pick field expressions like `'meta:a'` by their json column.
+function isPickedFieldExpression(pick, key) {
+  const column = getFieldExpressionColumn(key, pick);
+  return column !== null && hasOwn(pick, column);
 }
 
 function assignVirtualAttributes(json, model, vAttr, opt) {

package/lib/model/modelUtils.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const hiddenProps = ['$$queryProps'];
+const hiddenProps = ['$$queryProps', '$$combinedQueryProps'];
 
 const staticHiddenProps = [
   '$$knex',

package/lib/model/modelValidate.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { clone } = require('./modelClone');
+const { omitQueryProps } = require('./modelQueryProps');
 
 function validate(model, json, options = {}) {
   json = json || model;
@@ -37,7 +38,7 @@ function validate(model, json, options = {}) {
 
   if (validatingModelInstance) {
     // If we cloned `json`, we need to copy the possible default values.
-    return inputJson.$set(json);
+    return inputJson.$set(omitQueryProps(inputJson, json));
   } else {
     return json;
   }

package/lib/model/modelValues.js

@@ -19,16 +19,33 @@ function values(model, args) {
   }
 }
 
+// Turn the props of a model into a key to look up models by.
 function propKey(model, props) {
-  switch (props.length) {
-    case 1:
-      return propKey1(model, props);
-    case 2:
-      return propKey2(model, props);
-    case 3:
-      return propKey3(model, props);
-    default:
-      return propKeyN(model, props);
+  if (props.length === 1) {
+    return PROP_KEY_PREFIX + propToStr(model[props[0]]);
+  } else {
+    let key = PROP_KEY_PREFIX + compositePropToStr(model[props[0]]);
+
+    for (let i = 1, l = props.length; i < l; ++i) {
+      key += ',' + compositePropToStr(model[props[i]]);
+    }
+
+    return key;
+  }
+}
+
+// Same as `propKey()`, for values that were already collected.
+function valuesKey(values) {
+  if (values.length === 1) {
+    return PROP_KEY_PREFIX + propToStr(values[0]);
+  } else {
+    let key = PROP_KEY_PREFIX + compositePropToStr(values[0]);
+
+    for (let i = 1, l = values.length; i < l; ++i) {
+      key += ',' + compositePropToStr(values[i]);
+    }
+
+    return key;
   }
 }
 
@@ -66,50 +83,27 @@ function valuesN(model, args) {
   return ret;
 }
 
-function propKey1(model, props) {
-  return PROP_KEY_PREFIX + propToStr(model[props[0]]);
-}
-
-function propKey2(model, props) {
-  return PROP_KEY_PREFIX + propToStr(model[props[0]]) + ',' + propToStr(model[props[1]]);
-}
-
-function propKey3(model, props) {
-  return (
-    PROP_KEY_PREFIX +
-    propToStr(model[props[0]]) +
-    ',' +
-    propToStr(model[props[1]]) +
-    ',' +
-    propToStr(model[props[2]])
-  );
-}
-
-function propKeyN(model, props) {
-  let key = PROP_KEY_PREFIX;
-
-  for (let i = 0, l = props.length; i < l; ++i) {
-    key += propToStr(model[props[i]]);
-
-    if (i < l - 1) {
-      key += ',';
-    }
-  }
-
-  return key;
-}
-
 function propToStr(value) {
-  if (value === null) {
-    return 'null';
-  } else if (value === undefined) {
-    return 'undefined';
+  if (!isObject(value)) {
+    return `${value}`;
+  } else if (value instanceof Date) {
+    // Key dates by their time, as some drivers return them as numbers.
+    return `${value.getTime()}`;
   } else if (Buffer.isBuffer(value)) {
     return value.toString('hex');
-  } else if (isObject(value)) {
-    return JSON.stringify(value);
   } else {
+    return JSON.stringify(value);
+  }
+}
+
+// Escape the separators in composite keys, since strings can contain them.
+// Numbers can't, so we don't need to check them.
+function compositePropToStr(value) {
+  if (typeof value === 'number') {
     return `${value}`;
+  } else {
+    const str = propToStr(value);
+    return str.includes(',') || str.includes('\\') ? str.replace(/[,\\]/g, '\\$&') : str;
   }
 }
 
@@ -118,9 +112,8 @@ function isNullOrUndefined(val) {
 }
 
 module.exports = {
-  PROP_KEY_PREFIX,
-  propToStr,
   values,
   hasProps,
   propKey,
+  valuesKey,
 };

package/lib/queryBuilder/QueryBuilder.js

@@ -50,6 +50,7 @@ const { LimitPerParentOperation } = require('./operations/LimitPerParentOperatio
 const { FromOperation } = require('./operations/FromOperation');
 const { KnexOperation } = require('./operations/KnexOperation');
 const { clearUnsupportedOrder } = require('./withoutOrder');
+const { addWhereGroup } = require('./whereGroups');
 
 class QueryBuilder extends QueryBuilderBase {
   static init(self, modelClass) {
@@ -512,7 +513,10 @@ class QueryBuilder extends QueryBuilderBase {
 
   async resultSize() {
     const knex = this.knex();
-    const builder = this.clone().clear(/orderBy|offset|limit/);
+    // `distinctOn` picks the first row of each group in the order of the query,
+    // which decides which root models are counted with to-many joins.
+    const shouldKeepOrder = this.has('distinctOn');
+    const builder = this.clone().clear(shouldKeepOrder ? /offset|limit/ : /orderBy|offset|limit/);
     const joinEagerOp = builder.findOperation(JoinEagerOperation);
 
     if (joinEagerOp) {
@@ -530,16 +534,20 @@ class QueryBuilder extends QueryBuilderBase {
 
     // Also clear the order on the built knex query, since `orderBy` calls
     // can be added at build time, e.g. by a relation's `modify`.
-    const buildQuery = (knexBuilder) => builder.toKnexQuery(knexBuilder).clearOrder();
+    const buildQuery = (knexBuilder) => {
+      const query = builder.toKnexQuery(knexBuilder);
+      return shouldKeepOrder ? query : query.clearOrder();
+    };
 
     const countQuery = knex.count('* as count').from((knexBuilder) => {
       if (!joinEagerOp || idColumns.length === 0) {
         // Without ids, the joins are all to-one and return one row per root
         // model.
         buildQuery(knexBuilder);
-      } else if (refersToSelections(builder)) {
+      } else if (refersToSelections(builder) || builder.has('distinctOn')) {
         // `having`, and `groupBy` with an alias, may refer to aliased
-        // selections, so keep them and select the distinct ids from the
+        // selections, and clearing the selections would also clear
+        // `distinctOn`, so keep them and select the distinct ids from the
         // result instead.
         knexBuilder.distinct(idColumns).from((knexBuilder) => {
           buildQuery(knexBuilder).as('inner');
@@ -1015,7 +1023,9 @@ class QueryBuilder extends QueryBuilderBase {
   }
 
   updateById(id, modelOrObject) {
-    return this.findById(id)
+    // Keep the id condition in its own where group, so that a user's `orWhere()`
+    // can't get around it. See `whereGroups.js`.
+    return addWhereGroup(this, () => this.findById(id))
       .update(modelOrObject)
       .runBefore((_, builder) => {
         if (!builder.internalOptions().skipUndefined) {
@@ -1025,7 +1035,7 @@ class QueryBuilder extends QueryBuilderBase {
   }
 
   patchById(id, modelOrObject) {
-    return this.findById(id)
+    return addWhereGroup(this, () => this.findById(id))
       .patch(modelOrObject)
       .runBefore((_, builder) => {
         if (!builder.internalOptions().skipUndefined) {
@@ -1035,7 +1045,7 @@ class QueryBuilder extends QueryBuilderBase {
   }
 
   deleteById(id) {
-    return this.findById(id)
+    return addWhereGroup(this, () => this.findById(id))
       .delete()
       .runBefore((_, builder) => {
         if (!builder.internalOptions().skipUndefined) {

package/lib/queryBuilder/graph/GraphData.js

@@ -6,7 +6,46 @@ class GraphData {
     this.currentGraph = currentGraph;
     this.graphOptions = graphOptions;
     this.nodeDbExistence = nodeDbExistence;
+    // Pruning replaces `graph.nodes`, but `keptNodeForRow()` also needs the
+    // branches below related nodes.
+    this._unprunedNodes = graph.nodes;
+    this._keptNodesByRowKey = null;
   }
+
+  // Find the node that keeps the row of `currentNode` in another place of the
+  // graph, by relating or patching it there, or by upserting it below a related
+  // node. Only call this after pruning.
+  keptNodeForRow(currentNode) {
+    if (!this._keptNodesByRowKey) {
+      this._keptNodesByRowKey = new Map();
+      const nodes = new Set(this.graph.nodes);
+
+      // Nested upserts write the pruned branches below related nodes, except
+      // for models marked with `#unrelate` or `#delete` and their descendants:
+      const isUpsertedBelowRelatedNode = (node) =>
+        !node.toBeUnrelatedOrDeleted &&
+        (nodes.has(node.parentNode)
+          ? this.graphOptions.shouldRelate(node.parentNode, this)
+          : isUpsertedBelowRelatedNode(node.parentNode));
+
+      for (const node of this._unprunedNodes) {
+        if (
+          node.obj.$hasId() &&
+          (nodes.has(node)
+            ? this.graphOptions.shouldPatchOrUpdateIgnoreDisable(node, this)
+            : isUpsertedBelowRelatedNode(node))
+        ) {
+          this._keptNodesByRowKey.set(getRowKey(node), node);
+        }
+      }
+    }
+
+    return this._keptNodesByRowKey.get(getRowKey(currentNode)) || null;
+  }
+}
+
+function getRowKey(node) {
+  return `${node.modelClass.getTableName()}:${node.obj.$idKey()}`;
 }
 
 module.exports = {

package/lib/queryBuilder/graph/GraphFetcher.js

@@ -4,6 +4,8 @@ const { asArray, groupBy } = require('../../utils/objectUtils');
 const { ModelGraph } = require('../../model/graph/ModelGraph');
 const { FetchStrategy } = require('./GraphOptions');
 const { RelationExpression } = require('../RelationExpression');
+const { getDeclaredColumns } = require('../operations/UpdateOperation');
+const { getFieldExpressionColumn } = require('../../utils/parseFieldExpression');
 
 class GraphFetcher {
   /**
@@ -106,8 +108,12 @@ function createInputColumnSelector(nodes) {
 
     for (const node of nodes) {
       const databaseJson = node.obj.$toDatabaseJson(builder);
+      const declaredColumns = getDeclaredColumns(node.modelClass);
+
+      for (const key of Object.keys(databaseJson)) {
+        // Select the json column of field expressions like `'meta:a'`:
+        const column = getFieldExpressionColumn(key, declaredColumns) ?? key;
 
-      for (const column of Object.keys(databaseJson)) {
         if (!shouldSelectColumn(column, selects, node)) {
           continue;
         }