objection 3.2.0 → 3.3.0

package/lib/queryBuilder/graph/insert/DeferredRelationGraphInsertAction.js

@@ -0,0 +1,57 @@
+'use strict';
+
+const { GraphAction } = require('../GraphAction');
+const { groupBy } = require('../../../utils/objectUtils');
+const promiseUtils = require('../../../utils/promiseUtils');
+
+/**
+ * Patches in the foreign keys of `BelongsToOneRelation` edges that `GraphInsert`
+ * had to defer to break a dependency cycle. The owner nodes of these edges were
+ * inserted without the foreign keys. `GraphInsertAction` already copied them to
+ * the owner objects once the related nodes were inserted, so all that's left to
+ * do is one patch query per owner.
+ */
+class DeferredRelationGraphInsertAction extends GraphAction {
+  constructor(graphData, { edges }) {
+    super(graphData);
+
+    // `BelongsToOneRelation` edges whose owner node needs to be patched.
+    this.edges = edges;
+  }
+
+  run(builder) {
+    const edgesByOwner = groupBy(this.edges, (edge) => edge.ownerNode);
+    const owners = [...edgesByOwner.keys()];
+    const concurrency = this._getConcurrency(builder, owners);
+
+    return promiseUtils.map(
+      owners,
+      (owner) => this._patchOwner(builder, owner, edgesByOwner.get(owner)),
+      { concurrency },
+    );
+  }
+
+  _patchOwner(builder, ownerNode, edges) {
+    const { modelClass } = ownerNode;
+    const ownerIdProp = modelClass.getIdRelationProperty();
+    const patch = {};
+
+    for (const { relation } of edges) {
+      const { ownerProp } = relation;
+
+      ownerProp.forEach((i) => {
+        ownerProp.patch(patch, i, ownerProp.getProp(ownerNode.obj, i));
+      });
+    }
+
+    return modelClass
+      .query()
+      .childQueryOf(builder)
+      .patch(patch)
+      .whereInComposite(ownerIdProp.refs(builder), [ownerIdProp.getProps(ownerNode.obj)]);
+  }
+}
+
+module.exports = {
+  DeferredRelationGraphInsertAction,
+};

package/lib/queryBuilder/graph/insert/GraphInsert.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { JoinRowGraphInsertAction } = require('./JoinRowGraphInsertAction');
+const { DeferredRelationGraphInsertAction } = require('./DeferredRelationGraphInsertAction');
 const { GraphInsertAction } = require('./GraphInsertAction');
 const { GraphOperation } = require('../GraphOperation');
 const { ModelGraphEdge } = require('../../../model/graph/ModelGraphEdge');
@@ -9,10 +10,15 @@ class GraphInsert extends GraphOperation {
   constructor(...args) {
     super(...args);
     this.dependencies = this._createDependencyMap();
+    this.deferredEdges = new Set();
   }
 
   createActions() {
-    return [...this._createNormalActions(), ...this._createJoinRowActions()];
+    return [
+      ...this._createNormalActions(),
+      ...this._createDeferredRelationActions(),
+      ...this._createJoinRowActions(),
+    ];
   }
 
   _createDependencyMap() {
@@ -76,6 +82,10 @@ class GraphInsert extends GraphOperation {
       });
 
       if (nodesToInsert.length === 0) {
+        if (handledNodes.size !== this.graph.nodes.length && this._breakCycle(handledNodes)) {
+          continue;
+        }
+
         break;
       }
 
@@ -83,6 +93,7 @@ class GraphInsert extends GraphOperation {
         new GraphInsertAction(this.graphData, {
           nodes: nodesToInsert,
           dependencies: this.dependencies,
+          deferredEdges: this.deferredEdges,
         }),
       );
 
@@ -98,19 +109,71 @@ class GraphInsert extends GraphOperation {
     return actions;
   }
 
-  _isHandled(node, handledNodes) {
-    return handledNodes.has(node);
+  // Tries to break a dependency cycle by deferring one `BelongsToOneRelation`
+  // edge that points to a `#ref` node. The owner node is then inserted without
+  // the foreign key, which is patched in by `DeferredRelationGraphInsertAction`
+  // once the referenced node has been inserted. Cycles can only exist through
+  // `#ref` nodes, since the graph is a tree otherwise.
+  _breakCycle(handledNodes) {
+    for (const node of this.graph.nodes) {
+      if (this._isHandled(node, handledNodes)) {
+        continue;
+      }
+
+      for (const edge of this._pendingDependencyEdges(node, handledNodes)) {
+        if (
+          this._canDeferEdge(edge, node) &&
+          this._isBlockedBy(edge.relatedNode, node, handledNodes, new Set())
+        ) {
+          this.deferredEdges.add(edge);
+          return true;
+        }
+      }
+    }
+
+    return false;
   }
 
-  _hasDependencies(node, handledNodes) {
-    if (!this.dependencies.has(node)) {
+  _canDeferEdge(edge, node) {
+    return (
+      edge.type === ModelGraphEdge.Type.Relation &&
+      edge.relation.isObjectionBelongsToOneRelation &&
+      edge.isOwnerNode(node) &&
+      edge.relatedNode.isReference &&
+      // Both ends need to be inserted for the foreign key to be patched in later.
+      this.graphOptions.shouldInsert(node, this.graphData) &&
+      this.graphOptions.shouldInsert(edge.relatedNode.referencedNode, this.graphData) &&
+      !this._isReadByPropertyReference(node, edge.relation.ownerProp)
+    );
+  }
+
+  // Returns true if a `#ref{}` property reference reads one of the `props` of
+  // `node`. Such references may be resolved before a deferred prop is set,
+  // which would resolve them to `undefined`.
+  _isReadByPropertyReference(node, props) {
+    return node.refEdges.some((edge) => {
+      return (
+        edge.refType === ModelGraphEdge.ReferenceType.Property &&
+        edge.isRelatedNode(node) &&
+        props.props.includes(edge.refRelatedDataPath[0])
+      );
+    });
+  }
+
+  // Returns true if `node` (transitively) waits for `blockingNode` to be handled.
+  _isBlockedBy(node, blockingNode, handledNodes, visitedNodes) {
+    if (node === blockingNode) {
+      return true;
+    }
+
+    if (visitedNodes.has(node)) {
       return false;
     }
 
-    for (const edge of this.dependencies.get(node)) {
-      const dependencyNode = edge.getOtherNode(node);
+    visitedNodes.add(node);
 
-      if (!handledNodes.has(dependencyNode) && !this.currentGraph.nodeForNode(dependencyNode)) {
+    for (const edge of this._pendingDependencyEdges(node, handledNodes)) {
+      if (this._isBlockedBy(edge.getOtherNode(node), blockingNode, handledNodes, visitedNodes)) {
         return true;
       }
     }
@@ -118,6 +181,42 @@ class GraphInsert extends GraphOperation {
     return false;
   }
 
+  _createDeferredRelationActions() {
+    if (this.deferredEdges.size === 0) {
+      return [];
+    }
+
+    return [
+      new DeferredRelationGraphInsertAction(this.graphData, {
+        edges: [...this.deferredEdges],
+      }),
+    ];
+  }
+
+  _isHandled(node, handledNodes) {
+    return handledNodes.has(node);
+  }
+
+  _hasDependencies(node, handledNodes) {
+    return this._pendingDependencyEdges(node, handledNodes).length > 0;
+  }
+
+  // Returns the dependency edges of `node` whose other node still needs to be
+  // inserted first. Deferred edges are ignored.
+  _pendingDependencyEdges(node, handledNodes) {
+    const edges = this.dependencies.get(node) || [];
+
+    return edges.filter((edge) => {
+      const dependencyNode = edge.getOtherNode(node);
+
+      return (
+        !this.deferredEdges.has(edge) &&
+        !handledNodes.has(dependencyNode) &&
+        !this.currentGraph.nodeForNode(dependencyNode)
+      );
+    });
+  }
+
   _markHandled(node, handledNodes) {
     handledNodes.add(node);
 

package/lib/queryBuilder/graph/insert/GraphInsertAction.js

@@ -14,12 +14,15 @@ const promiseUtils = require('../../../utils/promiseUtils');
  * when needed.
  */
 class GraphInsertAction extends GraphAction {
-  constructor(graphData, { nodes, dependencies }) {
+  constructor(graphData, { nodes, dependencies, deferredEdges }) {
     super(graphData);
 
     // Nodes to insert.
     this.nodes = nodes;
     this.dependencies = dependencies;
+    // Dependency edges that are resolved once the dependency node has been
+    // inserted, and are patched in by `DeferredRelationGraphInsertAction`.
+    this.deferredEdges = deferredEdges;
   }
 
   run(builder) {
@@ -63,6 +66,10 @@ class GraphInsertAction extends GraphAction {
 
       if (edges) {
         for (const edge of edges) {
+          if (this.deferredEdges.has(edge)) {
+            continue;
+          }
+
           // `node` needs `dependencyNode` to have been inserted (and it has been).
           const dependencyNode = edge.getOtherNode(node);
 

package/lib/queryBuilder/join/JoinResultParser.js

@@ -22,6 +22,8 @@ class JoinResultParser {
       return flatRows;
     }
 
+    this._checkIdColumns(flatRows[0]);
+
     this.columnsByTableNode = this._createColumns(flatRows[0]);
     this.parentMap = new Map();
     this.rootModels = [];
@@ -93,6 +95,26 @@ class JoinResultParser {
     }
   }
 
+  _checkIdColumns(row) {
+    for (const tableNode of this.tableTree.nodes) {
+      const idColumns = tableNode.modelClass.getIdColumnArray();
+      const hasIdColumns =
+        idColumns.length > 0 &&
+        idColumns.every((column) =>
+          Object.prototype.hasOwnProperty.call(row, tableNode.getColumnAliasForColumn(column)),
+        );
+
+      if (!hasIdColumns) {
+        throw new Error(
+          `withGraphJoined: the id column(s) [${idColumns.join(', ')}] of model ${tableNode.modelClass.name} ` +
+            `are missing from the result, so the joined rows can't be grouped into models. ` +
+            `withGraphJoined requires all models in the relation expression to have a primary key ` +
+            `(idColumn) that exists in the table. Use withGraphFetched for models without one.`,
+        );
+      }
+    }
+  }
+
   _createColumns(row) {
     const columns = Object.keys(row).map((columnAlias) =>
       JoinResultColumn.create({ tableTree: this.tableTree, columnAlias }),

package/lib/queryBuilder/join/RelationJoiner.js

@@ -24,6 +24,8 @@ class RelationJoiner {
     this.modifiers = null;
 
     this.options = defaultOptions();
+    // The `joinOperation` options of the individual `withGraphJoined` calls.
+    this.joinOperations = [];
     this.tableTree = null;
     this.internalSelections = null;
   }
@@ -49,6 +51,11 @@ class RelationJoiner {
     return this;
   }
 
+  setJoinOperations(joinOperations) {
+    this.joinOperations = joinOperations;
+    return this;
+  }
+
   /**
    * Fetches the column information needed for building the select clauses.
    *
@@ -81,23 +88,6 @@ class RelationJoiner {
     }
   }
 
-  releaseInternalRootSelection(column) {
-    const rootTableNode = this.tableTree && this.tableTree.rootNode;
-
-    if (!rootTableNode || !this.internalSelections) {
-      return false;
-    }
-
-    const alias = rootTableNode.getColumnAliasForColumn(column);
-    const count = this.internalSelections.length;
-
-    this.internalSelections = this.internalSelections.filter(
-      (it) => it.table !== rootTableNode.alias || it.alias !== alias,
-    );
-
-    return this.internalSelections.length !== count;
-  }
-
   parseResult(builder, flatRows) {
     const parser = JoinResultParser.create({
       tableTree: this._getTableTree(builder),
@@ -134,12 +124,18 @@ class RelationJoiner {
     // also contain selects.
     userSelectQuery.modify(tableNode.relation.modify);
 
+    if (selectsOnlyAliasesKeepingSelectAll(userSelectQuery.findAllSelections())) {
+      // All columns are selected from the joined subquery in the main query,
+      // so the subquery needs to select them too.
+      subqueryToJoin.select(`${subqueryToJoin.tableRef()}.*`);
+    }
+
     // Save the query that contains the user specified selects
     // for later use.
     userSelectQueries.set(tableNode, userSelectQuery);
 
     tableNode.relation.join(builder, {
-      joinOperation: this.options.joinOperation,
+      joinOperation: this._getJoinOperation(tableNode),
 
       ownerTable: tableNode.parentNode.alias,
       relatedTableAlias: tableNode.alias,
@@ -156,6 +152,26 @@ class RelationJoiner {
     }
   }
 
+  // Returns the `joinOperation` of the last `withGraphJoined` call that
+  // contains the relation of `tableNode`, falling back to the default.
+  _getJoinOperation(tableNode) {
+    const path = [];
+
+    for (let node = tableNode; node.parentNode; node = node.parentNode) {
+      path.unshift(node.relationProperty);
+    }
+
+    for (let i = this.joinOperations.length - 1; i >= 0; --i) {
+      const { expression, joinOperation } = this.joinOperations[i];
+
+      if (containsPath(expression, path)) {
+        return joinOperation;
+      }
+    }
+
+    return this.options.joinOperation;
+  }
+
   _buildSelects({ builder, tableNode, userSelectQueries }) {
     const { selections, internalSelections } = getSelectionsForNode({
       builder,
@@ -168,9 +184,21 @@ class RelationJoiner {
     }
 
     // Save the selections that were added internally (not by the user)
-    // so that we can later remove the corresponding properties when
-    // parsing the result.
-    this.internalSelections = internalSelections;
+    // so that we can later remove the corresponding properties. The root
+    // ones are registered with the builder, which removes them from the
+    // result models after all eager operations have run, as they may still
+    // need them. The nested ones are removed when parsing the result.
+    this.internalSelections = [];
+
+    for (const selection of internalSelections) {
+      if (selection.table === tableNode.alias) {
+        builder.addInternalSelection(
+          tableNode.modelClass.columnNameToPropertyName(selection.column),
+        );
+      } else {
+        this.internalSelections.push(selection);
+      }
+    }
 
     builder.select(selectionsToStrings(selections));
   }
@@ -185,9 +213,21 @@ function defaultOptions() {
   };
 }
 
+function containsPath(expression, path) {
+  for (const name of path) {
+    expression = expression.childExpression(name);
+
+    if (!expression) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 function createSubqueryToJoin({ builder, tableNode, modifiers }) {
   const { relation, expression, modelClass } = tableNode;
-  const modifierQuery = modelClass.query().childQueryOf(builder);
+  const modifierQuery = modelClass.query().childQueryOf(builder, { isJoinChildQuery: true });
 
   for (const modifierName of expression.node.$modify) {
     const modifier = createModifier({
@@ -341,9 +381,18 @@ function isNotSelectAll(selection) {
 }
 
 function isSelectAllSelectionSet(selections) {
+  selections = selections.filter(isNotKeepingSelectAll);
   return selections.length === 0 || selections.some(isSelectAll);
 }
 
+function isNotKeepingSelectAll(selection) {
+  return !selection.keepsSelectAll;
+}
+
+function selectsOnlyAliasesKeepingSelectAll(selections) {
+  return selections.length > 0 && selections.every((it) => it.keepsSelectAll);
+}
+
 function checkAliasLength(modelClass, alias) {
   if (alias.length > ID_LENGTH_LIMIT) {
     throw modelClass.createValidationError({

package/lib/queryBuilder/operations/InsertAndFetchOperation.js

@@ -22,10 +22,16 @@ class InsertAndFetchOperation extends DelegateOperation {
     const modelClass = builder.modelClass();
     const insertedModels = await super.onAfter2(builder, inserted);
 
-    const insertedModelArray = asArray(insertedModels);
+    // Models without identifiers can't be fetched. This happens when a row
+    // with a database generated identifier is ignored by `onConflict()`.
+    const insertedModelArray = asArray(insertedModels).filter((model) => model && model.$hasId());
     const idProps = modelClass.getIdPropertyArray();
     const ids = insertedModelArray.map((model) => model.$id());
 
+    if (ids.length === 0) {
+      return insertedModels;
+    }
+
     const fetchedModels = await modelClass
       .query()
       .childQueryOf(builder)

package/lib/queryBuilder/operations/InsertGraphOperation.js

@@ -51,8 +51,8 @@ class InsertGraphOperation extends DelegateOperation {
     return retVal;
   }
 
-  onBefore1(_, result) {
-    // Do nothing.
+  onBefore1(builder, result) {
+    GraphUpsert.warnOnConflict(builder, this.name);
     return result;
   }
 

package/lib/queryBuilder/operations/InsertOperation.js

@@ -4,7 +4,7 @@ const { QueryBuilderOperation } = require('./QueryBuilderOperation');
 const { StaticHookArguments } = require('../StaticHookArguments');
 const { after, mapAfterAllReturn } = require('../../utils/promiseUtils');
 const { isPostgres, isSqlite, isMySql, isMsSql } = require('../../utils/knexUtils');
-const { isObject } = require('../../utils/objectUtils');
+const { isObject, isString } = require('../../utils/objectUtils');
 
 // Base class for all insert operations.
 class InsertOperation extends QueryBuilderOperation {
@@ -36,26 +36,46 @@ class InsertOperation extends QueryBuilderOperation {
   }
 
   onBuildKnex(knexBuilder, builder) {
-    if (!isSqlite(builder.knex()) && !isMySql(builder.knex()) && !builder.has(/returning/)) {
-      // If the user hasn't specified a `returning` clause, we make sure
-      // that at least the identifier is returned.
-      knexBuilder = knexBuilder.returning(builder.modelClass().getIdColumn());
+    const knex = builder.knex();
+
+    if (!builder.has(/returning/)) {
+      if (!isSqlite(knex) && !isMySql(knex)) {
+        // If the user hasn't specified a `returning` clause, we make sure
+        // that at least the identifier is returned. With `onConflict()` the
+        // conflict columns are returned too, so that the returned rows can
+        // be matched to the models when some of the rows are ignored.
+        knexBuilder = knexBuilder.returning(getDefaultReturning(builder));
+      } else if (isSqlite(knex) && builder.has('onConflict')) {
+        // Without a `returning` clause, SQLite returns the id of the last
+        // inserted row, which belongs to a previous insert if the row was
+        // ignored. Use `returning` to find out what was actually inserted.
+        knexBuilder = knexBuilder.returning(getDefaultReturning(builder));
+      }
     }
 
     return knexBuilder.insert(this.models.map((model) => model.$toDatabaseJson(builder)));
   }
 
-  onAfter1(_, ret) {
+  onAfter1(builder, ret) {
     if (!Array.isArray(ret) || !ret.length || ret === this.models) {
       // Early exit if there is nothing to do.
       return this.models;
     }
 
+    const isOnConflict = builder.has('onConflict');
+
     if (isObject(ret[0])) {
-      // If the user specified a `returning` clause the result may be an array of objects.
-      // Merge all values of the objects to our models.
-      for (let i = 0, l = this.models.length; i < l; ++i) {
-        this.models[i].$setDatabaseJson(ret[i]);
+      if (isOnConflict && ret.length !== this.models.length) {
+        // With `onConflict()`, ignored rows (and rows not updated because of a
+        // `where` clause after `merge()`) are not returned, so the returned rows
+        // can't be merged to the models based on their position.
+        mergeMatchingRows(builder, this.models, ret);
+      } else {
+        // If the user specified a `returning` clause the result may be an array of objects.
+        // Merge all values of the objects to our models.
+        for (let i = 0, l = this.models.length; i < l; ++i) {
+          this.models[i].$setDatabaseJson(ret[i]);
+        }
       }
     } else {
       // If the return value is not an array of objects, we assume it is an array of identifiers.
@@ -63,8 +83,9 @@ class InsertOperation extends QueryBuilderOperation {
         const model = this.models[i];
 
         // Don't set the id if the model already has one. MySQL and Sqlite don't return the correct
-        // primary key value if the id is not generated in db, but given explicitly.
-        if (!model.$id()) {
+        // primary key value if the id is not generated in db, but given explicitly. With
+        // `onConflict()`, MySQL returns `0` for ignored rows, which is not a valid id.
+        if (!model.$id() && !(isOnConflict && !ret[i])) {
           model.$id(ret[i]);
         }
       }
@@ -92,6 +113,94 @@ class InsertOperation extends QueryBuilderOperation {
   }
 }
 
+function getDefaultReturning(builder) {
+  const idColumns = builder.modelClass().getIdColumnArray();
+  const columns = Array.from(new Set(idColumns.concat(getConflictColumns(builder))));
+
+  return columns.length === 1 ? columns[0] : columns;
+}
+
+function getConflictColumns(builder) {
+  if (!builder.has('onConflict')) {
+    return [];
+  }
+
+  const op = builder.findOperation('onConflict');
+  const arg = op.args ? op.args[0] : undefined;
+  const columns = Array.isArray(arg) ? arg : [arg];
+
+  // Raw conflict targets (or no target at all) can't be used for matching.
+  return columns.every(isString) ? columns : [];
+}
+
+// Merges the rows returned by an insert with `onConflict()` into the models with
+// matching key values. Models whose rows were not returned are left untouched.
+function mergeMatchingRows(builder, models, rows) {
+  const jsons = models.map((model) => model.$toDatabaseJson(builder));
+  const idColumns = builder.modelClass().getIdColumnArray();
+
+  // Use the conflict columns, or else the identifiers, to match the rows to
+  // the models. The columns must be present in the input and the result.
+  const keyColumns = [getConflictColumns(builder), idColumns].find((columns) => {
+    return (
+      columns.length > 0 &&
+      columns.every((col) => {
+        return (
+          rows.every((row) => row[col] !== undefined) &&
+          jsons.every((json) => json[col] !== undefined)
+        );
+      })
+    );
+  });
+
+  if (!keyColumns) {
+    throw createMatchError();
+  }
+
+  const createKey = (obj) => JSON.stringify(keyColumns.map((col) => normalizeKey(obj[col])));
+  const indicesByKey = new Map();
+
+  jsons.forEach((json, index) => {
+    const key = createKey(json);
+    const indices = indicesByKey.get(key);
+
+    if (indices) {
+      indices.push(index);
+    } else {
+      indicesByKey.set(key, [index]);
+    }
+  });
+
+  for (const row of rows) {
+    const indices = indicesByKey.get(createKey(row));
+    const index = indices ? indices.shift() : undefined;
+
+    if (index === undefined) {
+      throw createMatchError();
+    }
+
+    models[index].$setDatabaseJson(row);
+  }
+}
+
+function normalizeKey(value) {
+  if (value === null || value === undefined) {
+    return null;
+  } else if (isObject(value)) {
+    return JSON.stringify(value);
+  } else {
+    return String(value);
+  }
+}
+
+function createMatchError() {
+  return new Error(
+    'Could not match the rows returned by an insert with `onConflict()` to the inserted models. ' +
+      'Pass the conflict columns to `onConflict()` and make sure they are returned, or provide ' +
+      'the identifiers of the inserted models.',
+  );
+}
+
 function callBeforeInsert(builder, models) {
   const maybePromise = callInstanceBeforeInsert(builder, models);
   return after(maybePromise, () => callStaticBeforeInsert(builder));

package/lib/queryBuilder/operations/QueryBuilderOperation.js

@@ -41,6 +41,9 @@ class QueryBuilderOperation {
 
     // Operations this operation added in any of its hooks.
     this.childOperations = [];
+
+    // The where group this operation belongs to. See `whereGroups.js`.
+    this.whereGroup = null;
   }
 
   is(OperationClass) {
@@ -217,6 +220,7 @@ class QueryBuilderOperation {
 
     clone.adderHookName = this.adderHookName;
     clone.parentOperation = this.parentOperation;
+    clone.whereGroup = this.whereGroup;
 
     clone.childOperations = this.childOperations.map((childOp) => {
       const childOpClone = childOp.clone();