objection 3.1.5 → 3.3.0

package/lib/queryBuilder/join/RelationJoiner.js

@@ -24,6 +24,8 @@ class RelationJoiner {
     this.modifiers = null;
 
     this.options = defaultOptions();
+    // The `joinOperation` options of the individual `withGraphJoined` calls.
+    this.joinOperations = [];
     this.tableTree = null;
     this.internalSelections = null;
   }
@@ -49,6 +51,11 @@ class RelationJoiner {
     return this;
   }
 
+  setJoinOperations(joinOperations) {
+    this.joinOperations = joinOperations;
+    return this;
+  }
+
   /**
    * Fetches the column information needed for building the select clauses.
    *
@@ -117,12 +124,18 @@ class RelationJoiner {
     // also contain selects.
     userSelectQuery.modify(tableNode.relation.modify);
 
+    if (selectsOnlyAliasesKeepingSelectAll(userSelectQuery.findAllSelections())) {
+      // All columns are selected from the joined subquery in the main query,
+      // so the subquery needs to select them too.
+      subqueryToJoin.select(`${subqueryToJoin.tableRef()}.*`);
+    }
+
     // Save the query that contains the user specified selects
     // for later use.
     userSelectQueries.set(tableNode, userSelectQuery);
 
     tableNode.relation.join(builder, {
-      joinOperation: this.options.joinOperation,
+      joinOperation: this._getJoinOperation(tableNode),
 
       ownerTable: tableNode.parentNode.alias,
       relatedTableAlias: tableNode.alias,
@@ -139,6 +152,26 @@ class RelationJoiner {
     }
   }
 
+  // Returns the `joinOperation` of the last `withGraphJoined` call that
+  // contains the relation of `tableNode`, falling back to the default.
+  _getJoinOperation(tableNode) {
+    const path = [];
+
+    for (let node = tableNode; node.parentNode; node = node.parentNode) {
+      path.unshift(node.relationProperty);
+    }
+
+    for (let i = this.joinOperations.length - 1; i >= 0; --i) {
+      const { expression, joinOperation } = this.joinOperations[i];
+
+      if (containsPath(expression, path)) {
+        return joinOperation;
+      }
+    }
+
+    return this.options.joinOperation;
+  }
+
   _buildSelects({ builder, tableNode, userSelectQueries }) {
     const { selections, internalSelections } = getSelectionsForNode({
       builder,
@@ -151,9 +184,21 @@ class RelationJoiner {
     }
 
     // Save the selections that were added internally (not by the user)
-    // so that we can later remove the corresponding properties when
-    // parsing the result.
-    this.internalSelections = internalSelections;
+    // so that we can later remove the corresponding properties. The root
+    // ones are registered with the builder, which removes them from the
+    // result models after all eager operations have run, as they may still
+    // need them. The nested ones are removed when parsing the result.
+    this.internalSelections = [];
+
+    for (const selection of internalSelections) {
+      if (selection.table === tableNode.alias) {
+        builder.addInternalSelection(
+          tableNode.modelClass.columnNameToPropertyName(selection.column),
+        );
+      } else {
+        this.internalSelections.push(selection);
+      }
+    }
 
     builder.select(selectionsToStrings(selections));
   }
@@ -168,9 +213,21 @@ function defaultOptions() {
   };
 }
 
+function containsPath(expression, path) {
+  for (const name of path) {
+    expression = expression.childExpression(name);
+
+    if (!expression) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 function createSubqueryToJoin({ builder, tableNode, modifiers }) {
   const { relation, expression, modelClass } = tableNode;
-  const modifierQuery = modelClass.query().childQueryOf(builder);
+  const modifierQuery = modelClass.query().childQueryOf(builder, { isJoinChildQuery: true });
 
   for (const modifierName of expression.node.$modify) {
     const modifier = createModifier({
@@ -324,9 +381,18 @@ function isNotSelectAll(selection) {
 }
 
 function isSelectAllSelectionSet(selections) {
+  selections = selections.filter(isNotKeepingSelectAll);
   return selections.length === 0 || selections.some(isSelectAll);
 }
 
+function isNotKeepingSelectAll(selection) {
+  return !selection.keepsSelectAll;
+}
+
+function selectsOnlyAliasesKeepingSelectAll(selections) {
+  return selections.length > 0 && selections.every((it) => it.keepsSelectAll);
+}
+
 function checkAliasLength(modelClass, alias) {
   if (alias.length > ID_LENGTH_LIMIT) {
     throw modelClass.createValidationError({

package/lib/queryBuilder/operations/DelegateOperation.js

@@ -11,6 +11,10 @@ class DelegateOperation extends QueryBuilderOperation {
     this.delegate = opt.delegate;
   }
 
+  get modelOptions() {
+    return this.delegate.modelOptions;
+  }
+
   is(OperationClass) {
     return super.is(OperationClass) || this.delegate.is(OperationClass);
   }

package/lib/queryBuilder/operations/InsertAndFetchOperation.js

@@ -22,10 +22,16 @@ class InsertAndFetchOperation extends DelegateOperation {
     const modelClass = builder.modelClass();
     const insertedModels = await super.onAfter2(builder, inserted);
 
-    const insertedModelArray = asArray(insertedModels);
+    // Models without identifiers can't be fetched. This happens when a row
+    // with a database generated identifier is ignored by `onConflict()`.
+    const insertedModelArray = asArray(insertedModels).filter((model) => model && model.$hasId());
     const idProps = modelClass.getIdPropertyArray();
     const ids = insertedModelArray.map((model) => model.$id());
 
+    if (ids.length === 0) {
+      return insertedModels;
+    }
+
     const fetchedModels = await modelClass
       .query()
       .childQueryOf(builder)

package/lib/queryBuilder/operations/InsertGraphOperation.js

@@ -51,8 +51,8 @@ class InsertGraphOperation extends DelegateOperation {
     return retVal;
   }
 
-  onBefore1(_, result) {
-    // Do nothing.
+  onBefore1(builder, result) {
+    GraphUpsert.warnOnConflict(builder, this.name);
     return result;
   }
 

package/lib/queryBuilder/operations/InsertOperation.js

@@ -4,7 +4,7 @@ const { QueryBuilderOperation } = require('./QueryBuilderOperation');
 const { StaticHookArguments } = require('../StaticHookArguments');
 const { after, mapAfterAllReturn } = require('../../utils/promiseUtils');
 const { isPostgres, isSqlite, isMySql, isMsSql } = require('../../utils/knexUtils');
-const { isObject } = require('../../utils/objectUtils');
+const { isObject, isString } = require('../../utils/objectUtils');
 
 // Base class for all insert operations.
 class InsertOperation extends QueryBuilderOperation {
@@ -36,26 +36,46 @@ class InsertOperation extends QueryBuilderOperation {
   }
 
   onBuildKnex(knexBuilder, builder) {
-    if (!isSqlite(builder.knex()) && !isMySql(builder.knex()) && !builder.has(/returning/)) {
-      // If the user hasn't specified a `returning` clause, we make sure
-      // that at least the identifier is returned.
-      knexBuilder = knexBuilder.returning(builder.modelClass().getIdColumn());
+    const knex = builder.knex();
+
+    if (!builder.has(/returning/)) {
+      if (!isSqlite(knex) && !isMySql(knex)) {
+        // If the user hasn't specified a `returning` clause, we make sure
+        // that at least the identifier is returned. With `onConflict()` the
+        // conflict columns are returned too, so that the returned rows can
+        // be matched to the models when some of the rows are ignored.
+        knexBuilder = knexBuilder.returning(getDefaultReturning(builder));
+      } else if (isSqlite(knex) && builder.has('onConflict')) {
+        // Without a `returning` clause, SQLite returns the id of the last
+        // inserted row, which belongs to a previous insert if the row was
+        // ignored. Use `returning` to find out what was actually inserted.
+        knexBuilder = knexBuilder.returning(getDefaultReturning(builder));
+      }
     }
 
     return knexBuilder.insert(this.models.map((model) => model.$toDatabaseJson(builder)));
   }
 
-  onAfter1(_, ret) {
+  onAfter1(builder, ret) {
     if (!Array.isArray(ret) || !ret.length || ret === this.models) {
       // Early exit if there is nothing to do.
       return this.models;
     }
 
+    const isOnConflict = builder.has('onConflict');
+
     if (isObject(ret[0])) {
-      // If the user specified a `returning` clause the result may be an array of objects.
-      // Merge all values of the objects to our models.
-      for (let i = 0, l = this.models.length; i < l; ++i) {
-        this.models[i].$setDatabaseJson(ret[i]);
+      if (isOnConflict && ret.length !== this.models.length) {
+        // With `onConflict()`, ignored rows (and rows not updated because of a
+        // `where` clause after `merge()`) are not returned, so the returned rows
+        // can't be merged to the models based on their position.
+        mergeMatchingRows(builder, this.models, ret);
+      } else {
+        // If the user specified a `returning` clause the result may be an array of objects.
+        // Merge all values of the objects to our models.
+        for (let i = 0, l = this.models.length; i < l; ++i) {
+          this.models[i].$setDatabaseJson(ret[i]);
+        }
       }
     } else {
       // If the return value is not an array of objects, we assume it is an array of identifiers.
@@ -63,8 +83,9 @@ class InsertOperation extends QueryBuilderOperation {
         const model = this.models[i];
 
         // Don't set the id if the model already has one. MySQL and Sqlite don't return the correct
-        // primary key value if the id is not generated in db, but given explicitly.
-        if (!model.$id()) {
+        // primary key value if the id is not generated in db, but given explicitly. With
+        // `onConflict()`, MySQL returns `0` for ignored rows, which is not a valid id.
+        if (!model.$id() && !(isOnConflict && !ret[i])) {
           model.$id(ret[i]);
         }
       }
@@ -92,6 +113,94 @@ class InsertOperation extends QueryBuilderOperation {
   }
 }
 
+function getDefaultReturning(builder) {
+  const idColumns = builder.modelClass().getIdColumnArray();
+  const columns = Array.from(new Set(idColumns.concat(getConflictColumns(builder))));
+
+  return columns.length === 1 ? columns[0] : columns;
+}
+
+function getConflictColumns(builder) {
+  if (!builder.has('onConflict')) {
+    return [];
+  }
+
+  const op = builder.findOperation('onConflict');
+  const arg = op.args ? op.args[0] : undefined;
+  const columns = Array.isArray(arg) ? arg : [arg];
+
+  // Raw conflict targets (or no target at all) can't be used for matching.
+  return columns.every(isString) ? columns : [];
+}
+
+// Merges the rows returned by an insert with `onConflict()` into the models with
+// matching key values. Models whose rows were not returned are left untouched.
+function mergeMatchingRows(builder, models, rows) {
+  const jsons = models.map((model) => model.$toDatabaseJson(builder));
+  const idColumns = builder.modelClass().getIdColumnArray();
+
+  // Use the conflict columns, or else the identifiers, to match the rows to
+  // the models. The columns must be present in the input and the result.
+  const keyColumns = [getConflictColumns(builder), idColumns].find((columns) => {
+    return (
+      columns.length > 0 &&
+      columns.every((col) => {
+        return (
+          rows.every((row) => row[col] !== undefined) &&
+          jsons.every((json) => json[col] !== undefined)
+        );
+      })
+    );
+  });
+
+  if (!keyColumns) {
+    throw createMatchError();
+  }
+
+  const createKey = (obj) => JSON.stringify(keyColumns.map((col) => normalizeKey(obj[col])));
+  const indicesByKey = new Map();
+
+  jsons.forEach((json, index) => {
+    const key = createKey(json);
+    const indices = indicesByKey.get(key);
+
+    if (indices) {
+      indices.push(index);
+    } else {
+      indicesByKey.set(key, [index]);
+    }
+  });
+
+  for (const row of rows) {
+    const indices = indicesByKey.get(createKey(row));
+    const index = indices ? indices.shift() : undefined;
+
+    if (index === undefined) {
+      throw createMatchError();
+    }
+
+    models[index].$setDatabaseJson(row);
+  }
+}
+
+function normalizeKey(value) {
+  if (value === null || value === undefined) {
+    return null;
+  } else if (isObject(value)) {
+    return JSON.stringify(value);
+  } else {
+    return String(value);
+  }
+}
+
+function createMatchError() {
+  return new Error(
+    'Could not match the rows returned by an insert with `onConflict()` to the inserted models. ' +
+      'Pass the conflict columns to `onConflict()` and make sure they are returned, or provide ' +
+      'the identifiers of the inserted models.',
+  );
+}
+
 function callBeforeInsert(builder, models) {
   const maybePromise = callInstanceBeforeInsert(builder, models);
   return after(maybePromise, () => callStaticBeforeInsert(builder));

package/lib/queryBuilder/operations/NoneOperation.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const { QueryBuilderOperation } = require('./QueryBuilderOperation');
+
+const AggregateSelector =
+  /^(count|countDistinct|min|max|sum|sumDistinct|avg|avgDistinct|groupBy\w*)$/;
+
+class NoneOperation extends QueryBuilderOperation {
+  onBefore1(builder, result) {
+    // Resolve the query without executing it, like `cancelQuery()` does.
+    if (builder.isFind()) {
+      // Aggregate queries are executed to get the result the database
+      // returns for no rows, e.g. `[{ count: 0 }]`.
+      if (!builder.has(AggregateSelector)) {
+        builder.resolve([]);
+      }
+    } else if (builder.isUpdate() || builder.isDelete()) {
+      builder.resolve(builder.has(/returning/) ? [] : 0);
+    } else {
+      throw new Error('none() can only be used with find, update and delete queries');
+    }
+
+    return result;
+  }
+
+  // Called by `buildKnexQuery()` after all other operations have been
+  // built, so that the query matches no rows when it does get executed,
+  // for example as a subquery or in `resultSize()`. The existing where
+  // clauses are removed so that `orWhere()` calls can't match any rows.
+  onAfterBuildKnex(knexBuilder) {
+    return knexBuilder.clearWhere().where(false);
+  }
+}
+
+module.exports = {
+  NoneOperation,
+};

package/lib/queryBuilder/operations/ObjectionToKnexConvertingOperation.js

@@ -106,6 +106,11 @@ function convertArray(arr, builder, i, opName, skipUndefined) {
           `undefined passed as an item in argument #${i} for '${opName}' operation. Call skipUndefined() method to ignore the undefined values.`,
         );
       }
+    } else if (opName === 'orderBy' && isObject(item)) {
+      // knex's `orderBy()` only reads `{ column, order, nulls }` objects from arrays,
+      // so wrap refs, raws and query builders in one and convert its properties.
+      const obj = isPlainObject(item) ? item : { column: item };
+      return convertPlainObject(obj, builder, i, opName, skipUndefined);
     } else if (hasToKnexRawMethod(item)) {
       return convertToKnexRaw(item, builder);
     } else if (isObjectionQueryBuilderBase(item)) {

package/lib/queryBuilder/operations/QueryBuilderOperation.js

@@ -41,6 +41,9 @@ class QueryBuilderOperation {
 
     // Operations this operation added in any of its hooks.
     this.childOperations = [];
+
+    // The where group this operation belongs to. See `whereGroups.js`.
+    this.whereGroup = null;
   }
 
   is(OperationClass) {
@@ -217,6 +220,7 @@ class QueryBuilderOperation {
 
     clone.adderHookName = this.adderHookName;
     clone.parentOperation = this.parentOperation;
+    clone.whereGroup = this.whereGroup;
 
     clone.childOperations = this.childOperations.map((childOp) => {
       const childOpClone = childOp.clone();

package/lib/queryBuilder/operations/ReturningOperation.js

@@ -1,14 +1,27 @@
 'use strict';
 
-const { flatten } = require('../../utils/objectUtils');
+const { flatten, isPlainObject } = require('../../utils/objectUtils');
 const { ObjectionToKnexConvertingOperation } = require('./ObjectionToKnexConvertingOperation');
 
-// This class's only purpose is to normalize the arguments into an array.
+// This class's purpose is to normalize the arguments into an array and to
+// pass an optional trailing options object (e.g. `includeTriggerModifications`)
+// on to knex.
 //
 // In knex, if a single column is given to `returning` it returns an array with the that column's value
 // in it. If an array is given with a one item inside, the return value is an object.
 class ReturningOperation extends ObjectionToKnexConvertingOperation {
+  constructor(name, opt) {
+    super(name, opt);
+    this.options = null;
+  }
+
   onAdd(builder, args) {
+    args = Array.from(args);
+
+    if (args.length > 1 && isPlainObject(args[args.length - 1])) {
+      this.options = args.pop();
+    }
+
     args = flatten(args);
 
     // Don't add an empty returning list.
@@ -21,7 +34,19 @@ class ReturningOperation extends ObjectionToKnexConvertingOperation {
 
   onBuildKnex(knexBuilder, builder) {
     // Always pass an array of columns to knex.returning.
-    return knexBuilder.returning(this.getKnexArgs(builder));
+    const columns = this.getKnexArgs(builder);
+
+    if (this.options) {
+      return knexBuilder.returning(columns, this.options);
+    } else {
+      return knexBuilder.returning(columns);
+    }
+  }
+
+  clone() {
+    const clone = super.clone();
+    clone.options = this.options;
+    return clone;
   }
 }
 

package/lib/queryBuilder/operations/UpdateAndFetchOperation.js

@@ -14,6 +14,7 @@ class UpdateAndFetchOperation extends DelegateOperation {
 
     this.id = null;
     this.skipIdWhere = false;
+    this.selectOperations = null;
   }
 
   get model() {
@@ -25,6 +26,19 @@ class UpdateAndFetchOperation extends DelegateOperation {
     return this.delegate.onAdd(builder, args.slice(1));
   }
 
+  onBefore3(builder, result) {
+    // Selects have no effect on the update query, so remember them here to
+    // apply them to the fetch query. Collect them before the query is built,
+    // so that selects added by `onBuild` hooks in the shared context aren't
+    // copied, as those hooks run again for the fetch query.
+    this.selectOperations = new Set();
+    builder.forEachOperation(builder.constructor.SelectSelector, (op) => {
+      this.selectOperations.add(op);
+    });
+
+    return super.onBefore3(builder, result);
+  }
+
   onBuild(builder) {
     if (!this.skipIdWhere) {
       builder.findById(this.id);
@@ -43,6 +57,7 @@ class UpdateAndFetchOperation extends DelegateOperation {
     const fetched = await builder
       .emptyInstance()
       .childQueryOf(builder)
+      .copyFrom(builder, (op) => !!this.selectOperations?.has(op))
       .modify((builder) => {
         if (!this.skipIdWhere) {
           builder.findById(this.id);
@@ -52,6 +67,11 @@ class UpdateAndFetchOperation extends DelegateOperation {
 
     if (fetched) {
       this.model.$set(fetched);
+      // A custom `select` may not include the id. The row was fetched by
+      // id, so make sure the result still has it.
+      if (!this.model.$hasId()) {
+        this.model.$id(this.id);
+      }
     }
 
     await super.onAfter2(builder, numUpdated);

package/lib/queryBuilder/operations/UpdateOperation.js

@@ -2,6 +2,7 @@
 
 const { ref } = require('../../queryBuilder/ReferenceBuilder');
 const { isEmpty } = require('../../utils/objectUtils');
+const { toPostgresJsonPathLiteral } = require('../../utils/parseFieldExpression');
 const { isKnexRaw, isKnexQueryBuilder } = require('../../utils/knexUtils');
 const { QueryBuilderOperation } = require('./QueryBuilderOperation');
 const { StaticHookArguments } = require('../StaticHookArguments');
@@ -104,7 +105,7 @@ function convertFieldExpressionsToRaw(builder, model, json) {
       // "col" : raw(`jsonb_set("col", '{attr}', to_jsonb("other"#>'{lol}'), true)`)
 
       let parsed = ref(key);
-      let jsonRefs = '{' + parsed.parsedExpr.access.map((it) => it.ref).join(',') + '}';
+      let jsonPath = toPostgresJsonPathLiteral(parsed.parsedExpr.access);
       let valuePlaceholder = '?';
 
       if (isKnexQueryBuilder(val) || isKnexRaw(val)) {
@@ -114,7 +115,7 @@ function convertFieldExpressionsToRaw(builder, model, json) {
       }
 
       convertedJson[parsed.column] = knex.raw(
-        `jsonb_set(??, '${jsonRefs}', ${valuePlaceholder}, true)`,
+        `jsonb_set(??, ${jsonPath}, ${valuePlaceholder}, true)`,
         [convertedJson[parsed.column] || parsed.column, val],
       );
 

package/lib/queryBuilder/operations/UpsertGraphOperation.js

@@ -40,6 +40,11 @@ class UpsertGraphOperation extends QueryBuilderOperation {
     return true;
   }
 
+  onBefore1(builder, result) {
+    GraphUpsert.warnOnConflict(builder, this.name);
+    return result;
+  }
+
   onAfter1(builder) {
     if (hasOtherSqlModifyingQueryBuilderCalls(builder)) {
       throw new Error(

package/lib/queryBuilder/operations/WhereGroupOperation.js

@@ -0,0 +1,41 @@
+'use strict';
+
+const { QueryBuilderOperation } = require('./QueryBuilderOperation');
+
+// Builds a set of where operations inside a single knex `where(callback)`
+// group, i.e. wraps them in parentheses. See `wrapWhereGroups`.
+class WhereGroupOperation extends QueryBuilderOperation {
+  constructor(name, opt) {
+    super(name, opt);
+    this.operations = [];
+  }
+
+  onBuildKnex(knexBuilder, builder) {
+    const operations = this.operations;
+
+    return knexBuilder.where(function () {
+      let knex = this;
+
+      const build = (op) => {
+        if (op.hasOnBuildKnex()) {
+          knex = builder.callOperationMethod(op, 'onBuildKnex', [knex, builder]) || knex;
+        }
+      };
+
+      for (const op of operations) {
+        build(op);
+        op.forEachDescendantOperation(build);
+      }
+    });
+  }
+
+  clone() {
+    const clone = super.clone();
+    clone.operations = this.operations.map((op) => op.clone());
+    return clone;
+  }
+}
+
+module.exports = {
+  WhereGroupOperation,
+};

package/lib/queryBuilder/operations/eager/EagerOperation.js

@@ -8,14 +8,16 @@ class EagerOperation extends QueryBuilderOperation {
     super(name, opt);
 
     this.expression = RelationExpression.create();
-    this.modifiersAtPath = [];
     this.graphOptions = this.opt.defaultGraphOptions;
+    // The `joinOperation` options passed to the `withGraphJoined` calls,
+    // together with the expressions of the calls.
+    this.joinOperations = [];
   }
 
-  buildFinalExpression() {
+  buildFinalExpression(builder) {
     const expression = this.expression.clone();
 
-    this.modifiersAtPath.forEach((modifier, i) => {
+    builder.graphModifiersAtPath().forEach((modifier, i) => {
       const modifierName = getModifierName(i);
 
       expression.expressionsAtPath(modifier.path).forEach((expr) => {
@@ -30,7 +32,7 @@ class EagerOperation extends QueryBuilderOperation {
     // `modifiers()` returns a clone so we can modify it.
     const modifiers = builder.modifiers();
 
-    this.modifiersAtPath.forEach((modifier, i) => {
+    builder.graphModifiersAtPath().forEach((modifier, i) => {
       const modifierName = getModifierName(i);
 
       modifiers[modifierName] = modifier.modifier;
@@ -41,8 +43,8 @@ class EagerOperation extends QueryBuilderOperation {
 
   cloneFrom(eagerOp) {
     this.expression = eagerOp.expression.clone();
-    this.modifiersAtPath = eagerOp.modifiersAtPath.slice();
     this.graphOptions = Object.assign({}, eagerOp.graphOptions);
+    this.joinOperations = eagerOp.joinOperations.slice();
   }
 
   clone() {

package/lib/queryBuilder/operations/eager/JoinEagerOperation.js

@@ -21,17 +21,19 @@ class JoinEagerOperation extends EagerOperation {
 
   onBefore3(builder) {
     return this.joiner
-      .setExpression(this.buildFinalExpression())
+      .setExpression(this.buildFinalExpression(builder))
       .setModifiers(this.buildFinalModifiers(builder))
       .setOptions(this.graphOptions)
+      .setJoinOperations(this.joinOperations)
       .fetchColumnInfo(builder);
   }
 
   onBuild(builder) {
     this.joiner
-      .setExpression(this.buildFinalExpression())
+      .setExpression(this.buildFinalExpression(builder))
       .setModifiers(this.buildFinalModifiers(builder))
       .setOptions(this.graphOptions)
+      .setJoinOperations(this.joinOperations)
       .build(builder);
   }