objection 3.8.0 → 3.9.1

package/lib/relations/manyToMany/insert/ManyToManyInsertOperation.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const { hasBatchInsert } = require('../../../utils/knexUtils');
 const { RelationInsertOperation } = require('../../RelationInsertOperation');
 const { ManyToManyRelateOperation } = require('../relate/ManyToManyRelateOperation');
 
@@ -19,13 +20,6 @@ class ManyToManyInsertOperation extends RelationInsertOperation {
   async onAfter1(builder, ret) {
     const inserted = await super.onAfter1(builder, ret);
 
-    const relateOp = new ManyToManyRelateOperation('relate', {
-      dontCopyReturning: true,
-      dontCopyOnConflict: true,
-      relation: this.relation,
-      owner: this.owner,
-    });
-
     const modelsToRelate = inserted.filter((it) => {
       return this.relation.relatedProp.hasProps(it);
     });
@@ -36,13 +30,26 @@ class ManyToManyInsertOperation extends RelationInsertOperation {
       }
     }
 
-    if (modelsToRelate.length === 0) {
-      return inserted;
+    if (modelsToRelate.length > 0) {
+      // Without batch inserts, `insertGraph()` inserts the related models one at
+      // a time, and so we need to do the same with their join rows:
+      const batches = hasBatchInsert(builder.knex())
+        ? [modelsToRelate]
+        : modelsToRelate.map((model) => [model]);
+
+      for (const batch of batches) {
+        const relateOp = new ManyToManyRelateOperation('relate', {
+          dontCopyReturning: true,
+          dontCopyOnConflict: true,
+          relation: this.relation,
+          owner: this.owner,
+        });
+
+        relateOp.onAdd(builder, [batch]);
+        await relateOp.queryExecutor(builder);
+      }
     }
 
-    relateOp.onAdd(builder, [modelsToRelate]);
-    await relateOp.queryExecutor(builder);
-
     return inserted;
   }
 }

package/lib/relations/manyToMany/update/ManyToManyUpdateMySqlOperation.js

@@ -3,24 +3,9 @@
 const { ManyToManyUpdateOperationBase } = require('./ManyToManyUpdateOperationBase');
 const { ManyToManyMySqlModifyMixin } = require('../ManyToManyMySqlModifyMixin');
 
-const LimitSelector = /^(orderBy\w*|limit|offset)$/;
-
 class ManyToManyUpdateMySqlOperation extends ManyToManyMySqlModifyMixin(
   ManyToManyUpdateOperationBase,
-) {
-  // MySQL is the only database that supports a limit in updates. Updates that
-  // modify the join rows too update the related rows by their ids, so the limit
-  // and the order pick these ids instead of limiting each batch of them. Without
-  // a limit or an offset, the order picks nothing and the query stays as it is.
-  onBuild(builder) {
-    super.onBuild(builder);
-
-    if (this.relatedIdsQuery && builder.has(/^(limit|offset)$/)) {
-      this.relatedIdsQuery.copyFrom(builder, LimitSelector);
-      builder.clear(LimitSelector);
-    }
-  }
-}
+) {}
 
 module.exports = {
   ManyToManyUpdateMySqlOperation,

package/lib/relations/manyToMany/update/ManyToManyUpdateOperationBase.js

@@ -3,11 +3,16 @@
 const { UpdateOperation } = require('../../../queryBuilder/operations/UpdateOperation');
 const { addWhereGroup } = require('../../../queryBuilder/whereGroups');
 const { chunk } = require('../../../utils/objectUtils');
+const { isMySql, isMsSql } = require('../../../utils/knexUtils');
 
 // Stay below the limits of bound values per statement of SQLite before 3.32
 // (999) and SQL Server (2100), leaving room for the patched values:
 const maxBatchValues = 500;
 
+const LimitSelector = /^(orderBy\w*|limit|offset)$/;
+// Without underscores, so that column name mappers don't change it.
+const relatedPropColumn = 'objectiontmprelated';
+
 class ManyToManyUpdateOperationBase extends UpdateOperation {
   constructor(name, opt) {
     super(name, opt);
@@ -59,9 +64,21 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
 
   onBuild(builder) {
     if (this.modifiesJoinTable && this.modifyFilterSubquery) {
+      // MySQL and SQL Server are the only databases that support a limit in
+      // updates. As the related rows are updated by their ids, the limit and
+      // the order pick these ids instead of limiting each batch of them.
+      // Without a limit or an offset, the order picks nothing and the query
+      // stays as it is.
+      const knex = builder.knex();
+      const hasLimit = (isMySql(knex) || isMsSql(knex)) && builder.has(/^(limit|offset)$/);
+
       // The related rows to update are identified by their ids first, see
       // `onAfterBuild()`.
-      this.relatedIdsQuery = this.createRelatedIdsQuery(builder);
+      this.relatedIdsQuery = this.createRelatedIdsQuery(builder, hasLimit);
+      if (hasLimit) {
+        this.relatedIdsQuery.copyFrom(builder, LimitSelector);
+        builder.clear(LimitSelector);
+      }
     }
 
     return super.onBuild(builder);
@@ -69,38 +86,78 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
 
   // Selects the ids of the related rows to update. Fetch raw rows, so that no
   // hooks run for them.
-  createRelatedIdsQuery(builder) {
-    const relatedIdsQuery = this.relation.relatedModelClass.query().childQueryOf(builder);
-    const idRefs = this.relation.relatedModelClass.getIdRelationProperty().refs(relatedIdsQuery);
-
-    return this.applyModifyFilterForRelatedTable(relatedIdsQuery).select(idRefs);
+  createRelatedIdsQuery(builder, hasLimit) {
+    const { relatedModelClass, relatedProp } = this.relation;
+
+    if (hasLimit) {
+      // The limit picks related rows, not join rows, so filter the related
+      // table by the ids of the matching join rows. The join rows refer to the
+      // related rows by their `relatedProp`, which isn't necessarily their id
+      // (#360). We select it after the ids, aliased as it can be the same
+      // columns:
+      const relatedIdsQuery = relatedModelClass.query().childQueryOf(builder);
+      const idRefs = relatedModelClass.getIdRelationProperty().refs(relatedIdsQuery);
+      const relatedPropRefs = relatedProp
+        .refs(relatedIdsQuery)
+        .map((ref, index) => ref.as(`${relatedPropColumn}${index}`));
+      return this.applyModifyFilterForRelatedTable(relatedIdsQuery).select([
+        ...idRefs,
+        ...relatedPropRefs,
+      ]);
+    } else {
+      // Select the ids from the filter subquery itself. Filtering the related
+      // table by them again lets Postgres pick plans that run the subquery
+      // once per related row, which takes minutes for a few thousand (#161).
+      const relatedIdsQuery = this.modifyFilterSubquery.clone();
+      const idRefs = relatedModelClass.getIdRelationProperty().refs(relatedIdsQuery);
+      return relatedIdsQuery.distinct(idRefs);
+    }
   }
 
   // Update the join rows before the related rows, while the filters still
   // match them (#161). A query resolved without updating anything, e.g. with
   // `none()`, leaves the join rows untouched too.
   async onAfterBuild(builder) {
+    // An offset that `onBuild()` moved to the ids query is honoured, and
+    // doesn't warn.
+    super.onAfterBuild(builder);
     if (this.hasExtraProps && !this.isResolvedWithoutUpdate(builder)) {
+      let relatedValues = null;
       if (this.relatedIdsQuery) {
         // The filters can also reference the extras that the join rows change,
         // so identify the related rows to modify first.
         const rows = await this.relatedIdsQuery.toKnexQuery();
-        this.relatedIds = rows.map((row) => Object.values(row));
+        const values = rows.map((row) => Object.values(row));
+        const { size } = this.relation.relatedModelClass.getIdRelationProperty();
+        this.relatedIds = values.map((row) => row.slice(0, size));
+        if (this.relatedIdsQuery.has(/^(limit|offset)$/)) {
+          relatedValues = values.map((row) => row.slice(size));
+        }
       }
 
-      if (this.relatedIdsQuery?.has(/^(limit|offset)$/)) {
+      let numJoinRowsUpdated;
+      if (relatedValues) {
         // The filters match the join rows of more related rows than the limit
-        // picks, so only update the join rows of the picked ones.
-        for (const ids of this.relatedIdBatches()) {
-          await this.updateJoinRows(builder, ids);
+        // picks, so only update the join rows of the picked ones, by the
+        // `relatedProp` values that follow their ids.
+        for (const values of chunkKeys(relatedValues, this.relation.relatedProp.size)) {
+          await this.updateJoinRows(builder, values);
         }
       } else {
-        await this.updateJoinRows(builder);
+        numJoinRowsUpdated = await this.updateJoinRows(builder);
+      }
+
+      if (this.hasNothingToUpdate) {
+        // Only the join rows change, so count the related rows that they
+        // belong to, as the update of the related rows does. Without their
+        // ids, the join rows all belong to the updated model, see
+        // `applyIdFilterForJoinTable()`.
+        builder.resolve(this.relatedIds?.length ?? Math.min(numJoinRowsUpdated, 1));
       }
     }
   }
 
-  updateJoinRows(builder, relatedIds = null) {
+  updateJoinRows(builder, relatedValues = null) {
     const joinTableUpdateQuery = this.relation
       .getJoinModelClass(builder.knex())
       .query()
@@ -112,11 +169,11 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
         ? this.applyIdFilterForJoinTable(joinTableUpdateQuery)
         : this.applyModifyFilterForJoinTable(joinTableUpdateQuery),
     );
-    if (relatedIds) {
+    if (relatedValues) {
       addWhereGroup(joinTableUpdateQuery, () =>
         joinTableUpdateQuery.whereInComposite(
           this.relation.joinTableRelatedProp.refs(joinTableUpdateQuery),
-          relatedIds,
+          relatedValues,
         ),
       );
     }
@@ -131,7 +188,10 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
   // number of values in one statement.
   queryBatches(builder) {
     if (this.relatedIds) {
-      const batches = this.relatedIdBatches();
+      const batches = chunkKeys(
+        this.relatedIds,
+        this.relation.relatedModelClass.getIdRelationProperty().size,
+      );
       // Without ids, still execute the query once for the dialect's result:
       return batches.length > 0 ? batches : [[]];
     } else {
@@ -139,11 +199,6 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     }
   }
 
-  relatedIdBatches() {
-    const idSize = this.relation.relatedModelClass.getIdRelationProperty().size;
-    return chunk(this.relatedIds, Math.floor(maxBatchValues / idSize));
-  }
-
   // Filters the join rows by the ids of the owner and the updated model.
   applyIdFilterForJoinTable(builder) {
     return builder
@@ -180,6 +235,10 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
   }
 }
 
+function chunkKeys(keys, size) {
+  return chunk(keys, Math.floor(maxBatchValues / size));
+}
+
 module.exports = {
   ManyToManyUpdateOperationBase,
 };

package/lib/utils/knexUtils.js

@@ -36,6 +36,12 @@ function isMsSql(knex) {
   return getDialect(knex) === 'mssql';
 }
 
+// Only Postgres and SQL Server return the identifiers of all the rows inserted
+// by one query.
+function hasBatchInsert(knex) {
+  return isPostgres(knex) || isMsSql(knex);
+}
+
 // MySQL supports window functions since 8.0, MariaDB since 10.2. Without a
 // version, knex assumes the latest one, and so do we.
 function hasWindowFunctions(knex) {
@@ -82,6 +88,7 @@ module.exports = {
   isSqlite,
   isMsSql,
   isOracle,
+  hasBatchInsert,
   hasWindowFunctions,
   isKnexQueryBuilder,
   isKnexJoinBuilder,

package/lib/utils/objectUtils.js

@@ -16,6 +16,10 @@ function isObject(value) {
   return value !== null && typeof value === 'object';
 }
 
+function hasOwn(obj, prop) {
+  return Object.prototype.hasOwnProperty.call(obj, prop);
+}
+
 // Quick and dirty check if an object is a plain object and not
 // for example an instance of some class.
 function isPlainObject(value) {
@@ -502,6 +506,7 @@ module.exports = {
   isString,
   isRegExp,
   isObject,
+  hasOwn,
   isNumber,
   isFunction,
   jsonEquals,

package/lib/utils/parseFieldExpression.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const jsonFieldExpressionParser = require('../queryBuilder/parsers/jsonFieldExpressionParser');
-const { isObject } = require('./objectUtils');
+const { isObject, hasOwn } = require('./objectUtils');
 const cache = new Map();
 
 function parseFieldExpression(expr) {
@@ -51,10 +51,21 @@ function freezeParsedExpr(parsedExpr) {
 // `'meta:a.b'` that write into json columns, unless they are `declared`: the
 // `properties` of the schema, or an object with their column names as keys.
 function isFieldExpressionKey(key, declared) {
-  return (
-    key.includes(':') &&
-    !(isObject(declared) && Object.prototype.hasOwnProperty.call(declared, key))
-  );
+  return key.includes(':') && !(isObject(declared) && hasOwn(declared, key));
+}
+
+// Returns the json column of a field expression key like `'meta:a'`, or `null`
+// if the key isn't a field expression or doesn't parse as one.
+function getFieldExpressionColumn(key, declared) {
+  if (isFieldExpressionKey(key, declared)) {
+    try {
+      return parseFieldExpression(key).column;
+    } catch (err) {
+      return null;
+    }
+  } else {
+    return null;
+  }
 }
 
 // Formats the json path of a parsed field expression as a Postgres text array
@@ -93,6 +104,7 @@ function toPostgresJsonPathLiteral(access) {
 module.exports = {
   parseFieldExpression,
   isFieldExpressionKey,
+  getFieldExpressionColumn,
   toPostgresJsonPath,
   toPostgresJsonPathLiteral,
 };

package/lib/utils/promiseUtils/map.js

@@ -43,7 +43,8 @@ function mapMany(items, mapper, opt = {}) {
     const ctx = {
       reject,
       resolve,
-      rejected: false,
+      failed: false,
+      error: null,
       index: 0,
       numFinished: 0,
       results: new Array(items.length),
@@ -51,7 +52,7 @@ function mapMany(items, mapper, opt = {}) {
       mapper,
     };
 
-    while (ctx.index < concurrency && ctx.index < items.length && !ctx.rejected) {
+    while (ctx.index < concurrency && ctx.index < items.length && !ctx.failed) {
       executeNext(ctx);
     }
   });
@@ -59,7 +60,7 @@ function mapMany(items, mapper, opt = {}) {
 
 function executeNext(ctx) {
   try {
-    if (ctx.rejected) {
+    if (ctx.failed) {
       return;
     }
 
@@ -82,12 +83,14 @@ function executeNext(ctx) {
 }
 
 function afterExecute(ctx, result, index) {
-  if (ctx.rejected) {
+  ctx.numFinished++;
+
+  if (ctx.failed) {
+    rejectWhenFinished(ctx);
     return null;
   }
 
   ctx.results[index] = result;
-  ctx.numFinished++;
 
   if (ctx.numFinished === ctx.items.length) {
     ctx.resolve(ctx.results);
@@ -101,8 +104,22 @@ function afterExecute(ctx, result, index) {
 }
 
 function onError(ctx, err) {
-  ctx.rejected = true;
-  ctx.reject(err);
+  ctx.numFinished++;
+
+  if (!ctx.failed) {
+    ctx.failed = true;
+    ctx.error = err;
+  }
+
+  rejectWhenFinished(ctx);
+}
+
+// Wait for all started mappers before rejecting: They may run queries in a
+// transaction that the caller rolls back on rejection.
+function rejectWhenFinished(ctx) {
+  if (ctx.numFinished === ctx.index) {
+    ctx.reject(ctx.error);
+  }
 }
 
 module.exports = {

package/lib/utils/wrapDbError.js

@@ -8,6 +8,15 @@ const MSSQL_PRIMARY_KEY_REGEX =
   /Violation of PRIMARY KEY constraint '(.+)'\. Cannot insert duplicate key in object '(.+)\.(.+)'\. The duplicate key value is \((.+)\)/;
 
 function wrapDbError(err) {
+  // tedious throws an AggregateError with an empty message when SQL Server
+  // reports several errors for one statement, and knex only prepends the SQL to
+  // it. Add the messages of its errors, to give the reasons.
+  if (isTediousAggregateError(err)) {
+    const wrapped = new DBError({ nativeError: err, client: 'mssql' });
+    wrapped.message += err.errors.map((error) => error.message).join(' ');
+    return wrapped;
+  }
+
   if (!isTediousError(err)) {
     return wrapError(err);
   }
@@ -45,6 +54,11 @@ function isTediousError(err) {
   return err?.code === 'EREQUEST' && Number.isInteger(err.number);
 }
 
+// Check the name, as AggregateError isn't defined on Node 14.
+function isTediousAggregateError(err) {
+  return err?.name === 'AggregateError' && err.errors.every(isTediousError);
+}
+
 module.exports = {
   wrapDbError,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "objection",
-  "version": "3.8.0",
+  "version": "3.9.1",
   "description": "An SQL-friendly ORM for Node.js",
   "main": "lib/objection.js",
   "license": "MIT",

package/typings/objection/index.d.ts

@@ -374,20 +374,30 @@ declare namespace Objection {
 
   /**
    * Converts the query builder NQB to the same result kind as QB:
-   * array, single, maybe-single or page. Any other result type, e.g. after
+   * array, page, single or maybe-single. Any other result type, e.g. after
    * `castTo()` or `patch()`, falls back to QB, without narrowing.
+   * QB's model type M and result type R are separate parameters so that QB
+   * only appears in the result, and TypeScript can relate the kinds of two
+   * query builders, e.g. of a custom query builder and its base class, through
+   * QB alone. The page and `object` checks come before the single ones, so
+   * that results like `number` or `Page<M>` are ruled out for a generic M too.
    */
-  type WithResultKind<QB extends AnyQueryBuilder, NQB extends AnyQueryBuilder> = [
-    ResultType<QB>,
-  ] extends [ModelType<QB>[]]
+  type WithResultKind<
+    QB extends AnyQueryBuilder,
+    NQB extends AnyQueryBuilder,
+    M extends Model = ModelType<QB>,
+    R = ResultType<QB>,
+  > = [R] extends [M[]]
     ? ArrayQueryBuilder<NQB>
-    : [ResultType<QB>] extends [ModelType<QB>]
-      ? SingleQueryBuilder<NQB>
-      : [ResultType<QB>] extends [ModelType<QB> | undefined]
-        ? MaybeSingleQueryBuilder<NQB>
-        : [ResultType<QB>] extends [Page<ModelType<QB>>]
-          ? PageQueryBuilder<NQB>
-          : QB;
+    : [R] extends [Page<M>]
+      ? PageQueryBuilder<NQB>
+      : [R] extends [object | undefined]
+        ? [R] extends [M]
+          ? SingleQueryBuilder<NQB>
+          : [R] extends [M | undefined]
+            ? MaybeSingleQueryBuilder<NQB>
+            : QB
+        : QB;
 
   /**
    * If T is an array, returns the item type, otherwise returns T.