objection 3.7.0 → 3.9.0

package/lib/relations/manyToMany/find/ManyToManyFindOperation.js

@@ -2,7 +2,7 @@
 
 const { RelationFindOperation } = require('../../RelationFindOperation');
 const { getTempColumn } = require('../../../utils/tmpColumnUtils');
-const { deprecate } = require('../../../utils/deprecate');
+const { warnOnce } = require('../../../utils/warnings');
 
 class ManyToManyFindOperation extends RelationFindOperation {
   constructor(name, opt) {
@@ -83,6 +83,11 @@ class ManyToManyFindOperation extends RelationFindOperation {
     return related;
   }
 
+  // @override
+  ownerKeys() {
+    return this.ownerJoinColumnAlias;
+  }
+
   // The related models are assigned to their owners by the owner join columns
   // selected from the join table. If they get lost, e.g. in a custom
   // `$parseDatabaseJson()`, the related models are silently dropped (#2258).
@@ -93,7 +98,7 @@ class ManyToManyFindOperation extends RelationFindOperation {
       const alias = this.ownerJoinColumnAlias[c];
 
       if (related.some((rel) => rel[alias] === undefined)) {
-        deprecate(
+        warnOnce(
           `Fetching relation "${this.relation.name}" of ${this.relation.ownerModelClass.name}: ` +
             `some related models are missing the join table column ` +
             `"${joinTableOwnerProp.propDescription(c)}" selected as "${alias}", so they are not ` +

package/lib/relations/manyToMany/insert/ManyToManyInsertOperation.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const { hasBatchInsert } = require('../../../utils/knexUtils');
 const { RelationInsertOperation } = require('../../RelationInsertOperation');
 const { ManyToManyRelateOperation } = require('../relate/ManyToManyRelateOperation');
 
@@ -19,13 +20,6 @@ class ManyToManyInsertOperation extends RelationInsertOperation {
   async onAfter1(builder, ret) {
     const inserted = await super.onAfter1(builder, ret);
 
-    const relateOp = new ManyToManyRelateOperation('relate', {
-      dontCopyReturning: true,
-      dontCopyOnConflict: true,
-      relation: this.relation,
-      owner: this.owner,
-    });
-
     const modelsToRelate = inserted.filter((it) => {
       return this.relation.relatedProp.hasProps(it);
     });
@@ -36,13 +30,26 @@ class ManyToManyInsertOperation extends RelationInsertOperation {
       }
     }
 
-    if (modelsToRelate.length === 0) {
-      return inserted;
+    if (modelsToRelate.length > 0) {
+      // Without batch inserts, `insertGraph()` inserts the related models one at
+      // a time, and so we need to do the same with their join rows:
+      const batches = hasBatchInsert(builder.knex())
+        ? [modelsToRelate]
+        : modelsToRelate.map((model) => [model]);
+
+      for (const batch of batches) {
+        const relateOp = new ManyToManyRelateOperation('relate', {
+          dontCopyReturning: true,
+          dontCopyOnConflict: true,
+          relation: this.relation,
+          owner: this.owner,
+        });
+
+        relateOp.onAdd(builder, [batch]);
+        await relateOp.queryExecutor(builder);
+      }
     }
 
-    relateOp.onAdd(builder, [modelsToRelate]);
-    await relateOp.queryExecutor(builder);
-
     return inserted;
   }
 }

package/lib/relations/manyToMany/update/ManyToManyUpdateOperationBase.js

@@ -2,6 +2,14 @@
 
 const { UpdateOperation } = require('../../../queryBuilder/operations/UpdateOperation');
 const { addWhereGroup } = require('../../../queryBuilder/whereGroups');
+const { chunk } = require('../../../utils/objectUtils');
+const { isMySql, isMsSql } = require('../../../utils/knexUtils');
+
+// Stay below the limits of bound values per statement of SQLite before 3.32
+// (999) and SQL Server (2100), leaving room for the patched values:
+const maxBatchValues = 500;
+
+const LimitSelector = /^(orderBy\w*|limit|offset)$/;
 
 class ManyToManyUpdateOperationBase extends UpdateOperation {
   constructor(name, opt) {
@@ -11,6 +19,8 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     this.owner = opt.owner;
 
     this.hasExtraProps = false;
+    this.relatedIdsQuery = null;
+    this.relatedIds = null;
     this.joinTablePatch = {};
     this.joinTablePatchFilterQuery = null;
   }
@@ -50,32 +60,126 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     return res;
   }
 
-  async onAfter1(builder, result) {
-    // A query resolved without updating anything, e.g. with `none()`, leaves
-    // the join rows untouched too.
+  onBuild(builder) {
+    if (this.modifiesJoinTable && this.modifyFilterSubquery) {
+      // MySQL and SQL Server are the only databases that support a limit in
+      // updates. As the related rows are updated by their ids, the limit and
+      // the order pick these ids instead of limiting each batch of them.
+      // Without a limit or an offset, the order picks nothing and the query
+      // stays as it is.
+      const knex = builder.knex();
+      const hasLimit = (isMySql(knex) || isMsSql(knex)) && builder.has(/^(limit|offset)$/);
+
+      // The related rows to update are identified by their ids first, see
+      // `onAfterBuild()`.
+      this.relatedIdsQuery = this.createRelatedIdsQuery(builder, hasLimit);
+      if (hasLimit) {
+        this.relatedIdsQuery.copyFrom(builder, LimitSelector);
+        builder.clear(LimitSelector);
+      }
+    }
+
+    return super.onBuild(builder);
+  }
+
+  // Selects the ids of the related rows to update. Fetch raw rows, so that no
+  // hooks run for them.
+  createRelatedIdsQuery(builder, hasLimit) {
+    const { relatedModelClass } = this.relation;
+
+    if (hasLimit) {
+      // The limit picks related rows, not join rows, so filter the related
+      // table by the ids of the matching join rows:
+      const relatedIdsQuery = relatedModelClass.query().childQueryOf(builder);
+      const idRefs = relatedModelClass.getIdRelationProperty().refs(relatedIdsQuery);
+      return this.applyModifyFilterForRelatedTable(relatedIdsQuery).select(idRefs);
+    } else {
+      // Select the ids from the filter subquery itself. Filtering the related
+      // table by them again lets Postgres pick plans that run the subquery
+      // once per related row, which takes minutes for a few thousand (#161).
+      const relatedIdsQuery = this.modifyFilterSubquery.clone();
+      const idRefs = relatedModelClass.getIdRelationProperty().refs(relatedIdsQuery);
+      return relatedIdsQuery.distinct(idRefs);
+    }
+  }
+
+  // Update the join rows before the related rows, while the filters still
+  // match them (#161). A query resolved without updating anything, e.g. with
+  // `none()`, leaves the join rows untouched too.
+  async onAfterBuild(builder) {
     if (this.hasExtraProps && !this.isResolvedWithoutUpdate(builder)) {
-      const joinTableUpdateQuery = this.relation
-        .getJoinModelClass(builder.knex())
-        .query()
-        .childQueryOf(builder)
-        .patch(this.joinTablePatch);
+      if (this.relatedIdsQuery) {
+        // The filters can also reference the extras that the join rows change,
+        // so identify the related rows to modify first.
+        const rows = await this.relatedIdsQuery.toKnexQuery();
+        this.relatedIds = rows.map((row) => Object.values(row));
+      }
 
+      let numJoinRowsUpdated;
+      if (this.relatedIdsQuery?.has(/^(limit|offset)$/)) {
+        // The filters match the join rows of more related rows than the limit
+        // picks, so only update the join rows of the picked ones.
+        for (const ids of this.relatedIdBatches()) {
+          await this.updateJoinRows(builder, ids);
+        }
+      } else {
+        numJoinRowsUpdated = await this.updateJoinRows(builder);
+      }
+
+      if (this.hasNothingToUpdate) {
+        // Only the join rows change, so count the related rows that they
+        // belong to, as the update of the related rows does. Without their
+        // ids, the join rows all belong to the updated model, see
+        // `applyIdFilterForJoinTable()`.
+        builder.resolve(this.relatedIds?.length ?? Math.min(numJoinRowsUpdated, 1));
+      }
+    }
+  }
+
+  updateJoinRows(builder, relatedIds = null) {
+    const joinTableUpdateQuery = this.relation
+      .getJoinModelClass(builder.knex())
+      .query()
+      .childQueryOf(builder)
+      .patch(this.joinTablePatch);
+
+    addWhereGroup(joinTableUpdateQuery, () =>
+      this.skipRelationFilter
+        ? this.applyIdFilterForJoinTable(joinTableUpdateQuery)
+        : this.applyModifyFilterForJoinTable(joinTableUpdateQuery),
+    );
+    if (relatedIds) {
       addWhereGroup(joinTableUpdateQuery, () =>
-        this.skipRelationFilter
-          ? this.applyIdFilterForJoinTable(joinTableUpdateQuery)
-          : this.applyModifyFilterForJoinTable(joinTableUpdateQuery),
-      );
-      addWhereGroup(joinTableUpdateQuery, () =>
-        joinTableUpdateQuery.modify(this.relation.joinTableModify),
+        joinTableUpdateQuery.whereInComposite(
+          this.relation.joinTableRelatedProp.refs(joinTableUpdateQuery),
+          relatedIds,
+        ),
       );
+    }
+    addWhereGroup(joinTableUpdateQuery, () =>
+      joinTableUpdateQuery.modify(this.relation.joinTableModify),
+    );
+
+    return joinTableUpdateQuery;
+  }
 
-      await joinTableUpdateQuery;
-      return result;
+  // Update the related rows in batches of their ids, as databases limit the
+  // number of values in one statement.
+  queryBatches(builder) {
+    if (this.relatedIds) {
+      const batches = this.relatedIdBatches();
+      // Without ids, still execute the query once for the dialect's result:
+      return batches.length > 0 ? batches : [[]];
     } else {
-      return result;
+      return null;
     }
   }
 
+  relatedIdBatches() {
+    const idSize = this.relation.relatedModelClass.getIdRelationProperty().size;
+    return chunk(this.relatedIds, Math.floor(maxBatchValues / idSize));
+  }
+
   // Filters the join rows by the ids of the owner and the updated model.
   applyIdFilterForJoinTable(builder) {
     return builder
@@ -103,6 +207,8 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     const clone = super.clone();
 
     clone.hasExtraProps = this.hasExtraProps;
+    clone.relatedIdsQuery = this.relatedIdsQuery;
+    clone.relatedIds = this.relatedIds;
     clone.joinTablePatch = this.joinTablePatch;
     clone.joinTablePatchFilterQuery = this.joinTablePatchFilterQuery;
 

package/lib/utils/identifierMapping.js

@@ -171,8 +171,8 @@ function keyMapper(mapper) {
 // Returns a function that only calls `mapper` for the part of the input string
 // before the first `separator`, and leaves the rest untouched. For field
 // expressions like `jsonColumn:some.jsonPath` this maps the column but keeps
-// the json path as is. Column names can't contain `:`, so the first one marks
-// the boundary.
+// the json path as is. The mapper doesn't know the schema, so the first `:`
+// marks the boundary also in a declared column name like `ns:name`.
 function mapFirstPart(mapper, separator) {
   return (str) => {
     const idx = str.indexOf(separator);

package/lib/utils/knexUtils.js

@@ -36,6 +36,25 @@ function isMsSql(knex) {
   return getDialect(knex) === 'mssql';
 }
 
+// Only Postgres and SQL Server return the identifiers of all the rows inserted
+// by one query.
+function hasBatchInsert(knex) {
+  return isPostgres(knex) || isMsSql(knex);
+}
+
+// MySQL supports window functions since 8.0, MariaDB since 10.2. Without a
+// version, knex assumes the latest one, and so do we.
+function hasWindowFunctions(knex) {
+  const { version, isMariaDB } = knex.client;
+
+  if (isMySql(knex) && version) {
+    const [major, minor] = version.split('.').map(Number);
+    return isMariaDB ? major > 10 || (major === 10 && minor >= 2) : major >= 8;
+  } else {
+    return true;
+  }
+}
+
 function isKnexQueryBuilder(value) {
   return (
     hasConstructor(value) &&
@@ -69,6 +88,8 @@ module.exports = {
   isSqlite,
   isMsSql,
   isOracle,
+  hasBatchInsert,
+  hasWindowFunctions,
   isKnexQueryBuilder,
   isKnexJoinBuilder,
   isKnexRaw,

package/lib/utils/objectUtils.js

@@ -16,6 +16,10 @@ function isObject(value) {
   return value !== null && typeof value === 'object';
 }
 
+function hasOwn(obj, prop) {
+  return Object.prototype.hasOwnProperty.call(obj, prop);
+}
+
 // Quick and dirty check if an object is a plain object and not
 // for example an instance of some class.
 function isPlainObject(value) {
@@ -502,6 +506,7 @@ module.exports = {
   isString,
   isRegExp,
   isObject,
+  hasOwn,
   isNumber,
   isFunction,
   jsonEquals,

package/lib/utils/parseFieldExpression.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const jsonFieldExpressionParser = require('../queryBuilder/parsers/jsonFieldExpressionParser');
+const { isObject, hasOwn } = require('./objectUtils');
 const cache = new Map();
 
 function parseFieldExpression(expr) {
@@ -46,6 +47,13 @@ function freezeParsedExpr(parsedExpr) {
   return parsedExpr;
 }
 
+// Keys with a ':' in the data of patches and updates are field expressions like
+// `'meta:a.b'` that write into json columns, unless they are `declared`: the
+// `properties` of the schema, or an object with their column names as keys.
+function isFieldExpressionKey(key, declared) {
+  return key.includes(':') && !(isObject(declared) && hasOwn(declared, key));
+}
+
 // Formats the json path of a parsed field expression as a Postgres text array
 // literal, e.g. `{a,"",b}`. Keys that would otherwise be misread (empty keys,
 // keys with whitespace, delimiters, quotes or backslashes and `NULL`) are
@@ -81,6 +89,7 @@ function toPostgresJsonPathLiteral(access) {
 
 module.exports = {
   parseFieldExpression,
+  isFieldExpressionKey,
   toPostgresJsonPath,
   toPostgresJsonPathLiteral,
 };

package/lib/utils/warnings.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const LOGGED_WARNINGS = new Set();
+
+// Log each warning only once.
+function warnOnce(message) {
+  if (!LOGGED_WARNINGS.has(message)) {
+    LOGGED_WARNINGS.add(message);
+    console.warn(`[objection] ${message}`);
+  }
+}
+
+function deprecate(message) {
+  warnOnce(message);
+}
+
+// Only meant for tests that need to see a warning again.
+function resetWarnings() {
+  LOGGED_WARNINGS.clear();
+}
+
+module.exports = {
+  warnOnce,
+  deprecate,
+  resetWarnings,
+};

package/lib/utils/wrapDbError.js

@@ -2,35 +2,49 @@
 
 const { wrapError, DBError, UniqueViolationError } = require('db-errors');
 
-// db-errors only recognizes mssql unique key violations, but the same error
-// number (2627) is also used for primary key violations (#2688).
+// db-errors only recognizes mssql unique key violations, but primary key
+// violations have the same error number, with a different message (#2688).
 const MSSQL_PRIMARY_KEY_REGEX =
   /Violation of PRIMARY KEY constraint '(.+)'\. Cannot insert duplicate key in object '(.+)\.(.+)'\. The duplicate key value is \((.+)\)/;
 
 function wrapDbError(err) {
-  const wrapped = wrapError(err);
-
-  if (wrapped.constructor === DBError && wrapped.client === 'mssql') {
-    const { nativeError } = wrapped;
-    const match =
-      nativeError.info &&
-      nativeError.info.number === 2627 &&
-      MSSQL_PRIMARY_KEY_REGEX.exec(nativeError.message);
-
-    if (match) {
-      return new UniqueViolationError({
-        nativeError,
-        client: 'mssql',
-        table: match[3],
-        schema: match[2],
-        constraint: match[1],
-      });
-    }
+  if (!isTediousError(err)) {
+    return wrapError(err);
+  }
+
+  // db-errors parses the errors of the `mssql` package, which wraps the errors
+  // of tedious and keeps their number and class in `originalError.info`. knex
+  // uses tedious directly, so its errors are passed on in that shape.
+  const wrapped = wrapError({
+    code: err.code,
+    originalError: { message: err.message, info: { number: err.number, class: err.class } },
+  });
+
+  if (!(wrapped instanceof DBError)) {
+    return err;
+  }
+
+  wrapped.nativeError = err;
+
+  const match = wrapped.constructor === DBError && MSSQL_PRIMARY_KEY_REGEX.exec(err.message);
+
+  if (match) {
+    return new UniqueViolationError({
+      nativeError: err,
+      client: 'mssql',
+      table: match[3],
+      schema: match[2],
+      constraint: match[1],
+    });
   }
 
   return wrapped;
 }
 
+function isTediousError(err) {
+  return err?.code === 'EREQUEST' && Number.isInteger(err.number);
+}
+
 module.exports = {
   wrapDbError,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "objection",
-  "version": "3.7.0",
+  "version": "3.9.0",
   "description": "An SQL-friendly ORM for Node.js",
   "main": "lib/objection.js",
   "license": "MIT",
@@ -9,7 +9,7 @@
     "test:fast": "mocha --slow 10 --timeout 15000 --reporter spec --recursive tests --bail --exclude \"tests/unit/relations/files/**\"",
     "test:typings": "tsc && prettier --check \"typings/**/*.ts\" \"tests/ts/**/*.ts\"",
     "db:up": "docker compose up -d --wait",
-    "db:down": "docker compose down -v",
+    "db:down": "docker compose --profile mssql down -v",
     "build:parsers": "npx --yes peggy@5.1.0 --format commonjs -o lib/queryBuilder/parsers/jsonFieldExpressionParser.js lib/queryBuilder/parsers/jsonFieldExpressionParser.pegjs && npx --yes peggy@5.1.0 --format commonjs -o lib/queryBuilder/parsers/relationExpressionParser.js lib/queryBuilder/parsers/relationExpressionParser.pegjs && prettier --write \"lib/queryBuilder/parsers/*Parser.js\"",
     "prettier": "prettier --write \"{examples,lib,tests,testUtils,typings,doc}/**/*.{js,ts}\"",
     "eslint": "eslint --format codeframe \"examples/**/*.js\" \"lib/**/*.js\" \"tests/**/*.js\" \"testUtils/**/*.js\" \"*.js\"",
@@ -66,7 +66,7 @@
     "db-errors": "^0.2.3"
   },
   "peerDependencies": {
-    "knex": ">=2.3.0"
+    "knex": ">=2.3.0 <4"
   },
   "devDependencies": {
     "@types/node": "^22.15.3",
@@ -82,7 +82,8 @@
     "pg": "^8.15.6",
     "prettier": "3.9.9",
     "sqlite3": "^5.1.7",
-    "typescript": ">=5.8.3",
+    "tedious": "^20.3.7",
+    "typescript": "^5.8.3",
     "vitepress": "1.6.4"
   }
 }

package/typings/objection/index.d.ts

@@ -374,20 +374,30 @@ declare namespace Objection {
 
   /**
    * Converts the query builder NQB to the same result kind as QB:
-   * array, single, maybe-single or page. Any other result type, e.g. after
+   * array, page, single or maybe-single. Any other result type, e.g. after
    * `castTo()` or `patch()`, falls back to QB, without narrowing.
+   * QB's model type M and result type R are separate parameters so that QB
+   * only appears in the result, and TypeScript can relate the kinds of two
+   * query builders, e.g. of a custom query builder and its base class, through
+   * QB alone. The page and `object` checks come before the single ones, so
+   * that results like `number` or `Page<M>` are ruled out for a generic M too.
    */
-  type WithResultKind<QB extends AnyQueryBuilder, NQB extends AnyQueryBuilder> = [
-    ResultType<QB>,
-  ] extends [ModelType<QB>[]]
+  type WithResultKind<
+    QB extends AnyQueryBuilder,
+    NQB extends AnyQueryBuilder,
+    M extends Model = ModelType<QB>,
+    R = ResultType<QB>,
+  > = [R] extends [M[]]
     ? ArrayQueryBuilder<NQB>
-    : [ResultType<QB>] extends [ModelType<QB>]
-      ? SingleQueryBuilder<NQB>
-      : [ResultType<QB>] extends [ModelType<QB> | undefined]
-        ? MaybeSingleQueryBuilder<NQB>
-        : [ResultType<QB>] extends [Page<ModelType<QB>>]
-          ? PageQueryBuilder<NQB>
-          : QB;
+    : [R] extends [Page<M>]
+      ? PageQueryBuilder<NQB>
+      : [R] extends [object | undefined]
+        ? [R] extends [M]
+          ? SingleQueryBuilder<NQB>
+          : [R] extends [M | undefined]
+            ? MaybeSingleQueryBuilder<NQB>
+            : QB
+        : QB;
 
   /**
    * If T is an array, returns the item type, otherwise returns T.
@@ -1022,8 +1032,11 @@ declare namespace Objection {
   /**
    * With `returning()`, `relate()` returns the inserted join rows for
    * many-to-many relations, the updated related rows for has-many relations
-   * and the updated owner rows for belongs-to-one relations. The typings can't
-   * tell these apart, so the rows are typed as `any`.
+   * and the updated owner rows for belongs-to-one relations. MySQL has no
+   * `returning`: there, many-to-many relations return the join rows built from
+   * the input, without columns generated by the database, and the other
+   * relations still return the number of affected rows. The typings can't tell
+   * these apart, so the rows are typed as `any`.
    */
   interface RelateReturningMethod<M extends Model> {
     (...args: ReturningArgs): QueryBuilder<M, any[]>;
@@ -1085,6 +1098,10 @@ declare namespace Objection {
   export interface InsertGraphOptions {
     relate?: boolean | string | string[];
     allowRefs?: boolean;
+    /**
+     * @deprecated Has no effect, since `insertGraph()` rejects `#unrelate` and
+     * `#delete` markers. Removed in objection 4.0.
+     */
     allowMarkers?: boolean;
   }
 
@@ -1140,6 +1157,7 @@ declare namespace Objection {
     aliases?: { [key: string]: string };
     joinOperation?: string;
     maxBatchSize?: number;
+    limitPerParent?: boolean;
   }
 
   export interface WithGraphOptions extends GraphOptions {

package/lib/utils/deprecate.js

@@ -1,21 +0,0 @@
-'use strict';
-
-const LOGGED_DEPRECATIONS = new Set();
-
-function deprecate(message) {
-  // Only log deprecation messages once.
-  if (!LOGGED_DEPRECATIONS.has(message)) {
-    LOGGED_DEPRECATIONS.add(message);
-    console.warn(message);
-  }
-}
-
-// Only meant for tests that need to see a deprecation message again.
-function resetDeprecations() {
-  LOGGED_DEPRECATIONS.clear();
-}
-
-module.exports = {
-  deprecate,
-  resetDeprecations,
-};