objection 3.5.0 → 3.7.0

package/lib/relations/manyToMany/update/ManyToManyUpdateOperationBase.js

@@ -15,9 +15,18 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     this.joinTablePatchFilterQuery = null;
   }
 
+  get modifiesJoinTable() {
+    return this.hasExtraProps;
+  }
+
   onAdd(builder, args) {
     const obj = args[0];
 
+    // The modified rows are known to be related to the owner, so they are
+    // filtered by their own ids instead of a subquery that joins the join
+    // table. On MySQL, concurrent updates with that subquery deadlock (#135).
+    this.skipRelationFilter = builder.internalOptions().skipRelationFilter;
+
     // Copy all extra properties to the `joinTablePatch` object.
     for (const extra of this.relation.joinTableExtras) {
       if (extra.aliasProp in obj) {
@@ -42,7 +51,9 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
   }
 
   async onAfter1(builder, result) {
-    if (this.hasExtraProps) {
+    // A query resolved without updating anything, e.g. with `none()`, leaves
+    // the join rows untouched too.
+    if (this.hasExtraProps && !this.isResolvedWithoutUpdate(builder)) {
       const joinTableUpdateQuery = this.relation
         .getJoinModelClass(builder.knex())
         .query()
@@ -50,7 +61,9 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
         .patch(this.joinTablePatch);
 
       addWhereGroup(joinTableUpdateQuery, () =>
-        this.applyModifyFilterForJoinTable(joinTableUpdateQuery),
+        this.skipRelationFilter
+          ? this.applyIdFilterForJoinTable(joinTableUpdateQuery)
+          : this.applyModifyFilterForJoinTable(joinTableUpdateQuery),
       );
       addWhereGroup(joinTableUpdateQuery, () =>
         joinTableUpdateQuery.modify(this.relation.joinTableModify),
@@ -63,6 +76,19 @@ class ManyToManyUpdateOperationBase extends UpdateOperation {
     }
   }
 
+  // Filters the join rows by the ids of the owner and the updated model.
+  applyIdFilterForJoinTable(builder) {
+    return builder
+      .whereInComposite(
+        this.relation.joinTableOwnerProp.refs(builder),
+        this.owner.getProps(this.relation),
+      )
+      .whereComposite(
+        this.relation.joinTableRelatedProp.refs(builder),
+        this.relation.relatedProp.getProps(this.model),
+      );
+  }
+
   /* istanbul ignore next */
   applyModifyFilterForRelatedTable(builder) {
     throw new Error('not implemented');

package/lib/utils/identifierMapping.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { isObject, isPlainObject } = require('./objectUtils');
+const { isObject } = require('./objectUtils');
 
 // Super fast memoize for single argument functions.
 function memoize(func) {
@@ -149,9 +149,8 @@ function mapLastPart(mapper, separator) {
 }
 
 // Returns a function that takes an object as an input and maps the object's keys
-// using `mapper`, and its values using the optional `valueMapper`. If the input
-// is not an object, the input is returned unchanged.
-function keyMapper(mapper, valueMapper) {
+// using `mapper`. If the input is not an object, the input is returned unchanged.
+function keyMapper(mapper) {
   return (obj) => {
     if (!isObject(obj) || Array.isArray(obj)) {
       return obj;
@@ -162,7 +161,7 @@ function keyMapper(mapper, valueMapper) {
 
     for (let i = 0, l = keys.length; i < l; ++i) {
       const key = keys[i];
-      out[mapper(key)] = valueMapper ? valueMapper(obj[key]) : obj[key];
+      out[mapper(key)] = obj[key];
     }
 
     return out;
@@ -190,16 +189,10 @@ function snakeCaseMappers(opt = {}) {
   };
 }
 
-function knexIdentifierMappers({ parse, format, idSeparator = ':', mapNestedKeys = false } = {}) {
+function knexIdentifierMappers({ parse, format, idSeparator = ':' } = {}) {
   const formatId = memoize(mapLastPart(format, idSeparator));
   const parseId = memoize(mapLastPart(parse, idSeparator));
   const parseKeys = keyMapper(parseId);
-  // With `mapNestedKeys`, also map the keys of plain objects one level down, as
-  // in the rows knex returns with its `nestTables: true` option on MySQL:
-  // `{ tableName: { columnName: value } }`.
-  const parseRow = mapNestedKeys
-    ? keyMapper(parseId, (value) => (isPlainObject(value) ? parseKeys(value) : value))
-    : parseKeys;
 
   return {
     wrapIdentifier(identifier, origWrap) {
@@ -211,12 +204,12 @@ function knexIdentifierMappers({ parse, format, idSeparator = ':', mapNestedKeys
         const output = new Array(result.length);
 
         for (let i = 0, l = result.length; i < l; ++i) {
-          output[i] = parseRow(result[i]);
+          output[i] = parseKeys(result[i]);
         }
 
         return output;
       } else {
-        return parseRow(result);
+        return parseKeys(result);
       }
     },
   };
@@ -226,11 +219,10 @@ function knexSnakeCaseMappers(opt = {}) {
   return knexIdentifierMappers({
     parse: (str) => camelCase(str, opt),
     format: (str) => snakeCase(str, opt),
-    mapNestedKeys: opt.mapNestedKeys,
   });
 }
 
-function knexIdentifierMapping(colToProp, { mapNestedKeys } = {}) {
+function knexIdentifierMapping(colToProp) {
   const propToCol = Object.keys(colToProp).reduce((propToCol, column) => {
     propToCol[colToProp[column]] = column;
     return propToCol;
@@ -239,7 +231,6 @@ function knexIdentifierMapping(colToProp, { mapNestedKeys } = {}) {
   return knexIdentifierMappers({
     parse: (column) => colToProp[column] || column,
     format: (prop) => propToCol[prop] || prop,
-    mapNestedKeys,
   });
 }
 

package/lib/utils/objectUtils.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const { clone, cloneDeep } = require('./clone');
 const SMALL_ARRAY_SIZE = 10;
 
 function isEmpty(item) {
@@ -389,6 +388,115 @@ function mergeMaps(map1, map2) {
   return map;
 }
 
+// Returns the tag of `value` like `Object.prototype.toString()`, but ignores a
+// `Symbol.toStringTag` that can be overwritten, like lodash's `baseGetTag()`.
+// Objects that define a writable tag, like decimal.js numbers, are plain
+// objects then, while built-ins like `URL` keep their own tag.
+function getRawTag(value) {
+  const tag = Object.prototype.toString.call(value);
+
+  if (!(Symbol.toStringTag in value)) {
+    return tag;
+  }
+
+  const isOwn = Object.prototype.hasOwnProperty.call(value, Symbol.toStringTag);
+  const ownTag = value[Symbol.toStringTag];
+
+  try {
+    value[Symbol.toStringTag] = undefined;
+  } catch (err) {
+    return tag;
+  }
+
+  const rawTag = Object.prototype.toString.call(value);
+
+  if (isOwn) {
+    value[Symbol.toStringTag] = ownTag;
+  } else {
+    delete value[Symbol.toStringTag];
+  }
+
+  return rawTag;
+}
+
+// Clones `value` deeply, like lodash's `cloneDeep()`: plain objects and class
+// instances keep their prototype and get copies of their own enumerable
+// properties, symbols included. Arrays, dates, regular expressions, maps,
+// sets, buffers, typed arrays and array buffers are copied, circular
+// references are kept. Anything else, like functions, errors and built-ins
+// with internal state like `URL`, is kept as it is.
+function cloneDeep(value, clones = new Map()) {
+  if (!isObject(value)) {
+    return value;
+  } else if (clones.has(value)) {
+    return clones.get(value);
+  }
+
+  const Ctor = value.constructor;
+  const tag = getRawTag(value);
+  let copy;
+
+  if (tag === '[object Object]') {
+    copy = Object.create(Object.getPrototypeOf(value));
+    clones.set(value, copy);
+
+    for (const key of Object.keys(value)) {
+      const item = cloneDeep(value[key], clones);
+
+      if (key === '__proto__') {
+        // Assigning it would set the prototype instead of the property.
+        Object.defineProperty(copy, key, {
+          value: item,
+          writable: true,
+          enumerable: true,
+          configurable: true,
+        });
+      } else {
+        copy[key] = item;
+      }
+    }
+
+    for (const symbol of Object.getOwnPropertySymbols(value)) {
+      if (Object.prototype.propertyIsEnumerable.call(value, symbol)) {
+        copy[symbol] = cloneDeep(value[symbol], clones);
+      }
+    }
+  } else if (tag === '[object Array]') {
+    copy = new Ctor(value.length);
+    clones.set(value, copy);
+
+    for (let i = 0, l = value.length; i < l; ++i) {
+      copy[i] = cloneDeep(value[i], clones);
+    }
+  } else if (Buffer.isBuffer(value)) {
+    copy = Buffer.from(value);
+  } else if (tag === '[object DataView]') {
+    copy = new Ctor(value.buffer.slice(0), value.byteOffset, value.byteLength);
+  } else if (ArrayBuffer.isView(value)) {
+    copy = value.slice();
+  } else if (tag === '[object ArrayBuffer]') {
+    copy = value.slice(0);
+  } else if (tag === '[object Date]') {
+    copy = new Ctor(+value);
+  } else if (tag === '[object RegExp]') {
+    copy = new Ctor(value.source, value.flags);
+    copy.lastIndex = value.lastIndex;
+  } else if (tag === '[object Map]') {
+    copy = new Ctor();
+    clones.set(value, copy);
+    value.forEach((item, key) => copy.set(key, cloneDeep(item, clones)));
+  } else if (tag === '[object Set]') {
+    copy = new Ctor();
+    clones.set(value, copy);
+    value.forEach((item) => copy.add(cloneDeep(item, clones)));
+  } else {
+    return value;
+  }
+
+  clones.set(value, copy);
+  return copy;
+}
+
 module.exports = {
   isEmpty,
   isString,
@@ -411,7 +519,6 @@ module.exports = {
   values,
   union,
   chunk,
-  clone,
   omit,
   once,
   last,

package/lib/utils/parseFieldExpression.js

@@ -66,10 +66,17 @@ function toPostgresJsonPath(access) {
 // `whereIn(ref(...), values)`), so the keys need to be escaped instead: single
 // quotes are doubled, and `?` is escaped so that knex doesn't read it as a
 // binding placeholder. Backslashes are always doubled by `toPostgresJsonPath()`,
-// so they can't escape the closing quote either.
+// so they can't escape the closing quote either. But knex's Postgres dialect
+// drops all backslashes in front of an escaped `?`, so if the path contains any,
+// an escape string literal is used in which backslashes are written as `\134`.
 function toPostgresJsonPathLiteral(access) {
-  const path = toPostgresJsonPath(access).replace(/'/g, "''").replace(/\?/g, '\\?');
-  return `'${path}'`;
+  const path = toPostgresJsonPath(access).replace(/'/g, "''");
+
+  if (path.includes('\\?')) {
+    return `E'${path.replace(/\\/g, '\\134').replace(/\?/g, '\\?')}'`;
+  } else {
+    return `'${path.replace(/\?/g, '\\?')}'`;
+  }
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "objection",
-  "version": "3.5.0",
+  "version": "3.7.0",
   "description": "An SQL-friendly ORM for Node.js",
   "main": "lib/objection.js",
   "license": "MIT",
@@ -8,8 +8,11 @@
     "test": "npm run eslint && mocha --slow 10 --timeout 15000 --reporter spec --recursive tests --exclude \"tests/unit/relations/files/**\" && npm run test:typings",
     "test:fast": "mocha --slow 10 --timeout 15000 --reporter spec --recursive tests --bail --exclude \"tests/unit/relations/files/**\"",
     "test:typings": "tsc && prettier --check \"typings/**/*.ts\" \"tests/ts/**/*.ts\"",
+    "db:up": "docker compose up -d --wait",
+    "db:down": "docker compose down -v",
+    "build:parsers": "npx --yes peggy@5.1.0 --format commonjs -o lib/queryBuilder/parsers/jsonFieldExpressionParser.js lib/queryBuilder/parsers/jsonFieldExpressionParser.pegjs && npx --yes peggy@5.1.0 --format commonjs -o lib/queryBuilder/parsers/relationExpressionParser.js lib/queryBuilder/parsers/relationExpressionParser.pegjs && prettier --write \"lib/queryBuilder/parsers/*Parser.js\"",
     "prettier": "prettier --write \"{examples,lib,tests,testUtils,typings,doc}/**/*.{js,ts}\"",
-    "eslint": "eslint --format codeframe \"examples/**/*.js\" \"lib/**/*.js\" \"tests/**/*.js\"",
+    "eslint": "eslint --format codeframe \"examples/**/*.js\" \"lib/**/*.js\" \"tests/**/*.js\" \"testUtils/**/*.js\" \"*.js\"",
     "docs:dev": "vitepress dev doc",
     "docs:build": "vitepress build doc",
     "docs:preview": "vitepress preview doc"

package/typings/objection/index.d.ts

@@ -374,7 +374,8 @@ declare namespace Objection {
 
   /**
    * Converts the query builder NQB to the same result kind as QB:
-   * array, single, maybe-single or page.
+   * array, single, maybe-single or page. Any other result type, e.g. after
+   * `castTo()` or `patch()`, falls back to QB, without narrowing.
    */
   type WithResultKind<QB extends AnyQueryBuilder, NQB extends AnyQueryBuilder> = [
     ResultType<QB>,
@@ -423,7 +424,9 @@ declare namespace Objection {
         ? I extends Model
           ? I[] | PartialModelObject<I>[]
           : Expression<T[K]>
-        : Expression<T[K]>;
+        : NonNullable<T[K]> extends Model
+          ? Expression<T[K]> | PartialModelObject<NonNullable<T[K]>> // nullable to-one relations
+          : Expression<T[K]>;
   };
 
   /**
@@ -1000,14 +1003,36 @@ declare namespace Objection {
     includeTriggerModifications?: boolean;
   }
 
+  type ReturningColumn = string | Raw | readonly (string | Raw)[];
+
+  /**
+   * The arguments of `returning()`: any number of columns or column arrays,
+   * optionally followed by an options object.
+   */
+  type ReturningArgs =
+    ReturningColumn[] | [ReturningColumn, ...ReturningColumn[], ReturningOptions | undefined];
+
   interface ReturningMethod {
     <QB extends AnyQueryBuilder>(
       this: QB,
-      column: string | Raw | (string | Raw)[] | readonly (string | Raw)[],
-      options?: ReturningOptions,
+      ...args: ReturningArgs
     ): QB extends NumberQueryBuilder<QB> ? ArrayQueryBuilder<QB> : QB;
   }
 
+  /**
+   * With `returning()`, `relate()` returns the inserted join rows for
+   * many-to-many relations, the updated related rows for has-many relations
+   * and the updated owner rows for belongs-to-one relations. The typings can't
+   * tell these apart, so the rows are typed as `any`.
+   */
+  interface RelateReturningMethod<M extends Model> {
+    (...args: ReturningArgs): QueryBuilder<M, any[]>;
+  }
+
+  type RelateQueryBuilder<QB extends AnyQueryBuilder> = {
+    returning: RelateReturningMethod<ModelType<QB>>;
+  } & NumberQueryBuilder<QB>;
+
   interface TimeoutOptions {
     cancel: boolean;
   }
@@ -1058,8 +1083,9 @@ declare namespace Objection {
   }
 
   export interface InsertGraphOptions {
-    relate?: boolean | string[];
+    relate?: boolean | string | string[];
     allowRefs?: boolean;
+    allowMarkers?: boolean;
   }
 
   interface InsertGraphMethod<M extends Model> {
@@ -1077,16 +1103,17 @@ declare namespace Objection {
   }
 
   export interface UpsertGraphOptions {
-    relate?: boolean | string[];
-    unrelate?: boolean | string[];
-    insertMissing?: boolean | string[];
-    update?: boolean | string[];
-    noInsert?: boolean | string[];
-    noUpdate?: boolean | string[];
-    noDelete?: boolean | string[];
-    noRelate?: boolean | string[];
-    noUnrelate?: boolean | string[];
+    relate?: boolean | string | string[];
+    unrelate?: boolean | string | string[];
+    insertMissing?: boolean | string | string[];
+    update?: boolean | string | string[];
+    noInsert?: boolean | string | string[];
+    noUpdate?: boolean | string | string[];
+    noDelete?: boolean | string | string[];
+    noRelate?: boolean | string | string[];
+    noUnrelate?: boolean | string | string[];
     allowRefs?: boolean;
+    allowMarkers?: boolean;
   }
 
   interface UpsertGraphMethod<M extends Model> {
@@ -1386,7 +1413,7 @@ declare namespace Objection {
 
     relate(
       ids: MaybeCompositeId | MaybeCompositeId[] | PartialModelObject<M> | PartialModelObject<M>[],
-    ): NumberQueryBuilder<this>;
+    ): RelateQueryBuilder<this>;
 
     unrelate(): NumberQueryBuilder<this>;
     for(ids: ForIdValue | ForIdValue[]): this;
@@ -1718,34 +1745,25 @@ declare namespace Objection {
     constructor(config: AjvConfig);
   }
 
-  export interface SnakeCaseMappersOptions {
+  /**
+   * The options shared by `snakeCaseMappers` and `knexSnakeCaseMappers`.
+   */
+  export interface SnakeCaseOptions {
     upperCase?: boolean;
     underscoreBeforeDigits?: boolean;
     underscoreBetweenUppercaseLetters?: boolean;
     noDoubleUnderscores?: boolean;
+  }
+
+  export interface SnakeCaseMappersOptions extends SnakeCaseOptions {
     /**
      * Only convert the column part of field expressions like
      * `jsonColumn:someKey` and keep their JSON keys as written.
-     * Only used by `snakeCaseMappers`.
      */
     preserveJsonKeys?: boolean;
-    /**
-     * Also convert the keys of plain objects one level down in the results,
-     * e.g. the rows of knex's `nestTables: true` option on MySQL. Also applies
-     * to JSON columns parsed by the driver.
-     * Only used by `knexSnakeCaseMappers`.
-     */
-    mapNestedKeys?: boolean;
   }
 
-  export interface KnexIdentifierMappingOptions {
-    /**
-     * Also convert the keys of plain objects one level down in the results,
-     * e.g. the rows of knex's `nestTables: true` option on MySQL. Also applies
-     * to JSON columns parsed by the driver.
-     */
-    mapNestedKeys?: boolean;
-  }
+  export interface KnexSnakeCaseMappersOptions extends SnakeCaseOptions {}
 
   export interface ColumnNameMappers {
     parse(json: Pojo): Pojo;
@@ -1762,11 +1780,11 @@ declare namespace Objection {
   }
 
   export interface KnexSnakeCaseMappersFactory {
-    (options?: SnakeCaseMappersOptions): KnexMappers;
+    (options?: KnexSnakeCaseMappersOptions): KnexMappers;
   }
 
   export interface KnexIdentifierMappingFactory {
-    (colToProp: Record<string, string>, options?: KnexIdentifierMappingOptions): KnexMappers;
+    (colToProp: Record<string, string>): KnexMappers;
   }
 
   export type ValidationErrorType =
@@ -2040,14 +2058,16 @@ declare namespace Objection {
     static bindKnex<M>(this: M, trxOrKnex: TransactionOrKnex): M;
     static bindTransaction<M>(this: M, trxOrKnex: TransactionOrKnex): M;
 
-    static fetchGraph<M extends Model, const E extends RelationExpression<M>>(
+    // E defaults to `string` so that `fetchGraph<Person>(...)` still compiles,
+    // without narrowing, as TypeScript doesn't infer the remaining arguments.
+    static fetchGraph<M extends Model, const E extends RelationExpression<M> = string>(
       this: ConstructorType<M>,
       modelOrObject: PartialModelObject<M>,
       expression: E,
       options?: FetchGraphOptions,
     ): SingleQueryBuilder<WithGraphModelQueryBuilder<M, E>>;
 
-    static fetchGraph<M extends Model, const E extends RelationExpression<M>>(
+    static fetchGraph<M extends Model, const E extends RelationExpression<M> = string>(
       this: ConstructorType<M>,
       modelOrObject: PartialModelObject<M>[],
       expression: E,

package/lib/relations/manyToMany/ManyToManyPostgresModifyMixin.js

@@ -1,27 +0,0 @@
-'use strict';
-
-const { ManyToManyModifyMixin } = require('./ManyToManyModifyMixin');
-
-// Like `ManyToManySqliteModifyMixin`, this identifies the join rows to modify
-// by their physical row id, so that only the join rows that match the filters
-// are modified (#1853). `tableoid` makes `ctid` unique for partitioned tables.
-const ManyToManyPostgresModifyMixin = (Operation) => {
-  return class extends ManyToManyModifyMixin(Operation) {
-    applyModifyFilterForJoinTable(builder) {
-      const joinTableOwnerRefs = this.relation.joinTableOwnerProp.refs(builder);
-      const tableRef = builder.tableRefFor(this.relation.getJoinModelClass(builder));
-      const rowIdRefs = [`${tableRef}.tableoid`, `${tableRef}.ctid`];
-
-      const ownerValues = this.owner.getProps(this.relation);
-      const subquery = this.modifyFilterSubquery.clone().select(rowIdRefs);
-
-      return builder
-        .whereInComposite(rowIdRefs, subquery)
-        .whereInComposite(joinTableOwnerRefs, ownerValues);
-    }
-  };
-};
-
-module.exports = {
-  ManyToManyPostgresModifyMixin,
-};

package/lib/relations/manyToMany/unrelate/ManyToManyUnrelatePostgresOperation.js

@@ -1,16 +0,0 @@
-'use strict';
-
-const { ManyToManyUnrelateOperationBase } = require('./ManyToManyUnrelateOperationBase');
-const { ManyToManyPostgresModifyMixin } = require('../ManyToManyPostgresModifyMixin');
-
-class ManyToManyUnrelatePostgresOperation extends ManyToManyPostgresModifyMixin(
-  ManyToManyUnrelateOperationBase,
-) {
-  get modifyMainQuery() {
-    return false;
-  }
-}
-
-module.exports = {
-  ManyToManyUnrelatePostgresOperation,
-};

package/lib/relations/manyToMany/update/ManyToManyUpdatePostgresOperation.js

@@ -1,12 +0,0 @@
-'use strict';
-
-const { ManyToManyUpdateOperationBase } = require('./ManyToManyUpdateOperationBase');
-const { ManyToManyPostgresModifyMixin } = require('../ManyToManyPostgresModifyMixin');
-
-class ManyToManyUpdatePostgresOperation extends ManyToManyPostgresModifyMixin(
-  ManyToManyUpdateOperationBase,
-) {}
-
-module.exports = {
-  ManyToManyUpdatePostgresOperation,
-};