objection 3.2.0 → 3.3.0

package/lib/queryBuilder/parsers/jsonFieldExpressionParser.js

@@ -166,10 +166,10 @@ function peg$parse(input, options) {
     peg$c4 = peg$literalExpectation('[', false),
     peg$c5 = '"',
     peg$c6 = peg$literalExpectation('"', false),
-    peg$c7 = "'",
-    peg$c8 = peg$literalExpectation("'", false),
-    peg$c9 = ']',
-    peg$c10 = peg$literalExpectation(']', false),
+    peg$c7 = ']',
+    peg$c8 = peg$literalExpectation(']', false),
+    peg$c9 = "'",
+    peg$c10 = peg$literalExpectation("'", false),
     peg$c11 = function (key) {
       return { type: 'object', ref: Array.isArray(key) ? key[1] : key };
     },
@@ -415,7 +415,7 @@ function peg$parse(input, options) {
   }
 
   function peg$parsebracketStringRef() {
-    var s0, s1, s2, s3, s4, s5;
+    var s0, s1, s2, s3, s4, s5, s6, s7;
 
     s0 = peg$currPos;
     if (input.charCodeAt(peg$currPos) === 91) {
@@ -451,8 +451,31 @@ function peg$parse(input, options) {
             }
           }
           if (s5 !== peg$FAILED) {
-            s3 = [s3, s4, s5];
-            s2 = s3;
+            s6 = peg$currPos;
+            peg$silentFails++;
+            if (input.charCodeAt(peg$currPos) === 93) {
+              s7 = peg$c7;
+              peg$currPos++;
+            } else {
+              s7 = peg$FAILED;
+              if (peg$silentFails === 0) {
+                peg$fail(peg$c8);
+              }
+            }
+            peg$silentFails--;
+            if (s7 !== peg$FAILED) {
+              peg$currPos = s6;
+              s6 = void 0;
+            } else {
+              s6 = peg$FAILED;
+            }
+            if (s6 !== peg$FAILED) {
+              s3 = [s3, s4, s5, s6];
+              s2 = s3;
+            } else {
+              peg$currPos = s2;
+              s2 = peg$FAILED;
+            }
           } else {
             peg$currPos = s2;
             s2 = peg$FAILED;
@@ -468,29 +491,52 @@ function peg$parse(input, options) {
       if (s2 === peg$FAILED) {
         s2 = peg$currPos;
         if (input.charCodeAt(peg$currPos) === 39) {
-          s3 = peg$c7;
+          s3 = peg$c9;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c8);
+            peg$fail(peg$c10);
           }
         }
         if (s3 !== peg$FAILED) {
           s4 = peg$parsestringWithoutSingleQuotes();
           if (s4 !== peg$FAILED) {
             if (input.charCodeAt(peg$currPos) === 39) {
-              s5 = peg$c7;
+              s5 = peg$c9;
               peg$currPos++;
             } else {
               s5 = peg$FAILED;
               if (peg$silentFails === 0) {
-                peg$fail(peg$c8);
+                peg$fail(peg$c10);
               }
             }
             if (s5 !== peg$FAILED) {
-              s3 = [s3, s4, s5];
-              s2 = s3;
+              s6 = peg$currPos;
+              peg$silentFails++;
+              if (input.charCodeAt(peg$currPos) === 93) {
+                s7 = peg$c7;
+                peg$currPos++;
+              } else {
+                s7 = peg$FAILED;
+                if (peg$silentFails === 0) {
+                  peg$fail(peg$c8);
+                }
+              }
+              peg$silentFails--;
+              if (s7 !== peg$FAILED) {
+                peg$currPos = s6;
+                s6 = void 0;
+              } else {
+                s6 = peg$FAILED;
+              }
+              if (s6 !== peg$FAILED) {
+                s3 = [s3, s4, s5, s6];
+                s2 = s3;
+              } else {
+                peg$currPos = s2;
+                s2 = peg$FAILED;
+              }
             } else {
               peg$currPos = s2;
               s2 = peg$FAILED;
@@ -509,12 +555,12 @@ function peg$parse(input, options) {
       }
       if (s2 !== peg$FAILED) {
         if (input.charCodeAt(peg$currPos) === 93) {
-          s3 = peg$c9;
+          s3 = peg$c7;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c10);
+            peg$fail(peg$c8);
           }
         }
         if (s3 !== peg$FAILED) {
@@ -554,12 +600,12 @@ function peg$parse(input, options) {
       s2 = peg$parseinteger();
       if (s2 !== peg$FAILED) {
         if (input.charCodeAt(peg$currPos) === 93) {
-          s3 = peg$c9;
+          s3 = peg$c7;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c10);
+            peg$fail(peg$c8);
           }
         }
         if (s3 !== peg$FAILED) {
@@ -719,21 +765,17 @@ function peg$parse(input, options) {
         peg$fail(peg$c22);
       }
     }
-    if (s2 !== peg$FAILED) {
-      while (s2 !== peg$FAILED) {
-        s1.push(s2);
-        if (peg$c21.test(input.charAt(peg$currPos))) {
-          s2 = input.charAt(peg$currPos);
-          peg$currPos++;
-        } else {
-          s2 = peg$FAILED;
-          if (peg$silentFails === 0) {
-            peg$fail(peg$c22);
-          }
+    while (s2 !== peg$FAILED) {
+      s1.push(s2);
+      if (peg$c21.test(input.charAt(peg$currPos))) {
+        s2 = input.charAt(peg$currPos);
+        peg$currPos++;
+      } else {
+        s2 = peg$FAILED;
+        if (peg$silentFails === 0) {
+          peg$fail(peg$c22);
         }
       }
-    } else {
-      s1 = peg$FAILED;
     }
     if (s1 !== peg$FAILED) {
       peg$savedPos = s0;
@@ -758,21 +800,17 @@ function peg$parse(input, options) {
         peg$fail(peg$c24);
       }
     }
-    if (s2 !== peg$FAILED) {
-      while (s2 !== peg$FAILED) {
-        s1.push(s2);
-        if (peg$c23.test(input.charAt(peg$currPos))) {
-          s2 = input.charAt(peg$currPos);
-          peg$currPos++;
-        } else {
-          s2 = peg$FAILED;
-          if (peg$silentFails === 0) {
-            peg$fail(peg$c24);
-          }
+    while (s2 !== peg$FAILED) {
+      s1.push(s2);
+      if (peg$c23.test(input.charAt(peg$currPos))) {
+        s2 = input.charAt(peg$currPos);
+        peg$currPos++;
+      } else {
+        s2 = peg$FAILED;
+        if (peg$silentFails === 0) {
+          peg$fail(peg$c24);
         }
       }
-    } else {
-      s1 = peg$FAILED;
     }
     if (s1 !== peg$FAILED) {
       peg$savedPos = s0;

package/lib/queryBuilder/parsers/jsonFieldExpressionParser.pegjs

@@ -31,6 +31,7 @@
  * 4. Keys containing square brackets and quotes
  *    `objectColumn:['Double."Quote".[]']` and `objectColumn:["Sinlge.'Quote'.[]"]`
  *    Column `{ "Double.\"Quote\".[]" : "I was referred",  "Sinlge.'Quote'.[]" : "Mee too!" }`
+ * 5. Empty keys `objectColumn:[""]` or `objectColumn:['']` Column `{ "" : "I was referred" }`
  * 99. Keys containing dots, square brackets, single quotes and double quotes in one json key is
  *     not currently supported
  *
@@ -60,8 +61,8 @@ start =
 bracketStringRef =
   '['
       key:(
-       '"' stringWithoutDoubleQuotes '"' /
-       "'" stringWithoutSingleQuotes "'" /
+       '"' stringWithoutDoubleQuotes '"' &']' /
+       "'" stringWithoutSingleQuotes "'" &']' /
        stringWithoutSquareBrackets
       )
   ']'
@@ -86,10 +87,10 @@ stringWithoutColon =
   chars:([^:])+ { return chars.join(""); }
 
 stringWithoutDoubleQuotes =
-  chars:([^"])+ { return chars.join(""); }
+  chars:([^"])* { return chars.join(""); }
 
 stringWithoutSingleQuotes =
-  chars:([^'])+ { return chars.join(""); }
+  chars:([^'])* { return chars.join(""); }
 
 stringWithoutSquareBracketsOrDots =
   chars:([^.\x5D\x5B])+ { return chars.join(""); }

package/lib/queryBuilder/whereGroups.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const { WhereGroupOperation } = require('./operations/WhereGroupOperation');
+const { isObject, isFunction } = require('../utils/objectUtils');
+const { isKnexRaw } = require('../utils/knexUtils');
+
+// Where operations of a query can come from different sources: the user, a
+// relation's owner condition, the relation's `modify`/`filter` and so on. Each
+// source forms its own "where group". All operations the user adds belong to
+// the default group, and `addWhereGroup` assigns the operations added inside
+// its callback to a new, separate group.
+//
+// Knex simply joins all where clauses with their `and` or `or` boolean. Since
+// `and` binds tighter than `or`, a user's `orWhere` would escape the relation's
+// owner condition: `where "ownerId" in (1) and a or b`. This would find, update
+// and delete rows of other owners too. See issues #2191 and #1909.
+//
+// To prevent this, `wrapWhereGroups` is called after all `onBuild` hooks have
+// run. If a query contains more than one where group, every group that may
+// contain an `or` at its top level is wrapped in parentheses, so that the
+// groups are always ANDed together: `where "ownerId" in (1) and (a or b)`.
+// Groups without an `or` are left alone so that their SQL doesn't change.
+function addWhereGroup(builder, callback) {
+  const parentWhereGroup = builder._whereGroup;
+  builder._whereGroup = {};
+
+  try {
+    return callback(builder);
+  } finally {
+    builder._whereGroup = parentWhereGroup;
+  }
+}
+
+function wrapWhereGroups(builder) {
+  const WhereSelector = builder.constructor.WhereSelector;
+  const whereOps = new Set();
+  const groups = new Map();
+
+  builder.forEachOperation(WhereSelector, (op) => {
+    // Only consider the top-most where operations. The descendants of a
+    // where operation are built as part of it.
+    if (op.isAncestorInSet(whereOps)) {
+      return;
+    }
+
+    whereOps.add(op);
+
+    const group = getWhereGroup(op);
+    let ops = groups.get(group);
+
+    if (!ops) {
+      ops = [];
+      groups.set(group, ops);
+    }
+
+    ops.push(op);
+  });
+
+  if (groups.size < 2) {
+    return builder;
+  }
+
+  for (const [group, ops] of groups) {
+    if (ops.some(mayContainTopLevelOr)) {
+      wrapOperations(builder, group, ops);
+    }
+  }
+
+  return builder;
+}
+
+function getWhereGroup(op) {
+  for (let it = op; it; it = it.parentOperation) {
+    if (it.whereGroup) {
+      return it.whereGroup;
+    }
+  }
+
+  return null;
+}
+
+function mayContainTopLevelOr(op) {
+  if (op instanceof WhereGroupOperation) {
+    return false;
+  }
+
+  // orWhere, orWhereIn, orWhereRaw, orWhereExists, orWhereJsonSupersetOf etc.
+  if (/^or[A-Z]/.test(op.name)) {
+    return true;
+  }
+
+  // Raw SQL isn't parenthesized by knex and can contain `or`.
+  if (/Raw$/.test(op.name)) {
+    return true;
+  }
+
+  return Array.isArray(op.args) && op.args.length === 1 && isRaw(op.args[0]);
+}
+
+function isRaw(arg) {
+  return isKnexRaw(arg) || (isObject(arg) && isFunction(arg.toKnexRaw));
+}
+
+function wrapOperations(builder, group, ops) {
+  const groupOp = new WhereGroupOperation('where');
+  groupOp.whereGroup = group;
+
+  builder.replaceOperation(ops[0], groupOp);
+
+  for (let i = 1; i < ops.length; ++i) {
+    builder.removeOperation(ops[i]);
+  }
+
+  for (const op of ops) {
+    op.parentOperation = null;
+    op.adderHookName = null;
+  }
+
+  groupOp.operations = ops;
+}
+
+module.exports = {
+  addWhereGroup,
+  wrapWhereGroups,
+};

package/lib/relations/Relation.js

@@ -10,6 +10,7 @@ const { resolveModel } = require('../utils/resolveModel');
 const { get, isFunction } = require('../utils/objectUtils');
 const { mapAfterAllReturn } = require('../utils/promiseUtils');
 const { createModifier } = require('../utils/createModifier');
+const { addWhereGroup } = require('../queryBuilder/whereGroups');
 
 class Relation {
   constructor(relationName, OwnerClass) {
@@ -115,14 +116,16 @@ class Relation {
 
   findQuery(builder, owner) {
     const relatedRefs = this.relatedProp.refs(builder);
-    owner.buildFindQuery(builder, this, relatedRefs);
+    addWhereGroup(builder, () => owner.buildFindQuery(builder, this, relatedRefs));
 
     return this.applyModify(builder);
   }
 
   applyModify(builder) {
     try {
-      return builder.modify(this.modify);
+      // Keep the relation's `modify` in a separate where group so that an
+      // `orWhere` in it can't escape the owner condition. See `whereGroups.js`.
+      return addWhereGroup(builder, () => builder.modify(this.modify));
     } catch (err) {
       if (err instanceof this.relatedModelClass.ModifierNotFoundError) {
         throw this.createError(err.message);

package/lib/relations/hasMany/HasManyRelateOperation.js

@@ -2,6 +2,7 @@
 
 const { normalizeIds } = require('../../utils/normalizeIds');
 const { RelateOperation } = require('../../queryBuilder/operations/RelateOperation');
+const { addWhereGroup } = require('../../queryBuilder/whereGroups');
 
 class HasManyRelateOperation extends RelateOperation {
   onAdd(_, args) {
@@ -31,8 +32,10 @@ class HasManyRelateOperation extends RelateOperation {
       .copyFrom(builder, /returning/)
       .copyFrom(builder, builder.constructor.JoinSelector)
       .copyFrom(builder, builder.constructor.WhereSelector)
-      .findByIds(this.ids)
-      .modify(this.relation.modify);
+      .modify((query) => {
+        addWhereGroup(query, () => query.findByIds(this.ids));
+        addWhereGroup(query, () => query.modify(this.relation.modify));
+      });
   }
 }
 

package/lib/relations/hasMany/HasManyUnrelateOperation.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { UnrelateOperation } = require('../../queryBuilder/operations/UnrelateOperation');
+const { addWhereGroup } = require('../../queryBuilder/whereGroups');
 
 class HasManyUnrelateOperation extends UnrelateOperation {
   queryExecutor(builder) {
@@ -19,8 +20,10 @@ class HasManyUnrelateOperation extends UnrelateOperation {
       .patch(patch)
       .copyFrom(builder, builder.constructor.JoinSelector)
       .copyFrom(builder, builder.constructor.WhereSelector)
-      .whereInComposite(relatedRefs, ownerValues)
-      .modify(this.relation.modify);
+      .modify((query) => {
+        addWhereGroup(query, () => query.whereInComposite(relatedRefs, ownerValues));
+        addWhereGroup(query, () => query.modify(this.relation.modify));
+      });
   }
 }
 

package/lib/relations/manyToMany/ManyToManyModifyMixin.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { ManyToManyFindOperation } = require('./find/ManyToManyFindOperation');
+const { addWhereGroup } = require('../../queryBuilder/whereGroups');
 
 // This mixin contains the shared code for all modify operations (update, delete, relate, unrelate)
 // for ManyToManyRelation operations.
@@ -31,7 +32,7 @@ const ManyToManyModifyMixin = (Operation) => {
         this.removeFiltersFromMainQuery(builder);
 
         // Add a single where clause that uses the created subquery.
-        this.applyModifyFilterForRelatedTable(builder);
+        addWhereGroup(builder, () => this.applyModifyFilterForRelatedTable(builder));
       }
 
       return super.onBuild(builder);
@@ -70,6 +71,10 @@ const ManyToManyModifyMixin = (Operation) => {
       return builder.whereInComposite(idRefs, subquery);
     }
 
+    // Modifies the join rows of all related rows that match the filters. If the filters
+    // reference join table columns, this also modifies join rows that don't match them
+    // but share a related row with one that does (#1853). The dialect specific mixins
+    // override this to only modify the join rows that match the filters themselves.
     applyModifyFilterForJoinTable(builder) {
       const joinTableOwnerRefs = this.relation.joinTableOwnerProp.refs(builder);
       const joinTableRelatedRefs = this.relation.joinTableRelatedProp.refs(builder);
@@ -85,10 +90,19 @@ const ManyToManyModifyMixin = (Operation) => {
     }
 
     toFindOperation() {
-      return new ManyToManyFindOperation('find', {
+      const opt = {
         relation: this.relation,
         owner: this.owner,
-      });
+      };
+
+      if (this.modifyMainQuery && this.modifyFilterSubquery) {
+        // `onBuild` has already moved the main query's filters into the
+        // subquery and removed them. Apply them to the find query too.
+        // See issue #2266.
+        return new ManyToManyModifyFindOperation('find', { ...opt, modifyOperation: this });
+      } else {
+        return new ManyToManyFindOperation('find', opt);
+      }
     }
 
     clone() {
@@ -99,6 +113,15 @@ const ManyToManyModifyMixin = (Operation) => {
   };
 };
 
+class ManyToManyModifyFindOperation extends ManyToManyFindOperation {
+  onBuild(builder) {
+    super.onBuild(builder);
+    addWhereGroup(builder, () =>
+      this.opt.modifyOperation.applyModifyFilterForRelatedTable(builder),
+    );
+  }
+}
+
 module.exports = {
   ManyToManyModifyMixin,
 };

package/lib/relations/manyToMany/ManyToManyMySqlModifyMixin.js

@@ -0,0 +1,66 @@
+'use strict';
+
+const { ManyToManyModifyMixin } = require('./ManyToManyModifyMixin');
+const { isObject } = require('../../utils/objectUtils');
+
+// We need to override this mixin for mysql because mysql doesn't allow a
+// statement that modifies the join table to read the related table in a
+// subquery, if a trigger on the join table modifies the related table:
+//
+//   ER_CANT_UPDATE_USED_TABLE_IN_SF_OR_TRG
+//
+// So the join rows that match the filters are fetched in a separate query
+// before the main query, which then only modifies these rows (#1853). Mysql
+// has no row ids and the join table's primary key isn't known, so the rows are
+// identified by the values of all their columns.
+//
+// https://stackoverflow.com/a/2314264/3729316
+const ManyToManyMySqlModifyMixin = (Operation) => {
+  return class extends ManyToManyModifyMixin(Operation) {
+    applyModifyFilterForJoinTable(builder) {
+      const joinTableOwnerRefs = this.relation.joinTableOwnerProp.refs(builder);
+      const tableRef = builder.tableRefFor(this.relation.getJoinModelClass(builder));
+
+      const ownerValues = this.owner.getProps(this.relation);
+      const subquery = this.modifyFilterSubquery.clone().select(`${tableRef}.*`);
+
+      return builder
+        .runBefore(async (_, builder) => {
+          // Fetch the raw rows: no models, so that no hooks run for them.
+          const rows = await subquery.toKnexQuery();
+
+          if (rows.length) {
+            builder.where((builder) => {
+              for (const row of rows) {
+                builder.orWhere((builder) => {
+                  for (const [column, value] of Object.entries(row)) {
+                    builder.whereRaw('?? <=> ?', [
+                      `${tableRef}.${column}`,
+                      toComparableValue(value),
+                    ]);
+                  }
+                });
+              }
+            });
+          } else {
+            // Still execute the query so that it returns the expected
+            // result for the dialect, e.g. a count of 0 for deletes.
+            builder.where(false);
+          }
+        })
+        .whereInComposite(joinTableOwnerRefs, ownerValues);
+    }
+  };
+};
+
+// Some drivers return JSON columns parsed. Mysql parses strings that are
+// compared to JSON columns, so the stringified value matches them again.
+function toComparableValue(value) {
+  return isObject(value) && !(value instanceof Date) && !Buffer.isBuffer(value)
+    ? JSON.stringify(value)
+    : value;
+}
+
+module.exports = {
+  ManyToManyMySqlModifyMixin,
+};

package/lib/relations/manyToMany/ManyToManyPostgresModifyMixin.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const { ManyToManyModifyMixin } = require('./ManyToManyModifyMixin');
+
+// Like `ManyToManySqliteModifyMixin`, this identifies the join rows to modify
+// by their physical row id, so that only the join rows that match the filters
+// are modified (#1853). `tableoid` makes `ctid` unique for partitioned tables.
+const ManyToManyPostgresModifyMixin = (Operation) => {
+  return class extends ManyToManyModifyMixin(Operation) {
+    applyModifyFilterForJoinTable(builder) {
+      const joinTableOwnerRefs = this.relation.joinTableOwnerProp.refs(builder);
+      const tableRef = builder.tableRefFor(this.relation.getJoinModelClass(builder));
+      const rowIdRefs = [`${tableRef}.tableoid`, `${tableRef}.ctid`];
+
+      const ownerValues = this.owner.getProps(this.relation);
+      const subquery = this.modifyFilterSubquery.clone().select(rowIdRefs);
+
+      return builder
+        .whereInComposite(rowIdRefs, subquery)
+        .whereInComposite(joinTableOwnerRefs, ownerValues);
+    }
+  };
+};
+
+module.exports = {
+  ManyToManyPostgresModifyMixin,
+};