objection 3.2.0 → 3.4.0

package/lib/queryBuilder/operations/select/Selection.js

@@ -1,14 +1,21 @@
 'use strict';
 
-const { isString, isObject } = require('../../../utils/objectUtils');
+const { isString, isObject, isPlainObject } = require('../../../utils/objectUtils');
+const { isKnexQueryBuilder, isKnexRaw } = require('../../../utils/knexUtils');
 
 const ALIAS_REGEX = /\s+as\s+/i;
 
+// Matches an alias at the end of a raw sql string: `as "alias"`, `as \`alias\``,
+// `as [alias]`, `as ??`, `as :name:` or `as alias`.
+const RAW_ALIAS_REGEX =
+  /\s+as\s+(?:"([^"]+)"|`([^`]+)`|\[([^\]]+)\]|(\?\?)|:(\w+):|([a-z_][\w$]*))\s*$/i;
+
 class Selection {
   constructor(table, column, alias) {
     this.table = table;
     this.column = column;
     this.alias = alias;
+    this.keepsSelectAll = false;
   }
 
   get name() {
@@ -23,6 +30,12 @@ class Selection {
         return createSelectionFromReference(selection);
       } else if (selection.isObjectionRawBuilder) {
         return createSelectionFromRaw(selection);
+      } else if (selection.isObjectionQueryBuilderBase) {
+        return createSelectionFromQueryBuilder(selection);
+      } else if (isKnexQueryBuilder(selection)) {
+        return createSelectionFromAlias(selection._single.as);
+      } else if (isKnexRaw(selection)) {
+        return createSelectionFromAlias(parseRawAlias(selection.sql, selection.bindings));
       } else {
         return null;
       }
@@ -50,10 +63,11 @@ class Selection {
     if (selectionInBuilder.column === '*') {
       if (selectionInBuilder.table) {
         if (selectionToTest.column === '*') {
-          return selectionToTest.table === selectionInBuilder.table;
+          return isSameTable(builder, selectionToTest.table, selectionInBuilder.table);
         } else {
           return (
-            selectionToTest.table === null || selectionToTest.table === selectionInBuilder.table
+            selectionToTest.table === null ||
+            isSameTable(builder, selectionToTest.table, selectionInBuilder.table)
           );
         }
       } else {
@@ -67,7 +81,8 @@ class Selection {
       } else {
         return (
           selectionToTest.column === selectionInBuilder.column &&
-          (selectionToTest.table === null || selectionToTest.table === selectionInBuilderTable)
+          (selectionToTest.table === null ||
+            isSameTable(builder, selectionToTest.table, selectionInBuilderTable))
         );
       }
     }
@@ -82,6 +97,36 @@ Object.defineProperties(Selection.prototype, {
   },
 });
 
+// Table names that only differ in a way that knex's `wrapIdentifier` mapping
+// (for example `knexSnakeCaseMappers()`) cancels out refer to the same table:
+// `dataExtension` and `data_extension` both end up as `data_extension`.
+function isSameTable(builder, table1, table2) {
+  if (table1 === table2) {
+    return true;
+  } else if (!table1 || !table2) {
+    return false;
+  }
+
+  const knex = builder.unsafeKnex();
+  const wrapIdentifier = knex && knex.client && knex.client.config.wrapIdentifier;
+
+  if (!wrapIdentifier) {
+    return false;
+  }
+
+  const mapTable = (table) =>
+    table
+      .split('.')
+      .map((part) => wrapIdentifier(part, identity))
+      .join('.');
+
+  return mapTable(table1) === mapTable(table2);
+}
+
+function identity(value) {
+  return value;
+}
+
 function createSelectionFromReference(ref) {
   return new Selection(ref.tableName, ref.column, ref.alias);
 }
@@ -89,6 +134,52 @@ function createSelectionFromReference(ref) {
 function createSelectionFromRaw(raw) {
   if (raw.alias) {
     return new Selection(null, null, raw.alias);
+  } else {
+    return createSelectionFromAlias(parseRawAlias(raw._sql, raw._args));
+  }
+}
+
+function parseRawAlias(sql, bindings) {
+  const match = isString(sql) && RAW_ALIAS_REGEX.exec(sql);
+
+  if (!match) {
+    return null;
+  }
+
+  const [, doubleQuoted, backQuoted, bracketQuoted, binding, namedBinding, unquoted] = match;
+
+  if (binding) {
+    return Array.isArray(bindings) ? bindings[bindings.length - 1] : null;
+  } else if (namedBinding) {
+    if (Array.isArray(bindings) && bindings.length === 1) {
+      // `raw(sql, { name: value })` of objection.
+      bindings = bindings[0];
+    }
+
+    return isPlainObject(bindings) ? bindings[namedBinding] : null;
+  } else if (unquoted) {
+    // Some databases, like postgres, fold unquoted identifiers to lower case
+    // and others don't, so only lower case ones can be used reliably.
+    return unquoted === unquoted.toLowerCase() ? unquoted : null;
+  } else {
+    return doubleQuoted || backQuoted || bracketQuoted;
+  }
+}
+
+// Subqueries aliased using `as()` select a column with the alias as name.
+function createSelectionFromQueryBuilder(builder) {
+  const asOperation = builder.findLastOperation('as');
+  return createSelectionFromAlias(asOperation && asOperation.args[0]);
+}
+
+function createSelectionFromAlias(alias) {
+  if (isString(alias)) {
+    const selection = new Selection(null, null, alias);
+    // These selections weren't recognized in the past. To not change which
+    // other columns `withGraphJoined` selects, they don't count as explicit
+    // selections when deciding whether all columns are selected.
+    selection.keepsSelectAll = true;
+    return selection;
   } else {
     return null;
   }

package/lib/queryBuilder/operations/whereInComposite/WhereInCompositeMsSqlOperation.js

@@ -26,6 +26,12 @@ class WhereInCompositeMsSqlOperation extends ObjectionToKnexConvertingOperation
   }
 
   buildComposite(knex, knexBuilder, columns, values) {
+    if (Array.isArray(values) && values.length === 0) {
+      // knex turns an empty `whereIn` into `1 = 0` and an empty `whereNotIn`
+      // into `1 = 1`.
+      return this.whereIn(knexBuilder, columns, []);
+    }
+
     const helperColumns = columns.map((_, index) => getTempColumn(index));
 
     if (Array.isArray(values)) {

package/lib/queryBuilder/operations/whereInComposite/WhereInCompositeOperation.js

@@ -48,7 +48,11 @@ function buildCompositeArgs(knex, columns, values) {
 }
 
 function buildCompositeValueArgs(columns, values) {
-  if (!Array.isArray(values[0])) {
+  if (values.length === 0) {
+    // knex turns an empty `whereIn` into `1 = 0` and an empty `whereNotIn`
+    // into `1 = 1`.
+    return [columns, []];
+  } else if (!Array.isArray(values[0])) {
     return [columns, [values]];
   } else {
     return [columns, values];

package/lib/queryBuilder/operations/whereInComposite/WhereInCompositeSqliteOperation.js

@@ -54,6 +54,15 @@ function buildCompositeArgs(columns, values) {
     throw new Error(`sqlite doesn't support multi-column where in clauses`);
   }
 
+  if (values.length === 0) {
+    // knex turns an empty `whereIn` into `1 = 0` and an empty `whereNotIn`
+    // into `1 = 1`.
+    return {
+      method: 'whereIn',
+      args: [columns, []],
+    };
+  }
+
   // Sqlite doesn't support the `where in` syntax for multiple columns but
   // we can emulate it using grouped `or` clauses.
   return {

package/lib/queryBuilder/parsers/jsonFieldExpressionParser.js

@@ -166,10 +166,10 @@ function peg$parse(input, options) {
     peg$c4 = peg$literalExpectation('[', false),
     peg$c5 = '"',
     peg$c6 = peg$literalExpectation('"', false),
-    peg$c7 = "'",
-    peg$c8 = peg$literalExpectation("'", false),
-    peg$c9 = ']',
-    peg$c10 = peg$literalExpectation(']', false),
+    peg$c7 = ']',
+    peg$c8 = peg$literalExpectation(']', false),
+    peg$c9 = "'",
+    peg$c10 = peg$literalExpectation("'", false),
     peg$c11 = function (key) {
       return { type: 'object', ref: Array.isArray(key) ? key[1] : key };
     },
@@ -415,7 +415,7 @@ function peg$parse(input, options) {
   }
 
   function peg$parsebracketStringRef() {
-    var s0, s1, s2, s3, s4, s5;
+    var s0, s1, s2, s3, s4, s5, s6, s7;
 
     s0 = peg$currPos;
     if (input.charCodeAt(peg$currPos) === 91) {
@@ -451,8 +451,31 @@ function peg$parse(input, options) {
             }
           }
           if (s5 !== peg$FAILED) {
-            s3 = [s3, s4, s5];
-            s2 = s3;
+            s6 = peg$currPos;
+            peg$silentFails++;
+            if (input.charCodeAt(peg$currPos) === 93) {
+              s7 = peg$c7;
+              peg$currPos++;
+            } else {
+              s7 = peg$FAILED;
+              if (peg$silentFails === 0) {
+                peg$fail(peg$c8);
+              }
+            }
+            peg$silentFails--;
+            if (s7 !== peg$FAILED) {
+              peg$currPos = s6;
+              s6 = void 0;
+            } else {
+              s6 = peg$FAILED;
+            }
+            if (s6 !== peg$FAILED) {
+              s3 = [s3, s4, s5, s6];
+              s2 = s3;
+            } else {
+              peg$currPos = s2;
+              s2 = peg$FAILED;
+            }
           } else {
             peg$currPos = s2;
             s2 = peg$FAILED;
@@ -468,29 +491,52 @@ function peg$parse(input, options) {
       if (s2 === peg$FAILED) {
         s2 = peg$currPos;
         if (input.charCodeAt(peg$currPos) === 39) {
-          s3 = peg$c7;
+          s3 = peg$c9;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c8);
+            peg$fail(peg$c10);
           }
         }
         if (s3 !== peg$FAILED) {
           s4 = peg$parsestringWithoutSingleQuotes();
           if (s4 !== peg$FAILED) {
             if (input.charCodeAt(peg$currPos) === 39) {
-              s5 = peg$c7;
+              s5 = peg$c9;
               peg$currPos++;
             } else {
               s5 = peg$FAILED;
               if (peg$silentFails === 0) {
-                peg$fail(peg$c8);
+                peg$fail(peg$c10);
               }
             }
             if (s5 !== peg$FAILED) {
-              s3 = [s3, s4, s5];
-              s2 = s3;
+              s6 = peg$currPos;
+              peg$silentFails++;
+              if (input.charCodeAt(peg$currPos) === 93) {
+                s7 = peg$c7;
+                peg$currPos++;
+              } else {
+                s7 = peg$FAILED;
+                if (peg$silentFails === 0) {
+                  peg$fail(peg$c8);
+                }
+              }
+              peg$silentFails--;
+              if (s7 !== peg$FAILED) {
+                peg$currPos = s6;
+                s6 = void 0;
+              } else {
+                s6 = peg$FAILED;
+              }
+              if (s6 !== peg$FAILED) {
+                s3 = [s3, s4, s5, s6];
+                s2 = s3;
+              } else {
+                peg$currPos = s2;
+                s2 = peg$FAILED;
+              }
             } else {
               peg$currPos = s2;
               s2 = peg$FAILED;
@@ -509,12 +555,12 @@ function peg$parse(input, options) {
       }
       if (s2 !== peg$FAILED) {
         if (input.charCodeAt(peg$currPos) === 93) {
-          s3 = peg$c9;
+          s3 = peg$c7;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c10);
+            peg$fail(peg$c8);
           }
         }
         if (s3 !== peg$FAILED) {
@@ -554,12 +600,12 @@ function peg$parse(input, options) {
       s2 = peg$parseinteger();
       if (s2 !== peg$FAILED) {
         if (input.charCodeAt(peg$currPos) === 93) {
-          s3 = peg$c9;
+          s3 = peg$c7;
           peg$currPos++;
         } else {
           s3 = peg$FAILED;
           if (peg$silentFails === 0) {
-            peg$fail(peg$c10);
+            peg$fail(peg$c8);
           }
         }
         if (s3 !== peg$FAILED) {
@@ -719,21 +765,17 @@ function peg$parse(input, options) {
         peg$fail(peg$c22);
       }
     }
-    if (s2 !== peg$FAILED) {
-      while (s2 !== peg$FAILED) {
-        s1.push(s2);
-        if (peg$c21.test(input.charAt(peg$currPos))) {
-          s2 = input.charAt(peg$currPos);
-          peg$currPos++;
-        } else {
-          s2 = peg$FAILED;
-          if (peg$silentFails === 0) {
-            peg$fail(peg$c22);
-          }
+    while (s2 !== peg$FAILED) {
+      s1.push(s2);
+      if (peg$c21.test(input.charAt(peg$currPos))) {
+        s2 = input.charAt(peg$currPos);
+        peg$currPos++;
+      } else {
+        s2 = peg$FAILED;
+        if (peg$silentFails === 0) {
+          peg$fail(peg$c22);
         }
       }
-    } else {
-      s1 = peg$FAILED;
     }
     if (s1 !== peg$FAILED) {
       peg$savedPos = s0;
@@ -758,21 +800,17 @@ function peg$parse(input, options) {
         peg$fail(peg$c24);
       }
     }
-    if (s2 !== peg$FAILED) {
-      while (s2 !== peg$FAILED) {
-        s1.push(s2);
-        if (peg$c23.test(input.charAt(peg$currPos))) {
-          s2 = input.charAt(peg$currPos);
-          peg$currPos++;
-        } else {
-          s2 = peg$FAILED;
-          if (peg$silentFails === 0) {
-            peg$fail(peg$c24);
-          }
+    while (s2 !== peg$FAILED) {
+      s1.push(s2);
+      if (peg$c23.test(input.charAt(peg$currPos))) {
+        s2 = input.charAt(peg$currPos);
+        peg$currPos++;
+      } else {
+        s2 = peg$FAILED;
+        if (peg$silentFails === 0) {
+          peg$fail(peg$c24);
         }
       }
-    } else {
-      s1 = peg$FAILED;
     }
     if (s1 !== peg$FAILED) {
       peg$savedPos = s0;

package/lib/queryBuilder/parsers/jsonFieldExpressionParser.pegjs

@@ -31,6 +31,7 @@
  * 4. Keys containing square brackets and quotes
  *    `objectColumn:['Double."Quote".[]']` and `objectColumn:["Sinlge.'Quote'.[]"]`
  *    Column `{ "Double.\"Quote\".[]" : "I was referred",  "Sinlge.'Quote'.[]" : "Mee too!" }`
+ * 5. Empty keys `objectColumn:[""]` or `objectColumn:['']` Column `{ "" : "I was referred" }`
  * 99. Keys containing dots, square brackets, single quotes and double quotes in one json key is
  *     not currently supported
  *
@@ -60,8 +61,8 @@ start =
 bracketStringRef =
   '['
       key:(
-       '"' stringWithoutDoubleQuotes '"' /
-       "'" stringWithoutSingleQuotes "'" /
+       '"' stringWithoutDoubleQuotes '"' &']' /
+       "'" stringWithoutSingleQuotes "'" &']' /
        stringWithoutSquareBrackets
       )
   ']'
@@ -86,10 +87,10 @@ stringWithoutColon =
   chars:([^:])+ { return chars.join(""); }
 
 stringWithoutDoubleQuotes =
-  chars:([^"])+ { return chars.join(""); }
+  chars:([^"])* { return chars.join(""); }
 
 stringWithoutSingleQuotes =
-  chars:([^'])+ { return chars.join(""); }
+  chars:([^'])* { return chars.join(""); }
 
 stringWithoutSquareBracketsOrDots =
   chars:([^.\x5D\x5B])+ { return chars.join(""); }

package/lib/queryBuilder/whereGroups.js

@@ -0,0 +1,125 @@
+'use strict';
+
+const { WhereGroupOperation } = require('./operations/WhereGroupOperation');
+const { isObject, isFunction } = require('../utils/objectUtils');
+const { isKnexRaw } = require('../utils/knexUtils');
+
+// Where operations of a query can come from different sources: the user, a
+// relation's owner condition, the relation's `modify`/`filter` and so on. Each
+// source forms its own "where group". All operations the user adds belong to
+// the default group, and `addWhereGroup` assigns the operations added inside
+// its callback to a new, separate group.
+//
+// Knex simply joins all where clauses with their `and` or `or` boolean. Since
+// `and` binds tighter than `or`, a user's `orWhere` would escape the relation's
+// owner condition: `where "ownerId" in (1) and a or b`. This would find, update
+// and delete rows of other owners too. See issues #2191 and #1909.
+//
+// To prevent this, `wrapWhereGroups` is called after all `onBuild` hooks have
+// run. If a query contains more than one where group, every group that may
+// contain an `or` at its top level is wrapped in parentheses, so that the
+// groups are always ANDed together: `where "ownerId" in (1) and (a or b)`.
+// Groups without an `or` are left alone so that their SQL doesn't change.
+function addWhereGroup(builder, callback) {
+  const parentWhereGroup = builder._whereGroup;
+  builder._whereGroup = {};
+
+  try {
+    return callback(builder);
+  } finally {
+    builder._whereGroup = parentWhereGroup;
+  }
+}
+
+function wrapWhereGroups(builder) {
+  const WhereSelector = builder.constructor.WhereSelector;
+  const whereOps = new Set();
+  const groups = new Map();
+
+  builder.forEachOperation(WhereSelector, (op) => {
+    // Only consider the top-most where operations. The descendants of a
+    // where operation are built as part of it.
+    if (op.isAncestorInSet(whereOps)) {
+      return;
+    }
+
+    whereOps.add(op);
+
+    const group = getWhereGroup(op);
+    let ops = groups.get(group);
+
+    if (!ops) {
+      ops = [];
+      groups.set(group, ops);
+    }
+
+    ops.push(op);
+  });
+
+  if (groups.size < 2) {
+    return builder;
+  }
+
+  for (const [group, ops] of groups) {
+    if (ops.some(mayContainTopLevelOr)) {
+      wrapOperations(builder, group, ops);
+    }
+  }
+
+  return builder;
+}
+
+function getWhereGroup(op) {
+  for (let it = op; it; it = it.parentOperation) {
+    if (it.whereGroup) {
+      return it.whereGroup;
+    }
+  }
+
+  return null;
+}
+
+function mayContainTopLevelOr(op) {
+  if (op instanceof WhereGroupOperation) {
+    return false;
+  }
+
+  // orWhere, orWhereIn, orWhereRaw, orWhereExists, orWhereJsonSupersetOf etc.
+  if (/^or[A-Z]/.test(op.name)) {
+    return true;
+  }
+
+  // Raw SQL isn't parenthesized by knex and can contain `or`.
+  if (/Raw$/.test(op.name)) {
+    return true;
+  }
+
+  return Array.isArray(op.args) && op.args.length === 1 && isRaw(op.args[0]);
+}
+
+function isRaw(arg) {
+  return isKnexRaw(arg) || (isObject(arg) && isFunction(arg.toKnexRaw));
+}
+
+function wrapOperations(builder, group, ops) {
+  const groupOp = new WhereGroupOperation('where');
+  groupOp.whereGroup = group;
+
+  builder.replaceOperation(ops[0], groupOp);
+
+  for (let i = 1; i < ops.length; ++i) {
+    builder.removeOperation(ops[i]);
+  }
+
+  for (const op of ops) {
+    op.parentOperation = null;
+    op.adderHookName = null;
+  }
+
+  groupOp.operations = ops;
+}
+
+module.exports = {
+  addWhereGroup,
+  wrapWhereGroups,
+};

package/lib/relations/Relation.js

@@ -10,6 +10,7 @@ const { resolveModel } = require('../utils/resolveModel');
 const { get, isFunction } = require('../utils/objectUtils');
 const { mapAfterAllReturn } = require('../utils/promiseUtils');
 const { createModifier } = require('../utils/createModifier');
+const { addWhereGroup } = require('../queryBuilder/whereGroups');
 
 class Relation {
   constructor(relationName, OwnerClass) {
@@ -115,14 +116,16 @@ class Relation {
 
   findQuery(builder, owner) {
     const relatedRefs = this.relatedProp.refs(builder);
-    owner.buildFindQuery(builder, this, relatedRefs);
+    addWhereGroup(builder, () => owner.buildFindQuery(builder, this, relatedRefs));
 
     return this.applyModify(builder);
   }
 
   applyModify(builder) {
     try {
-      return builder.modify(this.modify);
+      // Keep the relation's `modify` in a separate where group so that an
+      // `orWhere` in it can't escape the owner condition. See `whereGroups.js`.
+      return addWhereGroup(builder, () => builder.modify(this.modify));
     } catch (err) {
       if (err instanceof this.relatedModelClass.ModifierNotFoundError) {
         throw this.createError(err.message);

package/lib/relations/RelationFindOperation.js

@@ -1,6 +1,7 @@
 'use strict';
 
 const { FindOperation } = require('../queryBuilder/operations/FindOperation');
+const { deprecate } = require('../utils/deprecate');
 
 class RelationFindOperation extends FindOperation {
   constructor(name, opt) {
@@ -19,8 +20,12 @@ class RelationFindOperation extends FindOperation {
     this.maybeApplyAlias(builder);
     this.relation.findQuery(builder, this.owner);
 
-    if (this.assignResultToOwner && this.owner.isModels) {
-      this.selectMissingJoinColumns(builder);
+    if (this.owner.isModels) {
+      this.warnOnMissingOwnerProps();
+
+      if (this.assignResultToOwner) {
+        this.selectMissingJoinColumns(builder);
+      }
     }
   }
 
@@ -44,6 +49,8 @@ class RelationFindOperation extends FindOperation {
         arr.push(rel);
       }
 
+      this.warnOnMissingRelatedProps(related);
+
       for (let i = 0, l = owners.length; i < l; ++i) {
         const own = owners[i];
         const key = this.relation.ownerProp.propKey(own);
@@ -75,6 +82,43 @@ class RelationFindOperation extends FindOperation {
     return super.onAfter3(builder, related);
   }
 
+  // Owner models without the join columns, e.g. loaded with a partial
+  // `select()`, silently get no related models (#1832). Only `undefined`
+  // is reported, as `null` is a legit value for a missing relation.
+  // TODO: Consider throwing an error instead in objection 4.0.
+  warnOnMissingOwnerProps() {
+    const { ownerProp } = this.relation;
+
+    for (let c = 0, lc = ownerProp.size; c < lc; ++c) {
+      if (this.owner.modelArray.some((owner) => ownerProp.getProp(owner, c) === undefined)) {
+        deprecate(
+          `Fetching relation "${this.relation.name}" of ${this.relation.ownerModelClass.name}: ` +
+            `some owner models are missing the join property "${ownerProp.propDescription(c)}", ` +
+            'so no related models are found for them. Select the column in the owner query.',
+        );
+      }
+    }
+  }
+
+  // Related models without the join properties can't be assigned to their
+  // owners and are silently dropped (#2258). This happens for example when the
+  // relation mapping uses column names but the rows come back with different
+  // property names, e.g. snake_case mappings with `knexSnakeCaseMappers`.
+  warnOnMissingRelatedProps(related) {
+    const { relatedProp } = this.relation;
+
+    for (let c = 0, lc = relatedProp.size; c < lc; ++c) {
+      if (related.some((rel) => relatedProp.getProp(rel, c) === undefined)) {
+        deprecate(
+          `Fetching relation "${this.relation.name}" of ${this.relation.ownerModelClass.name}: ` +
+            `some related models are missing the join property "${relatedProp.propDescription(c)}", ` +
+            'so they are not assigned to any owner. Check that the relation mapping uses the ' +
+            'property names of the models, e.g. camelCase names when using knexSnakeCaseMappers.',
+        );
+      }
+    }
+  }
+
   selectMissingJoinColumns(builder) {
     const relatedProp = this.relation.relatedProp;
     const addedSelects = [];

package/lib/relations/RelationProperty.js

@@ -31,10 +31,11 @@ class RelationProperty {
   //
   // modelClassResolver must be a function that takes a table name
   // and returns a model class.
-  constructor(references, modelClassResolver) {
+  constructor(references, modelClassResolver, { allowEmpty = false } = {}) {
     const refs = createRefs(asArray(references));
     const paths = createPaths(refs, modelClassResolver);
-    const modelClass = resolveModelClass(paths);
+    const modelClass =
+      allowEmpty && refs.length === 0 ? modelClassResolver(null) : resolveModelClass(paths);
 
     this._refs = refs.map((ref) => ref.model(modelClass));
     this._modelClass = modelClass;