objection 3.9.0 → 4.0.0-rc.1

package/lib/queryBuilder/operations/DelegateOperation.js

@@ -1,10 +1,8 @@
-'use strict';
-
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
 
 // Operation that simply delegates all calls to the operation passed
 // to to the constructor in `opt.delegate`.
-class DelegateOperation extends QueryBuilderOperation {
+export class DelegateOperation extends QueryBuilderOperation {
   constructor(name, opt) {
     super(name, opt);
 
@@ -157,7 +155,3 @@ class DelegateOperation extends QueryBuilderOperation {
     return clone;
   }
 }
-
-module.exports = {
-  DelegateOperation,
-};

package/lib/queryBuilder/operations/DeleteOperation.js

@@ -1,23 +1,24 @@
-'use strict';
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
+import { StaticHookArguments } from '../StaticHookArguments.js';
+import { isMySql } from '../../utils/knexUtils.js';
 
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-const { StaticHookArguments } = require('../StaticHookArguments');
-const { isMySql } = require('../../utils/knexUtils');
-const { warnOnce } = require('../../utils/warnings');
-
-class DeleteOperation extends QueryBuilderOperation {
+export class DeleteOperation extends QueryBuilderOperation {
   async onBefore2(builder, result) {
     await callBeforeDelete(builder);
     return result;
   }
 
-  // knex leaves the order out of deletes on MySQL, but keeps their limit, so
-  // the order doesn't pick the deleted rows. Warn when the query is executed.
+  // knex leaves the offset out of deletes. On MySQL, it leaves out the order as
+  // well, but keeps the limit, so the order doesn't pick the deleted rows.
+  // Throw when the query is executed.
   onAfterBuild(builder) {
+    if (builder.has('offset')) {
+      throw new Error('offset() is not supported in delete queries, as knex leaves it out.');
+    }
     if (isMySql(builder.knex()) && builder.has('limit') && builder.has(/^orderBy/)) {
-      warnOnce(
-        'orderBy() has no effect in delete queries with limit() on MySQL, so the limit ' +
-          'deletes rows in any order. This will throw an error in objection 4.0.',
+      throw new Error(
+        'orderBy() is not supported in delete queries with limit() on MySQL, ' +
+          'as knex leaves out the order.',
       );
     }
   }
@@ -26,10 +27,6 @@ class DeleteOperation extends QueryBuilderOperation {
     return knexBuilder.delete();
   }
 
-  onAfter2(builder, result) {
-    return callAfterDelete(builder, result);
-  }
-
   toFindOperation() {
     return null;
   }
@@ -43,22 +40,3 @@ function callStaticBeforeDelete(builder) {
   const args = StaticHookArguments.create({ builder });
   return builder.modelClass().beforeDelete(args);
 }
-
-function callAfterDelete(builder, result) {
-  return callStaticAfterDelete(builder, result);
-}
-
-async function callStaticAfterDelete(builder, result) {
-  const args = StaticHookArguments.create({ builder, result });
-  const maybeResult = await builder.modelClass().afterDelete(args);
-
-  if (maybeResult === undefined) {
-    return result;
-  } else {
-    return maybeResult;
-  }
-}
-
-module.exports = {
-  DeleteOperation,
-};

package/lib/queryBuilder/operations/FindByIdOperation.js

@@ -1,12 +1,13 @@
-'use strict';
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
+import { assertIdNotUndefined } from '../../utils/assert.js';
 
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-const { assertIdNotUndefined } = require('../../utils/assert');
-
-class FindByIdOperation extends QueryBuilderOperation {
+export class FindByIdOperation extends QueryBuilderOperation {
   constructor(name, opt) {
     super(name, opt);
     this.id = this.opt.id;
+    // Keep the id condition in its own where group, so that it applies to
+    // whatever the user adds, `orWhere()` included. See `whereGroups.js`.
+    this.whereGroup = {};
   }
 
   onAdd(builder, args) {
@@ -18,9 +19,7 @@ class FindByIdOperation extends QueryBuilderOperation {
   }
 
   onBuild(builder) {
-    if (!builder.internalOptions().skipUndefined) {
-      assertIdNotUndefined(this.id, `undefined was passed to ${this.name}`);
-    }
+    assertIdNotUndefined(this.id, `undefined was passed to ${this.name}`);
 
     builder.whereComposite(builder.fullIdColumn(), this.id);
   }
@@ -31,7 +30,3 @@ class FindByIdOperation extends QueryBuilderOperation {
     return clone;
   }
 }
-
-module.exports = {
-  FindByIdOperation,
-};

package/lib/queryBuilder/operations/FindByIdsOperation.js

@@ -1,11 +1,11 @@
-'use strict';
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
 
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-
-class FindByIdsOperation extends QueryBuilderOperation {
+export class FindByIdsOperation extends QueryBuilderOperation {
   constructor(name, opt) {
     super(name, opt);
     this.ids = null;
+    // Its own where group, like `FindByIdOperation`.
+    this.whereGroup = {};
   }
 
   onAdd(builder, args) {
@@ -23,7 +23,3 @@ class FindByIdsOperation extends QueryBuilderOperation {
     return clone;
   }
 }
-
-module.exports = {
-  FindByIdsOperation,
-};

package/lib/queryBuilder/operations/FindOperation.js

@@ -1,22 +1,26 @@
-'use strict';
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
+import { StaticHookArguments, isGraphQuery } from '../StaticHookArguments.js';
+import { isPromise, afterReturn } from '../../utils/promiseUtils/index.js';
+import { isObject } from '../../utils/objectUtils.js';
 
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-const { StaticHookArguments } = require('../StaticHookArguments');
-const { isPromise, after, afterReturn } = require('../../utils/promiseUtils');
-const { isObject } = require('../../utils/objectUtils');
-
-class FindOperation extends QueryBuilderOperation {
+export class FindOperation extends QueryBuilderOperation {
   onBefore2(builder, result) {
-    return afterReturn(callStaticBeforeFind(builder), result);
+    if (isGraphQuery(builder) || builder.findOptions().dontCallFindHooks) {
+      return result;
+    } else {
+      return afterReturn(callStaticBeforeFind(builder), result);
+    }
   }
 
   onAfter3(builder, results) {
     const opt = builder.findOptions();
 
-    if (opt.dontCallFindHooks) {
+    // The outer queries of `insertGraph()` and `upsertGraph()` return the
+    // input models, unless they read the written rows back with `returning()`:
+    if ((isGraphQuery(builder) && !builder.has('returning')) || opt.dontCallFindHooks) {
       return results;
     } else {
-      return callAfterFind(builder, results);
+      return callInstanceAfterFind(builder.context(), results, opt.callAfterFindDeeply);
     }
   }
 }
@@ -26,26 +30,6 @@ function callStaticBeforeFind(builder) {
   return builder.modelClass().beforeFind(args);
 }
 
-function callAfterFind(builder, result) {
-  const opt = builder.findOptions();
-  const maybePromise = callInstanceAfterFind(builder.context(), result, opt.callAfterFindDeeply);
-
-  return after(maybePromise, () => callStaticAfterFind(builder, result));
-}
-
-function callStaticAfterFind(builder, result) {
-  const args = StaticHookArguments.create({ builder, result });
-  const maybePromise = builder.modelClass().afterFind(args);
-
-  return after(maybePromise, (maybeResult) => {
-    if (maybeResult === undefined) {
-      return result;
-    } else {
-      return maybeResult;
-    }
-  });
-}
-
 function callInstanceAfterFind(ctx, results, deep) {
   if (Array.isArray(results)) {
     if (results.length === 1) {
@@ -158,7 +142,3 @@ function getAfterFindHook(model) {
     return null;
   }
 }
-
-module.exports = {
-  FindOperation,
-};

package/lib/queryBuilder/operations/FirstOperation.js

@@ -1,8 +1,7 @@
-'use strict';
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
+import { LimitPerParentOperation } from './LimitPerParentOperation.js';
 
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-
-class FirstOperation extends QueryBuilderOperation {
+export class FirstOperation extends QueryBuilderOperation {
   shouldLimit(builder) {
     return builder.isFind() && builder.modelClass().useLimitInFirst;
   }
@@ -15,15 +14,12 @@ class FirstOperation extends QueryBuilderOperation {
     return knexBuilder;
   }
 
-  onAfter3(_, result) {
-    if (Array.isArray(result)) {
+  onAfter3(builder, result) {
+    // Limited per parent, the query still returns the models of all parents:
+    if (Array.isArray(result) && !builder.has(LimitPerParentOperation)) {
       return result[0];
     } else {
       return result;
     }
   }
 }
-
-module.exports = {
-  FirstOperation,
-};

package/lib/queryBuilder/operations/FromOperation.js

@@ -1,7 +1,5 @@
-'use strict';
-
-const { ObjectionToKnexConvertingOperation } = require('./ObjectionToKnexConvertingOperation');
-const { isPlainObject, isString } = require('../../utils/objectUtils');
+import { ObjectionToKnexConvertingOperation } from './ObjectionToKnexConvertingOperation.js';
+import { isPlainObject, isString } from '../../utils/objectUtils.js';
 
 const ALIAS_REGEX = /\s+as\s+/i;
 
@@ -9,7 +7,7 @@ const ALIAS_REGEX = /\s+as\s+/i;
 // knex, but we first try to parse the arguments so that we can determine which
 // tables have been mentioned in a query's from clause. We only parse string
 // references and not `raw` or `ref` etc. references at this point thouhg.
-class FromOperation extends ObjectionToKnexConvertingOperation {
+export class FromOperation extends ObjectionToKnexConvertingOperation {
   constructor(name, opt) {
     super(name, opt);
 
@@ -95,7 +93,3 @@ function parseTableAndAliasFromObject(arg, builder) {
     `one of the tables in ${JSON.stringify(arg)} must be the query's model class's table.`,
   );
 }
-
-module.exports = {
-  FromOperation,
-};

package/lib/queryBuilder/operations/InsertAndFetchOperation.js

@@ -1,11 +1,9 @@
-'use strict';
+import { InsertOperation } from './InsertOperation.js';
+import { DelegateOperation } from './DelegateOperation.js';
+import { keyByProps } from '../../model/modelUtils.js';
+import { asArray } from '../../utils/objectUtils.js';
 
-const { InsertOperation } = require('./InsertOperation');
-const { DelegateOperation } = require('./DelegateOperation');
-const { keyByProps } = require('../../model/modelUtils');
-const { asArray } = require('../../utils/objectUtils');
-
-class InsertAndFetchOperation extends DelegateOperation {
+export class InsertAndFetchOperation extends DelegateOperation {
   constructor(name, opt) {
     super(name, opt);
 
@@ -24,7 +22,7 @@ class InsertAndFetchOperation extends DelegateOperation {
 
     // Models without identifiers can't be fetched. This happens when a row
     // with a database generated identifier is ignored by `onConflict()`.
-    const insertedModelArray = asArray(insertedModels).filter((model) => model && model.$hasId());
+    const insertedModelArray = asArray(insertedModels).filter((model) => model.$hasId());
     const idProps = modelClass.getIdPropertyArray();
     const ids = insertedModelArray.map((model) => model.$id());
 
@@ -38,6 +36,13 @@ class InsertAndFetchOperation extends DelegateOperation {
       .findByIds(ids)
       .castTo(builder.resultModelClass());
 
+    if (!Array.isArray(fetchedModels) || !fetchedModels.every((model) => model.$isObjectionModel)) {
+      throw new Error(
+        `Unable to merge the result of ${modelClass.name}.afterFind() in ` +
+          `${this.name}(): expected an array of models.`,
+      );
+    }
+
     const modelsById = keyByProps(fetchedModels, idProps);
 
     // Instead of returning the freshly fetched models, update the input
@@ -49,7 +54,3 @@ class InsertAndFetchOperation extends DelegateOperation {
     return insertedModels;
   }
 }
-
-module.exports = {
-  InsertAndFetchOperation,
-};

package/lib/queryBuilder/operations/InsertGraphAndFetchOperation.js

@@ -1,10 +1,8 @@
-'use strict';
+import { DelegateOperation } from './DelegateOperation.js';
+import { InsertGraphOperation } from './InsertGraphOperation.js';
+import { RelationExpression } from '../RelationExpression.js';
 
-const { DelegateOperation } = require('./DelegateOperation');
-const { InsertGraphOperation } = require('./InsertGraphOperation');
-const { RelationExpression } = require('../RelationExpression');
-
-class InsertGraphAndFetchOperation extends DelegateOperation {
+export class InsertGraphAndFetchOperation extends DelegateOperation {
   constructor(name, opt) {
     super(name, opt);
 
@@ -39,7 +37,3 @@ class InsertGraphAndFetchOperation extends DelegateOperation {
     return this.isArray ? models : models[0] || null;
   }
 }
-
-module.exports = {
-  InsertGraphAndFetchOperation,
-};

package/lib/queryBuilder/operations/InsertGraphOperation.js

@@ -1,10 +1,8 @@
-'use strict';
+import { DelegateOperation } from './DelegateOperation.js';
+import { InsertOperation } from './InsertOperation.js';
+import { GraphUpsert } from '../graph/GraphUpsert.js';
 
-const { DelegateOperation } = require('./DelegateOperation');
-const { InsertOperation } = require('./InsertOperation');
-const { GraphUpsert } = require('../graph/GraphUpsert');
-
-class InsertGraphOperation extends DelegateOperation {
+export class InsertGraphOperation extends DelegateOperation {
   constructor(name, opt = null) {
     super(name, opt);
 
@@ -33,15 +31,10 @@ class InsertGraphOperation extends DelegateOperation {
     const retVal = super.onAdd(builder, args);
 
     this.upsert = new GraphUpsert({
-      objects: this.models,
+      objects: this.isArray ? this.models : this.models[0],
       rootModelClass: builder.modelClass(),
 
-      upsertOptions: Object.assign({}, this.upsertOptions, {
-        noUpdate: true,
-        noDelete: true,
-        noUnrelate: true,
-        insertMissing: true,
-      }),
+      upsertOptions: this.upsertOptions,
       insertOnly: true,
     });
 
@@ -53,7 +46,7 @@ class InsertGraphOperation extends DelegateOperation {
   }
 
   onBefore1(builder, result) {
-    GraphUpsert.warnOnConflict(builder, this.name);
+    GraphUpsert.checkForOnConflict(builder, this.name);
     return result;
   }
 
@@ -108,7 +101,3 @@ class InsertGraphOperation extends DelegateOperation {
     return clone;
   }
 }
-
-module.exports = {
-  InsertGraphOperation,
-};

package/lib/queryBuilder/operations/InsertOperation.js

@@ -1,17 +1,24 @@
-'use strict';
-
-const { QueryBuilderOperation } = require('./QueryBuilderOperation');
-const { StaticHookArguments } = require('../StaticHookArguments');
-const { after, mapAfterAllReturn } = require('../../utils/promiseUtils');
-const { isSqlite, isMySql, hasBatchInsert } = require('../../utils/knexUtils');
-const { isObject, isString } = require('../../utils/objectUtils');
+import { QueryBuilderOperation } from './QueryBuilderOperation.js';
+import { StaticHookArguments } from '../StaticHookArguments.js';
+import { after, mapAfterAllReturn } from '../../utils/promiseUtils/index.js';
+import {
+  isSqlite,
+  isMySql,
+  hasBatchInsert,
+  hasBatchInsertDefaults,
+  getMaxInsertRows,
+} from '../../utils/knexUtils.js';
+import { chunk, groupBy, isObject, isPlainObject, isString } from '../../utils/objectUtils.js';
+import { removeOverriddenValues } from '../../model/modelQueryProps.js';
+import { checkFieldExpressions } from './fieldExpressions.js';
 
 // Base class for all insert operations.
-class InsertOperation extends QueryBuilderOperation {
+export class InsertOperation extends QueryBuilderOperation {
   constructor(name, opt) {
     super(name, opt);
 
     this.models = null;
+    this.batches = null;
     this.isArray = false;
     this.modelOptions = Object.assign({}, this.opt.modelOptions || {});
   }
@@ -27,12 +34,58 @@ class InsertOperation extends QueryBuilderOperation {
   }
 
   async onBefore2(builder, result) {
-    if (this.models.length > 1 && !hasBatchInsert(builder.knex())) {
-      throw new Error('batch insert only works with Postgresql and SQL Server');
-    } else {
-      await callBeforeInsert(builder, this.models);
-      return result;
+    await callBeforeInsert(builder, this.models);
+
+    // Without ids to return, rows can be inserted in batches on all databases.
+    // The hooks may give the models their ids, or cancel the query.
+    if (
+      this.models.length > 1 &&
+      !hasBatchInsert(builder.knex()) &&
+      !builder.isExplicitlyResolvedOrRejected() &&
+      this._hasIdsToReturn(builder)
+    ) {
+      throw new Error(
+        'batch insert only works with Postgresql, SQL Server and SQLite, or with models that have their ids or no idColumn',
+      );
+    }
+
+    this.models.forEach(removeOverriddenValues);
+    return result;
+  }
+
+  // Insert the models in several queries if they don't fit into one: Models
+  // without any column can't be inserted in batches (#1694), where the database
+  // doesn't fill in defaults (SQLite) the rows of a batch need the same columns,
+  // and a query can only bind so many values. The queries run in the caller's
+  // transaction, if there is one. Without it they are not atomic.
+  queryBatches(builder) {
+    const models = this.models;
+    this.batches = null;
+
+    if (models.length < 2) {
+      return null;
+    }
+
+    const knex = builder.knex();
+    const columnsByModel = new Map(
+      models.map((model) => [model, Object.keys(model.$toDatabaseJson(builder))]),
+    );
+    const batches = [];
+
+    for (const group of groupBy(models, (model) =>
+      getColumnsKey(knex, columnsByModel.get(model)),
+    ).values()) {
+      const columnCount = Math.max(...group.map((model) => columnsByModel.get(model).length));
+
+      if (columnCount === 0) {
+        batches.push(...group.map((model) => [model]));
+      } else {
+        batches.push(...chunk(group, getMaxInsertRows(knex, columnCount)));
+      }
     }
+
+    this.batches = batches.length > 1 ? batches : null;
+    return this.batches;
   }
 
   onBuildKnex(knexBuilder, builder) {
@@ -47,15 +100,37 @@ class InsertOperation extends QueryBuilderOperation {
         // conflict columns are returned too, so that the returned rows can
         // be matched to the models when some of the rows are ignored.
         knexBuilder = knexBuilder.returning(defaultReturning);
-      } else if (isSqlite(knex) && builder.has('onConflict')) {
+      } else if (
+        isSqlite(knex) &&
+        (builder.has('onConflict') || (this.models.length > 1 && this._hasIdsToReturn(builder)))
+      ) {
         // Without a `returning` clause, SQLite returns the id of the last
         // inserted row, which belongs to a previous insert if the row was
-        // ignored. Use `returning` to find out what was actually inserted.
+        // ignored, and only one id for batch inserts. Use `returning` to find
+        // out what was actually inserted.
         knexBuilder = knexBuilder.returning(defaultReturning);
       }
     }
 
-    return knexBuilder.insert(this.models.map((model) => model.$toDatabaseJson(builder)));
+    const modelClass = builder.modelClass();
+    // knex leaves arrays unserialized in SQLite inserts, and also objects when
+    // it inserts several rows with `select ... union all`.
+    const shouldSerialize = isSqlite(knex);
+
+    return knexBuilder.insert(
+      (builder._queryBatch || this.models).map((model) => {
+        checkFieldExpressions(modelClass, model, 'insert');
+        const json = model.$toDatabaseJson(builder);
+        return shouldSerialize ? serializeObjects(json) : json;
+      }),
+    );
+  }
+
+  _hasIdsToReturn(builder) {
+    return (
+      builder.modelClass().getIdColumnArray().length > 0 &&
+      !this.models.every((model) => model.$hasId())
+    );
   }
 
   onAfter1(builder, ret) {
@@ -65,6 +140,8 @@ class InsertOperation extends QueryBuilderOperation {
     }
 
     const isOnConflict = builder.has('onConflict');
+    // The results of the batches are in the order of the batches.
+    const models = this.batches ? this.batches.flat() : this.models;
 
     if (isObject(ret[0])) {
       if (isOnConflict && ret.length !== this.models.length) {
@@ -75,14 +152,15 @@ class InsertOperation extends QueryBuilderOperation {
       } else {
         // If the user specified a `returning` clause the result may be an array of objects.
         // Merge all values of the objects to our models.
-        for (let i = 0, l = this.models.length; i < l; ++i) {
-          this.models[i].$setDatabaseJson(ret[i]);
+        const options = { knex: builder.knex() };
+        for (let i = 0, l = models.length; i < l; ++i) {
+          models[i].$setDatabaseJson(ret[i], options);
         }
       }
     } else if (builder.modelClass().getIdColumnArray().length > 0) {
       // If the return value is not an array of objects, we assume it is an array of identifiers.
-      for (let i = 0, l = this.models.length; i < l; ++i) {
-        const model = this.models[i];
+      for (let i = 0, l = models.length; i < l; ++i) {
+        const model = models[i];
 
         // Don't set the id if the model already has one. MySQL and Sqlite don't return the correct
         // primary key value if the id is not generated in db, but given explicitly. With
@@ -96,9 +174,9 @@ class InsertOperation extends QueryBuilderOperation {
     return this.models;
   }
 
-  onAfter2(builder, models) {
-    const result = this.isArray ? models : models[0] || null;
-    return callAfterInsert(builder, this.models, result);
+  async onAfter2(builder, models) {
+    await callInstanceAfterInsert(builder, this.models);
+    return this.isArray ? models : models[0] || null;
   }
 
   toFindOperation() {
@@ -109,12 +187,19 @@ class InsertOperation extends QueryBuilderOperation {
     const clone = super.clone();
 
     clone.models = this.models;
+    clone.batches = this.batches;
     clone.isArray = this.isArray;
 
     return clone;
   }
 }
 
+// Rows are only grouped where the database doesn't fill in the defaults of the
+// columns that a row leaves out.
+function getColumnsKey(knex, columns) {
+  return hasBatchInsertDefaults(knex) ? '' : JSON.stringify([...columns].sort());
+}
+
 // Returns `null` if there's nothing to return, e.g. for models without an `idColumn`.
 function getDefaultReturning(builder) {
   const idColumns = builder.modelClass().getIdColumnArray();
@@ -178,6 +263,8 @@ function mergeMatchingRows(builder, models, rows) {
     }
   });
 
+  const options = { knex: builder.knex() };
+
   for (const row of rows) {
     const indices = indicesByKey.get(createKey(row));
     const index = indices ? indices.shift() : undefined;
@@ -186,7 +273,7 @@ function mergeMatchingRows(builder, models, rows) {
       throw createMatchError();
     }
 
-    models[index].$setDatabaseJson(row);
+    models[index].$setDatabaseJson(row, options);
   }
 }
 
@@ -222,28 +309,16 @@ function callStaticBeforeInsert(builder) {
   return builder.modelClass().beforeInsert(args);
 }
 
-function callAfterInsert(builder, models, result) {
-  const maybePromise = callInstanceAfterInsert(builder, models);
-  return after(maybePromise, () => callStaticAfterInsert(builder, result));
-}
-
 function callInstanceAfterInsert(builder, models) {
   return mapAfterAllReturn(models, (model) => model.$afterInsert(builder.context()), models);
 }
 
-function callStaticAfterInsert(builder, result) {
-  const args = StaticHookArguments.create({ builder, result });
-  const maybePromise = builder.modelClass().afterInsert(args);
-
-  return after(maybePromise, (maybeResult) => {
-    if (maybeResult === undefined) {
-      return result;
-    } else {
-      return maybeResult;
+function serializeObjects(json) {
+  for (const [key, value] of Object.entries(json)) {
+    if (isPlainObject(value) || Array.isArray(value)) {
+      json[key] = JSON.stringify(value);
     }
-  });
-}
+  }
 
-module.exports = {
-  InsertOperation,
-};
+  return json;
+}