@strapi/database 4.6.0-beta.1 → 4.6.0

package/lib/entity-manager/regular-relations.js

@@ -198,38 +198,73 @@ const cleanOrderColumns = async ({ id, attribute, db, inverseRelIds, transaction
     return;
   }
 
+  // Handle databases that don't support window function ROW_NUMBER (here it's MySQL 5)
+  if (!strapi.db.dialect.supportsWindowFunctions()) {
+    await cleanOrderColumnsForOldDatabases({ id, attribute, db, inverseRelIds, transaction: trx });
+    return;
+  }
+
+  const { joinTable } = attribute;
+  const { joinColumn, inverseJoinColumn, orderColumnName, inverseOrderColumnName } = joinTable;
+  const update = [];
+  const updateBinding = [];
+  const select = ['??'];
+  const selectBinding = ['id'];
+  const where = [];
+  const whereBinding = [];
+
+  if (hasOrderColumn(attribute) && id) {
+    update.push('?? = b.src_order');
+    updateBinding.push(orderColumnName);
+    select.push('ROW_NUMBER() OVER (PARTITION BY ?? ORDER BY ??) AS src_order');
+    selectBinding.push(joinColumn.name, orderColumnName);
+    where.push('?? = ?');
+    whereBinding.push(joinColumn.name, id);
+  }
+
+  if (hasInverseOrderColumn(attribute) && !isEmpty(inverseRelIds)) {
+    update.push('?? = b.inv_order');
+    updateBinding.push(inverseOrderColumnName);
+    select.push('ROW_NUMBER() OVER (PARTITION BY ?? ORDER BY ??) AS inv_order');
+    selectBinding.push(inverseJoinColumn.name, inverseOrderColumnName);
+    where.push(`?? IN (${inverseRelIds.map(() => '?').join(', ')})`);
+    whereBinding.push(inverseJoinColumn.name, ...inverseRelIds);
+  }
+
   switch (strapi.db.dialect.client) {
     case 'mysql':
-      await cleanOrderColumnsForInnoDB({ id, attribute, db, inverseRelIds, transaction: trx });
+      // Here it's MariaDB and MySQL 8
+      await db
+        .getConnection()
+        .raw(
+          `UPDATE
+            ?? as a,
+            (
+              SELECT ${select.join(', ')}
+              FROM ??
+              WHERE ${where.join(' OR ')}
+            ) AS b
+          SET ${update.join(', ')}
+          WHERE b.id = a.id`,
+          [joinTable.name, ...selectBinding, joinTable.name, ...whereBinding, ...updateBinding]
+        )
+        .transacting(trx);
       break;
+    /*
+      UPDATE
+        :joinTable: as a,
+        (
+          SELECT
+            id,
+            ROW_NUMBER() OVER ( PARTITION BY :joinColumn: ORDER BY :orderColumn:) AS src_order,
+            ROW_NUMBER() OVER ( PARTITION BY :inverseJoinColumn: ORDER BY :inverseOrderColumn:) AS inv_order
+          FROM :joinTable:
+          WHERE :joinColumn: = :id OR :inverseJoinColumn: IN (:inverseRelIds)
+        ) AS b
+      SET :orderColumn: = b.src_order, :inverseOrderColumn: = b.inv_order
+      WHERE b.id = a.id;
+    */
     default: {
-      const { joinTable } = attribute;
-      const { joinColumn, inverseJoinColumn, orderColumnName, inverseOrderColumnName } = joinTable;
-      const update = [];
-      const updateBinding = [];
-      const select = ['??'];
-      const selectBinding = ['id'];
-      const where = [];
-      const whereBinding = [];
-
-      if (hasOrderColumn(attribute) && id) {
-        update.push('?? = b.src_order');
-        updateBinding.push(orderColumnName);
-        select.push('ROW_NUMBER() OVER (PARTITION BY ?? ORDER BY ??) AS src_order');
-        selectBinding.push(joinColumn.name, orderColumnName);
-        where.push('?? = ?');
-        whereBinding.push(joinColumn.name, id);
-      }
-
-      if (hasInverseOrderColumn(attribute) && !isEmpty(inverseRelIds)) {
-        update.push('?? = b.inv_order');
-        updateBinding.push(inverseOrderColumnName);
-        select.push('ROW_NUMBER() OVER (PARTITION BY ?? ORDER BY ??) AS inv_order');
-        selectBinding.push(inverseJoinColumn.name, inverseOrderColumnName);
-        where.push(`?? IN (${inverseRelIds.map(() => '?').join(', ')})`);
-        whereBinding.push(inverseJoinColumn.name, ...inverseRelIds);
-      }
-
       const joinTableName = addSchema(joinTable.name);
 
       // raw query as knex doesn't allow updating from a subquery
@@ -249,17 +284,17 @@ const cleanOrderColumns = async ({ id, attribute, db, inverseRelIds, transaction
         .transacting(trx);
 
       /*
-        `UPDATE :joinTable: as a
-          SET :orderColumn: = b.src_order, :inverseOrderColumn: = b.inv_order
-          FROM (
-            SELECT
-              id,
-              ROW_NUMBER() OVER ( PARTITION BY :joinColumn: ORDER BY :orderColumn:) AS src_order,
-              ROW_NUMBER() OVER ( PARTITION BY :inverseJoinColumn: ORDER BY :inverseOrderColumn:) AS inv_order
-            FROM :joinTable:
-            WHERE :joinColumn: = :id OR :inverseJoinColumn: IN (:inverseRelIds)
-          ) AS b
-          WHERE b.id = a.id`,
+        UPDATE :joinTable: as a
+        SET :orderColumn: = b.src_order, :inverseOrderColumn: = b.inv_order
+        FROM (
+          SELECT
+            id,
+            ROW_NUMBER() OVER ( PARTITION BY :joinColumn: ORDER BY :orderColumn:) AS src_order,
+            ROW_NUMBER() OVER ( PARTITION BY :inverseJoinColumn: ORDER BY :inverseOrderColumn:) AS inv_order
+          FROM :joinTable:
+          WHERE :joinColumn: = :id OR :inverseJoinColumn: IN (:inverseRelIds)
+        ) AS b
+        WHERE b.id = a.id;
       */
     }
   }
@@ -267,9 +302,9 @@ const cleanOrderColumns = async ({ id, attribute, db, inverseRelIds, transaction
 
 /*
  * Ensure that orders are following a 1, 2, 3 sequence, without gap.
- * The use of a temporary table instead of a window function makes the query compatible with MySQL 5 and prevents some deadlocks to happen in innoDB databases
+ * The use of a session variable instead of a window function makes the query compatible with MySQL 5
  */
-const cleanOrderColumnsForInnoDB = async ({
+const cleanOrderColumnsForOldDatabases = async ({
   id,
   attribute,
   db,
@@ -279,90 +314,68 @@ const cleanOrderColumnsForInnoDB = async ({
   const { joinTable } = attribute;
   const { joinColumn, inverseJoinColumn, orderColumnName, inverseOrderColumnName } = joinTable;
 
-  const now = new Date().valueOf();
-  const randomHex = randomBytes(16).toString('hex');
+  const randomSuffix = `${new Date().valueOf()}_${randomBytes(16).toString('hex')}`;
 
   if (hasOrderColumn(attribute) && id) {
-    const tempOrderTableName = `orderTable_${now}_${randomHex}`;
-    try {
-      await db.connection
-        .raw(
-          `
-          CREATE TABLE :tempOrderTableName:
-            SELECT
-              id,
-              (
-                SELECT count(*)
-                FROM :joinTableName: b
-                WHERE a.:orderColumnName: >= b.:orderColumnName: AND a.:joinColumnName: = b.:joinColumnName: AND a.:joinColumnName: = :id
-              ) AS src_order
-            FROM :joinTableName: a`,
-          {
-            tempOrderTableName,
-            joinTableName: joinTable.name,
-            orderColumnName,
-            joinColumnName: joinColumn.name,
-            id,
-          }
-        )
-        .transacting(trx);
-
-      // raw query as knex doesn't allow updating from a subquery
-      // https://github.com/knex/knex/issues/2504
-      await db.connection
-        .raw(
-          `UPDATE ?? as a, (SELECT * FROM ??) AS b
-          SET ?? = b.src_order
-          WHERE a.id = b.id`,
-          [joinTable.name, tempOrderTableName, orderColumnName]
-        )
-        .transacting(trx);
-    } finally {
-      await db.connection.raw(`DROP TABLE IF EXISTS ??`, [tempOrderTableName]).transacting(trx);
-    }
+    // raw query as knex doesn't allow updating from a subquery
+    // https://github.com/knex/knex/issues/2504
+    const orderVar = `order_${randomSuffix}`;
+    await db.connection.raw(`SET @${orderVar} = 0;`).transacting(trx);
+    await db.connection
+      .raw(
+        `UPDATE :joinTableName: as a, (
+          SELECT id, (@${orderVar}:=@${orderVar} + 1) AS src_order
+          FROM :joinTableName:
+	        WHERE :joinColumnName: = :id
+	        ORDER BY :orderColumnName:
+        ) AS b
+        SET :orderColumnName: = b.src_order
+        WHERE a.id = b.id
+        AND a.:joinColumnName: = :id`,
+        {
+          joinTableName: joinTable.name,
+          orderColumnName,
+          joinColumnName: joinColumn.name,
+          id,
+        }
+      )
+      .transacting(trx);
   }
 
   if (hasInverseOrderColumn(attribute) && !isEmpty(inverseRelIds)) {
-    const tempInvOrderTableName = `invOrderTable_${now}_${randomHex}`;
-    try {
-      await db.connection
-        .raw(
-          `
-          CREATE TABLE ??
-            SELECT
-              id,
-              (
-                SELECT count(*)
-                FROM ?? b
-                WHERE a.?? >= b.?? AND a.?? = b.?? AND a.?? IN (${inverseRelIds
-                  .map(() => '?')
-                  .join(', ')})
-              ) AS inv_order
-            FROM ?? a`,
-          [
-            tempInvOrderTableName,
-            joinTable.name,
-            inverseOrderColumnName,
-            inverseOrderColumnName,
-            inverseJoinColumn.name,
-            inverseJoinColumn.name,
-            inverseJoinColumn.name,
-            ...inverseRelIds,
-            joinTable.name,
-          ]
-        )
-        .transacting(trx);
-      await db.connection
-        .raw(
-          `UPDATE ?? as a, (SELECT * FROM ??) AS b
-            SET ?? = b.inv_order
-            WHERE a.id = b.id`,
-          [joinTable.name, tempInvOrderTableName, inverseOrderColumnName]
-        )
-        .transacting(trx);
-    } finally {
-      await db.connection.raw(`DROP TABLE IF EXISTS ??`, [tempInvOrderTableName]).transacting(trx);
-    }
+    const orderVar = `order_${randomSuffix}`;
+    const columnVar = `col_${randomSuffix}`;
+    await db.connection.raw(`SET @${orderVar} = 0;`).transacting(trx);
+    await db.connection
+      .raw(
+        `UPDATE ?? as a, (
+          SELECT
+          	id,
+            @${orderVar}:=CASE WHEN @${columnVar} = ?? THEN @${orderVar} + 1 ELSE 1 END AS inv_order,
+        	  @${columnVar}:=?? ??
+        	FROM ?? a
+        	WHERE ?? IN(${inverseRelIds.map(() => '?').join(', ')})
+        	ORDER BY ??, ??
+        ) AS b
+        SET ?? = b.inv_order
+        WHERE a.id = b.id
+        AND a.?? IN(${inverseRelIds.map(() => '?').join(', ')})`,
+        [
+          joinTable.name,
+          inverseJoinColumn.name,
+          inverseJoinColumn.name,
+          inverseJoinColumn.name,
+          joinTable.name,
+          inverseJoinColumn.name,
+          ...inverseRelIds,
+          inverseJoinColumn.name,
+          joinColumn.name,
+          inverseOrderColumnName,
+          inverseJoinColumn.name,
+          ...inverseRelIds,
+        ]
+      )
+      .transacting(trx);
   }
 };
 

package/lib/entity-manager/relations-orderer.js

@@ -1,6 +1,107 @@
 'use strict';
 
+const { castArray } = require('lodash/fp');
 const _ = require('lodash/fp');
+const { InvalidRelationError } = require('../errors');
+/**
+ * When connecting relations, the order you connect them matters.
+ *
+ * Example, if you connect the following relations:
+ *   { id: 5, position: { before: 1 } }
+ *   { id: 1, position: { before: 2 } }
+ *   { id: 2, position: { end: true } }
+ *
+ * Going through the connect array, id 5 has to be connected before id 1,
+ * so the order of id5 = id1 - 1. But the order value of id 1 is unknown.
+ * The only way to know the order of id 1 is to connect it first.
+ *
+ * This function makes sure the relations are connected in the right order:
+ *   { id: 2, position: { end: true } }
+ *   { id: 1, position: { before: 2 } }
+ *   { id: 5, position: { before: 1 } }
+ *
+ */
+const sortConnectArray = (connectArr, initialArr = [], strictSort = true) => {
+  const sortedConnect = [];
+  // Boolean to know if we have to recalculate the order of the relations
+  let needsSorting = false;
+  // Map to validate if relation is already in sortedConnect or DB.
+  const relationInInitialArray = initialArr.reduce((acc, rel) => ({ ...acc, [rel.id]: true }), {});
+  // Map to store the first index where a relation id is connected
+  const mappedRelations = connectArr.reduce((mapper, relation) => {
+    const adjacentRelId = relation.position?.before || relation.position?.after;
+
+    if (!relationInInitialArray[adjacentRelId] && !mapper[adjacentRelId]) {
+      needsSorting = true;
+    }
+
+    // If the relation is already in the array, throw an error
+    if (mapper[relation.id]) {
+      throw new InvalidRelationError(
+        `The relation with id ${relation.id} is already connected. ` +
+          'You cannot connect the same relation twice.'
+      );
+    }
+
+    return {
+      [relation.id]: { ...relation, computed: false },
+      ...mapper,
+    };
+  }, {});
+
+  // If we don't need to sort the connect array, we can return it as is
+  if (!needsSorting) return connectArr;
+
+  // Recursively compute in which order the relation should be connected
+  const computeRelation = (relation, relationsSeenInBranch) => {
+    const adjacentRelId = relation.position?.before || relation.position?.after;
+    const adjacentRelation = mappedRelations[adjacentRelId];
+
+    // If the relation has already been seen in the current branch,
+    // it means there is a circular reference
+    if (relationsSeenInBranch[adjacentRelId]) {
+      throw new InvalidRelationError(
+        'A circular reference was found in the connect array. ' +
+          'One relation is trying to connect before/after another one that is trying to connect before/after it'
+      );
+    }
+
+    // This relation has already been computed
+    if (mappedRelations[relation.id]?.computed) return;
+
+    mappedRelations[relation.id].computed = true;
+
+    // Relation does not have a before or after attribute or is in the initial array
+    if (!adjacentRelId || relationInInitialArray[adjacentRelId]) {
+      sortedConnect.push(relation);
+      return;
+    }
+
+    // Look if id is referenced elsewhere in the array
+    if (mappedRelations[adjacentRelId]) {
+      computeRelation(adjacentRelation, { ...relationsSeenInBranch, [relation.id]: true });
+      sortedConnect.push(relation);
+    } else if (strictSort) {
+      // If we reach this point, it means that the adjacent relation is not in the connect array
+      // and it is not in the database.
+      throw new InvalidRelationError(
+        `There was a problem connecting relation with id ${
+          relation.id
+        } at position ${JSON.stringify(
+          relation.position
+        )}. The relation with id ${adjacentRelId} needs to be connected first.`
+      );
+    } else {
+      // We are in non-strict mode so we can push the relation.
+      sortedConnect.push({ id: relation.id, position: { end: true } });
+    }
+  };
+
+  // Iterate over connectArr and populate sortedConnect
+  connectArr.forEach((relation) => computeRelation(relation, {}));
+
+  return sortedConnect;
+};
 
 /**
  * Responsible for calculating the relations order when connecting them.
@@ -23,34 +124,31 @@ const _ = require('lodash/fp');
  * - The final step would be to recalculate fractional order values.
  *      [ { id: 2, order: 4 }, { id: 5, order: 3.33 },  { id: 4, order: 3.66 }, { id: 3, order: 10 } ]
  *
- * Constraints:
- * - Expects you will never connect a relation before / after one that does not exist
- * - Expect initArr to have all relations referenced in the positional attributes
- *
  * @param {Array<*>} initArr - array of relations to initialize the class with
  * @param {string} idColumn - the column name of the id
  * @param {string} orderColumn - the column name of the order
+ * @param {boolean} strict - if true, will throw an error if a relation is connected adjacent to
+ *                               another one that does not exist
  * @return {*}
  */
-const relationsOrderer = (initArr, idColumn, orderColumn) => {
-  const arr = _.castArray(initArr || []).map((r) => ({
+const relationsOrderer = (initArr, idColumn, orderColumn, strict) => {
+  const computedRelations = _.castArray(initArr || []).map((r) => ({
     init: true,
     id: r[idColumn],
     order: r[orderColumn],
   }));
 
-  const maxOrder = _.maxBy('order', arr)?.order || 0;
+  const maxOrder = _.maxBy('order', computedRelations)?.order || 0;
 
-  // TODO: Improve performance by using a map
   const findRelation = (id) => {
-    const idx = arr.findIndex((r) => r.id === id);
-    return { idx, relation: arr[idx] };
+    const idx = computedRelations.findIndex((r) => r.id === id);
+    return { idx, relation: computedRelations[idx] };
   };
 
   const removeRelation = (r) => {
     const { idx } = findRelation(r.id);
     if (idx >= 0) {
-      arr.splice(idx, 1);
+      computedRelations.splice(idx, 1);
     }
   };
 
@@ -72,45 +170,46 @@ const relationsOrderer = (initArr, idColumn, orderColumn) => {
       idx = 0;
     } else {
       r.order = maxOrder + 0.5;
-      idx = arr.length;
+      idx = computedRelations.length;
     }
 
     // Insert the relation in the array
-    arr.splice(idx, 0, r);
+    computedRelations.splice(idx, 0, r);
   };
 
   return {
     disconnect(relations) {
-      _.castArray(relations).forEach((relation) => {
+      castArray(relations).forEach((relation) => {
         removeRelation(relation);
       });
       return this;
     },
     connect(relations) {
-      _.castArray(relations).forEach((relation) => {
+      sortConnectArray(castArray(relations), computedRelations, strict).forEach((relation) => {
         this.disconnect(relation);
 
         try {
           insertRelation(relation);
         } catch (err) {
-          strapi.log.error(err);
           throw new Error(
-            `Could not connect ${relation.id}, position ${JSON.stringify(
+            `There was a problem connecting relation with id ${
+              relation.id
+            } at position ${JSON.stringify(
               relation.position
-            )} is invalid`
+            )}. The list of connect relations is not valid`
           );
         }
       });
       return this;
     },
     get() {
-      return arr;
+      return computedRelations;
     },
     /**
      * Get a map between the relation id and its order
      */
     getOrderMap() {
-      return _(arr)
+      return _(computedRelations)
         .groupBy('order')
         .reduce((acc, relations) => {
           if (relations[0]?.init) return acc;
@@ -123,4 +222,4 @@ const relationsOrderer = (initArr, idColumn, orderColumn) => {
   };
 };
 
-module.exports = relationsOrderer;
+module.exports = { relationsOrderer, sortConnectArray };

package/lib/errors/index.js

@@ -5,6 +5,7 @@ const NotNullError = require('./not-null');
 const InvalidTimeError = require('./invalid-time');
 const InvalidDateError = require('./invalid-date');
 const InvalidDateTimeError = require('./invalid-datetime');
+const InvalidRelationError = require('./invalid-relation');
 
 module.exports = {
   DatabaseError,
@@ -12,4 +13,5 @@ module.exports = {
   InvalidTimeError,
   InvalidDateError,
   InvalidDateTimeError,
+  InvalidRelationError,
 };

package/lib/errors/invalid-relation.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const DatabaseError = require('./database');
+
+class InvalidRelationError extends DatabaseError {
+  constructor(message) {
+    super();
+    this.name = 'InvalidRelationFormat';
+    this.message = message || 'Invalid relation format';
+    this.details = {};
+  }
+}
+
+module.exports = InvalidRelationError;

package/lib/index.d.ts

@@ -163,6 +163,15 @@ export interface Database {
   connection: Knex;
 
   query<T extends keyof AllTypes>(uid: T): QueryFromContentType<T>;
+  transaction(
+    cb?: (params: {
+      trx: Knex.Transaction;
+      rollback: () => Promise<void>;
+      commit: () => Promise<void>;
+    }) => Promise<unknown>
+  ):
+    | Promise<unknown>
+    | { get: () => Knex.Transaction; rollback: () => Promise<void>; commit: () => Promise<void> };
 }
 export class Database implements Database {
   static transformContentTypes(contentTypes: any[]): ModelConfig[];

package/lib/index.js

@@ -8,9 +8,11 @@ const { createMigrationsProvider } = require('./migrations');
 const { createLifecyclesProvider } = require('./lifecycles');
 const createConnection = require('./connection');
 const errors = require('./errors');
+const transactionCtx = require('./transaction-context');
 
 // TODO: move back into strapi
 const { transformContentTypes } = require('./utils/content-types');
+const { validateDatabase } = require('./validations');
 
 class Database {
   constructor(config) {
@@ -20,6 +22,8 @@ class Database {
       connection: {},
       settings: {
         forceMigration: true,
+        runMigrations: true,
+        ...config.settings,
       },
       ...config,
     };
@@ -47,6 +51,44 @@ class Database {
     return this.entityManager.getRepository(uid);
   }
 
+  async transaction(cb) {
+    const notNestedTransaction = !transactionCtx.get();
+    const trx = notNestedTransaction ? await this.connection.transaction() : transactionCtx.get();
+
+    async function commit() {
+      if (notNestedTransaction) {
+        await trx.commit();
+      }
+    }
+
+    async function rollback() {
+      if (notNestedTransaction) {
+        await trx.rollback();
+      }
+    }
+    if (!cb) {
+      return {
+        commit,
+        rollback,
+        get() {
+          return trx;
+        },
+      };
+    }
+
+    return transactionCtx.run(trx, async () => {
+      try {
+        const callbackParams = { trx, commit, rollback };
+        const res = await cb(callbackParams);
+        await commit();
+        return res;
+      } catch (error) {
+        await rollback();
+        throw error;
+      }
+    });
+  }
+
   getConnection(tableName) {
     const schema = this.connection.getSchemaName();
     const connection = tableName ? this.connection(tableName) : this.connection;
@@ -58,10 +100,6 @@ class Database {
     return schema ? trx.schema.withSchema(schema) : trx.schema;
   }
 
-  transaction() {
-    return this.connection.transaction();
-  }
-
   queryBuilder(uid) {
     return this.entityManager.createQueryBuilder(uid);
   }
@@ -74,7 +112,11 @@ class Database {
 
 // TODO: move into strapi
 Database.transformContentTypes = transformContentTypes;
-Database.init = async (config) => new Database(config);
+Database.init = async (config) => {
+  const db = new Database(config);
+  await validateDatabase(db);
+  return db;
+};
 
 module.exports = {
   Database,

package/lib/metadata/relations.js

@@ -16,6 +16,8 @@ const isAnyToMany = (attribute) => ['oneToMany', 'manyToMany'].includes(attribut
 const isBidirectional = (attribute) => hasInversedBy(attribute) || hasMappedBy(attribute);
 const isOwner = (attribute) => !isBidirectional(attribute) || hasInversedBy(attribute);
 const shouldUseJoinTable = (attribute) => attribute.useJoinTable !== false;
+const getJoinTableName = (tableName, attributeName) =>
+  _.snakeCase(`${tableName}_${attributeName}_links`);
 
 /**
  * Creates a oneToOne relation metadata
@@ -397,7 +399,7 @@ const createJoinTable = (metadata, { attributeName, attribute, meta }) => {
     throw new Error(`Unknown target ${attribute.target}`);
   }
 
-  const joinTableName = _.snakeCase(`${meta.tableName}_${attributeName}_links`);
+  const joinTableName = getJoinTableName(meta.tableName, attributeName);
 
   const joinColumnName = _.snakeCase(`${meta.singularName}_id`);
   let inverseJoinColumnName = _.snakeCase(`${targetMeta.singularName}_id`);
@@ -560,4 +562,5 @@ module.exports = {
   isAnyToMany,
   hasOrderColumn,
   hasInverseOrderColumn,
+  getJoinTableName,
 };

package/lib/migrations/index.js

@@ -61,7 +61,7 @@ const createMigrationsProvider = (db) => {
     async shouldRun() {
       const pending = await migrations.pending();
 
-      return pending.length > 0;
+      return pending.length > 0 && db.config.settings.runMigrations;
     },
     async up() {
       await migrations.up();