@strapi/database 4.0.0-next.6 → 4.0.0

package/lib/schema/diff.js

@@ -0,0 +1,376 @@
+'use strict';
+
+const _ = require('lodash/fp');
+
+const RESERVED_TABLE_NAMES = ['strapi_migrations', 'strapi_database_schema'];
+
+const statuses = {
+  CHANGED: 'CHANGED',
+  UNCHANGED: 'UNCHANGED',
+};
+
+// NOTE:We could move the schema to use maps of tables & columns instead of arrays to make it easier to diff
+// => this will make the creation a bit more complicated (ordering, Object.values(tables | columns)) -> not a big pbl
+
+const helpers = {
+  hasTable(schema, tableName) {
+    return schema.tables.findIndex(table => table.name === tableName) !== -1;
+  },
+  findTable(schema, tableName) {
+    return schema.tables.find(table => table.name === tableName);
+  },
+
+  hasColumn(table, columnName) {
+    return table.columns.findIndex(column => column.name === columnName) !== -1;
+  },
+  findColumn(table, columnName) {
+    return table.columns.find(column => column.name === columnName);
+  },
+
+  hasIndex(table, columnName) {
+    return table.indexes.findIndex(column => column.name === columnName) !== -1;
+  },
+  findIndex(table, columnName) {
+    return table.indexes.find(column => column.name === columnName);
+  },
+
+  hasForeignKey(table, columnName) {
+    return table.foreignKeys.findIndex(column => column.name === columnName) !== -1;
+  },
+  findForeignKey(table, columnName) {
+    return table.foreignKeys.find(column => column.name === columnName);
+  },
+};
+
+module.exports = db => {
+  const hasChangedStatus = diff => diff.status === statuses.CHANGED;
+
+  /**
+   * Compares two indexes info
+   * @param {Object} oldIndex - index info read from DB
+   * @param {Object} index - newly generate index info
+   */
+  const diffIndexes = (oldIndex, index) => {
+    const changes = [];
+
+    if (_.difference(oldIndex.columns, index.columns).length > 0) {
+      changes.push('columns');
+    }
+
+    if (_.toLower(oldIndex.type) !== _.toLower(index.type)) {
+      changes.push('type');
+    }
+
+    return {
+      status: changes.length > 0 ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        name: index.name,
+        object: index,
+      },
+    };
+  };
+
+  /**
+   * Compares two foreign keys info
+   * @param {Object} oldForeignKey - foreignKey info read from DB
+   * @param {Object} foreignKey - newly generate foreignKey info
+   */
+  const diffForeignKeys = (oldForeignKey, foreignKey) => {
+    const changes = [];
+
+    if (_.difference(oldForeignKey.columns, foreignKey.columns).length > 0) {
+      changes.push('columns');
+    }
+
+    if (_.difference(oldForeignKey.referencedColumns, foreignKey.referencedColumns).length > 0) {
+      changes.push('referencedColumns');
+    }
+
+    if (oldForeignKey.referencedTable !== foreignKey.referencedTable) {
+      changes.push('referencedTable');
+    }
+
+    if (_.isNil(oldForeignKey.onDelete) || _.toUpper(oldForeignKey.onDelete) === 'NO ACTION') {
+      if (!_.isNil(foreignKey.onDelete) && _.toUpper(oldForeignKey.onDelete) !== 'NO ACTION') {
+        changes.push('onDelete');
+      }
+    } else if (_.toUpper(oldForeignKey.onDelete) !== _.toUpper(foreignKey.onDelete)) {
+      changes.push('onDelete');
+    }
+
+    if (_.isNil(oldForeignKey.onUpdate) || _.toUpper(oldForeignKey.onUpdate) === 'NO ACTION') {
+      if (!_.isNil(foreignKey.onUpdate) && _.toUpper(oldForeignKey.onUpdate) !== 'NO ACTION') {
+        changes.push('onUpdate');
+      }
+    } else if (_.toUpper(oldForeignKey.onUpdate) !== _.toUpper(foreignKey.onUpdate)) {
+      changes.push('onUpdate');
+    }
+
+    return {
+      status: changes.length > 0 ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        name: foreignKey.name,
+        object: foreignKey,
+      },
+    };
+  };
+
+  const diffDefault = (oldColumn, column) => {
+    const oldDefaultTo = oldColumn.defaultTo;
+    const defaultTo = column.defaultTo;
+
+    if (oldDefaultTo === null || _.toLower(oldDefaultTo) === 'null') {
+      return _.isNil(defaultTo) || _.toLower(defaultTo) === 'null';
+    }
+
+    return (
+      _.toLower(oldDefaultTo) === _.toLower(column.defaultTo) ||
+      _.toLower(oldDefaultTo) === _.toLower(`'${column.defaultTo}'`)
+    );
+  };
+
+  /**
+   * Compares two columns info
+   * @param {Object} oldColumn - column info read from DB
+   * @param {Object} column - newly generate column info
+   */
+  const diffColumns = (oldColumn, column) => {
+    const changes = [];
+
+    const isIgnoredType = ['increments', 'enum'].includes(column.type);
+
+    // NOTE: enum aren't updated, they need to be dropped & recreated. Knex doesn't handle it
+    const oldType = oldColumn.type;
+    const type = db.dialect.getSqlType(column.type);
+
+    if (oldType !== type && !isIgnoredType) {
+      changes.push('type');
+    }
+
+    // NOTE: compare args at some point and split them into specific properties instead
+
+    if (oldColumn.notNullable !== column.notNullable) {
+      changes.push('notNullable');
+    }
+
+    const hasSameDefault = diffDefault(oldColumn, column);
+    if (!hasSameDefault) {
+      changes.push('defaultTo');
+    }
+
+    if (oldColumn.unsigned !== column.unsigned && db.dialect.supportsUnsigned()) {
+      changes.push('unsigned');
+    }
+
+    return {
+      status: changes.length > 0 ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        name: column.name,
+        object: column,
+      },
+    };
+  };
+
+  const diffTableColumns = (srcTable, destTable) => {
+    const addedColumns = [];
+    const updatedColumns = [];
+    const unchangedColumns = [];
+    const removedColumns = [];
+
+    for (const destColumn of destTable.columns) {
+      if (!helpers.hasColumn(srcTable, destColumn.name)) {
+        addedColumns.push(destColumn);
+        continue;
+      }
+
+      const srcColumn = helpers.findColumn(srcTable, destColumn.name);
+      const { status, diff } = diffColumns(srcColumn, destColumn);
+
+      if (status === statuses.CHANGED) {
+        updatedColumns.push(diff);
+      } else {
+        unchangedColumns.push(srcColumn);
+      }
+    }
+
+    for (const srcColumn of srcTable.columns) {
+      if (!helpers.hasColumn(destTable, srcColumn.name)) {
+        removedColumns.push(srcColumn);
+      }
+    }
+
+    const hasChanged = [addedColumns, updatedColumns, removedColumns].some(arr => arr.length > 0);
+
+    return {
+      status: hasChanged ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        added: addedColumns,
+        updated: updatedColumns,
+        unchanged: unchangedColumns,
+        removed: removedColumns,
+      },
+    };
+  };
+
+  const diffTableIndexes = (srcTable, destTable) => {
+    const addedIndexes = [];
+    const updatedIndexes = [];
+    const unchangedIndexes = [];
+    const removedIndexes = [];
+
+    for (const destIndex of destTable.indexes) {
+      if (helpers.hasIndex(srcTable, destIndex.name)) {
+        const srcIndex = helpers.findIndex(srcTable, destIndex.name);
+        const { status, diff } = diffIndexes(srcIndex, destIndex);
+
+        if (status === statuses.CHANGED) {
+          updatedIndexes.push(diff);
+        } else {
+          unchangedIndexes.push(srcIndex);
+        }
+      } else {
+        addedIndexes.push(destIndex);
+      }
+    }
+
+    for (const srcIndex of srcTable.indexes) {
+      if (!helpers.hasIndex(destTable, srcIndex.name)) {
+        removedIndexes.push(srcIndex);
+      }
+    }
+
+    const hasChanged = [addedIndexes, updatedIndexes, removedIndexes].some(arr => arr.length > 0);
+
+    return {
+      status: hasChanged ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        added: addedIndexes,
+        updated: updatedIndexes,
+        unchanged: unchangedIndexes,
+        removed: removedIndexes,
+      },
+    };
+  };
+
+  const diffTableForeignKeys = (srcTable, destTable) => {
+    const addedForeignKeys = [];
+    const updatedForeignKeys = [];
+    const unchangedForeignKeys = [];
+    const removedForeignKeys = [];
+
+    if (!db.dialect.usesForeignKeys()) {
+      return {
+        status: statuses.UNCHANGED,
+        diff: {
+          added: addedForeignKeys,
+          updated: updatedForeignKeys,
+          unchanged: unchangedForeignKeys,
+          removed: removedForeignKeys,
+        },
+      };
+    }
+
+    for (const destForeignKey of destTable.foreignKeys) {
+      if (helpers.hasForeignKey(srcTable, destForeignKey.name)) {
+        const srcForeignKey = helpers.findForeignKey(srcTable, destForeignKey.name);
+        const { status, diff } = diffForeignKeys(srcForeignKey, destForeignKey);
+
+        if (status === statuses.CHANGED) {
+          updatedForeignKeys.push(diff);
+        } else {
+          unchangedForeignKeys.push(srcForeignKey);
+        }
+      } else {
+        addedForeignKeys.push(destForeignKey);
+      }
+    }
+
+    for (const srcForeignKey of srcTable.foreignKeys) {
+      if (!helpers.hasForeignKey(destTable, srcForeignKey.name)) {
+        removedForeignKeys.push(srcForeignKey);
+      }
+    }
+
+    const hasChanged = [addedForeignKeys, updatedForeignKeys, removedForeignKeys].some(
+      arr => arr.length > 0
+    );
+
+    return {
+      status: hasChanged ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        added: addedForeignKeys,
+        updated: updatedForeignKeys,
+        unchanged: unchangedForeignKeys,
+        removed: removedForeignKeys,
+      },
+    };
+  };
+
+  const diffTables = (srcTable, destTable) => {
+    const columnsDiff = diffTableColumns(srcTable, destTable);
+    const indexesDiff = diffTableIndexes(srcTable, destTable);
+    const foreignKeysDiff = diffTableForeignKeys(srcTable, destTable);
+
+    const hasChanged = [columnsDiff, indexesDiff, foreignKeysDiff].some(hasChangedStatus);
+
+    return {
+      status: hasChanged ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        name: srcTable.name,
+        indexes: indexesDiff.diff,
+        foreignKeys: foreignKeysDiff.diff,
+        columns: columnsDiff.diff,
+      },
+    };
+  };
+
+  const diffSchemas = (srcSchema, destSchema) => {
+    const addedTables = [];
+    const updatedTables = [];
+    const unchangedTables = [];
+    const removedTables = [];
+
+    for (const destTable of destSchema.tables) {
+      if (helpers.hasTable(srcSchema, destTable.name)) {
+        const srcTable = helpers.findTable(srcSchema, destTable.name);
+
+        const { status, diff } = diffTables(srcTable, destTable);
+
+        if (status === statuses.CHANGED) {
+          updatedTables.push(diff);
+        } else {
+          unchangedTables.push(srcTable);
+        }
+      } else {
+        addedTables.push(destTable);
+      }
+    }
+
+    for (const srcTable of srcSchema.tables) {
+      if (
+        !helpers.hasTable(destSchema, srcTable.name) &&
+        !RESERVED_TABLE_NAMES.includes(srcTable.name)
+      ) {
+        removedTables.push(srcTable);
+      }
+    }
+
+    const hasChanged = [addedTables, updatedTables, removedTables].some(arr => arr.length > 0);
+
+    return {
+      status: hasChanged ? statuses.CHANGED : statuses.UNCHANGED,
+      diff: {
+        tables: {
+          added: addedTables,
+          updated: updatedTables,
+          unchanged: unchangedTables,
+          removed: removedTables,
+        },
+      },
+    };
+  };
+
+  return {
+    diff: diffSchemas,
+  };
+};

package/lib/schema/index.d.ts

@@ -0,0 +1,49 @@
+import { Database } from '../';
+import { Action } from '../lifecycles';
+
+type Type =
+  | 'string'
+  | 'text'
+  | 'richtext'
+  | 'json'
+  | 'enumeration'
+  | 'password'
+  | 'email'
+  | 'integer'
+  | 'biginteger'
+  | 'float'
+  | 'decimal'
+  | 'date'
+  | 'time'
+  | 'datetime'
+  | 'timestamp'
+  | 'boolean'
+  | 'relation';
+
+export interface Attribute {
+  type: Type;
+}
+
+export interface Model {
+  uid: string;
+  tableName: string;
+  attributes: {
+    id: {
+      type: 'increments';
+    };
+    [k: string]: Attribute;
+  };
+  lifecycles?: {
+    [k in Action]: () => void;
+  };
+}
+
+export interface SchemaProvideer {
+  sync(): Promise<void>;
+  syncSchema(): Promise<void>;
+  reset(): Promise<void>;
+  create(): Promise<void>;
+  drop(): Promise<void>;
+}
+
+export default function(db: Database): SchemaProvideer;

package/lib/schema/index.js

@@ -1,49 +1,30 @@
 'use strict';
 
+const debug = require('debug')('strapi::database');
+
 const createSchemaBuilder = require('./builder');
-const createSchemaStorage = require('./schema-storage');
-const { diffSchemas } = require('./schema-diff');
-const { metadataToSchema, createTable } = require('./schema');
-
-const addInternalTables = schema => {
-  schema.addTable(
-    createTable({
-      tableName: 'strapi_database_schema',
-      attributes: {
-        id: {
-          type: 'increments',
-        },
-        schema: {
-          type: 'json',
-        },
-        createdAt: {
-          type: 'datetime',
-        },
-      },
-    })
-  );
-};
+const createSchemaDiff = require('./diff');
+const createSchemaStorage = require('./storage');
+const { metadataToSchema } = require('./schema');
 
+/**
+ * @type {import('.').default}
+ */
 const createSchemaProvider = db => {
-  const currentSchema = metadataToSchema(db.metadata);
-
-  // Add Internal tables to schema
-  addInternalTables(currentSchema);
+  const schema = metadataToSchema(db.metadata);
 
   return {
     builder: createSchemaBuilder(db),
+    schemaDiff: createSchemaDiff(db),
     schemaStorage: createSchemaStorage(db),
 
     /**
      * Drops the database schema
      */
     async drop() {
-      const DBSchema = await this.schemaStorage.read();
-
-      if (!DBSchema) {
-        return;
-      }
+      debug('Dropping database schema');
 
+      const DBSchema = await db.dialect.schemaInspector.getSchema();
       await this.builder.dropSchema(DBSchema);
     },
 
@@ -51,46 +32,62 @@ const createSchemaProvider = db => {
      * Creates the database schema
      */
     async create() {
-      await this.builder.createSchema(currentSchema);
-      await this.schemaStorage.create(currentSchema);
+      debug('Created database schema');
+      await this.builder.createSchema(schema);
     },
 
     /**
      * Resets the database schema
      */
     async reset() {
+      debug('Resetting database schema');
       await this.drop();
       await this.create();
     },
 
+    async syncSchema() {
+      debug('Synchronizing database schema');
+
+      const DBSchema = await db.dialect.schemaInspector.getSchema();
+
+      const { status, diff } = this.schemaDiff.diff(DBSchema, schema);
+
+      if (status === 'CHANGED') {
+        await this.builder.updateSchema(diff);
+      }
+
+      await this.schemaStorage.add(schema);
+    },
+
     // TODO: support options to migrate softly or forcefully
     // TODO: support option to disable auto migration & run a CLI command instead to avoid doing it at startup
+    // TODO: Allow keeping extra indexes / extra tables / extra columns (globally or on a per table basis)
     async sync() {
-      // load previous schema
-      const DBSchema = await this.schemaStorage.read();
+      if (await db.migrations.shouldRun()) {
+        debug('Found migrations to run');
+        await db.migrations.up();
 
-      if (!DBSchema) {
-        return this.create();
+        return this.syncSchema();
       }
 
-      // run migrations
+      const oldSchema = await this.schemaStorage.read();
 
-      // reload updated schema
+      if (!oldSchema) {
+        debug('Schema not persisted yet');
+        return this.syncSchema();
+      }
 
-      // diff schema
-      const { status, diff } = diffSchemas(DBSchema, currentSchema);
+      const { hash: oldHash } = oldSchema;
+      const hash = await this.schemaStorage.hashSchema(schema);
 
-      // TODO: replace by schemaDiff.hasChanged()
-      if (status === 'UNCHANGED') {
-        // NOTE: should we still update the schema in DB ?
-        return;
-      }
+      if (oldHash !== hash) {
+        debug('Schema changed');
 
-      // update schema
-      await this.builder.updateSchema(diff);
+        return this.syncSchema();
+      }
 
-      // persist new schema
-      await this.schemaStorage.update(currentSchema);
+      debug('Schema unchanged');
+      return;
     },
   };
 };

package/lib/schema/schema.js

@@ -9,19 +9,16 @@ const createColumn = (name, attribute) => {
     name,
     type,
     args,
+    defaultTo: null,
+    notNullable: false,
+    unsigned: false,
     ...opts,
     ...(attribute.column || {}),
-    // TODO: allow passing custom params to the DB from the model definition
   };
 };
 
-const shouldCreateColumn = attribute => {
-  return types.isScalar(attribute.type);
-};
-
 const createTable = meta => {
   const table = {
-    // TODO: allow passing custom params to the DB from the model definition
     name: meta.tableName,
     indexes: meta.indexes || [],
     foreignKeys: meta.foreignKeys || [],
@@ -67,9 +64,14 @@ const createTable = meta => {
           // NOTE: could allow configuration
           onDelete: 'SET NULL',
         });
+
+        table.indexes.push({
+          name: `${table.name}_${columnName}_fk`,
+          columns: [columnName],
+        });
       }
-    } else if (shouldCreateColumn(attribute)) {
-      const column = createColumn(key, meta.attributes[key]);
+    } else if (types.isScalar(attribute.type)) {
+      const column = createColumn(attribute.columnName || key, attribute);
 
       if (column.unique) {
         table.indexes.push({
@@ -101,8 +103,13 @@ const getColumnType = attribute => {
 
   switch (attribute.type) {
     case 'increments': {
-      return { type: 'increments', args: [{ primary: true }] };
+      return {
+        type: 'increments',
+        args: [{ primary: true }],
+        notNullable: true,
+      };
     }
+
     // We might want to convert email/password to string types before going into the orm with specific validators & transformers
     case 'password':
     case 'email':
@@ -129,29 +136,26 @@ const getColumnType = attribute => {
       return {
         type: 'enum',
         args: [
-          attribute.enum /*,{ useNative: true, existingType: true, enumName: 'foo_type', schemaName: 'public' }*/,
+          attribute.enum,
+          /*,{ useNative: true, existingType: true, enumName: 'foo_type', schemaName: 'public' }*/
         ],
       };
     }
-
     case 'integer': {
       return { type: 'integer' };
     }
     case 'biginteger': {
       return { type: 'bigInteger' };
     }
-    // TODO: verify usage of double vs float
     case 'float': {
-      return { type: 'double', args: [] };
+      return { type: 'double' };
     }
-    // TODO: define precision
     case 'decimal': {
       return { type: 'decimal', args: [10, 2] };
     }
     case 'date': {
       return { type: 'date' };
     }
-    // TODO: define precision
     case 'time': {
       return { type: 'time', args: [{ precision: 3 }] };
     }
@@ -161,19 +165,18 @@ const getColumnType = attribute => {
         args: [
           {
             useTz: false,
-            precision: 6, // TODO: to define
+            precision: 6,
           },
         ],
       };
     }
-    // TODO: handle defaults
     case 'timestamp': {
       return {
         type: 'timestamp',
         args: [
           {
             useTz: false,
-            precision: 6, // TODO: to define
+            precision: 6,
           },
         ],
       };