@nocobase/database 0.7.0-alpha.82 → 0.7.1-alpha.4

package/src/__tests__/option-parser.test.ts

@@ -1,7 +1,7 @@
 import { Collection } from '../collection';
-import { mockDatabase } from './index';
-import { OptionsParser } from '../options-parser';
 import { Database } from '../database';
+import { OptionsParser } from '../options-parser';
+import { mockDatabase } from './index';
 
 describe('option parser', () => {
   let db: Database;
@@ -83,6 +83,10 @@ describe('option parser', () => {
   });
 
   test('with sort option', () => {
+    if (db.inDialect('mysql'))  {
+      expect(1).toBe(1);
+      return;
+    }
     let options: any = {
       sort: ['id'],
     };
@@ -91,7 +95,7 @@ describe('option parser', () => {
       collection: User,
     });
     let params = parser.toSequelizeParams();
-    expect(params['order']).toEqual([['id', 'ASC']]);
+    expect(params['order']).toEqual([['id', 'ASC NULLS LAST']]);
 
     options = {
       sort: ['id', '-posts.title', 'posts.comments.createdAt'],
@@ -102,9 +106,9 @@ describe('option parser', () => {
     });
     params = parser.toSequelizeParams();
     expect(params['order']).toEqual([
-      ['id', 'ASC'],
-      [Post.model, 'title', 'DESC'],
-      [Post.model, Comment.model, 'createdAt', 'ASC'],
+      ['id', 'ASC NULLS LAST'],
+      [Post.model, 'title', 'DESC NULLS LAST'],
+      [Post.model, Comment.model, 'createdAt', 'ASC NULLS LAST'],
     ]);
   });
 

package/src/__tests__/relation-repository/belongs-to-many-repository.test.ts

@@ -308,7 +308,7 @@ describe('belongs to many', () => {
       offset: 0,
     });
 
-    console.log(tags);
+    // console.log(tags);
   });
 
   test('update raw attribute', async () => {

package/src/__tests__/sequelize-hooks.test.ts

@@ -0,0 +1,69 @@
+import { Database } from '../database';
+import { mockDatabase } from './index';
+
+// TODO
+describe('sequelize-hooks', () => {
+  let db: Database;
+
+  beforeEach(async () => {
+    db = mockDatabase();
+    await db.sync();
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  test('exec order', async () => {
+    const collection = db.collection({
+      name: 't_test',
+    });
+    const orders = [];
+    db.on('beforeCreate', () => {
+      orders.push('beforeCreate');
+    });
+    db.on('t_test.beforeCreate', () => {
+      orders.push('model.beforeCreate');
+    });
+    db.on('afterCreate', () => {
+      orders.push('afterCreate');
+    });
+    db.on('t_test.afterCreate', () => {
+      orders.push('model.afterCreate');
+    });
+    await collection.sync();
+    await collection.model.create();
+    expect(orders).toEqual([
+      'model.beforeCreate',
+      'beforeCreate',
+      'model.afterCreate',
+      'afterCreate'
+    ]);
+  });
+
+  describe('afterSync', () => {
+    test('singular name', async () => {
+      const collection = db.collection({
+        name: 't_test',
+      });
+      const spy = jest.fn();
+      db.on('t_test.afterSync', () => {
+        spy('afterSync');
+      });
+      await collection.sync();
+      expect(spy).toHaveBeenCalledTimes(1);
+    });
+
+    test('plural name', async () => {
+      const collection = db.collection({
+        name: 't_tests',
+      });
+      const spy = jest.fn();
+      db.on('t_tests.afterSync', () => {
+        spy('afterSync');
+      });
+      await collection.sync();
+      expect(spy).toHaveBeenCalledTimes(1);
+    });
+  });
+});

package/src/__tests__/sort.test.ts

@@ -0,0 +1,51 @@
+import { Database } from '../database';
+import { mockDatabase } from './';
+
+describe('sort', function () {
+  let db: Database;
+
+  beforeEach(async () => {
+    db = mockDatabase();
+  });
+
+  afterEach(async () => {
+    await db.close();
+  });
+
+  test('order nulls last', async () => {
+    const Test = db.collection({
+      name: 'test',
+      fields: [
+        {
+          type: 'integer',
+          name: 'num',
+        },
+      ],
+    });
+    await Test.sync();
+    await Test.model.bulkCreate([
+      {
+        num: 3,
+      },
+      {
+        num: 2,
+      },
+      {
+        num: null,
+      },
+      {
+        num: 1,
+      },
+    ]);
+    const items = await Test.repository.find({
+      sort: '-num',
+    });
+    const nums = items.map((item) => item.get('num'));
+    expect(nums).toEqual([3,2,1,null]);
+    const items2 = await Test.repository.find({
+      sort: 'num',
+    });
+    const nums2 = items2.map((item) => item.get('num'));
+    expect(nums2).toEqual([1,2,3,null]);
+  });
+});

package/src/__tests__/update-associations.test.ts

@@ -173,7 +173,7 @@ describe('update associations', () => {
       await updateAssociations(user1, {
         posts: post1,
       });
-      console.log(JSON.stringify(user1, null, 2));
+
       expect(user1.toJSON()).toMatchObject({
         name: 'user1',
       });
@@ -191,7 +191,7 @@ describe('update associations', () => {
           name: 'post111',
         },
       });
-      console.log(JSON.stringify(user1, null, 2));
+
       expect(user1.toJSON()).toMatchObject({
         name: 'user1',
       });
@@ -216,7 +216,7 @@ describe('update associations', () => {
           post3,
         ],
       });
-      console.log(JSON.stringify(user1, null, 2));
+
       expect(user1.toJSON()).toMatchObject({
         name: 'user1',
       });

package/src/collection-importer.ts

@@ -1,20 +1,10 @@
-import * as fs from 'fs';
-import lodash from 'lodash';
 import path from 'path';
+import { readdir } from 'fs/promises';
+import { isPlainObject } from 'lodash';
+import { requireModule } from '@nocobase/utils';
 
 export type ImportFileExtension = 'js' | 'ts' | 'json';
 
-async function requireModule(module: any) {
-  if (typeof module === 'string') {
-    module = require(module);
-  }
-
-  if (typeof module !== 'object') {
-    return module;
-  }
-  return module.__esModule ? module.default : module;
-}
-
 export class ImporterReader {
   directory: string;
   extensions: Set<string>;
@@ -30,11 +20,10 @@ export class ImporterReader {
   }
 
   async read() {
-    const modules = (
-      await fs.promises.readdir(this.directory, {
-        encoding: 'utf-8',
-      })
-    )
+    const files = await readdir(this.directory, {
+      encoding: 'utf-8',
+    });
+    const modules = files
       .filter((fileName) => {
         if (fileName.endsWith('.d.ts')) {
           return false;
@@ -42,8 +31,11 @@ export class ImporterReader {
         const ext = path.parse(fileName).ext.replace('.', '');
         return this.extensions.has(ext);
       })
-      .map(async (fileName) => await requireModule(path.join(this.directory, fileName)));
+      .map((fileName) => {
+        const mod = requireModule(path.join(this.directory, fileName));
+        return typeof mod === 'function' ? mod() : mod;
+      });
 
-    return (await Promise.all(modules)).filter((module) => lodash.isPlainObject(module));
+    return (await Promise.all(modules)).filter((module) => isPlainObject(module));
   }
 }

package/src/collection.ts

@@ -1,7 +1,7 @@
 import merge from 'deepmerge';
 import { EventEmitter } from 'events';
 import { default as lodash, default as _ } from 'lodash';
-import { col, ModelCtor, ModelOptions, SyncOptions } from 'sequelize';
+import { ModelCtor, ModelOptions, QueryInterfaceDropTableOptions, SyncOptions, Transactionable } from 'sequelize';
 import { Database } from './database';
 import { Field, FieldOptions } from './fields';
 import { Model } from './model';
@@ -53,6 +53,10 @@ export class Collection<
     return this.options.name;
   }
 
+  get db() {
+    return this.context.database;
+  }
+
   constructor(options: CollectionOptions, context?: CollectionContext) {
     super();
     this.context = context;
@@ -186,6 +190,26 @@ export class Collection<
     }
   }
 
+  remove() {
+    this.context.database.removeCollection(this.name);
+  }
+
+  async removeFromDb(options?: QueryInterfaceDropTableOptions) {
+    if (
+      await this.existsInDb({
+        transaction: options?.transaction,
+      })
+    ) {
+      const queryInterface = this.db.sequelize.getQueryInterface();
+      await queryInterface.dropTable(this.model.tableName, options);
+    }
+    this.remove();
+  }
+
+  async existsInDb(options?: Transactionable) {
+    return this.db.collectionExistsInDb(this.name, options);
+  }
+
   removeField(name) {
     if (!this.fields.has(name)) {
       return;
@@ -195,7 +219,7 @@ export class Collection<
     if (bool) {
       this.emit('field.afterRemove', field);
     }
-    return bool;
+    return field as Field;
   }
 
   /**

package/src/database.ts

@@ -1,8 +1,9 @@
 import { applyMixins, AsyncEmitter } from '@nocobase/utils';
 import merge from 'deepmerge';
 import { EventEmitter } from 'events';
+import glob from 'glob';
 import lodash from 'lodash';
-import { isAbsolute, resolve } from 'path';
+import { basename, isAbsolute, resolve } from 'path';
 import {
   ModelCtor,
   Op,
@@ -11,12 +12,15 @@ import {
   QueryOptions,
   Sequelize,
   SyncOptions,
+  Transactionable,
   Utils
 } from 'sequelize';
+import { SequelizeStorage, Umzug } from 'umzug';
 import { Collection, CollectionOptions, RepositoryType } from './collection';
 import { ImporterReader, ImportFileExtension } from './collection-importer';
 import * as FieldTypes from './fields';
 import { Field, FieldContext, RelationField } from './fields';
+import { Migrations } from './migration';
 import { Model } from './model';
 import { ModelHook } from './model-hook';
 import extendOperators from './operators';
@@ -36,9 +40,10 @@ interface MapOf<T> {
 
 export interface IDatabaseOptions extends Options {
   tablePrefix?: string;
+  migrator?: any;
 }
 
-export type DatabaseOptions = IDatabaseOptions | Sequelize;
+export type DatabaseOptions = IDatabaseOptions;
 
 interface RegisterOperatorsContext {
   db?: Database;
@@ -51,10 +56,19 @@ export interface CleanOptions extends QueryInterfaceDropAllTablesOptions {
   drop?: boolean;
 }
 
+export type AddMigrationsOptions = {
+  context?: any;
+  namespace?: string;
+  extensions?: string[];
+  directory: string;
+};
+
 type OperatorFunc = (value: any, ctx?: RegisterOperatorsContext) => any;
 
 export class Database extends EventEmitter implements AsyncEmitter {
   sequelize: Sequelize;
+  migrator: Umzug;
+  migrations: Migrations;
   fieldTypes = new Map();
   options: IDatabaseOptions;
   models = new Map<string, ModelCtor<Model>>();
@@ -71,27 +85,30 @@ export class Database extends EventEmitter implements AsyncEmitter {
   constructor(options: DatabaseOptions) {
     super();
 
-    if (options instanceof Sequelize) {
-      this.sequelize = options;
-    } else {
-      const opts = {
-        sync: {
-          alter: {
-            drop: false,
-          },
-          force: false,
+    const opts = {
+      sync: {
+        alter: {
+          drop: false,
         },
-        ...options,
-      };
-      if (options.storage && options.storage !== ':memory:') {
-        if (!isAbsolute(options.storage)) {
-          opts.storage = resolve(process.cwd(), options.storage);
-        }
+        force: false,
+      },
+      ...options,
+    };
+
+    if (options.storage && options.storage !== ':memory:') {
+      if (!isAbsolute(options.storage)) {
+        opts.storage = resolve(process.cwd(), options.storage);
       }
-      this.sequelize = new Sequelize(opts);
-      this.options = opts;
     }
 
+    if (options.dialect === 'sqlite') {
+      delete opts.timezone;
+    } else if (!opts.timezone) {
+      opts.timezone = '+00:00';
+    }
+
+    this.sequelize = new Sequelize(opts);
+    this.options = opts;
     this.collections = new Map();
     this.modelHook = new ModelHook(this);
 
@@ -116,6 +133,62 @@ export class Database extends EventEmitter implements AsyncEmitter {
     }
 
     this.initOperators();
+
+    const migratorOptions: any = this.options.migrator || {};
+
+    const context = {
+      db: this,
+      sequelize: this.sequelize,
+      queryInterface: this.sequelize.getQueryInterface(),
+      ...migratorOptions.context,
+    };
+
+    this.migrations = new Migrations(context);
+    this.migrator = new Umzug({
+      logger: migratorOptions.logger || console,
+      migrations: this.migrations.callback(),
+      context,
+      storage: new SequelizeStorage({
+        modelName: `${this.options.tablePrefix || ''}migrations`,
+        ...migratorOptions.storage,
+        sequelize: this.sequelize,
+      }),
+    });
+  }
+
+  addMigration(item) {
+    return this.migrations.add(item);
+  }
+
+  addMigrations(options: AddMigrationsOptions) {
+    const { namespace, context, extensions = ['js', 'ts'], directory } = options;
+    const patten = `${directory}/*.{${extensions.join(',')}}`;
+    const files = glob.sync(patten, {
+      ignore: ['**/*.d.ts'],
+    });
+    for (const file of files) {
+      let filename = basename(file);
+      filename = filename.substring(0, filename.lastIndexOf('.')) || filename;
+      this.migrations.add({
+        name: namespace ? `${namespace}/${filename}` : filename,
+        migration: this.requireModule(file),
+        context,
+      });
+    }
+  }
+
+  inDialect(...dialect: string[]) {
+    return dialect.includes(this.sequelize.getDialect());
+  }
+
+  private requireModule(module: any) {
+    if (typeof module === 'string') {
+      module = require(module);
+    }
+    if (typeof module !== 'object') {
+      return module;
+    }
+    return module.__esModule ? module.default : module;
   }
 
   /**
@@ -161,9 +234,13 @@ export class Database extends EventEmitter implements AsyncEmitter {
 
     const result = this.collections.delete(name);
 
+    this.sequelize.modelManager.removeModel(collection.model);
+
     if (result) {
       this.emit('afterRemoveCollection', collection);
     }
+
+    return collection;
   }
 
   getModel<M extends Model>(name: string) {
@@ -268,12 +345,19 @@ export class Database extends EventEmitter implements AsyncEmitter {
     }
   }
 
+  async collectionExistsInDb(name, options?: Transactionable) {
+    const tables = await this.sequelize.getQueryInterface().showAllTables({
+      transaction: options?.transaction,
+    });
+    return !!tables.find((table) => table === `${this.getTablePrefix()}${name}`);
+  }
+
   public isSqliteMemory() {
     return this.sequelize.getDialect() === 'sqlite' && lodash.get(this.options, 'storage') == ':memory:';
   }
 
-  async auth(options: QueryOptions & { repeat?: number } = {}) {
-    const { repeat = 10, ...others } = options;
+  async auth(options: QueryOptions & { retry?: number } = {}) {
+    const { retry = 10, ...others } = options;
     const delay = (ms) => new Promise((yea) => setTimeout(yea, ms));
     let count = 1;
     const authenticate = async () => {
@@ -282,7 +366,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
         console.log('Connection has been established successfully.');
         return true;
       } catch (error) {
-        if (count >= repeat) {
+        if (count >= retry) {
           throw error;
         }
         console.log('reconnecting...', count);
@@ -291,7 +375,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
         return await authenticate();
       }
     };
-
     return await authenticate();
   }
 
@@ -322,13 +405,13 @@ export class Database extends EventEmitter implements AsyncEmitter {
     return this.sequelize.close();
   }
 
-  on(event: string | symbol, listener: (...args: any[]) => void): this {
-    const modelEventName = this.modelHook.isModelHook(event);
-
-    if (modelEventName && !this.modelHook.hasBindEvent(modelEventName)) {
-      this.sequelize.addHook(modelEventName, this.modelHook.sequelizeHookBuilder(modelEventName));
+  on(event: string | symbol, listener): this {
+    // NOTE: to match if event is a sequelize or model type
+    const type = this.modelHook.match(event);
 
-      this.modelHook.bindEvent(modelEventName);
+    if (type && !this.modelHook.hasBoundEvent(type)) {
+      this.sequelize.addHook(type, this.modelHook.buildSequelizeHook(type));
+      this.modelHook.bindEvent(type);
     }
 
     return super.on(event, listener);

package/src/fields/field.ts

@@ -1,5 +1,12 @@
 import _ from 'lodash';
-import { DataType, ModelAttributeColumnOptions, ModelIndexesOptions, SyncOptions } from 'sequelize';
+import {
+  DataType,
+  ModelAttributeColumnOptions,
+  ModelIndexesOptions,
+  QueryInterfaceOptions,
+  SyncOptions,
+  Transactionable
+} from 'sequelize';
 import { Collection } from '../collection';
 import { Database } from '../database';
 
@@ -75,6 +82,86 @@ export abstract class Field {
     return this.options[name];
   }
 
+  remove() {
+    return this.collection.removeField(this.name);
+  }
+
+  async removeFromDb(options?: QueryInterfaceOptions) {
+    if (!this.collection.model.rawAttributes[this.name]) {
+      this.remove();
+      // console.log('field is not attribute');
+      return;
+    }
+    if ((this.collection.model as any)._virtualAttributes.has(this.name)) {
+      this.remove();
+      // console.log('field is virtual attribute');
+      return;
+    }
+    if (this.collection.model.primaryKeyAttributes.includes(this.name)) {
+      // 主键不能删除
+      return;
+    }
+    if (this.collection.model.options.timestamps !== false) {
+      // timestamps 相关字段不删除
+      if (['createdAt', 'updatedAt', 'deletedAt'].includes(this.name)) {
+        return;
+      }
+    }
+    // 排序字段通过 sortable 控制
+    const sortable = this.collection.options.sortable;
+    if (sortable) {
+      let sortField: string;
+      if (sortable === true) {
+        sortField = 'sort';
+      } else if (typeof sortable === 'string') {
+        sortField = sortable;
+      } else if (sortable.name) {
+        sortField = sortable.name || 'sort';
+      }
+      if (this.name === sortField) {
+        return;
+      }
+    }
+    if (this.options.field && this.name !== this.options.field) {
+      // field 指向的是真实的字段名，如果与 name 不一样，说明字段只是引用
+      this.remove();
+      return;
+    }
+    if (
+      await this.existsInDb({
+        transaction: options?.transaction,
+      })
+    ) {
+      const queryInterface = this.database.sequelize.getQueryInterface();
+      await queryInterface.removeColumn(this.collection.model.tableName, this.name, options);
+    }
+    this.remove();
+  }
+
+  async existsInDb(options?: Transactionable) {
+    const opts = {
+      transaction: options?.transaction,
+    };
+    let sql;
+    if (this.database.sequelize.getDialect() === 'sqlite') {
+      sql = `SELECT * from pragma_table_info('${this.collection.model.tableName}') WHERE name = '${this.name}'`;
+    } else if (this.database.inDialect('mysql')) {
+      sql = `
+        select column_name
+        from INFORMATION_SCHEMA.COLUMNS
+        where TABLE_SCHEMA='${this.database.options.database}' AND TABLE_NAME='${this.collection.model.tableName}' AND column_name='${this.name}'
+      `;
+    } else {
+      sql = `
+        select column_name
+        from INFORMATION_SCHEMA.COLUMNS
+        where TABLE_NAME='${this.collection.model.tableName}' AND column_name='${this.name}'
+      `;
+    }
+    const [rows] = await this.database.sequelize.query(sql, opts);
+    return rows.length > 0;
+  }
+
   merge(obj: any) {
     Object.assign(this.options, obj);
   }