@nocobase/database 0.9.0-alpha.2 → 0.9.1-alpha.1

package/src/collection.ts

@@ -3,17 +3,17 @@ import { EventEmitter } from 'events';
 import { default as lodash, default as _ } from 'lodash';
 import {
   ModelOptions,
+  ModelStatic,
   QueryInterfaceDropTableOptions,
   SyncOptions,
   Transactionable,
-  ModelStatic,
   Utils,
 } from 'sequelize';
 import { Database } from './database';
 import { Field, FieldOptions } from './fields';
 import { Model } from './model';
 import { Repository } from './repository';
-import { checkIdentifier, md5 } from './utils';
+import { checkIdentifier, md5, snakeCase } from './utils';
 
 export type RepositoryType = typeof Repository;
 
@@ -36,6 +36,7 @@ export interface CollectionOptions extends Omit<ModelOptions, 'name' | 'hooks'>
    * @default 'options'
    */
   magicAttribute?: string;
+
   [key: string]: any;
 }
 
@@ -62,17 +63,21 @@ export class Collection<
     return this.options.name;
   }
 
+  get titleField() {
+    return (this.options.titleField as string) || this.model.primaryKeyAttribute;
+  }
+
   get db() {
     return this.context.database;
   }
 
   constructor(options: CollectionOptions, context: CollectionContext) {
     super();
-    this.checkOptions(options);
-
     this.context = context;
     this.options = options;
 
+    this.checkOptions(options);
+
     this.bindFieldEventListener();
     this.modelInit();
 
@@ -89,15 +94,31 @@ export class Collection<
 
   private checkOptions(options: CollectionOptions) {
     checkIdentifier(options.name);
+    this.checkTableName();
   }
 
-  private sequelizeModelOptions() {
+  private checkTableName() {
+    const tableName = this.tableName();
+    for (const [k, collection] of this.db.collections) {
+      if (collection.name != this.options.name && tableName === collection.tableName()) {
+        throw new Error(`collection ${collection.name} and ${this.name} have same tableName "${tableName}"`);
+      }
+    }
+  }
+
+  tableName() {
     const { name, tableName } = this.options;
+    const tName = tableName || name;
+    return this.db.options.underscored ? snakeCase(tName) : tName;
+  }
+
+  private sequelizeModelOptions() {
+    const { name } = this.options;
     return {
       ..._.omit(this.options, ['name', 'fields', 'model', 'targetKey']),
       modelName: name,
       sequelize: this.context.database.sequelize,
-      tableName: tableName || name,
+      tableName: this.tableName(),
     };
   }
 
@@ -108,8 +129,10 @@ export class Collection<
     if (this.model) {
       return;
     }
+
     const { name, model, autoGenId = true } = this.options;
     let M: ModelStatic<Model> = Model;
+
     if (this.context.database.sequelize.isDefined(name)) {
       const m = this.context.database.sequelize.model(name);
       if ((m as any).isThrough) {
@@ -122,11 +145,13 @@ export class Collection<
         return;
       }
     }
+
     if (typeof model === 'string') {
       M = this.context.database.models.get(model) || Model;
     } else if (model) {
       M = model;
     }
+
     // @ts-ignore
     this.model = class extends M {};
     this.model.init(null, this.sequelizeModelOptions());
@@ -179,8 +204,31 @@ export class Collection<
     return this.setField(name, options);
   }
 
+  checkFieldType(name: string, options: FieldOptions) {
+    if (!this.db.options.underscored) {
+      return;
+    }
+    const fieldName = options.field || snakeCase(name);
+    const field = this.findField((f) => {
+      if (f.name === name) {
+        return false;
+      }
+      if (f.field) {
+        return f.field === fieldName;
+      }
+      return snakeCase(f.name) === fieldName;
+    });
+    if (!field) {
+      return;
+    }
+    if (options.type !== field.type) {
+      throw new Error(`fields with same column must be of the same type ${JSON.stringify(options)}`);
+    }
+  }
+
   setField(name: string, options: FieldOptions): Field {
     checkIdentifier(name);
+    this.checkFieldType(name, options);
 
     const { database } = this.context;
 
@@ -305,6 +353,7 @@ export class Collection<
     newOptions = merge(this.options, newOptions, mergeOptions);
 
     this.context.database.emit('beforeUpdateCollection', this, newOptions);
+    this.options = newOptions;
 
     this.setFields(options.fields, false);
     this.setRepository(options.repository);
@@ -357,9 +406,13 @@ export class Collection<
     if (!index) {
       return;
     }
+
+    // collection defined indexes
     let indexes: any = this.model.options.indexes || [];
+
     let indexName = [];
     let indexItem;
+
     if (typeof index === 'string') {
       indexItem = {
         fields: [index],
@@ -374,13 +427,17 @@ export class Collection<
       indexItem = index;
       indexName = index.fields;
     }
+
     if (lodash.isEqual(this.model.primaryKeyAttributes, indexName)) {
       return;
     }
+
     const name: string = this.model.primaryKeyAttributes.join(',');
+
     if (name.startsWith(`${indexName.join(',')},`)) {
       return;
     }
+
     for (const item of indexes) {
       if (lodash.isEqual(item.fields, indexName)) {
         return;
@@ -390,11 +447,13 @@ export class Collection<
         return;
       }
     }
+
     if (!indexItem) {
       return;
     }
+
     indexes.push(indexItem);
-    this.model.options.indexes = indexes;
+
     const tableName = this.model.getTableName();
     // @ts-ignore
     this.model._indexes = this.model.options.indexes
@@ -406,6 +465,7 @@ export class Collection<
         }
         return item;
       });
+
     this.refreshIndexes();
   }
 
@@ -425,15 +485,23 @@ export class Collection<
   refreshIndexes() {
     // @ts-ignore
     const indexes: any[] = this.model._indexes;
+
     // @ts-ignore
-    this.model._indexes = indexes.filter((item) => {
-      for (const field of item.fields) {
-        if (!this.model.rawAttributes[field]) {
-          return false;
-        }
-      }
-      return true;
-    });
+    this.model._indexes = lodash.uniqBy(
+      indexes
+        .filter((item) => {
+          return item.fields.every((field) =>
+            Object.values(this.model.rawAttributes).find((fieldVal) => fieldVal.field === field),
+          );
+        })
+        .map((item) => {
+          if (this.options.underscored) {
+            item.fields = item.fields.map((field) => snakeCase(field));
+          }
+          return item;
+        }),
+      'name',
+    );
   }
 
   async sync(syncOptions?: SyncOptions) {
@@ -470,4 +538,18 @@ export class Collection<
   public isParent() {
     return this.context.database.inheritanceMap.isParentNode(this.name);
   }
+
+  public addSchemaTableName() {
+    const tableName = this.model.tableName;
+
+    if (this.options.schema) {
+      return this.db.utils.addSchema(tableName, this.options.schema);
+    }
+
+    return tableName;
+  }
+
+  public quotedTableName() {
+    return this.db.utils.quoteTable(this.addSchemaTableName());
+  }
 }

package/src/database-utils/index.ts

@@ -0,0 +1,38 @@
+import Database from '../database';
+import lodash from 'lodash';
+
+export default class DatabaseUtils {
+  constructor(public db: Database) {}
+
+  addSchema(tableName, schema?) {
+    if (this.db.options.schema && !schema) {
+      schema = this.db.options.schema;
+    }
+
+    if (schema) {
+      // @ts-ignore
+      tableName = this.db.sequelize.getQueryInterface().queryGenerator.addSchema({
+        tableName,
+        _schema: schema,
+      });
+    }
+
+    return tableName;
+  }
+
+  quoteTable(tableName) {
+    const queryGenerator = this.db.sequelize.getQueryInterface().queryGenerator;
+    // @ts-ignore
+    tableName = queryGenerator.quoteTable(lodash.isPlainObject(tableName) ? tableName : this.addSchema(tableName));
+
+    return tableName;
+  }
+
+  schema() {
+    if (!this.db.inDialect('postgres')) {
+      return undefined;
+    }
+
+    return this.db.options.schema || 'public';
+  }
+}

package/src/database.ts

@@ -15,7 +15,7 @@ import {
   Sequelize,
   SyncOptions,
   Transactionable,
-  Utils
+  Utils,
 } from 'sequelize';
 import { SequelizeStorage, Umzug } from 'umzug';
 import { Collection, CollectionOptions, RepositoryType } from './collection';
@@ -58,8 +58,12 @@ import {
   SyncListener,
   UpdateListener,
   UpdateWithAssociationsListener,
-  ValidateListener
+  ValidateListener,
 } from './types';
+import { patchSequelizeQueryInterface, snakeCase } from './utils';
+
+import DatabaseUtils from './database-utils';
+import { BaseValueParser, registerFieldValueParsers } from './value-parsers';
 
 export interface MergeOptions extends merge.Options {}
 
@@ -76,6 +80,7 @@ export interface IDatabaseOptions extends Options {
   tablePrefix?: string;
   migrator?: any;
   usingBigIntForId?: boolean;
+  underscored?: boolean;
 }
 
 export type DatabaseOptions = IDatabaseOptions;
@@ -100,6 +105,30 @@ export type AddMigrationsOptions = {
 
 type OperatorFunc = (value: any, ctx?: RegisterOperatorsContext) => any;
 
+export const DialectVersionAccessors = {
+  sqlite: {
+    sql: 'select sqlite_version() as version',
+    get: (v: string) => v,
+  },
+  mysql: {
+    sql: 'select version() as version',
+    get: (v: string) => {
+      if (v.toLowerCase().includes('mariadb')) {
+        return '';
+      }
+      const m = /([\d+\.]+)/.exec(v);
+      return m[0];
+    },
+  },
+  postgres: {
+    sql: 'select version() as version',
+    get: (v: string) => {
+      const m = /([\d+\.]+)/.exec(v);
+      return semver.minVersion(m[0]).version;
+    },
+  },
+};
+
 class DatabaseVersion {
   db: Database;
 
@@ -108,33 +137,14 @@ class DatabaseVersion {
   }
 
   async satisfies(versions) {
-    const dialects = {
-      sqlite: {
-        sql: 'select sqlite_version() as version',
-        get: (v) => v,
-      },
-      mysql: {
-        sql: 'select version() as version',
-        get: (v) => {
-          const m = /([\d+\.]+)/.exec(v);
-          return m[0];
-        },
-      },
-      postgres: {
-        sql: 'select version() as version',
-        get: (v) => {
-          const m = /([\d+\.]+)/.exec(v);
-          return semver.minVersion(m[0]).version;
-        },
-      },
-    };
-    for (const dialect of Object.keys(dialects)) {
+    const accessors = DialectVersionAccessors;
+    for (const dialect of Object.keys(accessors)) {
       if (this.db.inDialect(dialect)) {
         if (!versions?.[dialect]) {
           return false;
         }
-        const [result] = (await this.db.sequelize.query(dialects[dialect].sql)) as any;
-        return semver.satisfies(dialects[dialect].get(result?.[0]?.version), versions[dialect]);
+        const [result] = (await this.db.sequelize.query(accessors[dialect].sql)) as any;
+        return semver.satisfies(accessors[dialect].get(result?.[0]?.version), versions[dialect]);
       }
     }
     return false;
@@ -146,6 +156,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
   migrator: Umzug;
   migrations: Migrations;
   fieldTypes = new Map();
+  fieldValueParsers = new Map();
   options: IDatabaseOptions;
   models = new Map<string, ModelStatic<Model>>();
   repositories = new Map<string, RepositoryType>();
@@ -155,6 +166,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
   modelCollection = new Map<ModelStatic<any>, Collection>();
   tableNameCollectionMap = new Map<string, Collection>();
 
+  utils = new DatabaseUtils(this);
   referenceMap = new ReferencesMap();
   inheritanceMap = new InheritanceMap();
 
@@ -196,9 +208,10 @@ export class Database extends EventEmitter implements AsyncEmitter {
       // https://github.com/sequelize/sequelize/issues/1774
       require('pg').defaults.parseInt8 = true;
     }
+    this.options = opts;
 
     this.sequelize = new Sequelize(opts);
-    this.options = opts;
+
     this.collections = new Map();
     this.modelHook = new ModelHook(this);
 
@@ -211,7 +224,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
     });
 
     // register database field types
-    for (const [name, field] of Object.entries(FieldTypes)) {
+    for (const [name, field] of Object.entries<any>(FieldTypes)) {
       if (['Field', 'RelationField'].includes(name)) {
         continue;
       }
@@ -222,6 +235,8 @@ export class Database extends EventEmitter implements AsyncEmitter {
       });
     }
 
+    registerFieldValueParsers(this);
+
     this.initOperators();
 
     const migratorOptions: any = this.options.migrator || {};
@@ -250,6 +265,8 @@ export class Database extends EventEmitter implements AsyncEmitter {
       name: 'migrations',
       autoGenId: false,
       timestamps: false,
+      namespace: 'core',
+      duplicator: 'required',
       fields: [{ type: 'string', name: 'name' }],
     });
 
@@ -260,9 +277,16 @@ export class Database extends EventEmitter implements AsyncEmitter {
     });
 
     this.initListener();
+    patchSequelizeQueryInterface(this);
   }
 
   initListener() {
+    this.on('beforeDefine', (model, options) => {
+      if (this.options.underscored) {
+        options.underscored = true;
+      }
+    });
+
     this.on('afterCreate', async (instance) => {
       instance?.toChangedWithAssociations?.();
     });
@@ -292,6 +316,37 @@ export class Database extends EventEmitter implements AsyncEmitter {
         }
       }
     });
+
+    this.on('afterUpdateCollection', (collection, options) => {
+      if (collection.options.schema) {
+        collection.model._schema = collection.options.schema;
+      }
+    });
+
+    this.on('beforeDefineCollection', (options) => {
+      if (options.underscored) {
+        if (lodash.get(options, 'sortable.scopeKey')) {
+          options.sortable.scopeKey = snakeCase(options.sortable.scopeKey);
+        }
+
+        if (lodash.get(options, 'indexes')) {
+          // change index fields to snake case
+          options.indexes = options.indexes.map((index) => {
+            if (index.fields) {
+              index.fields = index.fields.map((field) => {
+                return snakeCase(field);
+              });
+            }
+
+            return index;
+          });
+        }
+      }
+
+      if (this.options.schema && !options.schema) {
+        options.schema = this.options.schema;
+      }
+    });
   }
 
   addMigration(item: MigrationItem) {
@@ -326,6 +381,10 @@ export class Database extends EventEmitter implements AsyncEmitter {
   collection<Attributes = any, CreateAttributes = Attributes>(
     options: CollectionOptions,
   ): Collection<Attributes, CreateAttributes> {
+    if (this.options.underscored) {
+      options.underscored = true;
+    }
+
     this.emit('beforeDefineCollection', options);
 
     const hasValidInheritsOptions = (() => {
@@ -351,6 +410,31 @@ export class Database extends EventEmitter implements AsyncEmitter {
     return this.options.tablePrefix || '';
   }
 
+  getFieldByPath(path: string) {
+    if (!path) {
+      return;
+    }
+
+    const [collectionName, associationName, ...args] = path.split('.');
+    let collection = this.getCollection(collectionName);
+
+    if (!collection) {
+      return;
+    }
+
+    const field = collection.getField(associationName);
+
+    if (!field) {
+      return;
+    }
+
+    if (args.length > 0) {
+      return this.getFieldByPath(`${field?.target}.${args.join('.')}`);
+    }
+
+    return field;
+  }
+
   /**
    * get exists collection by its name
    * @param name
@@ -430,6 +514,20 @@ export class Database extends EventEmitter implements AsyncEmitter {
     }
   }
 
+  registerFieldValueParsers(parsers: MapOf<any>) {
+    for (const [type, parser] of Object.entries(parsers)) {
+      this.fieldValueParsers.set(type, parser);
+    }
+  }
+
+  buildFieldValueParser<T extends BaseValueParser>(field: Field, ctx: any) {
+    const Parser = this.fieldValueParsers.has(field.type)
+      ? this.fieldValueParsers.get(field.type)
+      : this.fieldValueParsers.get('default');
+    const parser = new Parser(field, ctx);
+    return parser as T;
+  }
+
   registerModels(models: MapOf<ModelStatic<any>>) {
     for (const [type, schemaType] of Object.entries(models)) {
       this.models.set(type, schemaType);
@@ -475,6 +573,10 @@ export class Database extends EventEmitter implements AsyncEmitter {
       throw Error(`unsupported field type ${type}`);
     }
 
+    if (options.field && this.options.underscored) {
+      options.field = snakeCase(options.field);
+    }
+
     return new Field(options, context);
   }
 
@@ -484,6 +586,10 @@ export class Database extends EventEmitter implements AsyncEmitter {
       await this.sequelize.query('SET FOREIGN_KEY_CHECKS = 0', null);
     }
 
+    if (this.options.schema && this.inDialect('postgres')) {
+      await this.sequelize.query(`CREATE SCHEMA IF NOT EXISTS "${this.options.schema}"`, null);
+    }
+
     const result = await this.sequelize.sync(options);
 
     if (isMySQL) {
@@ -494,24 +600,49 @@ export class Database extends EventEmitter implements AsyncEmitter {
   }
 
   async clean(options: CleanOptions) {
-    const { drop, ...others } = options;
-    if (drop) {
-      await this.sequelize.getQueryInterface().dropAllTables(others);
+    const { drop, ...others } = options || {};
+    if (drop !== true) {
+      return;
     }
+
+    if (this.options.schema) {
+      const tableNames = (await this.sequelize.getQueryInterface().showAllTables()).map((table) => {
+        return `"${this.options.schema}"."${table}"`;
+      });
+
+      const skip = options.skip || [];
+
+      // @ts-ignore
+      for (const tableName of tableNames) {
+        if (skip.includes(tableName)) {
+          continue;
+        }
+        await this.sequelize.query(`DROP TABLE IF EXISTS ${tableName} CASCADE`);
+      }
+      return;
+    }
+
+    await this.sequelize.getQueryInterface().dropAllTables(others);
   }
 
-  async collectionExistsInDb(name, options?: Transactionable) {
+  async collectionExistsInDb(name: string, options?: Transactionable) {
+    const collection = this.getCollection(name);
+    if (!collection) {
+      return false;
+    }
+
     const tables = await this.sequelize.getQueryInterface().showAllTables({
       transaction: options?.transaction,
     });
-    return !!tables.find((table) => table === `${this.getTablePrefix()}${name}`);
+
+    return tables.includes(this.getCollection(name).model.tableName);
   }
 
   public isSqliteMemory() {
     return this.sequelize.getDialect() === 'sqlite' && lodash.get(this.options, 'storage') == ':memory:';
   }
 
-  async auth(options: QueryOptions & { retry?: number } = {}) {
+  async auth(options: Omit<QueryOptions, 'retry'> & { retry?: number | Pick<QueryOptions, 'retry'> } = {}) {
     const { retry = 10, ...others } = options;
     const delay = (ms) => new Promise((yea) => setTimeout(yea, ms));
     let count = 1;
@@ -533,6 +664,12 @@ export class Database extends EventEmitter implements AsyncEmitter {
     return await authenticate();
   }
 
+  async prepare() {
+    if (this.inDialect('postgres') && this.options.schema && this.options.schema != 'public') {
+      await this.sequelize.query(`CREATE SCHEMA IF NOT EXISTS "${this.options.schema}"`, null);
+    }
+  }
+
   async reconnect() {
     if (this.isSqliteMemory()) {
       return;
@@ -589,6 +726,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
   }
 
   extendCollection(collectionOptions: CollectionOptions, mergeOptions?: MergeOptions) {
+    collectionOptions = lodash.cloneDeep(collectionOptions);
     const collectionName = collectionOptions.name;
     const existCollection = this.getCollection(collectionName);
     if (existCollection) {

package/src/fields/array-field.ts

@@ -1,5 +1,5 @@
-import { BaseColumnFieldOptions, Field } from './field';
 import { DataTypes } from 'sequelize';
+import { BaseColumnFieldOptions, Field } from './field';
 
 export class ArrayField extends Field {
   get dataType() {

package/src/fields/belongs-to-field.ts

@@ -1,5 +1,5 @@
 import { omit } from 'lodash';
-import { BelongsTo, BelongsToOptions as SequelizeBelongsToOptions, Utils } from 'sequelize';
+import { BelongsToOptions as SequelizeBelongsToOptions, Utils } from 'sequelize';
 import { Reference } from '../features/ReferencesMap';
 import { checkIdentifier } from '../utils';
 import { BaseRelationFieldOptions, RelationField } from './relation-field';

package/src/fields/belongs-to-many-field.ts

@@ -57,7 +57,6 @@ export class BelongsToManyField extends RelationField {
     } else {
       Through = database.collection({
         name: through,
-        // timestamps: false,
       });
 
       Object.defineProperty(Through.model, 'isThrough', { value: true });