@nocobase/database 0.9.3-alpha.1 → 0.9.4-alpha.1

package/src/database.ts

@@ -62,13 +62,13 @@ import {
 } from './types';
 import { patchSequelizeQueryInterface, snakeCase } from './utils';
 
+import { Logger } from '@nocobase/logger';
+import { CollectionGroupManager } from './collection-group-manager';
 import DatabaseUtils from './database-utils';
 import { registerBuiltInListeners } from './listeners';
-import { BaseValueParser, registerFieldValueParsers } from './value-parsers';
-import buildQueryInterface from './query-interface/query-interface-builder';
 import QueryInterface from './query-interface/query-interface';
-import { Logger } from '@nocobase/logger';
-import { CollectionGroupManager } from './collection-group-manager';
+import buildQueryInterface from './query-interface/query-interface-builder';
+import { BaseValueParser, registerFieldValueParsers } from './value-parsers';
 import { ViewCollection } from './view-collection';
 
 export type MergeOptions = merge.Options;
@@ -188,6 +188,7 @@ export class Database extends EventEmitter implements AsyncEmitter {
   logger: Logger;
 
   collectionGroupManager = new CollectionGroupManager(this);
+  declare emitAsync: (event: string | symbol, ...args: any[]) => Promise<boolean>;
 
   constructor(options: DatabaseOptions) {
     super();
@@ -275,6 +276,12 @@ export class Database extends EventEmitter implements AsyncEmitter {
       }),
     });
 
+    this.sequelize.beforeDefine((model, opts) => {
+      if (this.options.tablePrefix) {
+        opts.tableName = `${this.options.tablePrefix}${opts.tableName || opts.modelName || opts.name.plural}`;
+      }
+    });
+
     this.collection({
       name: 'migrations',
       autoGenId: false,
@@ -284,12 +291,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
       fields: [{ type: 'string', name: 'name' }],
     });
 
-    this.sequelize.beforeDefine((model, opts) => {
-      if (this.options.tablePrefix) {
-        opts.tableName = `${this.options.tablePrefix}${opts.tableName || opts.modelName || opts.name.plural}`;
-      }
-    });
-
     this.initListener();
     patchSequelizeQueryInterface(this);
   }
@@ -387,36 +388,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
       }
     });
 
-    this.on('afterRepositoryFind', ({ findOptions, dataCollection, data }) => {
-      if (dataCollection.isParent()) {
-        for (const row of data) {
-          const rowCollection = this.tableNameCollectionMap.get(
-            findOptions.raw
-              ? `${row['__schemaName']}.${row['__tableName']}`
-              : `${row.get('__schemaName')}.${row.get('__tableName')}`,
-          );
-
-          if (!rowCollection) {
-            this.logger.warn(
-              `Can not find collection by table name ${JSON.stringify(row)}, current collections: ${Array.from(
-                this.tableNameCollectionMap.keys(),
-              ).join(', ')}`,
-            );
-
-            return;
-          }
-
-          const rowCollectionName = rowCollection.name;
-
-          findOptions.raw
-            ? (row['__collection'] = rowCollectionName)
-            : row.set('__collection', rowCollectionName, {
-                raw: true,
-              });
-        }
-      }
-    });
-
     registerBuiltInListeners(this);
   }
 
@@ -565,7 +536,9 @@ export class Database extends EventEmitter implements AsyncEmitter {
   }
 
   getRepository<R extends Repository>(name: string): R;
+
   getRepository<R extends RelationRepository>(name: string, relationId: string | number): R;
+
   getRepository<R extends ArrayFieldRepository>(name: string, relationId: string | number): R;
 
   getRepository<R extends RelationRepository>(name: string, relationId?: string | number): Repository | R {
@@ -779,21 +752,34 @@ export class Database extends EventEmitter implements AsyncEmitter {
   }
 
   on(event: EventType, listener: any): this;
+
   on(event: ModelValidateEventTypes, listener: SyncListener): this;
+
   on(event: ModelValidateEventTypes, listener: ValidateListener): this;
+
   on(event: ModelCreateEventTypes, listener: CreateListener): this;
+
   on(event: ModelUpdateEventTypes, listener: UpdateListener): this;
+
   on(event: ModelSaveEventTypes, listener: SaveListener): this;
+
   on(event: ModelDestroyEventTypes, listener: DestroyListener): this;
+
   on(event: ModelCreateWithAssociationsEventTypes, listener: CreateWithAssociationsListener): this;
+
   on(event: ModelUpdateWithAssociationsEventTypes, listener: UpdateWithAssociationsListener): this;
+
   on(event: ModelSaveWithAssociationsEventTypes, listener: SaveWithAssociationsListener): this;
+
   on(event: DatabaseBeforeDefineCollectionEventType, listener: BeforeDefineCollectionListener): this;
+
   on(event: DatabaseAfterDefineCollectionEventType, listener: AfterDefineCollectionListener): this;
+
   on(
     event: DatabaseBeforeRemoveCollectionEventType | DatabaseAfterRemoveCollectionEventType,
     listener: RemoveCollectionListener,
   ): this;
+
   on(event: EventType, listener: any): this {
     // NOTE: to match if event is a sequelize or model type
     const type = this.modelHook.match(event);
@@ -844,8 +830,6 @@ export class Database extends EventEmitter implements AsyncEmitter {
 
     return result;
   }
-
-  declare emitAsync: (event: string | symbol, ...args: any[]) => Promise<boolean>;
 }
 
 export function extendCollection(collectionOptions: CollectionOptions, mergeOptions?: MergeOptions) {

package/src/eager-loading/eager-loading-tree.ts

@@ -0,0 +1,304 @@
+import { Association, Includeable, Model, ModelStatic, Transaction } from 'sequelize';
+import lodash from 'lodash';
+
+interface EagerLoadingNode {
+  model: ModelStatic<any>;
+  association: Association;
+  attributes: Array<string>;
+  rawAttributes: Array<string>;
+  children: Array<EagerLoadingNode>;
+  parent?: EagerLoadingNode;
+  instances?: Array<Model>;
+  order?: any;
+}
+
+export class EagerLoadingTree {
+  public root: EagerLoadingNode;
+
+  constructor(root: EagerLoadingNode) {
+    this.root = root;
+  }
+
+  static buildFromSequelizeOptions(options: {
+    model: ModelStatic<any>;
+    rootAttributes: Array<string>;
+    rootOrder?: any;
+    includeOption: Includeable | Includeable[];
+  }): EagerLoadingTree {
+    const { model, rootAttributes, includeOption } = options;
+
+    const root = {
+      model,
+      association: null,
+      rawAttributes: lodash.cloneDeep(rootAttributes),
+      attributes: lodash.cloneDeep(rootAttributes),
+      order: options.rootOrder,
+      children: [],
+    };
+
+    const pushAttribute = (node, attribute) => {
+      if (lodash.isArray(node.attributes) && !node.attributes.includes(attribute)) {
+        node.attributes.push(attribute);
+      }
+    };
+
+    const traverseIncludeOption = (includeOption, eagerLoadingTreeParent) => {
+      const includeOptions = lodash.castArray(includeOption);
+
+      if (includeOption.length > 0) {
+        const modelPrimaryKey = eagerLoadingTreeParent.model.primaryKeyAttribute;
+        pushAttribute(eagerLoadingTreeParent, modelPrimaryKey);
+      }
+
+      for (const include of includeOptions) {
+        // skip fromFilter include option
+        if (include.fromFilter) {
+          continue;
+        }
+
+        const association = eagerLoadingTreeParent.model.associations[include.association];
+        const associationType = association.associationType;
+
+        const child = {
+          model: association.target,
+          association,
+          rawAttributes: lodash.cloneDeep(include.attributes),
+          attributes: lodash.cloneDeep(include.attributes),
+          parent: eagerLoadingTreeParent,
+          children: [],
+        };
+
+        if (associationType == 'HasOne' || associationType == 'HasMany') {
+          const { sourceKey, foreignKey } = association;
+
+          pushAttribute(eagerLoadingTreeParent, sourceKey);
+          pushAttribute(child, foreignKey);
+        }
+
+        if (associationType == 'BelongsTo') {
+          const { targetKey, foreignKey } = association;
+
+          pushAttribute(eagerLoadingTreeParent, foreignKey);
+          pushAttribute(child, targetKey);
+        }
+
+        if (associationType == 'BelongsToMany') {
+          const { sourceKey } = association;
+          pushAttribute(eagerLoadingTreeParent, sourceKey);
+        }
+
+        eagerLoadingTreeParent.children.push(child);
+
+        if (include.include) {
+          traverseIncludeOption(include.include, child);
+        }
+      }
+    };
+
+    traverseIncludeOption(includeOption, root);
+
+    return new EagerLoadingTree(root);
+  }
+
+  async load(pks: Array<string | number>, transaction?: Transaction) {
+    const result = {};
+
+    const loadRecursive = async (node, ids) => {
+      const modelPrimaryKey = node.model.primaryKeyAttribute;
+
+      let instances = [];
+
+      // load instances from database
+      if (!node.parent) {
+        const findOptions = {
+          where: { [modelPrimaryKey]: ids },
+          attributes: node.attributes,
+        };
+
+        if (node.order) {
+          findOptions['order'] = node.order;
+        }
+
+        instances = await node.model.findAll({
+          ...findOptions,
+          transaction,
+        });
+      } else if (ids.length > 0) {
+        const association = node.association;
+        const associationType = association.associationType;
+
+        if (associationType == 'HasOne' || associationType == 'HasMany') {
+          const foreignKey = association.foreignKey;
+          const foreignKeyValues = node.parent.instances.map((instance) => instance.get(association.sourceKey));
+
+          const findOptions = {
+            where: { [foreignKey]: foreignKeyValues },
+            attributes: node.attributes,
+            transaction,
+          };
+
+          instances = await node.model.findAll(findOptions);
+        }
+
+        if (associationType == 'BelongsTo') {
+          const foreignKey = association.foreignKey;
+
+          const parentInstancesForeignKeyValues = node.parent.instances.map((instance) => instance.get(foreignKey));
+
+          instances = await node.model.findAll({
+            transaction,
+            where: {
+              [association.targetKey]: parentInstancesForeignKeyValues,
+            },
+            attributes: node.attributes,
+          });
+        }
+
+        if (associationType == 'BelongsToMany') {
+          const foreignKeyValues = node.parent.instances.map((instance) => instance.get(association.sourceKey));
+
+          instances = await node.model.findAll({
+            transaction,
+            attributes: node.attributes,
+            include: [
+              {
+                association: association.oneFromTarget,
+                where: {
+                  [association.foreignKey]: foreignKeyValues,
+                },
+              },
+            ],
+          });
+        }
+      }
+
+      node.instances = instances;
+
+      for (const child of node.children) {
+        const nodeIds = instances.map((instance) => instance.get(modelPrimaryKey));
+
+        await loadRecursive(child, nodeIds);
+      }
+
+      // merge instances to parent
+      if (!node.parent) {
+        return;
+      } else {
+        const association = node.association;
+        const associationType = association.associationType;
+
+        const setParentAccessor = (parentInstance) => {
+          const key = association.as;
+
+          const children = parentInstance.getDataValue(association.as);
+
+          if (association.isSingleAssociation) {
+            const isEmpty = !children;
+            parentInstance[key] = parentInstance.dataValues[key] = isEmpty ? null : children;
+          } else {
+            const isEmpty = !children || children.length == 0;
+            parentInstance[key] = parentInstance.dataValues[key] = isEmpty ? [] : children;
+          }
+        };
+
+        if (associationType == 'HasMany' || associationType == 'HasOne') {
+          const foreignKey = association.foreignKey;
+          const sourceKey = association.sourceKey;
+
+          for (const instance of node.instances) {
+            const parentInstance = node.parent.instances.find(
+              (parentInstance) => parentInstance.get(sourceKey) == instance.get(foreignKey),
+            );
+
+            if (parentInstance) {
+              if (associationType == 'HasMany') {
+                const children = parentInstance.getDataValue(association.as);
+                if (!children) {
+                  parentInstance.setDataValue(association.as, [instance]);
+                } else {
+                  children.push(instance);
+                }
+              }
+
+              if (associationType == 'HasOne') {
+                parentInstance.setDataValue(association.as, instance);
+              }
+            }
+          }
+        }
+
+        if (associationType == 'BelongsTo') {
+          const foreignKey = association.foreignKey;
+          const targetKey = association.targetKey;
+
+          for (const instance of node.instances) {
+            const parentInstances = node.parent.instances.filter(
+              (parentInstance) => parentInstance.get(foreignKey) == instance.get(targetKey),
+            );
+
+            for (const parentInstance of parentInstances) {
+              parentInstance.setDataValue(association.as, instance);
+            }
+          }
+        }
+
+        if (associationType == 'BelongsToMany') {
+          const sourceKey = association.sourceKey;
+          const foreignKey = association.foreignKey;
+
+          const oneFromTarget = association.oneFromTarget;
+
+          for (const instance of node.instances) {
+            const parentInstance = node.parent.instances.find(
+              (parentInstance) => parentInstance.get(sourceKey) == instance[oneFromTarget.as].get(foreignKey),
+            );
+
+            if (parentInstance) {
+              const children = parentInstance.getDataValue(association.as);
+
+              if (!children) {
+                parentInstance.setDataValue(association.as, [instance]);
+              } else {
+                children.push(instance);
+              }
+            }
+          }
+        }
+
+        for (const parent of node.parent.instances) {
+          setParentAccessor(parent);
+        }
+      }
+    };
+
+    await loadRecursive(this.root, pks);
+
+    const setInstanceAttributes = (node) => {
+      const nodeRawAttributes = node.rawAttributes;
+
+      if (!lodash.isArray(nodeRawAttributes)) {
+        return;
+      }
+
+      const nodeChildrenAs = node.children.map((child) => child.association.as);
+      const includeAttributes = [...nodeRawAttributes, ...nodeChildrenAs];
+
+      for (const instance of node.instances) {
+        const attributes = lodash.pick(instance.dataValues, includeAttributes);
+        instance.dataValues = attributes;
+      }
+    };
+
+    // traverse tree and set instance attributes
+    const traverse = (node) => {
+      setInstanceAttributes(node);
+
+      for (const child of node.children) {
+        traverse(child);
+      }
+    };
+
+    traverse(this.root);
+    return result;
+  }
+}

package/src/filter-parser.ts

@@ -56,7 +56,7 @@ export default class FilterParser {
     return filter;
   }
 
-  toSequelizeParams() {
+  toSequelizeParams(): any {
     debug('filter %o', this.filter);
 
     if (!this.filter) {
@@ -230,7 +230,21 @@ export default class FilterParser {
       });
     };
     debug('where %o, include %o', where, include);
-    return { where, include: toInclude(include) };
+    const results = { where, include: toInclude(include) };
+
+    //traverse filter include, set fromFiler to true
+    const traverseInclude = (include) => {
+      for (const item of include) {
+        if (item.include) {
+          traverseInclude(item.include);
+        }
+        item.fromFilter = true;
+      }
+    };
+
+    traverseInclude(results.include);
+
+    return results;
   }
 
   private getFieldNameFromQueryPath(queryPath: string) {

package/src/listeners/adjacency-list.ts

@@ -1,6 +1,4 @@
-import lodash from 'lodash';
-import { Collection, CollectionOptions } from '../collection';
-import { Model } from '../model';
+import { CollectionOptions } from '../collection';
 
 export const beforeDefineAdjacencyListCollection = (options: CollectionOptions) => {
   if (!options.tree) {

package/src/listeners/append-child-collection-name-after-repository-find.ts

@@ -0,0 +1,31 @@
+export const appendChildCollectionNameAfterRepositoryFind = (db) => {
+  return ({ findOptions, dataCollection, data }) => {
+    if (dataCollection.isParent()) {
+      for (const row of data) {
+        const rowCollection = db.tableNameCollectionMap.get(
+          findOptions.raw
+            ? `${row['__schemaName']}.${row['__tableName']}`
+            : `${row.get('__schemaName')}.${row.get('__tableName')}`,
+        );
+
+        if (!rowCollection) {
+          db.logger.warn(
+            `Can not find collection by table name ${JSON.stringify(row)}, current collections: ${Array.from(
+              db.tableNameCollectionMap.keys(),
+            ).join(', ')}`,
+          );
+
+          return;
+        }
+
+        const rowCollectionName = rowCollection.name;
+
+        findOptions.raw
+          ? (row['__collection'] = rowCollectionName)
+          : row.set('__collection', rowCollectionName, {
+              raw: true,
+            });
+      }
+    }
+  };
+};

package/src/listeners/index.ts

@@ -1,6 +1,8 @@
 import { Database } from '../database';
 import { beforeDefineAdjacencyListCollection } from './adjacency-list';
+import { appendChildCollectionNameAfterRepositoryFind } from './append-child-collection-name-after-repository-find';
 
 export const registerBuiltInListeners = (db: Database) => {
   db.on('beforeDefineCollection', beforeDefineAdjacencyListCollection);
+  db.on('afterRepositoryFind', appendChildCollectionNameAfterRepositoryFind(db));
 };

package/src/mock-database.ts

@@ -45,7 +45,9 @@ export function getConfigByEnv() {
 
 function customLogger(queryString, queryObject) {
   console.log(queryString); // outputs a string
-  console.log(queryObject.bind); // outputs an array
+  if (queryObject.bind) {
+    console.log(queryObject.bind); // outputs an array
+  }
 }
 
 export function mockDatabase(options: IDatabaseOptions = {}): MockDatabase {

package/src/operators/notIn.ts

@@ -10,3 +10,4 @@ export default {
     };
   },
 } as Record<string, any>;
+

package/src/operators/string.ts

@@ -1,5 +1,5 @@
-import { isPg } from './utils';
 import { Op } from 'sequelize';
+import { isPg } from './utils';
 
 export default {
   $includes(value, ctx) {

package/src/options-parser.ts

@@ -269,6 +269,11 @@ export class OptionsParser {
         (include) => include['association'] == appendAssociation,
       );
 
+      // if include from filter, remove fromFilter attribute
+      if (existIncludeIndex != -1) {
+        delete queryParams['include'][existIncludeIndex]['fromFilter'];
+      }
+
       // if association not exist, create it
       if (existIncludeIndex == -1) {
         // association not exists

package/src/query-interface/postgres-query-interface.ts

@@ -1,6 +1,6 @@
 import QueryInterface from './query-interface';
 import { Collection } from '../collection';
-import sqlParser from '../sql-parser';
+import sqlParser from '../sql-parser/postgres';
 
 export default class PostgresQueryInterface extends QueryInterface {
   constructor(db) {

package/src/relation-repository/belongs-to-many-repository.ts

@@ -1,7 +1,15 @@
 import lodash from 'lodash';
 import { BelongsToMany, Op, Transaction } from 'sequelize';
 import { Model } from '../model';
-import { CreateOptions, DestroyOptions, FindOneOptions, FindOptions, TargetKey, UpdateOptions } from '../repository';
+import {
+  AggregateOptions,
+  CreateOptions,
+  DestroyOptions,
+  FindOneOptions,
+  FindOptions,
+  TargetKey,
+  UpdateOptions,
+} from '../repository';
 import { updateThroughTableValue } from '../update-associations';
 import { FindAndCountOptions, MultipleRelationRepository } from './multiple-relation-repository';
 import { transaction } from './relation-repository';
@@ -38,6 +46,30 @@ interface IBelongsToManyRepository<M extends Model> {
 }
 
 export class BelongsToManyRepository extends MultipleRelationRepository implements IBelongsToManyRepository<any> {
+  async aggregate(options: AggregateOptions) {
+    const targetRepository = this.targetCollection.repository;
+
+    const sourceModel = await this.getSourceModel();
+
+    const association = this.association as any;
+
+    return await targetRepository.aggregate({
+      ...options,
+      optionsTransformer: (modelOptions) => {
+        modelOptions.include = modelOptions.include || [];
+        const throughWhere = {};
+        throughWhere[association.foreignKey] = sourceModel.get(association.sourceKey);
+
+        modelOptions.include.push({
+          association: association.oneFromTarget,
+          required: true,
+          attributes: [],
+          where: throughWhere,
+        });
+      },
+    });
+  }
+
   @transaction()
   async create(options?: CreateBelongsToManyOptions): Promise<any> {
     if (Array.isArray(options.values)) {

package/src/relation-repository/hasmany-repository.ts

@@ -2,6 +2,7 @@ import { omit } from 'lodash';
 import { HasMany, Op } from 'sequelize';
 import { Model } from '../model';
 import {
+  AggregateOptions,
   CreateOptions,
   DestroyOptions,
   FindOneOptions,
@@ -53,6 +54,22 @@ export class HasManyRepository extends MultipleRelationRepository implements IHa
     return await targetRepository.find(findOptions);
   }
 
+  async aggregate(options: AggregateOptions) {
+    const targetRepository = this.targetCollection.repository;
+    const addFilter = {
+      [this.association.foreignKey]: this.sourceKeyValue,
+    };
+
+    const aggOptions = {
+      ...options,
+      filter: {
+        $and: [options.filter || {}, addFilter],
+      },
+    };
+
+    return await targetRepository.aggregate(aggOptions);
+  }
+
   @transaction((args, transaction) => {
     return {
       filterByTk: args[0],