@nocobase/plugin-data-visualization 0.10.1-alpha.1

package/src/server/__tests__/api.test.ts

@@ -0,0 +1,105 @@
+import { Database } from '@nocobase/database';
+import { MockServer, mockServer } from '@nocobase/test';
+import { queryData } from '../actions/query';
+import ChartsV2Plugin from '../plugin';
+
+describe('api', () => {
+  let app: MockServer;
+  let db: Database;
+
+  beforeAll(async () => {
+    app = mockServer({
+      acl: true,
+      plugins: ['users', 'auth'],
+    });
+    app.plugin(ChartsV2Plugin);
+    await app.loadAndInstall({ clean: true });
+    db = app.db;
+
+    db.collection({
+      name: 'chart_test',
+      fields: [
+        {
+          type: 'double',
+          name: 'price',
+        },
+        {
+          type: 'bigInt',
+          name: 'count',
+        },
+        {
+          type: 'string',
+          name: 'title',
+        },
+        {
+          type: 'date',
+          name: 'createdAt',
+        },
+      ],
+    });
+    await db.sync();
+    const repo = db.getRepository('chart_test');
+    await repo.create({
+      values: [
+        { price: 1, count: 1, title: 'title1' },
+        { price: 2, count: 2, title: 'title2' },
+      ],
+    });
+  });
+
+  afterAll(async () => {
+    await db.close();
+  });
+
+  test('query', () => {
+    expect.assertions(1);
+    return expect(
+      queryData({ db } as any, {
+        collection: 'chart_test',
+        measures: [
+          {
+            field: ['price'],
+            alias: 'Price',
+          },
+          {
+            field: ['count'],
+            alias: 'Count',
+          },
+        ],
+        dimensions: [
+          {
+            field: ['title'],
+            alias: 'Title',
+          },
+        ],
+      }),
+    ).resolves.toBeDefined();
+  });
+
+  test('query with sort', () => {
+    expect.assertions(1);
+    return expect(
+      queryData({ db } as any, {
+        collection: 'chart_test',
+        measures: [
+          {
+            field: ['price'],
+            aggregation: 'sum',
+            alias: 'Price',
+          },
+        ],
+        dimensions: [
+          {
+            field: ['title'],
+            alias: 'Title',
+          },
+          {
+            field: ['createdAt'],
+            format: 'YYYY',
+          },
+        ],
+        orders: [{ field: 'createdAt', order: 'asc' }],
+      }),
+    ).resolves.toBeDefined();
+  });
+});

package/src/server/__tests__/formatter.test.ts

@@ -0,0 +1,49 @@
+import { dateFormatFn } from '../actions/formatter';
+
+describe('formatter', () => {
+  const field = 'field';
+  const format = 'YYYY-MM-DD hh:mm:ss';
+  describe('dateFormatFn', () => {
+    it('should return correct format for sqlite', () => {
+      const sequelize = {
+        fn: jest.fn().mockImplementation((fn: string, format: string, field: string) => ({
+          fn,
+          format,
+          field,
+        })),
+        col: jest.fn().mockImplementation((field: string) => field),
+      };
+      const dialect = 'sqlite';
+      const result = dateFormatFn(sequelize, dialect, field, format);
+      expect(result.format).toEqual('%Y-%m-%d %H:%M:%S');
+    });
+
+    it('should return correct format for mysql', () => {
+      const sequelize = {
+        fn: jest.fn().mockImplementation((fn: string, field: string, format: string) => ({
+          fn,
+          format,
+          field,
+        })),
+        col: jest.fn().mockImplementation((field: string) => field),
+      };
+      const dialect = 'mysql';
+      const result = dateFormatFn(sequelize, dialect, field, format);
+      expect(result.format).toEqual('%Y-%m-%d %H:%i:%S');
+    });
+
+    it('should return correct format for postgres', () => {
+      const sequelize = {
+        fn: jest.fn().mockImplementation((fn: string, field: string, format: string) => ({
+          fn,
+          format,
+          field,
+        })),
+        col: jest.fn().mockImplementation((field: string) => field),
+      };
+      const dialect = 'postgres';
+      const result = dateFormatFn(sequelize, dialect, field, format);
+      expect(result.format).toEqual('YYYY-MM-DD HH24:MI:SS');
+    });
+  });
+});

package/src/server/__tests__/query.test.ts

@@ -0,0 +1,220 @@
+import { MockServer, mockServer } from '@nocobase/test';
+import * as formatter from '../actions/formatter';
+import { cacheWrap, parseBuilder, parseFieldAndAssociations } from '../actions/query';
+
+describe('query', () => {
+  describe('parseBuilder', () => {
+    const sequelize = {
+      fn: jest.fn().mockImplementation((fn: string, field: string) => [fn, field]),
+      col: jest.fn().mockImplementation((field: string) => field),
+    };
+    let ctx: any;
+    let app: MockServer;
+
+    beforeAll(() => {
+      app = mockServer();
+      app.db.collection({
+        name: 'orders',
+        fields: [
+          {
+            name: 'id',
+            type: 'bigInt',
+          },
+          {
+            name: 'price',
+            type: 'double',
+          },
+          {
+            name: 'createdAt',
+            type: 'date',
+          },
+          {
+            type: 'belongsTo',
+            name: 'user',
+            target: 'users',
+            targetKey: 'id',
+            foreignKey: 'userId',
+          },
+        ],
+      });
+      app.db.collection({
+        name: 'users',
+        fields: [
+          {
+            name: 'id',
+            type: 'bigInt',
+          },
+          {
+            name: 'name',
+            type: 'string',
+          },
+        ],
+      });
+      ctx = {
+        db: {
+          sequelize,
+          getRepository: (name: string) => app.db.getRepository(name),
+          getModel: (name: string) => app.db.getModel(name),
+          getCollection: (name: string) => app.db.getCollection(name),
+          options: {
+            underscored: true,
+          },
+        },
+      };
+    });
+
+    it('should parse field and associations', () => {
+      const associations = parseFieldAndAssociations(ctx, {
+        collection: 'orders',
+        measures: [{ field: ['price'], aggregation: 'sum', alias: 'price' }],
+        dimensions: [{ field: ['createdAt'] }, { field: ['user', 'name'] }],
+      });
+      expect(associations).toMatchObject({
+        measures: [{ field: 'orders.price', aggregation: 'sum', alias: 'price', type: 'double' }],
+        dimensions: [
+          { field: 'orders.created_at', alias: 'createdAt', type: 'date' },
+          { field: 'user.name', alias: 'user.name' },
+        ],
+        include: [{ association: 'user' }],
+      });
+    });
+
+    it('should parse measures', () => {
+      const measures1 = [
+        {
+          field: ['price'],
+        },
+      ];
+      const { queryParams: result1 } = parseBuilder(ctx, { collection: 'orders', measures: measures1 });
+      expect(result1.attributes).toEqual([['orders.price', 'price']]);
+
+      const measures2 = [
+        {
+          field: ['price'],
+          aggregation: 'sum',
+          alias: 'price-alias',
+        },
+      ];
+      const { queryParams: result2 } = parseBuilder(ctx, { collection: 'orders', measures: measures2 });
+      expect(result2.attributes).toEqual([[['sum', 'orders.price'], 'price-alias']]);
+    });
+
+    it('should parse dimensions', () => {
+      jest.spyOn(formatter, 'formatter').mockReturnValue('formatted-field');
+      const dimensions = [
+        {
+          field: ['createdAt'],
+          format: 'YYYY-MM-DD',
+          alias: 'Created at',
+        },
+      ];
+      const { queryParams: result } = parseBuilder(ctx, { collection: 'orders', dimensions });
+      expect(result.attributes).toEqual([['formatted-field', 'Created at']]);
+      expect(result.group).toEqual([]);
+
+      const measures = [
+        {
+          field: ['field'],
+          aggregation: 'sum',
+        },
+      ];
+      const { queryParams: result2 } = parseBuilder(ctx, { collection: 'orders', measures, dimensions });
+      expect(result2.group).toEqual(['formatted-field']);
+    });
+
+    it('should parse filter', () => {
+      const filter = {
+        createdAt: {
+          $gt: '2020-01-01',
+        },
+      };
+      const { queryParams: result } = parseBuilder(ctx, { collection: 'orders', filter });
+      expect(result.where.createdAt).toBeDefined();
+    });
+  });
+
+  describe('cacheWrap', () => {
+    const key = 'test-key';
+    const value = 'test-val';
+    class MockCache {
+      map: Map<string, any> = new Map();
+      async func() {
+        return value;
+      }
+
+      get(key: string) {
+        return this.map.get(key);
+      }
+      set(key: string, value: any) {
+        this.map.set(key, value);
+      }
+    }
+    let cache: any;
+    let query: () => Promise<any>;
+
+    beforeEach(() => {
+      cache = new MockCache();
+    });
+
+    it('should use cache', async () => {
+      query = async () =>
+        await cacheWrap(cache, {
+          key,
+          func: cache.func,
+          useCache: true,
+          refresh: false,
+        });
+
+      const spy = jest.spyOn(cache, 'func');
+      expect(cache.get(key)).toBeUndefined();
+      const result = await query();
+      expect(cache.func).toBeCalled();
+      expect(result).toEqual(value);
+      expect(cache.get(key)).toEqual(value);
+
+      spy.mockReset();
+      const result2 = await query();
+      expect(result2).toEqual(value);
+      expect(cache.func).not.toBeCalled();
+    });
+
+    it('should not use cache', async () => {
+      query = async () =>
+        await cacheWrap(cache, {
+          key,
+          func: cache.func,
+          useCache: false,
+          refresh: false,
+        });
+
+      cache.set(key, value);
+      expect(cache.get(key)).toBeDefined();
+      jest.spyOn(cache, 'func');
+      const result = await query();
+      expect(cache.func).toBeCalled();
+      expect(result).toEqual(value);
+    });
+
+    it('should refresh', async () => {
+      query = async () =>
+        await cacheWrap(cache, {
+          key,
+          func: cache.func,
+          useCache: true,
+          refresh: true,
+        });
+
+      const spy = jest.spyOn(cache, 'func');
+      expect(cache.get(key)).toBeUndefined();
+      const result = await query();
+      expect(cache.func).toBeCalled();
+      expect(result).toEqual(value);
+      expect(cache.get(key)).toEqual(value);
+
+      spy.mockClear();
+      const result2 = await query();
+      expect(cache.func).toBeCalled();
+      expect(result2).toEqual(value);
+    });
+  });
+});

package/src/server/actions/formatter.ts

@@ -0,0 +1,49 @@
+export const dateFormatFn = (sequelize: any, dialect: string, field: string, format: string) => {
+  switch (dialect) {
+    case 'sqlite':
+      format = format
+        .replace(/YYYY/g, '%Y')
+        .replace(/MM/g, '%m')
+        .replace(/DD/g, '%d')
+        .replace(/hh/g, '%H')
+        .replace(/mm/g, '%M')
+        .replace(/ss/g, '%S');
+      return sequelize.fn('strftime', format, sequelize.col(field));
+    case 'mysql':
+      format = format
+        .replace(/YYYY/g, '%Y')
+        .replace(/MM/g, '%m')
+        .replace(/DD/g, '%d')
+        .replace(/hh/g, '%H')
+        .replace(/mm/g, '%i')
+        .replace(/ss/g, '%S');
+      return sequelize.fn('date_format', sequelize.col(field), format);
+    case 'postgres':
+      format = format.replace(/hh/g, 'HH24').replace(/mm/g, 'MI').replace(/ss/g, 'SS');
+      return sequelize.fn('to_char', sequelize.col(field), format);
+    default:
+      return field;
+  }
+};
+
+export const formatFn = (sequelize: any, dialect: string, field: string, format: string) => {
+  switch (dialect) {
+    case 'sqlite':
+    case 'postgres':
+      return sequelize.fn('format', format, sequelize.col(field));
+    default:
+      return field;
+  }
+};
+
+export const formatter = (sequelize: any, type: string, field: string, format: string) => {
+  const dialect = sequelize.getDialect();
+  switch (type) {
+    case 'date':
+    case 'datetime':
+    case 'time':
+      return dateFormatFn(sequelize, dialect, field, format);
+    default:
+      return formatFn(sequelize, dialect, field, format);
+  }
+};

package/src/server/actions/query.ts

@@ -0,0 +1,285 @@
+import { Context, Next } from '@nocobase/actions';
+import { Cache } from '@nocobase/cache';
+import { FilterParser, snakeCase } from '@nocobase/database';
+import ChartsV2Plugin from '../plugin';
+import { formatter } from './formatter';
+
+type MeasureProps = {
+  field: string | string[];
+  type?: string;
+  aggregation?: string;
+  alias?: string;
+};
+
+type DimensionProps = {
+  field: string | string[];
+  type?: string;
+  alias?: string;
+  format?: string;
+};
+
+type OrderProps = {
+  field: string | string[];
+  alias?: string;
+  order?: 'asc' | 'desc';
+};
+
+type QueryParams = Partial<{
+  uid: string;
+  collection: string;
+  measures: MeasureProps[];
+  dimensions: DimensionProps[];
+  orders: OrderProps[];
+  filter: any;
+  limit: number;
+  sql: {
+    fields?: string;
+    clauses?: string;
+  };
+  cache: {
+    enabled: boolean;
+    ttl: number;
+  };
+  // Get the latest data from the database
+  refresh: boolean;
+}>;
+
+export const parseFieldAndAssociations = (ctx: Context, params: QueryParams) => {
+  const { collection: collectionName, measures, dimensions, orders, filter } = params;
+  const collection = ctx.db.getCollection(collectionName);
+  const fields = collection.fields;
+  const underscored = ctx.db.options.underscored;
+  const models: {
+    [target: string]: {
+      type: string;
+    };
+  } = {};
+  const parseField = (selected: { field: string | string[]; alias?: string }) => {
+    let target: string;
+    let name: string;
+    if (!Array.isArray(selected.field)) {
+      name = selected.field;
+    } else if (selected.field.length === 1) {
+      name = selected.field[0];
+    } else if (selected.field.length > 1) {
+      [target, name] = selected.field;
+    }
+    let field = underscored ? snakeCase(name) : name;
+    let type = fields.get(name)?.type;
+    if (target) {
+      field = `${target}.${field}`;
+      name = `${target}.${name}`;
+      type = fields.get(target)?.type;
+      if (!models[target]) {
+        models[target] = { type };
+      }
+    } else {
+      field = `${collectionName}.${field}`;
+    }
+    return {
+      ...selected,
+      field,
+      name,
+      type,
+      alias: selected.alias || name,
+    };
+  };
+
+  const parsedMeasures = measures?.map(parseField) || [];
+  const parsedDimensions = dimensions?.map(parseField) || [];
+  const parsedOrders = orders?.map(parseField) || [];
+  const include = Object.entries(models).map(([target, { type }]) => ({
+    association: target,
+    attributes: [],
+    ...(type === 'belongsToMany' ? { through: { attributes: [] } } : {}),
+  }));
+
+  const filterParser = new FilterParser(filter, {
+    collection,
+  });
+  const { where, include: filterInclude } = filterParser.toSequelizeParams();
+  const parsedFilterInclude = filterInclude?.map((item) => {
+    if (fields.get(item.association)?.type === 'belongsToMany') {
+      item.through = { attributes: [] };
+    }
+    return item;
+  });
+
+  return {
+    where,
+    measures: parsedMeasures,
+    dimensions: parsedDimensions,
+    orders: parsedOrders,
+    include: [...include, ...(parsedFilterInclude || [])],
+  };
+};
+
+export const parseBuilder = (ctx: Context, builder: QueryParams) => {
+  const { sequelize } = ctx.db;
+  const { limit } = builder;
+  const { measures, dimensions, orders, include, where } = parseFieldAndAssociations(ctx, builder);
+  const attributes = [];
+  const group = [];
+  const order = [];
+  const fieldMap = {};
+  let hasAgg = false;
+
+  measures.forEach((measure: MeasureProps & { field: string }) => {
+    const { field, aggregation, alias } = measure;
+    const attribute = [];
+    const col = sequelize.col(field);
+    if (aggregation) {
+      hasAgg = true;
+      attribute.push(sequelize.fn(aggregation, col));
+    } else {
+      attribute.push(col);
+    }
+    if (alias) {
+      attribute.push(alias);
+    }
+    attributes.push(attribute.length > 1 ? attribute : attribute[0]);
+    fieldMap[alias || field] = measure;
+  });
+
+  dimensions.forEach((dimension: DimensionProps & { field: string }) => {
+    const { field, format, alias, type } = dimension;
+    const attribute = [];
+    const col = sequelize.col(field);
+    if (format) {
+      attribute.push(formatter(sequelize, type, field, format));
+    } else {
+      attribute.push(col);
+    }
+    if (alias) {
+      attribute.push(alias);
+    }
+    attributes.push(attribute.length > 1 ? attribute : attribute[0]);
+    if (hasAgg) {
+      group.push(attribute[0]);
+    }
+    fieldMap[alias || field] = dimension;
+  });
+
+  orders.forEach((item: OrderProps) => {
+    const name = hasAgg ? sequelize.literal(`"${item.alias}"`) : sequelize.col(item.field as string);
+    order.push([name, item.order || 'ASC']);
+  });
+
+  return {
+    queryParams: {
+      where,
+      attributes,
+      include,
+      group,
+      order,
+      limit: limit > 2000 ? 2000 : limit,
+      raw: true,
+    },
+    fieldMap,
+  };
+};
+
+export const processData = (ctx: Context, data: any[], fieldMap: { [source: string]: { type?: string } }) => {
+  const { sequelize } = ctx.db;
+  const dialect = sequelize.getDialect();
+  switch (dialect) {
+    case 'postgres':
+      // https://github.com/sequelize/sequelize/issues/4550
+      return data.map((record) => {
+        const result = {};
+        Object.entries(record).forEach(([key, value]) => {
+          const { type } = fieldMap[key] || {};
+          switch (type) {
+            case 'bigInt':
+            case 'integer':
+            case 'float':
+            case 'double':
+              value = Number(value);
+              break;
+          }
+          result[key] = value;
+        });
+        return result;
+      });
+    default:
+      return data;
+  }
+};
+
+export const queryData = async (ctx: Context, builder: QueryParams) => {
+  const { collection, measures, dimensions, orders, filter, limit, sql } = builder;
+  const model = ctx.db.getModel(collection);
+  const { queryParams, fieldMap } = parseBuilder(ctx, { collection, measures, dimensions, orders, filter, limit });
+  const data = await model.findAll(queryParams);
+  return processData(ctx, data, fieldMap);
+  // if (!sql) {
+  //   return await repository.find(parseBuilder(ctx, { collection, measures, dimensions, orders, filter, limit }));
+  // }
+
+  // const statement = `SELECT ${sql.fields} FROM ${collection} ${sql.clauses}`;
+  // const [data] = await ctx.db.sequelize.query(statement);
+  // return data;
+};
+
+export const cacheWrap = async (
+  cache: Cache,
+  options: {
+    func: () => Promise<any>;
+    key: string;
+    ttl?: number;
+    useCache?: boolean;
+    refresh?: boolean;
+  },
+) => {
+  const { func, key, ttl, useCache, refresh } = options;
+  if (useCache && !refresh) {
+    const data = await cache.get(key);
+    if (data) {
+      return data;
+    }
+  }
+  const data = await func();
+  if (useCache) {
+    await cache.set(key, data, ttl);
+  }
+  return data;
+};
+
+export const query = async (ctx: Context, next: Next) => {
+  const {
+    uid,
+    collection,
+    measures,
+    dimensions,
+    orders,
+    filter,
+    limit,
+    sql,
+    cache: cacheConfig,
+    refresh,
+  } = ctx.action.params.values as QueryParams;
+  const roleName = ctx.state.currentRole || 'anonymous';
+  const can = ctx.app.acl.can({ role: roleName, resource: collection, action: 'list' });
+  if (!can && roleName !== 'root') {
+    ctx.throw(403, 'No permissions');
+  }
+
+  const plugin = ctx.app.getPlugin('data-visualization') as ChartsV2Plugin;
+  const cache = plugin.cache;
+  const useCache = cacheConfig?.enabled && uid;
+
+  try {
+    ctx.body = await cacheWrap(cache, {
+      func: async () => await queryData(ctx, { collection, measures, dimensions, orders, filter, limit, sql }),
+      key: uid,
+      ttl: cacheConfig?.ttl || 30,
+      useCache: useCache ? true : false,
+      refresh,
+    });
+  } catch (err) {
+    ctx.app.logger.error('charts query: ', err);
+    ctx.throw(500, err);
+  }
+
+  await next();
+};

package/src/server/index.ts

@@ -0,0 +1 @@
+export { default } from './plugin';