@nocobase/test 2.0.0-beta.5 → 2.0.0-beta.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nocobase/test",
-  "version": "2.0.0-beta.5",
+  "version": "2.0.0-beta.7",
   "main": "lib/index.js",
   "module": "./src/index.ts",
   "types": "./lib/index.d.ts",
@@ -51,7 +51,7 @@
   },
   "dependencies": {
     "@faker-js/faker": "8.1.0",
-    "@nocobase/server": "2.0.0-beta.5",
+    "@nocobase/server": "2.0.0-beta.7",
     "@playwright/test": "^1.45.3",
     "@testing-library/jest-dom": "^6.4.2",
     "@testing-library/react": "^14.0.0",
@@ -62,6 +62,7 @@
     "@vitest/coverage-istanbul": "^1.5.0",
     "@vitest/coverage-v8": "^1.5.0",
     "axios-mock-adapter": "1.22.0",
+    "exceljs": "^4.4.0",
     "jsdom": "^25.0.1",
     "jsdom-worker": "^0.3.0",
     "mariadb": "^2.5.6",
@@ -75,5 +76,5 @@
     "vitest-dom": "^0.1.1",
     "ws": "^8.13.0"
   },
-  "gitHead": "ea028f41293607ac0a20d054107e2ba772d639b2"
+  "gitHead": "17b6e4042a4814e38b256c7b3ae34e6d0b08fd57"
 }

package/perf/scenarios/import/concurrency-import.ts

@@ -0,0 +1,353 @@
+import { sleep, check } from 'k6';
+import { vu } from 'k6/execution';
+import http from 'k6/http';
+
+import { setup as baseSetup } from './setup.js';
+
+const appCount = Number.parseInt(__ENV.IMPORT_APP_COUNT, 10);
+const iterations = Number.parseInt(__ENV.IMPORT_CONCURRENCY_PER_APP, 10);
+
+const appNames = [...Array(appCount).keys()].map((index) => `import-test-apps-${index}`);
+const importDataTestCollection = JSON.parse(open(__ENV.COLLECTION_DEFINED_JSON_PATH));
+const importDataTestData = open(__ENV.IMPORT_TEMPLATE_WITH_MOCK_DATA, 'b');
+
+export const options = {
+  setupTimeout: '30m',
+  scenarios: {
+    'import-async': {
+      executor: 'per-vu-iterations',
+      vus: appNames.length,
+      iterations,
+      maxDuration: '30m',
+    },
+  },
+};
+
+export function setup() {
+  const { token } = baseSetup();
+
+  let appList = getAppList({ token, appNames });
+  const appMap = Object.fromEntries(appList.data.map((app) => [app.name, app]));
+  for (const appName of appNames) {
+    if (appMap[appName]) {
+      startApp({ token, name: appName });
+      continue;
+    }
+    createApp({ token, name: appName });
+    startApp({ token, name: appName });
+  }
+  do {
+    appList = getAppList({ token, appNames });
+    sleep(15);
+  } while (!appList.data.every((app) => app.status === 'running'));
+
+  for (const appName of appNames) {
+    let enabledPluginList = listEnabledPlugin({ token, appName });
+    let enabledPluginNames = (enabledPluginList.data ?? []).map(({ name }) => name);
+    if (enabledPluginNames.includes('action-import-pro')) {
+      continue;
+    }
+
+    enablePlugins({ token, appName, pluginNames: ['action-import-pro'] });
+    do {
+      enabledPluginList = listEnabledPlugin({ token, appName });
+      enabledPluginNames = (enabledPluginList.data ?? []).map(({ name }) => name);
+      sleep(15);
+    } while (!enabledPluginNames.includes('action-import-pro'));
+  }
+
+  for (const appName of appNames) {
+    const collectionList = findCollection({ token, appName, collectionName: 'importDataTest' });
+    const [testCollection] = collectionList.data;
+    if (testCollection?.name === importDataTestCollection.name) {
+      destroyCollection({ token, appName });
+    }
+    createCollection({ token, appName });
+  }
+
+  return { token };
+}
+
+const getAppList = ({ token, appNames = [] }) => {
+  const filterName = appNames.length ? `&${appNames.map((name) => `filter[name][$in]=${name}`).join('&')}` : '';
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/applications:list?pageSize=50&sort[]=-createdAt${filterName}`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+    },
+    cookies: {},
+  };
+  const resp = http.request('GET', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const createApp = ({ token, name }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/applications:create`;
+  const payload = JSON.stringify({
+    name,
+    options: { autoStart: false, authManager: {} },
+    displayName: name,
+  });
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, payload, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const enablePlugins = ({ token, appName, pluginNames = [] }) => {
+  if (!appNames.length) {
+    return;
+  }
+  const filterByTk = `${pluginNames.map((name) => `filterByTk[]=${name}`).join('&')}`;
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/pm:enable?${filterByTk}`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const listEnabledPlugin = ({ token, appName }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/pm:listEnabled`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('GET', url, null, params);
+  return JSON.parse(resp.body);
+};
+
+const startApp = ({ token, name }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/applications:start?filterByTk=${name}`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const findCollection = ({ token, appName, collectionName }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/collections:list?pageSize=50&filter[name]=${collectionName}`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('GET', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const destroyCollection = ({ token, appName }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/collections:destroy?filterByTk=${importDataTestCollection.name}&cascade=true`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const createCollection = ({ token, appName }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/collections:create`;
+  const payload = JSON.stringify({
+    name: importDataTestCollection.name,
+    title: importDataTestCollection.title,
+    template: importDataTestCollection.template,
+    autoGenId: importDataTestCollection.autoGenId,
+    fields: importDataTestCollection.fields,
+    logging: true,
+    view: false,
+  });
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, payload, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const importData = ({ token, appName }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/importDataTest:importXlsx?mode=auto`;
+  const columns = JSON.stringify(
+    importDataTestCollection.fields
+      .filter((field) => field.name.startsWith('column') || field.name === 'id')
+      .map((field) => ({
+        dataIndex: [field.name],
+        defaultTitle: field.uiSchema.title,
+      })),
+  );
+  const payload = {
+    file: http.file(
+      importDataTestData,
+      'importDataTest-import-template.xlsx',
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    ),
+    columns,
+    explain: '',
+  };
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('POST', url, payload, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+const getTask = ({ token, appName, taskId }) => {
+  const url = http.url`${__ENV.TARGET_ORIGIN}/api/asyncTasks:list?sort=-createdAt&filter[id]=${taskId}`;
+  const params = {
+    headers: {
+      'Proxy-Connection': `keep-alive`,
+      Authorization: `Bearer ${token}`,
+      Accept: `application/json, text/plain, */*`,
+      'X-Timezone': `+08:00`,
+      'Content-Type': `application/json`,
+      'X-With-ACL-Meta': `true`,
+      'X-Locale': `en-US`,
+      Origin: `${__ENV.ORIGIN}`,
+      Referer: `${__ENV.ORIGIN}/admin/settings/multi-app-manager`,
+      'Accept-Encoding': `gzip, deflate`,
+      'Accept-Language': `zh-CN,zh;q=0.9`,
+      'X-App': appName,
+    },
+    cookies: {},
+  };
+  const resp = http.request('GET', url, null, params);
+  check(resp, { 'status equals 200': (r) => r.status === 200 });
+  return JSON.parse(resp.body);
+};
+
+export default function ({ token }) {
+  const appName = appNames[vu.idInTest - 1];
+  const receipt = importData({ token, appName });
+  console.log(`app id: ${appName} taskId: ${receipt?.data?.taskId}`);
+  let status;
+  do {
+    const taskResp = getTask({ token, appName, taskId: receipt.data.taskId });
+    const [task] = taskResp.data ?? [];
+    status = task?.status;
+    sleep(60);
+  } while (status !== 1);
+}

package/perf/scenarios/import/init.ts

@@ -0,0 +1,236 @@
+import { createMockServer } from '@nocobase/test';
+import { CollectionRepository } from '@nocobase/plugin-data-source-main';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import ExcelJS from 'exceljs';
+import { faker } from '@faker-js/faker';
+import request from 'superagent';
+
+const storagePath = path.resolve(process.cwd(), 'storage', 'perf', 'importDataTest');
+
+export default async function main() {
+  const app = await createMockServer({
+    plugins: ['nocobase'],
+  });
+  const columns = [...Array(30).keys()];
+  const importDataTestCollection = {
+    name: 'importDataTest',
+    title: 'ImportDataTest',
+    template: 'general',
+    autoGenId: false,
+    filterTargetKey: 'id',
+    fields: [
+      ...columns.map((index) => ({
+        type: 'string',
+        name: 'column' + index,
+        interface: 'input',
+        uiSchema: {
+          type: 'string',
+          'x-component': 'Input',
+          title: 'column' + index,
+        },
+      })),
+      {
+        name: 'id',
+        type: 'bigInt',
+        autoIncrement: true,
+        primaryKey: true,
+        allowNull: false,
+        uiSchema: {
+          type: 'number',
+          title: 'ID',
+          'x-component': 'InputNumber',
+          'x-read-pretty': true,
+        },
+        interface: 'integer',
+      },
+      {
+        name: 'createdAt',
+        interface: 'createdAt',
+        type: 'date',
+        field: 'createdAt',
+        uiSchema: {
+          type: 'datetime',
+          title: '{{t("Created at")}}',
+          'x-component': 'DatePicker',
+          'x-component-props': {},
+          'x-read-pretty': true,
+        },
+      },
+      {
+        name: 'createdBy',
+        interface: 'createdBy',
+        type: 'belongsTo',
+        target: 'users',
+        foreignKey: 'createdById',
+        uiSchema: {
+          type: 'object',
+          title: '{{t("Created by")}}',
+          'x-component': 'AssociationField',
+          'x-component-props': {
+            fieldNames: {
+              value: 'id',
+              label: 'nickname',
+            },
+          },
+          'x-read-pretty': true,
+        },
+      },
+      {
+        type: 'date',
+        field: 'updatedAt',
+        name: 'updatedAt',
+        interface: 'updatedAt',
+        uiSchema: {
+          type: 'datetime',
+          title: '{{t("Last updated at")}}',
+          'x-component': 'DatePicker',
+          'x-component-props': {},
+          'x-read-pretty': true,
+        },
+      },
+      {
+        type: 'belongsTo',
+        target: 'users',
+        foreignKey: 'updatedById',
+        name: 'updatedBy',
+        interface: 'updatedBy',
+        uiSchema: {
+          type: 'object',
+          title: '{{t("Last updated by")}}',
+          'x-component': 'AssociationField',
+          'x-component-props': {
+            fieldNames: {
+              value: 'id',
+              label: 'nickname',
+            },
+          },
+          'x-read-pretty': true,
+        },
+      },
+    ],
+  };
+  const importDataTestCollectionPath = path.resolve(storagePath, 'importDataTest.collection.json');
+  const writeStream = fs.createWriteStream(importDataTestCollectionPath);
+  writeStream.write(JSON.stringify(importDataTestCollection, null, 4));
+  app.logger.info('generate collection json on:' + importDataTestCollectionPath);
+
+  const db = app.db;
+  db.collection(importDataTestCollection);
+  await db.sync();
+  const CollectionRepo = db.getRepository('collections') as CollectionRepository;
+  await CollectionRepo.db2cm('importDataTest');
+
+  const importTemplateXlsxPath = await download(
+    app
+      .agent()
+      .post('/importDataTest:downloadXlsxTemplate')
+      .send({
+        title: 'ImportDataTest',
+        explain: '',
+        columns: [
+          ...columns.map((index) => ({
+            dataIndex: ['column' + index],
+            defaultTitle: 'column' + index,
+          })),
+          {
+            dataIndex: ['id'],
+            defaultTitle: 'ID',
+          },
+        ],
+      }),
+  );
+  app.logger.info('download import template on:' + importTemplateXlsxPath);
+
+  const workbook = new ExcelJS.Workbook();
+  await workbook.xlsx.readFile(importTemplateXlsxPath);
+  const worksheet = workbook.getWorksheet('Sheet 1');
+  if (!worksheet) {
+    throw new Error('Sheet 1 not exist');
+  }
+  for (let i = 0; i < 10000; i++) {
+    const data = [];
+    for (let i = 0; i < columns.length; i++) {
+      switch (i) {
+        case 0:
+          data[i] = faker.string.uuid();
+          break;
+        case 1:
+          data[i] = faker.person.firstName();
+          break;
+        case 2:
+          data[i] = faker.person.middleName();
+          break;
+        case 3:
+          data[i] = faker.person.lastName();
+          break;
+        case 4:
+          data[i] = faker.person.gender();
+          break;
+        case 5:
+          data[i] = faker.person.bio();
+          break;
+        case 6:
+          data[i] = faker.internet.email();
+          break;
+        default:
+          data[i] = faker.word.words({ count: { min: 5, max: 10 } });
+          break;
+      }
+    }
+    data[data.length] = i + 1;
+    worksheet.addRow(data);
+  }
+
+  await workbook.xlsx.writeFile(importTemplateXlsxPath);
+  app.logger.info('fill import template with random data on:' + importTemplateXlsxPath);
+}
+
+function getFilename(res) {
+  const disposition = res.headers['content-disposition'];
+  if (!disposition) return 'download.file';
+
+  const match = disposition.match(/filename\*?=(?:UTF-8'')?["']?([^"';]+)/i);
+  return decodeURIComponent(match?.[1] ?? 'download.file');
+}
+
+async function download(request: request.SuperAgentRequest): Promise<string> {
+  return new Promise<string>((resolve, reject) => {
+    let importTemplateXlsxPath = '';
+    const tmpDir = os.tmpdir();
+    const tmpFile = path.join(tmpDir, `download-${Date.now()}-${crypto.randomUUID()}`);
+    const writeStream = fs.createWriteStream(tmpFile);
+    request
+      .on('response', (res) => {
+        const filename = getFilename(res);
+        importTemplateXlsxPath = path.resolve(storagePath, filename);
+      })
+      .pipe(writeStream)
+      .on('finish', () => {
+        fs.mkdir(storagePath, { recursive: true }, (err) => {
+          if (err) {
+            reject(err);
+            return;
+          }
+          try {
+            fs.renameSync(tmpFile, importTemplateXlsxPath);
+            resolve(importTemplateXlsxPath);
+          } catch (e) {
+            reject(e);
+          }
+        });
+      })
+      .on('error', reject);
+  });
+}
+
+main()
+  .then(() => {
+    console.log('Data generation completed.');
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error('Error during data generation:', error);
+    process.exit(1);
+  });

package/perf/scenarios/import/setup.js

@@ -0,0 +1,16 @@
+const http = require('k6/http');
+const { check } = require('k6');
+
+export function setup() {
+  // 在测试开始前执行一次
+  let res = http.post(`${__ENV.TARGET_ORIGIN}/api/auth:signIn`, {
+    account: 'nocobase',
+    password: 'admin123',
+  });
+
+  check(res, { 'login succeeded': (r) => r.status === 200 });
+
+  // 假设返回里有 token
+  let data = res.json('data');
+  return { token: data.token };
+}