@nocobase/plugin-action-import 2.1.0-alpha.1 → 2.1.0-alpha.11

package/dist/server/index.d.ts

@@ -6,9 +6,11 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
+import type { FlowSchemaContribution } from '@nocobase/flow-engine';
 import { Plugin } from '@nocobase/server';
 export { ImportError, ImportValidationError } from './errors';
 export declare class PluginActionImportServer extends Plugin {
+    getFlowSchemaContributions(): FlowSchemaContribution;
     beforeLoad(): void;
     load(): Promise<void>;
 }

package/dist/server/index.js

@@ -37,10 +37,14 @@ var import_server = require("@nocobase/server");
 var import_actions = require("./actions");
 var import_middleware = require("./middleware");
 var import_errors = require("./errors");
+var import_flow_schema_contributions = require("./flow-schema-contributions");
 var import_errors2 = require("./errors");
 __reExport(server_exports, require("./services/xlsx-importer"), module.exports);
 __reExport(server_exports, require("./services/template-creator"), module.exports);
 class PluginActionImportServer extends import_server.Plugin {
+  getFlowSchemaContributions() {
+    return import_flow_schema_contributions.flowSchemaContribution;
+  }
   beforeLoad() {
     this.app.on("afterInstall", async () => {
       if (!this.app.db.getRepository("roles")) {

package/dist/server/services/xlsx-importer.d.ts

@@ -24,7 +24,15 @@ export type ImporterOptions = {
     collectionManager: ICollectionManager;
     collection: ICollection;
     columns: Array<ImportColumn>;
-    workbook: any;
+    /** Parsed SheetJS workbook. Used for the synchronous (small-file) import path. */
+    workbook?: any;
+    /**
+     * Absolute path to the XLSX file on disk. When set, the importer uses ExcelJS
+     * streaming reader so the file is never fully loaded into memory — rows are
+     * yielded one by one and processed in chunks. This is the preferred path for
+     * large async imports.
+     */
+    filePath?: string;
     chunkSize?: number;
     explain?: string;
     repository?: any;
@@ -47,6 +55,7 @@ export declare class XlsxImporter extends EventEmitter {
     run(options?: RunOptions): Promise<any>;
     resetSeq(options?: RunOptions): Promise<void>;
     private getColumnsByPermission;
+    protected validateColumns(ctx?: Context): void;
     performImport(data: string[][], options?: RunOptions): Promise<any>;
     protected getModel(): typeof Model;
     handleRowValuesWithColumns(row: any, rowValues: any, options: RunOptions, columns: ImportColumn[]): Promise<void>;
@@ -63,7 +72,7 @@ export declare class XlsxImporter extends EventEmitter {
     associateRecords(targets: Model[], options?: any): Promise<void>;
     renderErrorMessage(error: any): any;
     trimString(str: string): string;
-    private getExpectedHeaders;
+    protected getExpectedHeaders(ctx?: Context): string[];
     getData(ctx?: Context): Promise<string[][]>;
     private alignWithHeaders;
     private findAndValidateHeaders;

package/dist/server/services/xlsx-importer.js

@@ -78,25 +78,39 @@ class XlsxImporter extends import_events.default {
     }
   }
   async validate(ctx) {
-    const columns = this.getColumnsByPermission(ctx);
-    if (columns.length == 0) {
-      throw new import_errors.ImportValidationError("Columns configuration is empty");
-    }
-    for (const column of this.options.columns) {
-      const field = this.options.collection.getField(column.dataIndex[0]);
-      if (!field) {
-        throw new import_errors.ImportValidationError("Field not found: {{field}}", { field: column.dataIndex[0] });
-      }
-    }
+    this.validateColumns(ctx);
     const data = await this.getData(ctx);
     await this.validateBySpaces(data, ctx);
     return data;
   }
   async run(options = {}) {
-    var _a, _b;
-    let transaction = options.transaction;
-    if (!transaction && this.options.collectionManager.db) {
-      transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction();
+    var _a, _b, _c, _d;
+    const hasExternalTransaction = !!options.transaction;
+    const { db } = this.options.collectionManager;
+    if (hasExternalTransaction) {
+      try {
+        const data = await this.loggerService.measureExecutedTime(
+          async () => this.validate(options.context),
+          "Validation completed in {time}ms"
+        );
+        const imported = await this.loggerService.measureExecutedTime(
+          async () => this.performImport(data, options),
+          "Data import completed in {time}ms"
+        );
+        (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
+        if (db) {
+          await this.loggerService.measureExecutedTime(
+            async () => this.resetSeq(options),
+            "Sequence reset completed in {time}ms"
+          );
+        }
+        return imported;
+      } catch (error) {
+        (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+          originalError: error.stack || error.toString()
+        });
+        throw error;
+      }
     }
     try {
       const data = await this.loggerService.measureExecutedTime(
@@ -107,18 +121,16 @@ class XlsxImporter extends import_events.default {
         async () => this.performImport(data, options),
         "Data import completed in {time}ms"
       );
-      (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
-      if (this.options.collectionManager.db) {
+      (_c = this.logger) == null ? void 0 : _c.info(`Import completed successfully, imported ${imported} records`);
+      if (db) {
         await this.loggerService.measureExecutedTime(
-          async () => this.resetSeq(options),
+          async () => this.resetSeq({}),
           "Sequence reset completed in {time}ms"
         );
       }
-      transaction && await transaction.commit();
       return imported;
     } catch (error) {
-      transaction && await transaction.rollback();
-      (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+      (_d = this.logger) == null ? void 0 : _d.error(`Import failed: ${this.renderErrorMessage(error)}`, {
         originalError: error.stack || error.toString()
       });
       throw error;
@@ -126,7 +138,7 @@ class XlsxImporter extends import_events.default {
   }
   async resetSeq(options) {
     const { transaction } = options;
-    const db = this.options.collectionManager.db;
+    const { db } = this.options.collectionManager;
     const collection = this.options.collection;
     const autoIncrementAttribute = collection.model.autoIncrementAttribute;
     if (!autoIncrementAttribute) {
@@ -158,6 +170,12 @@ class XlsxImporter extends import_events.default {
       transaction
     });
     const maxVal = await collection.model.max(autoIncrementAttribute, { transaction });
+    if (maxVal == null) {
+      return;
+    }
+    if (typeof autoIncrInfo.currentVal === "number" && maxVal <= autoIncrInfo.currentVal) {
+      return;
+    }
     const queryInterface = db.queryInterface;
     await queryInterface.setAutoIncrementVal({
       tableInfo,
@@ -177,6 +195,56 @@ class XlsxImporter extends import_events.default {
       }
     );
   }
+  validateColumns(ctx) {
+    var _a, _b, _c;
+    const columns = this.getColumnsByPermission(ctx);
+    if (columns.length === 0) {
+      throw new import_errors.ImportValidationError("Columns configuration is empty");
+    }
+    for (const column of columns) {
+      if (!Array.isArray(column == null ? void 0 : column.dataIndex) || column.dataIndex.length === 0) {
+        throw new import_errors.ImportValidationError("Columns configuration is empty");
+      }
+      if (column.dataIndex.length > 2) {
+        throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
+          field: column.dataIndex.join(".")
+        });
+      }
+      const [fieldName, filterKey] = column.dataIndex;
+      if (typeof fieldName !== "string" || fieldName.trim() === "") {
+        throw new import_errors.ImportValidationError("Invalid field: {{field}}", { field: String(fieldName) });
+      }
+      const field = this.options.collection.getField(fieldName);
+      if (!field) {
+        throw new import_errors.ImportValidationError("Field not found: {{field}}", { field: fieldName });
+      }
+      if (column.dataIndex.length > 1) {
+        if (typeof field.isRelationField !== "function" || !field.isRelationField()) {
+          throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
+            field: column.dataIndex.join(".")
+          });
+        }
+        if (typeof filterKey !== "string" || filterKey.trim() === "") {
+          throw new import_errors.ImportValidationError("Invalid field: {{field}}", {
+            field: column.dataIndex.join(".")
+          });
+        }
+        const targetCollection = (_a = field.targetCollection) == null ? void 0 : _a.call(field);
+        if (!targetCollection) {
+          throw new import_errors.ImportValidationError("Field not found: {{field}}", {
+            field: column.dataIndex.join(".")
+          });
+        }
+        const targetField = targetCollection.getField(filterKey);
+        const isValidAttribute = (_c = (_b = targetCollection.model) == null ? void 0 : _b.getAttributes()) == null ? void 0 : _c[filterKey];
+        if (!targetField && !isValidAttribute) {
+          throw new import_errors.ImportValidationError("Field not found: {{field}}", {
+            field: `${fieldName}.${filterKey}`
+          });
+        }
+      }
+    }
+  }
   async performImport(data, options) {
     const chunkSize = this.options.chunkSize || 1e3;
     const chunks = import_lodash.default.chunk(data.slice(1), chunkSize);
@@ -186,7 +254,8 @@ class XlsxImporter extends import_events.default {
     if (this.options.explain) {
       handingRowIndex += 1;
     }
-    for (const chunkRows of chunks) {
+    let chunkRows;
+    while ((chunkRows = chunks.shift()) !== void 0) {
       await this.handleChuckRows(chunkRows, options, { handingRowIndex, context: options == null ? void 0 : options.context });
       imported += chunkRows.length;
       this.emit("progress", {
@@ -247,20 +316,31 @@ class XlsxImporter extends import_events.default {
     rowValues = this.repository instanceof import_database.RelationRepository ? model.callSetters(guard.sanitize(rowValues || {}), options) : guard.sanitize(rowValues);
   }
   async handleChuckRows(chunkRows, runOptions, options) {
-    var _a, _b, _c;
+    var _a, _b;
     let { handingRowIndex = 1 } = options;
-    const { transaction } = runOptions;
+    const db = this.options.collectionManager.db;
+    const externalTransaction = runOptions == null ? void 0 : runOptions.transaction;
+    const transaction = externalTransaction || (db ? await db.sequelize.transaction() : null);
+    const chunkRunOptions = { ...runOptions, transaction };
     const columns = this.getColumnsByPermission(options == null ? void 0 : options.context);
+    const translate = (message) => {
+      var _a2;
+      if ((_a2 = options.context) == null ? void 0 : _a2.t) {
+        return options.context.t(message, { ns: "action-import" });
+      } else {
+        return message;
+      }
+    };
     const rows = [];
-    for (const row of chunkRows) {
-      const rowValues = {};
-      await this.handleRowValuesWithColumns(row, rowValues, runOptions, columns);
-      rows.push({
-        ...this.options.rowDefaultValues || {},
-        ...rowValues
-      });
-    }
     try {
+      for (const row of chunkRows) {
+        const rowValues = {};
+        await this.handleRowValuesWithColumns(row, rowValues, chunkRunOptions, columns);
+        rows.push({
+          ...this.options.rowDefaultValues || {},
+          ...rowValues
+        });
+      }
       await this.loggerService.measureExecutedTime(
         async () => this.performInsert({
           values: rows,
@@ -271,19 +351,24 @@ class XlsxImporter extends import_events.default {
       );
       await new Promise((resolve) => setTimeout(resolve, 5));
       handingRowIndex += chunkRows.length;
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.commit());
+      }
     } catch (error) {
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.rollback());
+      }
+      if (error.name === "ImportValidationError") {
+        throw error;
+      }
       if (error.name === "SequelizeUniqueConstraintError") {
-        throw new Error(
-          `${(_a = options.context) == null ? void 0 : _a.t("Unique constraint error, fields:", { ns: "action-import" })} ${JSON.stringify(
-            error.fields
-          )}`
-        );
+        throw new Error(`${translate("Unique constraint error, fields:")} ${JSON.stringify(error.fields)}`);
       }
-      if ((_b = error.params) == null ? void 0 : _b.rowIndex) {
+      if ((_a = error.params) == null ? void 0 : _a.rowIndex) {
         handingRowIndex += error.params.rowIndex;
         error.params.rowIndex = handingRowIndex;
       }
-      (_c = this.logger) == null ? void 0 : _c.error(`Import error at row ${handingRowIndex}: ${error.message}`, {
+      (_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${handingRowIndex}: ${error.message}`, {
         rowIndex: handingRowIndex,
         rowData: rows[handingRowIndex],
         originalError: error.stack || error.toString()
@@ -349,6 +434,7 @@ class XlsxImporter extends import_events.default {
           "debug"
         );
       }
+      instances[i] = null;
     }
     return instances;
   }
@@ -397,6 +483,7 @@ class XlsxImporter extends import_events.default {
     const workbook = this.options.workbook;
     const worksheet = workbook.Sheets[workbook.SheetNames[0]];
     let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null, blankrows: false });
+    this.options.workbook = null;
     const expectedHeaders = this.getExpectedHeaders(ctx);
     const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders });
     if (headerRowIndex === -1) {

package/package.json

@@ -6,12 +6,19 @@
   "description": "Import records using excel templates. You can configure which fields to import and templates will be generated automatically.",
   "description.ru-RU": "Импорт записей с помощью шаблонов Excel: можно настроить, какие поля импортировать, шаблоны будут генерироваться автоматически.",
   "description.zh-CN": "使用 Excel 模板导入数据，可以配置导入哪些字段，自动生成模板。",
-  "version": "2.1.0-alpha.1",
-  "license": "AGPL-3.0",
+  "version": "2.1.0-alpha.11",
+  "license": "Apache-2.0",
   "main": "./dist/server/index.js",
   "homepage": "https://docs.nocobase.com/handbook/action-import",
   "homepage.ru-RU": "https://docs.nocobase.ru/handbook/action-import",
   "homepage.zh-CN": "https://docs-cn.nocobase.com/handbook/action-import",
+  "nocobase": {
+    "supportedVersions": [
+      "1.x",
+      "2.x"
+    ],
+    "editionLevel": 0
+  },
   "devDependencies": {
     "@ant-design/icons": "5.x",
     "@formily/antd-v5": "1.x",
@@ -24,7 +31,7 @@
     "async-mutex": "^0.5.0",
     "exceljs": "^4.4.0",
     "file-saver": "^2.0.5",
-    "mathjs": "^10.6.0",
+    "mathjs": "^15.1.0",
     "node-xlsx": "^0.16.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
@@ -39,7 +46,7 @@
     "@nocobase/test": "2.x",
     "@nocobase/utils": "2.x"
   },
-  "gitHead": "d27baf21569643d6fa83f882233f4e90eb5b89f1",
+  "gitHead": "bb96d633a6371afb586072ff516bd0613c757db0",
   "keywords": [
     "Actions"
   ]