@nocobase/plugin-action-import 2.1.0-beta.8 → 2.2.0-alpha.1

package/dist/server/actions/import-xlsx.d.ts

@@ -6,5 +6,8 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
+/// <reference types="node" />
 import { Context, Next } from '@nocobase/actions';
+import * as XLSX from 'xlsx';
+export declare function readImportWorkbook(buffer: Buffer, sheetRows: number): XLSX.WorkBook;
 export declare function importXlsx(ctx: Context, next: Next): Promise<void>;

package/dist/server/actions/import-xlsx.js

@@ -36,14 +36,25 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var import_xlsx_exports = {};
 __export(import_xlsx_exports, {
-  importXlsx: () => importXlsx
+  importXlsx: () => importXlsx,
+  readImportWorkbook: () => readImportWorkbook
 });
 module.exports = __toCommonJS(import_xlsx_exports);
-var import_xlsx = __toESM(require("xlsx"));
+var XLSX = __toESM(require("xlsx"));
 var import_async_mutex = require("async-mutex");
 var import_xlsx_importer = require("../services/xlsx-importer");
 const IMPORT_LIMIT_COUNT = 2e3;
 const mutex = new import_async_mutex.Mutex();
+function readImportWorkbook(buffer, sheetRows) {
+  return XLSX.read(buffer, {
+    type: "buffer",
+    sheetRows,
+    // Keep Excel date cells as serial numbers. SheetJS `cellDates: true` converts
+    // them to local-time Date objects through a timezone-sensitive path, which can
+    // shift date-only values in non-UTC environments (for example Asia/Shanghai).
+    cellDates: false
+  });
+}
 async function importXlsxAction(ctx, next) {
   var _a;
   let columns = ctx.request.body.columns;
@@ -55,11 +66,7 @@ async function importXlsxAction(ctx, next) {
   if (ctx.request.body.explain) {
     readLimit += 1;
   }
-  const workbook = import_xlsx.default.read(ctx.file.buffer, {
-    type: "buffer",
-    sheetRows: readLimit,
-    cellDates: true
-  });
+  const workbook = readImportWorkbook(ctx.file.buffer, readLimit);
   const repository = ctx.getCurrentRepository();
   const dataSource = ctx.dataSource;
   const collection = repository.collection;
@@ -96,5 +103,6 @@ async function importXlsx(ctx, next) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  importXlsx
+  importXlsx,
+  readImportWorkbook
 });

package/dist/server/services/xlsx-importer.d.ts

@@ -24,7 +24,15 @@ export type ImporterOptions = {
     collectionManager: ICollectionManager;
     collection: ICollection;
     columns: Array<ImportColumn>;
-    workbook: any;
+    /** Parsed SheetJS workbook. Used for the synchronous (small-file) import path. */
+    workbook?: any;
+    /**
+     * Absolute path to the XLSX file on disk. When set, the importer uses ExcelJS
+     * streaming reader so the file is never fully loaded into memory — rows are
+     * yielded one by one and processed in chunks. This is the preferred path for
+     * large async imports.
+     */
+    filePath?: string;
     chunkSize?: number;
     explain?: string;
     repository?: any;
@@ -47,10 +55,10 @@ export declare class XlsxImporter extends EventEmitter {
     run(options?: RunOptions): Promise<any>;
     resetSeq(options?: RunOptions): Promise<void>;
     private getColumnsByPermission;
-    private validateColumns;
+    protected validateColumns(ctx?: Context): void;
     performImport(data: string[][], options?: RunOptions): Promise<any>;
     protected getModel(): typeof Model;
-    handleRowValuesWithColumns(row: any, rowValues: any, options: RunOptions, columns: ImportColumn[]): Promise<void>;
+    handleRowValuesWithColumns(row: any, rowValues: any, options: RunOptions, columns: ImportColumn[], rowIndex?: number): Promise<void>;
     handleChuckRows(chunkRows: string[][], runOptions?: RunOptions, options?: {
         handingRowIndex: number;
         context: any;
@@ -64,7 +72,7 @@ export declare class XlsxImporter extends EventEmitter {
     associateRecords(targets: Model[], options?: any): Promise<void>;
     renderErrorMessage(error: any): any;
     trimString(str: string): string;
-    private getExpectedHeaders;
+    protected getExpectedHeaders(ctx?: Context): string[];
     getData(ctx?: Context): Promise<string[][]>;
     private alignWithHeaders;
     private findAndValidateHeaders;

package/dist/server/services/xlsx-importer.js

@@ -84,10 +84,33 @@ class XlsxImporter extends import_events.default {
     return data;
   }
   async run(options = {}) {
-    var _a, _b;
-    let transaction = options.transaction;
-    if (!transaction && this.options.collectionManager.db) {
-      transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction();
+    var _a, _b, _c, _d;
+    const hasExternalTransaction = !!options.transaction;
+    const { db } = this.options.collectionManager;
+    if (hasExternalTransaction) {
+      try {
+        const data = await this.loggerService.measureExecutedTime(
+          async () => this.validate(options.context),
+          "Validation completed in {time}ms"
+        );
+        const imported = await this.loggerService.measureExecutedTime(
+          async () => this.performImport(data, options),
+          "Data import completed in {time}ms"
+        );
+        (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
+        if (db) {
+          await this.loggerService.measureExecutedTime(
+            async () => this.resetSeq(options),
+            "Sequence reset completed in {time}ms"
+          );
+        }
+        return imported;
+      } catch (error) {
+        (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+          originalError: error.stack || error.toString()
+        });
+        throw error;
+      }
     }
     try {
       const data = await this.loggerService.measureExecutedTime(
@@ -98,18 +121,16 @@ class XlsxImporter extends import_events.default {
         async () => this.performImport(data, options),
         "Data import completed in {time}ms"
       );
-      (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
-      if (this.options.collectionManager.db) {
+      (_c = this.logger) == null ? void 0 : _c.info(`Import completed successfully, imported ${imported} records`);
+      if (db) {
         await this.loggerService.measureExecutedTime(
-          async () => this.resetSeq(options),
+          async () => this.resetSeq({}),
           "Sequence reset completed in {time}ms"
         );
       }
-      transaction && await transaction.commit();
       return imported;
     } catch (error) {
-      transaction && await transaction.rollback();
-      (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+      (_d = this.logger) == null ? void 0 : _d.error(`Import failed: ${this.renderErrorMessage(error)}`, {
         originalError: error.stack || error.toString()
       });
       throw error;
@@ -117,7 +138,7 @@ class XlsxImporter extends import_events.default {
   }
   async resetSeq(options) {
     const { transaction } = options;
-    const db = this.options.collectionManager.db;
+    const { db } = this.options.collectionManager;
     const collection = this.options.collection;
     const autoIncrementAttribute = collection.model.autoIncrementAttribute;
     if (!autoIncrementAttribute) {
@@ -166,13 +187,13 @@ class XlsxImporter extends import_events.default {
     this.emit("seqReset", { maxVal, seqName: autoIncrInfo.seqName });
   }
   getColumnsByPermission(ctx) {
+    var _a, _b, _c;
     const columns = this.options.columns;
-    return columns.filter(
-      (x) => {
-        var _a, _b, _c, _d, _e;
-        return import_lodash2.default.isEmpty((_b = (_a = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _a.can) == null ? void 0 : _b.params) ? true : import_lodash2.default.includes(((_e = (_d = (_c = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _c.can) == null ? void 0 : _d.params) == null ? void 0 : _e.fields) || [], x.dataIndex[0]);
-      }
-    );
+    const fields = (_c = (_b = (_a = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _a.can) == null ? void 0 : _b.params) == null ? void 0 : _c.fields;
+    if (!Array.isArray(fields)) {
+      return columns;
+    }
+    return columns.filter((x) => import_lodash2.default.includes(fields, x.dataIndex[0]));
   }
   validateColumns(ctx) {
     var _a, _b, _c;
@@ -233,8 +254,10 @@ class XlsxImporter extends import_events.default {
     if (this.options.explain) {
       handingRowIndex += 1;
     }
-    for (const chunkRows of chunks) {
+    let chunkRows;
+    while ((chunkRows = chunks.shift()) !== void 0) {
       await this.handleChuckRows(chunkRows, options, { handingRowIndex, context: options == null ? void 0 : options.context });
+      handingRowIndex += chunkRows.length;
       imported += chunkRows.length;
       this.emit("progress", {
         total,
@@ -246,8 +269,7 @@ class XlsxImporter extends import_events.default {
   getModel() {
     return this.repository instanceof import_database.RelationRepository ? this.repository.targetModel : this.repository.model;
   }
-  async handleRowValuesWithColumns(row, rowValues, options, columns) {
-    var _a;
+  async handleRowValuesWithColumns(row, rowValues, options, columns, rowIndex = 1) {
     for (let index = 0; index < columns.length; index++) {
       const column = columns[index];
       const field = this.options.collection.getField(column.dataIndex[0]);
@@ -266,10 +288,9 @@ class XlsxImporter extends import_events.default {
         continue;
       }
       const interfaceInstance = new InterfaceClass(field.options);
-      const ctx = {
-        transaction: options.transaction,
-        field
-      };
+      const ctx = options.context ? Object.create(options.context) : {};
+      ctx.transaction = options.transaction;
+      ctx.field = field;
       if (column.dataIndex.length > 1) {
         ctx.associationField = field;
         ctx.targetCollection = field.targetCollection();
@@ -279,7 +300,7 @@ class XlsxImporter extends import_events.default {
         rowValues[dataKey] = str == null ? null : await interfaceInstance.toValue(this.trimString(str), ctx);
       } catch (error) {
         throw new import_errors.ImportValidationError("Failed to parse field {{field}} in row {{rowIndex}}: {{message}}", {
-          rowIndex: ((_a = options == null ? void 0 : options.context) == null ? void 0 : _a.handingRowIndex) || 1,
+          rowIndex,
           field: dataKey,
           message: error.message
         });
@@ -295,18 +316,12 @@ class XlsxImporter extends import_events.default {
   }
   async handleChuckRows(chunkRows, runOptions, options) {
     var _a, _b;
-    let { handingRowIndex = 1 } = options;
-    const { transaction } = runOptions;
+    const { handingRowIndex: chunkStartRowIndex = 1 } = options;
+    const db = this.options.collectionManager.db;
+    const externalTransaction = runOptions == null ? void 0 : runOptions.transaction;
+    const transaction = externalTransaction || (db ? await db.sequelize.transaction() : null);
+    const chunkRunOptions = { ...runOptions, transaction };
     const columns = this.getColumnsByPermission(options == null ? void 0 : options.context);
-    const rows = [];
-    for (const row of chunkRows) {
-      const rowValues = {};
-      await this.handleRowValuesWithColumns(row, rowValues, runOptions, columns);
-      rows.push({
-        ...this.options.rowDefaultValues || {},
-        ...rowValues
-      });
-    }
     const translate = (message) => {
       var _a2;
       if ((_a2 = options.context) == null ? void 0 : _a2.t) {
@@ -315,7 +330,19 @@ class XlsxImporter extends import_events.default {
         return message;
       }
     };
+    const rows = [];
+    let inChunkRowIndex = 0;
     try {
+      for (let i = 0; i < chunkRows.length; i++) {
+        inChunkRowIndex = i;
+        const row = chunkRows[i];
+        const rowValues = {};
+        await this.handleRowValuesWithColumns(row, rowValues, chunkRunOptions, columns, chunkStartRowIndex + i);
+        rows.push({
+          ...this.options.rowDefaultValues || {},
+          ...rowValues
+        });
+      }
       await this.loggerService.measureExecutedTime(
         async () => this.performInsert({
           values: rows,
@@ -325,23 +352,31 @@ class XlsxImporter extends import_events.default {
         "Record insertion completed in {time}ms"
       );
       await new Promise((resolve) => setTimeout(resolve, 5));
-      handingRowIndex += chunkRows.length;
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.commit());
+      }
     } catch (error) {
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.rollback());
+      }
+      if (error.name === "ImportValidationError") {
+        throw error;
+      }
       if (error.name === "SequelizeUniqueConstraintError") {
         throw new Error(`${translate("Unique constraint error, fields:")} ${JSON.stringify(error.fields)}`);
       }
+      const failedRowIndex = chunkStartRowIndex + inChunkRowIndex;
       if ((_a = error.params) == null ? void 0 : _a.rowIndex) {
-        handingRowIndex += error.params.rowIndex;
-        error.params.rowIndex = handingRowIndex;
+        error.params.rowIndex = failedRowIndex;
       }
-      (_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${handingRowIndex}: ${error.message}`, {
-        rowIndex: handingRowIndex,
-        rowData: rows[handingRowIndex],
+      (_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${failedRowIndex}: ${error.message}`, {
+        rowIndex: failedRowIndex,
+        rowData: chunkRows[inChunkRowIndex],
         originalError: error.stack || error.toString()
       });
-      throw new import_errors.ImportError(`Import failed at row ${handingRowIndex}`, {
-        rowIndex: handingRowIndex,
-        rowData: rows[handingRowIndex - (this.options.explain ? 2 : 1)],
+      throw new import_errors.ImportError(`Import failed at row ${failedRowIndex}`, {
+        rowIndex: failedRowIndex,
+        rowData: chunkRows[inChunkRowIndex],
         cause: error
       });
     }
@@ -400,6 +435,7 @@ class XlsxImporter extends import_events.default {
           "debug"
         );
       }
+      instances[i] = null;
     }
     return instances;
   }
@@ -448,6 +484,7 @@ class XlsxImporter extends import_events.default {
     const workbook = this.options.workbook;
     const worksheet = workbook.Sheets[workbook.SheetNames[0]];
     let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null, blankrows: false });
+    this.options.workbook = null;
     const expectedHeaders = this.getExpectedHeaders(ctx);
     const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders });
     if (headerRowIndex === -1) {

package/package.json

@@ -6,7 +6,7 @@
   "description": "Import records using excel templates. You can configure which fields to import and templates will be generated automatically.",
   "description.ru-RU": "Импорт записей с помощью шаблонов Excel: можно настроить, какие поля импортировать, шаблоны будут генерироваться автоматически.",
   "description.zh-CN": "使用 Excel 模板导入数据，可以配置导入哪些字段，自动生成模板。",
-  "version": "2.1.0-beta.8",
+  "version": "2.2.0-alpha.1",
   "license": "Apache-2.0",
   "main": "./dist/server/index.js",
   "homepage": "https://docs.nocobase.com/handbook/action-import",
@@ -36,17 +36,18 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-i18next": "^11.15.1",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz"
   },
   "peerDependencies": {
     "@nocobase/actions": "2.x",
     "@nocobase/client": "2.x",
+    "@nocobase/client-v2": "2.x",
     "@nocobase/database": "2.x",
     "@nocobase/server": "2.x",
     "@nocobase/test": "2.x",
     "@nocobase/utils": "2.x"
   },
-  "gitHead": "5099d561c5467292414c1e77ad6bad3730d97344",
+  "gitHead": "303663aba6c6eefa27e6a6435b4c0352074ec40f",
   "keywords": [
     "Actions"
   ]