@nocobase/plugin-action-import 2.1.0-alpha.3 → 2.1.0-alpha.30

package/dist/node_modules/xlsx/xlsxworker.js

@@ -8,7 +8,7 @@ postMessage({t:"ready"});
 onmessage = function (evt) {
   var v;
   try {
-    v = XLSX.read(evt.data.d, {type: evt.data.b, codepage: evt.data.c});
+    v = XLSX.read(evt.data.d, {type: evt.data.b});
 postMessage({t:"xlsx", d:JSON.stringify(v)});
   } catch(e) { postMessage({t:"e",d:e.stack||e}); }
 };

package/dist/server/actions/import-xlsx.js

@@ -39,7 +39,7 @@ __export(import_xlsx_exports, {
   importXlsx: () => importXlsx
 });
 module.exports = __toCommonJS(import_xlsx_exports);
-var import_xlsx = __toESM(require("xlsx"));
+var XLSX = __toESM(require("xlsx"));
 var import_async_mutex = require("async-mutex");
 var import_xlsx_importer = require("../services/xlsx-importer");
 const IMPORT_LIMIT_COUNT = 2e3;
@@ -55,7 +55,7 @@ async function importXlsxAction(ctx, next) {
   if (ctx.request.body.explain) {
     readLimit += 1;
   }
-  const workbook = import_xlsx.default.read(ctx.file.buffer, {
+  const workbook = XLSX.read(ctx.file.buffer, {
     type: "buffer",
     sheetRows: readLimit,
     cellDates: true

package/dist/server/services/xlsx-importer.d.ts

@@ -24,7 +24,15 @@ export type ImporterOptions = {
     collectionManager: ICollectionManager;
     collection: ICollection;
     columns: Array<ImportColumn>;
-    workbook: any;
+    /** Parsed SheetJS workbook. Used for the synchronous (small-file) import path. */
+    workbook?: any;
+    /**
+     * Absolute path to the XLSX file on disk. When set, the importer uses ExcelJS
+     * streaming reader so the file is never fully loaded into memory — rows are
+     * yielded one by one and processed in chunks. This is the preferred path for
+     * large async imports.
+     */
+    filePath?: string;
     chunkSize?: number;
     explain?: string;
     repository?: any;
@@ -47,10 +55,10 @@ export declare class XlsxImporter extends EventEmitter {
     run(options?: RunOptions): Promise<any>;
     resetSeq(options?: RunOptions): Promise<void>;
     private getColumnsByPermission;
-    private validateColumns;
+    protected validateColumns(ctx?: Context): void;
     performImport(data: string[][], options?: RunOptions): Promise<any>;
     protected getModel(): typeof Model;
-    handleRowValuesWithColumns(row: any, rowValues: any, options: RunOptions, columns: ImportColumn[]): Promise<void>;
+    handleRowValuesWithColumns(row: any, rowValues: any, options: RunOptions, columns: ImportColumn[], rowIndex?: number): Promise<void>;
     handleChuckRows(chunkRows: string[][], runOptions?: RunOptions, options?: {
         handingRowIndex: number;
         context: any;
@@ -64,7 +72,7 @@ export declare class XlsxImporter extends EventEmitter {
     associateRecords(targets: Model[], options?: any): Promise<void>;
     renderErrorMessage(error: any): any;
     trimString(str: string): string;
-    private getExpectedHeaders;
+    protected getExpectedHeaders(ctx?: Context): string[];
     getData(ctx?: Context): Promise<string[][]>;
     private alignWithHeaders;
     private findAndValidateHeaders;

package/dist/server/services/xlsx-importer.js

@@ -84,10 +84,33 @@ class XlsxImporter extends import_events.default {
     return data;
   }
   async run(options = {}) {
-    var _a, _b;
-    let transaction = options.transaction;
-    if (!transaction && this.options.collectionManager.db) {
-      transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction();
+    var _a, _b, _c, _d;
+    const hasExternalTransaction = !!options.transaction;
+    const { db } = this.options.collectionManager;
+    if (hasExternalTransaction) {
+      try {
+        const data = await this.loggerService.measureExecutedTime(
+          async () => this.validate(options.context),
+          "Validation completed in {time}ms"
+        );
+        const imported = await this.loggerService.measureExecutedTime(
+          async () => this.performImport(data, options),
+          "Data import completed in {time}ms"
+        );
+        (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
+        if (db) {
+          await this.loggerService.measureExecutedTime(
+            async () => this.resetSeq(options),
+            "Sequence reset completed in {time}ms"
+          );
+        }
+        return imported;
+      } catch (error) {
+        (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+          originalError: error.stack || error.toString()
+        });
+        throw error;
+      }
     }
     try {
       const data = await this.loggerService.measureExecutedTime(
@@ -98,18 +121,16 @@ class XlsxImporter extends import_events.default {
         async () => this.performImport(data, options),
         "Data import completed in {time}ms"
       );
-      (_a = this.logger) == null ? void 0 : _a.info(`Import completed successfully, imported ${imported} records`);
-      if (this.options.collectionManager.db) {
+      (_c = this.logger) == null ? void 0 : _c.info(`Import completed successfully, imported ${imported} records`);
+      if (db) {
         await this.loggerService.measureExecutedTime(
-          async () => this.resetSeq(options),
+          async () => this.resetSeq({}),
           "Sequence reset completed in {time}ms"
         );
       }
-      transaction && await transaction.commit();
       return imported;
     } catch (error) {
-      transaction && await transaction.rollback();
-      (_b = this.logger) == null ? void 0 : _b.error(`Import failed: ${this.renderErrorMessage(error)}`, {
+      (_d = this.logger) == null ? void 0 : _d.error(`Import failed: ${this.renderErrorMessage(error)}`, {
         originalError: error.stack || error.toString()
       });
       throw error;
@@ -117,7 +138,7 @@ class XlsxImporter extends import_events.default {
   }
   async resetSeq(options) {
     const { transaction } = options;
-    const db = this.options.collectionManager.db;
+    const { db } = this.options.collectionManager;
     const collection = this.options.collection;
     const autoIncrementAttribute = collection.model.autoIncrementAttribute;
     if (!autoIncrementAttribute) {
@@ -166,13 +187,13 @@ class XlsxImporter extends import_events.default {
     this.emit("seqReset", { maxVal, seqName: autoIncrInfo.seqName });
   }
   getColumnsByPermission(ctx) {
+    var _a, _b, _c;
     const columns = this.options.columns;
-    return columns.filter(
-      (x) => {
-        var _a, _b, _c, _d, _e;
-        return import_lodash2.default.isEmpty((_b = (_a = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _a.can) == null ? void 0 : _b.params) ? true : import_lodash2.default.includes(((_e = (_d = (_c = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _c.can) == null ? void 0 : _d.params) == null ? void 0 : _e.fields) || [], x.dataIndex[0]);
-      }
-    );
+    const fields = (_c = (_b = (_a = ctx == null ? void 0 : ctx.permission) == null ? void 0 : _a.can) == null ? void 0 : _b.params) == null ? void 0 : _c.fields;
+    if (!Array.isArray(fields)) {
+      return columns;
+    }
+    return columns.filter((x) => import_lodash2.default.includes(fields, x.dataIndex[0]));
   }
   validateColumns(ctx) {
     var _a, _b, _c;
@@ -233,8 +254,10 @@ class XlsxImporter extends import_events.default {
     if (this.options.explain) {
       handingRowIndex += 1;
     }
-    for (const chunkRows of chunks) {
+    let chunkRows;
+    while ((chunkRows = chunks.shift()) !== void 0) {
       await this.handleChuckRows(chunkRows, options, { handingRowIndex, context: options == null ? void 0 : options.context });
+      handingRowIndex += chunkRows.length;
       imported += chunkRows.length;
       this.emit("progress", {
         total,
@@ -246,8 +269,7 @@ class XlsxImporter extends import_events.default {
   getModel() {
     return this.repository instanceof import_database.RelationRepository ? this.repository.targetModel : this.repository.model;
   }
-  async handleRowValuesWithColumns(row, rowValues, options, columns) {
-    var _a;
+  async handleRowValuesWithColumns(row, rowValues, options, columns, rowIndex = 1) {
     for (let index = 0; index < columns.length; index++) {
       const column = columns[index];
       const field = this.options.collection.getField(column.dataIndex[0]);
@@ -279,7 +301,7 @@ class XlsxImporter extends import_events.default {
         rowValues[dataKey] = str == null ? null : await interfaceInstance.toValue(this.trimString(str), ctx);
       } catch (error) {
         throw new import_errors.ImportValidationError("Failed to parse field {{field}} in row {{rowIndex}}: {{message}}", {
-          rowIndex: ((_a = options == null ? void 0 : options.context) == null ? void 0 : _a.handingRowIndex) || 1,
+          rowIndex,
           field: dataKey,
           message: error.message
         });
@@ -295,18 +317,12 @@ class XlsxImporter extends import_events.default {
   }
   async handleChuckRows(chunkRows, runOptions, options) {
     var _a, _b;
-    let { handingRowIndex = 1 } = options;
-    const { transaction } = runOptions;
+    const { handingRowIndex: chunkStartRowIndex = 1 } = options;
+    const db = this.options.collectionManager.db;
+    const externalTransaction = runOptions == null ? void 0 : runOptions.transaction;
+    const transaction = externalTransaction || (db ? await db.sequelize.transaction() : null);
+    const chunkRunOptions = { ...runOptions, transaction };
     const columns = this.getColumnsByPermission(options == null ? void 0 : options.context);
-    const rows = [];
-    for (const row of chunkRows) {
-      const rowValues = {};
-      await this.handleRowValuesWithColumns(row, rowValues, runOptions, columns);
-      rows.push({
-        ...this.options.rowDefaultValues || {},
-        ...rowValues
-      });
-    }
     const translate = (message) => {
       var _a2;
       if ((_a2 = options.context) == null ? void 0 : _a2.t) {
@@ -315,7 +331,19 @@ class XlsxImporter extends import_events.default {
         return message;
       }
     };
+    const rows = [];
+    let inChunkRowIndex = 0;
     try {
+      for (let i = 0; i < chunkRows.length; i++) {
+        inChunkRowIndex = i;
+        const row = chunkRows[i];
+        const rowValues = {};
+        await this.handleRowValuesWithColumns(row, rowValues, chunkRunOptions, columns, chunkStartRowIndex + i);
+        rows.push({
+          ...this.options.rowDefaultValues || {},
+          ...rowValues
+        });
+      }
       await this.loggerService.measureExecutedTime(
         async () => this.performInsert({
           values: rows,
@@ -325,23 +353,31 @@ class XlsxImporter extends import_events.default {
         "Record insertion completed in {time}ms"
       );
       await new Promise((resolve) => setTimeout(resolve, 5));
-      handingRowIndex += chunkRows.length;
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.commit());
+      }
     } catch (error) {
+      if (!externalTransaction) {
+        await (transaction == null ? void 0 : transaction.rollback());
+      }
+      if (error.name === "ImportValidationError") {
+        throw error;
+      }
       if (error.name === "SequelizeUniqueConstraintError") {
         throw new Error(`${translate("Unique constraint error, fields:")} ${JSON.stringify(error.fields)}`);
       }
+      const failedRowIndex = chunkStartRowIndex + inChunkRowIndex;
       if ((_a = error.params) == null ? void 0 : _a.rowIndex) {
-        handingRowIndex += error.params.rowIndex;
-        error.params.rowIndex = handingRowIndex;
+        error.params.rowIndex = failedRowIndex;
       }
-      (_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${handingRowIndex}: ${error.message}`, {
-        rowIndex: handingRowIndex,
-        rowData: rows[handingRowIndex],
+      (_b = this.logger) == null ? void 0 : _b.error(`Import error at row ${failedRowIndex}: ${error.message}`, {
+        rowIndex: failedRowIndex,
+        rowData: chunkRows[inChunkRowIndex],
         originalError: error.stack || error.toString()
       });
-      throw new import_errors.ImportError(`Import failed at row ${handingRowIndex}`, {
-        rowIndex: handingRowIndex,
-        rowData: rows[handingRowIndex - (this.options.explain ? 2 : 1)],
+      throw new import_errors.ImportError(`Import failed at row ${failedRowIndex}`, {
+        rowIndex: failedRowIndex,
+        rowData: chunkRows[inChunkRowIndex],
         cause: error
       });
     }
@@ -400,6 +436,7 @@ class XlsxImporter extends import_events.default {
           "debug"
         );
       }
+      instances[i] = null;
     }
     return instances;
   }
@@ -448,6 +485,7 @@ class XlsxImporter extends import_events.default {
     const workbook = this.options.workbook;
     const worksheet = workbook.Sheets[workbook.SheetNames[0]];
     let data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null, blankrows: false });
+    this.options.workbook = null;
     const expectedHeaders = this.getExpectedHeaders(ctx);
     const { headerRowIndex, headers } = this.findAndValidateHeaders({ data, expectedHeaders });
     if (headerRowIndex === -1) {

package/package.json

@@ -6,8 +6,8 @@
   "description": "Import records using excel templates. You can configure which fields to import and templates will be generated automatically.",
   "description.ru-RU": "Импорт записей с помощью шаблонов Excel: можно настроить, какие поля импортировать, шаблоны будут генерироваться автоматически.",
   "description.zh-CN": "使用 Excel 模板导入数据，可以配置导入哪些字段，自动生成模板。",
-  "version": "2.1.0-alpha.3",
-  "license": "AGPL-3.0",
+  "version": "2.1.0-alpha.30",
+  "license": "Apache-2.0",
   "main": "./dist/server/index.js",
   "homepage": "https://docs.nocobase.com/handbook/action-import",
   "homepage.ru-RU": "https://docs.nocobase.ru/handbook/action-import",
@@ -36,7 +36,7 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-i18next": "^11.15.1",
-    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
+    "xlsx": "^0.18.5"
   },
   "peerDependencies": {
     "@nocobase/actions": "2.x",
@@ -46,7 +46,7 @@
     "@nocobase/test": "2.x",
     "@nocobase/utils": "2.x"
   },
-  "gitHead": "b4d7448b938c1c3be8b2299ad32c6cbe012dd4ea",
+  "gitHead": "292ae0ad87f195ed201b274902d21ecd96f5ddd0",
   "keywords": [
     "Actions"
   ]

package/dist/node_modules/xlsx/dist/cpexcel.d.ts

@@ -1,39 +0,0 @@
-/* codepage.js (C) 2013-present SheetJS -- http://sheetjs.com */
-// TypeScript Version: 2.2
-
-/** Codepage index type (integer or string representation) */
-export type CP$Index = number | string;
-
-/* Individual codepage converter */
-export interface CP$Conv {
-	enc: {[n: string]: number; };
-	dec: {[n: number]: string; };
-}
-
-/** Encode input type (string, array of characters, Buffer) */
-export type CP$String = string | string[] | Uint8Array;
-
-/** Encode output / decode input type */
-export type CP$Data = string | number[] | Uint8Array;
-
-/** General utilities */
-export interface CP$Utils {
-	decode(cp: CP$Index, data: CP$Data): string;
-	encode(cp: CP$Index, data: CP$String, opts?: any): CP$Data;
-	hascp(n: number): boolean;
-	magic: {[cp: string]: string};
-}
-
-/* note: TS cannot export top-level indexer, hence default workaround */
-export interface CP$Module {
-	/** Version string */
-	version: string;
-
-	/** Utility Functions */
-	utils: CP$Utils;
-
-	/** Codepage Converters */
-	[cp: number]: CP$Conv;
-}
-export const cptable: CP$Module;
-export default cptable;

package/dist/node_modules/xlsx/dist/zahl.d.ts

@@ -1,4 +0,0 @@
-/* zahl.d.ts (C) 2022-present SheetJS */
-// TypeScript Version: 2.2
-declare const XLSX_ZAHL_PAYLOAD: string;
-export default XLSX_ZAHL_PAYLOAD;