@nocobase/plugin-action-import 1.5.0-beta.2 → 1.5.0-beta.21

package/dist/server/services/template-creator.js

@@ -7,11 +7,9 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 
-var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
-var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -25,42 +23,117 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
-var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
-  // If the importer is in node compatibility mode or this is not an ESM
-  // file that has been converted to a CommonJS file using a Babel-
-  // compatible transform (i.e. "__esModule" has not been set), then set
-  // "default" to the CommonJS "module.exports" for node compatibility.
-  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
-  mod
-));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var template_creator_exports = {};
 __export(template_creator_exports, {
   TemplateCreator: () => TemplateCreator
 });
 module.exports = __toCommonJS(template_creator_exports);
-var import_xlsx = __toESM(require("xlsx"));
+var import_workbook_converter = require("../utils/workbook-converter");
+var import_exceljs = require("exceljs");
 class TemplateCreator {
   constructor(options) {
     this.options = options;
   }
-  async run() {
+  headerRowIndex;
+  getHeaderRowIndex() {
+    return this.headerRowIndex;
+  }
+  async run(options) {
     var _a;
-    const workbook = import_xlsx.default.utils.book_new();
-    const worksheet = import_xlsx.default.utils.sheet_new();
-    const data = [this.renderHeaders()];
+    const workbook = new import_exceljs.Workbook();
+    const worksheet = workbook.addWorksheet("Sheet 1");
+    const headers = this.renderHeaders();
+    let currentRow = 1;
+    let explainText = "";
     if (this.options.explain && ((_a = this.options.explain) == null ? void 0 : _a.trim()) !== "") {
-      data.unshift([this.options.explain]);
+      explainText = this.options.explain;
+    }
+    const fieldDescriptions = this.options.columns.filter((col) => col.description).map((col) => `${col.title || col.defaultTitle}\uFF1A${col.description}`);
+    if (fieldDescriptions.length > 0) {
+      if (explainText) {
+        explainText += "\n\n";
+      }
+      explainText += fieldDescriptions.join("\n");
+    }
+    if (explainText.trim() !== "") {
+      const lines = explainText.split("\n");
+      const EXPLAIN_MERGE_COLUMNS = 5;
+      const columnsNeeded = Math.max(EXPLAIN_MERGE_COLUMNS, headers.length);
+      for (let i = 1; i <= columnsNeeded; i++) {
+        const col = worksheet.getColumn(i);
+        col.width = i === 1 ? 60 : 30;
+      }
+      lines.forEach((line, index) => {
+        const row = worksheet.getRow(index + 1);
+        worksheet.mergeCells(index + 1, 1, index + 1, EXPLAIN_MERGE_COLUMNS);
+        const cell = row.getCell(1);
+        cell.value = line;
+        cell.alignment = {
+          vertical: "middle",
+          horizontal: "left",
+          indent: 1,
+          wrapText: true
+        };
+      });
+      currentRow = lines.length + 1;
     }
-    import_xlsx.default.utils.sheet_add_aoa(worksheet, data, {
-      origin: "A1"
+    const headerRow = worksheet.getRow(currentRow);
+    headerRow.height = 25;
+    headers.forEach((header, index) => {
+      const cell = headerRow.getCell(index + 1);
+      cell.value = header;
+      cell.font = {
+        bold: true,
+        size: 10,
+        name: "Arial"
+      };
+      cell.border = {
+        top: { style: "thin" },
+        bottom: { style: "thin" },
+        left: { style: "thin" },
+        right: { style: "thin" }
+      };
+      cell.fill = {
+        type: "pattern",
+        pattern: "solid",
+        fgColor: { argb: "FFE9ECEF" }
+        // Darker gray background for headers
+      };
+      cell.alignment = {
+        vertical: "middle",
+        horizontal: "left",
+        indent: 1,
+        wrapText: true
+      };
+      if (index === 0) {
+        worksheet.getColumn(1).width = 30;
+      }
     });
-    import_xlsx.default.utils.book_append_sheet(workbook, worksheet, "Sheet 1");
+    headers.forEach((_, index) => {
+      const col = worksheet.getColumn(index + 1);
+      col.width = 30;
+      if (index === 0) {
+        col.eachCell({ includeEmpty: false }, (cell, rowNumber) => {
+          if (rowNumber === currentRow) {
+            cell.fill = {
+              type: "pattern",
+              pattern: "solid",
+              fgColor: { argb: "FFE9ECEF" }
+            };
+          }
+        });
+      }
+    });
+    this.headerRowIndex = currentRow;
+    if (options == null ? void 0 : options.returnXLSXWorkbook) {
+      return await import_workbook_converter.WorkbookConverter.excelJSToXLSX(workbook);
+    }
     return workbook;
   }
   renderHeaders() {
     return this.options.columns.map((col) => {
-      return col.defaultTitle;
+      return col.title || col.defaultTitle;
     });
   }
 }

package/dist/server/services/xlsx-importer.d.ts

@@ -7,35 +7,45 @@
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
 /// <reference types="node" />
-import XLSX, { WorkBook } from 'xlsx';
 import { ICollection, ICollectionManager } from '@nocobase/data-source-manager';
 import { Transaction } from 'sequelize';
 import EventEmitter from 'events';
 export type ImportColumn = {
     dataIndex: Array<string>;
     defaultTitle: string;
+    title?: string;
+    description?: string;
 };
-type ImporterOptions = {
+export type ImporterOptions = {
     collectionManager: ICollectionManager;
     collection: ICollection;
     columns: Array<ImportColumn>;
-    workbook: WorkBook;
+    workbook: any;
     chunkSize?: number;
     explain?: string;
+    repository?: any;
 };
-type RunOptions = {
+export type RunOptions = {
     transaction?: Transaction;
     context?: any;
 };
 export declare class XlsxImporter extends EventEmitter {
-    private options;
+    protected options: ImporterOptions;
+    private repository;
     constructor(options: ImporterOptions);
-    run(options?: RunOptions): Promise<number>;
+    validate(): Promise<void>;
+    run(options?: RunOptions): Promise<any>;
     resetSeq(options?: RunOptions): Promise<void>;
-    performImport(options?: RunOptions): Promise<number>;
+    performImport(options?: RunOptions): Promise<any>;
+    performInsert(insertOptions: {
+        values: any;
+        transaction: Transaction;
+        context: any;
+        hooks?: boolean;
+    }): Promise<any>;
     renderErrorMessage(error: any): any;
     trimString(str: string): string;
-    getData(): unknown[];
-    firstSheet(): XLSX.WorkSheet;
+    private getExpectedHeaders;
+    getData(): Promise<string[][]>;
+    private findAndValidateHeaders;
 }
-export {};

package/dist/server/services/xlsx-importer.js

@@ -39,16 +39,34 @@ __export(xlsx_importer_exports, {
   XlsxImporter: () => XlsxImporter
 });
 module.exports = __toCommonJS(xlsx_importer_exports);
-var import_xlsx = __toESM(require("xlsx"));
+var XLSX = __toESM(require("xlsx"));
 var import_lodash = __toESM(require("lodash"));
 var import_events = __toESM(require("events"));
+var import_errors = require("../errors");
 class XlsxImporter extends import_events.default {
   constructor(options) {
     super();
     this.options = options;
+    if (typeof options.columns === "string") {
+      options.columns = JSON.parse(options.columns);
+    }
     if (options.columns.length == 0) {
       throw new Error(`columns is empty`);
     }
+    this.repository = options.repository ? options.repository : options.collection.repository;
+  }
+  repository;
+  async validate() {
+    if (this.options.columns.length == 0) {
+      throw new import_errors.ImportValidationError("Columns configuration is empty");
+    }
+    for (const column of this.options.columns) {
+      const field = this.options.collection.getField(column.dataIndex[0]);
+      if (!field) {
+        throw new import_errors.ImportValidationError("Field not found: {{field}}", { field: column.dataIndex[0] });
+      }
+    }
+    await this.getData();
   }
   async run(options = {}) {
     let transaction = options.transaction;
@@ -56,6 +74,7 @@ class XlsxImporter extends import_events.default {
       transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction();
     }
     try {
+      await this.validate();
       const imported = await this.performImport(options);
       if (this.options.collectionManager.db) {
         await this.resetSeq(options);
@@ -109,13 +128,14 @@ class XlsxImporter extends import_events.default {
   }
   async performImport(options) {
     const transaction = options == null ? void 0 : options.transaction;
-    const rows = this.getData();
-    const chunks = import_lodash.default.chunk(rows, this.options.chunkSize || 200);
+    const data = await this.getData();
+    const chunks = import_lodash.default.chunk(data.slice(1), this.options.chunkSize || 200);
     let handingRowIndex = 1;
+    let imported = 0;
+    const total = data.length - 1;
     if (this.options.explain) {
       handingRowIndex += 1;
     }
-    let imported = 0;
     for (const chunkRows of chunks) {
       for (const row of chunkRows) {
         const rowValues = {};
@@ -125,7 +145,9 @@ class XlsxImporter extends import_events.default {
             const column = this.options.columns[index];
             const field = this.options.collection.getField(column.dataIndex[0]);
             if (!field) {
-              throw new Error(`Field not found: ${column.dataIndex[0]}`);
+              throw new import_errors.ImportValidationError("Import validation.Field not found", {
+                field: column.dataIndex[0]
+              });
             }
             const str = row[index];
             const dataKey = column.dataIndex[0];
@@ -148,26 +170,38 @@ class XlsxImporter extends import_events.default {
             }
             rowValues[dataKey] = await interfaceInstance.toValue(this.trimString(str), ctx);
           }
-          await this.options.collection.repository.create({
+          await this.performInsert({
             values: rowValues,
-            context: options == null ? void 0 : options.context,
-            transaction
+            transaction,
+            context: options == null ? void 0 : options.context
           });
           imported += 1;
+          this.emit("progress", {
+            total,
+            current: imported
+          });
           await new Promise((resolve) => setTimeout(resolve, 5));
         } catch (error) {
-          throw new Error(
-            `failed to import row ${handingRowIndex}, ${this.renderErrorMessage(error)}, rowData: ${JSON.stringify(
-              rowValues
-            )}`,
-            { cause: error }
-          );
+          throw new import_errors.ImportError(`Import failed at row ${handingRowIndex}`, {
+            rowIndex: handingRowIndex,
+            rowData: Object.entries(rowValues).map(([key, value]) => `${key}: ${value}`).join(", "),
+            cause: error
+          });
         }
       }
       await new Promise((resolve) => setTimeout(resolve, 50));
     }
     return imported;
   }
+  async performInsert(insertOptions) {
+    const { values, transaction, context } = insertOptions;
+    return this.repository.create({
+      values,
+      context,
+      transaction,
+      hooks: insertOptions.hooks == void 0 ? true : insertOptions.hooks
+    });
+  }
   renderErrorMessage(error) {
     let message = error.message;
     if (error.parent) {
@@ -181,25 +215,52 @@ class XlsxImporter extends import_events.default {
     }
     return str;
   }
-  getData() {
-    const firstSheet = this.firstSheet();
-    const rows = import_xlsx.default.utils.sheet_to_json(firstSheet, { header: 1, defval: null });
-    if (this.options.explain) {
-      rows.shift();
-    }
-    const headers = rows[0];
-    const columns = this.options.columns;
-    for (let i = 0; i < columns.length; i++) {
-      const column = columns[i];
-      if (column.defaultTitle !== headers[i]) {
-        throw new Error(`Invalid header: ${column.defaultTitle} !== ${headers[i]}`);
-      }
+  getExpectedHeaders() {
+    return this.options.columns.map((col) => col.title || col.defaultTitle);
+  }
+  async getData() {
+    const workbook = this.options.workbook;
+    const worksheet = workbook.Sheets[workbook.SheetNames[0]];
+    const data = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: null });
+    const expectedHeaders = this.getExpectedHeaders();
+    const { headerRowIndex, headers } = this.findAndValidateHeaders(data);
+    if (headerRowIndex === -1) {
+      throw new import_errors.ImportValidationError("Headers not found. Expected headers: {{headers}}", {
+        headers: expectedHeaders.join(", ")
+      });
+    }
+    const rows = data.slice(headerRowIndex + 1);
+    if (rows.length === 0) {
+      throw new import_errors.ImportValidationError("No data to import");
     }
-    rows.shift();
-    return rows;
+    return [headers, ...rows];
   }
-  firstSheet() {
-    return this.options.workbook.Sheets[this.options.workbook.SheetNames[0]];
+  findAndValidateHeaders(data) {
+    const expectedHeaders = this.getExpectedHeaders();
+    for (let rowIndex = 0; rowIndex < data.length; rowIndex++) {
+      const row = data[rowIndex];
+      const actualHeaders = row.filter((cell) => cell !== null && cell !== "");
+      const allHeadersFound = expectedHeaders.every((header) => actualHeaders.includes(header));
+      const noExtraHeaders = actualHeaders.length === expectedHeaders.length;
+      if (allHeadersFound && noExtraHeaders) {
+        const mismatchIndex = expectedHeaders.findIndex((title, index) => actualHeaders[index] !== title);
+        if (mismatchIndex === -1) {
+          return { headerRowIndex: rowIndex, headers: actualHeaders };
+        } else {
+          throw new import_errors.ImportValidationError(
+            'Header mismatch at column {{column}}: expected "{{expected}}", but got "{{actual}}"',
+            {
+              column: mismatchIndex + 1,
+              expected: expectedHeaders[mismatchIndex],
+              actual: actualHeaders[mismatchIndex] || "empty"
+            }
+          );
+        }
+      }
+    }
+    throw new import_errors.ImportValidationError("Headers not found. Expected headers: {{headers}}", {
+      headers: expectedHeaders.join(", ")
+    });
   }
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/server/utils/workbook-converter.d.ts

@@ -0,0 +1,8 @@
+import * as XLSX from 'xlsx';
+import { Workbook as ExcelJSWorkbook } from 'exceljs';
+export declare class WorkbookConverter {
+    /**
+     * Convert ExcelJS Workbook to XLSX Workbook
+     */
+    static excelJSToXLSX(workbook: ExcelJSWorkbook): Promise<XLSX.WorkBook>;
+}

package/dist/server/utils/workbook-converter.js

@@ -0,0 +1,55 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var workbook_converter_exports = {};
+__export(workbook_converter_exports, {
+  WorkbookConverter: () => WorkbookConverter
+});
+module.exports = __toCommonJS(workbook_converter_exports);
+var XLSX = __toESM(require("xlsx"));
+class WorkbookConverter {
+  /**
+   * Convert ExcelJS Workbook to XLSX Workbook
+   */
+  static async excelJSToXLSX(workbook) {
+    const buffer = await workbook.xlsx.writeBuffer();
+    return XLSX.read(buffer, { type: "buffer" });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  WorkbookConverter
+});

package/package.json

@@ -4,7 +4,7 @@
   "displayName.zh-CN": "操作：导入记录",
   "description": "Import records using excel templates. You can configure which fields to import and templates will be generated automatically.",
   "description.zh-CN": "使用 Excel 模板导入数据，可以配置导入哪些字段，自动生成模板。",
-  "version": "1.5.0-beta.2",
+  "version": "1.5.0-beta.21",
   "license": "AGPL-3.0",
   "main": "./dist/server/index.js",
   "homepage": "https://docs.nocobase.com/handbook/action-import",
@@ -18,6 +18,8 @@
     "@koa/multer": "^3.0.2",
     "@types/node-xlsx": "^0.15.1",
     "antd": "5.x",
+    "async-mutex": "^0.5.0",
+    "exceljs": "^4.4.0",
     "file-saver": "^2.0.5",
     "mathjs": "^10.6.0",
     "node-xlsx": "^0.16.1",
@@ -34,7 +36,7 @@
     "@nocobase/test": "1.x",
     "@nocobase/utils": "1.x"
   },
-  "gitHead": "8a9c29fdac5c6295dcc7abf00c3fa81bd9e01a36",
+  "gitHead": "6a2c545d9e5e4f073592c810e704e6b0fe9aa096",
   "keywords": [
     "Actions"
   ]