@nocobase/plugin-action-import 1.0.0-alpha.9 → 1.0.1-alpha.2

package/dist/node_modules/xlsx/xlsxworker.js

@@ -2,14 +2,13 @@
 importScripts('dist/shim.min.js');
 /* uncomment the next line for encoding support */
 importScripts('dist/cpexcel.js');
-importScripts('jszip.js');
 importScripts('xlsx.js');
 postMessage({t:"ready"});
 
 onmessage = function (evt) {
   var v;
   try {
-    v = XLSX.read(evt.data.d, {type: evt.data.b});
+    v = XLSX.read(evt.data.d, {type: evt.data.b, codepage: evt.data.c});
 postMessage({t:"xlsx", d:JSON.stringify(v)});
   } catch(e) { postMessage({t:"e",d:e.stack||e}); }
 };

package/dist/server/actions/{downloadXlsxTemplate.js → download-xlsx-template.js}

@@ -34,33 +34,29 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   mod
 ));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-var downloadXlsxTemplate_exports = {};
-__export(downloadXlsxTemplate_exports, {
+var download_xlsx_template_exports = {};
+__export(download_xlsx_template_exports, {
   downloadXlsxTemplate: () => downloadXlsxTemplate
 });
-module.exports = __toCommonJS(downloadXlsxTemplate_exports);
-var import_node_xlsx = __toESM(require("node-xlsx"));
+module.exports = __toCommonJS(download_xlsx_template_exports);
+var import_template_creator = require("../services/template-creator");
+var import_xlsx = __toESM(require("xlsx"));
 async function downloadXlsxTemplate(ctx, next) {
   let { columns } = ctx.request.body;
   const { explain, title } = ctx.request.body;
   if (typeof columns === "string") {
     columns = JSON.parse(columns);
   }
-  const header = columns == null ? void 0 : columns.map((column) => column.defaultTitle);
-  const data = [header];
-  if ((explain == null ? void 0 : explain.trim()) !== "") {
-    data.unshift([explain]);
-  }
-  ctx.body = import_node_xlsx.default.build([
-    {
-      name: "Sheet 1",
-      data,
-      options: {}
-    }
-  ]);
+  const templateCreator = new import_template_creator.TemplateCreator({
+    explain,
+    title,
+    columns
+  });
+  const workbook = await templateCreator.run();
+  const buffer = import_xlsx.default.write(workbook, { type: "buffer", bookType: "xlsx" });
+  ctx.body = buffer;
   ctx.set({
     "Content-Type": "application/octet-stream",
-    // to avoid "invalid character" error in header (RFC)
     "Content-Disposition": `attachment; filename=${encodeURIComponent(title)}.xlsx`
   });
   await next();

package/dist/server/actions/import-xlsx.js

@@ -0,0 +1,89 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var import_xlsx_exports = {};
+__export(import_xlsx_exports, {
+  importXlsx: () => importXlsx
+});
+module.exports = __toCommonJS(import_xlsx_exports);
+var import_xlsx = __toESM(require("xlsx"));
+var import_xlsx_importer = require("../services/xlsx-importer");
+var import_async_mutex = require("async-mutex");
+const mutex = new import_async_mutex.Mutex();
+const IMPORT_LIMIT_COUNT = 2e3;
+async function importXlsxAction(ctx, next) {
+  let columns = ctx.request.body.columns;
+  if (typeof columns === "string") {
+    columns = JSON.parse(columns);
+  }
+  let readLimit = IMPORT_LIMIT_COUNT;
+  readLimit += 1;
+  if (ctx.request.body.explain) {
+    readLimit += 1;
+  }
+  const workbook = import_xlsx.default.read(ctx.file.buffer, {
+    type: "buffer",
+    sheetRows: readLimit
+  });
+  const repository = ctx.getCurrentRepository();
+  const dataSource = ctx.dataSource;
+  const collection = repository.collection;
+  const importer = new import_xlsx_importer.XlsxImporter({
+    collectionManager: dataSource.collectionManager,
+    collection,
+    columns,
+    workbook
+  });
+  const importedCount = await importer.run();
+  ctx.bodyMeta = { successCount: importedCount };
+  ctx.body = ctx.bodyMeta;
+}
+async function importXlsx(ctx, next) {
+  if (mutex.isLocked()) {
+    throw new Error(`another import action is running, please try again later.`);
+  }
+  const release = await mutex.acquire();
+  try {
+    await importXlsxAction(ctx, next);
+  } finally {
+    release();
+  }
+  await next();
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  importXlsx
+});

package/dist/server/actions/index.d.ts

@@ -6,5 +6,5 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
-export * from './downloadXlsxTemplate';
-export * from './importXlsx';
+export * from './download-xlsx-template';
+export * from './import-xlsx';

package/dist/server/actions/index.js

@@ -23,10 +23,10 @@ var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "defau
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 var actions_exports = {};
 module.exports = __toCommonJS(actions_exports);
-__reExport(actions_exports, require("./downloadXlsxTemplate"), module.exports);
-__reExport(actions_exports, require("./importXlsx"), module.exports);
+__reExport(actions_exports, require("./download-xlsx-template"), module.exports);
+__reExport(actions_exports, require("./import-xlsx"), module.exports);
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  ...require("./downloadXlsxTemplate"),
-  ...require("./importXlsx")
+  ...require("./download-xlsx-template"),
+  ...require("./import-xlsx")
 });

package/dist/server/index.d.ts

@@ -6,10 +6,9 @@
  * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
  * For more information, please refer to: https://www.nocobase.com/agreement.
  */
-import { InstallOptions, Plugin } from '@nocobase/server';
+import { Plugin } from '@nocobase/server';
 export declare class PluginActionImportServer extends Plugin {
     beforeLoad(): void;
     load(): Promise<void>;
-    install(options: InstallOptions): Promise<void>;
 }
 export default PluginActionImportServer;

package/dist/server/index.js

@@ -42,10 +42,6 @@ class PluginActionImportServer extends import_server.Plugin {
   }
   async load() {
     this.app.dataSourceManager.afterAddDataSource((dataSource) => {
-      var _a;
-      if (!((_a = dataSource.collectionManager) == null ? void 0 : _a.db)) {
-        return;
-      }
       dataSource.resourceManager.use(import_middleware.importMiddleware);
       dataSource.resourceManager.registerActionHandler("downloadXlsxTemplate", import_actions.downloadXlsxTemplate);
       dataSource.resourceManager.registerActionHandler("importXlsx", import_actions.importXlsx);
@@ -58,8 +54,6 @@ class PluginActionImportServer extends import_server.Plugin {
       dataSource.acl.allow("*", "downloadXlsxTemplate", "loggedIn");
     });
   }
-  async install(options) {
-  }
 }
 var server_default = PluginActionImportServer;
 // Annotate the CommonJS export names for ESM import in node:

package/dist/server/services/template-creator.d.ts

@@ -0,0 +1,24 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import { ICollection } from '@nocobase/data-source-manager';
+import { ImportColumn } from './xlsx-importer';
+import { WorkBook } from 'xlsx';
+type TemplateCreatorOptions = {
+    collection?: ICollection;
+    title?: string;
+    explain?: string;
+    columns: Array<ImportColumn>;
+};
+export declare class TemplateCreator {
+    private options;
+    constructor(options: TemplateCreatorOptions);
+    run(): Promise<WorkBook>;
+    renderHeaders(): string[];
+}
+export {};

package/dist/server/services/template-creator.js

@@ -0,0 +1,70 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var template_creator_exports = {};
+__export(template_creator_exports, {
+  TemplateCreator: () => TemplateCreator
+});
+module.exports = __toCommonJS(template_creator_exports);
+var import_xlsx = __toESM(require("xlsx"));
+class TemplateCreator {
+  constructor(options) {
+    this.options = options;
+  }
+  async run() {
+    var _a;
+    const workbook = import_xlsx.default.utils.book_new();
+    const worksheet = import_xlsx.default.utils.sheet_new();
+    const data = [this.renderHeaders()];
+    if (this.options.explain && ((_a = this.options.explain) == null ? void 0 : _a.trim()) !== "") {
+      data.unshift([this.options.explain]);
+    }
+    import_xlsx.default.utils.sheet_add_aoa(worksheet, data, {
+      origin: "A1"
+    });
+    import_xlsx.default.utils.book_append_sheet(workbook, worksheet, "Sheet 1");
+    return workbook;
+  }
+  renderHeaders() {
+    return this.options.columns.map((col) => {
+      return col.defaultTitle;
+    });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  TemplateCreator
+});

package/dist/server/services/xlsx-importer.d.ts

@@ -0,0 +1,37 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+import XLSX, { WorkBook } from 'xlsx';
+import { ICollection, ICollectionManager } from '@nocobase/data-source-manager';
+import { Transaction } from 'sequelize';
+export type ImportColumn = {
+    dataIndex: Array<string>;
+    defaultTitle: string;
+};
+type ImporterOptions = {
+    collectionManager: ICollectionManager;
+    collection: ICollection;
+    columns: Array<ImportColumn>;
+    workbook: WorkBook;
+    chunkSize?: number;
+    explain?: string;
+};
+type RunOptions = {
+    transaction?: Transaction;
+};
+export declare class XlsxImporter {
+    private options;
+    constructor(options: ImporterOptions);
+    run(options?: RunOptions): Promise<number>;
+    resetSeq(options?: RunOptions): Promise<void>;
+    performImport(options?: RunOptions): Promise<number>;
+    trimString(str: string): string;
+    getData(): unknown[];
+    firstSheet(): XLSX.WorkSheet;
+}
+export {};

package/dist/server/services/xlsx-importer.js

@@ -0,0 +1,185 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var xlsx_importer_exports = {};
+__export(xlsx_importer_exports, {
+  XlsxImporter: () => XlsxImporter
+});
+module.exports = __toCommonJS(xlsx_importer_exports);
+var import_xlsx = __toESM(require("xlsx"));
+var import_lodash = __toESM(require("lodash"));
+class XlsxImporter {
+  constructor(options) {
+    this.options = options;
+    if (options.columns.length == 0) {
+      throw new Error(`columns is empty`);
+    }
+  }
+  async run(options = {}) {
+    let transaction = options.transaction;
+    if (!transaction && this.options.collectionManager.db) {
+      transaction = options.transaction = await this.options.collectionManager.db.sequelize.transaction();
+    }
+    try {
+      const imported = await this.performImport(options);
+      if (this.options.collectionManager.db) {
+        await this.resetSeq(options);
+      }
+      transaction && await transaction.commit();
+      return imported;
+    } catch (error) {
+      transaction && await transaction.rollback();
+      throw error;
+    }
+  }
+  async resetSeq(options) {
+    const { transaction } = options;
+    const db = this.options.collectionManager.db;
+    const collection = this.options.collection;
+    const autoIncrementAttribute = collection.model.autoIncrementAttribute;
+    if (!autoIncrementAttribute) {
+      return;
+    }
+    let tableInfo = collection.getTableNameWithSchema();
+    if (typeof tableInfo === "string") {
+      tableInfo = {
+        tableName: tableInfo
+      };
+    }
+    const autoIncrInfo = await db.queryInterface.getAutoIncrementInfo({
+      tableInfo,
+      fieldName: autoIncrementAttribute,
+      transaction
+    });
+    const maxVal = await collection.model.max(autoIncrementAttribute, { transaction });
+    const queryInterface = db.queryInterface;
+    await queryInterface.setAutoIncrementVal({
+      tableInfo,
+      columnName: collection.model.rawAttributes[autoIncrementAttribute].field,
+      currentVal: maxVal,
+      seqName: autoIncrInfo.seqName,
+      transaction
+    });
+  }
+  async performImport(options) {
+    const transaction = options == null ? void 0 : options.transaction;
+    const rows = this.getData();
+    const chunks = import_lodash.default.chunk(rows, this.options.chunkSize || 200);
+    let handingRowIndex = 1;
+    if (this.options.explain) {
+      handingRowIndex += 1;
+    }
+    let imported = 0;
+    for (const chunkRows of chunks) {
+      for (const row of chunkRows) {
+        const rowValues = {};
+        handingRowIndex += 1;
+        try {
+          for (let index = 0; index < this.options.columns.length; index++) {
+            const column = this.options.columns[index];
+            const field = this.options.collection.getField(column.dataIndex[0]);
+            if (!field) {
+              throw new Error(`Field not found: ${column.dataIndex[0]}`);
+            }
+            const str = row[index];
+            const dataKey = column.dataIndex[0];
+            const fieldOptions = field.options;
+            const interfaceName = fieldOptions.interface;
+            const InterfaceClass = this.options.collectionManager.getFieldInterface(interfaceName);
+            if (!InterfaceClass) {
+              rowValues[dataKey] = str;
+              continue;
+            }
+            const interfaceInstance = new InterfaceClass(field.options);
+            const ctx = {
+              transaction,
+              field
+            };
+            if (column.dataIndex.length > 1) {
+              ctx.targetCollection = field.targetCollection();
+              ctx.filterKey = column.dataIndex[1];
+            }
+            rowValues[dataKey] = await interfaceInstance.toValue(this.trimString(str), ctx);
+          }
+          await this.options.collection.repository.create({
+            values: rowValues,
+            transaction
+          });
+          imported += 1;
+          await new Promise((resolve) => setTimeout(resolve, 5));
+        } catch (error) {
+          console.log(error);
+          throw new Error(
+            `failed to import row ${handingRowIndex}, rowData: ${JSON.stringify(rowValues)} message: ${error.message}`,
+            { cause: error }
+          );
+        }
+      }
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    }
+    return imported;
+  }
+  trimString(str) {
+    if (typeof str === "string") {
+      return str.trim();
+    }
+    return str;
+  }
+  getData() {
+    const firstSheet = this.firstSheet();
+    const rows = import_xlsx.default.utils.sheet_to_json(firstSheet, { header: 1, defval: null, raw: false });
+    if (this.options.explain) {
+      rows.shift();
+    }
+    const headers = rows[0];
+    const columns = this.options.columns;
+    for (let i = 0; i < columns.length; i++) {
+      const column = columns[i];
+      if (column.defaultTitle !== headers[i]) {
+        throw new Error(`Invalid header: ${column.defaultTitle} !== ${headers[i]}`);
+      }
+    }
+    rows.shift();
+    return rows;
+  }
+  firstSheet() {
+    return this.options.workbook.Sheets[this.options.workbook.SheetNames[0]];
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  XlsxImporter
+});

package/package.json

@@ -4,7 +4,7 @@
   "displayName.zh-CN": "操作：导入记录",
   "description": "Import records using excel templates. You can configure which fields to import and templates will be generated automatically.",
   "description.zh-CN": "使用 Excel 模板导入数据，可以配置导入哪些字段，自动生成模板。",
-  "version": "1.0.0-alpha.9",
+  "version": "1.0.1-alpha.2",
   "license": "AGPL-3.0",
   "main": "./dist/server/index.js",
   "homepage": "https://docs.nocobase.com/handbook/action-import",
@@ -24,7 +24,7 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-i18next": "^11.15.1",
-    "xlsx": "^0.17.0"
+    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
   },
   "peerDependencies": {
     "@nocobase/actions": "1.x",
@@ -34,7 +34,7 @@
     "@nocobase/test": "1.x",
     "@nocobase/utils": "1.x"
   },
-  "gitHead": "562d6ae965317098df05db756d160e03363dcec5",
+  "gitHead": "fc5a8e3c812516f787cb22d3d198f058f45b1963",
   "keywords": [
     "Actions"
   ]