@tachybase/module-backup 0.23.22 → 0.23.40

package/dist/server/dumper.js

@@ -196,11 +196,16 @@ class Dumper extends import_app_migrator.AppMigrator {
       });
       Dumper.dumpTasks.set(backupFileName, promise);
     } else {
-      await this.dump({
-        groups: options.groups,
-        fileName: backupFileName
-      });
-      await this.cleanLockFile(backupFileName);
+      try {
+        await this.dump({
+          groups: options.groups,
+          fileName: backupFileName
+        });
+      } catch (err) {
+        throw err;
+      } finally {
+        this.cleanLockFile(backupFileName);
+      }
     }
     return backupFileName;
   }
@@ -281,14 +286,14 @@ class Dumper extends import_app_migrator.AppMigrator {
     await import_promises.default.writeFile(metaPath, JSON.stringify(metaObj), "utf8");
   }
   async dumpCollection(options) {
-    var _a;
+    var _a, _b;
     const app = this.app;
     const dir = this.workDir;
     const collectionName = options.name;
-    app.logger.info(`dumping collection ${collectionName}`);
+    app.logger.info(`Dumping collection ${collectionName}`);
     const collection = app.db.getCollection(collectionName);
     if (!collection) {
-      this.app.logger.warn(`collection ${collectionName} not found`);
+      this.app.logger.warn(`Collection ${collectionName} not found`);
       return;
     }
     const collectionOnDumpOption = (_a = this.app.db.collectionFactory.collectionTypes.get(
@@ -302,35 +307,29 @@ class Dumper extends import_app_migrator.AppMigrator {
     const collectionDataDir = import_path.default.resolve(dir, "collections", collectionName);
     await import_promises.default.mkdir(collectionDataDir, { recursive: true });
     let count = 0;
-    if (columns.length !== 0) {
-      const dataFilePath = import_path.default.resolve(collectionDataDir, "data");
-      const dataStream = import_fs.default.createWriteStream(dataFilePath);
-      const rows = await app.db.sequelize.query(
-        (0, import_utils.sqlAdapter)(
-          app.db,
-          `SELECT *
-           FROM ${collection.isParent() ? "ONLY" : ""} ${collection.quotedTableName()}`
-        ),
-        {
-          type: "SELECT"
-        }
+    const dataFilePath = import_path.default.resolve(collectionDataDir, "data");
+    const dataStream = import_fs.default.createWriteStream(dataFilePath);
+    const rows = await app.db.sequelize.query(
+      (0, import_utils.sqlAdapter)(app.db, `SELECT * FROM ${collection.isParent() ? "ONLY" : ""} ${collection.quotedTableName()}`),
+      { type: "SELECT" }
+    );
+    for (const row of rows) {
+      const rowData = JSON.stringify(
+        columns.map((col) => {
+          const val = row[col];
+          const field = collection.getField(col);
+          return field ? import_field_value_writer.FieldValueWriter.toDumpedValue(field, val) : val;
+        })
       );
-      for (const row of rows) {
-        const rowData = JSON.stringify(
-          columns.map((col) => {
-            const val = row[col];
-            const field = collection.getField(col);
-            return field ? import_field_value_writer.FieldValueWriter.toDumpedValue(field, val) : val;
-          })
-        );
-        dataStream.write(rowData + "\r\n", "utf8");
+      if (!dataStream.write(rowData + "\r\n", "utf8")) {
+        await new Promise((resolve) => dataStream.once("drain", resolve));
       }
-      dataStream.end();
-      await finished(dataStream);
-      count = rows.length;
+      count++;
     }
+    dataStream.end();
+    await finished(dataStream);
     const metaAttributes = import_lodash.default.mapValues(attributes, (attr, key) => {
-      var _a2, _b, _c;
+      var _a2, _b2, _c;
       const collectionField = collection.getField(key);
       const fieldOptionKeys = ["field", "primaryKey", "autoIncrement", "allowNull", "defaultValue", "unique"];
       if (collectionField) {
@@ -340,7 +339,7 @@ class Dumper extends import_app_migrator.AppMigrator {
           type: collectionField.type,
           typeOptions: collectionField.options
         };
-        if (((_c = (_b = (_a2 = fieldAttributes.typeOptions) == null ? void 0 : _a2.defaultValue) == null ? void 0 : _b.constructor) == null ? void 0 : _c.name) === "UUIDV4") {
+        if (((_c = (_b2 = (_a2 = fieldAttributes.typeOptions) == null ? void 0 : _a2.defaultValue) == null ? void 0 : _b2.constructor) == null ? void 0 : _c.name) === "UUIDV4") {
           delete fieldAttributes.typeOptions.defaultValue;
         }
         return fieldAttributes;
@@ -363,7 +362,7 @@ class Dumper extends import_app_migrator.AppMigrator {
       meta["inherits"] = import_lodash.default.uniq(collection.options.inherits);
     }
     const autoIncrAttr = collection.model.autoIncrementAttribute;
-    if (autoIncrAttr && collection.model.rawAttributes[autoIncrAttr] && collection.model.rawAttributes[autoIncrAttr].autoIncrement) {
+    if (autoIncrAttr && ((_b = collection.model.rawAttributes[autoIncrAttr]) == null ? void 0 : _b.autoIncrement)) {
       const queryInterface = app.db.queryInterface;
       const autoIncrInfo = await queryInterface.getAutoIncrementInfo({
         tableInfo: {

package/dist/server/resourcers/backup-files.js

@@ -112,14 +112,16 @@ var backup_files_default = {
      * @param next
      */
     async create(ctx, next) {
+      var _a, _b;
       const data = ctx.request.body;
       let taskId;
       const app = ctx.app;
-      if (data.method === "worker") {
-        if (!app.worker.available) {
-          ctx.throw(500, ctx.t("No worker thread", { ns: "worker-thread" }));
-          return next();
-        }
+      if (data.method === "worker" && !((_a = app.worker) == null ? void 0 : _a.available)) {
+        ctx.throw(500, ctx.t("No worker thread", { ns: "worker-thread" }));
+        return next();
+      }
+      let useWorker = data.method === "worker" || data.method === "priority" && ((_b = app.worker) == null ? void 0 : _b.available);
+      if (useWorker) {
         try {
           taskId = await app.worker.callPluginMethod({
             plugin: import_server2.default,
@@ -132,7 +134,6 @@ var backup_files_default = {
           });
           app.noticeManager.notify("backup", { level: "info", msg: ctx.t("Done") });
         } catch (error) {
-          ctx.logger.warn(error);
           ctx.throw(500, ctx.t(error.message, { ns: "worker-thread" }));
         }
       } else {

package/dist/server/restorer.d.ts

@@ -22,14 +22,23 @@ export declare class Restorer extends AppMigrator {
     getImportMeta(): Promise<any>;
     checkMeta(): Promise<void>;
     importCollections(options: RestoreOptions): Promise<void>;
-    decompressBackup(backupFilePath: string): Promise<void>;
+    decompressBackup(backupFilePath: string): Promise<unknown>;
     readCollectionMeta(collectionName: string): Promise<any>;
     importCollection(options: {
         name: string;
         insert?: boolean;
         clear?: boolean;
         rowCondition?: (row: any) => boolean;
-    }): Promise<any>;
+    }): Promise<void>;
     importDb(options: RestoreOptions): Promise<void>;
+    insertMetaRows({ rows, collectionName, columns, fieldAttributes, rawAttributes, addSchemaTableName, options }: {
+        rows: any;
+        collectionName: any;
+        columns: any;
+        fieldAttributes: any;
+        rawAttributes: any;
+        addSchemaTableName: any;
+        options: any;
+    }): Promise<any>;
 }
 export {};

package/dist/server/restorer.js

@@ -35,9 +35,9 @@ var import_promises = __toESM(require("fs/promises"));
 var import_path = __toESM(require("path"));
 var import_database = require("@tachybase/database");
 var Topo = __toESM(require("@hapi/topo"));
-var import_decompress = __toESM(require("decompress"));
 var import_lodash = __toESM(require("lodash"));
 var import_semver = __toESM(require("semver"));
+var import_yauzl = __toESM(require("yauzl"));
 var import_app_migrator = require("./app-migrator");
 var import_restore_check_error = require("./errors/restore-check-error");
 var import_field_value_writer = require("./field-value-writer");
@@ -154,7 +154,38 @@ class Restorer extends import_app_migrator.AppMigrator {
     await this.emitAsync("restoreCollectionsFinished");
   }
   async decompressBackup(backupFilePath) {
-    if (!this.decompressed) await (0, import_decompress.default)(backupFilePath, this.workDir);
+    if (!this.decompressed) {
+      return new Promise((resolve, reject) => {
+        import_yauzl.default.open(backupFilePath, { lazyEntries: true }, (err, zipfile) => {
+          if (err) return reject(err);
+          zipfile.readEntry();
+          zipfile.on("entry", (entry) => {
+            const filePath = import_path.default.join(this.workDir, entry.fileName);
+            if (/\/$/.test(entry.fileName)) {
+              import_fs.default.mkdir(filePath, { recursive: true }, (err2) => {
+                if (err2) return reject(err2);
+                zipfile.readEntry();
+              });
+            } else {
+              zipfile.openReadStream(entry, (err2, readStream) => {
+                if (err2) return reject(err2);
+                const writeStream = import_fs.default.createWriteStream(filePath);
+                readStream.pipe(writeStream);
+                writeStream.on("close", () => {
+                  zipfile.readEntry();
+                });
+              });
+            }
+          });
+          zipfile.on("end", () => {
+            this.decompressed = true;
+            resolve(1);
+          });
+        });
+      });
+    } else {
+      return Promise.resolve();
+    }
   }
   async readCollectionMeta(collectionName) {
     const dir = this.workDir;
@@ -225,52 +256,45 @@ class Restorer extends import_app_migrator.AppMigrator {
       if (meta.inherits) {
         for (const inherit of import_lodash.default.uniq(meta.inherits)) {
           const parentMeta = await this.readCollectionMeta(inherit);
-          const sql2 = `ALTER TABLE ${app.db.utils.quoteTable(addSchemaTableName)} INHERIT ${app.db.utils.quoteTable(
+          const sql = `ALTER TABLE ${app.db.utils.quoteTable(addSchemaTableName)} INHERIT ${app.db.utils.quoteTable(
             parentMeta.tableName
           )};`;
-          await db.sequelize.query(sql2);
+          await db.sequelize.query(sql);
         }
       }
     }
-    const rows = await (0, import_utils.readLines)(collectionDataPath);
-    if (rows.length === 0) {
-      app.logger.info(`${collectionName} has no data to import`);
-      this.importedCollections.push(collectionName);
-      return;
-    }
-    const rowsWithMeta = rows.map(
-      (row) => JSON.parse(row).map((val, index) => [columns[index], val]).reduce((carry, [column, val]) => {
-        const field = fieldAttributes[column];
-        carry[column] = field ? import_field_value_writer.FieldValueWriter.write(field, val) : val;
-        return carry;
-      }, {})
-    ).filter((row) => {
-      if (options.rowCondition) {
-        return options.rowCondition(row);
+    const batchSize = 1e3;
+    let batch = [];
+    let allLength = 0;
+    await (0, import_utils.readEveryLines)(collectionDataPath, async (line) => {
+      batch.push(line);
+      if (batch.length >= batchSize) {
+        await this.insertMetaRows({
+          rows: batch,
+          collectionName,
+          columns,
+          fieldAttributes,
+          rawAttributes,
+          addSchemaTableName,
+          options
+        });
+        allLength += batchSize;
+        batch = [];
       }
-      return true;
     });
-    if (rowsWithMeta.length === 0) {
-      app.logger.info(`${collectionName} has no data to import`);
-      this.importedCollections.push(collectionName);
-      return;
-    }
-    const insertGeneratorAttributes = import_lodash.default.mapKeys(rawAttributes, (value, key) => {
-      return value.field;
-    });
-    const sql = db.sequelize.queryInterface.queryGenerator.bulkInsertQuery(
-      addSchemaTableName,
-      rowsWithMeta,
-      {},
-      insertGeneratorAttributes
-    );
-    if (options.insert === false) {
-      return sql;
+    if (!this.importedCollections.includes(collectionName)) {
+      await this.insertMetaRows({
+        rows: batch,
+        collectionName,
+        columns,
+        fieldAttributes,
+        rawAttributes,
+        addSchemaTableName,
+        options
+      });
+      allLength += batch.length;
     }
-    await app.db.sequelize.query(sql, {
-      type: "INSERT"
-    });
-    app.logger.info(`${collectionName} imported with ${rowsWithMeta.length} rows`);
+    app.logger.info(`${collectionName} imported with ${allLength} rows`);
     if (meta.autoIncrement) {
       const queryInterface = app.db.queryInterface;
       await queryInterface.setAutoIncrementVal({
@@ -313,6 +337,47 @@ class Restorer extends import_app_migrator.AppMigrator {
       }
     }
   }
+  async insertMetaRows({ rows, collectionName, columns, fieldAttributes, rawAttributes, addSchemaTableName, options }) {
+    const app = this.app;
+    const db = app.db;
+    if (rows.length === 0) {
+      app.logger.info(`${collectionName} has no data to import`);
+      this.importedCollections.push(collectionName);
+      return;
+    }
+    const rowsWithMeta = rows.map(
+      (row) => JSON.parse(row).map((val, index) => [columns[index], val]).reduce((carry, [column, val]) => {
+        const field = fieldAttributes[column];
+        carry[column] = field ? import_field_value_writer.FieldValueWriter.write(field, val) : val;
+        return carry;
+      }, {})
+    ).filter((row) => {
+      if (options.rowCondition) {
+        return options.rowCondition(row);
+      }
+      return true;
+    });
+    if (rowsWithMeta.length === 0) {
+      app.logger.info(`${collectionName} has no data to import`);
+      this.importedCollections.push(collectionName);
+      return;
+    }
+    const insertGeneratorAttributes = import_lodash.default.mapKeys(rawAttributes, (value, key) => {
+      return value.field;
+    });
+    const sql = db.sequelize.queryInterface.queryGenerator.bulkInsertQuery(
+      addSchemaTableName,
+      rowsWithMeta,
+      {},
+      insertGeneratorAttributes
+    );
+    if (options.insert === false) {
+      return sql;
+    }
+    await app.db.sequelize.query(sql, {
+      type: "INSERT"
+    });
+  }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {

package/dist/server/utils.d.ts

@@ -2,4 +2,5 @@ import { Database } from '@tachybase/database';
 export declare const DUMPED_EXTENSION = "tbdump";
 export declare function sqlAdapter(database: Database, sql: string): string;
 export declare function readLines(filePath: string): Promise<any[]>;
+export declare function readEveryLines(filePath: string, processLine: (line: string) => void): Promise<void>;
 export declare function humanFileSize(bytes: any, si?: boolean, dp?: number): string;

package/dist/server/utils.js

@@ -29,6 +29,7 @@ var utils_exports = {};
 __export(utils_exports, {
   DUMPED_EXTENSION: () => DUMPED_EXTENSION,
   humanFileSize: () => humanFileSize,
+  readEveryLines: () => readEveryLines,
   readLines: () => readLines,
   sqlAdapter: () => sqlAdapter
 });
@@ -55,6 +56,16 @@ async function readLines(filePath) {
   }
   return results;
 }
+async function readEveryLines(filePath, processLine) {
+  const fileStream = import_fs.default.createReadStream(filePath);
+  const rl = import_readline.default.createInterface({
+    input: fileStream,
+    crlfDelay: Infinity
+  });
+  for await (const line of rl) {
+    await processLine(line);
+  }
+}
 function humanFileSize(bytes, si = false, dp = 1) {
   const thresh = si ? 1e3 : 1024;
   if (Math.abs(bytes) < thresh) {
@@ -73,6 +84,7 @@ function humanFileSize(bytes, si = false, dp = 1) {
 0 && (module.exports = {
   DUMPED_EXTENSION,
   humanFileSize,
+  readEveryLines,
   readLines,
   sqlAdapter
 });

package/package.json

@@ -1,14 +1,13 @@
 {
   "name": "@tachybase/module-backup",
   "displayName": "App backup & restore",
-  "version": "0.23.22",
+  "version": "0.23.40",
   "description": "Backup and restore applications for scenarios such as application replication, migration, and upgrades.",
   "keywords": [
     "System management"
   ],
   "license": "Apache-2.0",
   "main": "./dist/server/index.js",
-  "dependencies": {},
   "devDependencies": {
     "@ant-design/icons": "^5.5.2",
     "@hapi/topo": "^6.0.2",
@@ -30,16 +29,17 @@
     "react-i18next": "^15.2.0",
     "semver": "^7.6.3",
     "tar": "^6.2.1",
-    "@tachybase/components": "0.23.22",
-    "@tachybase/module-worker-thread": "0.23.22"
+    "yauzl": "^3.2.0",
+    "@tachybase/components": "0.23.40",
+    "@tachybase/module-worker-thread": "0.23.40"
   },
   "peerDependencies": {
-    "@tachybase/actions": "0.23.22",
-    "@tachybase/client": "0.23.22",
-    "@tachybase/database": "0.23.22",
-    "@tachybase/server": "0.23.22",
-    "@tachybase/test": "0.23.22",
-    "@tachybase/utils": "0.23.22"
+    "@tachybase/actions": "0.23.40",
+    "@tachybase/client": "0.23.40",
+    "@tachybase/database": "0.23.40",
+    "@tachybase/test": "0.23.40",
+    "@tachybase/server": "0.23.40",
+    "@tachybase/utils": "0.23.40"
   },
   "description.zh-CN": "备份和还原应用，可用于应用的复制、迁移、升级等场景。",
   "displayName.zh-CN": "应用的备份与还原",