@nocobase/plugin-backups 2.0.56

package/dist/server/index.js

@@ -0,0 +1,42 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var server_exports = {};
+__export(server_exports, {
+  default: () => import_plugin.default
+});
+module.exports = __toCommonJS(server_exports);
+var import_plugin = __toESM(require("./plugin"));

package/dist/server/managers/backup.d.ts

@@ -0,0 +1,41 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+/// <reference types="node" />
+import { ResourcerContext } from '@nocobase/resourcer';
+import Application from '@nocobase/server';
+import { Readable } from 'stream';
+export interface BackupSettings {
+    storageId?: string;
+    encryptionPassword: string;
+    enableFilesBackup: boolean;
+    keep?: number;
+    scheduled: boolean;
+    cron: string;
+}
+export interface BackupFile {
+    name: string;
+    fileSize?: string;
+    createdAt?: Date;
+    inProgress: boolean;
+}
+export interface BackupTaskResult {
+    message?: string;
+    inProgress: boolean;
+}
+export declare class BackupManager {
+    #private;
+    app: Application;
+    ctx: ResourcerContext | null;
+    constructor(app: Application, ctx: ResourcerContext | null, settings: BackupSettings);
+    createBackupName(): Promise<string>;
+    backup(fileBaseName: string, opts?: BackupSettings): Promise<string>;
+    destroy(fileName: string): Promise<void>;
+    list(): Promise<BackupFile[]>;
+    createReadStream(fileName: string): Promise<Readable>;
+}

package/dist/server/managers/backup.js

@@ -0,0 +1,448 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var backup_exports = {};
+__export(backup_exports, {
+  BackupManager: () => BackupManager
+});
+module.exports = __toCommonJS(backup_exports);
+var import_plugin_file_manager = __toESM(require("@nocobase/plugin-file-manager"));
+var import_archiver = __toESM(require("archiver"));
+var crypto = __toESM(require("crypto"));
+var import_dayjs = __toESM(require("dayjs"));
+var fs = __toESM(require("fs"));
+var import_promises = __toESM(require("fs/promises"));
+var import_path = __toESM(require("path"));
+var import_utils = require("@nocobase/utils");
+var import_util = require("util");
+var import_database = require("../adapters/database");
+var import_utils2 = require("../utils");
+class BackupManager {
+  app;
+  ctx;
+  // when triggered by cron job, ctx is null
+  #settings;
+  #dbAdapter;
+  #backupDir;
+  #tempDir;
+  #uploadDir;
+  #aesKeyPath;
+  constructor(app, ctx, settings) {
+    this.app = app;
+    this.ctx = ctx;
+    this.#settings = settings;
+    this.#dbAdapter = (0, import_database.getDBAdapter)(app.db.options);
+    this.#backupDir = (0, import_utils.storagePathJoin)("backups", app.name);
+    this.#tempDir = (0, import_utils.storagePathJoin)("tmp", "backups", app.name);
+    this.#uploadDir = (0, import_utils.storagePathJoin)("uploads");
+    this.#aesKeyPath = (0, import_utils.storagePathJoin)("apps", app.name, "aes_key.dat");
+  }
+  async createBackupName() {
+    await this.#dbAdapter.check("backup");
+    await import_promises.default.mkdir(this.#backupDir, { recursive: true });
+    await import_promises.default.mkdir(this.#tempDir, { recursive: true });
+    await import_promises.default.mkdir(this.#uploadDir, { recursive: true });
+    return this.#generateFileBaseName();
+  }
+  async backup(fileBaseName, opts = this.#settings) {
+    const contentPath = import_path.default.join(this.#tempDir, fileBaseName);
+    return this.#runBackupTask(opts, fileBaseName, contentPath);
+  }
+  async destroy(fileName) {
+    const filePath = this.#getValidatedFilePath(fileName);
+    await import_promises.default.unlink(filePath);
+  }
+  async list() {
+    const inProgressBackups = await this.#listProgressBackups();
+    const completedBackups = await this.#listCompletedBackups(inProgressBackups);
+    const cleanStaleLockFiles = async () => {
+      const statusCache = this.app.cacheManager.getCache(import_utils2.BACKUP_TASKS_CACHE_NAME);
+      for (const backup of inProgressBackups) {
+        if (!await statusCache.get(backup.name)) {
+          await this.#removeLockFile(import_path.default.basename(backup.name, `.${import_utils2.BACKUP_EXTENSION}`));
+          await statusCache.del(backup.name);
+        }
+      }
+    };
+    cleanStaleLockFiles().catch(() => {
+    });
+    return [...inProgressBackups, ...completedBackups];
+  }
+  async createReadStream(fileName) {
+    const filePath = this.#getValidatedFilePath(fileName);
+    return fs.createReadStream(filePath);
+  }
+  #getValidatedFilePath(fileName) {
+    const filePath = (0, import_utils2.resolvePathWithinBase)(this.#backupDir, fileName);
+    if (import_path.default.basename(fileName) !== fileName || !fileName.endsWith(`.${import_utils2.BACKUP_EXTENSION}`) || !filePath || !fs.existsSync(filePath)) {
+      throw new Error(this.#t("FILE_NOT_FOUND", fileName));
+    }
+    return filePath;
+  }
+  async #runBackupTask(opts, fileBaseName, contentPath) {
+    try {
+      await this.#createLockFile(fileBaseName);
+      await this.#createContentPath(contentPath);
+      await this.#dbAdapter.backup(contentPath, !this.app.pm.has("collection-fdw"));
+      await this.#metadataBackup(opts, contentPath);
+      const password = opts.encryptionPassword || void 0;
+      const filePath = await this.#compressFiles(contentPath, fileBaseName, password, opts);
+      await this.#uploadFiles(filePath, opts.storageId);
+      return filePath;
+    } catch (error) {
+      this.app.logger.error(`Error running backup task: ${error.message}`, { module: import_utils2.BACKUPS });
+      import_promises.default.unlink(import_path.default.join(this.#backupDir, `${fileBaseName}.${import_utils2.BACKUP_EXTENSION}`)).catch(() => {
+      });
+      throw new Error(this.#t("ERROR_TRIGGERING_BACKUP", this.#readableErrorMsg(error)));
+    } finally {
+      this.#removeContentPath(contentPath).catch(() => {
+      });
+      this.#removeLockFile(fileBaseName).catch(() => {
+      }).finally(() => {
+        if (opts.keep > 0) {
+          this.#dropOldBackups(opts.keep).catch(() => {
+          });
+        }
+      });
+    }
+  }
+  #readableErrorMsg(error) {
+    const message = error.message;
+    const { database } = this.#dbAdapter.dbOpts;
+    const msg = message.replace(new RegExp(`.*data\\s+${database}`), "");
+    if (msg.indexOf("pg_dump: error: aborting because of server version") !== -1) {
+      return this.#t("PG_DUMP_SERVER_VERSION_MISMATCH");
+    }
+    return msg;
+  }
+  async #metadataBackup(opts, dir) {
+    const { dialect, underscored, tablePrefix, schema } = this.#dbAdapter.dbOpts;
+    const installedPlugins = this.app.pm.getPlugins();
+    const plugins = [...installedPlugins.values()].filter((p) => p.enabled).map((installed) => {
+      return {
+        name: installed.options.name,
+        version: installed.options.version,
+        packageName: installed.options.packageName
+      };
+    });
+    const metadata = {
+      enableFilesBackup: opts.enableFilesBackup,
+      version: await this.app.version.get(),
+      database: {
+        dialect,
+        underscored,
+        tablePrefix,
+        schema,
+        version: await (0, import_utils2.getDBVersion)(this.app.db),
+        backupClientVersion: await this.#dbAdapter.clientVersion("backup")
+      },
+      plugins
+    };
+    const metadataFilePath = import_path.default.join(dir, "_metadata.json");
+    try {
+      await import_promises.default.writeFile(metadataFilePath, JSON.stringify(metadata, null, 2));
+    } catch (error) {
+      throw new Error(this.#t("ERROR_SAVING_MEATADATA", error.message));
+    }
+  }
+  async #compressFiles(dir, fileBaseName, password, opts) {
+    const archive = (0, import_archiver.default)("zip", {
+      zlib: { level: 9 }
+    });
+    const filePath = import_path.default.join(this.#backupDir, `${fileBaseName}.${import_utils2.BACKUP_EXTENSION}`);
+    const outputFileStream = fs.createWriteStream(filePath);
+    try {
+      const output = await this.#createEncryptedStream(outputFileStream, password);
+      const onClose = new Promise((resolve, reject) => {
+        output.on("close", () => {
+          this.app.logger.info(`Backup file created: ${filePath}`, { module: import_utils2.BACKUPS });
+          resolve();
+        });
+        output.on("end", () => {
+          this.app.logger.info(`File ${filePath} has been drained`, { module: import_utils2.BACKUPS });
+        });
+        output.on("error", reject);
+        archive.on("warning", (err) => {
+          if (err.code === "ENOENT") {
+            this.app.logger.warn(err.message, { module: import_utils2.BACKUPS });
+          } else {
+            reject(err);
+          }
+        });
+        archive.on("error", (err) => {
+          reject(err);
+        });
+      });
+      archive.pipe(output);
+      archive.directory(dir, false);
+      if (opts.enableFilesBackup) {
+        const fileCollections = [...this.app.db.collections.values()].filter((item) => {
+          return item.options.template === "file";
+        });
+        const filesPromises = fileCollections.map(async (collection) => {
+          try {
+            const files = await this.app.db.getRepository(collection.name).find({
+              filter: {
+                "storage.type": "local"
+              }
+            });
+            return {
+              collectionName: collection.name,
+              files
+            };
+          } catch (error) {
+            this.app.logger.warn(`Skip backing up files from collection "${collection.name}": ${error.message}`, {
+              module: import_utils2.BACKUPS
+            });
+            return {
+              collectionName: collection.name,
+              files: []
+            };
+          }
+        });
+        const filesList = await Promise.all(filesPromises);
+        for (const { collectionName, files } of filesList) {
+          for (const file of files) {
+            const backupFile = this.#getLocalFileForBackup(collectionName, file);
+            if (!backupFile) {
+              continue;
+            }
+            try {
+              archive.file(backupFile.filePath, { name: backupFile.archiveName });
+            } catch (error) {
+              this.#warnSkipFileBackup(collectionName, file, error.message);
+            }
+          }
+        }
+      }
+      if (fs.existsSync(this.#aesKeyPath)) {
+        archive.file(this.#aesKeyPath, { name: "aes_key.dat" });
+      }
+      await archive.finalize();
+      await onClose;
+    } catch (error) {
+      this.app.logger.error(`Error compressing files: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_COMPRESSING_FILES", error.message));
+    } finally {
+      outputFileStream.close();
+    }
+    return filePath;
+  }
+  #getLocalFileForBackup(collectionName, file) {
+    const filename = file.filename;
+    if (typeof filename !== "string" || filename.length === 0) {
+      this.#warnSkipFileBackup(collectionName, file, "Invalid filename");
+      return null;
+    }
+    const filePath = file.path ?? "";
+    if (typeof filePath !== "string") {
+      this.#warnSkipFileBackup(collectionName, file, "Invalid path");
+      return null;
+    }
+    const resolvedPath = import_path.default.resolve(this.#uploadDir, filePath, filename);
+    const relativePath = import_path.default.relative(this.#uploadDir, resolvedPath);
+    if (!relativePath || this.#hasPathTraversal(relativePath, import_path.default.sep) || import_path.default.isAbsolute(relativePath)) {
+      this.#warnSkipFileBackup(collectionName, file, "Path traversal is not allowed");
+      return null;
+    }
+    const archiveName = import_path.default.posix.join("uploads", ...relativePath.split(import_path.default.sep));
+    if (archiveName.includes("\\")) {
+      this.#warnSkipFileBackup(collectionName, file, "Invalid path");
+      return null;
+    }
+    if (this.#hasPathTraversal(archiveName, import_path.default.posix.sep)) {
+      this.#warnSkipFileBackup(collectionName, file, "Path traversal is not allowed");
+      return null;
+    }
+    return {
+      filePath: resolvedPath,
+      archiveName
+    };
+  }
+  #hasPathTraversal(filePath, separator) {
+    return filePath === ".." || filePath.startsWith(`..${separator}`) || filePath.split(separator).includes("..");
+  }
+  #warnSkipFileBackup(collectionName, file, reason) {
+    const recordId = file.id == null ? "" : ` id=${file.id}`;
+    this.app.logger.warn(
+      `Skip backing up invalid local file record from collection "${collectionName}"${recordId}: ${reason}`,
+      {
+        module: import_utils2.BACKUPS
+      }
+    );
+  }
+  async #createEncryptedStream(output, password) {
+    if (password) {
+      const key = await (0, import_util.promisify)(crypto.scrypt)(password, import_utils2.FILE_ENCRYPTION_SALT, 32);
+      const iv = crypto.randomBytes(16);
+      const cipher = crypto.createCipheriv("aes-256-cbc", key, iv);
+      try {
+        const written = output.write(iv);
+        if (!written) {
+          await new Promise((resolve, reject) => {
+            output.once("drain", resolve);
+            output.once("error", reject);
+          });
+        }
+      } catch (err) {
+        this.app.logger.error(`Failed to write IV to the output stream: ${err.message}`, { module: import_utils2.BACKUPS });
+        throw new Error(this.#t("ERROR_DECRYPTING_FILES", err.message));
+      }
+      cipher.pipe(output);
+      return cipher;
+    }
+    return output;
+  }
+  async #uploadFiles(filePath, storageId) {
+    if (!storageId) {
+      return;
+    }
+    try {
+      const storage = await this.app.db.getRepository("storages").findOne({
+        filterByTk: storageId
+      });
+      if (!storage || storage.type === "local") {
+        this.app.logger.info(`Has not set cloud storage, skip backup file upload process`, { module: import_utils2.BACKUPS });
+        return;
+      }
+      const Plugin = this.app.pm.get(import_plugin_file_manager.default);
+      const data = await Plugin.uploadFile({
+        filePath,
+        storageName: storage.name
+      });
+      this.app.logger.info(`Backup file uploaded to: ${data.filename}`, { module: import_utils2.BACKUPS });
+    } catch (error) {
+      this.app.logger.error(`Error uploading files: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_UPLOADING_FILES", error.message));
+    }
+  }
+  async #createContentPath(contentPath) {
+    try {
+      await import_promises.default.mkdir(contentPath, { recursive: true });
+    } catch (error) {
+      this.app.logger.error(`Error creating content path: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_CREATING_PATH", error.message));
+    }
+  }
+  async #removeContentPath(contentPath) {
+    try {
+      await import_promises.default.rm(contentPath, { recursive: true });
+    } catch (error) {
+      this.app.logger.error(`Error removing content path: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_REMOVING_FILES", error.message));
+    }
+  }
+  async #createLockFile(fileBaseName) {
+    const filePath = import_path.default.join(this.#tempDir, `${fileBaseName}.${import_utils2.BACKUP_EXTENSION}.lock`);
+    try {
+      await import_promises.default.mkdir(this.#tempDir, { recursive: true });
+      await import_promises.default.writeFile(filePath, "lock", "utf8");
+    } catch (error) {
+      this.app.logger.error(`Error creating lock file: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_CREATING_LOCK_FILE", error.message));
+    }
+  }
+  async #removeLockFile(fileBaseName) {
+    const filePath = import_path.default.join(this.#tempDir, `${fileBaseName}.${import_utils2.BACKUP_EXTENSION}.lock`);
+    try {
+      await import_promises.default.unlink(filePath);
+    } catch (error) {
+      this.app.logger.error(`Error removing lock file: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_REMOVING_FILES", error.message));
+    }
+  }
+  #generateFileBaseName() {
+    return `backup_${(0, import_dayjs.default)().format(`YYYYMMDD_HHmmss_${Math.floor(1e3 + Math.random() * 9e3)}`)}`;
+  }
+  async #listCompletedBackups(inProgressFiles = []) {
+    let files;
+    try {
+      files = await import_promises.default.readdir(this.#backupDir);
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return [];
+      }
+      this.app.logger.error(`Error listing backup files: ${error.message}`, { module: import_utils2.BACKUPS });
+      throw new Error(this.#t("ERROR_LISTING_BACKUPS", error.message));
+    }
+    const inProgressFileNames = inProgressFiles.map((file) => file.name);
+    const backupPromises = files.filter((file) => file.endsWith(`.${import_utils2.BACKUP_EXTENSION}`) && !inProgressFileNames.includes(file)).map(async (file) => {
+      const stats = await import_promises.default.stat(import_path.default.join(this.#backupDir, file));
+      return {
+        name: file,
+        fileSize: (0, import_utils2.humanFileSize)(stats.size),
+        createdAt: stats.ctime,
+        inProgress: false
+      };
+    });
+    const backups = await Promise.all(backupPromises);
+    return backups.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
+  }
+  async #listProgressBackups() {
+    try {
+      const files = await import_promises.default.readdir(this.#tempDir);
+      const backups = files.filter((file) => file.endsWith(`.${import_utils2.BACKUP_EXTENSION}.lock`)).map((file) => {
+        return {
+          name: file.replace(".lock", ""),
+          inProgress: true
+        };
+      });
+      return backups;
+    } catch (error) {
+      if (error.code === "ENOENT") {
+        return [];
+      } else {
+        this.app.logger.error(`Error listing lock files: ${error.message}`, { module: import_utils2.BACKUPS });
+        throw new Error(this.#t("ERROR_LISTING_BACKUPS", error.message));
+      }
+    }
+  }
+  async #dropOldBackups(keep) {
+    const inProgressBackups = await this.#listProgressBackups();
+    const backups = await this.#listCompletedBackups(inProgressBackups);
+    const files = backups.slice(keep);
+    await Promise.all(files.map((file) => this.destroy(file.name)));
+  }
+  #t(message, detail) {
+    var _a;
+    const t = ((_a = this.ctx) == null ? void 0 : _a.i18n.t) || this.app.i18n.t;
+    return t(message, { detail, ns: import_utils2.PLUGIN_BACKUPS_NAME, interpolation: { escapeValue: false } });
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  BackupManager
+});

package/dist/server/managers/restore.d.ts

@@ -0,0 +1,24 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+/// <reference types="multer" />
+import { ResourcerContext } from '@nocobase/resourcer';
+interface RestoreOptions {
+    forceSchemaRestore?: boolean;
+}
+export declare class RestoreManager {
+    #private;
+    ctx: ResourcerContext;
+    constructor(ctx: ResourcerContext, dbOptions?: any);
+    restoreFromBackup(backupFileName: string, taskId: string, password?: string, tolerentMode?: boolean, options?: RestoreOptions): Promise<void>;
+    restoreFromUpload(backupFile: Express.Multer.File, taskId: string, password?: string, tolerentMode?: boolean, options?: RestoreOptions): Promise<void>;
+    restoreCLI(filePath: string, password?: string, tolerentMode?: boolean, skipRevertOnError?: boolean, options?: RestoreOptions): Promise<void>;
+    private getStatusCache;
+    restore(filePath: string, taskId: string, password?: string, tolerentMode?: boolean, skipRevertOnError?: boolean, options?: RestoreOptions): Promise<void>;
+}
+export {};