@tachybase/plugin-database-clean 1.6.2

package/dist/server/services/filtered-backup-service.js

@@ -0,0 +1,269 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var filtered_backup_service_exports = {};
+__export(filtered_backup_service_exports, {
+  FilteredBackupService: () => FilteredBackupService
+});
+module.exports = __toCommonJS(filtered_backup_service_exports);
+var import_node_fs = __toESM(require("node:fs"));
+var import_promises = __toESM(require("node:fs/promises"));
+var import_node_path = __toESM(require("node:path"));
+var import_node_stream = __toESM(require("node:stream"));
+var import_node_util = __toESM(require("node:util"));
+var import_archiver = __toESM(require("archiver"));
+var import_dayjs = __toESM(require("dayjs"));
+var import_lodash = __toESM(require("lodash"));
+var import_adapters = require("../adapters");
+var import_utils = require("../utils");
+const finished = import_node_util.default.promisify(import_node_stream.default.finished);
+class FieldValueWriter {
+  static toDumpedValue(field, val) {
+    if (val === null) return val;
+    if (field.type === "point" || field.type === "lineString" || field.type === "circle" || field.type === "polygon") {
+      const mockObj = {
+        getDataValue: () => val
+      };
+      const newValue = field.options.get.call(mockObj);
+      return newValue;
+    }
+    return val;
+  }
+}
+class FilteredBackupService {
+  constructor(app) {
+    this.app = app;
+    this.workDir = import_node_path.default.resolve(process.env.TEGO_RUNTIME_HOME || process.cwd(), "storage", "temp", "database-clean");
+  }
+  /**
+   * 生成备份文件名
+   */
+  static generateFileName(collectionName, filter) {
+    var _a, _b, _c, _d, _e, _f;
+    const random = Math.floor(1e3 + Math.random() * 9e3);
+    const timestamp = (0, import_dayjs.default)().format("YYYYMMDD_HHmmss");
+    let rangeSuffix = "";
+    if (((_a = filter == null ? void 0 : filter.createdAt) == null ? void 0 : _a.$gte) || ((_b = filter == null ? void 0 : filter.createdAt) == null ? void 0 : _b.$lte)) {
+      const from = filter.createdAt.$gte ? (0, import_dayjs.default)(filter.createdAt.$gte).format("YYYYMMDD") : "start";
+      const to = filter.createdAt.$lte ? (0, import_dayjs.default)(filter.createdAt.$lte).format("YYYYMMDD") : "end";
+      rangeSuffix = `_createdAt_${from}_${to}`;
+    } else if (((_c = filter == null ? void 0 : filter.updatedAt) == null ? void 0 : _c.$gte) || ((_d = filter == null ? void 0 : filter.updatedAt) == null ? void 0 : _d.$lte)) {
+      const from = filter.updatedAt.$gte ? (0, import_dayjs.default)(filter.updatedAt.$gte).format("YYYYMMDD") : "start";
+      const to = filter.updatedAt.$lte ? (0, import_dayjs.default)(filter.updatedAt.$lte).format("YYYYMMDD") : "end";
+      rangeSuffix = `_updatedAt_${from}_${to}`;
+    } else if (((_e = filter == null ? void 0 : filter.id) == null ? void 0 : _e.$gte) !== void 0 || ((_f = filter == null ? void 0 : filter.id) == null ? void 0 : _f.$lte) !== void 0) {
+      const from = filter.id.$gte !== void 0 ? filter.id.$gte : "start";
+      const to = filter.id.$lte !== void 0 ? filter.id.$lte : "end";
+      rangeSuffix = `_id_${from}_${to}`;
+    }
+    return `db-clean_${collectionName}${rangeSuffix}_${timestamp}_${random}.tbdump`;
+  }
+  /**
+   * 备份单个集合的筛选数据
+   */
+  async backupCollection(options) {
+    import_adapters.DatabaseAdapterFactory.getAdapter(this.app);
+    const { collectionName, filter, fileName, appName } = options;
+    const collection = this.app.db.getCollection(collectionName);
+    if (!collection) {
+      throw new Error(`Collection ${collectionName} not found`);
+    }
+    await import_promises.default.mkdir(this.workDir, { recursive: true });
+    const collectionDataDir = import_node_path.default.resolve(this.workDir, "collections", collectionName);
+    await import_promises.default.mkdir(collectionDataDir, { recursive: true });
+    const attributes = collection.model.tableAttributes;
+    const columns = [...new Set(import_lodash.default.map(attributes, "field"))];
+    const whereConditions = [];
+    const replacements = {};
+    if (filter == null ? void 0 : filter.createdAt) {
+      if (filter.createdAt.$gte) {
+        whereConditions.push('"createdAt" >= :createdAtGte');
+        replacements.createdAtGte = filter.createdAt.$gte;
+      }
+      if (filter.createdAt.$lte) {
+        whereConditions.push('"createdAt" <= :createdAtLte');
+        replacements.createdAtLte = filter.createdAt.$lte;
+      }
+    }
+    if (filter == null ? void 0 : filter.updatedAt) {
+      if (filter.updatedAt.$gte) {
+        whereConditions.push('"updatedAt" >= :updatedAtGte');
+        replacements.updatedAtGte = filter.updatedAt.$gte;
+      }
+      if (filter.updatedAt.$lte) {
+        whereConditions.push('"updatedAt" <= :updatedAtLte');
+        replacements.updatedAtLte = filter.updatedAt.$lte;
+      }
+    }
+    if (filter == null ? void 0 : filter.id) {
+      const primaryKey = collection.model.primaryKeyAttribute || "id";
+      if (filter.id.$gte !== void 0) {
+        whereConditions.push(`"${primaryKey}" >= :idGte`);
+        replacements.idGte = filter.id.$gte;
+      }
+      if (filter.id.$lte !== void 0) {
+        whereConditions.push(`"${primaryKey}" <= :idLte`);
+        replacements.idLte = filter.id.$lte;
+      }
+    }
+    const whereClause = whereConditions.length > 0 ? `WHERE ${whereConditions.join(" AND ")}` : "";
+    const dataFilePath = import_node_path.default.resolve(collectionDataDir, "data");
+    const dataStream = import_node_fs.default.createWriteStream(dataFilePath);
+    let count = 0;
+    const sql = `SELECT * FROM ${collection.isParent() ? "ONLY" : ""} ${collection.quotedTableName()} ${whereClause}`;
+    const rows = await this.app.db.sequelize.query((0, import_utils.sqlAdapter)(this.app.db, sql), {
+      type: "SELECT",
+      replacements
+    });
+    for (const row of rows) {
+      const rowData = JSON.stringify(
+        columns.map((col) => {
+          const val = row[col];
+          const field = collection.getField(col);
+          return field ? FieldValueWriter.toDumpedValue(field, val) : val;
+        })
+      );
+      if (!dataStream.write(rowData + "\r\n", "utf8")) {
+        await new Promise((resolve) => dataStream.once("drain", resolve));
+      }
+      count++;
+    }
+    dataStream.end();
+    await finished(dataStream);
+    const metaAttributes = import_lodash.default.mapValues(attributes, (attr, key) => {
+      var _a, _b, _c;
+      const collectionField = collection.getField(key);
+      const fieldOptionKeys = ["field", "primaryKey", "autoIncrement", "allowNull", "defaultValue", "unique"];
+      if (collectionField) {
+        const fieldAttributes = {
+          field: attr.field,
+          isCollectionField: true,
+          type: collectionField.type,
+          typeOptions: collectionField.options
+        };
+        if (((_c = (_b = (_a = fieldAttributes.typeOptions) == null ? void 0 : _a.defaultValue) == null ? void 0 : _b.constructor) == null ? void 0 : _c.name) === "UUIDV4") {
+          delete fieldAttributes.typeOptions.defaultValue;
+        }
+        return fieldAttributes;
+      }
+      return {
+        ...import_lodash.default.pick(attr, fieldOptionKeys),
+        type: attr.type.constructor.toString(),
+        isCollectionField: false,
+        typeOptions: attr.type.options
+      };
+    });
+    const meta = {
+      name: collectionName,
+      tableName: collection.getTableNameWithSchema(),
+      count,
+      columns,
+      attributes: metaAttributes,
+      filter
+      // 保存筛选条件
+    };
+    if (collection.options.inherits) {
+      meta["inherits"] = import_lodash.default.uniq(collection.options.inherits);
+    }
+    await import_promises.default.writeFile(import_node_path.default.resolve(collectionDataDir, "meta"), JSON.stringify(meta), "utf8");
+    const rootMeta = {
+      version: this.app.getVersion(),
+      dialect: this.app.db.sequelize.getDialect(),
+      DB_UNDERSCORED: process.env.DB_UNDERSCORED || "false",
+      DB_SCHEMA: process.env.DB_SCHEMA || "",
+      COLLECTION_MANAGER_SCHEMA: process.env.COLLECTION_MANAGER_SCHEMA || "",
+      DB_TABLE_PREFIX: process.env.DB_TABLE_PREFIX || "",
+      // 标记为 database-clean 插件生成的部分备份
+      backupType: "database-clean",
+      dumpableCollectionsGroupByGroup: {
+        custom: [
+          {
+            name: collectionName,
+            group: "custom",
+            origin: "@tachybase/plugin-database-clean",
+            title: collectionName,
+            isView: false
+          }
+        ]
+      },
+      dumpedGroups: ["custom"],
+      delayCollections: [],
+      filter
+      // 保存筛选条件供参考
+    };
+    await import_promises.default.writeFile(import_node_path.default.resolve(this.workDir, "meta"), JSON.stringify(rootMeta), "utf8");
+    const backupFileName = fileName || FilteredBackupService.generateFileName(collectionName, filter);
+    const filePath = await this.packDumpedDir(backupFileName, appName);
+    await this.clearWorkDir();
+    return {
+      fileName: backupFileName,
+      filePath: filePath.filePath
+    };
+  }
+  async packDumpedDir(fileName, appName) {
+    const storageDir = this.getStorageDir(appName);
+    await import_promises.default.mkdir(storageDir, { recursive: true });
+    const filePath = import_node_path.default.resolve(storageDir, fileName);
+    const output = import_node_fs.default.createWriteStream(filePath);
+    const archive = (0, import_archiver.default)("zip", {
+      zlib: { level: 9 }
+    });
+    const onClose = new Promise((resolve, reject) => {
+      output.on("close", function() {
+        resolve({
+          filePath,
+          dirname: storageDir
+        });
+      });
+      archive.on("error", function(err) {
+        reject(err);
+      });
+    });
+    archive.pipe(output);
+    archive.directory(this.workDir, false);
+    await archive.finalize();
+    await onClose;
+    return { filePath, dirname: storageDir };
+  }
+  getStorageDir(appName) {
+    if (appName && appName !== "main") {
+      return import_node_path.default.resolve(process.env.TEGO_RUNTIME_HOME || process.cwd(), "storage", "backups", appName);
+    }
+    return import_node_path.default.resolve(process.env.TEGO_RUNTIME_HOME || process.cwd(), "storage", "backups");
+  }
+  async clearWorkDir() {
+    try {
+      await import_promises.default.rm(this.workDir, { recursive: true, force: true });
+    } catch (error) {
+      this.app.logger.warn("Failed to clear work directory:", error);
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  FilteredBackupService
+});

package/dist/server/utils/lock.d.ts

@@ -0,0 +1,23 @@
+import { Application } from '@tego/server';
+/**
+ * 基于缓存的锁实现，用于防止并发操作
+ */
+export declare class DatabaseCleanLock {
+    private app;
+    private cache;
+    private readonly LOCK_PREFIX;
+    private readonly LOCK_TTL;
+    constructor(app: Application);
+    /**
+     * 获取锁
+     */
+    acquire(collectionName: string): Promise<boolean>;
+    /**
+     * 释放锁
+     */
+    release(collectionName: string): Promise<void>;
+    /**
+     * 检查锁是否存在
+     */
+    isLocked(collectionName: string): Promise<boolean>;
+}

package/dist/server/utils/lock.js

@@ -0,0 +1,62 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var lock_exports = {};
+__export(lock_exports, {
+  DatabaseCleanLock: () => DatabaseCleanLock
+});
+module.exports = __toCommonJS(lock_exports);
+class DatabaseCleanLock {
+  // 2小时
+  constructor(app) {
+    this.app = app;
+    this.LOCK_PREFIX = "database-clean:lock:";
+    this.LOCK_TTL = 2 * 60 * 60 * 1e3;
+    this.cache = app.cache;
+  }
+  /**
+   * 获取锁
+   */
+  async acquire(collectionName) {
+    const lockKey = `${this.LOCK_PREFIX}${collectionName}`;
+    const existingLock = await this.cache.get(lockKey);
+    if (existingLock) {
+      return false;
+    }
+    await this.cache.set(lockKey, Date.now(), this.LOCK_TTL);
+    return true;
+  }
+  /**
+   * 释放锁
+   */
+  async release(collectionName) {
+    const lockKey = `${this.LOCK_PREFIX}${collectionName}`;
+    await this.cache.del(lockKey);
+  }
+  /**
+   * 检查锁是否存在
+   */
+  async isLocked(collectionName) {
+    const lockKey = `${this.LOCK_PREFIX}${collectionName}`;
+    const lock = await this.cache.get(lockKey);
+    return !!lock;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DatabaseCleanLock
+});

package/dist/server/utils.d.ts

@@ -0,0 +1,2 @@
+import { Database } from '@tego/server';
+export declare function sqlAdapter(database: Database, sql: string): string;

package/dist/server/utils.js

@@ -0,0 +1,43 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var utils_exports = {};
+__export(utils_exports, {
+  sqlAdapter: () => sqlAdapter
+});
+module.exports = __toCommonJS(utils_exports);
+var import_lodash = __toESM(require("lodash"));
+function sqlAdapter(database, sql) {
+  if (database.isMySQLCompatibleDialect()) {
+    return import_lodash.default.replace(sql, /"/g, "`");
+  }
+  return sql;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  sqlAdapter
+});

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@tachybase/plugin-database-clean",
+  "displayName": "Database Cleanup Tool",
+  "version": "1.6.2",
+  "description": "Clean up database.",
+  "main": "dist/server/index.js",
+  "devDependencies": {
+    "@ant-design/icons": "^5.6.1",
+    "@tachybase/client": "1.6.1",
+    "@tachybase/test": "1.6.1",
+    "@tego/client": "1.6.1",
+    "@tego/server": "1.6.1",
+    "@types/archiver": "^5.3.4",
+    "@types/file-saver": "^2.0.7",
+    "@types/lodash": "^4.17.20",
+    "antd": "5.22.5",
+    "archiver": "^7.0.1",
+    "dayjs": "1.11.13",
+    "file-saver": "^2.0.5",
+    "lodash": "4.17.21",
+    "react-router-dom": "6.28.1",
+    "sequelize": "^6.37.5"
+  },
+  "description.zh-CN": "数据库清理。",
+  "displayName.zh-CN": "数据库清理工具"
+}

package/server.d.ts

@@ -0,0 +1,2 @@
+export * from './dist/server';
+export { default } from './dist/server';

package/server.js

@@ -0,0 +1 @@
+module.exports = require('./dist/server/index.js');