@underpostnet/cyberia 3.2.80

package/src/cli/db.js

@@ -0,0 +1,1533 @@
+/**
+ * UnderpostDB CLI module
+ * @module src/cli/db.js
+ * @namespace UnderpostDB
+ * @description Manages database operations, backups, and cluster metadata for Kubernetes deployments.
+ * Supports MariaDB and MongoDB with import/export capabilities, Git integration, and multi-pod operations.
+ */
+
+import { mergeFile, splitFileFactory, loadConfServerJson, resolveConfSecrets, loadConfInstances } from '../server/conf.js';
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+import { DataBaseProviderService } from '../db/DataBaseProvider.js';
+import { loadReplicas, pathPortAssignmentFactory, loadCronDeployEnv } from '../server/conf.js';
+import { MongoBootstrap } from '../db/mongo/MongoBootstrap.js';
+import Underpost from '../index.js';
+import { timer } from '../client/components/core/CommonJs.js';
+const logger = loggerFactory(import.meta);
+
+/**
+ * Constants for database operations
+ * @constant {number} MAX_BACKUP_RETENTION - Maximum number of backups to retain
+ * @memberof UnderpostDB
+ */
+const MAX_BACKUP_RETENTION = 5;
+
+/**
+ * @typedef {Object} DatabaseOptions
+ * @memberof UnderpostDB
+ * @property {boolean} [import=false] - Flag to import data from a backup
+ * @property {boolean} [export=false] - Flag to export data to a backup
+ * @property {string} [podName=''] - Comma-separated list of pod names or patterns
+ * @property {string} [ns='default'] - Kubernetes namespace
+ * @property {string} [collections=''] - Comma-separated list of collections to include
+ * @property {string} [outPath=''] - Output path for backup files
+ * @property {boolean} [drop=false] - Flag to drop the database before importing
+ * @property {boolean} [preserveUUID=false] - Flag to preserve UUIDs during import
+ * @property {boolean} [git=false] - Flag to enable Git integration
+ * @property {string} [hosts=''] - Comma-separated list of hosts to include
+ * @property {string} [paths=''] - Comma-separated list of paths to include
+ * @property {boolean} [allPods=false] - Flag to target all matching pods
+ * @property {boolean} [primaryPod=false] - Flag to automatically detect and use MongoDB primary pod
+ * @property {boolean} [stats=false] - Flag to display collection/table statistics
+ * @property {boolean} [forceClone=false] - Flag to force remove and re-clone cron backup repository
+ */
+
+/**
+ * @typedef {Object} PodInfo
+ * @memberof UnderpostDB
+ * @property {string} NAME - Pod name
+ * @property {string} NAMESPACE - Pod namespace
+ * @property {string} NODE - Node where pod is running
+ * @property {string} STATUS - Pod status
+ * @property {string} [IP] - Pod IP address
+ */
+
+/**
+ * @typedef {Object} DatabaseConfig
+ * @memberof UnderpostDB
+ * @property {string} provider - Database provider (mariadb, mongoose)
+ * @property {string} name - Database name
+ * @property {string} user - Database user
+ * @property {string} password - Database password
+ * @property {string} hostFolder - Host folder path
+ * @property {string} host - Host identifier
+ * @property {string} path - Path identifier
+ * @property {number} [currentBackupTimestamp] - Timestamp of current backup
+ */
+
+/**
+ * @class UnderpostDB
+ * @description Manages database operations and backups for Kubernetes-based deployments.
+ * Provides comprehensive database management including import/export, multi-pod targeting,
+ * Git integration, and cluster metadata management.
+ */
+/**
+ * UnderpostDB class for managing database operations and backups.
+ * @class UnderpostDB
+ * @memberof UnderpostDB
+ */
+class UnderpostDB {
+  /**
+   * Static API object containing all database operation methods.
+   * @static
+   * @memberof UnderpostDB
+   */
+  static API = {
+    /**
+     * Helper: Resolves the latest backup timestamp from an existing backup directory.
+     * Scans the directory for numeric (epoch) sub-folders and returns the most recent one.
+     * @method _getLatestBackupTimestamp
+     * @memberof UnderpostDB
+     * @param {string} backupDir - Path to the host-folder backup directory.
+     * @return {string|null} The latest timestamp string, or null if none found.
+     */
+    _getLatestBackupTimestamp(backupDir) {
+      if (!fs.existsSync(backupDir)) return null;
+      const entries = fs.readdirSync(backupDir).filter((e) => /^\d+$/.test(e));
+      if (entries.length === 0) return null;
+      return entries.sort((a, b) => parseInt(b) - parseInt(a))[0];
+    },
+
+    /**
+     * Helper: Performs MariaDB import operation.
+     * @method _importMariaDB
+     * @memberof UnderpostDB
+     * @param {Object} params - Import parameters.
+     * @param {PodInfo} params.pod - Target pod.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+     * @param {string} params.user - Database user.
+     * @param {string} params.password - Database password.
+     * @param {string} params.sqlPath - SQL file path.
+     * @return {boolean} Success status.
+     */
+    _importMariaDB({ pod, namespace, dbName, user, password, sqlPath }) {
+      try {
+        const podName = pod.NAME;
+        const containerSqlPath = `/${dbName}.sql`;
+
+        logger.info('Importing MariaDB database', { podName, dbName });
+
+        // Always ensure the database exists first — required for WP even when no backup is available
+        Underpost.kubectl.run(
+          `kubectl exec -n ${namespace} -i ${podName} -- mariadb -p${password} -e 'CREATE DATABASE IF NOT EXISTS ${dbName};'`,
+          { context: `create database ${dbName}` },
+        );
+
+        // If no SQL file is available, the empty database is enough — return early
+        if (!sqlPath || !fs.existsSync(sqlPath)) {
+          logger.warn('No SQL backup file found — empty database ensured', { podName, dbName, sqlPath });
+          return true;
+        }
+
+        // Remove existing SQL file in container
+        Underpost.kubectl.exec({
+          podName,
+          namespace,
+          command: `rm -rf ${containerSqlPath}`,
+        });
+
+        // Copy SQL file to pod
+        if (
+          !Underpost.kubectl.cpTo({
+            sourcePath: sqlPath,
+            podName,
+            namespace,
+            destPath: containerSqlPath,
+          })
+        ) {
+          return false;
+        }
+
+        // Import SQL file
+        const importCmd = `mariadb -u ${user} -p${password} ${dbName} < ${containerSqlPath}`;
+        Underpost.kubectl.exec({ podName, namespace, command: importCmd });
+
+        logger.info('Successfully imported MariaDB database', { podName, dbName });
+        return true;
+      } catch (error) {
+        logger.error('MariaDB import failed', { podName: pod.NAME, dbName, error: error.message });
+        return false;
+      }
+    },
+
+    /**
+     * Helper: Performs MariaDB export operation.
+     * @method _exportMariaDB
+     * @memberof UnderpostDB
+     * @param {Object} params - Export parameters.
+     * @param {PodInfo} params.pod - Source pod.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+     * @param {string} params.user - Database user.
+     * @param {string} params.password - Database password.
+     * @param {string} params.outputPath - Output file path.
+     * @return {Promise<boolean>} A promise that resolves with success status.
+     */
+    async _exportMariaDB({ pod, namespace, dbName, user, password, outputPath }) {
+      try {
+        const podName = pod.NAME;
+        const containerSqlPath = `/home/${dbName}.sql`;
+
+        logger.info('Exporting MariaDB database', { podName, dbName });
+
+        // Remove existing SQL file in container
+        Underpost.kubectl.exec({
+          podName,
+          namespace,
+          command: `rm -rf ${containerSqlPath}`,
+        });
+
+        // Dump database
+        const dumpCmd = `mariadb-dump --user=${user} --password=${password} --lock-tables ${dbName} > ${containerSqlPath}`;
+        Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
+
+        // Copy SQL file from pod
+        if (
+          !Underpost.kubectl.cpFrom({
+            podName,
+            namespace,
+            sourcePath: containerSqlPath,
+            destPath: outputPath,
+          })
+        ) {
+          return false;
+        }
+
+        // Split file if it exists
+        if (fs.existsSync(outputPath)) {
+          await splitFileFactory(dbName, outputPath);
+        }
+
+        logger.info('Successfully exported MariaDB database', { podName, dbName, outputPath });
+        return true;
+      } catch (error) {
+        logger.error('MariaDB export failed', { podName: pod.NAME, dbName, error: error.message });
+        return false;
+      }
+    },
+
+    /**
+     * Helper: Performs MongoDB import operation.
+     * @method _importMongoDB
+     * @memberof UnderpostDB
+     * @param {Object} params - Import parameters.
+     * @param {PodInfo} params.pod - Target pod.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+     * @param {string} params.bsonPath - BSON directory path.
+     * @param {boolean} params.drop - Whether to drop existing database.
+     * @param {boolean} params.preserveUUID - Whether to preserve UUIDs.
+      * @param {string} [params.user=''] - MongoDB username for authenticated restore.
+      * @param {string} [params.password=''] - MongoDB password for authenticated restore.
+      * @param {string} [params.authDatabase='admin'] - Auth database for restore command.
+     * @return {boolean} Success status.
+     */
+    _importMongoDB({ pod, namespace, dbName, bsonPath, drop, preserveUUID, user = '', password = '', authDatabase = 'admin' }) {
+      try {
+        const podName = pod.NAME;
+        const containerBsonPath = `/${dbName}`;
+
+        logger.info('Importing MongoDB database', { podName, dbName });
+
+        // If no BSON directory is available, skip — MongoDB creates the DB on first write
+        if (!bsonPath || !fs.existsSync(bsonPath)) {
+          logger.warn('No BSON backup directory found — database will be created on first write', {
+            podName,
+            dbName,
+            bsonPath,
+          });
+          return true;
+        }
+
+        // Remove existing BSON directory in container
+        Underpost.kubectl.exec({
+          podName,
+          namespace,
+          command: `rm -rf ${containerBsonPath}`,
+        });
+
+        // Copy BSON directory to pod
+        if (
+          !Underpost.kubectl.cpTo({
+            sourcePath: bsonPath,
+            podName,
+            namespace,
+            destPath: containerBsonPath,
+          })
+        ) {
+          return false;
+        }
+
+        // Restore database
+        const authFlags = user && password
+          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
+          : '';
+        const restoreCmd = `mongorestore -d ${dbName} ${containerBsonPath}${drop ? ' --drop' : ''}${preserveUUID ? ' --preserveUUID' : ''}${authFlags}`;
+        Underpost.kubectl.exec({ podName, namespace, command: restoreCmd });
+
+        logger.info('Successfully imported MongoDB database', { podName, dbName });
+        return true;
+      } catch (error) {
+        logger.error('MongoDB import failed', { podName: pod.NAME, dbName, error: error.message });
+        return false;
+      }
+    },
+
+    /**
+     * Helper: Performs MongoDB export operation.
+     * @method _exportMongoDB
+     * @memberof UnderpostDB
+     * @param {Object} params - Export parameters.
+     * @param {PodInfo} params.pod - Source pod.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+     * @param {string} params.outputPath - Output directory path.
+     * @param {string} [params.collections=''] - Comma-separated collection list.
+      * @param {string} [params.user=''] - MongoDB username for authenticated dump.
+      * @param {string} [params.password=''] - MongoDB password for authenticated dump.
+      * @param {string} [params.authDatabase='admin'] - Auth database for dump command.
+     * @return {boolean} Success status.
+     */
+    _exportMongoDB({ pod, namespace, dbName, outputPath, collections = '', user = '', password = '', authDatabase = 'admin' }) {
+      try {
+        const podName = pod.NAME;
+        const containerBsonPath = `/${dbName}`;
+
+        logger.info('Exporting MongoDB database', { podName, dbName, collections });
+
+        // Remove existing BSON directory in container
+        Underpost.kubectl.exec({
+          podName,
+          namespace,
+          command: `rm -rf ${containerBsonPath}`,
+        });
+
+        // Dump database or specific collections
+        const authFlags = user && password
+          ? ` --username ${JSON.stringify(user)} --password ${JSON.stringify(password)} --authenticationDatabase ${JSON.stringify(authDatabase)}`
+          : '';
+
+        if (collections) {
+          const collectionList = collections.split(',').map((c) => c.trim());
+          for (const collection of collectionList) {
+            const dumpCmd = `mongodump -d ${dbName} --collection ${collection} -o /${authFlags}`;
+            Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
+          }
+        } else {
+          const dumpCmd = `mongodump -d ${dbName} -o /${authFlags}`;
+          Underpost.kubectl.exec({ podName, namespace, command: dumpCmd });
+        }
+
+        // Copy BSON directory from pod
+        if (
+          !Underpost.kubectl.cpFrom({
+            podName,
+            namespace,
+            sourcePath: containerBsonPath,
+            destPath: outputPath,
+          })
+        ) {
+          return false;
+        }
+
+        logger.info('Successfully exported MongoDB database', { podName, dbName, outputPath });
+        return true;
+      } catch (error) {
+        logger.error('MongoDB export failed', { podName: pod.NAME, dbName, error: error.message });
+        return false;
+      }
+    },
+
+    /**
+     * Helper: Gets MongoDB collection statistics.
+     * @method _getMongoStats
+     * @memberof UnderpostDB
+     * @param {Object} params - Parameters.
+     * @param {string} params.podName - Pod name.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+      * @param {string} [params.user=''] - MongoDB username for authenticated stats query.
+      * @param {string} [params.password=''] - MongoDB password for authenticated stats query.
+      * @param {string} [params.authDatabase='admin'] - Auth database for stats query.
+     * @return {Object|null} Collection statistics or null on error.
+     */
+    _getMongoStats({ podName, namespace, dbName, user = '', password = '', authDatabase = 'admin' }) {
+      try {
+        logger.info('Getting MongoDB collection statistics', { podName, dbName });
+
+        // Use db.getSiblingDB() instead of 'use' command
+        const script = `db.getSiblingDB('${dbName}').getCollectionNames().map(function(c) { return { collection: c, count: db.getSiblingDB('${dbName}')[c].countDocuments() }; })`;
+
+        // Execute the script
+        const authFlags = user && password
+          ? ` --authenticationDatabase ${JSON.stringify(authDatabase)} -u ${JSON.stringify(user)} -p ${JSON.stringify(password)}`
+          : '';
+        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mongosh --quiet${authFlags} --eval "${script}"`;
+        const output = shellExec(command, { stdout: true, silent: true, silentOnError: true });
+
+        if (!output || output.trim() === '') {
+          logger.warn('No collections found or empty output');
+          return null;
+        }
+
+        // Clean the output: remove newlines, handle EJSON format, replace single quotes with double quotes
+        let cleanedOutput = output
+          .trim()
+          .replace(/\n/g, '')
+          .replace(/\s+/g, ' ')
+          .replace(/NumberLong\("(\d+)"\)/g, '$1')
+          .replace(/NumberLong\((\d+)\)/g, '$1')
+          .replace(/NumberInt\("(\d+)"\)/g, '$1')
+          .replace(/NumberInt\((\d+)\)/g, '$1')
+          .replace(/ISODate\("([^"]+)"\)/g, '"$1"')
+          .replace(/'/g, '"')
+          .replace(/(\w+):/g, '"$1":');
+
+        try {
+          const stats = JSON.parse(cleanedOutput);
+          logger.info('MongoDB statistics retrieved', { dbName, collections: stats.length });
+          return stats;
+        } catch (parseError) {
+          logger.error('Failed to parse MongoDB output', {
+            podName,
+            dbName,
+            error: parseError.message,
+            rawOutput: output.substring(0, 200),
+            cleanedOutput: cleanedOutput.substring(0, 200),
+          });
+          return null;
+        }
+      } catch (error) {
+        logger.error('Failed to get MongoDB statistics', { podName, dbName, error: error.message });
+        return null;
+      }
+    },
+
+    /**
+     * Helper: Gets MariaDB table statistics.
+     * @method _getMariaDBStats
+     * @memberof UnderpostDB
+     * @param {Object} params - Parameters.
+     * @param {string} params.podName - Pod name.
+     * @param {string} params.namespace - Namespace.
+     * @param {string} params.dbName - Database name.
+     * @param {string} params.user - Database user.
+     * @param {string} params.password - Database password.
+     * @return {Object|null} Table statistics or null on error.
+     */
+    _getMariaDBStats({ podName, namespace, dbName, user, password }) {
+      try {
+        logger.info('Getting MariaDB table statistics', { podName, dbName });
+
+        const command = `sudo kubectl exec -n ${namespace} -i ${podName} -- mariadb -u ${user} -p${password} ${dbName} -e "SELECT TABLE_NAME as 'table', TABLE_ROWS as 'count' FROM information_schema.TABLES WHERE TABLE_SCHEMA = '${dbName}' ORDER BY TABLE_NAME;" --skip-column-names --batch`;
+        const output = shellExec(command, { stdout: true, silent: true, disableLog: true, silentOnError: true });
+
+        if (!output || output.trim() === '') {
+          logger.warn('No tables found or empty output');
+          return null;
+        }
+
+        // Parse the output (tab-separated values)
+        const lines = output.trim().split('\n');
+        const stats = lines.map((line) => {
+          const [table, count] = line.split('\t');
+          return { table, count: parseInt(count) || 0 };
+        });
+
+        logger.info('MariaDB statistics retrieved', { dbName, tables: stats.length });
+        return stats;
+      } catch (error) {
+        logger.error('Failed to get MariaDB statistics', { podName, dbName, error: error.message });
+        return null;
+      }
+    },
+
+    /**
+     * Helper: Displays database statistics in table format.
+     * @method _displayStats
+     * @memberof UnderpostDB
+     * @param {Object} params - Parameters.
+     * @param {string} params.provider - Database provider.
+     * @param {string} params.dbName - Database name.
+     * @param {Array<Object>} params.stats - Statistics array.
+     * @return {void}
+     */
+    _displayStats({ provider, dbName, stats }) {
+      if (!stats || stats.length === 0) {
+        logger.warn('No statistics to display', { provider, dbName });
+        return;
+      }
+
+      const title = provider === 'mongoose' ? 'Collections' : 'Tables';
+      const itemKey = provider === 'mongoose' ? 'collection' : 'table';
+
+      console.log('\n' + '='.repeat(70));
+      console.log(`DATABASE: ${dbName} (${provider.toUpperCase()})`);
+      console.log('='.repeat(70));
+      console.log(`${title.padEnd(50)} ${'Documents/Rows'.padStart(18)}`);
+      console.log('-'.repeat(70));
+
+      let totalCount = 0;
+      stats.forEach((item) => {
+        const name = item[itemKey] || 'Unknown';
+        const count = item.count || 0;
+        totalCount += count;
+        console.log(`${name.padEnd(50)} ${count.toString().padStart(18)}`);
+      });
+
+      console.log('-'.repeat(70));
+      console.log(`${'TOTAL'.padEnd(50)} ${totalCount.toString().padStart(18)}`);
+      console.log('='.repeat(70) + '\n');
+    },
+
+
+
+    /**
+     * Main callback: Initiates database backup workflow.
+     * Orchestrates the backup process for multiple deployments, handling
+     * database connections, backup storage, and optional Git integration for version control.
+     * Supports targeting multiple specific pods, nodes, and namespaces with advanced filtering.
+     * @method callback
+     * @memberof UnderpostDB
+     * @param {string} [deployList='default'] - Comma-separated list of deployment IDs.
+     * @param {Object} [options={}] - Backup options.
+     * @param {boolean} [options.import=false] - Whether to perform import operation.
+     * @param {boolean} [options.export=false] - Whether to perform export operation.
+     * @param {string} [options.podName=''] - Comma-separated pod name patterns to target.
+     * @param {string} [options.ns='default'] - Kubernetes namespace.
+     * @param {string} [options.collections=''] - Comma-separated MongoDB collections for export.
+     * @param {string} [options.outPath=''] - Output path for backups.
+     * @param {boolean} [options.drop=false] - Whether to drop existing database on import.
+     * @param {boolean} [options.preserveUUID=false] - Whether to preserve UUIDs on MongoDB import.
+     * @param {boolean} [options.git=false] - Whether to use Git for backup versioning.
+     * @param {string} [options.hosts=''] - Comma-separated list of hosts to filter databases.
+     * @param {string} [options.paths=''] - Comma-separated list of paths to filter databases.
+     * @param {boolean} [options.allPods=false] - Whether to target all pods in deployment.
+     * @param {boolean} [options.primaryPod=false] - Whether to target MongoDB primary pod only.
+     * @param {string} [options.primaryPodEnsure=''] - Pod name to ensure MongoDB primary pod is running.
+     * @param {boolean} [options.stats=false] - Whether to display database statistics.
+     * @param {number} [options.macroRollbackExport=1] - Number of commits to rollback in macro export.
+     * @param {boolean} [options.forceClone=false] - Whether to force re-clone Git repository.
+     * @param {boolean} [options.cleanFsCollection=false] - Clean orphaned File documents flag.
+     * @param {boolean} [options.cleanFsDryRun=false] - Dry run mode flag (use with cleanFsCollection).
+     * @param {boolean} [options.dev=false] - Development mode flag.
+     * @param {boolean} [options.k3s=false] - k3s cluster flag.
+     * @param {boolean} [options.kubeadm=false] - kubeadm cluster flag.
+     * @param {boolean} [options.kind=false] - kind cluster flag.
+     * @param {boolean} [options.repoBackup=false] - Backs up repositories (git commit+push) inside deployment pods via kubectl exec.
+     * @return {Promise<void>} Resolves when operation is complete.
+     */
+    async callback(
+      deployList = 'default',
+      options = {
+        import: false,
+        export: false,
+        podName: '',
+        ns: 'default',
+        collections: '',
+        outPath: '',
+        drop: false,
+        preserveUUID: false,
+        git: false,
+        hosts: '',
+        paths: '',
+        allPods: false,
+        primaryPod: false,
+        primaryPodEnsure: '',
+        stats: false,
+        macroRollbackExport: 1,
+        forceClone: false,
+        cleanFsCollection: false,
+        cleanFsDryRun: false,
+        dev: false,
+        k3s: false,
+        kubeadm: false,
+        kind: false,
+        repoBackup: false,
+      },
+    ) {
+      // Ensure engine-private is available (clone if inside a deployment
+      // container where globalSecretClean has already removed it).
+      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
+      try {
+        loadCronDeployEnv();
+        const newBackupTimestamp = new Date().getTime();
+        const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+
+        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
+
+        // Handle repository backup (git commit+push inside deployment pod)
+        if (options.repoBackup) {
+          const namespace = options.ns && typeof options.ns === 'string' ? options.ns : 'default';
+          for (const _deployId of deployList.split(',')) {
+            const deployId = _deployId.trim();
+            if (!deployId) continue;
+            logger.info('Starting pod repository backup', { deployId, namespace });
+            Underpost.repo.backupPodRepositories({
+              deployId,
+              namespace,
+              env: options.dev ? 'development' : 'production',
+            });
+          }
+          return;
+        }
+
+        // Handle clean-fs-collection operation
+        if (options.cleanFsCollection || options.cleanFsDryRun) {
+          logger.info('Starting File collection cleanup operation', { deployList });
+          await Underpost.db.cleanFsCollection(deployList, {
+            hosts: options.hosts,
+            paths: options.paths,
+            dryRun: options.cleanFsDryRun,
+          });
+          return;
+        }
+
+        logger.info('Starting database operation', {
+          deployList,
+          namespace,
+          import: options.import,
+          export: options.export,
+        });
+
+        if (options.primaryPodEnsure) {
+          const primaryPodName = MongoBootstrap.getPrimaryPodName({
+            namespace,
+            podName: options.primaryPodEnsure,
+            username: process.env.MONGODB_USERNAME || process.env.DB_USER || '',
+            password: process.env.MONGODB_PASSWORD || process.env.DB_PASSWORD || '',
+            authDatabase: process.env.MONGODB_AUTH_DB || 'admin',
+          });
+          if (!primaryPodName) {
+            const baseCommand = options.dev ? 'node bin' : 'underpost';
+            const baseClusterCommand = options.dev ? ' --dev' : '';
+            let clusterFlag = options.k3s ? ' --k3s' : options.kubeadm ? ' --kubeadm' : '';
+            shellExec(`${baseCommand} cluster${baseClusterCommand}${clusterFlag} --mongodb`);
+          }
+          return;
+        }
+
+        // Track processed repositories to avoid duplicate Git operations
+        const processedRepos = new Set();
+        // Track processed host+path combinations to avoid duplicates
+        const processedHostPaths = new Set();
+
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+
+          logger.info('Processing deployment', { deployId });
+
+          /** @type {Object.<string, Object.<string, DatabaseConfig>>} */
+          const dbs = {};
+          const repoName = `engine-${deployId.includes('dd-') ? deployId.split('dd-')[1] : deployId}-cron-backups`;
+
+          // Load server configuration
+          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+          if (!fs.existsSync(confServerPath)) {
+            logger.error('Configuration file not found', { path: confServerPath });
+            continue;
+          }
+
+          const confServer = loadConfServerJson(confServerPath, { resolve: true });
+
+          // Build database configuration map
+          for (const host of Object.keys(confServer)) {
+            for (const path of Object.keys(confServer[host])) {
+              const { db } = confServer[host][path];
+              if (db) {
+                const { provider, name, user, password } = db;
+                if (!dbs[provider]) dbs[provider] = {};
+
+                if (!(name in dbs[provider])) {
+                  dbs[provider][name] = {
+                    user,
+                    password,
+                    hostFolder: host + path.replaceAll('/', '-'),
+                    host,
+                    path,
+                  };
+                }
+              }
+            }
+          }
+
+          // Handle Git operations - execute only once per repository
+          if (!processedRepos.has(repoName)) {
+            logger.info('Processing Git operations for repository', { repoName, deployId });
+            if (options.git === true) {
+              Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+              Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
+            }
+
+            if (options.macroRollbackExport) {
+              // Only clone if not already done by git option above
+              if (options.git !== true) {
+                Underpost.repo.manageBackupRepo({ repoName, operation: 'clone', forceClone: options.forceClone });
+                Underpost.repo.manageBackupRepo({ repoName, operation: 'pull' });
+              }
+
+              const nCommits = parseInt(options.macroRollbackExport);
+              const repoPath = `../${repoName}`;
+              const username = process.env.GITHUB_USERNAME;
+
+              if (fs.existsSync(repoPath) && username) {
+                logger.info('Executing macro rollback export', { repoName, nCommits });
+                shellExec(`cd ${repoPath} && underpost cmt . reset ${nCommits}`);
+                shellExec(`cd ${repoPath} && git reset`);
+                shellExec(`cd ${repoPath} && git checkout .`);
+                shellExec(`cd ${repoPath} && git clean -f -d`);
+                shellExec(`cd ${repoPath} && underpost push . ${username}/${repoName} -f`);
+              } else {
+                if (!username) logger.error('GITHUB_USERNAME environment variable not set');
+                logger.warn('Repository not found for macro rollback', { repoPath });
+              }
+            }
+
+            processedRepos.add(repoName);
+            logger.info('Repository marked as processed', { repoName });
+          } else {
+            logger.info('Skipping Git operations for already processed repository', { repoName, deployId });
+          }
+
+          // Process each database provider
+          for (const provider of Object.keys(dbs)) {
+            for (const dbName of Object.keys(dbs[provider])) {
+              const { hostFolder, user, password, host, path } = dbs[provider][dbName];
+
+              // Create unique identifier for host+path combination
+              const hostPathKey = `${deployId}:${host}:${path}`;
+
+              // Skip if this host+path combination was already processed
+              if (processedHostPaths.has(hostPathKey)) {
+                logger.info('Skipping already processed host/path', { dbName, host, path, deployId });
+                continue;
+              }
+
+              // Filter by hosts and paths if specified
+              if (
+                (options.hosts &&
+                  !options.hosts
+                    .split(',')
+                    .map((h) => h.trim())
+                    .includes(host)) ||
+                (options.paths &&
+                  !options.paths
+                    .split(',')
+                    .map((p) => p.trim())
+                    .includes(path))
+              ) {
+                logger.info('Skipping database due to host/path filter', { dbName, host, path });
+                continue;
+              }
+
+              if (!hostFolder) {
+                logger.warn('No hostFolder defined for database', { dbName, provider });
+                continue;
+              }
+
+              logger.info('Processing database', { hostFolder, provider, dbName, deployId });
+
+              const latestBackupTimestamp = Underpost.db._getLatestBackupTimestamp(`../${repoName}/${hostFolder}`);
+
+              dbs[provider][dbName].currentBackupTimestamp = latestBackupTimestamp;
+
+              const currentTimestamp = latestBackupTimestamp || newBackupTimestamp;
+              const sqlContainerPath = `/home/${dbName}.sql`;
+              const fromPartsPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}-parths.json`;
+              const toSqlPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}.sql`;
+              const toNewSqlPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}.sql`;
+              const toBsonPath = `../${repoName}/${hostFolder}/${currentTimestamp}/${dbName}`;
+              const toNewBsonPath = `../${repoName}/${hostFolder}/${newBackupTimestamp}/${dbName}`;
+
+              // Merge split SQL files if needed for import
+              if (options.import === true && fs.existsSync(fromPartsPath) && !fs.existsSync(toSqlPath)) {
+                const names = JSON.parse(fs.readFileSync(fromPartsPath, 'utf8')).map((_path) => {
+                  return `../${repoName}/${hostFolder}/${currentTimestamp}/${_path.split('/').pop()}`;
+                });
+                logger.info('Merging backup parts', { fromPartsPath, toSqlPath, parts: names.length });
+                await mergeFile(names, toSqlPath);
+              }
+
+              // Get target pods based on provider and options
+              let targetPods = [];
+              const podCriteria = {
+                podNames: options.podName,
+                namespace,
+                deployId: provider === 'mariadb' ? 'mariadb' : 'mongo',
+              };
+
+              targetPods = Underpost.kubectl.getFilteredPods(podCriteria);
+
+              // Fallback to default if no custom pods specified
+              if (targetPods.length === 0 && !options.podName) {
+                const defaultPods = Underpost.kubectl.get(
+                  provider === 'mariadb' ? 'mariadb' : 'mongo',
+                  'pods',
+                  namespace,
+                );
+                console.log('defaultPods', defaultPods);
+                targetPods = defaultPods;
+              }
+
+              if (targetPods.length === 0) {
+                logger.warn('No pods found matching criteria', { provider, criteria: podCriteria });
+                continue;
+              }
+
+              // Handle primary pod detection for MongoDB
+              let podsToProcess = [];
+              if (provider === 'mongoose' && !options.allPods) {
+                // For MongoDB, always use primary pod unless allPods is true
+                if (!targetPods || targetPods.length === 0) {
+                  logger.warn('No MongoDB pods available to check for primary');
+                  podsToProcess = [];
+                } else {
+                  const firstPod = targetPods[0].NAME;
+                  const primaryPodName = MongoBootstrap.getPrimaryPodName({
+                    namespace,
+                    podName: firstPod,
+                    username: user,
+                    password,
+                    authDatabase: 'admin',
+                  });
+
+                  if (primaryPodName) {
+                    const primaryPod = targetPods.find((p) => p.NAME === primaryPodName);
+                    if (primaryPod) {
+                      podsToProcess = [primaryPod];
+                      logger.info('Using MongoDB primary pod', { primaryPod: primaryPodName });
+                    } else {
+                      logger.warn('Primary pod not in filtered list, using first pod', { primaryPodName });
+                      podsToProcess = [targetPods[0]];
+                    }
+                  } else {
+                    logger.warn('Could not detect primary pod, using first pod');
+                    podsToProcess = [targetPods[0]];
+                  }
+                }
+              } else {
+                // For MariaDB or when allPods is true, limit to first pod unless allPods is true
+                podsToProcess = options.allPods === true ? targetPods : [targetPods[0]];
+              }
+
+              logger.info(`Processing ${podsToProcess.length} pod(s) for ${provider}`, {
+                dbName,
+                pods: podsToProcess.map((p) => p.NAME),
+              });
+
+              let exportSucceeded = false;
+
+              // Process each pod
+              for (const pod of podsToProcess) {
+                logger.info('Processing pod', { podName: pod.NAME, node: pod.NODE, status: pod.STATUS });
+
+                switch (provider) {
+                  case 'mariadb': {
+                    if (options.stats === true) {
+                      const stats = Underpost.db._getMariaDBStats({
+                        podName: pod.NAME,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                      });
+                      if (stats) {
+                        Underpost.db._displayStats({ provider, dbName, stats });
+                      }
+                    }
+
+                    if (options.import === true) {
+                      Underpost.db._importMariaDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                        sqlPath: toSqlPath,
+                      });
+                    }
+
+                    if (options.export === true) {
+                      const outputPath = options.outPath || toNewSqlPath;
+                      const success = await Underpost.db._exportMariaDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                        outputPath,
+                      });
+                      exportSucceeded = exportSucceeded || success;
+                    }
+                    break;
+                  }
+
+                  case 'mongoose': {
+                    if (options.stats === true) {
+                      const stats = Underpost.db._getMongoStats({
+                        podName: pod.NAME,
+                        namespace,
+                        dbName,
+                        user,
+                        password,
+                        authDatabase: 'admin',
+                      });
+                      if (stats) {
+                        Underpost.db._displayStats({ provider, dbName, stats });
+                      }
+                    }
+
+                    if (options.import === true) {
+                      const bsonPath = options.outPath || toBsonPath;
+                      Underpost.db._importMongoDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        bsonPath,
+                        drop: options.drop,
+                        preserveUUID: options.preserveUUID,
+                        user,
+                        password,
+                        authDatabase: 'admin',
+                      });
+                    }
+
+                    if (options.export === true) {
+                      const outputPath = options.outPath || toNewBsonPath;
+                      const success = Underpost.db._exportMongoDB({
+                        pod,
+                        namespace,
+                        dbName,
+                        outputPath,
+                        collections: options.collections,
+                        user,
+                        password,
+                        authDatabase: 'admin',
+                      });
+                      exportSucceeded = exportSucceeded || success;
+                    }
+                    break;
+                  }
+
+                  default:
+                    logger.warn('Unsupported database provider', { provider });
+                    break;
+                }
+              }
+
+              if (options.export === true && exportSucceeded === true) {
+                Underpost.db._enforceBackupRetention(`../${repoName}/${hostFolder}`);
+              }
+
+              // Mark this host+path combination as processed
+              processedHostPaths.add(hostPathKey);
+            }
+          }
+
+          // Commit and push to Git if enabled - execute only once per repository
+          if (options.export === true && options.git === true && !processedRepos.has(`${repoName}-committed`)) {
+            const commitMessage = `${new Date(newBackupTimestamp).toLocaleDateString()} ${new Date(
+              newBackupTimestamp,
+            ).toLocaleTimeString()}`;
+            Underpost.repo.manageBackupRepo({ repoName, operation: 'commit', message: commitMessage });
+            Underpost.repo.manageBackupRepo({ repoName, operation: 'push' });
+            processedRepos.add(`${repoName}-committed`);
+          }
+        }
+
+        logger.info('Database operation completed successfully');
+      } catch (error) {
+        logger.error('Database operation failed', { error: error.message });
+        throw error;
+      }
+    },
+    /**
+     * Helper: Removes old timestamp backup folders and keeps only the newest ones.
+     * @method _enforceBackupRetention
+     * @memberof UnderpostDB
+     * @param {string} backupDir - Path to host-folder backup directory.
+     * @param {number} [maxRetention=MAX_BACKUP_RETENTION] - Maximum folders to keep.
+     * @return {number} Number of removed backup folders.
+     */
+    _enforceBackupRetention(backupDir, maxRetention = MAX_BACKUP_RETENTION) {
+      try {
+        if (!fs.existsSync(backupDir)) return 0;
+
+        const timestamps = fs
+          .readdirSync(backupDir)
+          .filter((entry) => /^\d+$/.test(entry))
+          .sort((a, b) => parseInt(b, 10) - parseInt(a, 10));
+
+        if (timestamps.length <= maxRetention) return 0;
+
+        const staleTimestamps = timestamps.slice(maxRetention);
+        staleTimestamps.forEach((timestamp) => {
+          fs.removeSync(`${backupDir}/${timestamp}`);
+        });
+
+        logger.info('Pruned old backup timestamp folders', {
+          backupDir,
+          kept: maxRetention,
+          removed: staleTimestamps.length,
+          removedTimestamps: staleTimestamps,
+        });
+
+        return staleTimestamps.length;
+      } catch (error) {
+        logger.error('Failed to enforce backup retention', { backupDir, maxRetention, error: error.message });
+        return 0;
+      }
+    },
+
+    /**
+     * Creates cluster metadata for the specified deployment.
+     * Loads database configuration and initializes cluster metadata including
+     * instances and cron jobs. This method populates the database with deployment information.
+     * @method clusterMetadataFactory
+     * @memberof UnderpostDB
+     * @param {string} [deployId=process.env.DEFAULT_DEPLOY_ID] - The deployment ID.
+     * @param {string} [host=process.env.DEFAULT_DEPLOY_HOST] - The host identifier.
+     * @param {string} [path=process.env.DEFAULT_DEPLOY_PATH] - The path identifier.
+     * @param {object} [options] - Options.
+     * @param {boolean} [options.dev=false] - Development mode flag.
+     * @return {Promise<void>} Resolves when metadata creation is complete.
+     * @throws {Error} If database configuration is invalid or connection fails.
+     */
+    async clusterMetadataFactory(
+      deployId = process.env.DEFAULT_DEPLOY_ID,
+      host = process.env.DEFAULT_DEPLOY_HOST,
+      path = process.env.DEFAULT_DEPLOY_PATH,
+      options = { dev: false },
+    ) {
+      try {
+        loadCronDeployEnv();
+        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
+        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
+        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
+
+        logger.info('Creating cluster metadata', { deployId, host, path });
+
+        const env = 'production';
+        const deployListPath = './engine-private/deploy/dd.router';
+
+        if (!fs.existsSync(deployListPath)) {
+          logger.error('Deploy router file not found', { path: deployListPath });
+          throw new Error(`Deploy router file not found: ${deployListPath}`);
+        }
+
+        const deployList = fs.readFileSync(deployListPath, 'utf8').split(',');
+
+        const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+        if (!fs.existsSync(confServerPath)) {
+          logger.error('Server configuration not found', { path: confServerPath });
+          throw new Error(`Server configuration not found: ${confServerPath}`);
+        }
+
+        const { db } = loadConfServerJson(confServerPath, { resolve: true })[host][path];
+
+        const maxRetries = 5;
+        const retryDelay = 3000;
+        for (let attempt = 1; attempt <= maxRetries; attempt++) {
+          try {
+            await DataBaseProviderService.load({ apis: ['instance', 'cron'], host, path, db });
+            break;
+          } catch (err) {
+            if (attempt === maxRetries) {
+              logger.error('Failed to connect to database after retries', { attempts: maxRetries, error: err.message });
+              throw err;
+            }
+            logger.warn('Database connection failed, retrying...', { attempt, maxRetries, error: err.message });
+            await timer(retryDelay);
+          }
+        }
+
+        try {
+          /** @type {import('../api/instance/instance.model.js').InstanceModel} */
+          const Instance = DataBaseProviderService.getModel('instance', { host, path });
+
+          await Instance.deleteMany();
+          logger.info('Cleared existing instance metadata');
+
+          for (const _deployId of deployList) {
+            const deployId = _deployId.trim();
+            if (!deployId) continue;
+
+            logger.info('Processing deployment for metadata', { deployId });
+
+            const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+            if (!fs.existsSync(confServerPath)) {
+              logger.warn('Configuration not found for deployment', { deployId, path: confServerPath });
+              continue;
+            }
+
+            const confServer = loadReplicas(deployId, loadConfServerJson(confServerPath, { resolve: true }));
+            const router = await Underpost.deploy.routerFactory(deployId, env);
+            const pathPortAssignmentData = await pathPortAssignmentFactory(deployId, router, confServer);
+
+            for (const host of Object.keys(confServer)) {
+              for (const { path, port } of pathPortAssignmentData[host]) {
+                if (!confServer[host][path]) continue;
+
+                const { client, runtime, apis, peer } = confServer[host][path];
+
+                // Save main instance
+                {
+                  const body = {
+                    deployId,
+                    host,
+                    path,
+                    port,
+                    client,
+                    runtime,
+                    apis,
+                  };
+
+                  logger.info('Saving instance metadata', body);
+                  await new Instance(body).save();
+                }
+
+                // Save peer instance if exists
+                if (peer) {
+                  const body = {
+                    deployId,
+                    host,
+                    path: path === '/' ? '/peer' : `${path}/peer`,
+                    port: port + 1,
+                    runtime: 'nodejs',
+                  };
+
+                  logger.info('Saving peer instance metadata', body);
+                  await new Instance(body).save();
+                }
+              }
+            }
+
+            // Process additional instances
+            const confInstancesPath = `./engine-private/conf/${deployId}/conf.instances.json`;
+            if (fs.existsSync(confInstancesPath)) {
+              // Expands multiInstance variants so each deployed instance is
+              // registered (handles both the bare-array and { instances } shapes).
+              const confInstances = loadConfInstances(deployId);
+              for (const instance of confInstances) {
+                const { id, host, path, fromPort, metadata } = instance;
+                const { runtime } = metadata;
+                const body = {
+                  deployId,
+                  host,
+                  path,
+                  port: fromPort,
+                  client: id,
+                  runtime,
+                };
+                logger.info('Saving additional instance metadata', body);
+                await new Instance(body).save();
+              }
+            }
+          }
+        } catch (error) {
+          logger.error('Failed to create instance metadata', { error: error.message });
+          throw error;
+        }
+
+        try {
+          const cronDeployPath = './engine-private/deploy/dd.cron';
+          if (!fs.existsSync(cronDeployPath)) {
+            logger.warn('Cron deploy file not found', { path: cronDeployPath });
+            return;
+          }
+
+          const cronDeployId = fs.readFileSync(cronDeployPath, 'utf8').trim();
+          const confCronPath = `./engine-private/conf/${cronDeployId}/conf.cron.json`;
+
+          if (!fs.existsSync(confCronPath)) {
+            logger.warn('Cron configuration not found', { path: confCronPath });
+            return;
+          }
+
+          const confCron = JSON.parse(fs.readFileSync(confCronPath, 'utf8'));
+
+          await DataBaseProviderService.load({ apis: ['cron'], host, path, db });
+
+          /** @type {import('../api/cron/cron.model.js').CronModel} */
+          const Cron = DataBaseProviderService.getModel('cron', { host, path });
+
+          await Cron.deleteMany();
+          logger.info('Cleared existing cron metadata');
+
+          for (const jobId of Object.keys(confCron.jobs)) {
+            const body = {
+              jobId,
+              deployId: Underpost.cron.getRelatedDeployIdList(jobId),
+              expression: confCron.jobs[jobId].expression,
+              enabled: confCron.jobs[jobId].enabled,
+            };
+            logger.info('Saving cron metadata', body);
+            await new Cron(body).save();
+          }
+        } catch (error) {
+          logger.error('Failed to create cron metadata', { error: error.message });
+        }
+
+        await DataBaseProviderService.getProvider({ host, path }, 'mongoose').close();
+        logger.info('Cluster metadata creation completed');
+      } catch (error) {
+        logger.error('Cluster metadata creation failed', { error: error.message });
+        throw error;
+      }
+    },
+
+    /**
+     * Cleans orphaned File references from database collections.
+     * Iterates over all deploy-ids and checks if File documents are actually referenced
+     * by other collections. Removes File documents that are not referenced anywhere.
+     * @method cleanFsCollection
+     * @memberof UnderpostDB
+     * @param {string} [deployList='dd'] - Comma-separated list of deployment IDs.
+     * @param {Object} [options={}] - Clean operation options.
+     * @param {string} [options.hosts=''] - Comma-separated list of hosts to filter.
+     * @param {string} [options.paths=''] - Comma-separated list of paths to filter.
+     * @param {boolean} [options.dryRun=false] - If true, only reports what would be deleted.
+     * @param {boolean} [options.dev=false] - Development mode flag.
+     * @return {Promise<void>} Resolves when clean operation is complete.
+     */
+    async cleanFsCollection(
+      deployList = 'dd',
+      options = {
+        hosts: '',
+        paths: '',
+        dryRun: false,
+        dev: false,
+      },
+    ) {
+      const firstDeployId = deployList !== 'dd' ? deployList.split(',')[0].trim() : '';
+      try {
+        loadCronDeployEnv();
+        if (deployList === 'dd') deployList = fs.readFileSync(`./engine-private/deploy/dd.router`, 'utf8');
+
+        logger.info('Starting File collection cleanup', { deployList, options });
+
+        // Load file.ref.json to know which models reference File
+        const fileRefPath = './src/api/file/file.ref.json';
+        if (!fs.existsSync(fileRefPath)) {
+          logger.error('file.ref.json not found', { path: fileRefPath });
+          return;
+        }
+
+        const fileRefData = JSON.parse(fs.readFileSync(fileRefPath, 'utf8'));
+        logger.info('Loaded file reference configuration', { apis: fileRefData.length });
+
+        // Filter hosts and paths if specified
+        const filterHosts = options.hosts ? options.hosts.split(',').map((h) => h.trim()) : [];
+        const filterPaths = options.paths ? options.paths.split(',').map((p) => p.trim()) : [];
+
+        // Track all connections to close them at the end
+        const connectionsToClose = [];
+
+        for (const _deployId of deployList.split(',')) {
+          const deployId = _deployId.trim();
+          if (!deployId) continue;
+
+          logger.info('Processing deployment for File cleanup', { deployId });
+
+          // Load server configuration
+          const confServerPath = `./engine-private/conf/${deployId}/conf.server.json`;
+          if (!fs.existsSync(confServerPath)) {
+            logger.error('Configuration file not found', { path: confServerPath });
+            continue;
+          }
+
+          const confServer = loadConfServerJson(confServerPath, { resolve: true });
+
+          // Process each host+path combination
+          for (const host of Object.keys(confServer)) {
+            if (filterHosts.length > 0 && !filterHosts.includes(host)) continue;
+
+            for (const path of Object.keys(confServer[host])) {
+              if (filterPaths.length > 0 && !filterPaths.includes(path)) continue;
+
+              const { db, apis } = confServer[host][path];
+              if (!db || !apis) continue;
+
+              // Check if 'file' api is in the apis list
+              if (!apis.includes('file')) {
+                logger.info('Skipping - no file api in configuration', { host, path });
+                continue;
+              }
+
+              // logger.info('Processing host+path with file api', { host, path, db: db.name });
+
+              try {
+                // Connect to database with retry
+                let dbProvider;
+                for (let attempt = 1; attempt <= 3; attempt++) {
+                  try {
+                    dbProvider = await DataBaseProviderService.load({ apis, host, path, db });
+                    break;
+                  } catch (err) {
+                    if (attempt === 3) throw err;
+                    logger.warn('Database connection failed, retrying...', { attempt, host, path, error: err.message });
+                    await timer(3000);
+                  }
+                }
+                if (!dbProvider || !dbProvider.models) {
+                  logger.error('Failed to load database provider', { host, path });
+                  continue;
+                }
+
+                const { models } = dbProvider;
+
+                // Track this connection for cleanup
+                connectionsToClose.push({ host, path, dbProvider });
+
+                // Check if File model exists
+                if (!models.File) {
+                  logger.warn('File model not loaded', { host, path });
+                  continue;
+                }
+
+                // Get all File documents
+                const allFiles = await models.File.find({}, '_id').lean();
+                logger.info('Found File documents', { count: allFiles.length, host, path });
+
+                if (allFiles.length === 0) continue;
+
+                // Track which File IDs are referenced
+                const referencedFileIds = new Set();
+
+                // Check each API from file.ref.json
+                for (const refConfig of fileRefData) {
+                  const { api, model: modelFields } = refConfig;
+
+                  // Check if this API is loaded in current context
+                  const modelName = api
+                    .split('-')
+                    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
+                    .join('');
+                  const Model = models[modelName];
+
+                  if (!Model) {
+                    logger.debug('Model not loaded in current context', { api, modelName, host, path });
+                    continue;
+                  }
+
+                  logger.info('Checking references in model', { api, modelName });
+
+                  // Helper function to recursively check field references
+                  const checkFieldReferences = async (fieldPath, fieldConfig) => {
+                    for (const [fieldName, fieldValue] of Object.entries(fieldConfig)) {
+                      const currentPath = fieldPath ? `${fieldPath}.${fieldName}` : fieldName;
+
+                      if (fieldValue === true) {
+                        // This is a File reference field
+                        const query = {};
+                        query[currentPath] = { $exists: true, $ne: null };
+
+                        const docs = await Model.find(query, currentPath).lean();
+
+                        for (const doc of docs) {
+                          // Navigate to the nested field
+                          const parts = currentPath.split('.');
+                          let value = doc;
+                          for (const part of parts) {
+                            value = value?.[part];
+                          }
+
+                          if (value) {
+                            if (Array.isArray(value)) {
+                              value.forEach((id) => id && referencedFileIds.add(id.toString()));
+                            } else {
+                              referencedFileIds.add(value.toString());
+                            }
+                          }
+                        }
+
+                        logger.info('Found references', {
+                          model: modelName,
+                          field: currentPath,
+                          count: docs.length,
+                        });
+                      } else if (typeof fieldValue === 'object') {
+                        // Nested object, recurse
+                        await checkFieldReferences(currentPath, fieldValue);
+                      }
+                    }
+                  };
+
+                  await checkFieldReferences('', modelFields);
+                }
+
+                logger.info('Total referenced File IDs', { count: referencedFileIds.size, host, path });
+
+                // Find orphaned files
+                const orphanedFiles = allFiles.filter((file) => !referencedFileIds.has(file._id.toString()));
+
+                if (orphanedFiles.length === 0) {
+                  logger.info('No orphaned files found', { host, path });
+                } else {
+                  logger.info('Found orphaned files', { count: orphanedFiles.length, host, path });
+
+                  if (options.dryRun) {
+                    logger.info('Dry run - would delete files', {
+                      count: orphanedFiles.length,
+                      ids: orphanedFiles.map((f) => f._id.toString()),
+                    });
+                  } else {
+                    const orphanedIds = orphanedFiles.map((f) => f._id);
+                    const deleteResult = await models.File.deleteMany({ _id: { $in: orphanedIds } });
+                    logger.info('Deleted orphaned files', {
+                      deletedCount: deleteResult.deletedCount,
+                      host,
+                      path,
+                    });
+                  }
+                }
+              } catch (error) {
+                logger.error('Error processing host+path', {
+                  host,
+                  path,
+                  error: error.message,
+                });
+              }
+            }
+          }
+        }
+
+        // Close all connections
+        logger.info('Closing all database connections', { count: connectionsToClose.length });
+        for (const { host, path, dbProvider } of connectionsToClose) {
+          try {
+            if (dbProvider && dbProvider.close) {
+              await dbProvider.close();
+              logger.info('Connection closed', { host, path });
+            }
+          } catch (error) {
+            logger.error('Error closing connection', { host, path, error: error.message });
+          }
+        }
+
+        logger.info('File collection cleanup completed');
+      } catch (error) {
+        logger.error('File collection cleanup failed', { error: error.message });
+        throw error;
+      }
+    },
+
+    /**
+     * Handles backup of cluster metadata.
+     * Orchestrates backup and restore operations for cluster metadata including
+     * instances and cron jobs. Supports import/export and metadata generation.
+     * @method clusterMetadataBackupCallback
+     * @memberof UnderpostDB
+     * @param {string} [deployId=process.env.DEFAULT_DEPLOY_ID] - The deployment ID.
+     * @param {string} [host=process.env.DEFAULT_DEPLOY_HOST] - The host identifier.
+     * @param {string} [path=process.env.DEFAULT_DEPLOY_PATH] - The path identifier.
+     * @param {Object} [options={}] - Backup operation options.
+     * @param {boolean} [options.generate=false] - Generate cluster metadata.
+     * @param {boolean} [options.itc=false] - Execute in container context.
+     * @param {boolean} [options.import=false] - Import metadata from backup.
+     * @param {boolean} [options.export=false] - Export metadata to backup.
+     * @param {boolean} [options.instances=false] - Process instances collection.
+     * @param {boolean} [options.crons=false] - Process crons collection.
+     * @param {boolean} [options.dev=false] - Development mode flag.
+     * @return {Promise<void>} Resolves when backup operation is complete.
+     */
+    async clusterMetadataBackupCallback(
+      deployId = process.env.DEFAULT_DEPLOY_ID,
+      host = process.env.DEFAULT_DEPLOY_HOST,
+      path = process.env.DEFAULT_DEPLOY_PATH,
+      options = {
+        generate: false,
+        itc: false,
+        import: false,
+        export: false,
+        instances: false,
+        crons: false,
+        dev: false,
+      },
+    ) {
+      try {
+        loadCronDeployEnv();
+        deployId = deployId ? deployId : process.env.DEFAULT_DEPLOY_ID;
+        host = host ? host : process.env.DEFAULT_DEPLOY_HOST;
+        path = path ? path : process.env.DEFAULT_DEPLOY_PATH;
+
+        logger.info('Starting cluster metadata backup operation', {
+          deployId,
+          host,
+          path,
+          options,
+        });
+
+        if (options.generate === true) {
+          logger.info('Generating cluster metadata');
+          await Underpost.db.clusterMetadataFactory(deployId, host, path);
+        }
+
+        if (options.instances === true) {
+          const outputPath = './engine-private/instances';
+          if (!fs.existsSync(outputPath)) {
+            fs.mkdirSync(outputPath, { recursive: true });
+          }
+          const collection = 'instances';
+
+          if (options.export === true) {
+            logger.info('Exporting instances collection', { outputPath });
+            shellExec(
+              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
+
+          if (options.import === true) {
+            logger.info('Importing instances collection', { outputPath });
+            shellExec(
+              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
+        }
+
+        if (options.crons === true) {
+          const outputPath = './engine-private/crons';
+          if (!fs.existsSync(outputPath)) {
+            fs.mkdirSync(outputPath, { recursive: true });
+          }
+          const collection = 'crons';
+
+          if (options.export === true) {
+            logger.info('Exporting crons collection', { outputPath });
+            shellExec(
+              `node bin db --export --primary-pod --collections ${collection} --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
+
+          if (options.import === true) {
+            logger.info('Importing crons collection', { outputPath });
+            shellExec(
+              `node bin db --import --primary-pod --drop --preserveUUID --out-path ${outputPath} --hosts ${host} --paths '${path}' ${deployId}`,
+            );
+          }
+        }
+
+        logger.info('Cluster metadata backup operation completed');
+      } catch (error) {
+        logger.error('Cluster metadata backup operation failed', { error: error.message });
+        throw error;
+      }
+    },
+  };
+}
+
+export default UnderpostDB;