mongodb-backup-service 1.0.0

package/src/core/backupManager.js

@@ -0,0 +1,114 @@
+const path = require('path');
+const configManager = require('../config/config');
+const BackupLock = require('./backupLock');
+const BackupStatus = require('./backupStatus');
+const StorageManager = require('../storage/storageManager');
+const MongoDump = require('../database/mongoDump');
+const ExcelExporter = require('../exporters/excelExporter');
+const logger = require('../utils/logger');
+const DateUtils = require('../utils/dateUtils');
+const RetentionManager = require('../retention/retentionManager');
+const EmailNotifier = require('../notifications/emailNotifier');
+
+class BackupManager {
+    constructor() {
+        this.config = configManager.get();
+        this.storageManager = new StorageManager(this.config);
+        this.storage = this.storageManager.getProvider();
+        this.lock = new BackupLock(this.config.storage.path);
+        this.retentionManager = new RetentionManager(this.config, this.storage);
+        this.emailNotifier = new EmailNotifier(this.config);
+        // Verify SMTP at startup so failures are visible immediately, not silently at send time.
+        // verifyConnection() is non-blocking and never exposes credentials.
+        this.emailNotifier.verifyConnection();
+    }
+
+    async createBackup(targetDateStr = null) {
+        if (!this.lock.acquire()) {
+            logger.warn("Backup aborted because another backup is already running.");
+            return false;
+        }
+
+        const dateStr = targetDateStr || DateUtils.getCurrentDateString(this.config.schedule.timezone);
+        logger.info(`Starting backup process for date: ${dateStr}`);
+        
+        // Prevent duplicate successful backups
+        const existingMetadata = this.storage.readMetadata(dateStr);
+        if (existingMetadata && existingMetadata.status === 'success') {
+            logger.info(`A successful backup already exists for ${dateStr}. Skipping.`);
+            this.lock.release();
+            return true;
+        }
+
+        const status = new BackupStatus(this.config.projectName, dateStr);
+        const backupDir = this.storage.ensureBackupDirectory(dateStr);
+        
+        // Initial pending metadata write
+        this.storage.writeMetadata(dateStr, status.get());
+
+        const mongoDumpFile = 'mongodb.archive.gz';
+        const mongoDumpPath = path.join(backupDir, mongoDumpFile);
+        
+        const excelFile = 'database.xlsx';
+        const excelPath = path.join(backupDir, excelFile);
+
+        let mongoSuccess = false;
+        let excelSuccess = false;
+
+        // 1. Run MongoDB Dump
+        try {
+            await MongoDump.execute(this.config.mongoUri, mongoDumpPath);
+            status.markMongoDumpSuccess(mongoDumpFile);
+            mongoSuccess = true;
+        } catch (error) {
+            status.markMongoDumpFailed(error.message);
+            logger.error("MongoDB dump component failed.");
+        }
+
+        // 2. Run Excel Export if enabled
+        if (this.config.excel.enabled) {
+            try {
+                await ExcelExporter.export(this.config.mongoUri, excelPath, this.config.excel.excludeCollections);
+                status.markExcelSuccess(excelFile);
+                excelSuccess = true;
+            } catch (error) {
+                status.markExcelFailed(error.message);
+                logger.error("Excel export component failed.");
+            }
+        } else {
+            status.markExcelNotApplicable();
+            excelSuccess = true; // Not required
+        }
+
+        // 3. Evaluate overall success. Mongo dump MUST succeed.
+        const isOverallSuccess = mongoSuccess && (!this.config.excel.enabled || excelSuccess);
+        
+        const finalMetadata = status.finish(isOverallSuccess);
+        this.storage.writeMetadata(dateStr, finalMetadata);
+
+        if (isOverallSuccess) {
+            logger.info("Backup completed successfully overall.");
+            
+            // Run Retention Cleanup
+            if (this.config.retention.enabled) {
+                try {
+                    this.retentionManager.runCleanup();
+                } catch (err) {
+                    logger.error("Retention cleanup failed, but backup was successful.", err);
+                }
+            }
+        } else {
+            logger.error("Backup process completed with failures.");
+        }
+
+        // Send Email Notification.
+        // sendNotification() handles the enabled/transporter guards internally.
+        // A failed notification must NEVER affect the backup result already written to metadata.
+        await this.emailNotifier.sendNotification(finalMetadata);
+
+        this.lock.release();
+        return isOverallSuccess;
+    }
+}
+
+module.exports = BackupManager;

package/src/core/backupStatus.js

@@ -0,0 +1,62 @@
+const DateUtils = require('../utils/dateUtils');
+
+class BackupStatus {
+    constructor(projectName, backupDate) {
+        this.metadata = {
+            projectName: projectName,
+            backupDate: backupDate,
+            startedAt: DateUtils.getIsoString(),
+            completedAt: null,
+            status: 'in_progress',
+            mongoDump: {
+                status: 'not_started',
+                file: null,
+                error: null
+            },
+            excel: {
+                status: 'not_started',
+                file: null,
+                error: null
+            },
+            durationMs: 0
+        };
+        this.startTime = Date.now();
+    }
+
+    markMongoDumpSuccess(fileName) {
+        this.metadata.mongoDump.status = 'success';
+        this.metadata.mongoDump.file = fileName;
+    }
+
+    markMongoDumpFailed(errorMsg) {
+        this.metadata.mongoDump.status = 'failed';
+        this.metadata.mongoDump.error = errorMsg;
+    }
+
+    markExcelSuccess(fileName) {
+        this.metadata.excel.status = 'success';
+        this.metadata.excel.file = fileName;
+    }
+
+    markExcelFailed(errorMsg) {
+        this.metadata.excel.status = 'failed';
+        this.metadata.excel.error = errorMsg;
+    }
+
+    markExcelNotApplicable() {
+        this.metadata.excel.status = 'not_started';
+    }
+
+    finish(isOverallSuccess) {
+        this.metadata.status = isOverallSuccess ? 'success' : 'failed';
+        this.metadata.completedAt = DateUtils.getIsoString();
+        this.metadata.durationMs = Date.now() - this.startTime;
+        return this.metadata;
+    }
+
+    get() {
+        return this.metadata;
+    }
+}
+
+module.exports = BackupStatus;

package/src/database/collectionReader.js

@@ -0,0 +1,48 @@
+const logger = require('../utils/logger');
+
+class CollectionReader {
+    /**
+     * Generator function to safely read collection documents in batches.
+     * @param {object} db MongoDB database instance
+     * @param {string} collectionName Name of the collection
+     * @param {number} batchSize Number of documents to process at once
+     */
+    static async *readCollectionBatches(db, collectionName, batchSize = 1000) {
+        const collection = db.collection(collectionName);
+        const cursor = collection.find({}).batchSize(batchSize);
+
+        try {
+            let batch = [];
+            for await (const doc of cursor) {
+                batch.push(doc);
+                if (batch.length >= batchSize) {
+                    yield batch;
+                    batch = [];
+                }
+            }
+            if (batch.length > 0) {
+                yield batch;
+            }
+        } catch (error) {
+            logger.error(`Error reading collection ${collectionName}`);
+            throw error;
+        } finally {
+            await cursor.close();
+        }
+    }
+
+    /**
+     * Gets a list of all collections excluding the system ones and those in the exclude list.
+     * @param {object} db MongoDB database instance
+     * @param {string[]} excludeCollections Array of collection names to exclude
+     * @returns {Promise<string[]>}
+     */
+    static async getCollections(db, excludeCollections = []) {
+        const collections = await db.listCollections().toArray();
+        return collections
+            .map(c => c.name)
+            .filter(name => !name.startsWith('system.') && !excludeCollections.includes(name));
+    }
+}
+
+module.exports = CollectionReader;

package/src/database/mongoConnection.js

@@ -0,0 +1,39 @@
+const { MongoClient } = require('mongodb');
+const logger = require('../utils/logger');
+
+class MongoConnection {
+    constructor() {
+        this.client = null;
+        this.db = null;
+    }
+
+    async connect(uri) {
+        try {
+            if (this.client) {
+                return this.db;
+            }
+            this.client = new MongoClient(uri, {
+                // Ensure we use safe reads and avoid long-running query timeouts for large backups
+                socketTimeoutMS: 300000,
+                connectTimeoutMS: 30000,
+                serverSelectionTimeoutMS: 30000
+            });
+            await this.client.connect();
+            this.db = this.client.db();
+            return this.db;
+        } catch (error) {
+            logger.error("MongoDB connection failed for Excel export.");
+            throw new Error(`MongoDB connection failed: ${error.message}`);
+        }
+    }
+
+    async disconnect() {
+        if (this.client) {
+            await this.client.close();
+            this.client = null;
+            this.db = null;
+        }
+    }
+}
+
+module.exports = MongoConnection;

package/src/database/mongoDump.js

@@ -0,0 +1,45 @@
+const fs = require('fs');
+const CommandRunner = require('../utils/commandRunner');
+const logger = require('../utils/logger');
+
+class MongoDump {
+    /**
+     * Executes mongodump.
+     * @param {string} uri MongoDB URI
+     * @param {string} outputPath Target file path (.gz)
+     * @returns {Promise<boolean>} True if successful
+     */
+    static async execute(uri, outputPath) {
+        try {
+            logger.info("MongoDB dump started");
+            
+            const args = [
+                '--uri', uri,
+                '--archive=' + outputPath,
+                '--gzip'
+            ];
+
+            // Run command without a shell to prevent injection
+            const result = await CommandRunner.run('mongodump', args);
+            
+            // Verify file exists and has size > 0
+            if (!fs.existsSync(outputPath)) {
+                throw new Error("mongodump completed but output file is missing.");
+            }
+
+            const stats = fs.statSync(outputPath);
+            if (stats.size === 0) {
+                throw new Error("mongodump created an empty file.");
+            }
+
+            logger.info("MongoDB dump completed successfully");
+            return true;
+        } catch (error) {
+            logger.error("MongoDB dump failed", error.error || error);
+            // Don't log sensitive arguments that might contain credentials
+            throw new Error(`mongodump execution failed: ${error.error ? error.error.message : error.message}`);
+        }
+    }
+}
+
+module.exports = MongoDump;

package/src/exporters/excelExporter.js

@@ -0,0 +1,152 @@
+const fs = require('fs');
+const exceljs = require('exceljs');
+const MongoConnection = require('../database/mongoConnection');
+const CollectionReader = require('../database/collectionReader');
+const logger = require('../utils/logger');
+
+class ExcelExporter {
+    /**
+     * Flattens a MongoDB document for Excel export.
+     */
+    static flattenDocument(doc) {
+        const result = {};
+        for (const [key, value] of Object.entries(doc)) {
+            if (value === null || value === undefined) {
+                result[key] = '';
+            } else if (value instanceof Date) {
+                result[key] = value;
+            } else if (typeof value === 'object') {
+                if (value.constructor && value.constructor.name === 'ObjectId') {
+                    result[key] = value.toString();
+                } else {
+                    // Stringify nested objects and arrays
+                    result[key] = JSON.stringify(value);
+                }
+            } else {
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Sanitizes a sheet name for Excel (max 31 chars, no invalid chars).
+     */
+    static sanitizeSheetName(name, existingNames) {
+        let safeName = name.replace(/[\\/*?:\[\]]/g, '_').substring(0, 31);
+        
+        let counter = 1;
+        let finalName = safeName;
+        while (existingNames.has(finalName)) {
+            const suffix = `_${counter}`;
+            finalName = safeName.substring(0, 31 - suffix.length) + suffix;
+            counter++;
+        }
+        existingNames.add(finalName);
+        return finalName;
+    }
+
+    /**
+     * Exports all collections to an Excel file.
+     * @param {string} uri MongoDB URI
+     * @param {string} outputPath Target Excel file path
+     * @param {string[]} excludeCollections Collections to skip
+     */
+    static async export(uri, outputPath, excludeCollections = []) {
+        logger.info("Excel export started");
+        const connection = new MongoConnection();
+        
+        try {
+            const db = await connection.connect(uri);
+            const collections = await CollectionReader.getCollections(db, excludeCollections);
+            
+            const workbook = new exceljs.stream.xlsx.WorkbookWriter({
+                filename: outputPath,
+                useStyles: true,
+                useSharedStrings: false
+            });
+
+            const sheetNames = new Set();
+
+            for (const collectionName of collections) {
+                const sheetName = this.sanitizeSheetName(collectionName, sheetNames);
+                const worksheet = workbook.addWorksheet(sheetName);
+                
+                let isFirstBatch = true;
+                let columns = new Set();
+
+                for await (const batch of CollectionReader.readCollectionBatches(db, collectionName)) {
+                    const flattenedBatch = batch.map(this.flattenDocument);
+                    
+                    if (isFirstBatch && flattenedBatch.length > 0) {
+                        // Dynamically discover columns from the first batch
+                        // Note: For fully robust schema-less exports, we'd need to scan all docs,
+                        // but doing it per-batch is memory efficient. We'll update columns dynamically.
+                        for (const doc of flattenedBatch) {
+                            Object.keys(doc).forEach(k => columns.add(k));
+                        }
+                        
+                        worksheet.columns = Array.from(columns).map(col => ({
+                            header: col,
+                            key: col,
+                            width: 20
+                        }));
+                        isFirstBatch = false;
+                    }
+                    
+                    // If subsequent batches have new fields, add them
+                    if (!isFirstBatch) {
+                        let newColsAdded = false;
+                        for (const doc of flattenedBatch) {
+                            Object.keys(doc).forEach(k => {
+                                if (!columns.has(k)) {
+                                    columns.add(k);
+                                    newColsAdded = true;
+                                }
+                            });
+                        }
+                        if (newColsAdded) {
+                            worksheet.columns = Array.from(columns).map(col => ({
+                                header: col,
+                                key: col,
+                                width: 20
+                            }));
+                        }
+                    }
+
+                    for (const doc of flattenedBatch) {
+                        worksheet.addRow(doc).commit();
+                    }
+                }
+                
+                // If collection was empty, just add a placeholder column
+                if (isFirstBatch) {
+                    worksheet.columns = [{ header: 'Empty Collection', key: 'empty', width: 20 }];
+                }
+                
+                worksheet.commit();
+            }
+
+            await workbook.commit();
+            
+            // Verify output
+            if (!fs.existsSync(outputPath)) {
+                throw new Error("Excel export completed but output file is missing.");
+            }
+            const stats = fs.statSync(outputPath);
+            if (stats.size === 0) {
+                throw new Error("Excel export created an empty file.");
+            }
+            
+            logger.info("Excel export completed successfully");
+            return true;
+        } catch (error) {
+            logger.error("Excel export failed", error);
+            throw new Error(`Excel export failed: ${error.message}`);
+        } finally {
+            await connection.disconnect();
+        }
+    }
+}
+
+module.exports = ExcelExporter;

package/src/index.js

@@ -0,0 +1,71 @@
+const configManager = require('./config/config');
+const Scheduler = require('./scheduler/scheduler');
+const BackupManager = require('./core/backupManager');
+const logger = require('./utils/logger');
+
+class DatabaseBackupService {
+    constructor() {
+        this.scheduler = null;
+        this.backupManager = null;
+    }
+
+    start(userConfig) {
+        try {
+            configManager.load(userConfig);
+            // Create BackupManager ONCE — Scheduler will share this same instance
+            // so EmailNotifier is constructed once and SMTP verify fires exactly once.
+            this.backupManager = new BackupManager();
+            this.scheduler = new Scheduler(this.backupManager);
+            this.scheduler.start();
+            logger.info("Backup service successfully started.");
+        } catch (error) {
+            logger.error("Failed to start backup service.", error);
+            // We don't rethrow to avoid crashing the consuming backend,
+            // but the service won't be running.
+        }
+    }
+
+    stop() {
+        if (this.scheduler) {
+            this.scheduler.stop();
+        }
+        logger.info("Backup service stopped.");
+    }
+
+    async createBackup() {
+        if (!this.backupManager) {
+            if (!configManager.get().mongoUri) {
+                throw new Error("Call backup.start(config) before backup.createBackup()");
+            }
+            this.backupManager = new BackupManager();
+        }
+        return await this.backupManager.createBackup();
+    }
+
+    getStatus(dateStr = null) {
+        if (!this.backupManager) {
+            this.backupManager = new BackupManager();
+        }
+        
+        const DateUtils = require('./utils/dateUtils');
+        const targetDate = dateStr || DateUtils.getCurrentDateString(configManager.get().schedule.timezone);
+        
+        return this.backupManager.storage.readMetadata(targetDate);
+    }
+
+    listBackups() {
+        if (!this.backupManager) {
+            this.backupManager = new BackupManager();
+        }
+        return this.backupManager.storage.getAllBackupDates();
+    }
+
+    async cleanup() {
+        if (!this.backupManager) {
+            this.backupManager = new BackupManager();
+        }
+        this.backupManager.retentionManager.runCleanup();
+    }
+}
+
+module.exports = new DatabaseBackupService();

package/src/notifications/emailNotifier.js

@@ -0,0 +1,86 @@
+const nodemailer = require('nodemailer');
+const logger = require('../utils/logger');
+const EmailTemplates = require('./emailTemplates');
+
+class EmailNotifier {
+    constructor(config) {
+        this.config = config.email;
+        this.transporter = null;
+
+        if (!this.config.enabled) {
+            logger.info("Email notifications are disabled (email.enabled = false).");
+            return;
+        }
+
+        // Validate required fields before creating the transporter
+        const missing = ['host', 'user', 'password', 'to'].filter(k => !this.config[k]);
+        if (missing.length > 0) {
+            logger.warn(`Email notifications enabled but missing required config: ${missing.join(', ')}. Notifications will be skipped.`);
+            return;
+        }
+
+        this.transporter = nodemailer.createTransport({
+            host: this.config.host,
+            port: this.config.port,
+            secure: this.config.secure,  // false = STARTTLS on port 587; true = TLS on port 465
+            auth: {
+                user: this.config.user,
+                pass: this.config.password  // password is never logged
+            }
+        });
+
+        logger.info(`Email notifier initialised. SMTP host: ${this.config.host}, port: ${this.config.port}, secure: ${this.config.secure}, recipient: ${this.config.to}`);
+    }
+
+    /**
+     * Verifies SMTP connectivity. Safe to call at startup — never logs credentials.
+     * Returns true if the connection succeeds, false otherwise.
+     */
+    async verifyConnection() {
+        if (!this.transporter) {
+            logger.warn("Email verify skipped — transporter not initialised.");
+            return false;
+        }
+
+        try {
+            await this.transporter.verify();
+            logger.info("SMTP connection verified successfully.");
+            return true;
+        } catch (error) {
+            // Log the error type and message only — nodemailer errors never include the password
+            logger.warn(`SMTP connection verification failed: ${error.code || error.message}. Check BACKUP_SMTP_HOST, BACKUP_SMTP_PORT, BACKUP_SMTP_USER, and BACKUP_SMTP_PASSWORD in your .env.`);
+            return false;
+        }
+    }
+
+    async sendNotification(metadata) {
+        if (!this.config.enabled) return;
+
+        if (!this.transporter) {
+            logger.warn("Email notification skipped — transporter not initialised (check config warnings above).");
+            return;
+        }
+
+        const { subject, text } = EmailTemplates.getBackupReport(metadata);
+
+        const mailOptions = {
+            from: this.config.from || `"Database Backup Service" <${this.config.user}>`,
+            to:   this.config.to,
+            subject,
+            text
+        };
+
+        logger.info(`Sending email notification to: ${this.config.to} | Subject: ${subject}`);
+
+        try {
+            const info = await this.transporter.sendMail(mailOptions);
+            logger.info(`Email notification sent successfully. Message ID: ${info.messageId}`);
+        } catch (error) {
+            // Mask authentication errors — they can expose user/host but never the password via message alone
+            logger.error(`Email notification failed [${error.code || 'UNKNOWN'}]: ${error.message}`);
+            // Do NOT throw — a failed email must never degrade a successful backup result
+        }
+    }
+}
+
+module.exports = EmailNotifier;

package/src/notifications/emailTemplates.js

@@ -0,0 +1,33 @@
+class EmailTemplates {
+    static getBackupReport(metadata) {
+        const isSuccess = metadata.status === 'success';
+        const subjectStatus = isSuccess ? 'Successful' : 'Failed';
+        
+        const durationMins = Math.floor(metadata.durationMs / 60000);
+        const durationSecs = Math.floor((metadata.durationMs % 60000) / 1000);
+        const durationStr = `${durationMins} minutes ${durationSecs} seconds`;
+
+        const subject = `Database Backup ${subjectStatus} - ${metadata.projectName}`;
+        
+        let text = `Subject:\nDatabase Backup ${subjectStatus} - ${metadata.projectName}\n\n`;
+        text += `Project:\n${metadata.projectName}\n\n`;
+        text += `Backup Date:\n${metadata.backupDate}\n\n`;
+        text += `Overall Status:\n${metadata.status.toUpperCase()}\n\n`;
+        
+        text += `MongoDB Dump:\n${metadata.mongoDump.status.toUpperCase()}\n`;
+        if (metadata.mongoDump.file) text += `File: ${metadata.mongoDump.file}\n`;
+        if (metadata.mongoDump.error) text += `Error: ${metadata.mongoDump.error}\n`;
+        text += `\n`;
+
+        text += `Excel Export:\n${metadata.excel.status.toUpperCase()}\n`;
+        if (metadata.excel.file) text += `File: ${metadata.excel.file}\n`;
+        if (metadata.excel.error) text += `Error: ${metadata.excel.error}\n`;
+        text += `\n`;
+
+        text += `Duration:\n${durationStr}\n`;
+
+        return { subject, text };
+    }
+}
+
+module.exports = EmailTemplates;