db-data-gd-uploader 1.0.0

package/src/server.js

@@ -0,0 +1,54 @@
+import { connectDatabase, disconnectDatabase } from './config/database.js'
+import config from './config/environment.js'
+import { createApp } from './app.js'
+import { startBackupScheduler } from './services/backupScheduler.js'
+
+async function startServer() {
+  try {
+    // Connect to database
+    await connectDatabase()
+
+    // Create Express app
+    const app = createApp()
+
+    // Start HTTP server
+    const server = app.listen(config.port, () => {
+      console.log(`\n${'═'.repeat(60)}`)
+      console.log(`🚀 Backend Server Started`)
+      console.log(`${'═'.repeat(60)}`)
+      console.log(`  URL: http://localhost:${config.port}`)
+      console.log(`  Environment: ${config.nodeEnv}`)
+      console.log(`  Database: Connected`)
+      console.log(`${'═'.repeat(60)}\n`)
+
+      // Start automatic backup scheduler if enabled
+      if (config.backup.enabled) {
+        console.log(`📦 Automatic backup scheduler enabled`)
+        console.log(`  Interval: Every ${config.backup.intervalMinutes} minute(s)\n`)
+        startBackupScheduler(config.backup.intervalMinutes)
+      } else {
+        console.log(`ℹ️  Automatic backup disabled`)
+        console.log(`  Set ENABLE_AUTO_BACKUP=true to enable\n`)
+      }
+    })
+
+    // Graceful shutdown
+    const shutdown = async (signal) => {
+      console.log(`\n⏸️  Received ${signal}, shutting down gracefully...`)
+      server.close(async () => {
+        console.log('🛑 Server closed')
+        await disconnectDatabase()
+        console.log('✅ Shutdown complete')
+        process.exit(0)
+      })
+    }
+
+    process.on('SIGTERM', () => shutdown('SIGTERM'))
+    process.on('SIGINT', () => shutdown('SIGINT'))
+  } catch (error) {
+    console.error('❌ Server startup failed:', error.message)
+    process.exitCode = 1
+  }
+}
+
+startServer()

package/src/services/backupScheduler.js

@@ -0,0 +1,142 @@
+import mongoose from 'mongoose'
+import { createAndUploadCompleteBackup } from './backupService.js'
+import dotenv from 'dotenv'
+
+dotenv.config()
+
+let isBackupRunning = false
+let lastBackupTime = null
+let backupCount = 0
+let totalBackups = 0
+
+/**
+ * Connect to MongoDB
+ */
+async function connectToDatabase() {
+  if (mongoose.connection.readyState === 1) {
+    return // Already connected
+  }
+  try {
+    await mongoose.connect(process.env.MONGODB_URI, {
+      maxPoolSize: 10,
+      serverSelectionTimeoutMS: 5000,
+    })
+    console.log('✅ MongoDB connected')
+  } catch (error) {
+    console.error('❌ MongoDB connection failed:', error.message)
+    throw error
+  }
+}
+
+/**
+ * Perform backup operation
+ */
+async function performBackup() {
+  if (isBackupRunning) {
+    console.log('⚠️  Backup already in progress, skipping...')
+    return
+  }
+
+  isBackupRunning = true
+  const startTime = Date.now()
+  const timestamp = new Date().toLocaleString()
+
+  try {
+    console.log(`\n${'═'.repeat(60)}`)
+    console.log(`📦 BACKUP #${++totalBackups} - ${timestamp}`)
+    console.log(`${'═'.repeat(60)}`)
+
+    await connectToDatabase()
+
+    const result = await createAndUploadCompleteBackup({
+      uploadToDrive: true,
+      excludedDatabases: ['admin', 'config', 'local'],
+    })
+
+    lastBackupTime = new Date()
+    backupCount++
+
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2)
+
+    console.log('\n📊 BACKUP SUMMARY:')
+    console.log(`  └─ Databases: ${result.databases.join(', ')}`)
+    console.log(`  └─ Collections: ${result.collections.length}`)
+    console.log(`  └─ Total Records: ${result.totalRecords.toLocaleString()}`)
+    console.log(`  └─ File Size: ${(result.driveFile?.size || 0).toLocaleString()} bytes`)
+    console.log(`  └─ Duration: ${duration}s`)
+    console.log(`  └─ Status: ✅ SUCCESS`)
+
+    if (result.uploaded && result.driveFile?.webViewLink) {
+      console.log(`\n☁️  Google Drive Link: ${result.driveFile.webViewLink}`)
+    }
+  } catch (error) {
+    console.error('\n❌ BACKUP FAILED:')
+    console.error(`  Error: ${error.message}`)
+  } finally {
+    isBackupRunning = false
+    const duration = ((Date.now() - startTime) / 1000).toFixed(2)
+    console.log(`\n⏱️  Execution time: ${duration}s`)
+    console.log(`${'═'.repeat(60)}\n`)
+  }
+}
+
+/**
+ * Start scheduler with specified interval
+ * @param {number} intervalMinutes - Interval in minutes
+ */
+export function startBackupScheduler(intervalMinutes = 60) {
+  console.log(`\n${'╔' + '═'.repeat(58) + '╗'}`)
+  console.log(`║ 🕐 BACKUP SCHEDULER STARTED                              ║`)
+  console.log(`║ Interval: Every ${intervalMinutes} minute(s)${' '.repeat(32 - String(intervalMinutes).length)}║`)
+  console.log(`║ Next backup: ${new Date(Date.now() + intervalMinutes * 60000).toLocaleString()}${' '.repeat(23 - new Date(Date.now() + intervalMinutes * 60000).toLocaleString().length)}║`)
+  console.log(`╚${'═'.repeat(58)}╝\n`)
+
+  // First backup immediately
+  performBackup()
+
+  // Then schedule periodic backups
+  const intervalMs = intervalMinutes * 60 * 1000
+  const scheduledInterval = setInterval(() => {
+    performBackup()
+  }, intervalMs)
+
+  // Handle graceful shutdown
+  process.on('SIGINT', async () => {
+    console.log('\n\n⏸️  Stopping backup scheduler...')
+    clearInterval(scheduledInterval)
+
+    // Print statistics
+    console.log(`\n📈 SCHEDULER STATISTICS:`)
+    console.log(`  └─ Total backups: ${totalBackups}`)
+    console.log(`  └─ Successful: ${backupCount}`)
+    console.log(`  └─ Failed: ${totalBackups - backupCount}`)
+    if (lastBackupTime) {
+      console.log(`  └─ Last backup: ${lastBackupTime.toLocaleString()}`)
+    }
+
+    await mongoose.disconnect()
+    console.log('✅ Scheduler stopped\n')
+    process.exit(0)
+  })
+
+  return scheduledInterval
+}
+
+/**
+ * Get scheduler status
+ */
+export function getBackupStatus() {
+  return {
+    isRunning: isBackupRunning,
+    lastBackupTime,
+    totalBackups,
+    successfulBackups: backupCount,
+    failedBackups: totalBackups - backupCount,
+  }
+}
+
+// If run directly
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const interval = process.env.BACKUP_INTERVAL_MINUTES || 60
+  startBackupScheduler(parseInt(interval))
+}

package/src/services/backupService.js

@@ -0,0 +1,85 @@
+import { createExcelWorkbook } from '../utils/excelFormatter.js'
+import { extractAllDatabasesCollections } from '../utils/dataExtract.js'
+import { uploadExcelToGoogleDrive } from './googleDrive.js'
+
+/**
+ * Extract data from ALL databases and collections ONLY
+ * This is the main function - extracts everything from MongoDB
+ */
+export async function createAndUploadCompleteBackup(options = {}) {
+  const {
+    uploadToDrive = true,
+    workbookName,
+    folderId,
+    excludedDatabases = ['admin', 'config', 'local'],
+  } = options
+
+  console.log('🔍 Extracting ALL databases and collections...')
+  const extractedData = await extractAllDatabasesCollections(
+    process.env.MONGODB_URI,
+    excludedDatabases,
+  )
+
+  if (extractedData.length === 0) {
+    throw new Error('No data found in any database')
+  }
+
+  console.log(`✅ Found ${extractedData.length} collections`)
+
+  // Format data for Excel creation - each collection gets its own sheet
+  const formattedData = extractedData.map((item) => ({
+    modelName: `${item.database}/${item.collectionName}`,
+    collectionName: item.collectionName,
+    database: item.database,
+    data: item.data || [],
+  }))
+
+  const fileName =
+    workbookName || `complete-backup-${new Date().toISOString().slice(0, 10)}`
+  const workbook = await createExcelWorkbook(formattedData, {
+    workbookName: fileName,
+  })
+
+  const result = {
+    uploaded: false,
+    uploadStatus: uploadToDrive === false ? 'skipped' : 'pending',
+    fileName: workbook.fileName,
+    databases: [...new Set(extractedData.map((d) => d.database))],
+    collections: extractedData.map(({ database, collectionName, documentCount, data }) => ({
+      database,
+      collectionName,
+      records: documentCount || data.length,
+    })),
+    totalRecords: extractedData.reduce((sum, item) => sum + (item.documentCount || item.data.length), 0),
+    sheetCount: extractedData.length,
+  }
+
+  if (uploadToDrive === false) {
+    console.log('✅ Backup created (upload skipped)')
+    return result
+  }
+
+  console.log(`📤 Uploading to Google Drive...`)
+  const driveFile = await uploadExcelToGoogleDrive({
+    ...workbook,
+    folderId: folderId || process.env.GOOGLE_DRIVE_FOLDER_ID,
+  })
+
+  console.log(`✅ Backup uploaded: ${driveFile.name}`)
+
+  return {
+    ...result,
+    uploaded: true,
+    uploadStatus: 'uploaded',
+    driveFile,
+  }
+}
+
+/**
+ * Old function - kept for backward compatibility (but not recommended)
+ * Use createAndUploadCompleteBackup instead
+ */
+export async function createAndUploadDatabaseExport(options = {}) {
+  console.warn('⚠️  createAndUploadDatabaseExport is deprecated. Use createAndUploadCompleteBackup instead.')
+  return createAndUploadCompleteBackup(options)
+}

package/src/services/googleAuth.js

@@ -0,0 +1,12 @@
+/**
+ * Google Authentication Service
+ * Uses Service Account - No OAuth needed
+ */
+
+export function getAuthStatus() {
+  return {
+    authenticated: true,
+    method: 'Service Account',
+    message: 'Direct upload to Google Drive enabled'
+  }
+}

package/src/services/googleDrive.js

@@ -0,0 +1,245 @@
+import { createWriteStream, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { Readable } from 'node:stream'
+import { google } from 'googleapis'
+import { join } from 'node:path'
+
+const EXPORTS_DIR = './exports'
+
+// Create exports directory if it doesn't exist
+if (!existsSync(EXPORTS_DIR)) {
+  mkdirSync(EXPORTS_DIR, { recursive: true })
+}
+
+/**
+ * Get authenticated Google Drive client
+ * 1. Checks for OAuth 2.0 tokens (google-tokens.json) -> Works with Personal Google Drive (15 GB free quota)
+ * 2. Fallback to Service Account (service-account.json) -> For Google Workspace / Shared Drives
+ */
+async function getDriveClient() {
+  const tokensPath = process.env.GOOGLE_TOKENS_PATH || join(process.cwd(), 'google-tokens.json')
+
+  // 1. Try OAuth 2.0 first if tokens exist
+  if (
+    process.env.GOOGLE_OAUTH_CLIENT_ID &&
+    process.env.GOOGLE_OAUTH_CLIENT_SECRET &&
+    existsSync(tokensPath)
+  ) {
+    try {
+      console.log('🔐 Authenticating with Google OAuth 2.0 (User Account)...')
+      const tokens = JSON.parse(readFileSync(tokensPath, 'utf-8'))
+
+      const oauth2Client = new google.auth.OAuth2(
+        process.env.GOOGLE_OAUTH_CLIENT_ID,
+        process.env.GOOGLE_OAUTH_CLIENT_SECRET,
+        process.env.GOOGLE_OAUTH_REDIRECT_URI || 'http://localhost:5000/auth/google/callback'
+      )
+
+      oauth2Client.setCredentials(tokens)
+
+      // Auto-save refreshed tokens when Google issues new tokens
+      oauth2Client.on('tokens', (newTokens) => {
+        try {
+          const currentTokens = existsSync(tokensPath)
+            ? JSON.parse(readFileSync(tokensPath, 'utf-8'))
+            : {}
+          const mergedTokens = { ...currentTokens, ...newTokens }
+          writeFileSync(tokensPath, JSON.stringify(mergedTokens, null, 2))
+          console.log('🔄 Google OAuth tokens refreshed and saved.')
+        } catch (saveErr) {
+          console.warn('⚠️ Could not auto-save refreshed tokens:', saveErr.message)
+        }
+      })
+
+      const drive = google.drive({ version: 'v3', auth: oauth2Client })
+      return { drive, authType: 'oauth' }
+    } catch (oauthError) {
+      console.warn('⚠️ OAuth authentication failed, trying Service Account fallback:', oauthError.message)
+    }
+  }
+
+  // 2. Service Account Auth (fallback)
+  let serviceAccount
+
+  const serviceAccountPath = join(process.cwd(), 'config', 'service-account.json')
+
+  if (existsSync(serviceAccountPath)) {
+    console.log('📂 Loading Service Account from config file...')
+    const fileContent = readFileSync(serviceAccountPath, 'utf-8')
+    serviceAccount = JSON.parse(fileContent)
+  } else if (process.env.GOOGLE_SERVICE_ACCOUNT_JSON) {
+    console.log('📂 Loading Service Account from .env...')
+    try {
+      serviceAccount = JSON.parse(process.env.GOOGLE_SERVICE_ACCOUNT_JSON)
+    } catch (error) {
+      throw new Error(
+        'Service Account JSON parse error in .env'
+      )
+    }
+  } else {
+    throw new Error(
+      'Authentication failed! No valid Google OAuth tokens (google-tokens.json) or Service Account found.'
+    )
+  }
+
+  const auth = new google.auth.GoogleAuth({
+    credentials: serviceAccount,
+    scopes: ['https://www.googleapis.com/auth/drive.file'],
+  })
+
+  const client = await auth.getClient()
+  const drive = google.drive({ version: 'v3', auth: client })
+  return { drive, authType: 'service_account' }
+}
+
+/**
+ * Upload Excel file to Google Drive
+ */
+export async function uploadExcelToGoogleDrive({ buffer, fileName, contentType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', folderId } = {}) {
+  if (!Buffer.isBuffer(buffer)) {
+    throw new TypeError('buffer must be a Buffer')
+  }
+  if (!fileName) {
+    throw new Error('fileName is required')
+  }
+
+  // Ensure fileName has .xlsx extension
+  if (!fileName.endsWith('.xlsx')) {
+    fileName = `${fileName}.xlsx`
+  }
+
+  // Step 1: Save locally first (always backup locally)
+  const localPath = join(EXPORTS_DIR, fileName)
+  const writeStream = createWriteStream(localPath)
+
+  await new Promise((resolve, reject) => {
+    writeStream.on('finish', resolve)
+    writeStream.on('error', reject)
+    writeStream.write(buffer)
+    writeStream.end()
+  })
+
+  console.log(`📁 File saved locally: ${localPath}`)
+
+  // Step 2: Upload to Google Drive
+  try {
+    const { drive, authType } = await getDriveClient()
+    const targetFolderId = folderId || process.env.GOOGLE_DRIVE_FOLDER_ID
+
+    console.log(`📤 Uploading to Google Drive (${authType}): ${fileName}`)
+
+    // Create readable stream from buffer
+    const bufferStream = Readable.from(buffer)
+
+    const requestBody = {
+      name: fileName,
+      ...(targetFolderId ? { parents: [targetFolderId] } : {}),
+    }
+
+    // Upload file
+    const response = await drive.files.create({
+      requestBody,
+      media: {
+        mimeType: contentType,
+        body: bufferStream,
+      },
+      fields: 'id, name, webViewLink, size',
+      supportsAllDrives: true,
+    })
+
+    const file = response.data
+
+    console.log(`✅ Uploaded to Google Drive!`)
+    console.log(`   File ID: ${file.id}`)
+    console.log(`   View: ${file.webViewLink}`)
+
+    return {
+      id: file.id,
+      name: file.name,
+      webViewLink: file.webViewLink,
+      localPath: localPath,
+      downloadPath: `/download/${fileName}`,
+      status: 'uploaded',
+      message: 'File uploaded to Google Drive successfully!',
+      size: file.size || buffer.length,
+    }
+  } catch (error) {
+    // Check if it's a quota error
+    if (error.message && error.message.includes('storage quota')) {
+      console.error('❌ Service Account has no storage quota!')
+      console.error('')
+      console.error('ℹ️ Google Service Accounts have 0 MB storage quota on personal Google Drive (@gmail.com).')
+      console.error('   Please use OAuth 2.0 authentication (google-tokens.json) or a Google Workspace Shared Drive.')
+      console.error('')
+    } else {
+      console.error('❌ Google Drive upload failed:', error.message)
+    }
+
+    // Return local file info even if Drive upload fails
+    return {
+      id: 'local-only',
+      name: fileName,
+      webViewLink: null,
+      localPath: localPath,
+      downloadPath: `/download/${fileName}`,
+      status: 'local_only',
+      message: `Saved locally. Drive upload failed: ${error.message}`,
+      size: buffer.length,
+      error: error.message,
+    }
+  }
+}
+
+/**
+ * List files in Google Drive folder
+ */
+export async function listDriveFiles(folderId, maxResults = 100) {
+  try {
+    const { drive } = await getDriveClient()
+    const targetFolderId = folderId || process.env.GOOGLE_DRIVE_FOLDER_ID
+
+    const response = await drive.files.list({
+      q: targetFolderId ? `'${targetFolderId}' in parents` : undefined,
+      pageSize: maxResults,
+      fields: 'files(id, name, createdTime, size, webViewLink)',
+      orderBy: 'createdTime desc',
+      supportsAllDrives: true,
+      includeItemsFromAllDrives: true,
+    })
+
+    return response.data.files
+  } catch (error) {
+    console.error('Failed to list files:', error.message)
+    return []
+  }
+}
+
+/**
+ * Delete old backup files (older than X days)
+ */
+export async function deleteOldBackups(folderId, daysOld = 30) {
+  try {
+    const { drive } = await getDriveClient()
+    const files = await listDriveFiles(folderId)
+
+    const cutoffDate = new Date()
+    cutoffDate.setDate(cutoffDate.getDate() - daysOld)
+
+    let deletedCount = 0
+
+    for (const file of files) {
+      const createdDate = new Date(file.createdTime)
+      if (createdDate < cutoffDate) {
+        await drive.files.delete({ fileId: file.id, supportsAllDrives: true })
+        console.log(`🗑️ Deleted old backup: ${file.name}`)
+        deletedCount++
+      }
+    }
+
+    console.log(`✅ Deleted ${deletedCount} old backup(s)`)
+    return deletedCount
+  } catch (error) {
+    console.error('Failed to delete old backups:', error.message)
+    return 0
+  }
+}
+