@khawer.ali/mongo-db-migrator 1.0.0

package/README.md

@@ -0,0 +1,173 @@
+# Mongo DB Migrator
+
+A powerful Node.js command-line tool for migrating MongoDB databases between servers. Perfect for copying production data to local development, server-to-server migrations, and database backups.
+
+## 🚀 Features
+
+- ✅ **Full Database Migration** - Migrate all collections from a database
+- ✅ **Single Collection Migration** - Migrate specific collections only
+- ✅ **Remote & Local MongoDB Support** - Works with any MongoDB instance
+- ✅ **Real-time Progress Tracking** - See migration progress as it happens
+- ✅ **Batch Processing** - Efficiently handles large datasets
+- ✅ **Error Handling** - Robust error reporting and connection management
+- ✅ **CLI Interactive Mode** - User-friendly prompts
+- ✅ **Command Line Arguments** - Scriptable for automation
+
+## 📦 Installation
+
+```bash
+# Clone the repository
+git clone https://github.com/alikhawer29/mongo-db-migrator
+cd mongo-db-migrator
+
+# Install dependencies
+npm install
+
+# Make the script executable (optional)
+chmod +x migrate.js
+```
+
+## 🛠️ Usage
+
+### Interactive Mode (Recommended)
+
+```bash
+npm run migrate
+```
+
+This will prompt you for:
+- Source MongoDB URI
+- Target MongoDB URI
+- Database name
+- Collection name (optional - leave empty for all collections)
+
+### Command Line Arguments
+
+```bash
+node migrate.js "source-uri" "target-uri" "database-name" "collection-name"
+```
+
+### Examples
+
+#### Migrate Entire Database
+```bash
+npm run migrate
+# Follow prompts to enter connection details
+```
+
+#### Migrate Specific Collection
+```bash
+node migrate.js "mongodb://admin:password@source.com:27017/" "mongodb://target.com:27017/" "mydb" "users"
+```
+
+#### Production to Local Development
+```bash
+node migrate.js "mongodb://admin:password@source-mongodb.com:27017/" "mongodb://localhost:27017/" "mydb" ""
+```
+
+## 📋 MongoDB URI Format
+
+```
+mongodb://[username:password@]host[:port]/[database]
+```
+
+### Examples:
+- **Local**: `mongodb://localhost:27017/`
+- **With Auth**: `mongodb://admin:password@host:27017/`
+- **Atlas Cloud**: `mongodb://user:pass@cluster.mongodb.net/`
+
+## 🎯 Recent Migration Success
+
+The tool successfully migrated a production database with 21 collections:
+
+```
+📊 Migration Summary:
+- Source: source-mongodb.com:27017
+- Target: target-mongodb.com:28028
+- Database: ltest
+- Collections: 21 total
+- Documents: 323 migrated
+- Duration: ~2 minutes
+```
+
+### Collections Migrated:
+- `users` (45 documents)
+- `notifications` (114 documents)
+- `payments` (26 documents)
+- `chatmessages` (27 documents)
+- `bookings` (26 documents)
+- And 16 more collections...
+
+## 🔧 Configuration
+
+No configuration required! The tool works out of the box. Simply provide the MongoDB connection URIs when prompted.
+
+## 🚨 Requirements
+
+- Node.js 14+ 
+- MongoDB Node.js Driver (included)
+- Access to source and target MongoDB databases
+
+## 🐛 Troubleshooting
+
+### "Found 0 collections"
+- Check database name spelling (case-sensitive)
+- Verify database exists on source server
+- Ensure proper authentication credentials
+
+### Connection Issues
+- Verify MongoDB URI format
+- Check network connectivity
+- Ensure MongoDB is running on target server
+- Validate username/password credentials
+
+### Large Databases
+- Tool uses batch processing (1000 documents per batch)
+- Progress is shown in real-time
+- Handles millions of documents efficiently
+
+## 📝 Development
+
+### Project Structure
+```
+mongo-db-migrator/
+├── migrate.js              # Main CLI migration script
+├── package.json            # Dependencies and scripts
+├── server/                 # Optional Express server
+│   ├── services/
+│   │   └── migrateService.js
+│   └── routes/
+│       └── migrateRoutes.js
+└── README.md               # This file
+```
+
+### Scripts
+- `npm run migrate` - Run migration tool
+- `npm run dev` - Start Express server (optional)
+- `npm start` - Start Express server
+
+## 🤝 Contributing
+
+1. Fork the repository
+2. Create a feature branch
+3. Make your changes
+4. Test thoroughly
+5. Submit a pull request
+
+## 📄 License
+
+ISC License - Feel free to use this tool in your projects!
+
+## 🆘 Support
+
+If you encounter issues:
+1. Check the troubleshooting section above
+2. Verify your MongoDB connection strings
+3. Ensure proper database permissions
+4. Test with small collections first
+
+---
+
+**Built with ❤️ by Khawer Ali using Node.js and MongoDB Driver**
+
+Perfect for developers who need to migrate databases between environments quickly and safely!

package/migrate.js

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+
+const { MongoClient } = require("mongodb");
+const readline = require('readline');
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout
+});
+
+function question(prompt) {
+  return new Promise((resolve) => {
+    rl.question(prompt, resolve);
+  });
+}
+
+async function migrateDatabase({ sourceUri, targetUri, dbName, collectionName }) {
+  const sourceClient = new MongoClient(sourceUri);
+  const targetClient = new MongoClient(targetUri);
+
+  try {
+    console.log("Connecting to source database...");
+    await sourceClient.connect();
+    console.log("Connected to source database.");
+
+    // List all databases to help with debugging
+    const admin = sourceClient.db().admin();
+    const databases = await admin.listDatabases();
+    console.log("Available databases on source:");
+    databases.databases.forEach(db => {
+      console.log(`  - ${db.name} (${db.sizeOnDisk} bytes)`);
+    });
+
+    console.log("Connecting to target database...");
+    await targetClient.connect();
+    console.log("Connected to target database.");
+
+    const sourceDb = sourceClient.db(dbName);
+    const targetDb = targetClient.db(dbName);
+
+    let collections = [];
+
+    if (collectionName) {
+      collections = [collectionName];
+    } else {
+      console.log("Fetching all collections from source database...");
+      collections = await sourceDb.listCollections().toArray();
+      collections = collections.map(col => col.name);
+      console.log(`Found ${collections.length} collections: ${collections.join(', ')}`);
+    }
+
+    for (const collection of collections) {
+      console.log(`\nMigrating collection: ${collection}`);
+      
+      const sourceCollection = sourceDb.collection(collection);
+      const targetCollection = targetDb.collection(collection);
+
+      // Get total documents count
+      const totalDocs = await sourceCollection.countDocuments();
+      console.log(`Total documents in ${collection}: ${totalDocs}`);
+
+      if (totalDocs === 0) {
+        console.log(`Collection ${collection} is empty, skipping...`);
+        continue;
+      }
+
+      // Create cursor for batch processing
+      const cursor = sourceCollection.find({});
+      let processedDocs = 0;
+      const batchSize = 1000;
+      let batch = [];
+
+      console.log("Starting migration...");
+      
+      for await (const doc of cursor) {
+        batch.push(doc);
+        processedDocs++;
+
+        // Insert batch when it reaches batch size or when we finish
+        if (batch.length === batchSize || processedDocs === totalDocs) {
+          if (batch.length > 0) {
+            await targetCollection.insertMany(batch);
+            console.log(`Progress: ${processedDocs}/${totalDocs} documents (${Math.round((processedDocs/totalDocs)*100)}%)`);
+            batch = [];
+          }
+        }
+      }
+
+      console.log(`✅ Successfully migrated ${processedDocs} documents from ${collection}`);
+    }
+
+    console.log("\n🎉 Migration completed successfully!");
+
+  } catch (error) {
+    console.error("❌ Migration failed:", error.message);
+  } finally {
+    await sourceClient.close();
+    await targetClient.close();
+    rl.close();
+  }
+}
+
+async function main() {
+  console.log("=== MongoDB Database Migration Tool ===\n");
+
+  try {
+    const sourceUri = await question("Enter source MongoDB URI: ");
+    const targetUri = await question("Enter target MongoDB URI: ");
+    const dbName = await question("Enter database name: ");
+    const collectionInput = await question("Enter collection name (leave empty for all collections): ");
+    
+    const collectionName = collectionInput.trim() === '' ? null : collectionInput.trim();
+
+    console.log("\nStarting migration...");
+    console.log(`Source: ${sourceUri}`);
+    console.log(`Target: ${targetUri}`);
+    console.log(`Database: ${dbName}`);
+    console.log(`Collection: ${collectionName || 'All collections'}`);
+    console.log("\n" + "=".repeat(50));
+
+    await migrateDatabase({ sourceUri, targetUri, dbName, collectionName });
+
+  } catch (error) {
+    console.error("Error:", error.message);
+    rl.close();
+  }
+}
+
+// Also support command line arguments
+if (process.argv.length > 2) {
+  const [, , sourceUri, targetUri, dbName, collectionName] = process.argv;
+  migrateDatabase({ sourceUri, targetUri, dbName, collectionName: collectionName || null });
+} else {
+  main();
+}

package/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "@khawer.ali/mongo-db-migrator",
+  "version": "1.0.0",
+  "description": "CLI tool to migrate MongoDB databases between servers",
+  "main": "migrate.js",
+  "bin": {
+    "mongo-db-migrator": "./migrate.js"
+  },
+  "keywords": [
+    "mongodb",
+    "migration",
+    "database",
+    "mongo",
+    "backup",
+    "cli",
+    "migrate"
+  ],
+  "author": "Khawer Ali",
+  "license": "ISC",
+  "scripts": {
+    "migrate": "node migrate.js",
+    "dev": "nodemon server/server.js",
+    "start": "node server/server.js"
+  },
+  "dependencies": {
+    "cors": "^2.8.6",
+    "express": "^5.2.1",
+    "mongodb": "^7.1.0",
+    "socket.io": "^4.8.3"
+  },
+  "devDependencies": {
+    "nodemon": "^3.1.14"
+  }
+}

package/server/routes/migrateRoutes.js

@@ -0,0 +1,10 @@
+const { migrateDatabase } = require("../services/migrateService");
+
+module.exports = async (req, res, io) => {
+    try {
+        await migrateDatabase({ ...req.body, io });
+        res.json({ message: "Migration started" });
+    } catch (err) {
+        res.status(500).json({ error: err.message });
+    }
+};

package/server/server.js

@@ -0,0 +1,21 @@
+const express = require("express");
+const cors = require("cors");
+const http = require("http");
+const { Server } = require("socket.io");
+
+const migrateRoutes = require("./routes/migrateRoutes");
+
+const app = express();
+app.use(cors());
+app.use(express.json());
+
+const server = http.createServer(app);
+const io = new Server(server, {
+    cors: { origin: "*" }
+});
+
+app.use("/api/migrate", (req, res) => migrateRoutes(req, res, io));
+
+server.listen(5000, () => {
+    console.log("Server running on port 5000");
+});

package/server/services/migrateService.js

@@ -0,0 +1,52 @@
+const { MongoClient } = require("mongodb");
+
+async function migrateDatabase({
+    sourceUri,
+    targetUri,
+    dbName,
+    collectionName,
+    io
+}) {
+    const sourceClient = new MongoClient(sourceUri);
+    const targetClient = new MongoClient(targetUri);
+
+    await sourceClient.connect();
+    await targetClient.connect();
+
+    const sourceDb = sourceClient.db(dbName);
+    const targetDb = targetClient.db(dbName);
+
+    let collections = [];
+
+    if (collectionName) {
+        collections = [collectionName];
+    } else {
+        const list = await sourceDb.listCollections().toArray();
+        collections = list.map(c => c.name);
+    }
+
+    for (const name of collections) {
+        const sourceCollection = sourceDb.collection(name);
+        const targetCollection = targetDb.collection(name);
+
+        const docs = await sourceCollection.find({}).toArray();
+
+        if (docs.length) {
+            await targetCollection.insertMany(docs);
+        }
+
+        if (io) {
+            io.emit("progress", {
+                collection: name,
+                count: docs.length
+            });
+        }
+
+        console.log(`Migrated ${name}: ${docs.length} records`);
+    }
+
+    await sourceClient.close();
+    await targetClient.close();
+}
+
+module.exports = { migrateDatabase };