appwrite-utils-cli 1.0.6 → 1.0.8

package/README.md

@@ -327,6 +327,70 @@ This updated CLI ensures that developers have robust tools at their fingertips t
 
 ## Changelog
 
+### 1.0.8 - Comprehensive Transfer System with Enhanced Rate Limiting
+
+**🚀 Complete Cross-Instance Transfer Solution**
+
+#### Comprehensive Transfer System
+- **New CLI Option**: `🚀 Comprehensive transfer (users → databases → buckets → functions)` in interactive mode
+- **Orchestrated Transfer Flow**: Proper execution order (users → databases → buckets → functions) for dependency management
+- **Cross-Instance Support**: Transfer entire Appwrite configurations between different instances/projects
+- **Selective Transfer**: Choose which components to transfer (users, databases, buckets, functions)
+- **Dry Run Mode**: Test transfers without making actual changes
+
+#### Enhanced Rate Limiting Strategy
+- **Configurable Limits**: 5 to 100 concurrent operations in steps of 5
+- **Differentiated Rates**: Smart rate limiting based on operation type:
+  - **General Operations**: Full rate (databases, functions)
+  - **User Operations**: Half rate (more sensitive operations)
+  - **File Operations**: Quarter rate (most bandwidth intensive)
+- **Visual Feedback**: Real-time rate limit display during transfers
+- **Intelligent Scaling**: Automatic calculation of optimal rates for different operations
+
+#### File Transfer Enhancements
+- **File Validation**: Comprehensive integrity checking (empty files, size limits)
+- **Retry Logic**: Exponential backoff for failed file transfers
+- **Error Handling**: Graceful handling of corrupt/invalid files
+- **Progress Tracking**: Real-time progress for large file transfers
+
+#### Function Transfer Integration
+- **Automated Function Migration**: Download from source, redeploy to target
+- **Temporary Management**: Automatic cleanup of downloaded function code
+- **Existing Code Integration**: Leverages existing deployment infrastructure
+- **Configuration Preservation**: Maintains function settings and variables
+
+#### User Experience Improvements
+- **Password Reset Warnings**: Clear notifications about Appwrite password limitations
+- **Interactive Configuration**: Step-by-step prompts for source/target setup
+- **Comprehensive Reporting**: Detailed transfer summaries with statistics
+- **Smart Confirmations**: Risk-based confirmations for destructive operations
+
+#### Technical Implementation
+- **Rate Limiting**: Uses p-limit for concurrent operation control
+- **Error Resilience**: Robust error handling with detailed user feedback
+- **Memory Management**: Efficient processing of large datasets
+- **Progress Tracking**: Real-time progress bars with ETA calculations
+
+#### Usage Examples
+```bash
+# Interactive mode - select comprehensive transfer
+npx appwrite-utils-cli@latest appwrite-migrate --it
+
+# Example rate limiting at 20 concurrent:
+# - General operations: 20 concurrent
+# - User operations: 10 concurrent  
+# - File operations: 5 concurrent
+```
+
+**Benefits**:
+- Complete Appwrite instance migration capability
+- Intelligent rate limiting prevents API throttling
+- Enhanced file transfer reliability
+- Comprehensive progress tracking and reporting
+- Maintains data integrity across transfers
+
+### 1.0.7 - Forgot to remove debug logs
+
 ### 1.0.6 - Cross-Language Constants Generation
 
 **🚀 Enhanced Developer Experience with Multi-Language Constants**

package/dist/config/yamlConfig.js

@@ -243,17 +243,13 @@ export const findYamlConfig = (startDir) => {
         path.join(startDir, "appwrite.yaml"),
         path.join(startDir, "appwrite.yml"),
     ];
-    console.log(`DEBUG: Checking YAML paths in ${startDir}:`);
     for (const configPath of possiblePaths) {
-        console.log(`  - ${configPath}: ${fs.existsSync(configPath)}`);
         if (fs.existsSync(configPath)) {
             return configPath;
         }
     }
     // Recursively search subdirectories for .appwrite folders
-    console.log(`DEBUG: Starting recursive search from ${startDir}`);
     const yamlConfigInSubdirs = findYamlConfigRecursive(startDir);
-    console.log(`DEBUG: Recursive search result: ${yamlConfigInSubdirs}`);
     if (yamlConfigInSubdirs) {
         return yamlConfigInSubdirs;
     }
@@ -316,23 +312,18 @@ const shouldIgnoreDirectory = (dirName) => {
 const findYamlConfigRecursive = (dir, depth = 0) => {
     // Limit search depth to prevent infinite recursion
     if (depth > 5) {
-        console.log(`DEBUG: Stopping search at depth ${depth} in ${dir}`);
         return null;
     }
     if (shouldIgnoreDirectory(path.basename(dir))) {
-        console.log(`DEBUG: Ignoring directory ${dir}`);
         return null;
     }
     try {
         const entries = fs.readdirSync(dir, { withFileTypes: true });
-        console.log(`DEBUG: Searching directory ${dir} at depth ${depth}, found ${entries.length} entries`);
         for (const entry of entries) {
             if (entry.isDirectory() && !shouldIgnoreDirectory(entry.name)) {
                 const fullPath = path.join(dir, entry.name);
-                console.log(`DEBUG: Checking subdirectory: ${fullPath}`);
                 // Check if this is an .appwrite directory
                 if (entry.name === ".appwrite") {
-                    console.log(`DEBUG: Found .appwrite directory at ${fullPath}`);
                     const configPaths = [
                         path.join(fullPath, "appwriteConfig.yaml"),
                         path.join(fullPath, "appwriteConfig.yml"),

package/dist/interactiveCLI.d.ts

@@ -34,4 +34,5 @@ export declare class InteractiveCLI {
     private detectConfigurationType;
     private buildChoicesList;
     private migrateTypeScriptConfig;
+    private comprehensiveTransfer;
 }

package/dist/interactiveCLI.js

@@ -6,6 +6,7 @@ import { fetchAllCollections } from "./collections/methods.js";
 import { listBuckets, createBucket } from "./storage/methods.js";
 import { Databases, Storage, Client, Compression, Query, Functions, } from "node-appwrite";
 import { getClient } from "./utils/getClientFromConfig.js";
+import { ComprehensiveTransfer } from "./migrations/comprehensiveTransfer.js";
 import { AppwriteFunctionSchema, parseAttribute, PermissionToAppwritePermission, RuntimeSchema, permissionSchema, } from "appwrite-utils";
 import { ulid } from "ulidx";
 import chalk from "chalk";
@@ -24,24 +25,25 @@ import { findYamlConfig, addFunctionToYamlConfig } from "./config/yamlConfig.js"
 var CHOICES;
 (function (CHOICES) {
     CHOICES["MIGRATE_CONFIG"] = "\uD83D\uDD04 Migrate TypeScript config to YAML (.appwrite structure)";
-    CHOICES["CREATE_COLLECTION_CONFIG"] = "Create collection config file";
-    CHOICES["CREATE_FUNCTION"] = "Create a new function, from scratch or using a template";
-    CHOICES["DEPLOY_FUNCTION"] = "Deploy function(s)";
-    CHOICES["DELETE_FUNCTION"] = "Delete function";
-    CHOICES["SETUP_DIRS_FILES"] = "Setup directories and files";
-    CHOICES["SETUP_DIRS_FILES_WITH_EXAMPLE_DATA"] = "Setup directories and files with example data";
-    CHOICES["SYNC_DB"] = "Push local config to Appwrite";
-    CHOICES["SYNCHRONIZE_CONFIGURATIONS"] = "Synchronize configurations - Pull from Appwrite and write to local config";
-    CHOICES["TRANSFER_DATA"] = "Transfer data";
-    CHOICES["BACKUP_DATABASE"] = "Backup database";
-    CHOICES["WIPE_DATABASE"] = "Wipe database";
-    CHOICES["WIPE_COLLECTIONS"] = "Wipe collections";
-    CHOICES["GENERATE_SCHEMAS"] = "Generate schemas";
+    CHOICES["CREATE_COLLECTION_CONFIG"] = "\uD83D\uDCC4 Create collection config file";
+    CHOICES["CREATE_FUNCTION"] = "\u26A1 Create a new function, from scratch or using a template";
+    CHOICES["DEPLOY_FUNCTION"] = "\uD83D\uDE80 Deploy function(s)";
+    CHOICES["DELETE_FUNCTION"] = "\uD83D\uDDD1\uFE0F Delete function";
+    CHOICES["SETUP_DIRS_FILES"] = "\uD83D\uDCC1 Setup directories and files";
+    CHOICES["SETUP_DIRS_FILES_WITH_EXAMPLE_DATA"] = "\uD83D\uDCC1\u2728 Setup directories and files with example data";
+    CHOICES["SYNC_DB"] = "\u2B06\uFE0F Push local config to Appwrite";
+    CHOICES["SYNCHRONIZE_CONFIGURATIONS"] = "\uD83D\uDD04 Synchronize configurations - Pull from Appwrite and write to local config";
+    CHOICES["TRANSFER_DATA"] = "\uD83D\uDCE6 Transfer data";
+    CHOICES["COMPREHENSIVE_TRANSFER"] = "\uD83D\uDE80 Comprehensive transfer (users \u2192 databases \u2192 buckets \u2192 functions)";
+    CHOICES["BACKUP_DATABASE"] = "\uD83D\uDCBE Backup database";
+    CHOICES["WIPE_DATABASE"] = "\uD83E\uDDF9 Wipe database";
+    CHOICES["WIPE_COLLECTIONS"] = "\uD83E\uDDF9 Wipe collections";
+    CHOICES["GENERATE_SCHEMAS"] = "\uD83C\uDFD7\uFE0F Generate schemas";
     CHOICES["GENERATE_CONSTANTS"] = "\uD83D\uDCCB Generate cross-language constants (TypeScript, Python, PHP, Dart, etc.)";
-    CHOICES["IMPORT_DATA"] = "Import data";
-    CHOICES["RELOAD_CONFIG"] = "Reload configuration files";
-    CHOICES["UPDATE_FUNCTION_SPEC"] = "Update function specifications";
-    CHOICES["EXIT"] = "Exit";
+    CHOICES["IMPORT_DATA"] = "\uD83D\uDCE5 Import data";
+    CHOICES["RELOAD_CONFIG"] = "\uD83D\uDD04 Reload configuration files";
+    CHOICES["UPDATE_FUNCTION_SPEC"] = "\u2699\uFE0F Update function specifications";
+    CHOICES["EXIT"] = "\uD83D\uDC4B Exit";
 })(CHOICES || (CHOICES = {}));
 export class InteractiveCLI {
     currentDir;
@@ -109,6 +111,9 @@ export class InteractiveCLI {
                     await this.initControllerIfNeeded();
                     await this.transferData();
                     break;
+                case CHOICES.COMPREHENSIVE_TRANSFER:
+                    await this.comprehensiveTransfer();
+                    break;
                 case CHOICES.BACKUP_DATABASE:
                     await this.initControllerIfNeeded();
                     await this.backupDatabase();
@@ -1396,13 +1401,13 @@ export class InteractiveCLI {
         }));
     }
     async reloadConfig() {
-        console.log(chalk.yellow("Reloading configuration files..."));
+        MessageFormatter.progress("Reloading configuration files...", { prefix: "Config" });
         try {
             await this.controller.reloadConfig();
-            console.log(chalk.green("Configuration files reloaded successfully."));
+            MessageFormatter.success("Configuration files reloaded successfully", { prefix: "Config" });
         }
         catch (error) {
-            console.error(chalk.red("Error reloading configuration files:"), error);
+            MessageFormatter.error("Failed to reload configuration files", error instanceof Error ? error : new Error(String(error)), { prefix: "Config" });
         }
     }
     async updateFunctionSpec() {
@@ -1452,7 +1457,6 @@ export class InteractiveCLI {
         try {
             // Check for YAML config first
             const yamlConfigPath = findYamlConfig(this.currentDir);
-            console.log(`DEBUG: YAML config search from ${this.currentDir}, found: ${yamlConfigPath}`);
             if (yamlConfigPath) {
                 this.isUsingTypeScriptConfig = false;
                 MessageFormatter.info("Using YAML configuration", { prefix: "Config" });
@@ -1509,4 +1513,164 @@ export class InteractiveCLI {
             MessageFormatter.error("Migration failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Migration" });
         }
     }
+    async comprehensiveTransfer() {
+        MessageFormatter.info("Starting comprehensive transfer configuration...", { prefix: "Transfer" });
+        try {
+            // Get source configuration
+            const sourceConfig = await inquirer.prompt([
+                {
+                    type: "input",
+                    name: "sourceEndpoint",
+                    message: "Enter the source Appwrite endpoint:",
+                    validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+                },
+                {
+                    type: "input",
+                    name: "sourceProject",
+                    message: "Enter the source project ID:",
+                    validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+                },
+                {
+                    type: "password",
+                    name: "sourceKey",
+                    message: "Enter the source API key:",
+                    validate: (input) => input.trim() !== "" || "API key cannot be empty",
+                },
+            ]);
+            // Get target configuration
+            const targetConfig = await inquirer.prompt([
+                {
+                    type: "input",
+                    name: "targetEndpoint",
+                    message: "Enter the target Appwrite endpoint:",
+                    validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+                },
+                {
+                    type: "input",
+                    name: "targetProject",
+                    message: "Enter the target project ID:",
+                    validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+                },
+                {
+                    type: "password",
+                    name: "targetKey",
+                    message: "Enter the target API key:",
+                    validate: (input) => input.trim() !== "" || "API key cannot be empty",
+                },
+            ]);
+            // Get transfer options
+            const transferOptions = await inquirer.prompt([
+                {
+                    type: "checkbox",
+                    name: "transferTypes",
+                    message: "Select what to transfer:",
+                    choices: [
+                        { name: "👥 Users", value: "users", checked: true },
+                        { name: "🗄️ Databases", value: "databases", checked: true },
+                        { name: "📦 Storage Buckets", value: "buckets", checked: true },
+                        { name: "⚡ Functions", value: "functions", checked: true },
+                    ],
+                    validate: (input) => input.length > 0 || "Select at least one transfer type",
+                },
+                {
+                    type: "list",
+                    name: "concurrencyLimit",
+                    message: "Select concurrency limit:",
+                    choices: [
+                        { name: "5 (Conservative) - Users: 2, Files: 1", value: 5 },
+                        { name: "10 (Balanced) - Users: 5, Files: 2", value: 10 },
+                        { name: "15 - Users: 7, Files: 3", value: 15 },
+                        { name: "20 - Users: 10, Files: 5", value: 20 },
+                        { name: "25 - Users: 12, Files: 6", value: 25 },
+                        { name: "30 - Users: 15, Files: 7", value: 30 },
+                        { name: "35 - Users: 17, Files: 8", value: 35 },
+                        { name: "40 - Users: 20, Files: 10", value: 40 },
+                        { name: "45 - Users: 22, Files: 11", value: 45 },
+                        { name: "50 - Users: 25, Files: 12", value: 50 },
+                        { name: "55 - Users: 27, Files: 13", value: 55 },
+                        { name: "60 - Users: 30, Files: 15", value: 60 },
+                        { name: "65 - Users: 32, Files: 16", value: 65 },
+                        { name: "70 - Users: 35, Files: 17", value: 70 },
+                        { name: "75 - Users: 37, Files: 18", value: 75 },
+                        { name: "80 - Users: 40, Files: 20", value: 80 },
+                        { name: "85 - Users: 42, Files: 21", value: 85 },
+                        { name: "90 - Users: 45, Files: 22", value: 90 },
+                        { name: "95 - Users: 47, Files: 23", value: 95 },
+                        { name: "100 (Aggressive) - Users: 50, Files: 25", value: 100 },
+                    ],
+                    default: 10,
+                },
+                {
+                    type: "confirm",
+                    name: "dryRun",
+                    message: "Run in dry-run mode (no actual changes)?",
+                    default: false,
+                },
+            ]);
+            // Confirmation
+            const { confirmed } = await inquirer.prompt([
+                {
+                    type: "confirm",
+                    name: "confirmed",
+                    message: `Are you sure you want to ${transferOptions.dryRun ? "dry-run" : "perform"} comprehensive transfer from ${sourceConfig.sourceEndpoint} to ${targetConfig.targetEndpoint}?`,
+                    default: false,
+                },
+            ]);
+            if (!confirmed) {
+                MessageFormatter.info("Transfer cancelled by user", { prefix: "Transfer" });
+                return;
+            }
+            // Important password warning
+            if (transferOptions.transferTypes.includes("users") && !transferOptions.dryRun) {
+                MessageFormatter.warning("IMPORTANT: User passwords cannot be transferred due to Appwrite security limitations.", { prefix: "Transfer" });
+                MessageFormatter.warning("Users will need to reset their passwords after transfer.", { prefix: "Transfer" });
+                const { continueWithUsers } = await inquirer.prompt([
+                    {
+                        type: "confirm",
+                        name: "continueWithUsers",
+                        message: "Continue with user transfer knowing passwords will be reset?",
+                        default: false,
+                    },
+                ]);
+                if (!continueWithUsers) {
+                    // Remove users from transfer types
+                    transferOptions.transferTypes = transferOptions.transferTypes.filter((type) => type !== "users");
+                    if (transferOptions.transferTypes.length === 0) {
+                        MessageFormatter.info("No transfer types selected, cancelling", { prefix: "Transfer" });
+                        return;
+                    }
+                }
+            }
+            // Execute comprehensive transfer
+            const comprehensiveTransferOptions = {
+                sourceEndpoint: sourceConfig.sourceEndpoint,
+                sourceProject: sourceConfig.sourceProject,
+                sourceKey: sourceConfig.sourceKey,
+                targetEndpoint: targetConfig.targetEndpoint,
+                targetProject: targetConfig.targetProject,
+                targetKey: targetConfig.targetKey,
+                transferUsers: transferOptions.transferTypes.includes("users"),
+                transferDatabases: transferOptions.transferTypes.includes("databases"),
+                transferBuckets: transferOptions.transferTypes.includes("buckets"),
+                transferFunctions: transferOptions.transferTypes.includes("functions"),
+                concurrencyLimit: transferOptions.concurrencyLimit,
+                dryRun: transferOptions.dryRun,
+            };
+            const transfer = new ComprehensiveTransfer(comprehensiveTransferOptions);
+            const results = await transfer.execute();
+            // Display results
+            if (transferOptions.dryRun) {
+                MessageFormatter.success("Dry run completed successfully!", { prefix: "Transfer" });
+            }
+            else {
+                MessageFormatter.success("Comprehensive transfer completed!", { prefix: "Transfer" });
+                if (transferOptions.transferTypes.includes("users") && results.users.transferred > 0) {
+                    MessageFormatter.info("Remember to notify users about password reset requirements", { prefix: "Transfer" });
+                }
+            }
+        }
+        catch (error) {
+            MessageFormatter.error("Comprehensive transfer failed", error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+        }
+    }
 }

package/dist/migrations/comprehensiveTransfer.d.ts

@@ -0,0 +1,66 @@
+export interface ComprehensiveTransferOptions {
+    sourceEndpoint: string;
+    sourceProject: string;
+    sourceKey: string;
+    targetEndpoint: string;
+    targetProject: string;
+    targetKey: string;
+    transferUsers?: boolean;
+    transferDatabases?: boolean;
+    transferBuckets?: boolean;
+    transferFunctions?: boolean;
+    concurrencyLimit?: number;
+    dryRun?: boolean;
+}
+export interface TransferResults {
+    users: {
+        transferred: number;
+        skipped: number;
+        failed: number;
+    };
+    databases: {
+        transferred: number;
+        skipped: number;
+        failed: number;
+    };
+    buckets: {
+        transferred: number;
+        skipped: number;
+        failed: number;
+    };
+    functions: {
+        transferred: number;
+        skipped: number;
+        failed: number;
+    };
+    totalTime: number;
+}
+export declare class ComprehensiveTransfer {
+    private options;
+    private sourceClient;
+    private targetClient;
+    private sourceUsers;
+    private targetUsers;
+    private sourceDatabases;
+    private targetDatabases;
+    private sourceStorage;
+    private targetStorage;
+    private sourceFunctions;
+    private targetFunctions;
+    private limit;
+    private userLimit;
+    private fileLimit;
+    private results;
+    private startTime;
+    private tempDir;
+    constructor(options: ComprehensiveTransferOptions);
+    execute(): Promise<TransferResults>;
+    private transferAllUsers;
+    private transferAllDatabases;
+    private transferAllBuckets;
+    private transferBucketFiles;
+    private validateAndDownloadFile;
+    private transferAllFunctions;
+    private downloadFunction;
+    private printSummary;
+}