appwrite-utils-cli 1.0.9 → 1.1.1

package/src/collections/methods.ts

@@ -23,6 +23,7 @@ import {
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 import { MessageFormatter } from "../shared/messageFormatter.js";
 import { ProgressManager } from "../shared/progressManager.js";
+import chalk from "chalk";
 
 export const documentExists = async (
   db: Databases,
@@ -615,6 +616,123 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
   );
 };
 
+/**
+ * Enhanced document transfer with fault tolerance and exponential backoff
+ */
+const transferDocumentWithRetry = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documentId: string,
+  documentData: any,
+  permissions: string[],
+  maxRetries: number = 3,
+  retryCount: number = 0
+): Promise<boolean> => {
+  try {
+    await db.createDocument(
+      dbId,
+      collectionId,
+      documentId,
+      documentData,
+      permissions
+    );
+    return true;
+  } catch (error: any) {
+    // Check if document already exists
+    if (error.code === 409 || error.message?.includes('already exists')) {
+      console.log(chalk.yellow(`Document ${documentId} already exists, skipping...`));
+      return true;
+    }
+    
+    if (retryCount < maxRetries) {
+      // Calculate exponential backoff: 1s, 2s, 4s
+      const exponentialDelay = Math.min(1000 * Math.pow(2, retryCount), 8000);
+      console.log(chalk.yellow(`Retrying document ${documentId} (attempt ${retryCount + 1}/${maxRetries}, backoff: ${exponentialDelay}ms)`));
+      
+      await delay(exponentialDelay);
+      
+      return await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        documentId,
+        documentData,
+        permissions,
+        maxRetries,
+        retryCount + 1
+      );
+    }
+    
+    console.log(chalk.red(`Failed to transfer document ${documentId} after ${maxRetries} retries: ${error.message}`));
+    return false;
+  }
+};
+
+/**
+ * Enhanced batch document transfer with fault tolerance
+ */
+const transferDocumentBatchWithRetry = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documents: any[],
+  batchSize: number = 10
+): Promise<{ successful: number; failed: number }> => {
+  let successful = 0;
+  let failed = 0;
+  
+  // Process documents in smaller batches to avoid overwhelming the server
+  const documentBatches = chunk(documents, batchSize);
+  
+  for (const batch of documentBatches) {
+    console.log(chalk.blue(`Processing batch of ${batch.length} documents...`));
+    
+    const batchPromises = batch.map(async (doc) => {
+      const toCreateObject: Partial<typeof doc> = { ...doc };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      
+      const result = await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        doc.$id,
+        toCreateObject,
+        doc.$permissions || []
+      );
+      
+      return { docId: doc.$id, success: result };
+    });
+    
+    const results = await Promise.allSettled(batchPromises);
+    
+    results.forEach((result, index) => {
+      if (result.status === 'fulfilled') {
+        if (result.value.success) {
+          successful++;
+        } else {
+          failed++;
+        }
+      } else {
+        console.log(chalk.red(`Batch promise rejected for document ${batch[index].$id}: ${result.reason}`));
+        failed++;
+      }
+    });
+    
+    // Add delay between batches to avoid rate limiting
+    if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+      await delay(500);
+    }
+  }
+  
+  return { successful, failed };
+};
+
 export const transferDocumentsBetweenDbsLocalToRemote = async (
   localDb: Databases,
   endpoint: string,
@@ -625,100 +743,70 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
   fromCollId: string,
   toCollId: string
 ) => {
+  console.log(chalk.blue(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`));
+  
   const client = new Client()
     .setEndpoint(endpoint)
     .setProject(projectId)
     .setKey(apiKey);
-  let totalDocumentsTransferred = 0;
+  
   const remoteDb = new Databases(client);
-  let fromCollDocs = await tryAwaitWithRetry(async () =>
-    localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
-  );
-
-  if (fromCollDocs.documents.length === 0) {
+  let totalDocumentsProcessed = 0;
+  let totalSuccessful = 0;
+  let totalFailed = 0;
+  
+  // Fetch documents in batches
+  let hasMoreDocuments = true;
+  let lastDocumentId: string | undefined;
+  
+  while (hasMoreDocuments) {
+    const queries = [Query.limit(50)];
+    if (lastDocumentId) {
+      queries.push(Query.cursorAfter(lastDocumentId));
+    }
+    
+    const fromCollDocs = await tryAwaitWithRetry(async () =>
+      localDb.listDocuments(fromDbId, fromCollId, queries)
+    );
+    
+    if (fromCollDocs.documents.length === 0) {
+      hasMoreDocuments = false;
+      break;
+    }
+    
+    console.log(chalk.blue(`Processing ${fromCollDocs.documents.length} documents...`));
+    
+    const { successful, failed } = await transferDocumentBatchWithRetry(
+      remoteDb,
+      toDbId,
+      toCollId,
+      fromCollDocs.documents
+    );
+    
+    totalDocumentsProcessed += fromCollDocs.documents.length;
+    totalSuccessful += successful;
+    totalFailed += failed;
+    
+    // Check if we have more documents to process
+    if (fromCollDocs.documents.length < 50) {
+      hasMoreDocuments = false;
+    } else {
+      lastDocumentId = fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
+    }
+    
+    console.log(chalk.gray(`Batch complete: ${successful} successful, ${failed} failed`));
+  }
+  
+  if (totalDocumentsProcessed === 0) {
     MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
     return;
-  } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: Partial<typeof doc> = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        remoteDb.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
+  }
+  
+  const message = `Total documents processed: ${totalDocumentsProcessed}, successful: ${totalSuccessful}, failed: ${totalFailed}`;
+  
+  if (totalFailed > 0) {
+    MessageFormatter.warning(message, { prefix: "Transfer" });
   } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: Partial<typeof doc> = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        remoteDb.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-    while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await tryAwaitWithRetry(async () =>
-        localDb.listDocuments(fromDbId, fromCollId, [
-          Query.limit(50),
-          Query.cursorAfter(
-            fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-          ),
-        ])
-      );
-      const batchedPromises = fromCollDocs.documents.map((doc) => {
-        const toCreateObject: Partial<typeof doc> = {
-          ...doc,
-        };
-        delete toCreateObject.$databaseId;
-        delete toCreateObject.$collectionId;
-        delete toCreateObject.$createdAt;
-        delete toCreateObject.$updatedAt;
-        delete toCreateObject.$id;
-        delete toCreateObject.$permissions;
-        return tryAwaitWithRetry(async () =>
-          remoteDb.createDocument(
-            toDbId,
-            toCollId,
-            doc.$id,
-            toCreateObject,
-            doc.$permissions
-          )
-        );
-      });
-      await Promise.all(batchedPromises);
-      totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
+    MessageFormatter.success(message, { prefix: "Transfer" });
   }
-  MessageFormatter.success(
-    `Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`,
-    { prefix: "Transfer" }
-  );
 };

package/src/interactiveCLI.ts

@@ -2052,49 +2052,144 @@ export class InteractiveCLI {
     MessageFormatter.info("Starting comprehensive transfer configuration...", { prefix: "Transfer" });
 
     try {
-      // Get source configuration
-      const sourceConfig = await inquirer.prompt([
-        {
-          type: "input",
-          name: "sourceEndpoint",
-          message: "Enter the source Appwrite endpoint:",
-          validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
-        },
-        {
-          type: "input",
-          name: "sourceProject",
-          message: "Enter the source project ID:",
-          validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
-        },
-        {
-          type: "password",
-          name: "sourceKey",
-          message: "Enter the source API key:",
-          validate: (input) => input.trim() !== "" || "API key cannot be empty",
-        },
-      ]);
+      // Initialize controller to optionally load config if available (supports both YAML and TypeScript configs)\n      await this.initControllerIfNeeded();\n      \n      // Check if user has an appwrite config for easier setup
+      const hasAppwriteConfig = this.controller?.config?.appwriteEndpoint && 
+                               this.controller?.config?.appwriteProject && 
+                               this.controller?.config?.appwriteKey;
 
-      // Get target configuration
-      const targetConfig = await inquirer.prompt([
-        {
-          type: "input",
-          name: "targetEndpoint",
-          message: "Enter the target Appwrite endpoint:",
-          validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
-        },
-        {
-          type: "input",
-          name: "targetProject",
-          message: "Enter the target project ID:",
-          validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
-        },
-        {
-          type: "password",
-          name: "targetKey",
-          message: "Enter the target API key:",
-          validate: (input) => input.trim() !== "" || "API key cannot be empty",
-        },
-      ]);
+      let sourceConfig: any;
+      let targetConfig: any;
+
+      if (hasAppwriteConfig) {
+        // Offer to use existing config for source
+        const { useConfigForSource } = await inquirer.prompt([
+          {
+            type: "confirm",
+            name: "useConfigForSource",
+            message: "Use your current appwriteConfig as the source?",
+            default: true,
+          },
+        ]);
+
+        if (useConfigForSource) {
+          sourceConfig = {
+            sourceEndpoint: this.controller!.config!.appwriteEndpoint,
+            sourceProject: this.controller!.config!.appwriteProject,
+            sourceKey: this.controller!.config!.appwriteKey,
+          };
+          MessageFormatter.info(`Using config source: ${sourceConfig.sourceEndpoint}`, { prefix: "Transfer" });
+        } else {
+          // Get source configuration manually
+          sourceConfig = await inquirer.prompt([
+            {
+              type: "input",
+              name: "sourceEndpoint",
+              message: "Enter the source Appwrite endpoint:",
+              validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+            },
+            {
+              type: "input",
+              name: "sourceProject",
+              message: "Enter the source project ID:",
+              validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+            },
+            {
+              type: "password",
+              name: "sourceKey",
+              message: "Enter the source API key:",
+              validate: (input) => input.trim() !== "" || "API key cannot be empty",
+            },
+          ]);
+        }
+
+        // Offer to use existing config for target
+        const { useConfigForTarget } = await inquirer.prompt([
+          {
+            type: "confirm",
+            name: "useConfigForTarget",
+            message: "Use your current appwriteConfig as the target?",
+            default: false,
+          },
+        ]);
+
+        if (useConfigForTarget) {
+          targetConfig = {
+            targetEndpoint: this.controller!.config!.appwriteEndpoint,
+            targetProject: this.controller!.config!.appwriteProject,
+            targetKey: this.controller!.config!.appwriteKey,
+          };
+          MessageFormatter.info(`Using config target: ${targetConfig.targetEndpoint}`, { prefix: "Transfer" });
+        } else {
+          // Get target configuration manually
+          targetConfig = await inquirer.prompt([
+            {
+              type: "input",
+              name: "targetEndpoint",
+              message: "Enter the target Appwrite endpoint:",
+              validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+            },
+            {
+              type: "input",
+              name: "targetProject",
+              message: "Enter the target project ID:",
+              validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+            },
+            {
+              type: "password",
+              name: "targetKey",
+              message: "Enter the target API key:",
+              validate: (input) => input.trim() !== "" || "API key cannot be empty",
+            },
+          ]);
+        }
+      } else {
+        // No appwrite config found, get both configurations manually
+        MessageFormatter.info("No appwriteConfig found, please enter source and target configurations manually", { prefix: "Transfer" });
+        
+        // Get source configuration
+        sourceConfig = await inquirer.prompt([
+          {
+            type: "input",
+            name: "sourceEndpoint",
+            message: "Enter the source Appwrite endpoint:",
+            validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+          },
+          {
+            type: "input",
+            name: "sourceProject",
+            message: "Enter the source project ID:",
+            validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+          },
+          {
+            type: "password",
+            name: "sourceKey",
+            message: "Enter the source API key:",
+            validate: (input) => input.trim() !== "" || "API key cannot be empty",
+          },
+        ]);
+
+        // Get target configuration
+        targetConfig = await inquirer.prompt([
+          {
+            type: "input",
+            name: "targetEndpoint",
+            message: "Enter the target Appwrite endpoint:",
+            validate: (input) => input.trim() !== "" || "Endpoint cannot be empty",
+          },
+          {
+            type: "input",
+            name: "targetProject",
+            message: "Enter the target project ID:",
+            validate: (input) => input.trim() !== "" || "Project ID cannot be empty",
+          },
+          {
+            type: "password",
+            name: "targetKey",
+            message: "Enter the target API key:",
+            validate: (input) => input.trim() !== "" || "API key cannot be empty",
+          },
+        ]);
+      }
 
       // Get transfer options
       const transferOptions = await inquirer.prompt([