appwrite-utils-cli 1.0.9 → 1.1.0

package/src/collections/indexes.ts

@@ -1,6 +1,270 @@
 import { indexSchema, type Index } from "appwrite-utils";
 import { Databases, IndexType, Query, type Models } from "node-appwrite";
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
+import chalk from "chalk";
+
+// Interface for index with status
+interface IndexWithStatus {
+  key: string;
+  type: string;
+  status: 'available' | 'processing' | 'deleting' | 'stuck' | 'failed';
+  error: string;
+  attributes: string[];
+  orders?: string[];
+  $createdAt: string;
+  $updatedAt: string;
+}
+
+/**
+ * Wait for index to become available, with retry logic for stuck indexes and exponential backoff
+ */
+const waitForIndexAvailable = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  indexKey: string,
+  maxWaitTime: number = 60000, // 1 minute
+  retryCount: number = 0,
+  maxRetries: number = 5
+): Promise<boolean> => {
+  const startTime = Date.now();
+  let checkInterval = 2000; // Start with 2 seconds
+  
+  // Calculate exponential backoff: 2s, 4s, 8s, 16s, 30s (capped at 30s)
+  if (retryCount > 0) {
+    const exponentialDelay = Math.min(2000 * Math.pow(2, retryCount), 30000);
+    console.log(chalk.blue(`Waiting for index '${indexKey}' to become available (retry ${retryCount}, backoff: ${exponentialDelay}ms)...`));
+    await delay(exponentialDelay);
+  } else {
+    console.log(chalk.blue(`Waiting for index '${indexKey}' to become available...`));
+  }
+  
+  while (Date.now() - startTime < maxWaitTime) {
+    try {
+      const indexList = await db.listIndexes(dbId, collectionId);
+      const index = indexList.indexes.find(
+        (idx: any) => idx.key === indexKey
+      ) as IndexWithStatus | undefined;
+      
+      if (!index) {
+        console.log(chalk.red(`Index '${indexKey}' not found`));
+        return false;
+      }
+      
+      console.log(chalk.gray(`Index '${indexKey}' status: ${index.status}`));
+      
+      switch (index.status) {
+        case 'available':
+          console.log(chalk.green(`✅ Index '${indexKey}' is now available`));
+          return true;
+          
+        case 'failed':
+          console.log(chalk.red(`❌ Index '${indexKey}' failed: ${index.error}`));
+          return false;
+          
+        case 'stuck':
+          console.log(chalk.yellow(`⚠️ Index '${indexKey}' is stuck, will retry...`));
+          return false;
+          
+        case 'processing':
+          // Continue waiting
+          break;
+          
+        case 'deleting':
+          console.log(chalk.yellow(`Index '${indexKey}' is being deleted`));
+          break;
+          
+        default:
+          console.log(chalk.yellow(`Unknown status '${index.status}' for index '${indexKey}'`));
+          break;
+      }
+      
+      await delay(checkInterval);
+    } catch (error) {
+      console.log(chalk.red(`Error checking index status: ${error}`));
+      return false;
+    }
+  }
+  
+  // Timeout reached
+  console.log(chalk.yellow(`⏰ Timeout waiting for index '${indexKey}' (${maxWaitTime}ms)`));
+  
+  // If we have retries left and this isn't the last retry, try recreating
+  if (retryCount < maxRetries) {
+    console.log(chalk.yellow(`🔄 Retrying index creation (attempt ${retryCount + 1}/${maxRetries})`));
+    return false; // Signal that we need to retry
+  }
+  
+  return false;
+};
+
+/**
+ * Delete collection and recreate for index retry (reused from attributes.ts)
+ */
+const deleteAndRecreateCollectionForIndex = async (
+  db: Databases,
+  dbId: string,
+  collection: Models.Collection,
+  retryCount: number
+): Promise<Models.Collection | null> => {
+  try {
+    console.log(chalk.yellow(`🗑️ Deleting collection '${collection.name}' for index retry ${retryCount}`));
+    
+    // Delete the collection
+    await db.deleteCollection(dbId, collection.$id);
+    console.log(chalk.yellow(`Deleted collection '${collection.name}'`));
+    
+    // Wait a bit before recreating
+    await delay(2000);
+    
+    // Recreate the collection
+    console.log(chalk.blue(`🔄 Recreating collection '${collection.name}'`));
+    const newCollection = await db.createCollection(
+      dbId,
+      collection.$id,
+      collection.name,
+      collection.$permissions,
+      collection.documentSecurity,
+      collection.enabled
+    );
+    
+    console.log(chalk.green(`✅ Recreated collection '${collection.name}'`));
+    return newCollection;
+    
+  } catch (error) {
+    console.log(chalk.red(`Failed to delete/recreate collection '${collection.name}': ${error}`));
+    return null;
+  }
+};
+
+/**
+ * Enhanced index creation with proper status monitoring and retry logic
+ */
+export const createOrUpdateIndexWithStatusCheck = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  collection: Models.Collection,
+  index: Index,
+  retryCount: number = 0,
+  maxRetries: number = 5
+): Promise<boolean> => {
+  console.log(chalk.blue(`Creating/updating index '${index.key}' (attempt ${retryCount + 1}/${maxRetries + 1})`));
+  
+  try {
+    // First, try to create/update the index using existing logic
+    await createOrUpdateIndex(dbId, db, collectionId, index);
+    
+    // Now wait for the index to become available
+    const success = await waitForIndexAvailable(
+      db, 
+      dbId, 
+      collectionId, 
+      index.key, 
+      60000, // 1 minute timeout
+      retryCount, 
+      maxRetries
+    );
+    
+    if (success) {
+      return true;
+    }
+    
+    // If not successful and we have retries left, delete collection and try again
+    if (retryCount < maxRetries) {
+      console.log(chalk.yellow(`Index '${index.key}' failed/stuck, retrying...`));
+      
+      // Get fresh collection data
+      const freshCollection = await db.getCollection(dbId, collectionId);
+      
+      // Delete and recreate collection
+      const newCollection = await deleteAndRecreateCollectionForIndex(db, dbId, freshCollection, retryCount + 1);
+      
+      if (newCollection) {
+        // Retry with the new collection
+        return await createOrUpdateIndexWithStatusCheck(
+          dbId, 
+          db, 
+          newCollection.$id, 
+          newCollection, 
+          index, 
+          retryCount + 1, 
+          maxRetries
+        );
+      }
+    }
+    
+    console.log(chalk.red(`❌ Failed to create index '${index.key}' after ${maxRetries + 1} attempts`));
+    return false;
+    
+  } catch (error) {
+    console.log(chalk.red(`Error creating index '${index.key}': ${error}`));
+    
+    if (retryCount < maxRetries) {
+      console.log(chalk.yellow(`Retrying index '${index.key}' due to error...`));
+      
+      // Wait a bit before retry
+      await delay(2000);
+      
+      return await createOrUpdateIndexWithStatusCheck(
+        dbId, 
+        db, 
+        collectionId, 
+        collection, 
+        index, 
+        retryCount + 1, 
+        maxRetries
+      );
+    }
+    
+    return false;
+  }
+};
+
+/**
+ * Enhanced index creation with status monitoring for all indexes
+ */
+export const createOrUpdateIndexesWithStatusCheck = async (
+  dbId: string,
+  db: Databases,
+  collectionId: string,
+  collection: Models.Collection,
+  indexes: Index[]
+): Promise<boolean> => {
+  console.log(chalk.blue(`Creating/updating ${indexes.length} indexes with status monitoring...`));
+  
+  const failedIndexes: string[] = [];
+  
+  for (const index of indexes) {
+    console.log(chalk.blue(`\n--- Processing index: ${index.key} ---`));
+    
+    const success = await createOrUpdateIndexWithStatusCheck(
+      dbId, 
+      db, 
+      collectionId, 
+      collection, 
+      index
+    );
+    
+    if (success) {
+      console.log(chalk.green(`✅ Successfully created index: ${index.key}`));
+      
+      // Add delay between successful indexes
+      await delay(1000);
+    } else {
+      console.log(chalk.red(`❌ Failed to create index: ${index.key}`));
+      failedIndexes.push(index.key);
+    }
+  }
+  
+  if (failedIndexes.length > 0) {
+    console.log(chalk.red(`\n❌ Failed to create ${failedIndexes.length} indexes: ${failedIndexes.join(', ')}`));
+    return false;
+  }
+  
+  console.log(chalk.green(`\n✅ Successfully created all ${indexes.length} indexes`));
+  return true;
+};
 
 export const createOrUpdateIndex = async (
   dbId: string,

package/src/collections/methods.ts

@@ -23,6 +23,7 @@ import {
 import { delay, tryAwaitWithRetry } from "../utils/helperFunctions.js";
 import { MessageFormatter } from "../shared/messageFormatter.js";
 import { ProgressManager } from "../shared/progressManager.js";
+import chalk from "chalk";
 
 export const documentExists = async (
   db: Databases,
@@ -615,6 +616,123 @@ export const transferDocumentsBetweenDbsLocalToLocal = async (
   );
 };
 
+/**
+ * Enhanced document transfer with fault tolerance and exponential backoff
+ */
+const transferDocumentWithRetry = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documentId: string,
+  documentData: any,
+  permissions: string[],
+  maxRetries: number = 3,
+  retryCount: number = 0
+): Promise<boolean> => {
+  try {
+    await db.createDocument(
+      dbId,
+      collectionId,
+      documentId,
+      documentData,
+      permissions
+    );
+    return true;
+  } catch (error: any) {
+    // Check if document already exists
+    if (error.code === 409 || error.message?.includes('already exists')) {
+      console.log(chalk.yellow(`Document ${documentId} already exists, skipping...`));
+      return true;
+    }
+    
+    if (retryCount < maxRetries) {
+      // Calculate exponential backoff: 1s, 2s, 4s
+      const exponentialDelay = Math.min(1000 * Math.pow(2, retryCount), 8000);
+      console.log(chalk.yellow(`Retrying document ${documentId} (attempt ${retryCount + 1}/${maxRetries}, backoff: ${exponentialDelay}ms)`));
+      
+      await delay(exponentialDelay);
+      
+      return await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        documentId,
+        documentData,
+        permissions,
+        maxRetries,
+        retryCount + 1
+      );
+    }
+    
+    console.log(chalk.red(`Failed to transfer document ${documentId} after ${maxRetries} retries: ${error.message}`));
+    return false;
+  }
+};
+
+/**
+ * Enhanced batch document transfer with fault tolerance
+ */
+const transferDocumentBatchWithRetry = async (
+  db: Databases,
+  dbId: string,
+  collectionId: string,
+  documents: any[],
+  batchSize: number = 10
+): Promise<{ successful: number; failed: number }> => {
+  let successful = 0;
+  let failed = 0;
+  
+  // Process documents in smaller batches to avoid overwhelming the server
+  const documentBatches = chunk(documents, batchSize);
+  
+  for (const batch of documentBatches) {
+    console.log(chalk.blue(`Processing batch of ${batch.length} documents...`));
+    
+    const batchPromises = batch.map(async (doc) => {
+      const toCreateObject: Partial<typeof doc> = { ...doc };
+      delete toCreateObject.$databaseId;
+      delete toCreateObject.$collectionId;
+      delete toCreateObject.$createdAt;
+      delete toCreateObject.$updatedAt;
+      delete toCreateObject.$id;
+      delete toCreateObject.$permissions;
+      
+      const result = await transferDocumentWithRetry(
+        db,
+        dbId,
+        collectionId,
+        doc.$id,
+        toCreateObject,
+        doc.$permissions || []
+      );
+      
+      return { docId: doc.$id, success: result };
+    });
+    
+    const results = await Promise.allSettled(batchPromises);
+    
+    results.forEach((result, index) => {
+      if (result.status === 'fulfilled') {
+        if (result.value.success) {
+          successful++;
+        } else {
+          failed++;
+        }
+      } else {
+        console.log(chalk.red(`Batch promise rejected for document ${batch[index].$id}: ${result.reason}`));
+        failed++;
+      }
+    });
+    
+    // Add delay between batches to avoid rate limiting
+    if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+      await delay(500);
+    }
+  }
+  
+  return { successful, failed };
+};
+
 export const transferDocumentsBetweenDbsLocalToRemote = async (
   localDb: Databases,
   endpoint: string,
@@ -625,100 +743,70 @@ export const transferDocumentsBetweenDbsLocalToRemote = async (
   fromCollId: string,
   toCollId: string
 ) => {
+  console.log(chalk.blue(`Starting enhanced document transfer from ${fromCollId} to ${toCollId}...`));
+  
   const client = new Client()
     .setEndpoint(endpoint)
     .setProject(projectId)
     .setKey(apiKey);
-  let totalDocumentsTransferred = 0;
+  
   const remoteDb = new Databases(client);
-  let fromCollDocs = await tryAwaitWithRetry(async () =>
-    localDb.listDocuments(fromDbId, fromCollId, [Query.limit(50)])
-  );
-
-  if (fromCollDocs.documents.length === 0) {
+  let totalDocumentsProcessed = 0;
+  let totalSuccessful = 0;
+  let totalFailed = 0;
+  
+  // Fetch documents in batches
+  let hasMoreDocuments = true;
+  let lastDocumentId: string | undefined;
+  
+  while (hasMoreDocuments) {
+    const queries = [Query.limit(50)];
+    if (lastDocumentId) {
+      queries.push(Query.cursorAfter(lastDocumentId));
+    }
+    
+    const fromCollDocs = await tryAwaitWithRetry(async () =>
+      localDb.listDocuments(fromDbId, fromCollId, queries)
+    );
+    
+    if (fromCollDocs.documents.length === 0) {
+      hasMoreDocuments = false;
+      break;
+    }
+    
+    console.log(chalk.blue(`Processing ${fromCollDocs.documents.length} documents...`));
+    
+    const { successful, failed } = await transferDocumentBatchWithRetry(
+      remoteDb,
+      toDbId,
+      toCollId,
+      fromCollDocs.documents
+    );
+    
+    totalDocumentsProcessed += fromCollDocs.documents.length;
+    totalSuccessful += successful;
+    totalFailed += failed;
+    
+    // Check if we have more documents to process
+    if (fromCollDocs.documents.length < 50) {
+      hasMoreDocuments = false;
+    } else {
+      lastDocumentId = fromCollDocs.documents[fromCollDocs.documents.length - 1].$id;
+    }
+    
+    console.log(chalk.gray(`Batch complete: ${successful} successful, ${failed} failed`));
+  }
+  
+  if (totalDocumentsProcessed === 0) {
     MessageFormatter.info(`No documents found in collection ${fromCollId}`, { prefix: "Transfer" });
     return;
-  } else if (fromCollDocs.documents.length < 50) {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: Partial<typeof doc> = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        remoteDb.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
+  }
+  
+  const message = `Total documents processed: ${totalDocumentsProcessed}, successful: ${totalSuccessful}, failed: ${totalFailed}`;
+  
+  if (totalFailed > 0) {
+    MessageFormatter.warning(message, { prefix: "Transfer" });
   } else {
-    const batchedPromises = fromCollDocs.documents.map((doc) => {
-      const toCreateObject: Partial<typeof doc> = {
-        ...doc,
-      };
-      delete toCreateObject.$databaseId;
-      delete toCreateObject.$collectionId;
-      delete toCreateObject.$createdAt;
-      delete toCreateObject.$updatedAt;
-      delete toCreateObject.$id;
-      delete toCreateObject.$permissions;
-      return tryAwaitWithRetry(async () =>
-        remoteDb.createDocument(
-          toDbId,
-          toCollId,
-          doc.$id,
-          toCreateObject,
-          doc.$permissions
-        )
-      );
-    });
-    await Promise.all(batchedPromises);
-    totalDocumentsTransferred += fromCollDocs.documents.length;
-    while (fromCollDocs.documents.length === 50) {
-      fromCollDocs = await tryAwaitWithRetry(async () =>
-        localDb.listDocuments(fromDbId, fromCollId, [
-          Query.limit(50),
-          Query.cursorAfter(
-            fromCollDocs.documents[fromCollDocs.documents.length - 1].$id
-          ),
-        ])
-      );
-      const batchedPromises = fromCollDocs.documents.map((doc) => {
-        const toCreateObject: Partial<typeof doc> = {
-          ...doc,
-        };
-        delete toCreateObject.$databaseId;
-        delete toCreateObject.$collectionId;
-        delete toCreateObject.$createdAt;
-        delete toCreateObject.$updatedAt;
-        delete toCreateObject.$id;
-        delete toCreateObject.$permissions;
-        return tryAwaitWithRetry(async () =>
-          remoteDb.createDocument(
-            toDbId,
-            toCollId,
-            doc.$id,
-            toCreateObject,
-            doc.$permissions
-          )
-        );
-      });
-      await Promise.all(batchedPromises);
-      totalDocumentsTransferred += fromCollDocs.documents.length;
-    }
+    MessageFormatter.success(message, { prefix: "Transfer" });
   }
-  MessageFormatter.success(
-    `Total documents transferred from database ${fromDbId} to database ${toDbId} -- collection ${fromCollId} to collection ${toCollId}: ${totalDocumentsTransferred}`,
-    { prefix: "Transfer" }
-  );
 };