appwrite-utils-cli 1.2.6 → 1.2.8

package/src/migrations/comprehensiveTransfer.ts

@@ -1,4 +1,4 @@
-import { converterFunctions, tryAwaitWithRetry, parseAttribute } from "appwrite-utils";
+import { converterFunctions, tryAwaitWithRetry, parseAttribute, objectNeedsUpdate } from "appwrite-utils";
 import {
   Client,
   Databases,
@@ -519,6 +519,45 @@ export class ComprehensiveTransfer {
               // Create bucket with fallback strategy for maximumFileSize
               await this.createBucketWithFallback(bucket);
               MessageFormatter.success(`Created bucket: ${bucket.name}`, { prefix: "Transfer" });
+            } else {
+              // Compare bucket permissions and update if needed
+              const sourcePermissions = JSON.stringify(bucket.$permissions?.sort() || []);
+              const targetPermissions = JSON.stringify(existingBucket.$permissions?.sort() || []);
+              
+              if (sourcePermissions !== targetPermissions ||
+                  existingBucket.name !== bucket.name ||
+                  existingBucket.fileSecurity !== bucket.fileSecurity ||
+                  existingBucket.enabled !== bucket.enabled) {
+                
+                MessageFormatter.warning(
+                  `Bucket ${bucket.name} exists but has different settings. Updating to match source.`, 
+                  { prefix: "Transfer" }
+                );
+                
+                try {
+                  await this.targetStorage.updateBucket(
+                    bucket.$id,
+                    bucket.name,
+                    bucket.$permissions,
+                    bucket.fileSecurity,
+                    bucket.enabled,
+                    bucket.maximumFileSize,
+                    bucket.allowedFileExtensions,
+                    bucket.compression as any,
+                    bucket.encryption,
+                    bucket.antivirus
+                  );
+                  MessageFormatter.success(`Updated bucket ${bucket.name} to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update bucket ${bucket.name}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`Bucket ${bucket.name} already exists with matching settings`, { prefix: "Transfer" });
+              }
             }
 
             // Transfer bucket files with enhanced validation
@@ -703,10 +742,40 @@ export class ComprehensiveTransfer {
       const fileTasks = files.files.map(file => 
         this.fileLimit(async () => {
           try {
-            // Check if file already exists
+            // Check if file already exists and compare permissions
+            let existingFile: Models.File | null = null;
             try {
-              await this.targetStorage.getFile(targetBucketId, file.$id);
-              MessageFormatter.info(`File ${file.name} already exists, skipping`, { prefix: "Transfer" });
+              existingFile = await this.targetStorage.getFile(targetBucketId, file.$id);
+              
+              // Compare permissions between source and target file
+              const sourcePermissions = JSON.stringify(file.$permissions?.sort() || []);
+              const targetPermissions = JSON.stringify(existingFile.$permissions?.sort() || []);
+              
+              if (sourcePermissions !== targetPermissions) {
+                MessageFormatter.warning(
+                  `File ${file.name} (${file.$id}) exists but has different permissions. Source: ${sourcePermissions}, Target: ${targetPermissions}`, 
+                  { prefix: "Transfer" }
+                );
+                
+                // Update file permissions to match source
+                try {
+                  await this.targetStorage.updateFile(
+                    targetBucketId,
+                    file.$id,
+                    file.name,
+                    file.$permissions
+                  );
+                  MessageFormatter.success(`Updated file ${file.name} permissions to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update permissions for file ${file.name}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`File ${file.name} already exists with matching permissions, skipping`, { prefix: "Transfer" });
+              }
               return;
             } catch (error) {
               // File doesn't exist, proceed with transfer
@@ -998,7 +1067,7 @@ export class ComprehensiveTransfer {
   }
 
   /**
-   * Helper method to transfer documents between databases
+   * Helper method to transfer documents between databases using bulk operations with content and permission-based filtering
    */
   private async transferDocumentsBetweenDatabases(
     sourceDb: Databases,
@@ -1008,69 +1077,407 @@ export class ComprehensiveTransfer {
     sourceCollectionId: string,
     targetCollectionId: string
   ): Promise<void> {
-    MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+    MessageFormatter.info(`Transferring documents from ${sourceCollectionId} to ${targetCollectionId} with bulk operations, content comparison, and permission filtering`, { prefix: "Transfer" });
 
     let lastId: string | undefined;
     let totalTransferred = 0;
+    let totalSkipped = 0;
+    let totalUpdated = 0;
+
+    // Check if bulk operations are supported
+    const supportsBulk = this.options.sourceEndpoint.includes('cloud.appwrite.io') || 
+                        this.options.targetEndpoint.includes('cloud.appwrite.io');
+    
+    if (supportsBulk) {
+      MessageFormatter.info(`Using bulk operations for enhanced performance`, { prefix: "Transfer" });
+    }
 
     while (true) {
-      const queries = [Query.limit(50)]; // Smaller batch size for better performance
+      // Fetch source documents in larger batches (1000 instead of 50)
+      const queries = [Query.limit(1000)];
       if (lastId) {
         queries.push(Query.cursorAfter(lastId));
       }
 
-      const documents = await tryAwaitWithRetry(async () => 
+      const sourceDocuments = await tryAwaitWithRetry(async () => 
         sourceDb.listDocuments(sourceDbId, sourceCollectionId, queries)
       );
 
-      if (documents.documents.length === 0) {
+      if (sourceDocuments.documents.length === 0) {
         break;
       }
 
-      // Transfer documents with rate limiting
-      const transferTasks = documents.documents.map(doc => 
-        this.limit(async () => {
-          try {
-            // Check if document already exists
-            try {
-              await targetDb.getDocument(targetDbId, targetCollectionId, doc.$id);
-              MessageFormatter.info(`Document ${doc.$id} already exists, skipping`, { prefix: "Transfer" });
-              return;
-            } catch (error) {
-              // Document doesn't exist, proceed with creation
-            }
+      MessageFormatter.info(`Processing batch of ${sourceDocuments.documents.length} source documents`, { prefix: "Transfer" });
 
-            // Create document in target
-            const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
-            
-            await tryAwaitWithRetry(async () =>
-              targetDb.createDocument(
-                targetDbId,
-                targetCollectionId,
-                doc.$id,
-                docData,
-                doc.$permissions
-              )
-            );
+      // Extract document IDs from the current batch
+      const sourceDocIds = sourceDocuments.documents.map(doc => doc.$id);
+      
+      // Fetch existing documents from target in a single query
+      const existingTargetDocs = await this.fetchTargetDocumentsBatch(
+        targetDb, 
+        targetDbId, 
+        targetCollectionId, 
+        sourceDocIds
+      );
 
-            totalTransferred++;
-            MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
-          } catch (error) {
-            MessageFormatter.error(`Failed to transfer document ${doc.$id}`, error instanceof Error ? error : new Error(String(error)), { prefix: "Transfer" });
+      // Create a map for quick lookup of existing documents
+      const existingDocsMap = new Map<string, Models.Document>();
+      existingTargetDocs.forEach(doc => {
+        existingDocsMap.set(doc.$id, doc);
+      });
+
+      // Filter documents based on existence, content comparison, and permission comparison
+      const documentsToTransfer: Models.Document[] = [];
+      const documentsToUpdate: { doc: Models.Document; targetDoc: Models.Document; reason: string }[] = [];
+
+      for (const sourceDoc of sourceDocuments.documents) {
+        const existingTargetDoc = existingDocsMap.get(sourceDoc.$id);
+        
+        if (!existingTargetDoc) {
+          // Document doesn't exist in target, needs to be transferred
+          documentsToTransfer.push(sourceDoc);
+        } else {
+          // Document exists, compare both content and permissions
+          const sourcePermissions = JSON.stringify((sourceDoc.$permissions || []).sort());
+          const targetPermissions = JSON.stringify((existingTargetDoc.$permissions || []).sort());
+          const permissionsDiffer = sourcePermissions !== targetPermissions;
+          
+          // Use objectNeedsUpdate to compare document content (excluding system fields)
+          const contentDiffers = objectNeedsUpdate(existingTargetDoc, sourceDoc);
+          
+          if (contentDiffers && permissionsDiffer) {
+            // Both content and permissions differ
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "content and permissions differ"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but content and permissions differ - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else if (contentDiffers) {
+            // Only content differs
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "content differs"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but content differs - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else if (permissionsDiffer) {
+            // Only permissions differ
+            documentsToUpdate.push({ 
+              doc: sourceDoc, 
+              targetDoc: existingTargetDoc, 
+              reason: "permissions differ"
+            });
+            MessageFormatter.info(
+              `Document ${sourceDoc.$id} exists but permissions differ - will update`, 
+              { prefix: "Transfer" }
+            );
+          } else {
+            // Document exists with identical content AND permissions, skip
+            totalSkipped++;
+            MessageFormatter.info(`Document ${sourceDoc.$id} exists with matching content and permissions - skipping`, { prefix: "Transfer" });
           }
-        })
+        }
+      }
+
+      MessageFormatter.info(
+        `Batch analysis: ${documentsToTransfer.length} to create, ${documentsToUpdate.length} to update, ${totalSkipped} skipped so far`, 
+        { prefix: "Transfer" }
       );
 
-      await Promise.all(transferTasks);
+      // Process new documents with bulk operations if supported and available
+      if (documentsToTransfer.length > 0) {
+        if (supportsBulk && documentsToTransfer.length >= 10) {
+          // Use bulk operations for large batches
+          await this.transferDocumentsBulk(
+            targetDb,
+            targetDbId,
+            targetCollectionId,
+            documentsToTransfer
+          );
+          totalTransferred += documentsToTransfer.length;
+          MessageFormatter.success(`Bulk transferred ${documentsToTransfer.length} new documents`, { prefix: "Transfer" });
+        } else {
+          // Use individual transfers for smaller batches or non-bulk endpoints
+          const transferCount = await this.transferDocumentsIndividual(
+            targetDb,
+            targetDbId,
+            targetCollectionId,
+            documentsToTransfer
+          );
+          totalTransferred += transferCount;
+        }
+      }
 
-      if (documents.documents.length < 50) {
+      // Process document updates (always individual since bulk update with permissions needs special handling)
+      if (documentsToUpdate.length > 0) {
+        const updateCount = await this.updateDocumentsIndividual(
+          targetDb,
+          targetDbId,
+          targetCollectionId,
+          documentsToUpdate
+        );
+        totalUpdated += updateCount;
+      }
+
+      if (sourceDocuments.documents.length < 1000) {
         break;
       }
 
-      lastId = documents.documents[documents.documents.length - 1].$id;
+      lastId = sourceDocuments.documents[sourceDocuments.documents.length - 1].$id;
     }
 
-    MessageFormatter.info(`Transferred ${totalTransferred} documents from ${sourceCollectionId} to ${targetCollectionId}`, { prefix: "Transfer" });
+    MessageFormatter.info(
+      `Transfer complete: ${totalTransferred} new, ${totalUpdated} updated, ${totalSkipped} skipped from ${sourceCollectionId} to ${targetCollectionId}`, 
+      { prefix: "Transfer" }
+    );
+  }
+
+  /**
+   * Fetch target documents by IDs in batches to check existence and permissions
+   */
+  private async fetchTargetDocumentsBatch(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    docIds: string[]
+  ): Promise<Models.Document[]> {
+    const documents: Models.Document[] = [];
+    
+    // Split IDs into chunks of 100 for Query.equal limitations
+    const idChunks = this.chunkArray(docIds, 100);
+    
+    for (const chunk of idChunks) {
+      try {
+        const result = await tryAwaitWithRetry(async () => 
+          targetDb.listDocuments(targetDbId, targetCollectionId, [
+            Query.equal('$id', chunk),
+            Query.limit(100)
+          ])
+        );
+        documents.push(...result.documents);
+      } catch (error) {
+        // If query fails, fall back to individual gets (less efficient but more reliable)
+        MessageFormatter.warning(
+          `Batch query failed for ${chunk.length} documents, falling back to individual checks`, 
+          { prefix: "Transfer" }
+        );
+        
+        for (const docId of chunk) {
+          try {
+            const doc = await targetDb.getDocument(targetDbId, targetCollectionId, docId);
+            documents.push(doc);
+          } catch (getError) {
+            // Document doesn't exist, which is fine
+          }
+        }
+      }
+    }
+    
+    return documents;
+  }
+
+  /**
+   * Transfer documents using bulk operations with proper batch size handling
+   */
+  private async transferDocumentsBulk(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documents: Models.Document[]
+  ): Promise<void> {
+    // Prepare documents for bulk upsert
+    const preparedDocs = documents.map(doc => {
+      const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+      return {
+        $id,
+        $permissions,
+        ...docData
+      };
+    });
+
+    // Process in smaller chunks for bulk operations (1000 for Pro, 100 for Free tier)
+    const batchSizes = [1000, 100]; // Start with Pro plan, fallback to Free
+    let processed = false;
+
+    for (const maxBatchSize of batchSizes) {
+      const documentBatches = this.chunkArray(preparedDocs, maxBatchSize);
+      
+      try {
+        for (const batch of documentBatches) {
+          MessageFormatter.info(`Bulk upserting ${batch.length} documents...`, { prefix: "Transfer" });
+          
+          await this.bulkUpsertDocuments(
+            this.targetClient,
+            targetDbId,
+            targetCollectionId,
+            batch
+          );
+          
+          MessageFormatter.success(`✅ Bulk upserted ${batch.length} documents`, { prefix: "Transfer" });
+          
+          // Add delay between batches to respect rate limits
+          if (documentBatches.indexOf(batch) < documentBatches.length - 1) {
+            await new Promise(resolve => setTimeout(resolve, 200));
+          }
+        }
+        
+        processed = true;
+        break; // Success, exit batch size loop
+      } catch (error) {
+        MessageFormatter.warning(
+          `Bulk upsert with batch size ${maxBatchSize} failed, trying smaller size...`, 
+          { prefix: "Transfer" }
+        );
+        continue; // Try next smaller batch size
+      }
+    }
+
+    if (!processed) {
+      MessageFormatter.warning(
+        `All bulk operations failed, falling back to individual transfers`, 
+        { prefix: "Transfer" }
+      );
+      
+      // Fall back to individual transfers
+      await this.transferDocumentsIndividual(targetDb, targetDbId, targetCollectionId, documents);
+    }
+  }
+
+  /**
+   * Direct HTTP implementation of bulk upsert API
+   */
+  private async bulkUpsertDocuments(
+    client: any,
+    dbId: string,
+    collectionId: string,
+    documents: any[]
+  ): Promise<any> {
+    const apiPath = `/databases/${dbId}/collections/${collectionId}/documents`;
+    const url = new URL(client.config.endpoint + apiPath);
+    
+    const headers = {
+      'Content-Type': 'application/json',
+      'X-Appwrite-Project': client.config.project,
+      'X-Appwrite-Key': client.config.key
+    };
+    
+    const response = await fetch(url.toString(), {
+      method: 'PUT',
+      headers,
+      body: JSON.stringify({ documents })
+    });
+    
+    if (!response.ok) {
+      const errorData: any = await response.json().catch(() => ({ message: 'Unknown error' }));
+      throw new Error(`Bulk upsert failed: ${response.status} - ${errorData.message || 'Unknown error'}`);
+    }
+    
+    return await response.json();
+  }
+
+  /**
+   * Transfer documents individually with rate limiting
+   */
+  private async transferDocumentsIndividual(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documents: Models.Document[]
+  ): Promise<number> {
+    let successCount = 0;
+
+    const transferTasks = documents.map(doc => 
+      this.limit(async () => {
+        try {
+          const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+          
+          await tryAwaitWithRetry(async () =>
+            targetDb.createDocument(
+              targetDbId,
+              targetCollectionId,
+              doc.$id,
+              docData,
+              doc.$permissions
+            )
+          );
+
+          successCount++;
+          MessageFormatter.success(`Transferred document ${doc.$id}`, { prefix: "Transfer" });
+        } catch (error) {
+          MessageFormatter.error(
+            `Failed to transfer document ${doc.$id}`, 
+            error instanceof Error ? error : new Error(String(error)), 
+            { prefix: "Transfer" }
+          );
+        }
+      })
+    );
+
+    await Promise.all(transferTasks);
+    return successCount;
+  }
+
+  /**
+   * Update documents individually with content and/or permission changes
+   */
+  private async updateDocumentsIndividual(
+    targetDb: Databases,
+    targetDbId: string,
+    targetCollectionId: string,
+    documentPairs: { doc: Models.Document; targetDoc: Models.Document; reason: string }[]
+  ): Promise<number> {
+    let successCount = 0;
+
+    const updateTasks = documentPairs.map(({ doc, targetDoc, reason }) => 
+      this.limit(async () => {
+        try {
+          const { $id, $createdAt, $updatedAt, $permissions, $databaseId, $collectionId, ...docData } = doc;
+          
+          await tryAwaitWithRetry(async () =>
+            targetDb.updateDocument(
+              targetDbId,
+              targetCollectionId,
+              doc.$id,
+              docData,
+              doc.$permissions
+            )
+          );
+
+          successCount++;
+          MessageFormatter.success(
+            `Updated document ${doc.$id} (${reason}) - permissions: [${targetDoc.$permissions?.join(', ')}] → [${doc.$permissions?.join(', ')}]`, 
+            { prefix: "Transfer" }
+          );
+        } catch (error) {
+          MessageFormatter.error(
+            `Failed to update document ${doc.$id} (${reason})`, 
+            error instanceof Error ? error : new Error(String(error)), 
+            { prefix: "Transfer" }
+          );
+        }
+      })
+    );
+
+    await Promise.all(updateTasks);
+    return successCount;
+  }
+
+  /**
+   * Utility method to chunk arrays
+   */
+  private chunkArray<T>(array: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
   }
 
   /**
@@ -1160,10 +1567,39 @@ export class ComprehensiveTransfer {
       const transferTasks = memberships.map(membership => 
         this.userLimit(async () => { // Use userLimit for team operations (more sensitive)
           try {
-            // Check if membership already exists
+            // Check if membership already exists and compare roles
+            let existingMembership: Models.Membership | null = null;
             try {
-              await this.targetTeams.getMembership(teamId, membership.$id);
-              MessageFormatter.info(`Membership ${membership.$id} already exists, skipping`, { prefix: "Transfer" });
+              existingMembership = await this.targetTeams.getMembership(teamId, membership.$id);
+              
+              // Compare roles between source and target membership
+              const sourceRoles = JSON.stringify(membership.roles?.sort() || []);
+              const targetRoles = JSON.stringify(existingMembership.roles?.sort() || []);
+              
+              if (sourceRoles !== targetRoles) {
+                MessageFormatter.warning(
+                  `Membership ${membership.$id} exists but has different roles. Source: ${sourceRoles}, Target: ${targetRoles}`, 
+                  { prefix: "Transfer" }
+                );
+                
+                // Update membership roles to match source
+                try {
+                  await this.targetTeams.updateMembership(
+                    teamId,
+                    membership.$id,
+                    membership.roles
+                  );
+                  MessageFormatter.success(`Updated membership ${membership.$id} roles to match source`, { prefix: "Transfer" });
+                } catch (updateError) {
+                  MessageFormatter.error(
+                    `Failed to update roles for membership ${membership.$id}`, 
+                    updateError instanceof Error ? updateError : new Error(String(updateError)), 
+                    { prefix: "Transfer" }
+                  );
+                }
+              } else {
+                MessageFormatter.info(`Membership ${membership.$id} already exists with matching roles, skipping`, { prefix: "Transfer" });
+              }
               return;
             } catch (error) {
               // Membership doesn't exist, proceed with creation

package/src/shared/schemaGenerator.ts

@@ -198,6 +198,37 @@ export class SchemaGenerator {
   }
 
   public updateConfig(config: AppwriteConfig): void {
+    // Check if user is using YAML config
+    const { findYamlConfig, writeYamlConfig } = require("../config/yamlConfig.js");
+    const yamlConfigPath = findYamlConfig(this.appwriteFolderPath);
+    
+    if (yamlConfigPath) {
+      // User has YAML config - update it and generate individual collection files
+      this.updateYamlConfig(config, yamlConfigPath);
+    } else {
+      // User has TypeScript config - update the TS file
+      this.updateTypeScriptConfig(config);
+    }
+  }
+
+  private async updateYamlConfig(config: AppwriteConfig, yamlConfigPath: string): Promise<void> {
+    try {
+      const { writeYamlConfig } = await import("../config/yamlConfig.js");
+      
+      // Write the main YAML config (without collections)
+      await writeYamlConfig(yamlConfigPath, config);
+      
+      // Generate individual collection YAML files
+      this.updateYamlCollections();
+      
+      console.log("✅ Updated YAML configuration and collection files");
+    } catch (error) {
+      console.error("❌ Error updating YAML config:", error instanceof Error ? error.message : error);
+      throw error;
+    }
+  }
+
+  private updateTypeScriptConfig(config: AppwriteConfig): void {
     const configPath = path.join(this.appwriteFolderPath, "appwriteConfig.ts");
     const configContent = `import { type AppwriteConfig } from "appwrite-utils";
 
@@ -219,7 +250,7 @@ const appwriteConfig: AppwriteConfig = {
       $id: func.$id || ulid(),
       name: func.name,
       runtime: func.runtime,
-      dirPath: func.dirPath || `functions/${func.name}`,
+      dirPath: func.dirPath || "functions/" + func.name,
       entrypoint: func.entrypoint || "src/index.ts",
       execute: func.execute || [],
       events: func.events || [],
@@ -238,12 +269,14 @@ const appwriteConfig: AppwriteConfig = {
     })),
     null,
     4
-  )}
+  )},
+  collections: ${JSON.stringify(config.collections, null, 4)}
 };
 
 export default appwriteConfig;
 `;
     fs.writeFileSync(configPath, configContent, { encoding: "utf-8" });
+    console.log("✅ Updated TypeScript configuration file");
   }
 
   private extractRelationships(): void {

package/src/utils/yamlConverter.ts

@@ -128,4 +128,5 @@ export function sanitizeFilename(name: string): string {
  */
 export function getCollectionYamlFilename(collection: Collection | CollectionCreate): string {
   return `${sanitizeFilename(collection.name)}.yaml`;
-}
+}
+